├── .gitignore ├── LICENSE ├── README.md ├── bin ├── run-cmd.sh ├── run-get-test.sh ├── run-local-cluster.sh └── run-put-test.sh ├── contrib └── YCSB │ ├── .gitignore │ ├── BUILD │ ├── CHANGELOG │ ├── README │ ├── bin │ ├── ycsb │ └── ycsb.sh │ ├── checkstyle.xml │ ├── core │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── yahoo │ │ │ └── ycsb │ │ │ ├── BasicDB.java │ │ │ ├── ByteArrayByteIterator.java │ │ │ ├── ByteIterator.java │ │ │ ├── Client.java │ │ │ ├── CommandLine.java │ │ │ ├── DB.java │ │ │ ├── DBException.java │ │ │ ├── DBFactory.java │ │ │ ├── DBWrapper.java │ │ │ ├── InputStreamByteIterator.java │ │ │ ├── RandomByteIterator.java │ │ │ ├── StringByteIterator.java │ │ │ ├── TerminatorThread.java │ │ │ ├── UnknownDBException.java │ │ │ ├── Utils.java │ │ │ ├── Workload.java │ │ │ ├── WorkloadException.java │ │ │ ├── generator │ │ │ ├── ConstantIntegerGenerator.java │ │ │ ├── CounterGenerator.java │ │ │ ├── DiscreteGenerator.java │ │ │ ├── ExponentialGenerator.java │ │ │ ├── FileGenerator.java │ │ │ ├── Generator.java │ │ │ ├── HistogramGenerator.java │ │ │ ├── HotspotIntegerGenerator.java │ │ │ ├── IntegerGenerator.java │ │ │ ├── ScrambledZipfianGenerator.java │ │ │ ├── SkewedLatestGenerator.java │ │ │ ├── UniformGenerator.java │ │ │ ├── UniformIntegerGenerator.java │ │ │ └── ZipfianGenerator.java │ │ │ ├── measurements │ │ │ ├── Measurements.java │ │ │ ├── OneMeasurement.java │ │ │ ├── OneMeasurementHistogram.java │ │ │ ├── OneMeasurementTimeSeries.java │ │ │ └── exporter │ │ │ │ ├── JSONMeasurementsExporter.java │ │ │ │ ├── MeasurementsExporter.java │ │ │ │ └── TextMeasurementsExporter.java │ │ │ └── workloads │ │ │ ├── ConstantOccupancyWorkload.java │ │ │ └── CoreWorkload.java │ │ └── test │ │ └── java │ │ └── com │ │ └── yahoo │ │ └── ycsb │ │ └── TestByteIterator.java │ ├── distribution │ ├── pom.xml │ └── src │ │ └── main │ │ └── assembly │ │ └── distribution.xml │ ├── doc │ ├── coreproperties.html │ ├── coreworkloads.html │ ├── dblayer.html │ ├── images │ │ ├── ycsb.jpg │ │ └── ycsblogo-small.png │ ├── index.html │ ├── javadoc │ │ ├── allclasses-frame.html │ │ ├── allclasses-noframe.html │ │ ├── com │ │ │ └── yahoo │ │ │ │ └── ycsb │ │ │ │ ├── BasicDB.html │ │ │ │ ├── Client.html │ │ │ │ ├── CounterGenerator.html │ │ │ │ ├── DB.html │ │ │ │ ├── DBException.html │ │ │ │ ├── DBFactory.html │ │ │ │ ├── DBWrapper.html │ │ │ │ ├── DiscreteGenerator.html │ │ │ │ ├── FindGoodAB.html │ │ │ │ ├── FindGoodScrambleVector.html │ │ │ │ ├── Generator.html │ │ │ │ ├── IntegerGenerator.html │ │ │ │ ├── Measurements.html │ │ │ │ ├── OneMeasurement.html │ │ │ │ ├── OneMeasurementHistogram.html │ │ │ │ ├── OneMeasurementTimeSeries.html │ │ │ │ ├── ScrambledZipfianGenerator.html │ │ │ │ ├── SkewedLatestGenerator.html │ │ │ │ ├── TestCollisions.html │ │ │ │ ├── TestExpandedZipfian.html │ │ │ │ ├── TestZipfian.html │ │ │ │ ├── UniformGenerator.html │ │ │ │ ├── UniformIntegerGenerator.html │ │ │ │ ├── UnknownDBException.html │ │ │ │ ├── Utils.html │ │ │ │ ├── Workload.html │ │ │ │ ├── WorkloadException.html │ │ │ │ ├── ZipfianGenerator.html │ │ │ │ ├── db │ │ │ │ ├── CassandraClient.html │ │ │ │ ├── HBaseClient.html │ │ │ │ ├── ShardClient.html │ │ │ │ ├── SherpaClient.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ └── package-tree.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ ├── package-tree.html │ │ │ │ └── workloads │ │ │ │ ├── CoreWorkload.html │ │ │ │ ├── package-frame.html │ │ │ │ ├── package-summary.html │ │ │ │ └── package-tree.html │ │ ├── constant-values.html │ │ ├── deprecated-list.html │ │ ├── help-doc.html │ │ ├── index-all.html │ │ ├── index.html │ │ ├── overview-frame.html │ │ ├── overview-summary.html │ │ ├── overview-tree.html │ │ ├── package-list │ │ ├── resources │ │ │ └── inherit.gif │ │ ├── serialized-form.html │ │ └── stylesheet.css │ ├── parallelclients.html │ ├── tipsfaq.html │ └── workload.html │ ├── install-kaiju-jar.sh │ ├── kaiju │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── yahoo │ │ └── ycsb │ │ └── db │ │ └── KaijuClient.java │ ├── pom.xml │ ├── run-ycsb.sh │ └── workloads │ ├── workloada │ ├── workloadb │ ├── workloadc │ ├── workloadd │ ├── workloade │ └── workloadf ├── experiment ├── .gitignore ├── analyze-serverlogs.py ├── common_funcs.py ├── example.txt ├── experiments.py ├── run-all.sh └── setup_hosts.py ├── pom.xml ├── py-impl ├── bloom_filter.py ├── data_item.py ├── demo.py ├── ramp_client.py └── ramp_server.py ├── src └── main │ ├── java │ ├── edu │ │ └── berkeley │ │ │ └── kaiju │ │ │ ├── KaijuServer.java │ │ │ ├── config │ │ │ ├── ClusterConverter.java │ │ │ ├── Config.java │ │ │ └── ShortConverter.java │ │ │ ├── data │ │ │ ├── DataItem.java │ │ │ └── ItemVersion.java │ │ │ ├── exception │ │ │ ├── AbortedException.java │ │ │ ├── ClientException.java │ │ │ ├── ConfigurationException.java │ │ │ ├── HandlerException.java │ │ │ ├── KaijuException.java │ │ │ ├── RemoteOperationException.java │ │ │ └── VersionNotFoundException.java │ │ │ ├── frontend │ │ │ ├── FrontendServer.java │ │ │ ├── KaijuClient.java │ │ │ ├── request │ │ │ │ ├── ClientGetAllRequest.java │ │ │ │ ├── ClientPutAllRequest.java │ │ │ │ ├── ClientRequest.java │ │ │ │ └── ClientSetIsolationRequest.java │ │ │ └── response │ │ │ │ ├── ClientError.java │ │ │ │ ├── ClientGetAllResponse.java │ │ │ │ ├── ClientPutAllResponse.java │ │ │ │ ├── ClientResponse.java │ │ │ │ └── ClientSetIsolationResponse.java │ │ │ ├── monitor │ │ │ └── MetricsManager.java │ │ │ ├── net │ │ │ ├── InboundMessagingService.java │ │ │ ├── callback │ │ │ │ ├── IMessageCallback.java │ │ │ │ ├── MultiMessageCallback.java │ │ │ │ └── SingleMessageCallback.java │ │ │ ├── routing │ │ │ │ ├── HashingRouter.java │ │ │ │ └── OutboundRouter.java │ │ │ └── tcp │ │ │ │ ├── InternalTCPReceiver.java │ │ │ │ └── InternalTCPSender.java │ │ │ ├── service │ │ │ ├── CooperativeCommitter.java │ │ │ ├── LockManager.java │ │ │ ├── MemoryStorageEngine.java │ │ │ └── request │ │ │ │ ├── RequestDispatcher.java │ │ │ │ ├── RequestExecutor.java │ │ │ │ ├── RequestExecutorFactory.java │ │ │ │ ├── eiger │ │ │ │ └── EigerExecutor.java │ │ │ │ ├── handler │ │ │ │ ├── EigerKaijuServiceHandler.java │ │ │ │ ├── IKaijuHandler.java │ │ │ │ ├── KaijuServiceHandler.java │ │ │ │ ├── LockBasedKaijuServiceHandler.java │ │ │ │ ├── ReadAtomicBloomBasedKaijuServiceHandler.java │ │ │ │ ├── ReadAtomicKaijuServiceHandler.java │ │ │ │ ├── ReadAtomicListBasedKaijuServiceHandler.java │ │ │ │ ├── ReadAtomicStampBasedKaijuServiceHandler.java │ │ │ │ └── ReadCommittedKaijuServiceHandler.java │ │ │ │ └── message │ │ │ │ ├── KaijuMessage.java │ │ │ │ ├── request │ │ │ │ ├── CheckPreparedRequest.java │ │ │ │ ├── CommitPutAllRequest.java │ │ │ │ ├── EigerCheckCommitRequest.java │ │ │ │ ├── EigerCommitRequest.java │ │ │ │ ├── EigerGetAllRequest.java │ │ │ │ ├── EigerPutAllRequest.java │ │ │ │ ├── GetAllByTimestampListRequest.java │ │ │ │ ├── GetAllByTimestampRequest.java │ │ │ │ ├── GetAllRequest.java │ │ │ │ ├── GetEachByTimestampListRequest.java │ │ │ │ ├── GetRequest.java │ │ │ │ ├── GetTimestampsRequest.java │ │ │ │ ├── IKaijuRequest.java │ │ │ │ ├── PreparePutAllRequest.java │ │ │ │ ├── PutAllRequest.java │ │ │ │ ├── PutRequest.java │ │ │ │ ├── ReadLockRequest.java │ │ │ │ ├── UnlockRequest.java │ │ │ │ └── WriteLockRequest.java │ │ │ │ └── response │ │ │ │ ├── EigerCheckCommitResponse.java │ │ │ │ ├── EigerPreparedResponse.java │ │ │ │ └── KaijuResponse.java │ │ │ └── util │ │ │ ├── KryoSerializer.java │ │ │ └── Timestamp.java │ └── org │ │ └── apache │ │ └── hadoop │ │ └── util │ │ └── bloom │ │ ├── BloomFilter.java │ │ ├── HashFunction.java │ │ └── MurmurHash.java │ └── resources │ └── log4j.properties └── tools └── cmdline ├── pom.xml └── src └── main └── java └── edu └── berkeley └── kaiju └── cmdline └── CommandLineClient.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.txt 3 | hosts 4 | *.idea 5 | *.iml 6 | target -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is licensed under the Apache 2 license, quoted below. 2 | 3 | Copyright 2014 Peter Bailis 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. -------------------------------------------------------------------------------- /bin/run-cmd.sh: -------------------------------------------------------------------------------- 1 | java -jar tools/cmdline/target/cmdline-1.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /bin/run-get-test.sh: -------------------------------------------------------------------------------- 1 | java -jar tools/cmdline/target/cmdline-1.0-SNAPSHOT.jar GA -------------------------------------------------------------------------------- /bin/run-local-cluster.sh: -------------------------------------------------------------------------------- 1 | java -ea -Djava.library.path=/usr/local/lib -Dlog4j.configuration=file:target/classes/log4j.properties -jar target/kaiju-1.0-SNAPSHOT.jar -kaiju_port 8081 -id 0 -cluster 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083 & 2 | java -ea -Djava.library.path=/usr/local/lib -Dlog4j.configuration=file:target/classes/log4j.properties -jar target/kaiju-1.0-SNAPSHOT.jar -kaiju_port 8082 -id 1 -cluster 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083 -thrift_port 9002 & 3 | java -ea -Djava.library.path=/usr/local/lib -Dlog4j.configuration=file:target/classes/log4j.properties -jar target/kaiju-1.0-SNAPSHOT.jar -kaiju_port 8083 -id 2 -cluster 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083 -thrift_port 9003 & -------------------------------------------------------------------------------- /bin/run-put-test.sh: -------------------------------------------------------------------------------- 1 | java -jar tools/cmdline/target/cmdline-1.0-SNAPSHOT.jar PA -------------------------------------------------------------------------------- /contrib/YCSB/.gitignore: -------------------------------------------------------------------------------- 1 | /elasticsearch/target/ 2 | -------------------------------------------------------------------------------- /contrib/YCSB/BUILD: -------------------------------------------------------------------------------- 1 | # Building YCSB 2 | 3 | To build YCSB, run: 4 | 5 | mvn clean package 6 | 7 | # Running YCSB 8 | 9 | Once `mvn clean package` succeeds, you can run `ycsb` command: 10 | 11 | ./bin/ycsb load basic workloads/workloada 12 | ./bin/ycsb run basic workloads/workloada 13 | 14 | # Oracle NoSQL Database 15 | 16 | Oracle NoSQL Database binding doesn't get built by default because there is no 17 | Maven repository for it. To build the binding: 18 | 19 | 1. Download kv-ce-1.2.123.tar.gz from here: 20 | 21 | http://www.oracle.com/technetwork/database/nosqldb/downloads/index.html 22 | 23 | 2. Untar kv-ce-1.2.123.tar.gz and install kvclient-1.2.123.jar in your local 24 | maven repository: 25 | 26 | tar xfvz kv-ce-1.2.123.tar.gz 27 | mvn install:install-file -Dfile=kv-1.2.123/lib/kvclient-1.2.123.jar \ 28 | -DgroupId=com.oracle -DartifactId=kvclient -Dversion=1.2.123 29 | -Dpackaging=jar 30 | 31 | 3. Uncomment `nosqldb` and run `mvn clean package`. 32 | -------------------------------------------------------------------------------- /contrib/YCSB/CHANGELOG: -------------------------------------------------------------------------------- 1 | - gh-95 Bump MongoDB version to 2.9.0 (allanbank) 2 | - gh-67 Use checkstyle (m1ch1) 3 | - gh-76 Implemented OrientDB client (lvca) 4 | - gh-88 YCSB client for Amazon DynamoDB (jananin) 5 | - gh-89 Patch for YCSB Cassandra Client version 1.0.6 (jananin) 6 | - gh-93 New ElasticSearch Database Implementation (saden1) 7 | - gh-97 Bug fixes in dynamodb plugin (jananin) 8 | 9 | 0.1.4 - 2/22/12 10 | 11 | - Fixes for Cassandra 0.7 client (tjake) 12 | - New generator FileGenerator (nono) 13 | - Fixes for MongoDB (nono) 14 | - Fixes for Voldemort (nono) 15 | - JDBC client (sudiptodas) 16 | - HotspotIntegerGenerator (sudiptodas) 17 | - Optimizing cassandra7 (joaquincasares) 18 | - Mysql key fix (joaquincasares) 19 | - Added a db plugin for Infinispan (maniksurtani) 20 | - gh-31 Support to stop benchmark after a maximum specified elapsed time. (sudiptodas) 21 | - gh-35 Cassandra0.8 support (joaquincasares) 22 | - gh-30 IllegalArgumentException with MongoDB (m1ch1) 23 | - gh-27 MongoDbClient was not working with non localhost URLs (arunxarun) 24 | - gh-29 Add simple sharding capabilities for JDBC driver (kibab) 25 | - gh-40 Merge Redis database interface layer (lehmannro) 26 | - gh-42 Response latencies are measured in microseconds (mikewied) 27 | - gh-43 Variable length fields (sears) 28 | - gh-44 Constant occupancy workload (sears) 29 | - gh-45 Modify DB API for efficient large object support (sears) 30 | - gh-46 Fixed typo in RedisClient (Zlender) 31 | - gh-49 Build fix (sears) 32 | - gh-50 Switch unordered key generation from FNV32 to FNV64 (sears) 33 | - gh-51 Improved Random Number Generation Performance and add Exponential distribution support (sears) 34 | - gh-52 Mongo db fix (sears) 35 | - gh-54 Add mapkeeper driver (m1ch1) 36 | - gh-57 voldemort - enable nio connector (akkumar) 37 | - gh-58 benchmarking with hbase 0.90.5 (akkumar) 38 | - gh-55 VMware vFabric GemFire (sbawaska) 39 | - gh-59 Cassandra 1.0.6 (akkumar) 40 | - gh-62 Oracle NoSQL Database Client (y-namiki) 41 | - gh-64 Mavenisation of YCSB with a (tar ball) distribution packager (hariprasad-k) 42 | - gh-65 Patch for improving the MongoDB Client (singhsiddharth) 43 | 44 | 0.1.3 - 10/26/10 45 | 46 | = Voldemort binding (rsumbaly) 47 | = HBase client improvements (ryanobjc) 48 | = Fixes to Cassandra 0.7 binding (johanoskarsson, nickmbailey) 49 | = Added an interface for exporting the measurements and a JSON implementation. It can write to both stdout and to a file (johanoskarsson) 50 | -Other minor fixes (brianfrankcooper) 51 | 52 | 0.1.2 - 5/12/10 53 | 54 | - MongoDB binding (ypai) 55 | - Cassandra 0.7 binding (johanoskarsson) 56 | - Simple command line interface (brianfrankcooper) 57 | - Faster string generation (tlipcon) 58 | - Avoid Bytes conversion in HBaseClient (tlipcon) 59 | 60 | 0.1.1 - 4/25/10 61 | 62 | - Compiles under 1.5 63 | - Fixes doc and HBaseClient bugs 64 | 65 | 0.1.0 - 4/23/10 66 | 67 | - Initial open source release 68 | -------------------------------------------------------------------------------- /contrib/YCSB/README: -------------------------------------------------------------------------------- 1 | Yahoo! Cloud System Benchmark (YCSB) 2 | ==================================== 3 | 4 | Links 5 | ----- 6 | http://wiki.github.com/brianfrankcooper/YCSB/ 7 | http://research.yahoo.com/Web_Information_Management/YCSB 8 | ycsb-users@yahoogroups.com 9 | 10 | Getting Started 11 | --------------- 12 | 13 | 1. Download the latest release of YCSB: 14 | 15 | wget https://github.com/downloads/brianfrankcooper/YCSB/ycsb-0.1.4.tar.gz 16 | tar xfvz ycsb-0.1.4 17 | cd ycsb-0.1.4 18 | 19 | 2. Set up a database to benchmark. There is a README file under each binding 20 | directory. 21 | 22 | 3. Run YCSB command. 23 | 24 | bin/ycsb load basic -P workloads/workloada 25 | bin/ycsb run basic -P workloads/workloada 26 | 27 | Running the `ycsb` command without any argument will print the usage. 28 | 29 | See https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload 30 | for a detailed documentation on how to run a workload. 31 | 32 | See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for 33 | the list of available workload properties. 34 | -------------------------------------------------------------------------------- /contrib/YCSB/core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.yahoo.ycsb 6 | root 7 | 0.1.4 8 | 9 | 10 | core 11 | Core YCSB 12 | jar 13 | 14 | 15 | 1.9.4 16 | 17 | 18 | 19 | 20 | org.codehaus.jackson 21 | jackson-mapper-asl 22 | ${jackson.api.version} 23 | 24 | 25 | org.codehaus.jackson 26 | jackson-core-asl 27 | ${jackson.api.version} 28 | 29 | 30 | org.testng 31 | testng 32 | 6.1.1 33 | test 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/ByteArrayByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | public class ByteArrayByteIterator extends ByteIterator { 20 | byte[] str; 21 | int off; 22 | final int len; 23 | public ByteArrayByteIterator(byte[] s) { 24 | this.str = s; 25 | this.off = 0; 26 | this.len = s.length; 27 | } 28 | 29 | public ByteArrayByteIterator(byte[] s, int off, int len) { 30 | this.str = s; 31 | this.off = off; 32 | this.len = off + len; 33 | } 34 | 35 | @Override 36 | public boolean hasNext() { 37 | return off < len; 38 | } 39 | 40 | @Override 41 | public byte nextByte() { 42 | byte ret = str[off]; 43 | off++; 44 | return ret; 45 | } 46 | 47 | @Override 48 | public long bytesLeft() { 49 | return len - off; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/DBException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | /** 21 | * Something bad happened while interacting with the database. 22 | */ 23 | public class DBException extends Exception 24 | { 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 6646883591588721475L; 29 | 30 | public DBException(String message) 31 | { 32 | super(message); 33 | } 34 | 35 | public DBException() 36 | { 37 | super(); 38 | } 39 | 40 | public DBException(String message, Throwable cause) 41 | { 42 | super(message,cause); 43 | } 44 | 45 | public DBException(Throwable cause) 46 | { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/DBFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.Properties; 21 | 22 | /** 23 | * Creates a DB layer by dynamically classloading the specified DB class. 24 | */ 25 | public class DBFactory 26 | { 27 | @SuppressWarnings("unchecked") 28 | public static DB newDB(String dbname, Properties properties) throws UnknownDBException 29 | { 30 | ClassLoader classLoader = DBFactory.class.getClassLoader(); 31 | 32 | DB ret=null; 33 | 34 | try 35 | { 36 | Class dbclass = classLoader.loadClass(dbname); 37 | //System.out.println("dbclass.getName() = " + dbclass.getName()); 38 | 39 | ret=(DB)dbclass.newInstance(); 40 | } 41 | catch (Exception e) 42 | { 43 | e.printStackTrace(); 44 | return null; 45 | } 46 | 47 | ret.setProperties(properties); 48 | 49 | return new DBWrapper(ret); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/InputStreamByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | import java.io.InputStream; 20 | 21 | public class InputStreamByteIterator extends ByteIterator { 22 | long len; 23 | InputStream ins; 24 | long off; 25 | 26 | public InputStreamByteIterator(InputStream ins, long len) { 27 | this.len = len; 28 | this.ins = ins; 29 | off = 0; 30 | } 31 | 32 | @Override 33 | public boolean hasNext() { 34 | return off < len; 35 | } 36 | 37 | @Override 38 | public byte nextByte() { 39 | int ret; 40 | try { 41 | ret = ins.read(); 42 | } catch(Exception e) { 43 | throw new IllegalStateException(e); 44 | } 45 | if(ret == -1) { throw new IllegalStateException("Past EOF!"); } 46 | off++; 47 | return (byte)ret; 48 | } 49 | 50 | @Override 51 | public long bytesLeft() { 52 | return len - off; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/RandomByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | /** 20 | * A ByteIterator that generates a random sequence of bytes. 21 | */ 22 | public class RandomByteIterator extends ByteIterator { 23 | private long len; 24 | private long off; 25 | private int bufOff; 26 | private byte[] buf; 27 | 28 | @Override 29 | public boolean hasNext() { 30 | return (off + bufOff) < len; 31 | } 32 | 33 | private void fillBytesImpl(byte[] buffer, int base) { 34 | int bytes = Utils.random().nextInt(); 35 | try { 36 | buffer[base+0] = (byte)(((bytes) & 31) + ' '); 37 | buffer[base+1] = (byte)(((bytes >> 5) & 31) + ' '); 38 | buffer[base+2] = (byte)(((bytes >> 10) & 31) + ' '); 39 | buffer[base+3] = (byte)(((bytes >> 15) & 31) + ' '); 40 | buffer[base+4] = (byte)(((bytes >> 20) & 31) + ' '); 41 | buffer[base+5] = (byte)(((bytes >> 25) & 31) + ' '); 42 | } catch (ArrayIndexOutOfBoundsException e) { /* ignore it */ } 43 | } 44 | 45 | private void fillBytes() { 46 | if(bufOff == buf.length) { 47 | fillBytesImpl(buf, 0); 48 | bufOff = 0; 49 | off += buf.length; 50 | } 51 | } 52 | 53 | public RandomByteIterator(long len) { 54 | this.len = len; 55 | this.buf = new byte[6]; 56 | this.bufOff = buf.length; 57 | fillBytes(); 58 | this.off = 0; 59 | } 60 | 61 | public byte nextByte() { 62 | fillBytes(); 63 | bufOff++; 64 | return buf[bufOff-1]; 65 | } 66 | 67 | @Override 68 | public int nextBuf(byte[] buffer, int bufferOffset) { 69 | int ret; 70 | if(len - off < buffer.length - bufferOffset) { 71 | ret = (int)(len - off); 72 | } else { 73 | ret = buffer.length - bufferOffset; 74 | } 75 | int i; 76 | for(i = 0; i < ret; i+=6) { 77 | fillBytesImpl(buffer, i + bufferOffset); 78 | } 79 | off+=ret; 80 | return ret + bufferOffset; 81 | } 82 | 83 | @Override 84 | public long bytesLeft() { 85 | return len - off - bufOff; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/UnknownDBException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | /** 21 | * Could not create the specified DB. 22 | */ 23 | public class UnknownDBException extends Exception 24 | { 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 459099842269616836L; 29 | 30 | public UnknownDBException(String message) 31 | { 32 | super(message); 33 | } 34 | 35 | public UnknownDBException() 36 | { 37 | super(); 38 | } 39 | 40 | public UnknownDBException(String message, Throwable cause) 41 | { 42 | super(message,cause); 43 | } 44 | 45 | public UnknownDBException(Throwable cause) 46 | { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/WorkloadException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | /** 21 | * The workload tried to do something bad. 22 | */ 23 | public class WorkloadException extends Exception 24 | { 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 8844396756042772132L; 29 | 30 | public WorkloadException(String message) 31 | { 32 | super(message); 33 | } 34 | 35 | public WorkloadException() 36 | { 37 | super(); 38 | } 39 | 40 | public WorkloadException(String message, Throwable cause) 41 | { 42 | super(message,cause); 43 | } 44 | 45 | public WorkloadException(Throwable cause) 46 | { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb.generator; 18 | 19 | /** 20 | * A trivial integer generator that always returns the same value. 21 | * 22 | * @author sears 23 | * 24 | */ 25 | public class ConstantIntegerGenerator extends IntegerGenerator { 26 | private final int i; 27 | /** 28 | * @param i The integer that this generator will always return. 29 | */ 30 | public ConstantIntegerGenerator(int i) { 31 | this.i = i; 32 | } 33 | 34 | @Override 35 | public int nextInt() { 36 | return i; 37 | } 38 | 39 | @Override 40 | public double mean() { 41 | return i; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/generator/CounterGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | 22 | /** 23 | * Generates a sequence of integers 0, 1, ... 24 | */ 25 | public class CounterGenerator extends IntegerGenerator 26 | { 27 | final AtomicInteger counter; 28 | 29 | /** 30 | * Create a counter that starts at countstart 31 | */ 32 | public CounterGenerator(int countstart) 33 | { 34 | counter=new AtomicInteger(countstart); 35 | setLastInt(counter.get()-1); 36 | } 37 | 38 | /** 39 | * If the generator returns numeric (integer) values, return the next value as an int. Default is to return -1, which 40 | * is appropriate for generators that do not return numeric values. 41 | */ 42 | public int nextInt() 43 | { 44 | int ret = counter.getAndIncrement(); 45 | setLastInt(ret); 46 | return ret; 47 | } 48 | @Override 49 | public int lastInt() 50 | { 51 | return counter.get() - 1; 52 | } 53 | @Override 54 | public double mean() { 55 | throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!"); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/generator/FileGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); you 3 | * may not use this file except in compliance with the License. You 4 | * may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 11 | * implied. See the License for the specific language governing 12 | * permissions and limitations under the License. See accompanying 13 | * LICENSE file. 14 | */ 15 | 16 | package com.yahoo.ycsb.generator; 17 | 18 | import java.io.File; 19 | import java.io.FileInputStream; 20 | import java.io.InputStreamReader; 21 | import java.io.BufferedReader; 22 | import java.io.IOException; 23 | 24 | /** 25 | * A generator, whose sequence is the lines of a file. 26 | */ 27 | public class FileGenerator extends Generator 28 | { 29 | String filename; 30 | String current; 31 | BufferedReader reader; 32 | 33 | /** 34 | * Create a FileGenerator with the given file. 35 | * @param _filename The file to read lines from. 36 | */ 37 | public FileGenerator(String _filename) 38 | { 39 | try { 40 | filename = _filename; 41 | File file = new File(filename); 42 | FileInputStream in = new FileInputStream(file); 43 | reader = new BufferedReader(new InputStreamReader(in)); 44 | } catch(IOException e) { 45 | System.err.println("Exception: " + e); 46 | } 47 | } 48 | 49 | /** 50 | * Return the next string of the sequence, ie the next line of the file. 51 | */ 52 | public synchronized String nextString() 53 | { 54 | try { 55 | return current = reader.readLine(); 56 | } catch(NullPointerException e) { 57 | System.err.println("NullPointerException: " + filename + ':' + current); 58 | throw e; 59 | } catch(IOException e) { 60 | System.err.println("Exception: " + e); 61 | return null; 62 | } 63 | } 64 | 65 | /** 66 | * Return the previous read line. 67 | */ 68 | public String lastString() 69 | { 70 | return current; 71 | } 72 | 73 | /** 74 | * Reopen the file to reuse values. 75 | */ 76 | public synchronized void reloadFile() 77 | { 78 | try { 79 | System.err.println("Reload " + filename); 80 | reader.close(); 81 | File file = new File(filename); 82 | FileInputStream in = new FileInputStream(file); 83 | reader = new BufferedReader(new InputStreamReader(in)); 84 | } catch(IOException e) { 85 | System.err.println("Exception: " + e); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/generator/Generator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | /** 21 | * An expression that generates a sequence of string values, following some distribution (Uniform, Zipfian, Sequential, etc.) 22 | */ 23 | public abstract class Generator 24 | { 25 | /** 26 | * Generate the next string in the distribution. 27 | */ 28 | public abstract String nextString(); 29 | 30 | /** 31 | * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. 32 | * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet 33 | * been called, lastString() should return something reasonable. 34 | */ 35 | public abstract String lastString(); 36 | } 37 | 38 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/generator/SkewedLatestGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | /** 21 | * Generate a popularity distribution of items, skewed to favor recent items significantly more than older items. 22 | */ 23 | public class SkewedLatestGenerator extends IntegerGenerator 24 | { 25 | CounterGenerator _basis; 26 | ZipfianGenerator _zipfian; 27 | 28 | public SkewedLatestGenerator(CounterGenerator basis) 29 | { 30 | _basis=basis; 31 | _zipfian=new ZipfianGenerator(_basis.lastInt()); 32 | nextInt(); 33 | } 34 | 35 | /** 36 | * Generate the next string in the distribution, skewed Zipfian favoring the items most recently returned by the basis generator. 37 | */ 38 | public int nextInt() 39 | { 40 | int max=_basis.lastInt(); 41 | int nextint=max-_zipfian.nextInt(max); 42 | setLastInt(nextint); 43 | return nextint; 44 | } 45 | 46 | public static void main(String[] args) 47 | { 48 | SkewedLatestGenerator gen=new SkewedLatestGenerator(new CounterGenerator(1000)); 49 | for (int i=0; i _values; 29 | String _laststring; 30 | UniformIntegerGenerator _gen; 31 | 32 | 33 | /** 34 | * Creates a generator that will return strings from the specified set uniformly randomly 35 | */ 36 | @SuppressWarnings( "unchecked" ) 37 | public UniformGenerator(Vector values) 38 | { 39 | _values=(Vector)values.clone(); 40 | _laststring=null; 41 | _gen=new UniformIntegerGenerator(0,values.size()-1); 42 | } 43 | 44 | /** 45 | * Generate the next string in the distribution. 46 | */ 47 | public String nextString() 48 | { 49 | _laststring=_values.elementAt(_gen.nextInt()); 50 | return _laststring; 51 | } 52 | 53 | /** 54 | * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. 55 | * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet 56 | * been called, lastString() should return something reasonable. 57 | */ 58 | public String lastString() 59 | { 60 | if (_laststring==null) 61 | { 62 | nextString(); 63 | } 64 | return _laststring; 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/generator/UniformIntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | import java.util.Random; 21 | 22 | import com.yahoo.ycsb.Utils; 23 | 24 | /** 25 | * Generates integers randomly uniform from an interval. 26 | */ 27 | public class UniformIntegerGenerator extends IntegerGenerator 28 | { 29 | int _lb,_ub,_interval; 30 | 31 | /** 32 | * Creates a generator that will return integers uniformly randomly from the interval [lb,ub] inclusive (that is, lb and ub are possible values) 33 | * 34 | * @param lb the lower bound (inclusive) of generated values 35 | * @param ub the upper bound (inclusive) of generated values 36 | */ 37 | public UniformIntegerGenerator(int lb, int ub) 38 | { 39 | _lb=lb; 40 | _ub=ub; 41 | _interval=_ub-_lb+1; 42 | } 43 | 44 | @Override 45 | public int nextInt() 46 | { 47 | int ret=Utils.random().nextInt(_interval)+_lb; 48 | setLastInt(ret); 49 | 50 | return ret; 51 | } 52 | 53 | @Override 54 | public double mean() { 55 | return ((double)((long)(_lb + (long)_ub))) / 2.0; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/measurements/OneMeasurement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.measurements; 19 | 20 | import java.io.IOException; 21 | 22 | import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; 23 | 24 | /** 25 | * A single measured metric (such as READ LATENCY) 26 | */ 27 | public abstract class OneMeasurement { 28 | 29 | String _name; 30 | 31 | public String getName() { 32 | return _name; 33 | } 34 | 35 | /** 36 | * @param _name 37 | */ 38 | public OneMeasurement(String _name) { 39 | this._name = _name; 40 | } 41 | 42 | public abstract void reportReturnCode(int code); 43 | 44 | public abstract void measure(int latency); 45 | 46 | public abstract String getSummary(); 47 | 48 | /** 49 | * Export the current measurements to a suitable format. 50 | * 51 | * @param exporter Exporter representing the type of format to write to. 52 | * @throws IOException Thrown if the export failed. 53 | */ 54 | public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException; 55 | } 56 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/measurements/exporter/MeasurementsExporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb.measurements.exporter; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | 22 | /** 23 | * Used to export the collected measurements into a useful format, for example 24 | * human readable text or machine readable JSON. 25 | */ 26 | public interface MeasurementsExporter extends Closeable 27 | { 28 | 29 | /** 30 | * Write a measurement to the exported format. 31 | * 32 | * @param metric Metric name, for example "READ LATENCY". 33 | * @param measurement Measurement name, for example "Average latency". 34 | * @param i Measurement to write. 35 | * @throws IOException if writing failed 36 | */ 37 | public void write(String metric, String measurement, int i) throws IOException; 38 | 39 | /** 40 | * Write a measurement to the exported format. 41 | * 42 | * @param metric Metric name, for example "READ LATENCY". 43 | * @param measurement Measurement name, for example "Average latency". 44 | * @param d Measurement to write. 45 | * @throws IOException if writing failed 46 | */ 47 | public void write(String metric, String measurement, double d) throws IOException; 48 | 49 | } 50 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/main/java/com/yahoo/ycsb/measurements/exporter/TextMeasurementsExporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb.measurements.exporter; 18 | 19 | import java.io.BufferedWriter; 20 | import java.io.IOException; 21 | import java.io.OutputStream; 22 | import java.io.OutputStreamWriter; 23 | 24 | /** 25 | * Write human readable text. Tries to emulate the previous print report method. 26 | */ 27 | public class TextMeasurementsExporter implements MeasurementsExporter 28 | { 29 | 30 | private BufferedWriter bw; 31 | 32 | public TextMeasurementsExporter(OutputStream os) 33 | { 34 | this.bw = new BufferedWriter(new OutputStreamWriter(os)); 35 | } 36 | 37 | public void write(String metric, String measurement, int i) throws IOException 38 | { 39 | bw.write("[" + metric + "], " + measurement + ", " + i); 40 | bw.newLine(); 41 | } 42 | 43 | public void write(String metric, String measurement, double d) throws IOException 44 | { 45 | bw.write("[" + metric + "], " + measurement + ", " + d); 46 | bw.newLine(); 47 | } 48 | 49 | public void close() throws IOException 50 | { 51 | this.bw.close(); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /contrib/YCSB/core/src/test/java/com/yahoo/ycsb/TestByteIterator.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.ycsb; 2 | 3 | import org.testng.annotations.Test; 4 | import static org.testng.AssertJUnit.*; 5 | 6 | public class TestByteIterator { 7 | @Test 8 | public void testRandomByteIterator() { 9 | int size = 100; 10 | ByteIterator itor = new RandomByteIterator(size); 11 | assertTrue(itor.hasNext()); 12 | assertEquals(size, itor.bytesLeft()); 13 | assertEquals(size, itor.toString().getBytes().length); 14 | assertFalse(itor.hasNext()); 15 | assertEquals(0, itor.bytesLeft()); 16 | 17 | itor = new RandomByteIterator(size); 18 | assertEquals(size, itor.toArray().length); 19 | assertFalse(itor.hasNext()); 20 | assertEquals(0, itor.bytesLeft()); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /contrib/YCSB/distribution/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.yahoo.ycsb 7 | root 8 | 0.1.4 9 | 10 | 11 | ycsb 12 | YCSB Release Distribution Builder 13 | pom 14 | 15 | 16 | This module creates the release package of the YCSB with all DB library bindings. 17 | It is only used by the build process and does not contain any real 18 | code of itself. 19 | 20 | 21 | 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-assembly-plugin 26 | ${maven.assembly.version} 27 | 28 | 29 | src/main/assembly/distribution.xml 30 | 31 | false 32 | 33 | 34 | 35 | package 36 | 37 | single 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /contrib/YCSB/distribution/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 3 | package 4 | 5 | tar.gz 6 | 7 | true 8 | 9 | 10 | .. 11 | . 12 | 0644 13 | 14 | README 15 | CHANGELOG 16 | LICENSE.txt 17 | NOTICE.txt 18 | 19 | 20 | 21 | ../bin 22 | bin 23 | 0755 24 | 25 | 26 | ../workloads 27 | workloads 28 | 0644 29 | 30 | 31 | 32 | 33 | true 34 | true 35 | 36 | true 37 | 38 | 39 | 40 | . 41 | 0644 42 | 43 | README 44 | 45 | 46 | 47 | src/main/conf 48 | conf 49 | 0644 50 | 51 | 52 | lib 53 | target 54 | 55 | *.jar 56 | 57 | 0644 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/coreproperties.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | YCSB - Core workload package properties 4 | 5 | 6 |

Yahoo! Cloud Serving Benchmark

7 |

Version 0.1.2

8 |
9 | Home - Core workloads - Tips and FAQ 10 |
11 |

Core workload package properties

12 | The property files used with the core workload generator can specify values for the following properties:

13 |

    14 |
  • fieldcount: the number of fields in a record (default: 10) 15 |
  • fieldlength: the size of each field (default: 100) 16 |
  • readallfields: should reads read all fields (true) or just one (false) (default: true) 17 |
  • readproportion: what proportion of operations should be reads (default: 0.95) 18 |
  • updateproportion: what proportion of operations should be updates (default: 0.05) 19 |
  • insertproportion: what proportion of operations should be inserts (default: 0) 20 |
  • scanproportion: what proportion of operations should be scans (default: 0) 21 |
  • readmodifywriteproportion: what proportion of operations should be read a record, modify it, write it back (default: 0) 22 |
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: uniform) 23 |
  • maxscanlength: for scans, what is the maximum number of records to scan (default: 1000) 24 |
  • scanlengthdistribution: for scans, what distribution should be used to choose the number of records to scan, for each scan, between 1 and maxscanlength (default: uniform) 25 |
  • insertorder: should records be inserted in order by key ("ordered"), or in hashed order ("hashed") (default: hashed) 26 |
27 |
28 | YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. 29 | 30 | 31 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/coreworkloads.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | YCSB - Core workloads 4 | 5 | 6 |

Yahoo! Cloud Serving Benchmark

7 |

Version 0.1.2

8 |
9 | Home - Core workloads - Tips and FAQ 10 |
11 |

Core workloads

12 | YCSB includes a set of core workloads that define a basic benchmark for cloud systems. Of course, you can define your own workloads, as described here. However, 13 | the core workloads are a useful first step, and obtaining these benchmark numbers for a variety of different systems would allow you to understand the performance 14 | tradeoffs of different systems. 15 |

16 | The core workloads consist of six different workloads: 17 |

18 | Workload A: Update heavy workload 19 |

20 | This workload has a mix of 50/50 reads and writes. An application example is a session store recording recent actions. 21 |

22 | Workload B: Read mostly workload 23 |

24 | This workload has a 95/5 reads/write mix. Application example: photo tagging; add a tag is an update, but most operations are to read tags. 25 |

26 | Workload C: Read only 27 |

28 | This workload is 100% read. Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop). 29 |

30 | Workload D: Read latest workload 31 |

32 | In this workload, new records are inserted, and the most recently inserted records are the most popular. Application example: user status updates; people want to read the latest. 33 |

34 | Workload E: Short ranges 35 |

36 | In this workload, short ranges of records are queried, instead of individual records. Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id). 37 |

38 | Workload F: Read-modify-write 39 |

40 | In this workload, the client will read a record, modify it, and write back the changes. Application example: user database, where user records are read and modified by the user or to record user activity. 41 | 42 |

43 |

Running the workloads

44 | All six workloads have a data set which is similar. Workloads D and E insert records during the test run. Thus, to keep the database size consistent, we recommend the following sequence: 45 |
    46 |
  1. Load the database, using workload A's parameter file (workloads/workloada) and the "-load" switch to the client. 47 |
  2. Run workload A (using workloads/workloada and "-t") for a variety of throughputs. 48 |
  3. Run workload B (using workloads/workloadb and "-t") for a variety of throughputs. 49 |
  4. Run workload C (using workloads/workloadc and "-t") for a variety of throughputs. 50 |
  5. Run workload F (using workloads/workloadf and "-t") for a variety of throughputs. 51 |
  6. Run workload D (using workloads/workloadd and "-t") for a variety of throughputs. This workload inserts records, increasing the size of the database. 52 |
  7. Delete the data in the database. 53 |
  8. Reload the database, using workload E's parameter file (workloads/workloade) and the "-load switch to the client. 54 |
  9. Run workload E (using workloads/workloadd and "-t") for a variety of throughputs. This workload inserts records, increasing the size of the database. 55 |
56 |
57 | YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. 58 | 59 | 60 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/images/ycsb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pbailis/ramp-sigmod2014-code/373594e584f4cca85dc2e8ebdb6ff19f3e9ccdd6/contrib/YCSB/doc/images/ycsb.jpg -------------------------------------------------------------------------------- /contrib/YCSB/doc/images/ycsblogo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pbailis/ramp-sigmod2014-code/373594e584f4cca85dc2e8ebdb6ff19f3e9ccdd6/contrib/YCSB/doc/images/ycsblogo-small.png -------------------------------------------------------------------------------- /contrib/YCSB/doc/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | YCSB - Yahoo! Cloud Serving Benchmark 4 | 5 | 6 |

Yahoo! Cloud Serving Benchmark

7 |

Version 0.1.2

8 |
9 | Home - Core workloads - Tips and FAQ 10 |
11 | 17 |
18 | 19 |

Overview

20 | There are many new serving databases available, including: 21 | 33 | It is difficult to decide which system is right for your application, partially because the features differ between 34 | systems, and partially because there is not an easy way to compare the performance of one system versus another. 35 |

36 | The goal of the YCSB project is to develop a framework and common set of workloads for evaluating the performance of 37 | different "key-value" and "cloud" serving stores. The project comprises two things: 38 |

    39 |
  • The YCSB Client, an extensible workload generator 40 |
  • The Core workloads, a set of workload scenarios to be executed by the generator 41 |
42 | Although the core workloads provide a well rounded picture of a system's performance, the Client is extensible so that 43 | you can define new and different workloads to examine system aspects, or application scenarios, not adequately covered by 44 | the core workload. Similarly, the Client is extensible to support benchmarking different databases. Although we include 45 | sample code for benchmarking HBase and Cassandra, it is straightforward to write a new interface layer to benchmark 46 | your favorite database. 47 |

48 | A common use of the tool is to benchmark multiple systems and compare them. For example, you can install multiple systems 49 | on the same hardward configuration, and run the same workloads against each system. Then you can plot the performance 50 | of each system (for example, as latency versus throughput curves) to see when one system does better than another. 51 |

52 | 53 |

Download YCSB

54 | YCSB is available 55 | at http://wiki.github.com/brianfrankcooper/YCSB/. 56 |
57 | 58 |

Getting started

59 | Detailed instructions for using YCSB are available on the GitHub wiki: 60 | http://wiki.github.com/brianfrankcooper/YCSB/getting-started. 61 |
62 | 63 |

Extending YCSB

64 | YCSB is designed to be extensible. It is easy to add a new database interface layer to support benchmarking a new database. It is also easy to define new workloads. 65 | 69 | More details about the entire class structure of YCSB is available here: 70 | 73 |
74 | YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. 75 | 76 | 77 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/allclasses-noframe.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | All Classes 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | All Classes 20 |
21 | 22 | 23 | 24 | 85 | 86 |
BasicDB 25 |
26 | CassandraClient5 27 |
28 | CassandraClient6 29 |
30 | CassandraClient7 31 |
32 | Client 33 |
34 | CommandLine 35 |
36 | CoreWorkload 37 |
38 | CounterGenerator 39 |
40 | DB 41 |
42 | DBException 43 |
44 | DBFactory 45 |
46 | DBWrapper 47 |
48 | DiscreteGenerator 49 |
50 | Generator 51 |
52 | HBaseClient 53 |
54 | IntegerGenerator 55 |
56 | Measurements 57 |
58 | MongoDbClient 59 |
60 | OneMeasurement 61 |
62 | OneMeasurementHistogram 63 |
64 | OneMeasurementTimeSeries 65 |
66 | ScrambledZipfianGenerator 67 |
68 | SkewedLatestGenerator 69 |
70 | UniformGenerator 71 |
72 | UniformIntegerGenerator 73 |
74 | UnknownDBException 75 |
76 | Utils 77 |
78 | Workload 79 |
80 | WorkloadException 81 |
82 | ZipfianGenerator 83 |
84 |
87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/com/yahoo/ycsb/db/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.yahoo.ycsb.db 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | com.yahoo.ycsb.db 20 | 21 | 22 | 35 | 36 |
23 | Classes  24 | 25 |
26 | CassandraClient5 27 |
28 | CassandraClient6 29 |
30 | CassandraClient7 31 |
32 | HBaseClient 33 |
34 | MongoDbClient
37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/com/yahoo/ycsb/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.yahoo.ycsb 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | com.yahoo.ycsb 20 | 21 | 22 | 41 | 42 |
23 | Classes  24 | 25 |
26 | BasicDB 27 |
28 | Client 29 |
30 | CommandLine 31 |
32 | DB 33 |
34 | DBFactory 35 |
36 | DBWrapper 37 |
38 | Utils 39 |
40 | Workload
43 | 44 | 45 | 46 | 47 | 56 | 57 |
48 | Exceptions  49 | 50 |
51 | DBException 52 |
53 | UnknownDBException 54 |
55 | WorkloadException
58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/com/yahoo/ycsb/workloads/package-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | com.yahoo.ycsb.workloads 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | com.yahoo.ycsb.workloads 20 | 21 | 22 | 27 | 28 |
23 | Classes  24 | 25 |
26 | CoreWorkload
29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Generated Documentation (Untitled) 8 | 9 | 20 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | <H2> 31 | Frame Alert</H2> 32 | 33 | <P> 34 | This document is designed to be viewed using the frames feature. If you see this message, you are using a non-frame-capable web client. 35 | <BR> 36 | Link to<A HREF="overview-summary.html">Non-frame version.</A> 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/overview-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Overview List 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 23 | 24 |
22 |
25 | 26 | 27 | 28 | 44 | 45 |
All Classes 29 |

30 | 31 | Packages 32 |
33 | com.yahoo.ycsb 34 |
35 | com.yahoo.ycsb.db 36 |
37 | com.yahoo.ycsb.generator 38 |
39 | com.yahoo.ycsb.measurements 40 |
41 | com.yahoo.ycsb.workloads 42 |
43 |

46 | 47 |

48 |   49 | 50 | 51 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/package-list: -------------------------------------------------------------------------------- 1 | com.yahoo.ycsb 2 | com.yahoo.ycsb.db 3 | com.yahoo.ycsb.generator 4 | com.yahoo.ycsb.measurements 5 | com.yahoo.ycsb.workloads 6 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/resources/inherit.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pbailis/ramp-sigmod2014-code/373594e584f4cca85dc2e8ebdb6ff19f3e9ccdd6/contrib/YCSB/doc/javadoc/resources/inherit.gif -------------------------------------------------------------------------------- /contrib/YCSB/doc/javadoc/stylesheet.css: -------------------------------------------------------------------------------- 1 | /* Javadoc style sheet */ 2 | 3 | /* Define colors, fonts and other style attributes here to override the defaults */ 4 | 5 | /* Page background color */ 6 | body { background-color: #FFFFFF; color:#000000 } 7 | 8 | /* Headings */ 9 | h1 { font-size: 145% } 10 | 11 | /* Table colors */ 12 | .TableHeadingColor { background: #CCCCFF; color:#000000 } /* Dark mauve */ 13 | .TableSubHeadingColor { background: #EEEEFF; color:#000000 } /* Light mauve */ 14 | .TableRowColor { background: #FFFFFF; color:#000000 } /* White */ 15 | 16 | /* Font used in left-hand frame lists */ 17 | .FrameTitleFont { font-size: 100%; font-family: Helvetica, Arial, sans-serif; color:#000000 } 18 | .FrameHeadingFont { font-size: 90%; font-family: Helvetica, Arial, sans-serif; color:#000000 } 19 | .FrameItemFont { font-size: 90%; font-family: Helvetica, Arial, sans-serif; color:#000000 } 20 | 21 | /* Navigation bar fonts and colors */ 22 | .NavBarCell1 { background-color:#EEEEFF; color:#000000} /* Light mauve */ 23 | .NavBarCell1Rev { background-color:#00008B; color:#FFFFFF} /* Dark Blue */ 24 | .NavBarFont1 { font-family: Arial, Helvetica, sans-serif; color:#000000;color:#000000;} 25 | .NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;color:#FFFFFF;} 26 | 27 | .NavBarCell2 { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF; color:#000000} 28 | .NavBarCell3 { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF; color:#000000} 29 | 30 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/parallelclients.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | YCSB - Parallel clients 4 | 5 | 6 |

Yahoo! Cloud Serving Benchmark

7 |

Version 0.1.2

8 |
9 | Home - Core workloads - Tips and FAQ 10 |
11 |

Running multiple clients in parallel

12 | It is straightforward to run the transaction phase of the workload from multiple servers - just start up clients on different servers, each running the same workload. Each client will 13 | produce performance statistics when it is done, and you'll have to aggregate these individual files into a single set of results. 14 |

15 | In some cases it makes sense to load the database using multiple servers. In this case, you will want to partition the records to be loaded among the clients. Normally, YCSB just loads 16 | all of the records (as defined by the recordcount property). However, if you want to partition the load you need to additionally specify two other properties for each client: 17 |

    18 |
  • insertstart: The index of the record to start at. 19 |
  • insertcount: The number of records to insert. 20 |
21 | These properties can be specified in a property file or on the command line using the -p option. 22 |

23 | For example, imagine you want to load 100 million records (so recordcount=100000000). Imagine you want to load with four clients. For the first client: 24 | 

25 | insertstart=0
26 | insertcount=25000000
27 |
28 | For the second client: 29 | 
30 | insertstart=25000000
31 | insertcount=25000000
32 |
33 | For the third client: 34 | 
35 | insertstart=50000000
36 | insertcount=25000000
37 |
38 | And for the fourth client: 39 | 
40 | insertstart=75000000
41 | insertcount=25000000
42 |
43 |
44 | YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. 45 | 46 | 47 | -------------------------------------------------------------------------------- /contrib/YCSB/doc/tipsfaq.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | YCSB - Tips and FAQ 4 | 5 | 6 |

Yahoo! Cloud Serving Benchmark

7 |

Version 0.1.2

8 |
9 | Home - Core workloads - Tips and FAQ 10 |
11 |

Tips

12 | Tip 1 - Carefully adjust the number of threads 13 |

14 | The number of threads determines how much workload you can generate against the database. Imagine that you are trying to run a test with 10,000 operations per second, 15 | but you are only achieving 8,000 operations per second. Is this because the database can't keep up with the load? Not necessarily. Imagine that you are running with 100 16 | client threads (e.g. "-threads 100") and each operation is taking 12 milliseconds on average. Each thread will only be able to generate 83 operations per second, because each 17 | thread operates sequentially. Over 100 threads, your client will only generate 8300 operations per second, even if the database can support more. Increasing the number of threads 18 | ensures there are enough parallel clients hitting the database so that the database, not the client, is the bottleneck. 19 |

20 | To calculate the number of threads needed, you should have some idea of the expected latency. For example, at 10,000 operations per second, we might expect the database 21 | to have a latency of 10-30 milliseconds on average. So you to generate 10,000 operations per second, you will need (Ops per sec / (1000 / avg latency in ms) ), or (10000/(1000/30))=300 threads. 22 | In fact, to be conservative, you might consider having 400 threads. Although this is a lot of threads, each thread will spend most of its time waiting for the database to respond, 23 | so the context switching overhead will be low. 24 |

25 | Experiment with increasing the number of threads, especially if you find you are not reaching your target throughput. Eventually, of course, you will saturate the database 26 | and there will be no way to increase the number of threads to get more throughput (in fact, increasing the number of client threads may make things worse) but you need to have 27 | enough threads to ensure it is the database, not the client, that is the bottleneck. 28 |

29 | YCSB - Yahoo! Research - Contact cooperb@yahoo-inc.com. 30 | 31 | 32 | -------------------------------------------------------------------------------- /contrib/YCSB/install-kaiju-jar.sh: -------------------------------------------------------------------------------- 1 | mvn install:install-file -Dfile=../../target/kaiju-1.0-SNAPSHOT.jar -DgroupId=edu.berkeley.kaiju -DartifactId=kaiju -Dversion=1.0-SNAPSHOT -Dpackaging=jar -------------------------------------------------------------------------------- /contrib/YCSB/kaiju/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.yahoo.ycsb 6 | root 7 | 0.1.4 8 | 9 | 10 | kaiju-binding 11 | Kaiju DB Binding 12 | jar 13 | 14 | 15 | 16 | 17 | org.apache.maven.plugins 18 | maven-assembly-plugin 19 | ${maven.assembly.version} 20 | 21 | 22 | jar-with-dependencies 23 | 24 | false 25 | 26 | 27 | 28 | package 29 | 30 | single 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | edu.berkeley.kaiju 41 | kaiju 42 | 1.0-SNAPSHOT 43 | compile 44 | 45 | 46 | 47 | com.yahoo.ycsb 48 | core 49 | ${project.version} 50 | 51 | 52 | 53 | org.slf4j 54 | slf4j-log4j12 55 | 1.6.4 56 | 57 | 58 | 59 | com.esotericsoftware.kryo 60 | kryo 61 | 2.20 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /contrib/YCSB/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.yahoo.ycsb 7 | root 8 | 0.1.4 9 | pom 10 | 11 | YCSB Root 12 | 13 | 14 | This is the top level project that builds, packages the core and all the DB bindings for YCSB infrastructure. 15 | 16 | 17 | 18 | org.slf4j 19 | slf4j-api 20 | 1.6.4 21 | 22 | 23 | 24 | 25 | 26 | 2.2.1 27 | 0.92.1 28 | 0.7.0 29 | 7.1.0.CR1 30 | 2.1.1 31 | 1.0 32 | 2.9.0 33 | 1.0.1 34 | 2.0.0 35 | 0.81 36 | UTF-8 37 | 0.8.0 38 | 0.9.5.6 39 | 40 | 41 | 42 | 43 | core 44 | kaiju 45 | distribution 46 | 47 | 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-compiler-plugin 53 | 2.3.2 54 | 55 | 1.6 56 | 1.6 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /contrib/YCSB/run-ycsb.sh: -------------------------------------------------------------------------------- 1 | bin/ycsb run kaiju -p hosts=127.0.0.1 -p port=8080 -P workloads/workloada -p operationcount=100000 -p maxexecutiontime=30 -p isolation_level=READ_ATOMIC -p read_atomic_algorithm=KEY_LIST -s -------------------------------------------------------------------------------- /contrib/YCSB/workloads/workloada: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Yahoo! Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. See accompanying 14 | # LICENSE file. 15 | 16 | 17 | # Yahoo! Cloud System Benchmark 18 | # Workload A: Update heavy workload 19 | # Application example: Session store recording recent actions 20 | # 21 | # Read/update ratio: 50/50 22 | # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) 23 | # Request distribution: zipfian 24 | 25 | recordcount=1000 26 | operationcount=1000 27 | workload=com.yahoo.ycsb.workloads.CoreWorkload 28 | 29 | readallfields=true 30 | 31 | readproportion=0.5 32 | updateproportion=0.5 33 | scanproportion=0 34 | insertproportion=0 35 | 36 | requestdistribution=zipfian 37 | 38 | -------------------------------------------------------------------------------- /contrib/YCSB/workloads/workloadb: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Yahoo! Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. See accompanying 14 | # LICENSE file. 15 | 16 | # Yahoo! Cloud System Benchmark 17 | # Workload B: Read mostly workload 18 | # Application example: photo tagging; add a tag is an update, but most operations are to read tags 19 | # 20 | # Read/update ratio: 95/5 21 | # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) 22 | # Request distribution: zipfian 23 | 24 | recordcount=1000 25 | operationcount=1000 26 | workload=com.yahoo.ycsb.workloads.CoreWorkload 27 | 28 | readallfields=true 29 | 30 | readproportion=0.95 31 | updateproportion=0.05 32 | scanproportion=0 33 | insertproportion=0 34 | 35 | requestdistribution=zipfian 36 | 37 | -------------------------------------------------------------------------------- /contrib/YCSB/workloads/workloadc: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Yahoo! Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. See accompanying 14 | # LICENSE file. 15 | 16 | # Yahoo! Cloud System Benchmark 17 | # Workload C: Read only 18 | # Application example: user profile cache, where profiles are constructed elsewhere (e.g., Hadoop) 19 | # 20 | # Read/update ratio: 100/0 21 | # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) 22 | # Request distribution: zipfian 23 | 24 | recordcount=1000 25 | operationcount=1000 26 | workload=com.yahoo.ycsb.workloads.CoreWorkload 27 | 28 | readallfields=true 29 | 30 | readproportion=1 31 | updateproportion=0 32 | scanproportion=0 33 | insertproportion=0 34 | 35 | requestdistribution=zipfian 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /contrib/YCSB/workloads/workloadd: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Yahoo! Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. See accompanying 14 | # LICENSE file. 15 | 16 | # Yahoo! Cloud System Benchmark 17 | # Workload D: Read latest workload 18 | # Application example: user status updates; people want to read the latest 19 | # 20 | # Read/update/insert ratio: 95/0/5 21 | # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) 22 | # Request distribution: latest 23 | 24 | # The insert order for this is hashed, not ordered. The "latest" items may be 25 | # scattered around the keyspace if they are keyed by userid.timestamp. A workload 26 | # which orders items purely by time, and demands the latest, is very different than 27 | # workload here (which we believe is more typical of how people build systems.) 28 | 29 | recordcount=1000 30 | operationcount=1000 31 | workload=com.yahoo.ycsb.workloads.CoreWorkload 32 | 33 | readallfields=true 34 | 35 | readproportion=0.95 36 | updateproportion=0 37 | scanproportion=0 38 | insertproportion=0.05 39 | 40 | requestdistribution=latest 41 | 42 | -------------------------------------------------------------------------------- /contrib/YCSB/workloads/workloade: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Yahoo! Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. See accompanying 14 | # LICENSE file. 15 | 16 | # Yahoo! Cloud System Benchmark 17 | # Workload E: Short ranges 18 | # Application example: threaded conversations, where each scan is for the posts in a given thread (assumed to be clustered by thread id) 19 | # 20 | # Scan/insert ratio: 95/5 21 | # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) 22 | # Request distribution: zipfian 23 | 24 | # The insert order is hashed, not ordered. Although the scans are ordered, it does not necessarily 25 | # follow that the data is inserted in order. For example, posts for thread 342 may not be inserted contiguously, but 26 | # instead interspersed with posts from lots of other threads. The way the YCSB client works is that it will pick a start 27 | # key, and then request a number of records; this works fine even for hashed insertion. 28 | 29 | recordcount=1000 30 | operationcount=1000 31 | workload=com.yahoo.ycsb.workloads.CoreWorkload 32 | 33 | readallfields=true 34 | 35 | readproportion=0 36 | updateproportion=0 37 | scanproportion=0.95 38 | insertproportion=0.05 39 | 40 | requestdistribution=zipfian 41 | 42 | maxscanlength=100 43 | 44 | scanlengthdistribution=uniform 45 | 46 | 47 | -------------------------------------------------------------------------------- /contrib/YCSB/workloads/workloadf: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Yahoo! Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you 4 | # may not use this file except in compliance with the License. You 5 | # may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | # implied. See the License for the specific language governing 13 | # permissions and limitations under the License. See accompanying 14 | # LICENSE file. 15 | 16 | # Yahoo! Cloud System Benchmark 17 | # Workload F: Read-modify-write workload 18 | # Application example: user database, where user records are read and modified by the user or to record user activity. 19 | # 20 | # Read/read-modify-write ratio: 50/50 21 | # Default data size: 1 KB records (10 fields, 100 bytes each, plus key) 22 | # Request distribution: zipfian 23 | 24 | recordcount=1000 25 | operationcount=1000 26 | workload=com.yahoo.ycsb.workloads.CoreWorkload 27 | 28 | readallfields=true 29 | 30 | readproportion=0.5 31 | updateproportion=0 32 | scanproportion=0 33 | insertproportion=0 34 | readmodifywriteproportion=0.5 35 | 36 | requestdistribution=zipfian 37 | 38 | -------------------------------------------------------------------------------- /experiment/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.txt 3 | hosts -------------------------------------------------------------------------------- /experiment/analyze-serverlogs.py: -------------------------------------------------------------------------------- 1 | from os import listdir 2 | from sys import argv 3 | 4 | d = argv[1] 5 | tofind = argv[2] 6 | var = argv[3] 7 | 8 | for f in listdir(argv[1]): 9 | if f.find("IT") == -1: 10 | continue 11 | print f 12 | f = argv[1]+'/'+f 13 | serverresults = {} 14 | for g in listdir(f): 15 | if g.find("S") == -1: 16 | continue 17 | g = f+'/'+g 18 | s = g.split("/S")[1] 19 | lookout = False 20 | for line in open(g+"/server.log"): 21 | if line.find("edu.berkeley.kaiju") != -1: 22 | 23 | if line.find(tofind) != -1: 24 | lookout=True 25 | else: 26 | lookout=False 27 | if lookout and line.find(tofind) == -1 and line.find(var) != -1: 28 | serverresults[s] = line.split("=")[1] 29 | 30 | for s in serverresults: 31 | print s, serverresults[s], 32 | 33 | -------------------------------------------------------------------------------- /experiment/common_funcs.py: -------------------------------------------------------------------------------- 1 | # Common helper functions 2 | 3 | import subprocess 4 | from os import system 5 | from time import sleep 6 | 7 | def run_cmd(hosts, cmd, user="ubuntu", time=1000): 8 | cmd = "parallel-ssh -t %d -O StrictHostKeyChecking=no -l %s -h hosts/%s.txt \"%s\"" % (time, user, hosts, cmd) 9 | if time != 1000: 10 | cmd = "timeout %d %s" % (time, cmd) 11 | print cmd 12 | system(cmd) 13 | 14 | def run_cmd_single(host, cmd, user="ubuntu", time = None): 15 | cmd = "ssh -o StrictHostKeyChecking=no %s@%s \"%s\"" % (user, host, cmd) 16 | if time: 17 | cmd = "timeout %d %s" % (time, cmd) 18 | print cmd 19 | system(cmd) 20 | 21 | def run_cmd_single_bg(host, cmd, user="ubuntu", time = None): 22 | cmd = "ssh -o StrictHostKeyChecking=no %s@%s \"%s\" &" % (user, host, cmd) 23 | print cmd 24 | system(cmd) 25 | 26 | 27 | def start_cmd_disown(host, cmd, user="ubuntu"): 28 | run_cmd_single_bg(host, cmd+" & disown", user) 29 | 30 | 31 | def start_cmd_disown_nobg(host, cmd, user="ubuntu"): 32 | run_cmd_single_bg(host, cmd+" disown", user) 33 | 34 | def run_process_single(host, cmd, user="ubuntu", stdout=None, stderr=None): 35 | subprocess.call("ssh %s@%s \"%s\"" % (user, host, cmd), 36 | stdout=stdout, stderr=stderr, shell=True) 37 | 38 | def upload_file(hosts, local_path, remote_path, user="ubuntu"): 39 | system("cp %s /tmp" % (local_path)) 40 | script = local_path.split("/")[-1] 41 | system("parallel-scp -O StrictHostKeyChecking=no -l %s -h hosts/%s.txt /tmp/%s %s" % (user, hosts, script, remote_path)) 42 | 43 | def run_script(hosts, script, user="ubuntu"): 44 | upload_file(hosts, script.split(" ")[0], "/tmp", user) 45 | run_cmd(hosts, "bash /tmp/%s" % (script.split("/")[-1]), user) 46 | 47 | def fetch_file_single(host, remote, local, user="ubuntu"): 48 | system("scp -o StrictHostKeyChecking=no %s@%s:%s '%s'" % (user, host, remote, local)) 49 | 50 | def fetch_file_single_compressed(host, remote, local, user="ubuntu"): 51 | system("scp -o StrictHostKeyChecking=no %s@%s:%s '%s'" % (user, host, remote, local)) 52 | 53 | def fetch_file_single_compressed_bg(host, remote, local, user="ubuntu"): 54 | system("scp -o StrictHostKeyChecking=no %s@%s:%s '%s' &" % (user, host, remote, local)) 55 | 56 | def get_host_ips(hosts): 57 | return open("hosts/%s.txt" % (hosts)).read().split('\n')[:-1] 58 | 59 | def sed(file, find, repl): 60 | iOpt = '' 61 | print 'sed -i -e %s \'s/%s/%s/g\' %s' % (iOpt, escape(find), escape(repl), file) 62 | system('sed -i -e %s \'s/%s/%s/g\' %s' % (iOpt, escape(find), escape(repl), file)) 63 | 64 | def escape(path): 65 | return path.replace('/', '\/') 66 | 67 | def get_node_ips(): 68 | ret = [] 69 | system("ec2-describe-instances > /tmp/instances.txt") 70 | system("ec2-describe-instances --region us-west-2 >> /tmp/instances.txt") 71 | for line in open("/tmp/instances.txt"): 72 | line = line.split() 73 | if line[0] != "INSTANCE" or line[5] != "running": 74 | continue 75 | # addr, externalip, internalip, ami 76 | ret.append((line[3], line[13], line[14], line[1])) 77 | return ret 78 | 79 | def get_matching_ip(host, hosts): 80 | cips = get_host_ips(hosts) 81 | #argh should use a comprehension/filter; i'm tired 82 | for h in get_node_ips(): 83 | if h[0] == host: 84 | return h[1] 85 | -------------------------------------------------------------------------------- /experiment/example.txt: -------------------------------------------------------------------------------- 1 | # launch a cluster with five clients and five hosts in us-west with ec2 tag 'example' 2 | python setup_hosts.py --color -c us-west-2 -nc 5 -ns 5 --launch --tag example 3 | 4 | # run experiment 'varythreads' on 'example' cluster 5 | python setup_hosts.py --color -c us-west-2 -nc 5 -ns 5 --experiment varythreads --tag example 6 | 7 | # terminate 8 | python setup_hosts.py --color -c us-west-2 -nc 5 -ns 5 --terminate --tag example 9 | -------------------------------------------------------------------------------- /experiment/run-all.sh: -------------------------------------------------------------------------------- 1 | 2 | TAG="testscalability" 3 | 4 | mkdir -p multitest 5 | 6 | for EXPERIMENT in threads rprop txnlen valuesize numkeys 7 | do 8 | python setup_hosts.py --color -c us-west-2 --experiment $EXPERIMENT --tag $TAG --output multitest/$EXPERIMENT 9 | printf "Subject: FUNZO DONEZO `date`\nExperiment $EXPERIMENT has completed, sir\n" > /tmp/msg.txt 10 | done 11 | 12 | printf "Subject: FUNZO DONEZO ALL `date`\nExperiment has completed, sir\n" > /tmp/msg.txt 13 | -------------------------------------------------------------------------------- /py-impl/bloom_filter.py: -------------------------------------------------------------------------------- 1 | 2 | # quick and dirty lift from http://glowingpython.blogspot.nl/2013/01/bloom-filter.html 3 | 4 | class BloomFilter: 5 | def __init__(self, m, k): 6 | """ 7 | m, size of the vector 8 | k, number of hash fnctions to compute 9 | """ 10 | self.m = m 11 | self.vector = [0]*m 12 | self.k = k 13 | 14 | def insert(self, key): 15 | for i in range(self.k): 16 | self.vector[hash(key+str(i)) % self.m] = 1 17 | 18 | def contains(self, key): 19 | for i in range(self.k): 20 | if self.vector[hash(key+str(i)) % self.m] == 0: 21 | return False 22 | return True 23 | 24 | def list_to_bloom(self, keys): 25 | for key in keys: 26 | self.insert(key) 27 | -------------------------------------------------------------------------------- /py-impl/data_item.py: -------------------------------------------------------------------------------- 1 | 2 | class DataItem: 3 | def __init__(self, value, timestamp, txn_keys=None, bloom_filter=None): 4 | self.value = value 5 | self.timestamp = timestamp 6 | self.txn_keys = txn_keys 7 | self.bloom_filter = bloom_filter 8 | -------------------------------------------------------------------------------- /py-impl/demo.py: -------------------------------------------------------------------------------- 1 | 2 | from ramp_server import Partition, RAMPAlgorithm 3 | from ramp_client import Client 4 | from random import sample, random, choice 5 | from string import ascii_uppercase 6 | from threading import Semaphore, Thread 7 | 8 | # RAMPAlgorithm.{Fast, Small, Hybrid} 9 | ALGORITHM = RAMPAlgorithm.Fast 10 | 11 | NUM_PARTITIONS = 5 12 | NUM_CLIENTS = 5 13 | 14 | NUM_TXNS = 1000 15 | READ_PROPORTION = .5 16 | TXN_LENGTH = 4 17 | NUM_KEYS = 100 18 | 19 | KEYS = [str(i) for i in range(0, NUM_KEYS)] 20 | PARTITIONS = [Partition() for _ in range(0, NUM_PARTITIONS)] 21 | 22 | request_sem = Semaphore(NUM_TXNS) 23 | finished_sem = Semaphore() 24 | 25 | def random_string(): 26 | return ''.join(choice(ascii_uppercase) for _ in range(6)) 27 | 28 | def run_client(client): 29 | while(request_sem.acquire(False)): 30 | txn_keys = sample(KEYS, TXN_LENGTH) 31 | 32 | if random() < READ_PROPORTION: 33 | client.get_all(txn_keys) 34 | else: 35 | kvps = {} 36 | value = random_string() 37 | for key in txn_keys: 38 | kvps[key] = value 39 | client.put_all(kvps) 40 | finished_sem.release() 41 | 42 | for c_id in range(0, NUM_CLIENTS): 43 | client = Client(c_id, PARTITIONS, ALGORITHM) 44 | t = Thread(target=run_client, args=(client,)) 45 | t.start() 46 | 47 | finished_sem.acquire(NUM_TXNS) 48 | 49 | print "DONE!" 50 | 51 | -------------------------------------------------------------------------------- /py-impl/ramp_server.py: -------------------------------------------------------------------------------- 1 | 2 | from collections import defaultdict 3 | from threading import Lock 4 | 5 | class RAMPAlgorithm: 6 | Fast, Small, Hybrid = range(3) 7 | 8 | class Partition: 9 | def __init__(self): 10 | # map of maps: key -> [version -> value] 11 | self.versions = defaultdict(lambda: defaultdict(lambda: None)) 12 | self.lastCommit = defaultdict(lambda: None) 13 | 14 | # used for synchronizing last-writer-wins 15 | self.lwwLock = Lock() 16 | 17 | def prepare(self, key, value, timestamp): 18 | self.versions[key][timestamp] = value 19 | 20 | def commit(self, key, timestamp): 21 | self.lwwLock.acquire() 22 | if self.lastCommit[key] < timestamp: 23 | self.lastCommit[key] = timestamp 24 | self.lwwLock.release() 25 | 26 | def getRAMPFast(self, key, ts_required): 27 | if ts_required == None: 28 | return self.versions[key][self.lastCommit[key]] 29 | else: 30 | return self.versions[key][ts_required] 31 | 32 | def getRAMPSmall(self, key, ts_set): 33 | if ts_set == None: 34 | return self.lastCommit[key] 35 | else: 36 | presentTimestamps = self.versions[key].keys() 37 | presentTimestamps.sort() 38 | presentTimestamps.reverse() 39 | for version in presentTimestamps: 40 | if version in ts_set: 41 | return self.versions[key][version] 42 | 43 | return None 44 | 45 | def getRAMPHybrid(self, key, ts_set): 46 | if ts_set == None: 47 | return self.versions[key][self.lastCommit[key]] 48 | else: 49 | return self.getRAMPSmall(key, ts_set) 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/KaijuServer.java: -------------------------------------------------------------------------------- 1 | 2 | package edu.berkeley.kaiju; 3 | 4 | import edu.berkeley.kaiju.config.Config; 5 | import edu.berkeley.kaiju.frontend.FrontendServer; 6 | import edu.berkeley.kaiju.monitor.MetricsManager; 7 | import edu.berkeley.kaiju.net.InboundMessagingService; 8 | import edu.berkeley.kaiju.net.routing.OutboundRouter; 9 | import edu.berkeley.kaiju.service.CooperativeCommitter; 10 | import edu.berkeley.kaiju.service.LockManager; 11 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 12 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 13 | import edu.berkeley.kaiju.service.request.RequestExecutorFactory; 14 | import edu.berkeley.kaiju.service.request.eiger.EigerExecutor; 15 | import edu.berkeley.kaiju.service.request.handler.KaijuServiceHandler; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import java.io.IOException; 20 | 21 | public class KaijuServer { 22 | 23 | private static Logger logger = LoggerFactory.getLogger(KaijuServer.class); 24 | 25 | public static void main(String[] args) { 26 | Config.serverSideInitialize(args); 27 | 28 | MetricsManager.initializeMetrics(); 29 | 30 | MemoryStorageEngine storage = new MemoryStorageEngine(); 31 | LockManager lockManager = new LockManager(); 32 | RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(storage, lockManager); 33 | RequestDispatcher dispatcher = new RequestDispatcher(requestExecutorFactory); 34 | requestExecutorFactory.setEigerExecutor(new EigerExecutor(dispatcher, storage)); 35 | new CooperativeCommitter(storage, new KaijuServiceHandler(dispatcher, storage, lockManager)); 36 | 37 | try { 38 | InboundMessagingService.start(dispatcher); 39 | } catch (IOException e) { 40 | logger.error("Error starting inbound messaging service", e); 41 | System.exit(-1); 42 | } 43 | 44 | logger.info("Started listening for connections..."); 45 | 46 | try { 47 | Thread.sleep(Config.getConfig().bootstrap_time); 48 | } catch (InterruptedException e) { 49 | logger.warn("Bootstrap interrupted", e); 50 | } 51 | 52 | try { 53 | OutboundRouter.initializeRouter(); 54 | } catch (IOException e) { 55 | logger.error("Error starting outbound messaging service", e); 56 | System.exit(-1); 57 | } 58 | 59 | logger.info("Initialized Kaiju internal services; starting Thrift server"); 60 | 61 | KaijuServiceHandler handler = new KaijuServiceHandler(dispatcher, storage, lockManager); 62 | 63 | try { 64 | new FrontendServer(handler, Config.getConfig().thrift_port).serve(); 65 | } catch (Exception e) { 66 | throw new RuntimeException(e); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/config/ClusterConverter.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.config; 2 | 3 | import com.beust.jcommander.IStringConverter; 4 | import com.google.common.collect.Lists; 5 | 6 | import java.net.InetSocketAddress; 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | public class ClusterConverter implements IStringConverter> { 11 | @Override 12 | public List convert(String value) { 13 | List ret = Lists.newArrayList(); 14 | List addressStringList = Arrays.asList(value.split(",")); 15 | for(String addressString : addressStringList) { 16 | ret.add(new InetSocketAddress(addressString.split(":")[0], 17 | Integer.parseInt(addressString.split(":")[1]))); 18 | } 19 | 20 | return ret; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/config/ShortConverter.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.config; 2 | 3 | import com.beust.jcommander.IStringConverter; 4 | 5 | public class ShortConverter implements IStringConverter { 6 | @Override 7 | public Short convert(String value) { 8 | return Short.parseShort(value); 9 | } 10 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/data/DataItem.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.data; 2 | 3 | import edu.berkeley.kaiju.util.Timestamp; 4 | import org.apache.hadoop.util.bloom.BloomFilter; 5 | 6 | import java.util.Collection; 7 | 8 | /* 9 | Since we implemented so many different algorithms for the paper, this 10 | class bloated quite a bit. Annotations inline. 11 | */ 12 | public class DataItem { 13 | // Every item has a version 14 | private long timestamp = Timestamp.NO_TIMESTAMP; 15 | private byte[] value; 16 | 17 | // Used in RAMP-Fast 18 | private Collection transactionKeys = null; 19 | 20 | // Used in RAMP-Hybrid 21 | private BloomFilter bloomTransactionKeys = null; 22 | 23 | public DataItem(long timestamp, byte[] value) { 24 | this.timestamp = timestamp; 25 | this.value = value; 26 | } 27 | 28 | public DataItem(long timestamp, byte[] value, Collection transactionKeys) { 29 | this(timestamp, value); 30 | this.transactionKeys = transactionKeys; 31 | } 32 | 33 | public DataItem(long timestamp, byte[] value, BloomFilter bloomTransactionKeys) { 34 | this(timestamp, value); 35 | this.bloomTransactionKeys = bloomTransactionKeys; 36 | } 37 | 38 | private DataItem() {} 39 | 40 | public long getTimestamp() { 41 | return timestamp; 42 | } 43 | 44 | public boolean hasTransactionKeys() { 45 | return transactionKeys != null; 46 | } 47 | 48 | public byte[] getValue() { 49 | return value; 50 | } 51 | 52 | public Collection getTransactionKeys() { 53 | return transactionKeys; 54 | } 55 | 56 | public BloomFilter getBloomTransactionKeys() { 57 | return bloomTransactionKeys; 58 | } 59 | 60 | public static DataItem getNullItem() { 61 | return new DataItem(Timestamp.NO_TIMESTAMP, new byte[0]); 62 | } 63 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/data/ItemVersion.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.data; 2 | 3 | /* 4 | Utility class for identifying a key-timestamp pair. 5 | In retrospect, not sure why we didn't override HashCode or equals(). 6 | */ 7 | public class ItemVersion { 8 | private String key; 9 | private long timestamp; 10 | byte[] keyBytes = null; 11 | 12 | private ItemVersion() {} 13 | 14 | public ItemVersion(String key, long timestamp) { 15 | this.timestamp = timestamp; 16 | this.key = key; 17 | } 18 | 19 | public String getKey() { 20 | return key; 21 | } 22 | 23 | public long getTimestamp() { 24 | return timestamp; 25 | } 26 | 27 | public byte[] getKeyBytes() { 28 | if(keyBytes == null) { 29 | keyBytes = key.getBytes(); 30 | } 31 | 32 | return keyBytes; 33 | } 34 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/AbortedException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public class AbortedException extends KaijuException { 4 | public AbortedException(String message) { 5 | super(message); 6 | } 7 | 8 | public AbortedException(String message, Throwable throwable) { 9 | super(message, throwable); 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/ClientException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public class ClientException extends KaijuException { 4 | public ClientException(String message) { 5 | super(message); 6 | } 7 | 8 | public ClientException(String message, Throwable throwable) { 9 | super(message, throwable); 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public class ConfigurationException extends KaijuException { 4 | public ConfigurationException(String message) { 5 | super(message); 6 | } 7 | 8 | public ConfigurationException(String message, Throwable throwable) { 9 | super(message, throwable); 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/HandlerException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public class HandlerException extends KaijuException { 4 | public HandlerException(String message) { 5 | super(message); 6 | } 7 | 8 | public HandlerException(String message, Throwable throwable) { 9 | super(message, throwable); 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/KaijuException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public abstract class KaijuException extends Exception 4 | { 5 | public KaijuException(String message) { 6 | super(message); 7 | } 8 | 9 | public KaijuException(String message, Throwable throwable) { 10 | super(message, throwable); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/RemoteOperationException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public class RemoteOperationException extends KaijuException { 4 | public RemoteOperationException(String message) { 5 | super(message); 6 | } 7 | 8 | public RemoteOperationException(String message, Throwable throwable) { 9 | super(message, throwable); 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/exception/VersionNotFoundException.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.exception; 2 | 3 | public class VersionNotFoundException extends KaijuException { 4 | public VersionNotFoundException(String message) { 5 | super(message); 6 | } 7 | 8 | public VersionNotFoundException(String message, Throwable throwable) { 9 | super(message, throwable); 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/KaijuClient.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend; 2 | 3 | import com.esotericsoftware.kryo.KryoException; 4 | import edu.berkeley.kaiju.config.Config; 5 | import edu.berkeley.kaiju.exception.ClientException; 6 | import edu.berkeley.kaiju.exception.KaijuException; 7 | import edu.berkeley.kaiju.frontend.request.ClientGetAllRequest; 8 | import edu.berkeley.kaiju.frontend.request.ClientPutAllRequest; 9 | import edu.berkeley.kaiju.frontend.request.ClientSetIsolationRequest; 10 | import edu.berkeley.kaiju.frontend.response.ClientError; 11 | import edu.berkeley.kaiju.frontend.response.ClientGetAllResponse; 12 | import edu.berkeley.kaiju.util.KryoSerializer; 13 | 14 | import java.io.IOException; 15 | import java.net.Socket; 16 | import java.nio.ByteBuffer; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | /* 21 | Client-side interface for interacting with Kaiju cluster. 22 | Clients can connect to *any* server in the cluster (like Dynamo/Cassandra). 23 | */ 24 | public class KaijuClient { 25 | private Socket clientSocket; 26 | private boolean hasClosed = false; 27 | private KryoSerializer serializer = new KryoSerializer(); 28 | 29 | public KaijuClient(String host, int port) throws IOException { 30 | Config.clientSideInitialize(); 31 | clientSocket = new Socket(host, port); 32 | serializer.setInputStream(clientSocket.getInputStream()); 33 | serializer.setOutputStream(clientSocket.getOutputStream()); 34 | } 35 | 36 | public void close() throws IOException { 37 | hasClosed = true; 38 | serializer.serialize("EXIT"); 39 | clientSocket.close(); 40 | } 41 | 42 | public Map get_all(List keys) throws IOException, KaijuException { 43 | try { 44 | serializer.serialize(new ClientGetAllRequest(keys)); 45 | Object ret = serializer.getObject(); 46 | if (ret instanceof ClientError) { 47 | throw new ClientException(((ClientError) ret).error); 48 | } 49 | 50 | return ((ClientGetAllResponse) ret).keyValuePairs; 51 | } catch (KryoException e) { 52 | if(!hasClosed) { 53 | throw e; 54 | } 55 | 56 | return null; 57 | } 58 | } 59 | 60 | 61 | public void put_all(Map keyValuePairs) throws IOException, KaijuException { 62 | try { 63 | serializer.serialize(new ClientPutAllRequest(keyValuePairs)); 64 | Object ret = serializer.getObject(); 65 | if (ret instanceof ClientError) { 66 | throw new ClientException(((ClientError) ret).error); 67 | } 68 | } catch (KryoException e) { 69 | if(!hasClosed) { 70 | throw e; 71 | } 72 | } 73 | } 74 | 75 | public void setIsolation(Config.IsolationLevel level, Config.ReadAtomicAlgorithm algorithm) throws IOException, KaijuException { 76 | serializer.serialize(new ClientSetIsolationRequest(level, algorithm)); 77 | Object ret = serializer.getObject(); 78 | if (ret instanceof ClientError) { 79 | throw new ClientException(((ClientError) ret).error); 80 | } 81 | } 82 | 83 | public void setIsolation(Config.IsolationLevel level) throws IOException, KaijuException { 84 | setIsolation(level, Config.ReadAtomicAlgorithm.KEY_LIST); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/request/ClientGetAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.request; 2 | 3 | import java.util.List; 4 | 5 | public class ClientGetAllRequest extends ClientRequest { 6 | public List keys; 7 | 8 | ClientGetAllRequest() {} 9 | 10 | public ClientGetAllRequest(List keys) { 11 | this.keys = keys; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/request/ClientPutAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.request; 2 | 3 | import java.util.Map; 4 | 5 | public class ClientPutAllRequest extends ClientRequest { 6 | public Map keyValuePairs; 7 | 8 | ClientPutAllRequest() {} 9 | 10 | public ClientPutAllRequest(Map keyValuePairs) { 11 | this.keyValuePairs = keyValuePairs; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/request/ClientRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.request; 2 | 3 | public abstract class ClientRequest { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/request/ClientSetIsolationRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.request; 2 | 3 | import edu.berkeley.kaiju.config.Config; 4 | 5 | public class ClientSetIsolationRequest extends ClientRequest { 6 | public Config.IsolationLevel isolationLevel; 7 | public Config.ReadAtomicAlgorithm readAtomicAlgorithm; 8 | 9 | ClientSetIsolationRequest() {}; 10 | 11 | public ClientSetIsolationRequest(Config.IsolationLevel isolationLevel, Config.ReadAtomicAlgorithm readAtomicAlgorithm) { 12 | this.isolationLevel = isolationLevel; 13 | this.readAtomicAlgorithm = readAtomicAlgorithm; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/response/ClientError.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.response; 2 | 3 | public class ClientError extends ClientResponse { 4 | public String error; 5 | 6 | ClientError() {} 7 | 8 | public ClientError(String error) { 9 | this.error = error; 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/response/ClientGetAllResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.response; 2 | 3 | import java.util.Map; 4 | 5 | public class ClientGetAllResponse extends ClientResponse { 6 | public Map keyValuePairs; 7 | 8 | ClientGetAllResponse() {} 9 | 10 | public ClientGetAllResponse(Map response) { 11 | this.keyValuePairs = response; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/response/ClientPutAllResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.response; 2 | 3 | public class ClientPutAllResponse extends ClientResponse { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/response/ClientResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.response; 2 | 3 | public abstract class ClientResponse { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/frontend/response/ClientSetIsolationResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.frontend.response; 2 | 3 | public class ClientSetIsolationResponse { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/InboundMessagingService.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net; 2 | 3 | import edu.berkeley.kaiju.config.Config; 4 | import edu.berkeley.kaiju.net.tcp.InternalTCPReceiver; 5 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 6 | 7 | import java.io.IOException; 8 | 9 | /* 10 | Probably didn't need an entire class for this. 11 | */ 12 | public class InboundMessagingService { 13 | public static void start(RequestDispatcher dispatcher) throws IOException { 14 | new InternalTCPReceiver(Config.getConfig().kaiju_port, dispatcher); 15 | } 16 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/callback/IMessageCallback.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.callback; 2 | 3 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 4 | 5 | public interface IMessageCallback { 6 | public void notifyResponse(KaijuMessage response); 7 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/callback/MultiMessageCallback.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.callback; 2 | 3 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 4 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 5 | 6 | import java.util.Collection; 7 | import java.util.Vector; 8 | import java.util.concurrent.Semaphore; 9 | 10 | /* 11 | In the future, I'd probably use futures here. 12 | */ 13 | public class MultiMessageCallback implements IMessageCallback { 14 | private Vector responses = new Vector(); 15 | private Semaphore responseSemaphore; 16 | private int numMessages; 17 | 18 | public MultiMessageCallback(int numMessages) { 19 | responseSemaphore = new Semaphore(0); 20 | this.numMessages = numMessages; 21 | } 22 | 23 | // TODO: add method for a timed block 24 | public Collection blockForResponses() throws InterruptedException { 25 | responseSemaphore.acquireUninterruptibly(numMessages); 26 | 27 | return responses; 28 | } 29 | 30 | public void notifyResponse(KaijuMessage response) { 31 | responses.add((KaijuResponse) response); 32 | responseSemaphore.release(); 33 | } 34 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/callback/SingleMessageCallback.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.callback; 2 | 3 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 4 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 5 | 6 | public class SingleMessageCallback implements IMessageCallback { 7 | private KaijuMessage response = null; 8 | 9 | // TODO: add method for a timed block 10 | public KaijuResponse blockForResponse() throws InterruptedException { 11 | synchronized (this) { 12 | if(response == null) { 13 | this.wait(); 14 | } 15 | } 16 | 17 | return (KaijuResponse) response; 18 | } 19 | 20 | public void notifyResponse(KaijuMessage response) { 21 | synchronized (this) { 22 | this.response = response; 23 | this.notify(); 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/routing/HashingRouter.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.routing; 2 | 3 | import java.io.IOException; 4 | 5 | public class HashingRouter extends OutboundRouter { 6 | public HashingRouter() throws IOException { 7 | super(); 8 | } 9 | 10 | @Override 11 | public int getServerIDByResourceID(int resourceID) { 12 | return Math.abs(resourceID) % senders.size(); 13 | } 14 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/routing/OutboundRouter.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.routing; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.google.common.collect.Maps; 5 | import edu.berkeley.kaiju.config.Config; 6 | import edu.berkeley.kaiju.net.tcp.InternalTCPSender; 7 | 8 | import java.io.IOException; 9 | import java.net.InetSocketAddress; 10 | import java.util.ArrayList; 11 | import java.util.Collection; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /* 16 | "Maintains" the consistent hashing for the cluster. 17 | With an all-to-all topology, we simply have to look up a partition by item. 18 | FWIW, every item has an integer "resource ID"; for strings, their HashCode. 19 | 20 | Replication is not supported. 21 | */ 22 | public abstract class OutboundRouter { 23 | protected static List senders = Lists.newArrayList(); 24 | 25 | private static OutboundRouter router = null; 26 | 27 | public abstract int getServerIDByResourceID(int resourceID); 28 | 29 | public InternalTCPSender getChannelByResourceID(int resourceID) { 30 | return senders.get(getServerIDByResourceID(resourceID)); 31 | } 32 | 33 | public static InternalTCPSender getChannelByServerID(int serverID) { 34 | return senders.get(serverID); 35 | } 36 | 37 | public OutboundRouter() throws IOException { 38 | for(InetSocketAddress serverAddress : Config.getConfig().cluster_servers) { 39 | senders.add(new InternalTCPSender(serverAddress)); 40 | } 41 | } 42 | 43 | public static void initializeRouter() throws IOException { 44 | Config.RoutingMode mode = Config.getConfig().routing_strategy; 45 | router = new HashingRouter(); 46 | } 47 | 48 | public static OutboundRouter getRouter() { 49 | return router; 50 | } 51 | 52 | public static boolean ownsResource(int resourceID) { 53 | return router.getServerIDByResourceID(resourceID) == Config.getConfig().server_id; 54 | } 55 | 56 | /* 57 | This is a very useful function. 58 | Given a set of keys (Strings), it groups the keys by the partition ID of the server 59 | that is responsible for each. Among other things, we use this for batching reads/writes that 60 | are sent to the same server (partition). 61 | */ 62 | public Map> groupKeysByServerID(Collection keys) { 63 | Map> ret = Maps.newHashMap(); 64 | for(String key : keys) { 65 | int serverID = getServerIDByResourceID(key.hashCode()); 66 | if(!ret.containsKey(serverID)) 67 | ret.put(serverID, new ArrayList()); 68 | ret.get(serverID).add(key); 69 | } 70 | 71 | return ret; 72 | } 73 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/tcp/InternalTCPReceiver.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.tcp; 2 | 3 | import com.esotericsoftware.kryo.KryoException; 4 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 5 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 6 | import edu.berkeley.kaiju.util.KryoSerializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.IOException; 11 | import java.net.InetSocketAddress; 12 | import java.nio.channels.Channels; 13 | import java.nio.channels.ServerSocketChannel; 14 | import java.nio.channels.SocketChannel; 15 | 16 | /* 17 | Thread per connection server connection handler for incoming requests. 18 | We separate the reader and the writer thread, though this isn't actually 19 | necessary. 20 | */ 21 | public class InternalTCPReceiver { 22 | private static Logger logger = LoggerFactory.getLogger(InternalTCPReceiver.class); 23 | 24 | public InternalTCPReceiver(final int port, final RequestDispatcher dispatcher) throws IOException { 25 | final ServerSocketChannel serverSocket = ServerSocketChannel.open(); 26 | serverSocket.socket().bind(new InetSocketAddress(port)); 27 | 28 | logger.info("Listening to internal connections on "+serverSocket); 29 | 30 | new Thread(new InternalIncomingServer(serverSocket, dispatcher)).start(); 31 | } 32 | 33 | private class InternalIncomingServer implements Runnable { 34 | ServerSocketChannel serverSocket; 35 | RequestDispatcher dispatcher; 36 | 37 | private InternalIncomingServer(ServerSocketChannel serverSocket, RequestDispatcher dispatcher) { 38 | this.serverSocket = serverSocket; 39 | this.dispatcher = dispatcher; 40 | } 41 | 42 | @Override 43 | public void run() { 44 | while(true) { 45 | try { 46 | SocketChannel clientSocket = serverSocket.accept(); 47 | 48 | new Thread(new InternalConnectionHandler(clientSocket, dispatcher)).start(); 49 | } catch(IOException e) { 50 | logger.warn("Error accepting socket on "+serverSocket, e); 51 | } 52 | } 53 | } 54 | } 55 | 56 | private class InternalConnectionHandler implements Runnable { 57 | RequestDispatcher dispatcher; 58 | KryoSerializer serializer = new KryoSerializer(); 59 | SocketChannel clientSocket; 60 | 61 | public InternalConnectionHandler(SocketChannel clientSocket, RequestDispatcher dispatcher) { 62 | this.dispatcher = dispatcher; 63 | this.clientSocket = clientSocket; 64 | 65 | serializer.setInputStream(Channels.newInputStream(clientSocket)); 66 | } 67 | 68 | @Override 69 | public void run() { 70 | try { 71 | while(true) { 72 | Object toRead = serializer.getObject(); 73 | dispatcher.processInbound((KaijuMessage) toRead); 74 | } 75 | } catch (KryoException e) { 76 | logger.error("Error reading from client "+clientSocket, e); 77 | try { 78 | clientSocket.close(); 79 | } catch(IOException ex) { 80 | logger.error("Exception closing connection: ", ex); 81 | } 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/net/tcp/InternalTCPSender.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.net.tcp; 2 | 3 | import com.google.common.collect.Queues; 4 | import com.google.common.util.concurrent.Uninterruptibles; 5 | import edu.berkeley.kaiju.config.Config; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.util.KryoSerializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | import java.io.OutputStream; 13 | import java.net.InetSocketAddress; 14 | import java.net.Socket; 15 | import java.util.concurrent.BlockingQueue; 16 | 17 | /* 18 | Thread per connection server connection handler for outgoing requests. 19 | We separate the reader and the writer thread, though this isn't actually 20 | necessary. 21 | */ 22 | public class InternalTCPSender { 23 | private static Logger logger = LoggerFactory.getLogger(InternalTCPSender.class); 24 | 25 | private BlockingQueue toSend = Queues.newLinkedBlockingQueue(); 26 | 27 | public InternalTCPSender(final InetSocketAddress socketAddress) throws IOException { 28 | for(int i = 0; i < Config.getConfig().outbound_internal_conn; ++i) { 29 | try { 30 | final KryoSerializer serializer = new KryoSerializer(); 31 | 32 | Socket remoteSocket = new Socket(socketAddress.getAddress(), socketAddress.getPort()); 33 | 34 | if(Config.getConfig().tcp_nodelay) { 35 | remoteSocket.setTcpNoDelay(true); 36 | } 37 | 38 | OutputStream outputStream = remoteSocket.getOutputStream(); 39 | serializer.setOutputStream(outputStream); 40 | 41 | new Thread(new Runnable() { 42 | @Override 43 | public void run() { 44 | while (true) { 45 | 46 | try { 47 | serializer.serialize(Uninterruptibles.takeUninterruptibly(toSend)); 48 | } catch (IOException e) { 49 | logger.error(String.format("Communication error with %s:%s", 50 | socketAddress.getHostName(), 51 | socketAddress.getPort()), e); 52 | } 53 | } 54 | } 55 | }).start(); 56 | } catch (IOException e) { 57 | throw new IOException("Error connecting to "+socketAddress, e); 58 | } 59 | } 60 | } 61 | 62 | public void enqueue(KaijuMessage message) { 63 | toSend.add(message); 64 | } 65 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/RequestExecutorFactory.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request; 2 | 3 | import edu.berkeley.kaiju.service.LockManager; 4 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 5 | import edu.berkeley.kaiju.service.request.eiger.EigerExecutor; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | 8 | /* 9 | Convenience wrapper for passing the right params to 10 | */ 11 | public class RequestExecutorFactory { 12 | private MemoryStorageEngine storageEngine; 13 | private LockManager lockManager; 14 | private EigerExecutor eigerExecutor; 15 | 16 | public RequestExecutorFactory(MemoryStorageEngine storageEngine, LockManager lockManager) { 17 | this.storageEngine = storageEngine; 18 | this.lockManager = lockManager; 19 | } 20 | 21 | /* 22 | The 2PC-CI code in general is a bit annoying/was added later on in the dev cycle. 23 | Artifacts like this are lamentable. 24 | */ 25 | public void setEigerExecutor(EigerExecutor eigerExecutor) { 26 | this.eigerExecutor = eigerExecutor; 27 | } 28 | 29 | public RequestExecutor createExecutor(RequestDispatcher responseDispatcher, 30 | KaijuMessage message) { 31 | return new RequestExecutor(responseDispatcher, storageEngine, lockManager, message, eigerExecutor); 32 | } 33 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/handler/EigerKaijuServiceHandler.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.handler; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.google.common.collect.Maps; 5 | import edu.berkeley.kaiju.data.DataItem; 6 | import edu.berkeley.kaiju.exception.HandlerException; 7 | import edu.berkeley.kaiju.net.routing.OutboundRouter; 8 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 9 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 10 | import edu.berkeley.kaiju.service.request.message.request.EigerGetAllRequest; 11 | import edu.berkeley.kaiju.service.request.message.request.EigerPutAllRequest; 12 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 13 | import edu.berkeley.kaiju.util.Timestamp; 14 | 15 | import java.util.Collection; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.Random; 19 | 20 | public class EigerKaijuServiceHandler implements IKaijuHandler { 21 | RequestDispatcher dispatcher; 22 | public EigerKaijuServiceHandler(RequestDispatcher dispatcher) { 23 | this.dispatcher = dispatcher; 24 | } 25 | 26 | Random random = new Random(); 27 | 28 | public Map get_all(List keys) throws HandlerException { 29 | try { 30 | long readStamp = Timestamp.assignNewTimestamp(); 31 | 32 | Map> keysByServerID = OutboundRouter.getRouter().groupKeysByServerID(keys); 33 | Map requestsByServerID = Maps.newHashMap(); 34 | 35 | for(int serverID : keysByServerID.keySet()) { 36 | requestsByServerID.put(serverID, new EigerGetAllRequest(keysByServerID.get(serverID), readStamp)); 37 | } 38 | 39 | Collection responses = dispatcher.multiRequest(requestsByServerID); 40 | 41 | Map ret = Maps.newHashMap(); 42 | 43 | KaijuResponse.coalesceErrorsIntoException(responses); 44 | 45 | for(KaijuResponse response : responses) { 46 | for(Map.Entry keyValuePair : response.keyValuePairs.entrySet()) { 47 | ret.put(keyValuePair.getKey(), keyValuePair.getValue().getValue()); 48 | } 49 | } 50 | 51 | return ret; 52 | } catch (Exception e) { 53 | throw new HandlerException("Error processing request", e); 54 | } 55 | } 56 | 57 | @Override 58 | public void put_all(Map keyValuePairs) throws HandlerException { 59 | try { 60 | List keys = Lists.newArrayList(keyValuePairs.keySet()); 61 | String coordinatorKey = keys.get(random.nextInt(keys.size())); 62 | 63 | Map> keysByServerID = OutboundRouter.getRouter().groupKeysByServerID( 64 | keyValuePairs.keySet()); 65 | Map requestsByServerID = Maps.newHashMap(); 66 | 67 | long timestamp = Timestamp.assignNewTimestamp(); 68 | 69 | for(int serverID : keysByServerID.keySet()) { 70 | Map keyValuePairsForServer = Maps.newHashMap(); 71 | for(String key : keysByServerID.get(serverID)) { 72 | keyValuePairsForServer.put(key, new DataItem(timestamp, keyValuePairs.get(key))); 73 | } 74 | 75 | requestsByServerID.put(serverID, new EigerPutAllRequest(keyValuePairsForServer, 76 | coordinatorKey, 77 | keyValuePairs.size())); 78 | } 79 | 80 | Collection responses = dispatcher.multiRequestBlockFor(requestsByServerID, 1); 81 | 82 | KaijuResponse.coalesceErrorsIntoException(responses); 83 | } catch (Exception e) { 84 | throw new HandlerException("Error processing request", e); 85 | } 86 | } 87 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/handler/IKaijuHandler.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.handler; 2 | 3 | import edu.berkeley.kaiju.exception.HandlerException; 4 | 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | /* 9 | Clients connect to a front-end server, which executes their transactions for them. 10 | 11 | We extend the IKaijuHandler interface with multiple implementations of read-only and write-only transactions. 12 | */ 13 | public interface IKaijuHandler { 14 | public Map get_all(List keys) throws HandlerException; 15 | public void put_all(Map keyValuePairs) throws HandlerException; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/handler/ReadAtomicListBasedKaijuServiceHandler.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.handler; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.google.common.collect.Maps; 5 | import edu.berkeley.kaiju.data.DataItem; 6 | import edu.berkeley.kaiju.exception.HandlerException; 7 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | import edu.berkeley.kaiju.util.Timestamp; 10 | 11 | import java.util.Collection; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.Random; 15 | 16 | public class ReadAtomicListBasedKaijuServiceHandler extends ReadAtomicKaijuServiceHandler { 17 | 18 | public ReadAtomicListBasedKaijuServiceHandler(RequestDispatcher dispatcher) { 19 | super(dispatcher); 20 | } 21 | 22 | public Map get_all(List keys) throws HandlerException { 23 | try { 24 | Collection first_round_responses = fetch_from_server(keys); 25 | 26 | Map latestUpdateForKey = Maps.newHashMap(); 27 | 28 | // latestUpdateForKey will contain v_latest in Algorithm 1 29 | for(KaijuResponse response : first_round_responses) { 30 | for(Map.Entry keyValuePair : response.keyValuePairs.entrySet()) { 31 | if(keyValuePair.getValue().getTransactionKeys() == null || 32 | keyValuePair.getValue().getTransactionKeys().isEmpty()) { 33 | 34 | if(!latestUpdateForKey.containsKey(keyValuePair.getKey())) { 35 | latestUpdateForKey.put(keyValuePair.getKey(), Timestamp.NO_TIMESTAMP); 36 | } 37 | continue; 38 | } 39 | 40 | for(String key : keyValuePair.getValue().getTransactionKeys()) { 41 | if(!latestUpdateForKey.containsKey(key) || latestUpdateForKey.get(key) < keyValuePair.getValue().getTimestamp()) { 42 | latestUpdateForKey.put(key, keyValuePair.getValue().getTimestamp()); 43 | } 44 | } 45 | 46 | 47 | } 48 | } 49 | 50 | Map ret = Maps.newHashMap(); 51 | 52 | Map second_round_required = Maps.newHashMap(); 53 | 54 | for(KaijuResponse response : first_round_responses) { 55 | for(Map.Entry keyValuePair : response.keyValuePairs.entrySet()) { 56 | long latestSeenValue = latestUpdateForKey.get(keyValuePair.getKey()); 57 | // if we need to fetch an updated value 58 | if(latestSeenValue > keyValuePair.getValue().getTimestamp()) { 59 | second_round_required.put(keyValuePair.getKey(), latestSeenValue); 60 | } 61 | // saw the last updated value 62 | else { 63 | ret.put(keyValuePair.getKey(), keyValuePair.getValue().getValue()); 64 | } 65 | } 66 | } 67 | 68 | // weren't missing any updates! 69 | if(second_round_required.isEmpty()) 70 | return ret; 71 | 72 | Collection second_round_responses = fetch_by_version_from_server(second_round_required); 73 | 74 | for(KaijuResponse response : second_round_responses) { 75 | for(Map.Entry keyValuePair : response.keyValuePairs.entrySet()) { 76 | ret.put(keyValuePair.getKey(), keyValuePair.getValue().getValue()); 77 | } 78 | } 79 | 80 | return ret; 81 | } catch (Exception e) { 82 | throw new HandlerException("Error processing request", e); 83 | } 84 | } 85 | 86 | // attach the write set as metadata 87 | public DataItem instantiateKaijuItem(byte[] value, 88 | Collection allKeys, 89 | long timestamp) { 90 | return new DataItem(timestamp, value, Lists.newArrayList(allKeys)); 91 | } 92 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/handler/ReadAtomicStampBasedKaijuServiceHandler.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.handler; 2 | 3 | import com.beust.jcommander.internal.Sets; 4 | import com.google.common.collect.Maps; 5 | import edu.berkeley.kaiju.data.DataItem; 6 | import edu.berkeley.kaiju.exception.HandlerException; 7 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Collection; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | public class ReadAtomicStampBasedKaijuServiceHandler extends ReadAtomicKaijuServiceHandler { 15 | public ReadAtomicStampBasedKaijuServiceHandler(RequestDispatcher dispatcher) { 16 | super(dispatcher); 17 | } 18 | 19 | public Map get_all(List keys) throws HandlerException { 20 | try { 21 | Collection first_round_responses = fetch_stamps_from_server(keys); 22 | 23 | Collection first_round_stamps = Sets.newHashSet(); 24 | 25 | for(KaijuResponse response : first_round_responses) { 26 | first_round_stamps.addAll(response.timestamps); 27 | } 28 | 29 | Map ret = Maps.newHashMap(); 30 | 31 | // didn't get any responses 32 | if(first_round_stamps.isEmpty()) { 33 | for(String key : keys) { 34 | ret.put(key, new byte[0]); 35 | } 36 | 37 | return ret; 38 | } 39 | 40 | Collection second_round_responses = fetch_by_stamplist_from_server(keys, first_round_stamps); 41 | 42 | for(KaijuResponse response : second_round_responses) { 43 | for(Map.Entry keyValuePair : response.keyValuePairs.entrySet()) { 44 | ret.put(keyValuePair.getKey(), keyValuePair.getValue().getValue()); 45 | } 46 | } 47 | 48 | return ret; 49 | } catch (Exception e) { 50 | throw new HandlerException("Error processing request", e); 51 | } 52 | } 53 | 54 | // RAMP-Small just uses a timestamp 55 | public DataItem instantiateKaijuItem(byte[] value, 56 | Collection allKeys, 57 | long timestamp) { 58 | return new DataItem(timestamp, value); 59 | } 60 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/handler/ReadCommittedKaijuServiceHandler.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.handler; 2 | 3 | import com.google.common.collect.Maps; 4 | import edu.berkeley.kaiju.data.DataItem; 5 | import edu.berkeley.kaiju.exception.HandlerException; 6 | import edu.berkeley.kaiju.net.routing.OutboundRouter; 7 | import edu.berkeley.kaiju.service.request.RequestDispatcher; 8 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 9 | import edu.berkeley.kaiju.service.request.message.request.GetAllRequest; 10 | import edu.berkeley.kaiju.service.request.message.request.PutAllRequest; 11 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 12 | import edu.berkeley.kaiju.util.Timestamp; 13 | 14 | import java.util.Collection; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | /* 19 | Vanilla NWNR implementation. 20 | */ 21 | public class ReadCommittedKaijuServiceHandler implements IKaijuHandler { 22 | RequestDispatcher dispatcher; 23 | public ReadCommittedKaijuServiceHandler(RequestDispatcher dispatcher) { 24 | this.dispatcher = dispatcher; 25 | } 26 | 27 | public Map get_all(List keys) throws HandlerException { 28 | try { 29 | Map> keysByServerID = OutboundRouter.getRouter().groupKeysByServerID(keys); 30 | Map requestsByServerID = Maps.newHashMap(); 31 | 32 | for(int serverID : keysByServerID.keySet()) { 33 | requestsByServerID.put(serverID, new GetAllRequest(keysByServerID.get(serverID))); 34 | } 35 | 36 | Collection responses = dispatcher.multiRequest(requestsByServerID); 37 | 38 | Map ret = Maps.newHashMap(); 39 | 40 | KaijuResponse.coalesceErrorsIntoException(responses); 41 | 42 | for(KaijuResponse response : responses) { 43 | for(Map.Entry keyValuePair : response.keyValuePairs.entrySet()) { 44 | ret.put(keyValuePair.getKey(), keyValuePair.getValue().getValue()); 45 | } 46 | } 47 | 48 | return ret; 49 | } catch (Exception e) { 50 | throw new HandlerException("Error processing request", e); 51 | } 52 | } 53 | 54 | public void put_all(Map keyValuePairs) throws HandlerException { 55 | try { 56 | Map> keysByServerID = OutboundRouter.getRouter().groupKeysByServerID(keyValuePairs.keySet()); 57 | Map requestsByServerID = Maps.newHashMap(); 58 | 59 | long timestamp = Timestamp.assignNewTimestamp(); 60 | 61 | for(int serverID : keysByServerID.keySet()) { 62 | Map keyValuePairsForServer = Maps.newHashMap(); 63 | for(String key : keysByServerID.get(serverID)) { 64 | keyValuePairsForServer.put(key, new DataItem(timestamp, keyValuePairs.get(key))); 65 | } 66 | 67 | requestsByServerID.put(serverID, new PutAllRequest(keyValuePairsForServer)); 68 | } 69 | 70 | Collection responses = dispatcher.multiRequest(requestsByServerID); 71 | 72 | KaijuResponse.coalesceErrorsIntoException(responses); 73 | } catch (Exception e) { 74 | throw new HandlerException("Error processing request", e); 75 | } 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/KaijuMessage.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message; 2 | 3 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 4 | 5 | /* 6 | To allow multiple outstanding RPCs on a given socket, each RPC message is assigned 7 | a requestID (by the sender; see util.Timestamp) and a sender ID (for dispatch). 8 | */ 9 | public abstract class KaijuMessage { 10 | public int requestID; 11 | public short senderID; 12 | 13 | public boolean isResponse() { 14 | return this instanceof KaijuResponse; 15 | } 16 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/CheckPreparedRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.DataItem; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Collection; 11 | import java.util.List; 12 | 13 | public class CheckPreparedRequest extends KaijuMessage implements IKaijuRequest { 14 | public long timestamp; 15 | public Collection toCheck; 16 | 17 | public CheckPreparedRequest() { } 18 | 19 | public CheckPreparedRequest(long timestamp, Collection toCheck) { 20 | this.timestamp = timestamp; 21 | this.toCheck = toCheck; 22 | } 23 | 24 | @Override 25 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 26 | KaijuException { 27 | for(String item : toCheck) { 28 | if(!storageEngine.isPreparedOrHigherCommitted(item, timestamp)) { 29 | return new KaijuResponse(false); 30 | } 31 | } 32 | 33 | return new KaijuResponse(true); 34 | } 35 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/CommitPutAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | public class CommitPutAllRequest extends KaijuMessage implements IKaijuRequest { 10 | public long timestamp; 11 | 12 | private CommitPutAllRequest() {} 13 | 14 | public CommitPutAllRequest(long timestamp) { 15 | this.timestamp = timestamp; 16 | } 17 | 18 | @Override 19 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 20 | KaijuException { 21 | storageEngine.commit(timestamp); 22 | return new KaijuResponse(); 23 | } 24 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/EigerCheckCommitRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import com.beust.jcommander.internal.Sets; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Collection; 11 | 12 | public class EigerCheckCommitRequest extends KaijuMessage implements IKaijuRequest { 13 | public Collection toCheck; 14 | public long readTimestamp; 15 | 16 | public EigerCheckCommitRequest() {} 17 | 18 | public EigerCheckCommitRequest(long readTimestamp) { 19 | toCheck = Sets.newHashSet(); 20 | this.readTimestamp = readTimestamp; 21 | } 22 | 23 | @Override 24 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 25 | KaijuException { 26 | throw new RuntimeException("SHOULD NOT BE CALLED"); 27 | } 28 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/EigerCommitRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 4 | 5 | public class EigerCommitRequest extends KaijuMessage { 6 | public long transactionID; 7 | public long commitTime; 8 | 9 | private EigerCommitRequest() {} 10 | 11 | public EigerCommitRequest(long transactionID, long commitTime) { 12 | this.transactionID = transactionID; 13 | this.commitTime = commitTime; 14 | } 15 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/EigerGetAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | import java.util.Collection; 10 | 11 | public class EigerGetAllRequest extends KaijuMessage implements IKaijuRequest { 12 | public Collection keys; 13 | public long readTimestamp; 14 | 15 | private EigerGetAllRequest() {} 16 | 17 | public EigerGetAllRequest(Collection keys, long readTimestamp) { 18 | this.keys = keys; 19 | this.readTimestamp = readTimestamp; 20 | } 21 | 22 | @Override 23 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 24 | KaijuException { 25 | throw new RuntimeException("SHOULD NOT BE CALLED"); 26 | } 27 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/EigerPutAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.DataItem; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Map; 11 | 12 | public class EigerPutAllRequest extends KaijuMessage implements IKaijuRequest { 13 | public Map keyValuePairs; 14 | public String coordinatorKey; 15 | public int totalNumKeys; 16 | 17 | private EigerPutAllRequest() {} 18 | 19 | public EigerPutAllRequest(Map keyValuePairs, String coordinatorKey, int totalNumKeys) { 20 | this.keyValuePairs = keyValuePairs; 21 | this.coordinatorKey = coordinatorKey; 22 | this.totalNumKeys = totalNumKeys; 23 | } 24 | 25 | @Override 26 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 27 | KaijuException { 28 | throw new RuntimeException("SHOULD NOT BE CALLED"); 29 | } 30 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/GetAllByTimestampListRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | import java.util.Collection; 10 | 11 | public class GetAllByTimestampListRequest extends KaijuMessage implements IKaijuRequest { 12 | Collection keys; 13 | Collection versions; 14 | 15 | private GetAllByTimestampListRequest() {} 16 | 17 | public GetAllByTimestampListRequest(Collection keys, Collection versions) { 18 | this.keys = keys; 19 | this.versions = versions; 20 | } 21 | 22 | @Override 23 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 24 | KaijuException { 25 | KaijuResponse ret = new KaijuResponse(); 26 | ret.keyValuePairs = storageEngine.getAllByVersionList(keys, versions); 27 | return ret; 28 | } 29 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/GetAllByTimestampRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.ItemVersion; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Collection; 11 | 12 | public class GetAllByTimestampRequest extends KaijuMessage implements IKaijuRequest { 13 | public Collection versions; 14 | 15 | private GetAllByTimestampRequest() {} 16 | 17 | public GetAllByTimestampRequest(Collection versions) { 18 | this.versions = versions; 19 | } 20 | 21 | @Override 22 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 23 | KaijuException { 24 | KaijuResponse ret = new KaijuResponse(); 25 | ret.keyValuePairs = storageEngine.getAllByVersion(versions); 26 | return ret; 27 | } 28 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/GetAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | import java.util.Collection; 10 | 11 | public class GetAllRequest extends KaijuMessage implements IKaijuRequest { 12 | public Collection keys; 13 | 14 | private GetAllRequest() {} 15 | 16 | public GetAllRequest(Collection keys) { 17 | this.keys = keys; 18 | } 19 | 20 | @Override 21 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 22 | KaijuException { 23 | return new KaijuResponse(storageEngine.getAll(keys)); 24 | } 25 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/GetEachByTimestampListRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | import java.util.Collection; 10 | import java.util.Map; 11 | 12 | public class GetEachByTimestampListRequest extends KaijuMessage implements IKaijuRequest { 13 | private Map> keyLists; 14 | 15 | private GetEachByTimestampListRequest() {} 16 | 17 | public GetEachByTimestampListRequest(Map> keyLists) { 18 | this.keyLists = keyLists; 19 | } 20 | 21 | @Override 22 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 23 | KaijuException { 24 | KaijuResponse ret = new KaijuResponse(); 25 | ret.keyValuePairs = storageEngine.getEachByVersionList(keyLists); 26 | return ret; 27 | } 28 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/GetRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.service.LockManager; 4 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 5 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 6 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 7 | 8 | public class GetRequest extends KaijuMessage implements IKaijuRequest { 9 | public String key; 10 | 11 | private GetRequest() {} 12 | 13 | public GetRequest(String key) { 14 | this.key = key; 15 | } 16 | 17 | @Override 18 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) { 19 | return new KaijuResponse(storageEngine.get(key)); 20 | } 21 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/GetTimestampsRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | import java.util.Collection; 10 | 11 | public class GetTimestampsRequest extends KaijuMessage implements IKaijuRequest { 12 | public Collection keys; 13 | 14 | private GetTimestampsRequest() {} 15 | 16 | public GetTimestampsRequest(Collection keys) { 17 | this.keys = keys; 18 | } 19 | 20 | @Override 21 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 22 | KaijuException { 23 | return new KaijuResponse(storageEngine.getTimestamps(keys)); 24 | } 25 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/IKaijuRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 7 | 8 | /* 9 | Every RPC (except for Eiger/E-PCI requests) simply extends the following interface, which is executed on the 10 | remote server (and passed a handle to its storage engine and lock manager.) 11 | */ 12 | public interface IKaijuRequest { 13 | public KaijuResponse processRequest(MemoryStorageEngine engine, LockManager lockManager) throws KaijuException; 14 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/PreparePutAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.DataItem; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Map; 11 | 12 | public class PreparePutAllRequest extends KaijuMessage implements IKaijuRequest { 13 | public Map keyValuePairs; 14 | 15 | private PreparePutAllRequest() {} 16 | 17 | public PreparePutAllRequest(Map keyValuePairs) { 18 | this.keyValuePairs = keyValuePairs; 19 | } 20 | 21 | @Override 22 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 23 | KaijuException { 24 | storageEngine.prepare(keyValuePairs); 25 | return new KaijuResponse(); 26 | } 27 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/PutAllRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.DataItem; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | import java.util.Map; 11 | 12 | public class PutAllRequest extends KaijuMessage implements IKaijuRequest { 13 | public Map keyValuePairs; 14 | 15 | private PutAllRequest() {} 16 | 17 | public PutAllRequest(Map keyValuePairs) { 18 | this.keyValuePairs = keyValuePairs; 19 | } 20 | 21 | @Override 22 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 23 | KaijuException { 24 | storageEngine.putAll(keyValuePairs); 25 | return new KaijuResponse(); 26 | } 27 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/PutRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.DataItem; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 9 | 10 | public class PutRequest extends KaijuMessage implements IKaijuRequest { 11 | public String key; 12 | public DataItem item; 13 | 14 | private PutRequest() {}; 15 | 16 | public PutRequest(String key, DataItem item) { 17 | this.key = key; 18 | this.item = item; 19 | } 20 | 21 | @Override 22 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 23 | KaijuException { 24 | storageEngine.put(key, item); 25 | return new KaijuResponse(); 26 | } 27 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/ReadLockRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.LockManager.LockDuration; 6 | import edu.berkeley.kaiju.service.LockManager.LockType; 7 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 8 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 9 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 10 | 11 | public class ReadLockRequest extends KaijuMessage implements IKaijuRequest { 12 | public String key; 13 | public LockDuration lockDuration; 14 | 15 | private ReadLockRequest() {} 16 | 17 | public ReadLockRequest(String key, LockDuration lockDuration) { 18 | this.key = key; 19 | this.lockDuration = lockDuration; 20 | } 21 | 22 | @Override 23 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 24 | KaijuException { 25 | KaijuResponse ret = new KaijuResponse(); 26 | lockManager.lock(LockType.READ, key); 27 | ret.dataItem = storageEngine.get(key); 28 | if(lockDuration == LockDuration.SHORT) { 29 | lockManager.unlock(key); 30 | } 31 | 32 | return ret; 33 | } 34 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/UnlockRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.exception.KaijuException; 4 | import edu.berkeley.kaiju.service.LockManager; 5 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 6 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 7 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 8 | 9 | import java.util.Collection; 10 | 11 | public class UnlockRequest extends KaijuMessage implements IKaijuRequest { 12 | public Collection keys; 13 | 14 | private UnlockRequest() {} 15 | 16 | public UnlockRequest(Collection keys) { 17 | this.keys = keys; 18 | } 19 | 20 | @Override 21 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 22 | KaijuException { 23 | lockManager.unlock(keys); 24 | return new KaijuResponse(); 25 | } 26 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/request/WriteLockRequest.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.request; 2 | 3 | import edu.berkeley.kaiju.data.DataItem; 4 | import edu.berkeley.kaiju.exception.KaijuException; 5 | import edu.berkeley.kaiju.service.LockManager; 6 | import edu.berkeley.kaiju.service.LockManager.LockDuration; 7 | import edu.berkeley.kaiju.service.LockManager.LockType; 8 | import edu.berkeley.kaiju.service.MemoryStorageEngine; 9 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 10 | import edu.berkeley.kaiju.service.request.message.response.KaijuResponse; 11 | 12 | public class WriteLockRequest extends KaijuMessage implements IKaijuRequest { 13 | public String key; 14 | public DataItem item; 15 | public LockDuration lockDuration; 16 | 17 | private WriteLockRequest() {} 18 | 19 | public WriteLockRequest(String key, DataItem item, LockDuration lockDuration) { 20 | this.key = key; 21 | this.item = item; 22 | this.lockDuration = lockDuration; 23 | } 24 | 25 | @Override 26 | public KaijuResponse processRequest(MemoryStorageEngine storageEngine, LockManager lockManager) throws 27 | KaijuException { 28 | lockManager.lock(LockType.WRITE, key); 29 | storageEngine.put(key, item); 30 | if(lockDuration == LockDuration.SHORT) { 31 | lockManager.unlock(key); 32 | } 33 | 34 | return new KaijuResponse(); 35 | } 36 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/response/EigerCheckCommitResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.response; 2 | 3 | import java.util.Map; 4 | 5 | public class EigerCheckCommitResponse extends KaijuResponse { 6 | public Map commitTimes; 7 | 8 | private EigerCheckCommitResponse() {} 9 | 10 | public EigerCheckCommitResponse(Map commitTimes) { 11 | this.commitTimes = commitTimes; 12 | } 13 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/response/EigerPreparedResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.response; 2 | 3 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 4 | 5 | public class EigerPreparedResponse extends KaijuMessage { 6 | public long transactionID; 7 | public int numKeys; 8 | public long preparedTime; 9 | 10 | private EigerPreparedResponse() {} 11 | 12 | public EigerPreparedResponse(long transactionID, int numKeys, long preparedTime) { 13 | this.transactionID = transactionID; 14 | this.numKeys = numKeys; 15 | this.preparedTime = preparedTime; 16 | } 17 | } -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/service/request/message/response/KaijuResponse.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.service.request.message.response; 2 | 3 | import com.google.common.base.Joiner; 4 | import com.google.common.collect.Lists; 5 | import edu.berkeley.kaiju.data.DataItem; 6 | import edu.berkeley.kaiju.exception.RemoteOperationException; 7 | import edu.berkeley.kaiju.service.request.message.KaijuMessage; 8 | 9 | import java.util.Collection; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | public class KaijuResponse extends KaijuMessage { 14 | // used in GET_ALL, GETALL_BY_TIMESTAMP 15 | public Map keyValuePairs; 16 | 17 | // used in GET_TIMESTAMPS_REQUEST 18 | public Collection timestamps; 19 | 20 | // used in GET_MESSAGE 21 | public DataItem dataItem; 22 | 23 | // used in CHECK_PREPARED 24 | public Boolean prepared; 25 | 26 | public List errors; 27 | 28 | public KaijuResponse(Map keyValuePairs) { 29 | this.keyValuePairs = keyValuePairs; 30 | } 31 | 32 | public KaijuResponse(Collection timestamps) { 33 | this.timestamps = timestamps; 34 | } 35 | 36 | public KaijuResponse(DataItem dataItem) { 37 | this.dataItem = dataItem; 38 | } 39 | 40 | public KaijuResponse(List errors) { 41 | this.errors = errors; 42 | } 43 | 44 | public KaijuResponse(boolean prepared) { 45 | this.prepared = prepared; 46 | } 47 | 48 | public KaijuResponse() {} 49 | 50 | public boolean hasErrors() { 51 | return errors != null; 52 | } 53 | 54 | public String getErrorString() { 55 | Joiner joiner = Joiner.on(";"); 56 | return joiner.join(errors); 57 | } 58 | 59 | public static void coalesceErrorsIntoException(Collection responses) throws 60 | RemoteOperationException { 61 | Collection errors = null; 62 | for(KaijuResponse response : responses) { 63 | if(response.hasErrors()) { 64 | if(errors == null) 65 | errors = Lists.newArrayList(); 66 | errors.add(response.senderID + ": " + response.getErrorString()); 67 | } 68 | } 69 | 70 | if(errors != null) 71 | throw new RemoteOperationException(Joiner.on("; ").join(errors)); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/edu/berkeley/kaiju/util/Timestamp.java: -------------------------------------------------------------------------------- 1 | package edu.berkeley.kaiju.util; 2 | 3 | import edu.berkeley.kaiju.config.Config; 4 | 5 | public class Timestamp { 6 | /* 7 | Timestamps are comprised of: 8 | 38 bits for realtime, measured in ms -> 8.7 year wraparound 9 | 14 bits for sequence no within a ms -> max 16,384,000 operations/sec 10 | 12 bits for serverid 11 | */ 12 | 13 | public final static long NO_TIMESTAMP = -1; 14 | 15 | private static volatile long latestMillis = -1; 16 | private static volatile int sequenceNo = 0; 17 | 18 | public static long assignNewTimestamp() { 19 | return assignNewTimestamp(-1); 20 | } 21 | 22 | public static synchronized long assignNewTimestamp(long min) { 23 | long chosenTime = Math.max(System.currentTimeMillis(), min >> 26); 24 | int chosenSeqNo = 0; 25 | 26 | if (latestMillis < chosenTime) { 27 | latestMillis = chosenTime; 28 | sequenceNo = chosenSeqNo = 0; 29 | } else if (latestMillis == chosenTime) { 30 | sequenceNo++; 31 | chosenSeqNo = sequenceNo; 32 | } else { 33 | chosenTime = latestMillis; 34 | sequenceNo++; 35 | chosenSeqNo = sequenceNo; 36 | } 37 | 38 | assert sequenceNo < 16384; 39 | 40 | return (chosenTime << 26) | (chosenSeqNo << 12) | (Config.getConfig().server_id); 41 | } 42 | } -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/util/bloom/MurmurHash.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.util.bloom; 20 | 21 | /** 22 | * This is a very fast, non-cryptographic hash suitable for general hash-based 23 | * lookup. See http://murmurhash.googlepages.com/ for more details. 24 | * 25 | *

The C version of MurmurHash 2.0 found at that site was ported 26 | * to Java by Andrzej Bialecki (ab at getopt org).

27 | */ 28 | public class MurmurHash { 29 | private static MurmurHash _instance = new MurmurHash(); 30 | 31 | private MurmurHash() {} 32 | 33 | public static MurmurHash getInstance() { 34 | return _instance; 35 | } 36 | 37 | public int hash(byte[] data, int length, int seed) { 38 | int m = 0x5bd1e995; 39 | int r = 24; 40 | 41 | int h = seed ^ length; 42 | 43 | int len_4 = length >> 2; 44 | 45 | for (int i = 0; i < len_4; i++) { 46 | int i_4 = i << 2; 47 | int k = data[i_4 + 3]; 48 | k = k << 8; 49 | k = k | (data[i_4 + 2] & 0xff); 50 | k = k << 8; 51 | k = k | (data[i_4 + 1] & 0xff); 52 | k = k << 8; 53 | k = k | (data[i_4 + 0] & 0xff); 54 | k *= m; 55 | k ^= k >>> r; 56 | k *= m; 57 | h *= m; 58 | h ^= k; 59 | } 60 | 61 | // avoid calculating modulo 62 | int len_m = len_4 << 2; 63 | int left = length - len_m; 64 | 65 | if (left != 0) { 66 | if (left >= 3) { 67 | h ^= (int) data[length - 3] << 16; 68 | } 69 | if (left >= 2) { 70 | h ^= (int) data[length - 2] << 8; 71 | } 72 | if (left >= 1) { 73 | h ^= (int) data[length - 1]; 74 | } 75 | 76 | h *= m; 77 | } 78 | 79 | h ^= h >>> 13; 80 | h *= m; 81 | h ^= h >>> 15; 82 | 83 | return h; 84 | } 85 | } -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /tools/cmdline/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | edu.berkeley.kaiju 5 | cmdline 6 | jar 7 | 1.0-SNAPSHOT 8 | Kaiju Data Store Command Line 9 | http://maven.apache.org 10 | 11 | 12 | 13 | 14 | org.apache.maven.plugins 15 | maven-compiler-plugin 16 | 3.0 17 | 18 | 1.6 19 | 1.6 20 | 21 | 22 | 23 | org.apache.maven.plugins 24 | maven-dependency-plugin 25 | 26 | 27 | copy-dependencies 28 | prepare-package 29 | 30 | copy-dependencies 31 | 32 | 33 | ${project.build.directory}/lib 34 | false 35 | false 36 | true 37 | 38 | 39 | 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-jar-plugin 44 | 2.4 45 | 46 | 47 | 48 | true 49 | ${project.build.directory}/lib/ 50 | edu.berkeley.kaiju.cmdline.CommandLineClient 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | edu.berkeley.kaiju 61 | kaiju 62 | 1.0-SNAPSHOT 63 | compile 64 | 65 | 66 | 67 | org.slf4j 68 | slf4j-log4j12 69 | 1.5.8 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /tools/cmdline/src/main/java/edu/berkeley/kaiju/cmdline/CommandLineClient.java: -------------------------------------------------------------------------------- 1 | 2 | package edu.berkeley.kaiju.cmdline; 3 | 4 | import edu.berkeley.kaiju.frontend.KaijuClient; 5 | 6 | import java.lang.Exception; 7 | import java.lang.Long; 8 | import java.lang.String; 9 | import java.lang.System; 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import java.lang.Thread; 15 | import java.nio.ByteBuffer; 16 | 17 | public class CommandLineClient { 18 | private static enum MODE { PUT_ALL, GET_ALL } 19 | 20 | public static void main(String[] args) { 21 | try { 22 | KaijuClient client = new KaijuClient("127.0.0.1", 8080); 23 | 24 | MODE mode = MODE.PUT_ALL; 25 | 26 | if(args.length > 0) { 27 | if(args[0].equalsIgnoreCase("PA")) { 28 | mode = MODE.PUT_ALL; 29 | } 30 | else if(args[0].equalsIgnoreCase("GA")) { 31 | mode = MODE.GET_ALL; 32 | } 33 | } 34 | 35 | if(mode == MODE.PUT_ALL) 36 | System.out.println("PUTALL"); 37 | else if(mode == MODE.GET_ALL) 38 | System.out.println("GETALL"); 39 | 40 | System.out.println("Hello, World"); 41 | 42 | long counter = 0; 43 | long startTime = System.currentTimeMillis(); 44 | while(true) { 45 | counter++; 46 | if(counter % 1000 == 0) { 47 | System.out.println(counter/(System.currentTimeMillis()-startTime+1.)*1000.); 48 | } 49 | 50 | if(mode == MODE.PUT_ALL) { 51 | Map toPut = new HashMap(); 52 | toPut.put("a", String.format("%d", counter).getBytes()); 53 | toPut.put("b", String.format("%d", counter).getBytes()); 54 | client.put_all(toPut); 55 | } else if(mode == MODE.GET_ALL) { 56 | ArrayList toGet = new ArrayList(); 57 | toGet.add("a"); 58 | toGet.add("b"); 59 | Map ret = client.get_all(toGet); 60 | if(!Arrays.equals(ret.get("a"), ret.get("b"))) { 61 | System.err.println("ERROR: 'a' was "+Arrays.toString(ret.get("a")) 62 | + " but 'b' was "+Arrays.toString(ret.get("b"))); 63 | } 64 | } 65 | } 66 | } catch (Exception e) { 67 | System.out.println(e); 68 | e.printStackTrace(); 69 | } 70 | } 71 | 72 | } 73 | --------------------------------------------------------------------------------