├── Makefile ├── README.md ├── YCSB └── src │ ├── README │ ├── com │ └── yahoo │ │ └── ycsb │ │ ├── BasicDB.java │ │ ├── ByteArrayByteIterator.java │ │ ├── ByteIterator.java │ │ ├── Client.java │ │ ├── CommandLine.java │ │ ├── DB.java │ │ ├── DBException.java │ │ ├── DBFactory.java │ │ ├── DBWrapper.java │ │ ├── InputStreamByteIterator.java │ │ ├── RandomByteIterator.java │ │ ├── StringByteIterator.java │ │ ├── TerminatorThread.java │ │ ├── UnknownDBException.java │ │ ├── Utils.class │ │ ├── Utils.java │ │ ├── Workload.java │ │ ├── WorkloadException.java │ │ ├── generator │ │ ├── ConstantIntegerGenerator.java │ │ ├── CounterGenerator.java │ │ ├── DiscreteGenerator.java │ │ ├── ExponentialGenerator.java │ │ ├── FileGenerator.java │ │ ├── Generator.class │ │ ├── Generator.java │ │ ├── HistogramGenerator.java │ │ ├── HotspotIntegerGenerator.java │ │ ├── IntegerGenerator.class │ │ ├── IntegerGenerator.java │ │ ├── ScrambledZipfianGenerator.class │ │ ├── ScrambledZipfianGenerator.java │ │ ├── SkewedLatestGenerator.java │ │ ├── UniformGenerator.java │ │ ├── UniformIntegerGenerator.java │ │ ├── ZipfianGenerator.class │ │ └── ZipfianGenerator.java │ │ ├── measurements │ │ ├── Measurements.java │ │ ├── OneMeasurement.java │ │ ├── OneMeasurementHistogram.java │ │ ├── OneMeasurementTimeSeries.java │ │ └── exporter │ │ │ ├── JSONMeasurementsExporter.java │ │ │ ├── MeasurementsExporter.java │ │ │ └── TextMeasurementsExporter.java │ │ └── workloads │ │ ├── ConstantOccupancyWorkload.java │ │ └── CoreWorkload.java │ ├── gen-zipf.sh │ ├── prop.sh │ ├── zipf.java │ └── zipf_properties.java ├── common.c ├── common.h ├── conn.c ├── do.sh ├── kill-remote.sh ├── local-kill.sh ├── main.c ├── run-machine.sh ├── run-servers.sh ├── scripts ├── average.sh ├── busy.sh ├── gen-servers.sh ├── hugepages-check.sh ├── hugepages-create.sh ├── shm-ls.sh ├── shm-rm.sh └── zipf-check.sh ├── servers ├── shm-init.sh └── sizes.h /Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS := -O3 -Wall -Werror -Wno-unused-result 2 | LD := gcc 3 | LDFLAGS := ${LDFLAGS} -lrdmacm -libverbs -lrt -lpthread 4 | 5 | main: common.o conn.o main.o 6 | ${LD} -o $@ $^ ${LDFLAGS} 7 | 8 | PHONY: clean 9 | clean: 10 | rm -f *.o main 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | HERD 2 | ==== 3 | 4 | A Highly Efficient key-value system for RDMA. The 5 | original [paper describing HERD](http://dl.acm.org/citation.cfm?id=2626299) 6 | appeared in *SIGCOMM'14*. 7 | 8 | **Important: This code is not maintained anymore. Please use the new and improved 9 | (up to 83% faster) version of HERD from our 10 | [USENIX ATC 16 paper](https://www.usenix.org/system/files/conference/atc16/atc16_paper-kalia.pdf). 11 | The code is available [here](https://github.com/efficient/rdma_bench/tree/master/herd).** 12 | 13 | This version of HERD has been tested for the following configuration: 14 | 15 | 1. Software 16 | * OS: Ubuntu 12.04 (kernel 3.2.0) 17 | * RDMA drivers: `mlx4` from MLNX OFED 2.2. I suggest using the MLNX OFED version for Ubuntu 12.04. 18 | 2. Hardware 19 | * RNICs: 20 | * ConnectX-3 353A (InfiniBand) 21 | * ConnectX-3 313A (RoCE) 22 | * ConnectX-3 354A (InfiniBand) 23 | 24 | Initial setup: 25 | ------------- 26 | 27 | * I assume that the machines are named: `node-i.RDMA.fawn.apt.emulab.net` starting from `i = 1`. 28 | * The experiment requires at least `(1 + (NUM_CLIENTS / num_processes))` machines. 29 | `node-1` is the server machine. 30 | `NUM_CLIENTS` is the total number of client processes, defined in `common.h`. 31 | `num_processes` is the number of client processes per machine, defined in 32 | `run-machine.sh`. 33 | * To modify HERD for your machine names: 34 | * Make appropriate changes in `kill-remote.sh` and `run-servers.sh`. 35 | * Change the server's machine name in the `servers` file. Clients use this file to 36 | connect to server processes. 37 | 38 | * Make sure that ports 5500 to 5515 are available on the server machine. Server process `i` 39 | listens for clients on port `5500 + i`. 40 | 41 | * Execute the following commands at the server machine: 42 | ```bash 43 | cd ~ 44 | git clone https://github.com/anujkaliaiitd/HERD.git 45 | export PATH=~/HERD/scripts:$PATH 46 | cd HERD 47 | sudo ./shm-init.sh # Increase shmmax and shmall 48 | sudo hugepages-create.sh 0 4096 # Create hugepages on socket 0. Do for all sockets. 49 | ``` 50 | 51 | * Mount the HERD folder on all client machines via NFS. 52 | 53 | Quick start: 54 | ----------- 55 | 56 | * Run `make` on the server machine to build the executables. 57 | 58 | * To run the clients automatically along with the server: 59 | 60 | ```bash 61 | # At node-1 (server) 62 | ./run-servers.sh 63 | ``` 64 | 65 | * If you do not want to run clients automatically from the server, delete the 66 | 2nd loop from `run-servers.sh`. Then: 67 | 68 | ```bash 69 | # At node-1 (server) 70 | ./run-servers.sh 71 | # At node-2 (client 0) 72 | ./run-machine.sh 0 73 | # At node-i (client i - 2) 74 | ./run-machine.sh (i - 2) 75 | ``` 76 | 77 | * To kill the server processes, run `local-kill.sh` at the server machine. To kill the 78 | client processes remotely, run `kill-remote.sh` at the server machine. 79 | 80 | License 81 | ------- 82 | 83 | Copyright 2014 Carnegie Mellon University 84 | 85 | Licensed under the Apache License, Version 2.0 (the "License"); 86 | you may not use this file except in compliance with the License. 87 | You may obtain a copy of the License at 88 | 89 | http://www.apache.org/licenses/LICENSE-2.0 90 | 91 | Unless required by applicable law or agreed to in writing, software 92 | distributed under the License is distributed on an "AS IS" BASIS, 93 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 94 | See the License for the specific language governing permissions and 95 | limitations under the License. 96 | 97 | 148 | -------------------------------------------------------------------------------- /YCSB/src/README: -------------------------------------------------------------------------------- 1 | Test: 2 | * Generate 20 million zipf .99 numbers and slot them into 64 buckets 3 | * Print percetile weights of buckets 4 | 5 | 1 buckets of 64 comprise 5 percent of samples 6 | 3 buckets of 64 comprise 10 percent of samples 7 | 5 buckets of 64 comprise 15 percent of samples 8 | 8 buckets of 64 comprise 20 percent of samples 9 | 11 buckets of 64 comprise 25 percent of samples 10 | 14 buckets of 64 comprise 30 percent of samples 11 | 17 buckets of 64 comprise 35 percent of samples 12 | 20 buckets of 64 comprise 40 percent of samples 13 | 23 buckets of 64 comprise 45 percent of samples 14 | 27 buckets of 64 comprise 50 percent of samples 15 | 30 buckets of 64 comprise 55 percent of samples 16 | 34 buckets of 64 comprise 60 percent of samples 17 | 37 buckets of 64 comprise 65 percent of samples 18 | 41 buckets of 64 comprise 70 percent of samples 19 | 45 buckets of 64 comprise 75 percent of samples 20 | 49 buckets of 64 comprise 80 percent of samples 21 | 53 buckets of 64 comprise 85 percent of samples 22 | 57 buckets of 64 comprise 90 percent of samples 23 | 61 buckets of 64 comprise 95 percent of samples 24 | 64 buckets of 64 comprise 100 percent of samples 25 | 26 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/BasicDB.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.HashMap; 21 | import java.util.Properties; 22 | import java.util.Set; 23 | import java.util.Enumeration; 24 | import java.util.Vector; 25 | 26 | 27 | /** 28 | * Basic DB that just prints out the requested operations, instead of doing them against a database. 29 | */ 30 | public class BasicDB extends DB 31 | { 32 | public static final String VERBOSE="basicdb.verbose"; 33 | public static final String VERBOSE_DEFAULT="true"; 34 | 35 | public static final String SIMULATE_DELAY="basicdb.simulatedelay"; 36 | public static final String SIMULATE_DELAY_DEFAULT="0"; 37 | 38 | 39 | boolean verbose; 40 | int todelay; 41 | 42 | public BasicDB() 43 | { 44 | todelay=0; 45 | } 46 | 47 | 48 | void delay() 49 | { 50 | if (todelay>0) 51 | { 52 | try 53 | { 54 | Thread.sleep((long)Utils.random().nextInt(todelay)); 55 | } 56 | catch (InterruptedException e) 57 | { 58 | //do nothing 59 | } 60 | } 61 | } 62 | 63 | /** 64 | * Initialize any state for this DB. 65 | * Called once per DB instance; there is one DB instance per client thread. 66 | */ 67 | @SuppressWarnings("unchecked") 68 | public void init() 69 | { 70 | verbose=Boolean.parseBoolean(getProperties().getProperty(VERBOSE, VERBOSE_DEFAULT)); 71 | todelay=Integer.parseInt(getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT)); 72 | 73 | if (verbose) 74 | { 75 | System.out.println("***************** properties *****************"); 76 | Properties p=getProperties(); 77 | if (p!=null) 78 | { 79 | for (Enumeration e=p.propertyNames(); e.hasMoreElements(); ) 80 | { 81 | String k=(String)e.nextElement(); 82 | System.out.println("\""+k+"\"=\""+p.getProperty(k)+"\""); 83 | } 84 | } 85 | System.out.println("**********************************************"); 86 | } 87 | } 88 | 89 | /** 90 | * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. 91 | * 92 | * @param table The name of the table 93 | * @param key The record key of the record to read. 94 | * @param fields The list of fields to read, or null for all of them 95 | * @param result A HashMap of field/value pairs for the result 96 | * @return Zero on success, a non-zero error code on error 97 | */ 98 | public int read(String table, String key, Set fields, HashMap result) 99 | { 100 | delay(); 101 | 102 | if (verbose) 103 | { 104 | System.out.print("READ "+table+" "+key+" [ "); 105 | if (fields!=null) 106 | { 107 | for (String f : fields) 108 | { 109 | System.out.print(f+" "); 110 | } 111 | } 112 | else 113 | { 114 | System.out.print(""); 115 | } 116 | 117 | System.out.println("]"); 118 | } 119 | 120 | return 0; 121 | } 122 | 123 | /** 124 | * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. 125 | * 126 | * @param table The name of the table 127 | * @param startkey The record key of the first record to read. 128 | * @param recordcount The number of records to read 129 | * @param fields The list of fields to read, or null for all of them 130 | * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record 131 | * @return Zero on success, a non-zero error code on error 132 | */ 133 | public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) 134 | { 135 | delay(); 136 | 137 | if (verbose) 138 | { 139 | System.out.print("SCAN "+table+" "+startkey+" "+recordcount+" [ "); 140 | if (fields!=null) 141 | { 142 | for (String f : fields) 143 | { 144 | System.out.print(f+" "); 145 | } 146 | } 147 | else 148 | { 149 | System.out.print(""); 150 | } 151 | 152 | System.out.println("]"); 153 | } 154 | 155 | return 0; 156 | } 157 | 158 | /** 159 | * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 160 | * record key, overwriting any existing values with the same field name. 161 | * 162 | * @param table The name of the table 163 | * @param key The record key of the record to write. 164 | * @param values A HashMap of field/value pairs to update in the record 165 | * @return Zero on success, a non-zero error code on error 166 | */ 167 | public int update(String table, String key, HashMap values) 168 | { 169 | delay(); 170 | 171 | if (verbose) 172 | { 173 | System.out.print("UPDATE "+table+" "+key+" [ "); 174 | if (values!=null) 175 | { 176 | for (String k : values.keySet()) 177 | { 178 | System.out.print(k+"="+values.get(k)+" "); 179 | } 180 | } 181 | System.out.println("]"); 182 | } 183 | 184 | return 0; 185 | } 186 | 187 | /** 188 | * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 189 | * record key. 190 | * 191 | * @param table The name of the table 192 | * @param key The record key of the record to insert. 193 | * @param values A HashMap of field/value pairs to insert in the record 194 | * @return Zero on success, a non-zero error code on error 195 | */ 196 | public int insert(String table, String key, HashMap values) 197 | { 198 | delay(); 199 | 200 | if (verbose) 201 | { 202 | System.out.print("INSERT "+table+" "+key+" [ "); 203 | if (values!=null) 204 | { 205 | for (String k : values.keySet()) 206 | { 207 | System.out.print(k+"="+values.get(k)+" "); 208 | } 209 | } 210 | 211 | System.out.println("]"); 212 | } 213 | 214 | return 0; 215 | } 216 | 217 | 218 | /** 219 | * Delete a record from the database. 220 | * 221 | * @param table The name of the table 222 | * @param key The record key of the record to delete. 223 | * @return Zero on success, a non-zero error code on error 224 | */ 225 | public int delete(String table, String key) 226 | { 227 | delay(); 228 | 229 | if (verbose) 230 | { 231 | System.out.println("DELETE "+table+" "+key); 232 | } 233 | 234 | return 0; 235 | } 236 | 237 | /** 238 | * Short test of BasicDB 239 | */ 240 | /* 241 | public static void main(String[] args) 242 | { 243 | BasicDB bdb=new BasicDB(); 244 | 245 | Properties p=new Properties(); 246 | p.setProperty("Sky","Blue"); 247 | p.setProperty("Ocean","Wet"); 248 | 249 | bdb.setProperties(p); 250 | 251 | bdb.init(); 252 | 253 | HashMap fields=new HashMap(); 254 | fields.put("A","X"); 255 | fields.put("B","Y"); 256 | 257 | bdb.read("table","key",null,null); 258 | bdb.insert("table","key",fields); 259 | 260 | fields=new HashMap(); 261 | fields.put("C","Z"); 262 | 263 | bdb.update("table","key",fields); 264 | 265 | bdb.delete("table","key"); 266 | }*/ 267 | } 268 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/ByteArrayByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | public class ByteArrayByteIterator extends ByteIterator { 20 | byte[] str; 21 | int off; 22 | final int len; 23 | public ByteArrayByteIterator(byte[] s) { 24 | this.str = s; 25 | this.off = 0; 26 | this.len = s.length; 27 | } 28 | 29 | public ByteArrayByteIterator(byte[] s, int off, int len) { 30 | this.str = s; 31 | this.off = off; 32 | this.len = off + len; 33 | } 34 | 35 | @Override 36 | public boolean hasNext() { 37 | return off < len; 38 | } 39 | 40 | @Override 41 | public byte nextByte() { 42 | byte ret = str[off]; 43 | off++; 44 | return ret; 45 | } 46 | 47 | @Override 48 | public long bytesLeft() { 49 | return len - off; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/ByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | import java.util.Iterator; 20 | import java.util.ArrayList; 21 | /** 22 | * YCSB-specific buffer class. ByteIterators are designed to support 23 | * efficient field generation, and to allow backend drivers that can stream 24 | * fields (instead of materializing them in RAM) to do so. 25 | *

26 | * YCSB originially used String objects to represent field values. This led to 27 | * two performance issues. 28 | *

29 | * First, it leads to unnecessary conversions between UTF-16 and UTF-8, both 30 | * during field generation, and when passing data to byte-based backend 31 | * drivers. 32 | *

33 | * Second, Java strings are represented internally using UTF-16, and are 34 | * built by appending to a growable array type (StringBuilder or 35 | * StringBuffer), then calling a toString() method. This leads to a 4x memory 36 | * overhead as field values are being built, which prevented YCSB from 37 | * driving large object stores. 38 | *

39 | * The StringByteIterator class contains a number of convenience methods for 40 | * backend drivers that convert between Map<String,String> and 41 | * Map<String,ByteBuffer>. 42 | * 43 | * @author sears 44 | */ 45 | public abstract class ByteIterator implements Iterator { 46 | 47 | @Override 48 | public abstract boolean hasNext(); 49 | 50 | @Override 51 | public Byte next() { 52 | throw new UnsupportedOperationException(); 53 | //return nextByte(); 54 | } 55 | 56 | public abstract byte nextByte(); 57 | /** @return byte offset immediately after the last valid byte */ 58 | public int nextBuf(byte[] buf, int buf_off) { 59 | int sz = buf_off; 60 | while(sz < buf.length && hasNext()) { 61 | buf[sz] = nextByte(); 62 | sz++; 63 | } 64 | return sz; 65 | } 66 | 67 | public abstract long bytesLeft(); 68 | 69 | @Override 70 | public void remove() { 71 | throw new UnsupportedOperationException(); 72 | } 73 | 74 | /** Consumes remaining contents of this object, and returns them as a string. */ 75 | public String toString() { 76 | StringBuilder sb = new StringBuilder(); 77 | while(this.hasNext()) { sb.append((char)nextByte()); } 78 | return sb.toString(); 79 | } 80 | /** Consumes remaining contents of this object, and returns them as a byte array. */ 81 | public byte[] toArray() { 82 | long left = bytesLeft(); 83 | if(left != (int)left) { throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!"); } 84 | byte[] ret = new byte[(int)left]; 85 | int off = 0; 86 | while(off < ret.length) { 87 | off = nextBuf(ret, off); 88 | } 89 | return ret; 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/DB.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.HashMap; 21 | import java.util.Properties; 22 | import java.util.Set; 23 | import java.util.Vector; 24 | 25 | /** 26 | * A layer for accessing a database to be benchmarked. Each thread in the client 27 | * will be given its own instance of whatever DB class is to be used in the test. 28 | * This class should be constructed using a no-argument constructor, so we can 29 | * load it dynamically. Any argument-based initialization should be 30 | * done by init(). 31 | * 32 | * Note that YCSB does not make any use of the return codes returned by this class. 33 | * Instead, it keeps a count of the return values and presents them to the user. 34 | * 35 | * The semantics of methods such as insert, update and delete vary from database 36 | * to database. In particular, operations may or may not be durable once these 37 | * methods commit, and some systems may return 'success' regardless of whether 38 | * or not a tuple with a matching key existed before the call. Rather than dictate 39 | * the exact semantics of these methods, we recommend you either implement them 40 | * to match the database's default semantics, or the semantics of your 41 | * target application. For the sake of comparison between experiments we also 42 | * recommend you explain the semantics you chose when presenting performance results. 43 | */ 44 | public abstract class DB 45 | { 46 | /** 47 | * Properties for configuring this DB. 48 | */ 49 | Properties _p=new Properties(); 50 | 51 | /** 52 | * Set the properties for this DB. 53 | */ 54 | public void setProperties(Properties p) 55 | { 56 | _p=p; 57 | 58 | } 59 | 60 | /** 61 | * Get the set of properties for this DB. 62 | */ 63 | public Properties getProperties() 64 | { 65 | return _p; 66 | } 67 | 68 | /** 69 | * Initialize any state for this DB. 70 | * Called once per DB instance; there is one DB instance per client thread. 71 | */ 72 | public void init() throws DBException 73 | { 74 | } 75 | 76 | /** 77 | * Cleanup any state for this DB. 78 | * Called once per DB instance; there is one DB instance per client thread. 79 | */ 80 | public void cleanup() throws DBException 81 | { 82 | } 83 | 84 | /** 85 | * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. 86 | * 87 | * @param table The name of the table 88 | * @param key The record key of the record to read. 89 | * @param fields The list of fields to read, or null for all of them 90 | * @param result A HashMap of field/value pairs for the result 91 | * @return Zero on success, a non-zero error code on error or "not found". 92 | */ 93 | public abstract int read(String table, String key, Set fields, HashMap result); 94 | 95 | /** 96 | * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. 97 | * 98 | * @param table The name of the table 99 | * @param startkey The record key of the first record to read. 100 | * @param recordcount The number of records to read 101 | * @param fields The list of fields to read, or null for all of them 102 | * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record 103 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 104 | */ 105 | public abstract int scan(String table, String startkey, int recordcount, Set fields, Vector> result); 106 | 107 | /** 108 | * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 109 | * record key, overwriting any existing values with the same field name. 110 | * 111 | * @param table The name of the table 112 | * @param key The record key of the record to write. 113 | * @param values A HashMap of field/value pairs to update in the record 114 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 115 | */ 116 | public abstract int update(String table, String key, HashMap values); 117 | 118 | /** 119 | * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 120 | * record key. 121 | * 122 | * @param table The name of the table 123 | * @param key The record key of the record to insert. 124 | * @param values A HashMap of field/value pairs to insert in the record 125 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 126 | */ 127 | public abstract int insert(String table, String key, HashMap values); 128 | 129 | /** 130 | * Delete a record from the database. 131 | * 132 | * @param table The name of the table 133 | * @param key The record key of the record to delete. 134 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 135 | */ 136 | public abstract int delete(String table, String key); 137 | } 138 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/DBException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | /** 21 | * Something bad happened while interacting with the database. 22 | */ 23 | public class DBException extends Exception 24 | { 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 6646883591588721475L; 29 | 30 | public DBException(String message) 31 | { 32 | super(message); 33 | } 34 | 35 | public DBException() 36 | { 37 | super(); 38 | } 39 | 40 | public DBException(String message, Throwable cause) 41 | { 42 | super(message,cause); 43 | } 44 | 45 | public DBException(Throwable cause) 46 | { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/DBFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.Properties; 21 | 22 | /** 23 | * Creates a DB layer by dynamically classloading the specified DB class. 24 | */ 25 | public class DBFactory 26 | { 27 | @SuppressWarnings("unchecked") 28 | public static DB newDB(String dbname, Properties properties) throws UnknownDBException 29 | { 30 | ClassLoader classLoader = DBFactory.class.getClassLoader(); 31 | 32 | DB ret=null; 33 | 34 | try 35 | { 36 | Class dbclass = classLoader.loadClass(dbname); 37 | //System.out.println("dbclass.getName() = " + dbclass.getName()); 38 | 39 | ret=(DB)dbclass.newInstance(); 40 | } 41 | catch (Exception e) 42 | { 43 | e.printStackTrace(); 44 | return null; 45 | } 46 | 47 | ret.setProperties(properties); 48 | 49 | return new DBWrapper(ret); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/DBWrapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.HashMap; 21 | import java.util.Properties; 22 | import java.util.Set; 23 | import java.util.Vector; 24 | 25 | import com.yahoo.ycsb.measurements.Measurements; 26 | 27 | /** 28 | * Wrapper around a "real" DB that measures latencies and counts return codes. 29 | */ 30 | public class DBWrapper extends DB 31 | { 32 | DB _db; 33 | Measurements _measurements; 34 | 35 | public DBWrapper(DB db) 36 | { 37 | _db=db; 38 | _measurements=Measurements.getMeasurements(); 39 | } 40 | 41 | /** 42 | * Set the properties for this DB. 43 | */ 44 | public void setProperties(Properties p) 45 | { 46 | _db.setProperties(p); 47 | } 48 | 49 | /** 50 | * Get the set of properties for this DB. 51 | */ 52 | public Properties getProperties() 53 | { 54 | return _db.getProperties(); 55 | } 56 | 57 | /** 58 | * Initialize any state for this DB. 59 | * Called once per DB instance; there is one DB instance per client thread. 60 | */ 61 | public void init() throws DBException 62 | { 63 | _db.init(); 64 | } 65 | 66 | /** 67 | * Cleanup any state for this DB. 68 | * Called once per DB instance; there is one DB instance per client thread. 69 | */ 70 | public void cleanup() throws DBException 71 | { 72 | long st=System.nanoTime(); 73 | _db.cleanup(); 74 | long en=System.nanoTime(); 75 | _measurements.measure("CLEANUP", (int)((en-st)/1000)); 76 | } 77 | 78 | /** 79 | * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. 80 | * 81 | * @param table The name of the table 82 | * @param key The record key of the record to read. 83 | * @param fields The list of fields to read, or null for all of them 84 | * @param result A HashMap of field/value pairs for the result 85 | * @return Zero on success, a non-zero error code on error 86 | */ 87 | public int read(String table, String key, Set fields, HashMap result) 88 | { 89 | long st=System.nanoTime(); 90 | int res=_db.read(table,key,fields,result); 91 | long en=System.nanoTime(); 92 | _measurements.measure("READ",(int)((en-st)/1000)); 93 | _measurements.reportReturnCode("READ",res); 94 | return res; 95 | } 96 | 97 | /** 98 | * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. 99 | * 100 | * @param table The name of the table 101 | * @param startkey The record key of the first record to read. 102 | * @param recordcount The number of records to read 103 | * @param fields The list of fields to read, or null for all of them 104 | * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record 105 | * @return Zero on success, a non-zero error code on error 106 | */ 107 | public int scan(String table, String startkey, int recordcount, Set fields, Vector> result) 108 | { 109 | long st=System.nanoTime(); 110 | int res=_db.scan(table,startkey,recordcount,fields,result); 111 | long en=System.nanoTime(); 112 | _measurements.measure("SCAN",(int)((en-st)/1000)); 113 | _measurements.reportReturnCode("SCAN",res); 114 | return res; 115 | } 116 | 117 | /** 118 | * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 119 | * record key, overwriting any existing values with the same field name. 120 | * 121 | * @param table The name of the table 122 | * @param key The record key of the record to write. 123 | * @param values A HashMap of field/value pairs to update in the record 124 | * @return Zero on success, a non-zero error code on error 125 | */ 126 | public int update(String table, String key, HashMap values) 127 | { 128 | long st=System.nanoTime(); 129 | int res=_db.update(table,key,values); 130 | long en=System.nanoTime(); 131 | _measurements.measure("UPDATE",(int)((en-st)/1000)); 132 | _measurements.reportReturnCode("UPDATE",res); 133 | return res; 134 | } 135 | 136 | /** 137 | * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 138 | * record key. 139 | * 140 | * @param table The name of the table 141 | * @param key The record key of the record to insert. 142 | * @param values A HashMap of field/value pairs to insert in the record 143 | * @return Zero on success, a non-zero error code on error 144 | */ 145 | public int insert(String table, String key, HashMap values) 146 | { 147 | long st=System.nanoTime(); 148 | int res=_db.insert(table,key,values); 149 | long en=System.nanoTime(); 150 | _measurements.measure("INSERT",(int)((en-st)/1000)); 151 | _measurements.reportReturnCode("INSERT",res); 152 | return res; 153 | } 154 | 155 | /** 156 | * Delete a record from the database. 157 | * 158 | * @param table The name of the table 159 | * @param key The record key of the record to delete. 160 | * @return Zero on success, a non-zero error code on error 161 | */ 162 | public int delete(String table, String key) 163 | { 164 | long st=System.nanoTime(); 165 | int res=_db.delete(table,key); 166 | long en=System.nanoTime(); 167 | _measurements.measure("DELETE",(int)((en-st)/1000)); 168 | _measurements.reportReturnCode("DELETE",res); 169 | return res; 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/InputStreamByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | import java.io.InputStream; 20 | 21 | public class InputStreamByteIterator extends ByteIterator { 22 | long len; 23 | InputStream ins; 24 | long off; 25 | 26 | public InputStreamByteIterator(InputStream ins, long len) { 27 | this.len = len; 28 | this.ins = ins; 29 | off = 0; 30 | } 31 | 32 | @Override 33 | public boolean hasNext() { 34 | return off < len; 35 | } 36 | 37 | @Override 38 | public byte nextByte() { 39 | int ret; 40 | try { 41 | ret = ins.read(); 42 | } catch(Exception e) { 43 | throw new IllegalStateException(e); 44 | } 45 | if(ret == -1) { throw new IllegalStateException("Past EOF!"); } 46 | off++; 47 | return (byte)ret; 48 | } 49 | 50 | @Override 51 | public long bytesLeft() { 52 | return len - off; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/RandomByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | /** 20 | * A ByteIterator that generates a random sequence of bytes. 21 | */ 22 | public class RandomByteIterator extends ByteIterator { 23 | private long len; 24 | private long off; 25 | private int bufOff; 26 | private byte[] buf; 27 | 28 | @Override 29 | public boolean hasNext() { 30 | return (off + bufOff) < len; 31 | } 32 | 33 | private void fillBytesImpl(byte[] buffer, int base) { 34 | int bytes = Utils.random().nextInt(); 35 | try { 36 | buffer[base+0] = (byte)(((bytes) & 31) + ' '); 37 | buffer[base+1] = (byte)(((bytes >> 5) & 31) + ' '); 38 | buffer[base+2] = (byte)(((bytes >> 10) & 31) + ' '); 39 | buffer[base+3] = (byte)(((bytes >> 15) & 31) + ' '); 40 | buffer[base+4] = (byte)(((bytes >> 20) & 31) + ' '); 41 | buffer[base+5] = (byte)(((bytes >> 25) & 31) + ' '); 42 | } catch (ArrayIndexOutOfBoundsException e) { /* ignore it */ } 43 | } 44 | 45 | private void fillBytes() { 46 | if(bufOff == buf.length) { 47 | fillBytesImpl(buf, 0); 48 | bufOff = 0; 49 | off += buf.length; 50 | } 51 | } 52 | 53 | public RandomByteIterator(long len) { 54 | this.len = len; 55 | this.buf = new byte[6]; 56 | this.bufOff = buf.length; 57 | fillBytes(); 58 | this.off = 0; 59 | } 60 | 61 | public byte nextByte() { 62 | fillBytes(); 63 | bufOff++; 64 | return buf[bufOff-1]; 65 | } 66 | 67 | @Override 68 | public int nextBuf(byte[] buffer, int bufferOffset) { 69 | int ret; 70 | if(len - off < buffer.length - bufferOffset) { 71 | ret = (int)(len - off); 72 | } else { 73 | ret = buffer.length - bufferOffset; 74 | } 75 | int i; 76 | for(i = 0; i < ret; i+=6) { 77 | fillBytesImpl(buffer, i + bufferOffset); 78 | } 79 | off+=ret; 80 | return ret + bufferOffset; 81 | } 82 | 83 | @Override 84 | public long bytesLeft() { 85 | return len - off - bufOff; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/StringByteIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.Map; 21 | import java.util.HashMap; 22 | 23 | public class StringByteIterator extends ByteIterator { 24 | String str; 25 | int off; 26 | 27 | /** 28 | * Put all of the entries of one map into the other, converting 29 | * String values into ByteIterators. 30 | */ 31 | public static void putAllAsByteIterators(Map out, Map in) { 32 | for(String s: in.keySet()) { out.put(s, new StringByteIterator(in.get(s))); } 33 | } 34 | 35 | /** 36 | * Put all of the entries of one map into the other, converting 37 | * ByteIterator values into Strings. 38 | */ 39 | public static void putAllAsStrings(Map out, Map in) { 40 | for(String s: in.keySet()) { out.put(s, in.get(s).toString()); } 41 | } 42 | 43 | /** 44 | * Create a copy of a map, converting the values from Strings to 45 | * StringByteIterators. 46 | */ 47 | public static HashMap getByteIteratorMap(Map m) { 48 | HashMap ret = 49 | new HashMap(); 50 | 51 | for(String s: m.keySet()) { 52 | ret.put(s, new StringByteIterator(m.get(s))); 53 | } 54 | return ret; 55 | } 56 | 57 | /** 58 | * Create a copy of a map, converting the values from 59 | * StringByteIterators to Strings. 60 | */ 61 | public static HashMap getStringMap(Map m) { 62 | HashMap ret = new HashMap(); 63 | 64 | for(String s: m.keySet()) { 65 | ret.put(s, m.get(s).toString());; 66 | } 67 | return ret; 68 | } 69 | 70 | public StringByteIterator(String s) { 71 | this.str = s; 72 | this.off = 0; 73 | } 74 | @Override 75 | public boolean hasNext() { 76 | return off < str.length(); 77 | } 78 | 79 | @Override 80 | public byte nextByte() { 81 | byte ret = (byte)str.charAt(off); 82 | off++; 83 | return ret; 84 | } 85 | 86 | @Override 87 | public long bytesLeft() { 88 | return str.length() - off; 89 | } 90 | 91 | /** 92 | * Specialization of general purpose toString() to avoid unnecessary 93 | * copies. 94 | *

95 | * Creating a new StringByteIterator, then calling toString() 96 | * yields the original String object, and does not perform any copies 97 | * or String conversion operations. 98 | *

99 | */ 100 | @Override 101 | public String toString() { 102 | if(off > 0) { 103 | return super.toString(); 104 | } else { 105 | return str; 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/TerminatorThread.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2011 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb; 18 | 19 | import java.util.Vector; 20 | 21 | /** 22 | * A thread that waits for the maximum specified time and then interrupts all the client 23 | * threads passed as the Vector at initialization of this thread. 24 | * 25 | * The maximum execution time passed is assumed to be in seconds. 26 | * 27 | * @author sudipto 28 | * 29 | */ 30 | public class TerminatorThread extends Thread { 31 | 32 | private Vector threads; 33 | private long maxExecutionTime; 34 | private Workload workload; 35 | private long waitTimeOutInMS; 36 | 37 | public TerminatorThread(long maxExecutionTime, Vector threads, 38 | Workload workload) { 39 | this.maxExecutionTime = maxExecutionTime; 40 | this.threads = threads; 41 | this.workload = workload; 42 | waitTimeOutInMS = 2000; 43 | System.err.println("Maximum execution time specified as: " + maxExecutionTime + " secs"); 44 | } 45 | 46 | public void run() { 47 | try { 48 | Thread.sleep(maxExecutionTime * 1000); 49 | } catch (InterruptedException e) { 50 | System.err.println("Could not wait until max specified time, TerminatorThread interrupted."); 51 | return; 52 | } 53 | System.err.println("Maximum time elapsed. Requesting stop for the workload."); 54 | workload.requestStop(); 55 | System.err.println("Stop requested for workload. Now Joining!"); 56 | for (Thread t : threads) { 57 | while (t.isAlive()) { 58 | try { 59 | t.join(waitTimeOutInMS); 60 | if (t.isAlive()) { 61 | System.err.println("Still waiting for thread " + t.getName() + " to complete. " + 62 | "Workload status: " + workload.isStopRequested()); 63 | } 64 | } catch (InterruptedException e) { 65 | // Do nothing. Don't know why I was interrupted. 66 | } 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/UnknownDBException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | /** 21 | * Could not create the specified DB. 22 | */ 23 | public class UnknownDBException extends Exception 24 | { 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 459099842269616836L; 29 | 30 | public UnknownDBException(String message) 31 | { 32 | super(message); 33 | } 34 | 35 | public UnknownDBException() 36 | { 37 | super(); 38 | } 39 | 40 | public UnknownDBException(String message, Throwable cause) 41 | { 42 | super(message,cause); 43 | } 44 | 45 | public UnknownDBException(Throwable cause) 46 | { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/Utils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/efficient/HERD/2fa86cb66c18adc22e3e8e6513cc3482f7c26242/YCSB/src/com/yahoo/ycsb/Utils.class -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.Random; 21 | 22 | /** 23 | * Utility functions. 24 | */ 25 | public class Utils 26 | { 27 | private static final Random rand = new Random(); 28 | private static final ThreadLocal rng = new ThreadLocal(); 29 | 30 | public static Random random() { 31 | Random ret = new Random(3185); 32 | return ret; 33 | } 34 | /** 35 | * Generate a random ASCII string of a given length. 36 | */ 37 | public static String ASCIIString(int length) 38 | { 39 | int interval='~'-' '+1; 40 | 41 | byte []buf = new byte[length]; 42 | random().nextBytes(buf); 43 | for (int i = 0; i < length; i++) { 44 | if (buf[i] < 0) { 45 | buf[i] = (byte)((-buf[i] % interval) + ' '); 46 | } else { 47 | buf[i] = (byte)((buf[i] % interval) + ' '); 48 | } 49 | } 50 | return new String(buf); 51 | } 52 | 53 | /** 54 | * Hash an integer value. 55 | */ 56 | public static long hash(long val) 57 | { 58 | return FNVhash64(val); 59 | } 60 | 61 | public static final int FNV_offset_basis_32=0x811c9dc5; 62 | public static final int FNV_prime_32=16777619; 63 | 64 | /** 65 | * 32 bit FNV hash. Produces more "random" hashes than (say) String.hashCode(). 66 | * 67 | * @param val The value to hash. 68 | * @return The hash value 69 | */ 70 | public static int FNVhash32(int val) 71 | { 72 | //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash 73 | int hashval = FNV_offset_basis_32; 74 | 75 | for (int i=0; i<4; i++) 76 | { 77 | int octet=val&0x00ff; 78 | val=val>>8; 79 | 80 | hashval = hashval ^ octet; 81 | hashval = hashval * FNV_prime_32; 82 | //hashval = hashval ^ octet; 83 | } 84 | return Math.abs(hashval); 85 | } 86 | 87 | public static final long FNV_offset_basis_64=0xCBF29CE484222325L; 88 | public static final long FNV_prime_64=1099511628211L; 89 | 90 | /** 91 | * 64 bit FNV hash. Produces more "random" hashes than (say) String.hashCode(). 92 | * 93 | * @param val The value to hash. 94 | * @return The hash value 95 | */ 96 | public static long FNVhash64(long val) 97 | { 98 | //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash 99 | long hashval = FNV_offset_basis_64; 100 | 101 | for (int i=0; i<8; i++) 102 | { 103 | long octet=val&0x00ff; 104 | val=val>>8; 105 | 106 | hashval = hashval ^ octet; 107 | hashval = hashval * FNV_prime_64; 108 | //hashval = hashval ^ octet; 109 | } 110 | return Math.abs(hashval); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/Workload.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | import java.util.Properties; 21 | import java.util.concurrent.atomic.AtomicBoolean; 22 | 23 | /** 24 | * One experiment scenario. One object of this type will 25 | * be instantiated and shared among all client threads. This class 26 | * should be constructed using a no-argument constructor, so we can 27 | * load it dynamically. Any argument-based initialization should be 28 | * done by init(). 29 | * 30 | * If you extend this class, you should support the "insertstart" property. This 31 | * allows the load phase to proceed from multiple clients on different machines, in case 32 | * the client is the bottleneck. For example, if we want to load 1 million records from 33 | * 2 machines, the first machine should have insertstart=0 and the second insertstart=500000. Additionally, 34 | * the "insertcount" property, which is interpreted by Client, can be used to tell each instance of the 35 | * client how many inserts to do. In the example above, both clients should have insertcount=500000. 36 | */ 37 | public abstract class Workload 38 | { 39 | public static final String INSERT_START_PROPERTY="insertstart"; 40 | 41 | public static final String INSERT_START_PROPERTY_DEFAULT="0"; 42 | 43 | private volatile AtomicBoolean stopRequested = new AtomicBoolean(false); 44 | 45 | /** 46 | * Initialize the scenario. Create any generators and other shared objects here. 47 | * Called once, in the main client thread, before any operations are started. 48 | */ 49 | public void init(Properties p) throws WorkloadException 50 | { 51 | } 52 | 53 | /** 54 | * Initialize any state for a particular client thread. Since the scenario object 55 | * will be shared among all threads, this is the place to create any state that is specific 56 | * to one thread. To be clear, this means the returned object should be created anew on each 57 | * call to initThread(); do not return the same object multiple times. 58 | * The returned object will be passed to invocations of doInsert() and doTransaction() 59 | * for this thread. There should be no side effects from this call; all state should be encapsulated 60 | * in the returned object. If you have no state to retain for this thread, return null. (But if you have 61 | * no state to retain for this thread, probably you don't need to override initThread().) 62 | * 63 | * @return false if the workload knows it is done for this thread. Client will terminate the thread. Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read traces from a file, return true when there are more to do, false when you are done. 64 | */ 65 | public Object initThread(Properties p, int mythreadid, int threadcount) throws WorkloadException 66 | { 67 | return null; 68 | } 69 | 70 | /** 71 | * Cleanup the scenario. Called once, in the main client thread, after all operations have completed. 72 | */ 73 | public void cleanup() throws WorkloadException 74 | { 75 | } 76 | 77 | /** 78 | * Do one insert operation. Because it will be called concurrently from multiple client threads, this 79 | * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each 80 | * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side 81 | * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be 82 | * synchronized, since each thread has its own threadstate instance. 83 | */ 84 | public abstract boolean doInsert(DB db, Object threadstate); 85 | 86 | /** 87 | * Do one transaction operation. Because it will be called concurrently from multiple client threads, this 88 | * function must be thread safe. However, avoid synchronized, or the threads will block waiting for each 89 | * other, and it will be difficult to reach the target throughput. Ideally, this function would have no side 90 | * effects other than DB operations and mutations on threadstate. Mutations to threadstate do not need to be 91 | * synchronized, since each thread has its own threadstate instance. 92 | * 93 | * @return false if the workload knows it is done for this thread. Client will terminate the thread. Return true otherwise. Return true for workloads that rely on operationcount. For workloads that read traces from a file, return true when there are more to do, false when you are done. 94 | */ 95 | public abstract boolean doTransaction(DB db, Object threadstate); 96 | 97 | /** 98 | * Allows scheduling a request to stop the workload. 99 | */ 100 | public void requestStop() { 101 | stopRequested.set(true); 102 | } 103 | 104 | /** 105 | * Check the status of the stop request flag. 106 | * @return true if stop was requested, false otherwise. 107 | */ 108 | public boolean isStopRequested() { 109 | if (stopRequested.get() == true) return true; 110 | else return false; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/WorkloadException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb; 19 | 20 | /** 21 | * The workload tried to do something bad. 22 | */ 23 | public class WorkloadException extends Exception 24 | { 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 8844396756042772132L; 29 | 30 | public WorkloadException(String message) 31 | { 32 | super(message); 33 | } 34 | 35 | public WorkloadException() 36 | { 37 | super(); 38 | } 39 | 40 | public WorkloadException(String message, Throwable cause) 41 | { 42 | super(message,cause); 43 | } 44 | 45 | public WorkloadException(Throwable cause) 46 | { 47 | super(cause); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb.generator; 18 | 19 | /** 20 | * A trivial integer generator that always returns the same value. 21 | * 22 | * @author sears 23 | * 24 | */ 25 | public class ConstantIntegerGenerator extends IntegerGenerator { 26 | private final int i; 27 | /** 28 | * @param i The integer that this generator will always return. 29 | */ 30 | public ConstantIntegerGenerator(int i) { 31 | this.i = i; 32 | } 33 | 34 | @Override 35 | public int nextInt() { 36 | return i; 37 | } 38 | 39 | @Override 40 | public double mean() { 41 | return i; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/CounterGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | 22 | /** 23 | * Generates a sequence of integers 0, 1, ... 24 | */ 25 | public class CounterGenerator extends IntegerGenerator 26 | { 27 | final AtomicInteger counter; 28 | 29 | /** 30 | * Create a counter that starts at countstart 31 | */ 32 | public CounterGenerator(int countstart) 33 | { 34 | counter=new AtomicInteger(countstart); 35 | setLastInt(counter.get()-1); 36 | } 37 | 38 | /** 39 | * If the generator returns numeric (integer) values, return the next value as an int. Default is to return -1, which 40 | * is appropriate for generators that do not return numeric values. 41 | */ 42 | public int nextInt() 43 | { 44 | int ret = counter.getAndIncrement(); 45 | setLastInt(ret); 46 | return ret; 47 | } 48 | @Override 49 | public int lastInt() 50 | { 51 | return counter.get() - 1; 52 | } 53 | @Override 54 | public double mean() { 55 | throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!"); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/DiscreteGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | import java.util.Vector; 21 | import java.util.Random; 22 | 23 | import com.yahoo.ycsb.Utils; 24 | import com.yahoo.ycsb.WorkloadException; 25 | 26 | /** 27 | * Generates a distribution by choosing from a discrete set of values. 28 | */ 29 | public class DiscreteGenerator extends Generator 30 | { 31 | class Pair 32 | { 33 | public double _weight; 34 | public String _value; 35 | 36 | Pair(double weight, String value) 37 | { 38 | _weight=weight; 39 | _value=value; 40 | } 41 | } 42 | 43 | Vector _values; 44 | String _lastvalue; 45 | 46 | public DiscreteGenerator() 47 | { 48 | _values=new Vector(); 49 | _lastvalue=null; 50 | } 51 | 52 | /** 53 | * Generate the next string in the distribution. 54 | */ 55 | public String nextString() 56 | { 57 | double sum=0; 58 | 59 | for (Pair p : _values) 60 | { 61 | sum+=p._weight; 62 | } 63 | 64 | double val=Utils.random().nextDouble(); 65 | 66 | for (Pair p : _values) 67 | { 68 | if (val a = new ArrayList(); 55 | 56 | str = in.readLine(); 57 | if(str == null) { 58 | throw new IOException("Empty input file!\n"); 59 | } 60 | line = str.split("\t"); 61 | if(line[0].compareTo("BlockSize") != 0) { 62 | throw new IOException("First line of histogram is not the BlockSize!\n"); 63 | } 64 | block_size = Integer.parseInt(line[1]); 65 | 66 | while((str = in.readLine()) != null){ 67 | // [0] is the bucket, [1] is the value 68 | line = str.split("\t"); 69 | 70 | a.add(Integer.parseInt(line[0]), Integer.parseInt(line[1])); 71 | } 72 | buckets = new long[a.size()]; 73 | for(int i = 0; i < a.size(); i++) { 74 | buckets[i] = a.get(i); 75 | } 76 | 77 | in.close(); 78 | init(); 79 | } 80 | 81 | public HistogramGenerator(long[] buckets, int block_size) { 82 | this.block_size = block_size; 83 | this.buckets = buckets; 84 | init(); 85 | } 86 | private void init() { 87 | for(int i = 0; i < buckets.length; i++) { 88 | area += buckets[i]; 89 | weighted_area = i * buckets[i]; 90 | } 91 | // calculate average file size 92 | mean_size = ((double)block_size) * ((double)weighted_area) / (double)(area); 93 | } 94 | 95 | @Override 96 | public int nextInt() { 97 | int number = Utils.random().nextInt((int)area); 98 | int i; 99 | 100 | for(i = 0; i < (buckets.length - 1); i++){ 101 | number -= buckets[i]; 102 | if(number <= 0){ 103 | return (int)((i+1)*block_size); 104 | } 105 | } 106 | 107 | return (int)(i * block_size); 108 | } 109 | 110 | @Override 111 | public double mean() { 112 | return mean_size; 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | package com.yahoo.ycsb.generator; 18 | 19 | import java.util.Random; 20 | 21 | import com.yahoo.ycsb.Utils; 22 | 23 | /** 24 | * Generate integers resembling a hotspot distribution where x% of operations 25 | * access y% of data items. The parameters specify the bounds for the numbers, 26 | * the percentage of the of the interval which comprises the hot set and 27 | * the percentage of operations that access the hot set. Numbers of the hot set are 28 | * always smaller than any number in the cold set. Elements from the hot set and 29 | * the cold set are chose using a uniform distribution. 30 | * 31 | * @author sudipto 32 | * 33 | */ 34 | public class HotspotIntegerGenerator extends IntegerGenerator { 35 | 36 | private final int lowerBound; 37 | private final int upperBound; 38 | private final int hotInterval; 39 | private final int coldInterval; 40 | private final double hotsetFraction; 41 | private final double hotOpnFraction; 42 | 43 | /** 44 | * Create a generator for Hotspot distributions. 45 | * 46 | * @param lowerBound lower bound of the distribution. 47 | * @param upperBound upper bound of the distribution. 48 | * @param hotsetFraction percentage of data item 49 | * @param hotOpnFraction percentage of operations accessing the hot set. 50 | */ 51 | public HotspotIntegerGenerator(int lowerBound, int upperBound, 52 | double hotsetFraction, double hotOpnFraction) { 53 | if (hotsetFraction < 0.0 || hotsetFraction > 1.0) { 54 | System.err.println("Hotset fraction out of range. Setting to 0.0"); 55 | hotsetFraction = 0.0; 56 | } 57 | if (hotOpnFraction < 0.0 || hotOpnFraction > 1.0) { 58 | System.err.println("Hot operation fraction out of range. Setting to 0.0"); 59 | hotOpnFraction = 0.0; 60 | } 61 | if (lowerBound > upperBound) { 62 | System.err.println("Upper bound of Hotspot generator smaller than the lower bound. " + 63 | "Swapping the values."); 64 | int temp = lowerBound; 65 | lowerBound = upperBound; 66 | upperBound = temp; 67 | } 68 | this.lowerBound = lowerBound; 69 | this.upperBound = upperBound; 70 | this.hotsetFraction = hotsetFraction; 71 | int interval = upperBound - lowerBound + 1; 72 | this.hotInterval = (int)(interval * hotsetFraction); 73 | this.coldInterval = interval - hotInterval; 74 | this.hotOpnFraction = hotOpnFraction; 75 | } 76 | 77 | @Override 78 | public int nextInt() { 79 | int value = 0; 80 | Random random = Utils.random(); 81 | if (random.nextDouble() < hotOpnFraction) { 82 | // Choose a value from the hot set. 83 | value = lowerBound + random.nextInt(hotInterval); 84 | } else { 85 | // Choose a value from the cold set. 86 | value = lowerBound + hotInterval + random.nextInt(coldInterval); 87 | } 88 | setLastInt(value); 89 | return value; 90 | } 91 | 92 | /** 93 | * @return the lowerBound 94 | */ 95 | public int getLowerBound() { 96 | return lowerBound; 97 | } 98 | 99 | /** 100 | * @return the upperBound 101 | */ 102 | public int getUpperBound() { 103 | return upperBound; 104 | } 105 | 106 | /** 107 | * @return the hotsetFraction 108 | */ 109 | public double getHotsetFraction() { 110 | return hotsetFraction; 111 | } 112 | 113 | /** 114 | * @return the hotOpnFraction 115 | */ 116 | public double getHotOpnFraction() { 117 | return hotOpnFraction; 118 | } 119 | @Override 120 | public double mean() { 121 | return hotOpnFraction * (lowerBound + hotInterval/2.0) 122 | + (1 - hotOpnFraction) * (lowerBound + hotInterval + coldInterval/2.0); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/IntegerGenerator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/efficient/HERD/2fa86cb66c18adc22e3e8e6513cc3482f7c26242/YCSB/src/com/yahoo/ycsb/generator/IntegerGenerator.class -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/IntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | /** 21 | * A generator that is capable of generating ints as well as strings 22 | * 23 | * @author cooperb 24 | * 25 | */ 26 | public abstract class IntegerGenerator extends Generator 27 | { 28 | int lastint; 29 | 30 | /** 31 | * Set the last value generated. IntegerGenerator subclasses must use this call 32 | * to properly set the last string value, or the lastString() and lastInt() calls won't work. 33 | */ 34 | protected void setLastInt(int last) 35 | { 36 | lastint=last; 37 | } 38 | 39 | /** 40 | * Return the next value as an int. When overriding this method, be sure to call setLastString() properly, or the lastString() call won't work. 41 | */ 42 | public abstract int nextInt(); 43 | 44 | /** 45 | * Generate the next string in the distribution. 46 | */ 47 | public String nextString() 48 | { 49 | return ""+nextInt(); 50 | } 51 | 52 | /** 53 | * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. 54 | * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet 55 | * been called, lastString() should return something reasonable. 56 | */ 57 | @Override 58 | public String lastString() 59 | { 60 | return ""+lastInt(); 61 | } 62 | 63 | /** 64 | * Return the previous int generated by the distribution. This call is unique to IntegerGenerator subclasses, and assumes 65 | * IntegerGenerator subclasses always return ints for nextInt() (e.g. not arbitrary strings). 66 | */ 67 | public int lastInt() 68 | { 69 | return lastint; 70 | } 71 | /** 72 | * Return the expected value (mean) of the values this generator will return. 73 | */ 74 | public abstract double mean(); 75 | } 76 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/efficient/HERD/2fa86cb66c18adc22e3e8e6513cc3482f7c26242/YCSB/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.class -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | import com.yahoo.ycsb.Utils; 21 | 22 | /** 23 | * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according 24 | * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either 25 | * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of 26 | * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount). 27 | * 28 | * Unlike @ZipfianGenerator, this class scatters the "popular" items across the itemspace. Use this, instead of @ZipfianGenerator, if you 29 | * don't want the head of the distribution (the popular items) clustered together. 30 | */ 31 | public class ScrambledZipfianGenerator extends IntegerGenerator 32 | { 33 | public static final double ZETAN=26.46902820178302; 34 | public static final double USED_ZIPFIAN_CONSTANT=0.99; 35 | public static final long ITEM_COUNT=10000000000L; 36 | 37 | ZipfianGenerator gen; 38 | long _min,_max,_itemcount; 39 | 40 | /******************************* Constructors **************************************/ 41 | 42 | /** 43 | * Create a zipfian generator for the specified number of items. 44 | * @param _items The number of items in the distribution. 45 | */ 46 | public ScrambledZipfianGenerator(long _items) 47 | { 48 | this(0,_items-1); 49 | } 50 | 51 | /** 52 | * Create a zipfian generator for items between min and max. 53 | * @param _min The smallest integer to generate in the sequence. 54 | * @param _max The largest integer to generate in the sequence. 55 | */ 56 | public ScrambledZipfianGenerator(long _min, long _max) 57 | { 58 | this(_min,_max,ZipfianGenerator.ZIPFIAN_CONSTANT); 59 | } 60 | 61 | /** 62 | * Create a zipfian generator for the specified number of items using the specified zipfian constant. 63 | * 64 | * @param _items The number of items in the distribution. 65 | * @param _zipfianconstant The zipfian constant to use. 66 | */ 67 | /* 68 | // not supported, as the value of zeta depends on the zipfian constant, and we have only precomputed zeta for one zipfian constant 69 | public ScrambledZipfianGenerator(long _items, double _zipfianconstant) 70 | { 71 | this(0,_items-1,_zipfianconstant); 72 | } 73 | */ 74 | 75 | /** 76 | * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. If you 77 | * use a zipfian constant other than 0.99, this will take a long time to complete because we need to recompute zeta. 78 | * @param min The smallest integer to generate in the sequence. 79 | * @param max The largest integer to generate in the sequence. 80 | * @param _zipfianconstant The zipfian constant to use. 81 | */ 82 | public ScrambledZipfianGenerator(long min, long max, double _zipfianconstant) 83 | { 84 | _min=min; 85 | _max=max; 86 | _itemcount=_max-_min+1; 87 | if (_zipfianconstant == USED_ZIPFIAN_CONSTANT) 88 | { 89 | gen=new ZipfianGenerator(0,ITEM_COUNT,_zipfianconstant,ZETAN); 90 | } else { 91 | gen=new ZipfianGenerator(0,ITEM_COUNT,_zipfianconstant); 92 | } 93 | } 94 | 95 | /**************************************************************************************************/ 96 | 97 | /** 98 | * Return the next int in the sequence. 99 | */ 100 | @Override 101 | public int nextInt() { 102 | return (int)nextLong(); 103 | } 104 | 105 | /** 106 | * Return the next long in the sequence. 107 | */ 108 | public long nextLong() 109 | { 110 | long ret=gen.nextLong(); 111 | ret=_min+Utils.FNVhash64(ret)%_itemcount; 112 | setLastInt((int)ret); 113 | return ret; 114 | } 115 | 116 | public static void main(String[] args) 117 | { 118 | double newzetan = ZipfianGenerator.zetastatic(ITEM_COUNT,ZipfianGenerator.ZIPFIAN_CONSTANT); 119 | System.out.println("zetan: "+newzetan); 120 | System.exit(0); 121 | 122 | ScrambledZipfianGenerator gen=new ScrambledZipfianGenerator(10000); 123 | 124 | for (int i=0; i<1000000; i++) 125 | { 126 | System.out.println(""+gen.nextInt()); 127 | } 128 | } 129 | 130 | /** 131 | * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range. 132 | */ 133 | @Override 134 | public double mean() { 135 | return ((double)(((long)_min) +(long)_max))/2.0; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | /** 21 | * Generate a popularity distribution of items, skewed to favor recent items significantly more than older items. 22 | */ 23 | public class SkewedLatestGenerator extends IntegerGenerator 24 | { 25 | CounterGenerator _basis; 26 | ZipfianGenerator _zipfian; 27 | 28 | public SkewedLatestGenerator(CounterGenerator basis) 29 | { 30 | _basis=basis; 31 | _zipfian=new ZipfianGenerator(_basis.lastInt()); 32 | nextInt(); 33 | } 34 | 35 | /** 36 | * Generate the next string in the distribution, skewed Zipfian favoring the items most recently returned by the basis generator. 37 | */ 38 | public int nextInt() 39 | { 40 | int max=_basis.lastInt(); 41 | int nextint=max-_zipfian.nextInt(max); 42 | setLastInt(nextint); 43 | return nextint; 44 | } 45 | 46 | public static void main(String[] args) 47 | { 48 | SkewedLatestGenerator gen=new SkewedLatestGenerator(new CounterGenerator(1000)); 49 | for (int i=0; i _values; 29 | String _laststring; 30 | UniformIntegerGenerator _gen; 31 | 32 | 33 | /** 34 | * Creates a generator that will return strings from the specified set uniformly randomly 35 | */ 36 | @SuppressWarnings( "unchecked" ) 37 | public UniformGenerator(Vector values) 38 | { 39 | _values=(Vector)values.clone(); 40 | _laststring=null; 41 | _gen=new UniformIntegerGenerator(0,values.size()-1); 42 | } 43 | 44 | /** 45 | * Generate the next string in the distribution. 46 | */ 47 | public String nextString() 48 | { 49 | _laststring=_values.elementAt(_gen.nextInt()); 50 | return _laststring; 51 | } 52 | 53 | /** 54 | * Return the previous string generated by the distribution; e.g., returned from the last nextString() call. 55 | * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet 56 | * been called, lastString() should return something reasonable. 57 | */ 58 | public String lastString() 59 | { 60 | if (_laststring==null) 61 | { 62 | nextString(); 63 | } 64 | return _laststring; 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.generator; 19 | 20 | import java.util.Random; 21 | 22 | import com.yahoo.ycsb.Utils; 23 | 24 | /** 25 | * Generates integers randomly uniform from an interval. 26 | */ 27 | public class UniformIntegerGenerator extends IntegerGenerator 28 | { 29 | int _lb,_ub,_interval; 30 | 31 | /** 32 | * Creates a generator that will return integers uniformly randomly from the interval [lb,ub] inclusive (that is, lb and ub are possible values) 33 | * 34 | * @param lb the lower bound (inclusive) of generated values 35 | * @param ub the upper bound (inclusive) of generated values 36 | */ 37 | public UniformIntegerGenerator(int lb, int ub) 38 | { 39 | _lb=lb; 40 | _ub=ub; 41 | _interval=_ub-_lb+1; 42 | } 43 | 44 | @Override 45 | public int nextInt() 46 | { 47 | int ret=Utils.random().nextInt(_interval)+_lb; 48 | setLastInt(ret); 49 | 50 | return ret; 51 | } 52 | 53 | @Override 54 | public double mean() { 55 | return ((double)((long)(_lb + (long)_ub))) / 2.0; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/generator/ZipfianGenerator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/efficient/HERD/2fa86cb66c18adc22e3e8e6513cc3482f7c26242/YCSB/src/com/yahoo/ycsb/generator/ZipfianGenerator.class -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/measurements/Measurements.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.measurements; 19 | 20 | import java.io.IOException; 21 | import java.util.HashMap; 22 | import java.util.Properties; 23 | 24 | import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; 25 | 26 | /** 27 | * Collects latency measurements, and reports them when requested. 28 | * 29 | * @author cooperb 30 | * 31 | */ 32 | public class Measurements 33 | { 34 | private static final String MEASUREMENT_TYPE = "measurementtype"; 35 | 36 | private static final String MEASUREMENT_TYPE_DEFAULT = "histogram"; 37 | 38 | static Measurements singleton=null; 39 | 40 | static Properties measurementproperties=null; 41 | 42 | public static void setProperties(Properties props) 43 | { 44 | measurementproperties=props; 45 | } 46 | 47 | /** 48 | * Return the singleton Measurements object. 49 | */ 50 | public synchronized static Measurements getMeasurements() 51 | { 52 | if (singleton==null) 53 | { 54 | singleton=new Measurements(measurementproperties); 55 | } 56 | return singleton; 57 | } 58 | 59 | HashMap data; 60 | boolean histogram=true; 61 | 62 | private Properties _props; 63 | 64 | /** 65 | * Create a new object with the specified properties. 66 | */ 67 | public Measurements(Properties props) 68 | { 69 | data=new HashMap(); 70 | 71 | _props=props; 72 | 73 | if (_props.getProperty(MEASUREMENT_TYPE, MEASUREMENT_TYPE_DEFAULT).compareTo("histogram")==0) 74 | { 75 | histogram=true; 76 | } 77 | else 78 | { 79 | histogram=false; 80 | } 81 | } 82 | 83 | OneMeasurement constructOneMeasurement(String name) 84 | { 85 | if (histogram) 86 | { 87 | return new OneMeasurementHistogram(name,_props); 88 | } 89 | else 90 | { 91 | return new OneMeasurementTimeSeries(name,_props); 92 | } 93 | } 94 | 95 | /** 96 | * Report a single value of a single metric. E.g. for read latency, operation="READ" and latency is the measured value. 97 | */ 98 | public synchronized void measure(String operation, int latency) 99 | { 100 | if (!data.containsKey(operation)) 101 | { 102 | synchronized(this) 103 | { 104 | if (!data.containsKey(operation)) 105 | { 106 | data.put(operation,constructOneMeasurement(operation)); 107 | } 108 | } 109 | } 110 | try 111 | { 112 | data.get(operation).measure(latency); 113 | } 114 | catch (java.lang.ArrayIndexOutOfBoundsException e) 115 | { 116 | System.out.println("ERROR: java.lang.ArrayIndexOutOfBoundsException - ignoring and continuing"); 117 | e.printStackTrace(); 118 | e.printStackTrace(System.out); 119 | } 120 | } 121 | 122 | /** 123 | * Report a return code for a single DB operaiton. 124 | */ 125 | public void reportReturnCode(String operation, int code) 126 | { 127 | if (!data.containsKey(operation)) 128 | { 129 | synchronized(this) 130 | { 131 | if (!data.containsKey(operation)) 132 | { 133 | data.put(operation,constructOneMeasurement(operation)); 134 | } 135 | } 136 | } 137 | data.get(operation).reportReturnCode(code); 138 | } 139 | 140 | /** 141 | * Export the current measurements to a suitable format. 142 | * 143 | * @param exporter Exporter representing the type of format to write to. 144 | * @throws IOException Thrown if the export failed. 145 | */ 146 | public void exportMeasurements(MeasurementsExporter exporter) throws IOException 147 | { 148 | for (OneMeasurement measurement : data.values()) 149 | { 150 | measurement.exportMeasurements(exporter); 151 | } 152 | } 153 | 154 | /** 155 | * Return a one line summary of the measurements. 156 | */ 157 | public String getSummary() 158 | { 159 | String ret=""; 160 | for (OneMeasurement m : data.values()) 161 | { 162 | ret+=m.getSummary()+" "; 163 | } 164 | 165 | return ret; 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/measurements/OneMeasurement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.measurements; 19 | 20 | import java.io.IOException; 21 | 22 | import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; 23 | 24 | /** 25 | * A single measured metric (such as READ LATENCY) 26 | */ 27 | public abstract class OneMeasurement { 28 | 29 | String _name; 30 | 31 | public String getName() { 32 | return _name; 33 | } 34 | 35 | /** 36 | * @param _name 37 | */ 38 | public OneMeasurement(String _name) { 39 | this._name = _name; 40 | } 41 | 42 | public abstract void reportReturnCode(int code); 43 | 44 | public abstract void measure(int latency); 45 | 46 | public abstract String getSummary(); 47 | 48 | /** 49 | * Export the current measurements to a suitable format. 50 | * 51 | * @param exporter Exporter representing the type of format to write to. 52 | * @throws IOException Thrown if the export failed. 53 | */ 54 | public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException; 55 | } 56 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.measurements; 19 | 20 | import java.io.IOException; 21 | import java.text.DecimalFormat; 22 | import java.util.HashMap; 23 | import java.util.Properties; 24 | 25 | import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; 26 | 27 | 28 | /** 29 | * Take measurements and maintain a histogram of a given metric, such as READ LATENCY. 30 | * 31 | * @author cooperb 32 | * 33 | */ 34 | public class OneMeasurementHistogram extends OneMeasurement 35 | { 36 | public static final String BUCKETS="histogram.buckets"; 37 | public static final String BUCKETS_DEFAULT="1000"; 38 | 39 | int _buckets; 40 | int[] histogram; 41 | int histogramoverflow; 42 | int operations; 43 | long totallatency; 44 | 45 | //keep a windowed version of these stats for printing status 46 | int windowoperations; 47 | long windowtotallatency; 48 | 49 | int min; 50 | int max; 51 | HashMap returncodes; 52 | 53 | public OneMeasurementHistogram(String name, Properties props) 54 | { 55 | super(name); 56 | _buckets=Integer.parseInt(props.getProperty(BUCKETS, BUCKETS_DEFAULT)); 57 | histogram=new int[_buckets]; 58 | histogramoverflow=0; 59 | operations=0; 60 | totallatency=0; 61 | windowoperations=0; 62 | windowtotallatency=0; 63 | min=-1; 64 | max=-1; 65 | returncodes=new HashMap(); 66 | } 67 | 68 | /* (non-Javadoc) 69 | * @see com.yahoo.ycsb.OneMeasurement#reportReturnCode(int) 70 | */ 71 | public synchronized void reportReturnCode(int code) 72 | { 73 | Integer Icode=code; 74 | if (!returncodes.containsKey(Icode)) 75 | { 76 | int[] val=new int[1]; 77 | val[0]=0; 78 | returncodes.put(Icode,val); 79 | } 80 | returncodes.get(Icode)[0]++; 81 | } 82 | 83 | 84 | /* (non-Javadoc) 85 | * @see com.yahoo.ycsb.OneMeasurement#measure(int) 86 | */ 87 | public synchronized void measure(int latency) 88 | { 89 | if (latency/1000>=_buckets) 90 | { 91 | histogramoverflow++; 92 | } 93 | else 94 | { 95 | histogram[latency/1000]++; 96 | } 97 | operations++; 98 | totallatency+=latency; 99 | windowoperations++; 100 | windowtotallatency+=latency; 101 | 102 | if ( (min<0) || (latencymax) ) 108 | { 109 | max=latency; 110 | } 111 | } 112 | 113 | 114 | @Override 115 | public void exportMeasurements(MeasurementsExporter exporter) throws IOException 116 | { 117 | exporter.write(getName(), "Operations", operations); 118 | exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations))); 119 | exporter.write(getName(), "MinLatency(us)", min); 120 | exporter.write(getName(), "MaxLatency(us)", max); 121 | 122 | int opcounter=0; 123 | boolean done95th=false; 124 | for (int i=0; i<_buckets; i++) 125 | { 126 | opcounter+=histogram[i]; 127 | if ( (!done95th) && (((double)opcounter)/((double)operations)>=0.95) ) 128 | { 129 | exporter.write(getName(), "95thPercentileLatency(ms)", i); 130 | done95th=true; 131 | } 132 | if (((double)opcounter)/((double)operations)>=0.99) 133 | { 134 | exporter.write(getName(), "99thPercentileLatency(ms)", i); 135 | break; 136 | } 137 | } 138 | 139 | for (Integer I : returncodes.keySet()) 140 | { 141 | int[] val=returncodes.get(I); 142 | exporter.write(getName(), "Return="+I, val[0]); 143 | } 144 | 145 | for (int i=0; i<_buckets; i++) 146 | { 147 | exporter.write(getName(), Integer.toString(i), histogram[i]); 148 | } 149 | exporter.write(getName(), ">"+_buckets, histogramoverflow); 150 | } 151 | 152 | @Override 153 | public String getSummary() { 154 | if (windowoperations==0) 155 | { 156 | return ""; 157 | } 158 | DecimalFormat d = new DecimalFormat("#.##"); 159 | double report=((double)windowtotallatency)/((double)windowoperations); 160 | windowtotallatency=0; 161 | windowoperations=0; 162 | return "["+getName()+" AverageLatency(us)="+d.format(report)+"]"; 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /YCSB/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you 5 | * may not use this file except in compliance with the License. You 6 | * may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. See accompanying 15 | * LICENSE file. 16 | */ 17 | 18 | package com.yahoo.ycsb.measurements; 19 | 20 | import java.io.IOException; 21 | import java.text.DecimalFormat; 22 | import java.util.HashMap; 23 | import java.util.Properties; 24 | import java.util.Vector; 25 | 26 | import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter; 27 | 28 | class SeriesUnit 29 | { 30 | /** 31 | * @param time 32 | * @param average 33 | */ 34 | public SeriesUnit(long time, double average) { 35 | this.time = time; 36 | this.average = average; 37 | } 38 | public long time; 39 | public double average; 40 | } 41 | 42 | /** 43 | * A time series measurement of a metric, such as READ LATENCY. 44 | */ 45 | public class OneMeasurementTimeSeries extends OneMeasurement 46 | { 47 | /** 48 | * Granularity for time series; measurements will be averaged in chunks of this granularity. Units are milliseconds. 49 | */ 50 | public static final String GRANULARITY="timeseries.granularity"; 51 | 52 | public static final String GRANULARITY_DEFAULT="1000"; 53 | 54 | int _granularity; 55 | Vector _measurements; 56 | 57 | long start=-1; 58 | long currentunit=-1; 59 | int count=0; 60 | int sum=0; 61 | int operations=0; 62 | long totallatency=0; 63 | 64 | //keep a windowed version of these stats for printing status 65 | int windowoperations=0; 66 | long windowtotallatency=0; 67 | 68 | int min=-1; 69 | int max=-1; 70 | 71 | private HashMap returncodes; 72 | 73 | public OneMeasurementTimeSeries(String name, Properties props) 74 | { 75 | super(name); 76 | _granularity=Integer.parseInt(props.getProperty(GRANULARITY,GRANULARITY_DEFAULT)); 77 | _measurements=new Vector(); 78 | returncodes=new HashMap(); 79 | } 80 | 81 | void checkEndOfUnit(boolean forceend) 82 | { 83 | long now=System.currentTimeMillis(); 84 | 85 | if (start<0) 86 | { 87 | currentunit=0; 88 | start=now; 89 | } 90 | 91 | long unit=((now-start)/_granularity)*_granularity; 92 | 93 | if ( (unit>currentunit) || (forceend) ) 94 | { 95 | double avg=((double)sum)/((double)count); 96 | _measurements.add(new SeriesUnit(currentunit,avg)); 97 | 98 | currentunit=unit; 99 | 100 | count=0; 101 | sum=0; 102 | } 103 | } 104 | 105 | @Override 106 | public void measure(int latency) 107 | { 108 | checkEndOfUnit(false); 109 | 110 | count++; 111 | sum+=latency; 112 | totallatency+=latency; 113 | operations++; 114 | windowoperations++; 115 | windowtotallatency+=latency; 116 | 117 | if (latency>max) 118 | { 119 | max=latency; 120 | } 121 | 122 | if ( (latency 28 | * Properties to control the client: 29 | *

30 | *
    31 | *
  • disksize: how many bytes of storage can the disk store? (default 100,000,000) 32 | *
  • occupancy: what fraction of the available storage should be used? (default 0.9) 33 | *
  • requestdistribution: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: histogram) 34 | *
35 | * 36 | * 37 | *

See also: 38 | * Russell Sears, Catharine van Ingen. 39 | * Fragmentation in Large Object Repositories, 40 | * CIDR 2006. [Presentation] 41 | *

42 | * 43 | * 44 | * @author sears 45 | * 46 | */ 47 | public class ConstantOccupancyWorkload extends CoreWorkload { 48 | long disksize; 49 | long storageages; 50 | IntegerGenerator objectsizes; 51 | double occupancy; 52 | 53 | long object_count; 54 | 55 | public static final String STORAGE_AGE_PROPERTY = "storageages"; 56 | public static final long STORAGE_AGE_PROPERTY_DEFAULT = 10; 57 | 58 | public static final String DISK_SIZE_PROPERTY = "disksize"; 59 | public static final long DISK_SIZE_PROPERTY_DEFAULT = 100 * 1000 * 1000; 60 | 61 | public static final String OCCUPANCY_PROPERTY = "occupancy"; 62 | public static final double OCCUPANCY_PROPERTY_DEFAULT = 0.9; 63 | 64 | @Override 65 | public void init(Properties p) throws WorkloadException 66 | { 67 | disksize = Long.parseLong( p.getProperty(DISK_SIZE_PROPERTY, DISK_SIZE_PROPERTY_DEFAULT+"")); 68 | storageages = Long.parseLong( p.getProperty(STORAGE_AGE_PROPERTY, STORAGE_AGE_PROPERTY_DEFAULT+"")); 69 | occupancy = Double.parseDouble(p.getProperty(OCCUPANCY_PROPERTY, OCCUPANCY_PROPERTY_DEFAULT+"")); 70 | 71 | if(p.getProperty(Client.RECORD_COUNT_PROPERTY) != null || 72 | p.getProperty(Client.INSERT_COUNT_PROPERTY) != null || 73 | p.getProperty(Client.OPERATION_COUNT_PROPERTY) != null) { 74 | System.err.println("Warning: record, insert or operation count was set prior to initting ConstantOccupancyWorkload. Overriding old values."); 75 | } 76 | IntegerGenerator g = CoreWorkload.getFieldLengthGenerator(p); 77 | double fieldsize = g.mean(); 78 | int fieldcount = Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT)); 79 | 80 | object_count = (long)(occupancy * ((double)disksize / (fieldsize * (double)fieldcount))); 81 | if(object_count == 0) { 82 | throw new IllegalStateException("Object count was zero. Perhaps disksize is too low?"); 83 | } 84 | p.setProperty(Client.RECORD_COUNT_PROPERTY, object_count+""); 85 | p.setProperty(Client.OPERATION_COUNT_PROPERTY, (storageages*object_count)+""); 86 | p.setProperty(Client.INSERT_COUNT_PROPERTY, object_count+""); 87 | 88 | super.init(p); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /YCSB/src/gen-zipf.sh: -------------------------------------------------------------------------------- 1 | sudo mkdir /dev/zipf 2 | sudo chown akalia:fawn /dev/zipf 3 | 4 | # As this script is executed by the server on all clients (via ssh), 5 | # do not recompile zipf.java. This can lead to some clients seeing 6 | # a bad zipf class (ToCToU). 7 | java zipf 8 | -------------------------------------------------------------------------------- /YCSB/src/prop.sh: -------------------------------------------------------------------------------- 1 | # Test zipf properties 2 | javac zipf_properties.java 3 | java zipf_properties 10000000 # Use 20 million samples 4 | -------------------------------------------------------------------------------- /YCSB/src/zipf.java: -------------------------------------------------------------------------------- 1 | import java.io.File; 2 | import java.io.FileNotFoundException; 3 | import java.io.PrintWriter; 4 | import java.util.Arrays; 5 | 6 | import com.yahoo.ycsb.generator.*; 7 | 8 | public class zipf { 9 | static int NUM_SERVERS = 7; 10 | static int NUM_CLIENTS = 51; 11 | public static void main(String args[]) throws FileNotFoundException { 12 | int Freq[] = new int[NUM_SERVERS]; 13 | 14 | PrintWriter out[] = new PrintWriter[NUM_CLIENTS]; 15 | for (int i = 0; i < NUM_CLIENTS; i++) { 16 | String filename = "/dev/zipf/data" + i + ".dat"; 17 | out[i] = new PrintWriter(new File(filename)); 18 | } 19 | 20 | ScrambledZipfianGenerator gen = new ScrambledZipfianGenerator(0, 21 | Long.MAX_VALUE, .99); 22 | 23 | int N = 1 * 1024 * 1024; 24 | for (int cn = 0; cn < NUM_CLIENTS; cn++) { 25 | for (int i = 0; i < N; i++) { 26 | long num = gen.nextLong(); 27 | long mask = 0xffL; 28 | if((num & mask) == 0) { // Do not allow last byte = 0 29 | i --; 30 | continue; 31 | } 32 | Freq[(int)((num >> 40) % (NUM_SERVERS - 1)) + 1] ++; 33 | out[cn].println(num); 34 | } 35 | System.out.println("Done for client " + cn); 36 | } 37 | 38 | Arrays.sort(Freq); 39 | System.out.println(Arrays.toString(Freq)); 40 | for (int cn = 0; cn < NUM_CLIENTS; cn++) { 41 | out[cn].close(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /YCSB/src/zipf_properties.java: -------------------------------------------------------------------------------- 1 | // Code to find some Zipf properties 2 | import java.io.File; 3 | import java.io.FileNotFoundException; 4 | import java.io.PrintWriter; 5 | import java.util.Arrays; 6 | import java.util.Collections; 7 | 8 | import com.yahoo.ycsb.generator.*; 9 | 10 | public class zipf_properties { 11 | public static void main(String args[]) throws FileNotFoundException { 12 | int count = Integer.parseInt(args[0]); // Number of samples 13 | 14 | int buckets = 64; 15 | int Freq[] = new int[buckets + 1]; 16 | 17 | // Initialize a generator for numbers between 0 and @buckets with 18 | // skeness .99 19 | ScrambledZipfianGenerator gen = new ScrambledZipfianGenerator(0, 20 | Long.MAX_VALUE, .99); 21 | 22 | for (int i = 0; i < count; i++) { 23 | long num = gen.nextLong(); 24 | Freq[(int) (num % buckets)]++; 25 | } 26 | 27 | // Sort the array - this is dumb.. 28 | Arrays.sort(Freq); 29 | for (int i = 0; i < Freq.length / 2; i++) { 30 | int temp = Freq[i]; 31 | Freq[i] = Freq[Freq.length - 1 - i]; 32 | Freq[Freq.length - 1 - i] = temp; 33 | } 34 | 35 | // Print percentiles 36 | for (int percentile = 5; percentile <= 100; percentile += 5) { 37 | int tot = 0; 38 | int i; 39 | for(i = 0; i < buckets; i++) { 40 | tot += Freq[i]; 41 | if(tot >= ((double) percentile * count) / 100) { 42 | i++; 43 | break; 44 | } 45 | } 46 | 47 | System.out.println(i + " buckets of " + buckets + " comprise " + 48 | percentile + " percent of samples"); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /common.h: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "sizes.h" 19 | 20 | #define FAIL_LIM 100 // # of failed polls before a server advances its pipeline 21 | #define ZIPF 0 // Use ZIPF distributed workload 22 | #define KEY_SIZE 2 // In long long units 23 | #define VALUE_SIZE 32 // In char units 24 | #define SLOTS_PER_BKT 8 // Number of pairs in a MICA-style index bkt 25 | 26 | #define PUT_PERCENT 5 // Percentage of PUT operations 27 | 28 | #define IB_PHYS_PORT 1 // Primary physical port number for qps 29 | #define CLIENT_PRINT_LAT 0 // Should clients sample request latency? 30 | 31 | #define USE_UC 1 // Use UC for requests. If 0, RC is ued 32 | #define USE_INLINE 1 // Use WQE inlining for requests and responses 33 | #define USE_HUGEPAGE 1 34 | 35 | #if (USE_INLINE == 0) 36 | #define MY_SEND_INLINE 0 37 | #else 38 | #define MY_SEND_INLINE IBV_SEND_INLINE 39 | #endif 40 | 41 | #define NUM_CLIENTS 36 // Number of client processes 42 | #define NUM_SERVERS 7 // Number of server processes 43 | 44 | #define Q_DEPTH 1024 // Size of all created queues 45 | #define S_DEPTH 512 46 | #define S_DEPTH_ 511 47 | 48 | /**** Performance params that need to be tuned with VALUE_SIZE ****/ 49 | #define WINDOW_SIZE 4 // Outstanding requests by a client 50 | #define WINDOW_SIZE_ 3 51 | 52 | #define WS_SERVER 64 // Outstanding responses by a server 53 | #define WS_SERVER_ 63 54 | /******************************************************************/ 55 | 56 | #define CL_BTCH_SZ 128 // Number of RECVs maintained by a client 57 | #define CL_BTCH_SZ_ 127 58 | 59 | #define CL_SEMI_BTCH_SZ 64 // Client posts new RECVs if the number drops below this 60 | #define CL_SEMI_BTCH_SZ_ 63 61 | 62 | #define NUM_ITER 1000000000 // Total number of iterations performed by a client 63 | 64 | #define REQ_AC (NUM_CLIENTS * WINDOW_SIZE * NUM_SERVERS) 65 | #define RESP_AC (WINDOW_SIZE * NUM_SERVERS) 66 | 67 | // SHM keys for servers. The request and response area is shared among server processes 68 | // and needs a single key. For lossy index and circular log, each server uses a separate 69 | // key := BASE + cb->id. Number of server processes must be less than 32. 70 | #define REQ_AREA_SHM_KEY 3185 71 | #define RESP_AREA_SHM_KEY 3186 72 | #define BASE_HT_INDEX_SHM_KEY 1 73 | #define BASE_HT_LOG_SHM_KEY 32 74 | 75 | // The number of keys written by each client process 76 | #define NUM_KEYS M_1 // 51 * M_4 ~ 200 M keys 77 | #define NUM_KEYS_ M_1_ 78 | 79 | // Number of index buckets (each with 8 slots) at each server process 80 | #define NUM_IDX_BKTS M_4 // Size = 256M. Support for 32M keys 81 | #define NUM_IDX_BKTS_ M_4_ 82 | 83 | // The size of the circular at each server process 84 | #define LOG_SIZE M_128 // Support for 32M keys 85 | #define LOG_SIZE_ M_128_ 86 | 87 | // Compare, print, and exit 88 | #define CPE(val, msg, err_code) \ 89 | if(val) { fprintf(stderr, msg); fprintf(stderr, " Error %d \n", err_code); \ 90 | exit(err_code);} 91 | 92 | // The key-value struct 93 | struct __attribute__((__packed__)) KV { 94 | uint16_t len; 95 | char value[VALUE_SIZE]; 96 | long long key[KEY_SIZE]; // <-- KV_KEY_OFFSET 97 | }; 98 | #define S_KV sizeof(struct KV) 99 | #define KV_KEY_OFFSET (2 + VALUE_SIZE) 100 | 101 | // For a RECV completion, clients get the KV and the UD GRH 102 | struct UD_KV { 103 | char grh[40]; 104 | struct KV kv; 105 | }; 106 | #define S_UD_KV sizeof(struct UD_KV) 107 | 108 | // Operation types for pipeline items 109 | #define GET_TYPE 81 110 | #define PUT_TYPE 82 111 | #define DUMMY_TYPE 83 112 | #define EMPTY_TYPE 84 113 | 114 | // If the client receives len < GET_FAIL_LEN_1 for a GET request, it assumes 115 | // that the request succeeded and checks the value. PUT requests always succeed. 116 | #define GET_FAIL_LEN_1 20001 // Denotes GET failure in pipeline stage 1 117 | #define GET_FAIL_LEN_2 20002 // Denotes GET failure in pipeline stage 2 118 | 119 | // PIpeline ITem 120 | struct PL_IT { 121 | struct KV *kv; 122 | int cn; // Client who sent the KV 123 | int req_area_slot; // Request area slot into which this request was received 124 | int get_slot; 125 | int req_type; 126 | // The rightmost part of KV that must be zeroed immediately after detecting 127 | // a new request. If this is not done, the server can loop around the window 128 | // and detect an old request again 129 | long long poll_val; 130 | }; 131 | #define S_PL_IT sizeof(struct PL_IT) 132 | struct PL_IT pipeline[2]; // The pipeline 133 | 134 | struct IDX_BKT { // An index bucket 135 | long long slot[SLOTS_PER_BKT]; 136 | }; 137 | #define S_IDX_BKT sizeof(struct IDX_BKT) 138 | 139 | // A slot contains a 6 byte offset and a 2 byte tag. 140 | // Invalid slots have their MSB set to 1. 141 | #define INVALID_SLOT 0x8000000000000000 142 | #define MIN_INT 0x80000000 143 | #define MIN_LL 0x8000000000000000 144 | 145 | struct qp_attr { 146 | uint64_t gid_global_interface_id; // Store the gid fields separately because I 147 | uint64_t gid_global_subnet_prefix; // don't like unions. Needed for RoCE only 148 | 149 | int lid; // A queue pair is identified by the local id (lid) 150 | int qpn; // of the device port and its queue pair number (qpn) 151 | int psn; 152 | }; 153 | #define S_QPA sizeof(struct qp_attr) 154 | 155 | struct ctrl_blk { 156 | struct ibv_context *context; 157 | struct ibv_pd *pd; 158 | 159 | struct ibv_cq **conn_cq; // Queue pairs and completion queues 160 | struct ibv_qp **conn_qp; 161 | 162 | struct ibv_cq **dgram_cq; 163 | struct ibv_qp **dgram_qp; 164 | 165 | struct ibv_ah *ah[NUM_CLIENTS]; // Per client address handles 166 | 167 | struct qp_attr *local_dgram_qp_attrs; // Local and remote queue pair attributes 168 | struct qp_attr *remote_dgram_qp_attrs; 169 | 170 | struct qp_attr *local_conn_qp_attrs; 171 | struct qp_attr *remote_conn_qp_attrs; 172 | 173 | struct ibv_send_wr wr; // A work request and its scatter-gather list 174 | struct ibv_sge sgl; 175 | 176 | int num_conn_qps, num_remote_dgram_qps, num_local_dgram_qps; 177 | int is_client, id; 178 | int sock_port; // The socket port a server uses for client connections 179 | }; 180 | 181 | struct stag { // An "stag" identifying an RDMA region 182 | uint64_t buf; 183 | uint32_t rkey; 184 | uint32_t size; 185 | }; 186 | #define S_STG sizeof(struct stag) 187 | 188 | // The lossy index and the circular log 189 | struct IDX_BKT *ht_index; 190 | char *ht_log; 191 | 192 | // Request and response regions, and their RDMA memory-region descriptors 193 | volatile struct KV *server_req_area; 194 | volatile struct KV *server_resp_area; 195 | volatile struct KV *client_req_area; 196 | volatile struct UD_KV *client_resp_area; 197 | 198 | struct ibv_mr *server_req_area_mr, *server_resp_area_mr; 199 | struct ibv_mr *client_resp_area_mr, *client_req_area_mr; 200 | 201 | struct stag server_req_area_stag[NUM_SERVERS], client_resp_area_stag[NUM_CLIENTS]; 202 | 203 | union ibv_gid get_gid(struct ibv_context *context); 204 | uint16_t get_local_lid(struct ibv_context *context); 205 | 206 | void create_qp(struct ctrl_blk *ctx); 207 | void modify_qp_to_init(struct ctrl_blk *ctx); 208 | int setup_buffers(struct ctrl_blk *cb); 209 | 210 | void client_exch_dest(struct ctrl_blk *ctx); 211 | void server_exch_dest(struct ctrl_blk *ctx); 212 | 213 | int modify_dgram_qp_to_rts(struct ctrl_blk *ctx); 214 | int connect_ctx(struct ctrl_blk *ctx, int my_psn, struct qp_attr dest, int qp_i); 215 | 216 | int close_ctx(struct ctrl_blk *ctx); 217 | 218 | void print_kv(struct KV); 219 | void print_ud_kv(struct UD_KV); 220 | void print_stag(struct stag); 221 | void print_qp_attr(struct qp_attr); 222 | void print_kv_array(volatile struct KV *kv, int size); 223 | void print_ud_kv_array(struct UD_KV *ud_kv, int size); 224 | void print_ht_index(); 225 | 226 | void nano_sleep(int ns); 227 | inline long long get_cycles(); 228 | 229 | void poll_conn_cq(int num_completions, struct ctrl_blk *cb, int cq_num); 230 | void poll_dgram_cq(int num_completions, struct ctrl_blk *cb, int cq_num); 231 | 232 | int valcheck(volatile char *val, long long exp); 233 | 234 | long long* gen_key_corpus(int cn); 235 | void init_ht(struct ctrl_blk *cb); 236 | int is_roce(void); 237 | inline uint32_t fastrand(uint64_t* seed); 238 | 239 | #define LL long long 240 | #define KEY_TO_BUCKET(k) ((int) (k >> 16) & NUM_IDX_BKTS_) // 3 bytes (up to 16 Mi buckets) 241 | #define KEY_TO_TAG(k) ((int) (k & 0xffff)) // 2 bytes 242 | #define KEY_TO_SERVER(k) ((int) ((k >> 40) % (NUM_SERVERS - 1) + 1)) // DO NOT shift by 48 243 | 244 | #define SLOT_TO_OFFSET(s) (s >> 16) 245 | #define SLOT_TO_TAG(s) ((int) (s & 0xffff)) 246 | 247 | #define SET_PL_IT_MEMCPY_DONE(pl_it) (pl_it.cn |= 0xf00) 248 | #define GET_PL_IT_MEMCPY_DONE(pl_it) (pl_it->cn & 0xf00) 249 | -------------------------------------------------------------------------------- /conn.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | // Transition all UD QPs to RTS 4 | int modify_dgram_qp_to_rts(struct ctrl_blk *ctx) 5 | { 6 | int i; 7 | for(i = 0; i < ctx->num_local_dgram_qps; i++) { 8 | struct ibv_qp_attr dgram_attr = { 9 | .qp_state = IBV_QPS_RTR, 10 | }; 11 | 12 | if (ibv_modify_qp(ctx->dgram_qp[i], &dgram_attr, IBV_QP_STATE)) { 13 | fprintf(stderr, "Failed to modify dgram QP to RTR\n"); 14 | return 1; 15 | } 16 | 17 | dgram_attr.qp_state = IBV_QPS_RTS; 18 | dgram_attr.sq_psn = ctx->local_dgram_qp_attrs[i].psn; 19 | 20 | if(ibv_modify_qp(ctx->dgram_qp[i], 21 | &dgram_attr, IBV_QP_STATE|IBV_QP_SQ_PSN)) { 22 | fprintf(stderr, "Failed to modify dgram QP to RTS\n"); 23 | return 1; 24 | } 25 | } 26 | 27 | return 0; 28 | } 29 | 30 | // Transition connected QP indexed qp_i through RTR and RTS stages 31 | int connect_ctx(struct ctrl_blk *ctx, int my_psn, struct qp_attr dest, 32 | int qp_i) 33 | { 34 | struct ibv_qp_attr conn_attr = { 35 | .qp_state = IBV_QPS_RTR, 36 | .path_mtu = IBV_MTU_4096, 37 | .dest_qp_num = dest.qpn, 38 | .rq_psn = dest.psn, 39 | .ah_attr = { 40 | .is_global = (is_roce() == 1) ? 1 : 0, 41 | .dlid = (is_roce() == 1) ? 0 : dest.lid, 42 | .sl = 0, 43 | .src_path_bits = 0, 44 | .port_num = IB_PHYS_PORT 45 | } 46 | }; 47 | 48 | if(is_roce()) { 49 | conn_attr.ah_attr.grh.dgid.global.interface_id = 50 | dest.gid_global_interface_id; 51 | conn_attr.ah_attr.grh.dgid.global.subnet_prefix = 52 | dest.gid_global_subnet_prefix; 53 | 54 | conn_attr.ah_attr.grh.sgid_index = 0; 55 | conn_attr.ah_attr.grh.hop_limit = 1; 56 | } 57 | 58 | int rtr_flags = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN 59 | | IBV_QP_RQ_PSN; 60 | if(!USE_UC) { 61 | conn_attr.max_dest_rd_atomic = 16; 62 | conn_attr.min_rnr_timer = 12; 63 | rtr_flags |= IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; 64 | } 65 | if (ibv_modify_qp(ctx->conn_qp[qp_i], &conn_attr, rtr_flags)) { 66 | fprintf(stderr, "Failed to modify QP to RTR\n"); 67 | return 1; 68 | } 69 | 70 | memset(&conn_attr, 0, sizeof(conn_attr)); 71 | conn_attr.qp_state = IBV_QPS_RTS; 72 | conn_attr.sq_psn = my_psn; 73 | int rts_flags = IBV_QP_STATE | IBV_QP_SQ_PSN; 74 | if(!USE_UC) { 75 | conn_attr.timeout = 14; 76 | conn_attr.retry_cnt = 7; 77 | conn_attr.rnr_retry = 7; 78 | conn_attr.max_rd_atomic = 16; 79 | rts_flags |= IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | 80 | IBV_QP_MAX_QP_RD_ATOMIC; 81 | } 82 | if (ibv_modify_qp(ctx->conn_qp[qp_i], &conn_attr, rts_flags)) { 83 | fprintf(stderr, "Failed to modify QP to RTS\n"); 84 | return 1; 85 | } 86 | 87 | return 0; 88 | } 89 | 90 | // Get server's request region STAG. Exchange queue pair attributes. 91 | void client_exch_dest(struct ctrl_blk *cb) 92 | { 93 | int sockfd, i, sock_port; 94 | 95 | struct sockaddr_in serv_addr; 96 | struct hostent *server; 97 | char server_name[20],sock_port_str[20]; 98 | 99 | for(i = 0; i < NUM_SERVERS; i++) { 100 | // Find the server name and port from the "servers" file 101 | scanf("%s", server_name); 102 | scanf("%s", sock_port_str); 103 | printf("At client %d, server_name = %s, port = %s\n", cb->id, 104 | server_name, sock_port_str); 105 | sock_port = atoi(sock_port_str); 106 | 107 | sockfd = socket(AF_INET, SOCK_STREAM, 0); 108 | CPE(sockfd < 0, "Error opening socket", 0); 109 | 110 | server = gethostbyname(server_name); 111 | CPE(server == NULL, "No such host", 0); 112 | 113 | bzero((char *) &serv_addr, sizeof(serv_addr)); 114 | serv_addr.sin_family = AF_INET; 115 | bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, 116 | server->h_length); 117 | serv_addr.sin_port = htons(sock_port); 118 | 119 | if(connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))) { 120 | fprintf(stderr, "ERROR connecting\n"); 121 | } 122 | 123 | // Get STAG 124 | if(read(sockfd, &server_req_area_stag[i], S_STG) < 0) { 125 | fprintf(stderr, "ERROR reading stag from socket\n"); 126 | } 127 | fprintf(stderr, "Client %d <-- Server %d's stag: ", cb->id, i); 128 | print_stag(server_req_area_stag[i]); 129 | 130 | // Exchange attributes for connected QPs 131 | if(write(sockfd, &cb->local_conn_qp_attrs[i], S_QPA) < 0) { 132 | fprintf(stderr, "ERROR writing conn qp_attr to socket\n"); 133 | } 134 | fprintf(stderr, "Client %d --> Server %d conn qp_attr: ", cb->id, i); 135 | print_qp_attr(cb->local_conn_qp_attrs[i]); 136 | 137 | if(read(sockfd, &cb->remote_conn_qp_attrs[i], S_QPA) < 0) { 138 | fprintf(stderr, "Error reading conn qp_attr from socket"); 139 | } 140 | fprintf(stderr, "Client %d <-- Server %d's conn qp_attr: ", cb->id, i); 141 | print_qp_attr(cb->remote_conn_qp_attrs[i]); 142 | 143 | // Send datagram QP attrs. Clients don't need server's UD QP attrs 144 | // The client sends a different UD QP to each server 145 | if(write(sockfd, &cb->local_dgram_qp_attrs[i], S_QPA) < 0) { 146 | fprintf(stderr, "ERROR writing dgram qp_attr to socket\n"); 147 | } 148 | fprintf(stderr, "Client %d --> Server %d UD qp_attr: ", cb->id, i); 149 | print_qp_attr(cb->local_dgram_qp_attrs[i]); 150 | 151 | close(sockfd); 152 | } 153 | } 154 | 155 | // Exchange QP information with clients 156 | void server_exch_dest(struct ctrl_blk *cb) 157 | { 158 | int sockfd, newsockfd, i; 159 | struct sockaddr_in serv_addr; 160 | 161 | sockfd = socket(AF_INET, SOCK_STREAM, 0); 162 | if (sockfd < 0) { 163 | fprintf(stderr, "ERROR opening socket"); 164 | } 165 | 166 | int on = 1, status = -1; 167 | status = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, 168 | (const char *) &on, sizeof(on)); 169 | if (-1 == status) { 170 | perror("setsockopt(...,SO_REUSEADDR,...)"); 171 | } 172 | 173 | bzero((char *) &serv_addr, sizeof(serv_addr)); 174 | serv_addr.sin_family = AF_INET; 175 | serv_addr.sin_addr.s_addr = INADDR_ANY; 176 | serv_addr.sin_port = htons(cb->sock_port); 177 | 178 | if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { 179 | fprintf(stderr, "ERROR on binding"); 180 | } 181 | printf("Server %d listening on port %d\n", cb->id, cb->sock_port); 182 | listen(sockfd, NUM_CLIENTS); 183 | 184 | for(i = 0; i < NUM_CLIENTS; i++) { 185 | 186 | printf("Server %d trying to accept()\n", cb->id); 187 | newsockfd = accept(sockfd, NULL, NULL); 188 | if (newsockfd < 0) { 189 | fprintf(stderr, "ERROR on accept"); 190 | exit(1); 191 | } 192 | 193 | // Exchange stag information 194 | server_req_area_stag[0].buf = (uint64_t) (unsigned long) 195 | server_req_area; 196 | server_req_area_stag[0].rkey = server_req_area_mr->rkey; 197 | server_req_area_stag[0].size = REQ_AC * S_KV; 198 | 199 | if(write(newsockfd, &server_req_area_stag[0], S_STG) < 0) { 200 | fprintf(stderr, "ERROR writing stag to socket\n"); 201 | } 202 | fprintf(stderr, "Server %d --> Client %d stag: ", cb->id, i); 203 | print_stag(server_req_area_stag[0]); 204 | 205 | // Exchange attributes for connected QPs 206 | if(read(newsockfd, &cb->remote_conn_qp_attrs[i], S_QPA) < 0) { 207 | fprintf(stderr, "ERROR reading conn qp_attr from socket\n"); 208 | } 209 | fprintf(stderr, "Server %d <-- Client %d's conn qp_attr: ", cb->id, i); 210 | print_qp_attr(cb->remote_conn_qp_attrs[i]); 211 | 212 | if(connect_ctx(cb, cb->local_conn_qp_attrs[i].psn, 213 | cb->remote_conn_qp_attrs[i], i)) { 214 | fprintf(stderr, "Couldn't connect to remote QP\n"); 215 | exit(0); 216 | } 217 | 218 | if(write(newsockfd, &cb->local_conn_qp_attrs[i], S_QPA) < 0 ) { 219 | fprintf(stderr, "Error writing conn qp_attr to socket\n"); 220 | } 221 | fprintf(stderr, "Server %d --> Client %d conn qp_attr: ", cb->id, i); 222 | print_qp_attr(cb->local_conn_qp_attrs[i]); 223 | 224 | // The server reads many clients' UD qp_attrs 225 | if(read(newsockfd, &cb->remote_dgram_qp_attrs[i], S_QPA) < 0) { 226 | fprintf(stderr, "ERROR reading dgram qp_attr from socket\n"); 227 | } 228 | fprintf(stderr, "Server %d <-- Client %d's UD qp_attr: ", cb->id, i); 229 | print_qp_attr(cb->remote_dgram_qp_attrs[i]); 230 | 231 | close(newsockfd); 232 | } 233 | close(sockfd); 234 | } 235 | -------------------------------------------------------------------------------- /do.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | make clean 3 | make 4 | -------------------------------------------------------------------------------- /kill-remote.sh: -------------------------------------------------------------------------------- 1 | # Action: 2 | # Kill clients by ssh-ing into client machines 3 | 4 | export APT=1 5 | 6 | if [ $APT -eq 1 ] 7 | then 8 | for i in `seq 2 110`; do 9 | ssh -oStrictHostKeyChecking=no node-$i.RDMA.fawn.apt.emulab.net "cd HERD; ./local-kill.sh" & 10 | done 11 | else 12 | for i in `seq 2 20`; do 13 | ssh anuj$i.RDMA.fawn.susitna.pdl.cmu.local "cd HERD; ./local-kill.sh" 14 | done 15 | fi 16 | -------------------------------------------------------------------------------- /local-kill.sh: -------------------------------------------------------------------------------- 1 | # Kill the HERD process at a machine 2 | 3 | sudo killall main 4 | sudo killall ssh 5 | -------------------------------------------------------------------------------- /run-machine.sh: -------------------------------------------------------------------------------- 1 | # Action: 2 | # 1. Run num_processes client processes 3 | 4 | num_processes=3 # Number of processes per client machine 5 | export ROCE=0 6 | export APT=1 7 | 8 | hi=`expr $num_processes - 1` 9 | for i in `seq 0 $hi`; do 10 | id=`expr $@ \* $num_processes + $i` 11 | echo "Running client id $id" 12 | touch client-tput/client-$id 13 | 14 | if [ $APT -eq 1 ] # There is only one socket on Apt's r320 nodes 15 | then 16 | sudo -E ./main $id < servers 1>client-tput/client-$id 2>client-tput/client-$id & 17 | else 18 | if [ $ROCE -eq 1 ] # Susitna's RoCE RNIC is connected to CPU 0 19 | then 20 | core=`expr 0 + $id` 21 | sudo -E numactl --physcpubind $core --interleave 0,1 ./main $id < servers & 22 | else # Susitna's IB RNIC is connected to CPU 3 23 | core=`expr 32 + $id` 24 | sudo -E numactl --physcpubind $core --interleave 4,5 ./main $id < servers & 25 | fi 26 | fi 27 | 28 | sleep .1 29 | done 30 | 31 | # When we run this script remotely, the client processes die when this script dies 32 | # So, sleep. 33 | sleep 10000 34 | -------------------------------------------------------------------------------- /run-servers.sh: -------------------------------------------------------------------------------- 1 | # Action: 2 | # 1. Run server processes on the server machine 3 | # 2. ssh into client machines and run the run-machine.sh script 4 | 5 | shm-rm.sh # Remove hugepages 6 | export ROCE=0 # Don't use RoCE on Apt 7 | export APT=1 8 | 9 | NUM_SERVERS=7 # Number of server processes on the server machine 10 | NUM_CLIENT_MACHINES=12 # Number of client machines 11 | 12 | rm -rf client-tput # Re-create a folder for clients to write their stuff into 13 | mkdir client-tput 14 | 15 | for i in `seq 1 $NUM_SERVERS`; do 16 | id=`expr $i - 1` 17 | sock_port=`expr 5500 + $i - 1` 18 | 19 | if [ $APT -eq 1 ] # There is only one socket on Apt's r320 nodes 20 | then 21 | sudo -E ./main $id $sock_port & 22 | else 23 | if [ $ROCE -eq 1 ] # Susitna's RoCE RNIC is connected to CPU 0 24 | then 25 | core=`expr 0 + $id` 26 | sudo -E numactl --physcpubind $core --interleave 0,1 ./main $id $sock_port & 27 | else # Susitna's IB RNIC is connected to CPU 3 28 | core=`expr 32 + $id` 29 | sudo -E numactl --physcpubind $core --interleave 4,5 ./main $id $sock_port & 30 | fi 31 | fi 32 | 33 | if [ $i -eq 1 ] # Give the master server plenty of time to setup 34 | then 35 | sleep 2 36 | else 37 | sleep .1 38 | fi 39 | done 40 | 41 | for i in `seq 1 $NUM_CLIENT_MACHINES`; do 42 | mc=`expr $i + 1` 43 | client_id=`expr $mc - 2` 44 | ssh -oStrictHostKeyChecking=no node-$mc.RDMA.fawn.apt.emulab.net "cd HERD; ./run-machine.sh $client_id" & 45 | echo "Starting client $client_id" 46 | 47 | # Removing this sleep sometimes causes the tput to drop drastically. 48 | # Bug: which part of the code requires clients to connect in order? 49 | sleep .5 50 | done 51 | 52 | -------------------------------------------------------------------------------- /scripts/average.sh: -------------------------------------------------------------------------------- 1 | rm temp 2 | sudo apt-get install bc 3 | DIR=$@ 4 | total_sum=0 5 | total_files=0 6 | for file in $DIR/*; do 7 | echo "Processing file $file" 8 | tail -5 $file >> temp 9 | total_files=`expr $total_files + 1` 10 | done 11 | 12 | sum=`awk '{ sum += $1 } END { print sum }' temp` 13 | echo "Sum = $sum" 14 | N=`wc -l temp | cut -f1 -d' '` 15 | avg=`python -c "print $sum / $N"` 16 | 17 | throughput=`python -c "print $avg * $total_files"` 18 | echo "Directory $DIR's average = $avg, throughput $throughput" 19 | -------------------------------------------------------------------------------- /scripts/busy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | while : 3 | do 4 | echo "Press [CTRL+C] to stop.." 5 | sleep 10 6 | done 7 | 8 | -------------------------------------------------------------------------------- /scripts/gen-servers.sh: -------------------------------------------------------------------------------- 1 | NUM_SERVERS=512 2 | hi=`expr $NUM_SERVERS - 1` 3 | echo $hi 4 | for i in `seq 0 $hi`; do 5 | echo "10.0.0.50" 6 | echo `expr 5500 + $i` 7 | done 8 | -------------------------------------------------------------------------------- /scripts/hugepages-check.sh: -------------------------------------------------------------------------------- 1 | cat /sys/devices/system/node/*/meminfo | grep Huge 2 | -------------------------------------------------------------------------------- /scripts/hugepages-create.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 2 ]; then 2 | echo "Illegal number of parameters" 3 | echo "Usage: ./hugepages-create.sh " 4 | exit 5 | fi 6 | 7 | echo "Creating $2 hugepages on socket $1" 8 | echo $2 > /sys/devices/system/node/node$1/hugepages/hugepages-2048kB/nr_hugepages 9 | -------------------------------------------------------------------------------- /scripts/shm-ls.sh: -------------------------------------------------------------------------------- 1 | ipcs -m 2 | -------------------------------------------------------------------------------- /scripts/shm-rm.sh: -------------------------------------------------------------------------------- 1 | for i in `seq 0 64`; do # Lossy index and circular log 2 | sudo ipcrm -M $i 3 | done 4 | sudo ipcrm -M 3185 # Request region at server 5 | sudo ipcrm -M 3186 # Response region at server 6 | -------------------------------------------------------------------------------- /scripts/zipf-check.sh: -------------------------------------------------------------------------------- 1 | ls -l /dev/zipf/zipf/ 2 | -------------------------------------------------------------------------------- /servers: -------------------------------------------------------------------------------- 1 | node-1.RDMA.fawn.apt.emulab.net 2 | 5500 3 | node-1.RDMA.fawn.apt.emulab.net 4 | 5501 5 | node-1.RDMA.fawn.apt.emulab.net 6 | 5502 7 | node-1.RDMA.fawn.apt.emulab.net 8 | 5503 9 | node-1.RDMA.fawn.apt.emulab.net 10 | 5504 11 | node-1.RDMA.fawn.apt.emulab.net 12 | 5505 13 | node-1.RDMA.fawn.apt.emulab.net 14 | 5506 15 | node-1.RDMA.fawn.apt.emulab.net 16 | 5507 17 | node-1.RDMA.fawn.apt.emulab.net 18 | 5508 19 | node-1.RDMA.fawn.apt.emulab.net 20 | 5509 21 | node-1.RDMA.fawn.apt.emulab.net 22 | 5510 23 | node-1.RDMA.fawn.apt.emulab.net 24 | 5511 25 | node-1.RDMA.fawn.apt.emulab.net 26 | 5512 27 | node-1.RDMA.fawn.apt.emulab.net 28 | 5513 29 | node-1.RDMA.fawn.apt.emulab.net 30 | 5514 31 | node-1.RDMA.fawn.apt.emulab.net 32 | 5515 33 | -------------------------------------------------------------------------------- /shm-init.sh: -------------------------------------------------------------------------------- 1 | # Increase the shmmax and shmall parameters so that a process can 2 | # map more memory via shmget (default is 32 MB) 3 | 4 | sudo sysctl -w kernel.shmmax=2147483648 # Bytes 5 | sudo sysctl -w kernel.shmall=2147483648 # Pages 6 | 7 | sudo sysctl -p /etc/sysctl.conf 8 | 9 | # If you want to generate the zipf workloads at the client machines, uncomment: 10 | ## for i in `seq 2 30`; do 11 | ## ssh node-$i.RDMA.fawn.apt.emulab.net "cd ~/HERD/YCSB/src; ./gen-zipf.sh" & 12 | ## done 13 | -------------------------------------------------------------------------------- /sizes.h: -------------------------------------------------------------------------------- 1 | #define K_128 131072 2 | #define K_128_ 131071 3 | 4 | #define K_512 524288 5 | #define K_512_ 524287 6 | 7 | #define M_1 1048576 8 | #define M_1_ 1048575 9 | 10 | #define M_2 2097152 11 | #define M_2_ 2097151 12 | 13 | #define M_4 4194304 14 | #define M_4_ 4194303 15 | 16 | #define M_8 8388608 17 | #define M_8_ 8388607 18 | 19 | #define M_16 16777216 20 | #define M_16_ 16777215 21 | 22 | #define M_32 33554432 23 | #define M_32_ 33554431 24 | 25 | #define M_128 134217728 26 | #define M_128_ 134217727 27 | 28 | #define M_256 268435456 29 | #define M_256_ 268435455 30 | 31 | #define M_512 536870912 32 | #define M_512_ 536870911 33 | 34 | #define M_1024 1073741824 35 | #define M_1024_ 1073741823 36 | 37 | #define M_2048 2147483648 38 | #define M_2048_ 2147483647 39 | 40 | 41 | --------------------------------------------------------------------------------