├── .gitignore ├── .travis.yml ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── TODO.md ├── all_unit_tests.sh ├── bench_tests.sh ├── cppperf ├── ABOUT.md ├── Makefile ├── main.cpp ├── memcache_client.cpp ├── memcache_client.h ├── tcp_client.cpp └── tcp_client.h ├── doc ├── Architecture.md ├── Performance-testing-ideas.md ├── Protocol-support.md └── Test-plan.md ├── fast_unit_tests.sh ├── format_all_code.sh ├── integration_tests.sh ├── pyemc ├── ABOUT.md ├── __init__.py ├── abstractions │ ├── __init__.py │ ├── task_api.py │ └── test_api.py ├── client.py ├── main.py ├── socket_stream.py ├── task_filler.py ├── test_integration.py ├── test_stress.py └── util.py ├── run_server.sh ├── rustfmt.toml └── src ├── common ├── consts.rs ├── conversions.rs └── mod.rs ├── main.rs ├── metrics ├── live_timers.rs ├── metric.rs ├── metrics.rs ├── mod.rs ├── recorder.rs ├── statistics │ ├── aggregate.rs │ ├── aggregated_metric.rs │ ├── mod.rs │ └── tests.rs ├── tests.rs ├── time_series.rs ├── timer.rs ├── timing.rs └── typedefs.rs ├── options.rs ├── orchestrator ├── driver_task.rs ├── listener_task.rs ├── metrics_task.rs ├── mod.rs ├── transport_task.rs └── typedefs.rs ├── platform ├── mod.rs ├── process.rs └── time.rs ├── protocol ├── cmd.rs ├── driver.rs ├── mod.rs ├── tests.rs ├── tests_bench.rs └── util.rs ├── storage ├── cache.rs ├── errors.rs ├── key.rs ├── macros.rs ├── mod.rs ├── tests.rs ├── tests_bench.rs ├── typedefs.rs └── value.rs ├── tcp_transport ├── conversions.rs ├── errors.rs ├── mod.rs ├── stats.rs ├── tests.rs ├── tests_bench.rs ├── transport.rs └── typedefs.rs └── testlib ├── cmp.rs ├── datagen.rs ├── mod.rs └── test_stream.rs /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.rs.bk 3 | cppperf/perftest 4 | cppperf/*.o 5 | target/ 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | sudo: false 3 | 4 | rust: 5 | - stable 6 | - beta 7 | - nightly 8 | 9 | matrix: 10 | allow_failures: 11 | - rust: stable 12 | - rust: beta 13 | 14 | addons: 15 | apt: 16 | packages: 17 | - libcurl4-openssl-dev 18 | - libelf-dev 19 | - libdw-dev 20 | 21 | cache: 22 | apt: true 23 | directories: 24 | - target/debug/deps 25 | 26 | script: 27 | - ./all_unit_tests.sh && ./integration_tests.sh 28 | 29 | after_success: | 30 | [ $TRAVIS_RUST_VERSION = nightly ] && 31 | wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && 32 | tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && make install DESTDIR=../tmp && cd ../.. && 33 | ls target/debug && 34 | ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern=/.cargo target/kcov target/debug/emcache-* 35 | ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/emcache-* --ignored 36 | 37 | notifications: 38 | email: true 39 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "0.5.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "bufstream" 16 | version = "0.1.4" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" 19 | 20 | [[package]] 21 | name = "cfg-if" 22 | version = "0.1.10" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" 25 | 26 | [[package]] 27 | name = "docopt" 28 | version = "0.6.86" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "4a7ef30445607f6fc8720f0a0a2c7442284b629cf0d049286860fae23e71c4d9" 31 | dependencies = [ 32 | "lazy_static", 33 | "regex", 34 | "rustc-serialize", 35 | "strsim", 36 | ] 37 | 38 | [[package]] 39 | name = "emcache" 40 | version = "0.1.0" 41 | dependencies = [ 42 | "bufstream", 43 | "docopt", 44 | "libc", 45 | "linked-hash-map", 46 | "maplit", 47 | "net2", 48 | "rand 0.3.23", 49 | "rustc-serialize", 50 | "time", 51 | ] 52 | 53 | [[package]] 54 | name = "fuchsia-cprng" 55 | version = "0.1.1" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 58 | 59 | [[package]] 60 | name = "kernel32-sys" 61 | version = "0.2.2" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" 64 | dependencies = [ 65 | "winapi 0.2.8", 66 | "winapi-build", 67 | ] 68 | 69 | [[package]] 70 | name = "lazy_static" 71 | version = "0.2.11" 72 | source = "registry+https://github.com/rust-lang/crates.io-index" 73 | checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" 74 | 75 | [[package]] 76 | name = "libc" 77 | version = "0.2.147" 78 | source = "registry+https://github.com/rust-lang/crates.io-index" 79 | checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" 80 | 81 | [[package]] 82 | name = "linked-hash-map" 83 | version = "0.0.9" 84 | source = "registry+https://github.com/rust-lang/crates.io-index" 85 | checksum = "83f7ff3baae999fdf921cccf54b61842bb3b26868d50d02dff48052ebec8dd79" 86 | 87 | [[package]] 88 | name = "maplit" 89 | version = "0.1.6" 90 | source = "registry+https://github.com/rust-lang/crates.io-index" 91 | checksum = "22593015b8df7747861c69c28acd32589fb96c1686369f3b661d12e409d4cf65" 92 | 93 | [[package]] 94 | name = "memchr" 95 | version = "0.1.11" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" 98 | dependencies = [ 99 | "libc", 100 | ] 101 | 102 | [[package]] 103 | name = "net2" 104 | version = "0.2.39" 105 | source = "registry+https://github.com/rust-lang/crates.io-index" 106 | checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" 107 | dependencies = [ 108 | "cfg-if", 109 | "libc", 110 | "winapi 0.3.9", 111 | ] 112 | 113 | [[package]] 114 | name = "rand" 115 | version = "0.3.23" 116 | source = "registry+https://github.com/rust-lang/crates.io-index" 117 | checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" 118 | dependencies = [ 119 | "libc", 120 | "rand 0.4.6", 121 | ] 122 | 123 | [[package]] 124 | name = "rand" 125 | version = "0.4.6" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 128 | dependencies = [ 129 | "fuchsia-cprng", 130 | "libc", 131 | "rand_core 0.3.1", 132 | "rdrand", 133 | "winapi 0.3.9", 134 | ] 135 | 136 | [[package]] 137 | name = "rand_core" 138 | version = "0.3.1" 139 | source = "registry+https://github.com/rust-lang/crates.io-index" 140 | checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 141 | dependencies = [ 142 | "rand_core 0.4.2", 143 | ] 144 | 145 | [[package]] 146 | name = "rand_core" 147 | version = "0.4.2" 148 | source = "registry+https://github.com/rust-lang/crates.io-index" 149 | checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 150 | 151 | [[package]] 152 | name = "rdrand" 153 | version = "0.4.0" 154 | source = "registry+https://github.com/rust-lang/crates.io-index" 155 | checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 156 | dependencies = [ 157 | "rand_core 0.3.1", 158 | ] 159 | 160 | [[package]] 161 | name = "regex" 162 | version = "0.1.80" 163 | source = "registry+https://github.com/rust-lang/crates.io-index" 164 | checksum = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" 165 | dependencies = [ 166 | "aho-corasick", 167 | "memchr", 168 | "regex-syntax", 169 | "thread_local", 170 | "utf8-ranges", 171 | ] 172 | 173 | [[package]] 174 | name = "regex-syntax" 175 | version = "0.3.9" 176 | source = "registry+https://github.com/rust-lang/crates.io-index" 177 | checksum = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" 178 | 179 | [[package]] 180 | name = "rustc-serialize" 181 | version = "0.3.24" 182 | source = "registry+https://github.com/rust-lang/crates.io-index" 183 | checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" 184 | 185 | [[package]] 186 | name = "strsim" 187 | version = "0.5.2" 188 | source = "registry+https://github.com/rust-lang/crates.io-index" 189 | checksum = "67f84c44fbb2f91db7fef94554e6b2ac05909c9c0b0bc23bb98d3a1aebfe7f7c" 190 | 191 | [[package]] 192 | name = "thread-id" 193 | version = "2.0.0" 194 | source = "registry+https://github.com/rust-lang/crates.io-index" 195 | checksum = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03" 196 | dependencies = [ 197 | "kernel32-sys", 198 | "libc", 199 | ] 200 | 201 | [[package]] 202 | name = "thread_local" 203 | version = "0.2.7" 204 | source = "registry+https://github.com/rust-lang/crates.io-index" 205 | checksum = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5" 206 | dependencies = [ 207 | "thread-id", 208 | ] 209 | 210 | [[package]] 211 | name = "time" 212 | version = "0.1.45" 213 | source = "registry+https://github.com/rust-lang/crates.io-index" 214 | checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" 215 | dependencies = [ 216 | "libc", 217 | "wasi", 218 | "winapi 0.3.9", 219 | ] 220 | 221 | [[package]] 222 | name = "utf8-ranges" 223 | version = "0.1.3" 224 | source = "registry+https://github.com/rust-lang/crates.io-index" 225 | checksum = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f" 226 | 227 | [[package]] 228 | name = "wasi" 229 | version = "0.10.0+wasi-snapshot-preview1" 230 | source = "registry+https://github.com/rust-lang/crates.io-index" 231 | checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" 232 | 233 | [[package]] 234 | name = "winapi" 235 | version = "0.2.8" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" 238 | 239 | [[package]] 240 | name = "winapi" 241 | version = "0.3.9" 242 | source = "registry+https://github.com/rust-lang/crates.io-index" 243 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 244 | dependencies = [ 245 | "winapi-i686-pc-windows-gnu", 246 | "winapi-x86_64-pc-windows-gnu", 247 | ] 248 | 249 | [[package]] 250 | name = "winapi-build" 251 | version = "0.1.1" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" 254 | 255 | [[package]] 256 | name = "winapi-i686-pc-windows-gnu" 257 | version = "0.4.0" 258 | source = "registry+https://github.com/rust-lang/crates.io-index" 259 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 260 | 261 | [[package]] 262 | name = "winapi-x86_64-pc-windows-gnu" 263 | version = "0.4.0" 264 | source = "registry+https://github.com/rust-lang/crates.io-index" 265 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 266 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "emcache" 3 | version = "0.1.0" 4 | authors = ["Martin Matusiak"] 5 | 6 | [dependencies] 7 | bufstream = "0.1.1" # for buffered tcp streams 8 | docopt = "0.6.78" # cmdline arguments 9 | libc = "0.2" 10 | linked-hash-map = "0.0.9" # hashmap that remembers order of insertion 11 | net2 = "0.2.20" # support for setting socket options 12 | rustc-serialize = "0.3.16" # needed for docopt 13 | time = "0.1" # timing primitives in unix time 14 | # Testing related 15 | maplit = "0.1.2" # hashmap literals 16 | rand = "0.3" # random number generator 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2016 Martin Matusiak 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # emcache 2 | 3 | [![Build Status](https://travis-ci.org/numerodix/emcache.svg?branch=master)](https://travis-ci.org/numerodix/emcache) 4 | [![Coverage Status](https://coveralls.io/repos/numerodix/emcache/badge.svg?branch=coverage)](https://coveralls.io/r/numerodix/emcache?branch=coverage) 5 | 6 | An implementation of memcached in Rust. 7 | 8 | 9 | ## Features and todo list 10 | 11 | * Implements the [memcached protocol](doc/Protocol-support.md). 12 | * Bounded cache with LRU behavior. 13 | * Concurrency model based on thread-per-connection. 14 | * [Modular architecture](doc/Architecture.md). Transport layer is separate from storage and is configured in a N:1 topology with communication using immutable Cmd/Resp values over async channels. 15 | * Fairly good test coverage. 16 | * No config file, logging or daemonization yet. 17 | * [Performance](pyemc/ABOUT.md) is generally within 0.5-1x of memcached. 18 | * Currently (Apr 2016) only builds against rust-nightly due to dependency on unstable "test" crate for benchmarking. 19 | 20 | 21 | ## Development 22 | 23 | To build: 24 | 25 | $ cargo build 26 | 27 | To run unit tests: 28 | 29 | $ ./all_unit_tests.sh 30 | 31 | To run the server: 32 | 33 | $ ./run_server.sh 34 | 35 | The Python integration test suite supplies various forms of tests. To run them 36 | against a running server: 37 | 38 | $ python -m pyemc.main -p $PORT # integration tests 39 | $ python -m pyemc.main -p $PORT --stress # burn in tests 40 | $ python -m pyemc.main -p $PORT --fill 45 # fill to 45% capacity 41 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | Low hanging fruit: 2 | 3 | * ... 4 | 5 | Medium: 6 | 7 | * Add a logging facility and start adding some basic log output. 8 | * Add fuzzing test to pyperf by supplying valid samples of command strings and randomly shuffling characters / shortening/elongating fields. 9 | 10 | Large: 11 | 12 | * Port process model from thread-per-connection w/channels to async with MetalIO. 13 | -------------------------------------------------------------------------------- /all_unit_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cargo test -- --ignored $@ 6 | ./fast_unit_tests.sh $@ 7 | -------------------------------------------------------------------------------- /bench_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cargo bench $@ 4 | -------------------------------------------------------------------------------- /cppperf/ABOUT.md: -------------------------------------------------------------------------------- 1 | # Performance testing in C++ 2 | 3 | The idea was to write a highly optimal client to measure transaction rates both 4 | against memcached and emcache. 5 | 6 | But during testing perftest runs at a higher cpu utilization than memcached, so 7 | perftest seems unable to achieve a high enough request rate to measure the peak 8 | rate for memcached. 9 | 10 | Furthermore, the transaction rate measured using pyemc is notably higher for 11 | the exact same parameters, so the expected performance benefits of C++ are 12 | called into question. 13 | 14 | 15 | ## Measurements 16 | 17 | Single threaded client, on the same machine. 18 | 19 | * key: 'x' 20 | * value: 'abc' 21 | 22 | memcached: 23 | 24 | * 10,000x get (constant key): 32k/s 25 | * 10,000x set (constant key): 19k/s 26 | 27 | emcache: 28 | 29 | * 10,000x get (constant key): 7.8k/s 30 | * 10,000x set (constant key): 7.6k/s 31 | -------------------------------------------------------------------------------- /cppperf/Makefile: -------------------------------------------------------------------------------- 1 | #CXX := g++ 2 | CXX := clang++-3.6 3 | CXX_FLAGS := -std=c++11 -Wall -pedantic 4 | 5 | all: perftest 6 | 7 | 8 | perftest: main.cpp memcache_client.o tcp_client.o 9 | $(CXX) $(CXX_FLAGS) main.cpp memcache_client.o tcp_client.o -o $@ 10 | 11 | memcache_client.o: memcache_client.h memcache_client.cpp tcp_client.o 12 | $(CXX) $(CXX_FLAGS) -c memcache_client.cpp -o $@ 13 | 14 | tcp_client.o: tcp_client.h tcp_client.cpp 15 | $(CXX) $(CXX_FLAGS) -c tcp_client.cpp -o $@ 16 | 17 | 18 | clean: 19 | -rm perftest *.o 20 | -------------------------------------------------------------------------------- /cppperf/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "memcache_client.h" 8 | #include "tcp_client.h" 9 | 10 | 11 | int main(int argc, char *argv[]) { 12 | MemcacheClient cli("127.0.0.1", 11311); 13 | cli.printStats(); 14 | 15 | uint32_t num_requests = 10000; 16 | std::vector val = {'a', 'b', 'c'}; 17 | 18 | auto begin = std::chrono::steady_clock::now(); 19 | for (int i=0; i < num_requests; i++) { 20 | assert( true == cli.set("x", val) ); 21 | std::vector val2 = cli.get("x"); 22 | assert( cli.equal(val, val2) ); 23 | } 24 | auto end = std::chrono::steady_clock::now(); 25 | auto dur_ms = std::chrono::duration_cast(end - begin).count(); 26 | double dur = (double) dur_ms / 1000; 27 | double rate = num_requests / dur; 28 | 29 | std::cout << "Made " << num_requests << " constant key set+get requests in " << dur << 30 | " seconds = " << rate << " requests/sec" << std::endl; 31 | 32 | return 0; 33 | } 34 | -------------------------------------------------------------------------------- /cppperf/memcache_client.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "memcache_client.h" 7 | 8 | 9 | MemcacheClient::MemcacheClient(std::string host, uint16_t port) : m_client(nullptr) { 10 | m_client = new TcpClient(host, port); 11 | } 12 | 13 | std::vector MemcacheClient::get(std::string key) { 14 | // Construct request 15 | std::stringstream srequest; 16 | 17 | srequest << "get " << key << "\r\n"; 18 | 19 | std::string request = srequest.str(); 20 | 21 | // Send request 22 | std::cout << "memcache: Loading key '" << key << "'" << std::endl; 23 | assert( request.length() == m_client->transmit(request.c_str(), request.length()) ); 24 | 25 | // Receive response 26 | std::vector response; 27 | char response_buf[4096] = {0}; // 4k is enough stats for everyone 28 | m_client->receive(response_buf, 4096); 29 | response.insert(response.end(), &response_buf[0], &response_buf[4096]); 30 | 31 | // Interpret response 32 | // format: VALUE x 0 3\r\nabc\r\n 33 | std::string prefix(&response_buf[0], &response_buf[512]); 34 | // see if it's a successful response 35 | if (prefix.substr(0, 5).compare("VALUE") == 0) { 36 | // find the space before the key 37 | std::size_t pos = prefix.find(" ", 5); 38 | 39 | // find the space before the flags 40 | pos = prefix.find(" ", pos + 1); 41 | 42 | // find the space before the bytecount 43 | pos = prefix.find(" ", pos + 1); 44 | uint32_t data_len = atoi(&prefix[pos]); 45 | 46 | // find the line ending 47 | pos = prefix.find("\r\n", pos + 1); 48 | 49 | // we know the indices of the data now 50 | std::vector data(&prefix[pos + 2], &prefix[pos + 2 + data_len]); 51 | 52 | std::string data_str(data.begin(), data.end()); 53 | std::cout << "memcache: Loaded key '" << key << "' with value <<<" 54 | << data_str << ">>>" << std::endl; 55 | 56 | return data; 57 | } 58 | 59 | std::cout << "memcache: Failed to load key '" << key << "'" << std::endl; 60 | std::vector empty; 61 | return empty; 62 | } 63 | 64 | bool MemcacheClient::set(std::string key, std::vector data) { 65 | // Construct request 66 | std::stringstream srequest; 67 | std::string data_str(data.begin(), data.end()); 68 | 69 | srequest << "set " << key << " 0 0 " << data.size() << " \r\n"; 70 | srequest << data_str << "\r\n"; 71 | 72 | std::string request = srequest.str(); 73 | 74 | // Send request 75 | std::cout << "memcache: Storing key '" << key << "' with value <<<" 76 | << data_str << ">>>" << std::endl; 77 | assert( request.length() == m_client->transmit(request.c_str(), request.length()) ); 78 | 79 | // Receive response 80 | char response_buf[101] = {0}; 81 | m_client->receive(response_buf, 100); 82 | 83 | // Interpret response 84 | std::string response(response_buf); 85 | if (response.compare("STORED\r\n") == 0) { 86 | std::cout << "memcache: Stored key '" << key << "'" << std::endl; 87 | return true; 88 | } 89 | 90 | std::cout << "memcache: Failed to store key '" << key << "'" << std::endl; 91 | return false; 92 | } 93 | 94 | void MemcacheClient::printStats() { 95 | // Construct request 96 | std::string cmd("stats\r\n"); 97 | 98 | // Send request 99 | std::cout << "memcache: Requesting stats" << std::endl; 100 | assert( cmd.length() == m_client->transmit(cmd.c_str(), cmd.length()) ); 101 | 102 | // Receive response 103 | char buf[4096] = {0}; // 4k is enough stats for everyone 104 | assert( 0 < m_client->receive(buf, 4095) ); 105 | std::cout << "memcache: Requested stats" << std::endl; 106 | 107 | // Interpret response 108 | std::string resp(buf); 109 | std::size_t pos = resp.find("\r\nEND"); // find the end marker 110 | std::cout << resp.substr(0, pos) << std::endl; 111 | } 112 | 113 | bool MemcacheClient::equal(std::vector data1, std::vector data2) { 114 | return std::equal(data1.begin(), data1.end(), data2.begin()); 115 | } 116 | -------------------------------------------------------------------------------- /cppperf/memcache_client.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "tcp_client.h" 7 | 8 | 9 | class MemcacheClient { 10 | public: 11 | MemcacheClient(std::string host, uint16_t port); 12 | std::vector get(std::string key); 13 | bool set(std::string key, std::vector data); 14 | void printStats(); 15 | 16 | bool equal(std::vector data1, std::vector data2); 17 | 18 | private: 19 | TcpClient *m_client; 20 | }; 21 | -------------------------------------------------------------------------------- /cppperf/tcp_client.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "tcp_client.h" 12 | 13 | 14 | TcpClient::TcpClient(std::string host, uint16_t port) 15 | : m_host(host), m_port(port), m_sockfd(-1) { 16 | } 17 | 18 | bool TcpClient::_connect() { 19 | // Already connected 20 | if (m_sockfd > -1) { 21 | return true; 22 | } 23 | 24 | // Create the socket first 25 | int32_t sockfd = socket(AF_INET, SOCK_STREAM, 0); 26 | if (sockfd == -1) { 27 | perror("Could not create socket"); 28 | return false; 29 | } 30 | 31 | // Do we need to resolve the hostname? 32 | struct sockaddr_in server_addr; 33 | if (inet_addr(m_host.c_str()) == INADDR_NONE) { 34 | struct hostent *he; 35 | struct in_addr **addr_list; 36 | 37 | if ((he = gethostbyname(m_host.c_str())) == NULL) { 38 | perror("gethostbyname() failed"); 39 | std::cout << "tcp: failed to resolve hostname " << m_host << "\n"; 40 | return false; 41 | } 42 | 43 | addr_list = (struct in_addr**) he->h_addr_list; 44 | for (int i=0; addr_list[i] != NULL; i++) { 45 | server_addr.sin_addr = *addr_list[i]; 46 | std::cout << "tcp: " << m_host << " resolved to " 47 | << inet_ntoa(*addr_list[i]) << std::endl; 48 | break; 49 | } 50 | 51 | // We have an ip address already 52 | } else { 53 | server_addr.sin_addr.s_addr = inet_addr(m_host.c_str()); 54 | } 55 | 56 | server_addr.sin_family = AF_INET; 57 | server_addr.sin_port = htons(m_port); 58 | 59 | // Establish connection 60 | if (connect(sockfd, (struct sockaddr*) &server_addr, sizeof(server_addr)) < 0) { 61 | perror("connect() failed"); 62 | return false; 63 | } 64 | 65 | // Set socket options 66 | int flags = 1; 67 | if (setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (void *)&flags, sizeof(flags)) != 0) { 68 | perror("setsockopt() failed"); 69 | } 70 | 71 | std::cout << "tcp: connected to " << m_host << ":" << m_port << "\n"; 72 | m_sockfd = sockfd; 73 | return true; 74 | } 75 | 76 | uint32_t TcpClient::transmit(const char* data, uint32_t len) { 77 | assert( this->_connect() == true ); 78 | 79 | ssize_t bytes_cnt = send(m_sockfd, data, strlen(data), 0); 80 | if (bytes_cnt < 0) { 81 | perror("send() failed"); 82 | } else { 83 | std::cout << "tcp: sent " << bytes_cnt << " bytes\n"; 84 | } 85 | 86 | return (uint32_t) bytes_cnt; 87 | } 88 | 89 | uint32_t TcpClient::receive(char* data, uint32_t len) { 90 | assert( this->_connect() == true ); 91 | 92 | ssize_t bytes_cnt = recv(m_sockfd, data, len, 0); 93 | if (bytes_cnt < 0) { 94 | perror("recv() failed"); 95 | } else { 96 | std::cout << "tcp: received " << bytes_cnt << " bytes\n"; 97 | } 98 | 99 | return (uint32_t) bytes_cnt; 100 | } 101 | -------------------------------------------------------------------------------- /cppperf/tcp_client.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | 7 | class TcpClient { 8 | public: 9 | TcpClient(std::string host, uint16_t port); 10 | bool _connect(); // to avoid clashing with imported symbol 'connect' 11 | uint32_t transmit(const char* data, uint32_t len); 12 | uint32_t receive(char *buf, uint32_t len); 13 | 14 | private: 15 | std::string m_host; 16 | uint16_t m_port; 17 | int32_t m_sockfd; 18 | }; 19 | -------------------------------------------------------------------------------- /doc/Architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | ---------- ---------- 4 | | Client | | Client | 5 | ---------- ---------- 6 | | | 7 | | | 8 | ------------- ------------- 9 | | Transport | | Transport | 10 | ------------- ------------- 11 | | | | 12 | ------------ | | | 13 | | Listener | | \----------\ /---------/ 14 | ------------ | | 15 | | ------------ 16 | | /-----| Protocol | 17 | ----------- | / ------------ 18 | | Metrics |-----------------------| Storage | 19 | ----------- ------------ 20 | 21 | 22 | 23 | ## Components 24 | 25 | * Storage: Bounded LRU map with single threaded access only, no locking. 26 | 27 | * Protocol: Implements the memcached protocol. Performs operations (Cmd) on the storage on behalf of clients and returns reponses (Resp). 28 | 29 | * Transport: Converts client requests (in bytes) into Cmd objects and transmits these to the Protocol. Receives Resp objects from the Protocol and writes them out to clients as responses (in bytes). 30 | 31 | * Listener: Manages the listening socket and spawns a Transport for each new client. When a client goes away the Transport simply dies (no cleanup is necessary). 32 | 33 | * Metrics: Collects server metrics from any other component and aggregates them/displays them. 34 | 35 | The Storage and Protocol run in the same thread. All other components run in 36 | separate threads. All communication between threads is done over async channels 37 | (ownership of the sent object is transfered from the sender to the receiver). 38 | 39 | 40 | ## Concepts 41 | 42 | * Stats: Part of the memcached protocol. Collected in the Storage, Protocol and Transport. For stats originating in the Transport (eg. bytes sent, bytes received) they are transmitted to (and aggregated at) the Protocol. (Keep in mind that Transports are concurrent, so these stats are always just snapshots and never fully accurate.) 43 | 44 | * Metrics: Internal server performance metrics. Any component may collect these and transmit them to the Metrics collector over a channel. 45 | -------------------------------------------------------------------------------- /doc/Performance-testing-ideas.md: -------------------------------------------------------------------------------- 1 | # Regression analysis 2 | 3 | Probable factors that influence performance: 4 | 5 | * cache use % 6 | * size of cache 7 | * size of items being written/read 8 | * number of clients participating in test 9 | 10 | Performance metrics: 11 | 12 | * operation latency 13 | * transactions/s 14 | * bytes/s 15 | 16 | Stats to track: avg, p90, p99, p99.9 17 | -------------------------------------------------------------------------------- /doc/Protocol-support.md: -------------------------------------------------------------------------------- 1 | # Protocol support 2 | 3 | We follow [the official protocol specification](https://github.com/memcached/memcached/blob/master/doc/protocol.txt). 4 | 5 | 6 | ## Fully supported 7 | 8 | * ADD 9 | * APPEND 10 | * CAS 11 | * DECR 12 | * DELETE 13 | * GET 14 | * GETS 15 | * INCR 16 | * QUIT 17 | * PREPEND 18 | * REPLACE 19 | * SET 20 | * TOUCH 21 | * VERSION 22 | 23 | 24 | ## Partial support 25 | 26 | * FLUSH_ALL (without options) 27 | * STATS (not all stats are present) 28 | 29 | 30 | ## No plan to support 31 | 32 | These commands are too implementation specific to memcached itself for it to 33 | make sense to support them. 34 | 35 | * SLABS 36 | * STATS [arg] 37 | * VERBOSITY 38 | -------------------------------------------------------------------------------- /doc/Test-plan.md: -------------------------------------------------------------------------------- 1 | # Test plan 2 | 3 | 4 | ## Correctness 5 | 6 | The primary vehicle for correctness testing is **unit tests**. Almost every 7 | component in emcache is unit tested, including the storage, protocol and 8 | transport. 9 | 10 | The only exception is tasks, which occupy a thread and directly manipulate a 11 | socket or channel, which makes them impractical to unit test. Evenso, they are 12 | merely sequences of steps performed on underlying components which are 13 | themselves unit tested. 14 | 15 | As a practical matter, we try to avoid time based unit tests when possible and 16 | mark tests that require sleeping as `ignore` so they are excluded from the 17 | default unit test run. All unit tests must run in Travis CI. 18 | 19 | We also use **integration tests** to verify that all the trusted pieces have been 20 | wired up correctly. The integration tests must cover the entire protocol. When 21 | extending integration tests the suite must also be run against memcached to 22 | verify that the tests themselves work correctly. 23 | 24 | 25 | ## Performance 26 | 27 | Performance testing of memcached, which is essentially an in-memory hashtable 28 | exposed on a socket, is a tricky dance, given just how performant the 29 | storage layer is. 30 | 31 | **Rust level mirco benchmarks** show that setting or getting a key runs on the 32 | order of 200 nanoseconds. So theoretically we can store 5M keys per second. 33 | Even tacking on a protocol layer only takes it up to about 300-400ns. The 34 | transport layer is much slower, showing about 1000ns for reading 35 | commands/writing responses. So that gives a total upper bound of 1400ns per 36 | operation. 37 | 38 | And yet **tcp based stress testing** over 127.0.0.1 shows a much lower rate of 39 | 150k/s for key insertion in `noreply` mode where the client just spits out set 40 | commands without receiving an acknowledgment, and an even lower rate of 15k 41 | without `noreply` when each request is followed by a response before making the 42 | next one. 43 | 44 | So clearly the bottleneck is TCP, and the game is won or lost by making the 45 | transport layer (in emcache terms) as efficient as possible. This is fair game 46 | for the server, but it is unfortunate for the client generating this traffic, 47 | which inescapably too is saddled with this responsibility. It bears keeping 48 | in mind, then, that a performance test can only go as high as the client 49 | itself can go, and at the end of the day we are merely writing items into a 50 | hashmap. 51 | 52 | ...todo... 53 | 54 | *NOTE:* All the numbers cited in this section are relative to one single 55 | machine, so numbers will be different between machines (obviously). 56 | -------------------------------------------------------------------------------- /fast_unit_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cargo test $@ 4 | -------------------------------------------------------------------------------- /format_all_code.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | find -iname '*.rs' -exec rustfmt {} \; 4 | find -iname '*.rs.bk' -delete 5 | -------------------------------------------------------------------------------- /integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | # Options we'll use 6 | PORT=11311 7 | 8 | # Build the server 9 | cargo build 10 | 11 | # Launch the server, give it 5sec to start up 12 | cargo run -- --port $PORT & 13 | sleep 5 14 | 15 | # Run the fill test 16 | echo -e "\n=== FILL TEST ===\n" 17 | python -m pyemc.main -p $PORT --fill 5.0 18 | 19 | # Run the stress test 20 | echo -e "\n=== STRESS TEST ===\n" 21 | python -m pyemc.main -p $PORT --stress 22 | 23 | # Run the integ tests 24 | echo -e "\n=== INTEG TEST ===\n" 25 | python -m pyemc.main -p $PORT 26 | exit_code=$? 27 | 28 | # Kill the server 29 | ps axf | grep 'target\/.*\/emcache' | awk '{print $1}' | xargs kill || true 30 | 31 | exit $exit_code 32 | -------------------------------------------------------------------------------- /pyemc/ABOUT.md: -------------------------------------------------------------------------------- 1 | # Performance testing in Python 2 | 3 | This test suite includes a Task abstraction which allows defining a task as a 4 | pair of Task (main job) + Tasklet (client run in seperate threads) to enable 5 | parallelism. 6 | 7 | 8 | ## Measurements 9 | 10 | 11 | ### Set/get a constant key 12 | 13 | * against commit 67c5cda (Jan 5, 2016) 14 | * single threaded client 15 | * connecting to 127.0.0.1 16 | * key: 'x' 17 | * value: 'abc' 18 | 19 | | Test | memcached/cpython | emcache/cpython | memcached/pypy | emcache/pypy | 20 | |------------------------|------------------:|----------------:|---------------:|-------------:| 21 | | 100,000x get | **26k/s** | 18k/s | **40k/s** | 20k/s | 22 | | 100,000x set | **27k/s** | 16k/s | **44k/s** | 31k/s | 23 | | 700,000x set w/noreply | 87k/s | **97k/s** | **1225k/s** | 87k/s | 24 | | 100,000x version | **37k/s** | 21k/s | **54k/s** | 33k/s | 25 | 26 | The pypy version was run 10x longer to account for jit warmup, but the rate is 27 | still averaged over the whole run. 28 | 29 | 30 | ### Fill cache to a certain percentage 31 | 32 | Test parameters: 33 | 34 | * Against commit d81b62a (Jan 6, 2016) 35 | * Using 4 client threads 36 | * Connecting to 127.0.0.1 37 | * Cache size: 512mb 38 | * Rate is averaged over the whole run (measuring items/s and bytes/s) 39 | 40 | Fill mode: 41 | 42 | * key: 10 printable chars, random 43 | * value: 100-1000 bytes, random 44 | * set w/noreply and pipelined in batches of 100 45 | 46 | | Task | memcached/cpython | emcache/cpython | memcached/pypy | emcache/pypy | 47 | |-------------------|----------------------:|------------------:|-----------------:|---------------------:| 48 | | Fill cache to 80% | **567k/s - 317mb/s** | 503k/s - 281mb/s | 594k/s - 332mb/s | **614k/s - 343mb/s** | 49 | 50 | 51 | ### Memory efficiency - per commit 8741b0b (Jan 6, 2016) 52 | 53 | Ideally, emcache would use only as much memory as the user actually stores. In 54 | practice, there is a certain overhead because we also need various 55 | datastructures, and each transport needs a few buffers to operate. 56 | 57 | If we look at storage alone, there is a certain overhead to storing keys (storing the vector is 24 bytes on 64bit) and values (vector + flags + exptime + atime + cas_unique). 58 | 59 | * A key is currently 24 bytes + the key data. 60 | * A value is currently 56 bytes + the value data. 61 | 62 | For really small keys (ints), the overhead completely dominates the user data. 63 | 64 | For the fill test we see these numbers: 65 | 66 | * Cache capacity is 1024mb 67 | * Cache utilization is 80% 68 | 69 | | User stored size | emcache bytes stat | process residental size | 70 | |-----------------:|-------------------:|------------------------:| 71 | | 716mb | 819mb (87% eff.) | 993mb (72% eff.) | 72 | -------------------------------------------------------------------------------- /pyemc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numerodix/emcache/e681301c623af8cbf127801e7b1400b571b3a67c/pyemc/__init__.py -------------------------------------------------------------------------------- /pyemc/abstractions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numerodix/emcache/e681301c623af8cbf127801e7b1400b571b3a67c/pyemc/abstractions/__init__.py -------------------------------------------------------------------------------- /pyemc/abstractions/task_api.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import threading 3 | import sys 4 | 5 | 6 | def write(msg): 7 | msg = msg + '\n' if not msg.endswith('\n') else msg 8 | sys.stderr.write(msg) 9 | sys.stderr.flush() 10 | 11 | 12 | class TaskletMetrics(object): 13 | '''This class allows a Tasklet to store any state that a Task may need to 14 | have after the execution of its Tasklets.''' 15 | 16 | def __init__(self): 17 | pass 18 | 19 | 20 | class Tasklet(object): 21 | '''A Tasklet represents a unit of work run in a separate thread.''' 22 | 23 | def __init__(self, id, client_params): 24 | self.id = id 25 | self.client_params = client_params 26 | 27 | self._metrics = TaskletMetrics() 28 | self._runnable = True 29 | 30 | def launch(self): 31 | client = self.client_params.create_client() 32 | self.run(client, self._metrics) 33 | 34 | def run(self, client): 35 | raise NotImplementedError 36 | 37 | def write(self, msg): 38 | msg = '[thread%s] %s' % (self.id, msg) 39 | write(msg) 40 | 41 | 42 | class TaskState(object): 43 | '''This class represents any state that a task needs to keep between task 44 | phases.''' 45 | 46 | def __init__(self): 47 | pass 48 | 49 | 50 | class Task(object): 51 | '''Represents a task run against the server, which internally delegates its 52 | work to Tasklets run in threads. 53 | 54 | Users need to implement a Task to do any setup/teardown of the task and 55 | define the creation of Tasklets.''' 56 | 57 | def __init__(self, client_params): 58 | self.client_params = client_params 59 | 60 | def create_tasklets(self): 61 | raise NotImplementedError 62 | 63 | def pre_tasklets(self): 64 | raise NotImplementedError 65 | 66 | def run_tasklets(self, tasklets): 67 | metrics_list = [] 68 | threads = [] 69 | 70 | def handle_signal(signal, frame): 71 | write("Got interrupt, stopping all tasklets") 72 | for tasklet in tasklets: 73 | tasklet._runnable = False 74 | 75 | signal.signal(signal.SIGINT, handle_signal) 76 | 77 | for tasklet in tasklets: 78 | thread = threading.Thread(target=tasklet.launch) 79 | thread.daemon = True 80 | metrics_list.append(tasklet._metrics) 81 | threads.append(thread) 82 | thread.start() 83 | 84 | while threads: 85 | for thread in threads: 86 | thread.join(.05) 87 | if not thread.isAlive(): 88 | threads.remove(thread) 89 | 90 | return metrics_list 91 | 92 | def post_tasklets(self): 93 | raise NotImplementedError 94 | 95 | def launch(self): 96 | client = self.client_params.create_client() 97 | state = TaskState() 98 | 99 | tasklets = self.create_tasklets(state) 100 | self.pre_tasklets(client, state) 101 | metrics_list = self.run_tasklets(tasklets) 102 | self.post_tasklets(client, state, metrics_list) 103 | 104 | def write(self, msg): 105 | write(msg) 106 | -------------------------------------------------------------------------------- /pyemc/abstractions/test_api.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | import re 3 | import time 4 | import traceback 5 | import sys 6 | 7 | 8 | def write(msg): 9 | msg = msg + '\n' if not msg.endswith('\n') else msg 10 | sys.stdout.write(msg) 11 | sys.stdout.flush() 12 | 13 | 14 | class TestFailedError(Exception): 15 | pass 16 | 17 | 18 | class TestRunner(object): 19 | def __init__(self, client_params, args): 20 | self.client_params = client_params 21 | 22 | self.test_filter = None 23 | if args: 24 | self.test_filter = re.compile(args[0]) 25 | 26 | def execute_all(self, test_cases_classes): 27 | test_id = -1 28 | 29 | failed = [] 30 | passed = [] 31 | 32 | for test_case_cls in test_cases_classes: 33 | atts = dir(test_case_cls) 34 | atts = [att for att in atts if att.startswith('test_')] 35 | 36 | for att in atts: 37 | unbound_method = getattr(test_case_cls, att) 38 | if not callable(unbound_method): 39 | continue 40 | 41 | # Does not match the filter -> not selected to run 42 | if self.test_filter and not self.test_filter.search(att): 43 | continue 44 | 45 | test_id += 1 46 | test_case = test_case_cls(id=test_id) 47 | 48 | bound_method = getattr(test_case, att) 49 | rv = self.execute_one(bound_method) 50 | 51 | if rv: 52 | passed.append(att) 53 | else: 54 | failed.append(att) 55 | 56 | self.write("%s test run: %s passed, %s failed" % 57 | (len(failed + passed), len(passed), len(failed))) 58 | 59 | if failed: 60 | return False 61 | 62 | return True 63 | 64 | def execute_one(self, method): 65 | test_case = method.im_self 66 | test_name = '%s.%s' % (test_case.__class__.__name__, method.__name__) 67 | exc = None 68 | 69 | self.write("%s..." % test_name) 70 | 71 | try: 72 | test_case.set_up(self.client_params) 73 | except Exception as e: 74 | traceback.print_exc() 75 | self.write("SETUP FAILED: %r" % e) 76 | return 77 | 78 | time_start = time.time() 79 | try: 80 | method() 81 | except Exception as e: 82 | exc = e 83 | traceback.print_exc() 84 | 85 | time_stop = time.time() 86 | dur = time_stop - time_start 87 | 88 | if exc is None: 89 | self.write("SUCCEEDED in %.4fs" % (dur)) 90 | return True 91 | else: 92 | self.write("FAILED in %.4fs: %r" % (dur, exc)) 93 | return False 94 | 95 | def write(self, msg): 96 | msg = '[runner] %s' % msg 97 | write(msg) 98 | 99 | 100 | class TestCase(object): 101 | def __init__(self, id): 102 | self.id = id 103 | 104 | def set_up(self, client_params): 105 | '''Subclasses should implement both set_up and get_client so the runner 106 | can get access to the client.''' 107 | self.client = client_params.create_client() 108 | 109 | def get_client(self): 110 | return self.client 111 | 112 | @contextmanager 113 | def assert_raises(self, exc_class): 114 | raised = False 115 | try: 116 | yield 117 | 118 | except exc_class: 119 | raised = True 120 | 121 | if not raised: 122 | raise AssertionError("%s not raised" % exc_class.__name__) 123 | 124 | def write(self, msg): 125 | msg = '[test%s] %s' % (self.id, msg) 126 | write(msg) 127 | -------------------------------------------------------------------------------- /pyemc/client.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import re 3 | 4 | from pyemc.socket_stream import BufferedSocketStream 5 | 6 | 7 | class MemcacheClientParams(object): 8 | def __init__(self, host, port, pipeline_mode=False): 9 | self.host = host 10 | self.port = port 11 | self.pipeline_mode = pipeline_mode 12 | 13 | def create_client(self): 14 | return MemcacheClient( 15 | host=self.host, 16 | port=self.port, 17 | pipeline_mode=self.pipeline_mode, 18 | ) 19 | 20 | 21 | class ClientError(Exception): 22 | pass 23 | 24 | class DeleteFailedError(Exception): 25 | pass 26 | 27 | class ExistsError(Exception): 28 | pass 29 | 30 | class NotFoundError(Exception): 31 | pass 32 | 33 | class NotStoredError(Exception): 34 | pass 35 | 36 | class ServerError(Exception): 37 | pass 38 | 39 | 40 | _exc_map = { 41 | 'ERROR': ServerError, 42 | 'EXISTS': ExistsError, 43 | 'NOT_FOUND': NotFoundError, 44 | 'NOT_STORED': NotStoredError, 45 | } 46 | def create_exc(server_resp, exc_msg): 47 | if server_resp.startswith('CLIENT_ERROR'): 48 | msg = server_resp.split(' ', 1)[-1] 49 | return ClientError(msg) 50 | 51 | elif server_resp.startswith('SERVER_ERROR'): 52 | msg = server_resp.split(' ', 1)[-1] 53 | return ServerError(msg) 54 | 55 | exc_cls = _exc_map.get(server_resp.strip()) 56 | 57 | if exc_cls is None: 58 | raise Exception("Could not create exception for: %s" % server_resp) 59 | 60 | return exc_cls(exc_msg) 61 | 62 | 63 | class Item(object): 64 | def __init__(self, key, flags, value, cas_unique=None): 65 | self.key = key 66 | self.flags = flags 67 | self.value = value 68 | self.cas_unique = cas_unique 69 | 70 | def __repr__(self): 71 | return '<%s key=%r, flags=%r, cas_unique=%r, value=%r>' % ( 72 | self.__class__.__name__, 73 | self.key, 74 | self.flags, 75 | self.cas_unique, 76 | self.value, 77 | ) 78 | 79 | 80 | class MemcacheClient(object): 81 | rx_get_value = re.compile('VALUE (?P[^ ]*) (?P\d+) (?P\d+)') 82 | rx_gets_value = re.compile('VALUE (?P[^ ]*) (?P\d+) (?P\d+) (?P\d+)') 83 | rx_inc_value = re.compile('(?P\d+)') 84 | 85 | def __init__(self, host, port, pipeline_mode=False): 86 | '''NOTE: pipeline_mode mode will queue up all requests that have noreply 87 | set *until* flush_pipeline is called. If you're interleaving those with 88 | get() etc they will not be sent in the order you expect!''' 89 | 90 | self.stream = BufferedSocketStream(host, port) 91 | self.pipeline_mode = pipeline_mode 92 | 93 | def add(self, key, value, flags=0, exptime=0, noreply=False): 94 | return self._set_family( 95 | 'add', 96 | key=key, 97 | value=value, 98 | flags=flags, 99 | exptime=exptime, 100 | noreply=noreply, 101 | ) 102 | 103 | def append(self, key, value, flags=0, exptime=0, noreply=False): 104 | return self._set_family( 105 | 'append', 106 | key=key, 107 | value=value, 108 | flags=flags, 109 | exptime=exptime, 110 | noreply=noreply, 111 | ) 112 | 113 | def cas(self, key, value, flags=0, exptime=0, cas_unique=None, noreply=False): 114 | return self._set_family( 115 | 'cas', 116 | key=key, 117 | value=value, 118 | flags=flags, 119 | exptime=exptime, 120 | cas_unique=cas_unique, 121 | noreply=noreply, 122 | ) 123 | 124 | def decr(self, key, delta=1, noreply=False): 125 | return self._inc_family('decr', key, delta, noreply) 126 | 127 | def delete(self, key, noreply=False): 128 | # prepare command 129 | command = 'delete %(key)s %(noreply)s\r\n' % { 130 | 'key': key, 131 | 'noreply': 'noreply' if noreply else '', 132 | } 133 | 134 | # execute command 135 | self.maybe_write_now(command, noreply=noreply) 136 | 137 | # parse the response 138 | if not noreply: 139 | resp = self.stream.read_line() 140 | if not resp == 'DELETED\r\n': 141 | raise create_exc(resp, 'Could not delete key %r' % key) 142 | 143 | def flush_all(self, exptime=None, noreply=False): 144 | # prepare command 145 | # TODO no support for parameters yet 146 | command = 'flush_all\r\n' 147 | 148 | # execute command 149 | self.maybe_write_now(command, noreply=noreply) 150 | 151 | # parse the response 152 | if not noreply: 153 | resp = self.stream.read_line() 154 | if not resp == 'OK\r\n': 155 | raise create_exc(resp, 'Could not perform flush_all') 156 | 157 | def get_multi(self, keys): 158 | return self._get_multi_family('get', keys) 159 | 160 | def gets_multi(self, keys): 161 | return self._get_multi_family('gets', keys) 162 | 163 | def _get_multi_family(self, instr, keys): 164 | # prepare command 165 | keys = ' '.join(keys) 166 | command = '%(instr)s %(keys)s\r\n' % { 167 | 'instr': instr, 168 | 'keys': keys, 169 | } 170 | 171 | # execute command 172 | self.stream.write(command) 173 | 174 | # parse the response 175 | dct = OrderedDict() 176 | stream_terminator = 'END\r\n' 177 | 178 | while True: 179 | line = self.stream.read_line() 180 | try: 181 | cas_unique = None 182 | 183 | if instr == 'get': 184 | key, flags, bytelen = self.rx_get_value.findall(line)[0] 185 | elif instr == 'gets': 186 | key, flags, bytelen, cas_unique = self.rx_gets_value.findall(line)[0] 187 | 188 | flags = int(flags) 189 | bytelen = int(bytelen) 190 | except IndexError: 191 | # no items were returned at all 192 | if line == stream_terminator: 193 | break 194 | 195 | # read value + line terminator 196 | data = self.stream.read_exact(bytelen + 2) 197 | 198 | data = data[:-2] 199 | item = Item(key, flags, data, cas_unique=cas_unique) 200 | dct[key] = item 201 | 202 | if self.stream.peek_contains(stream_terminator, consume=True): 203 | break 204 | 205 | return dct 206 | 207 | def get(self, key): 208 | keys = (key,) 209 | dct = self.get_multi(keys) 210 | 211 | try: 212 | return dct[key] 213 | except KeyError: 214 | raise NotFoundError('The item with key %r was not found' % key) 215 | 216 | def gets(self, key): 217 | keys = (key,) 218 | dct = self.gets_multi(keys) 219 | 220 | try: 221 | return dct[key] 222 | except KeyError: 223 | raise NotFoundError('The item with key %r was not found' % key) 224 | 225 | def get_stats(self): 226 | dct = OrderedDict() 227 | 228 | # prepare command 229 | command = 'stats\r\n' 230 | 231 | # execute command 232 | self.stream.write(command) 233 | 234 | # read response line by line 235 | stream_terminator = 'END\r\n' 236 | 237 | line = self.stream.read_line() 238 | while line != stream_terminator: 239 | kw, key, value = line.split(' ', 2) 240 | dct[key] = value.strip() 241 | 242 | line = self.stream.read_line() 243 | 244 | return dct 245 | 246 | def incr(self, key, delta=1, noreply=False): 247 | return self._inc_family('incr', key, delta, noreply) 248 | 249 | def _inc_family(self, instr, key, delta=1, noreply=False): 250 | # prepare command 251 | command = '%(instr)s %(key)s %(delta)d %(noreply)s\r\n' % { 252 | 'instr': instr, 253 | 'key': key, 254 | 'delta': int(delta), 255 | 'noreply': 'noreply' if noreply else '', 256 | } 257 | 258 | # execute command 259 | self.maybe_write_now(command, noreply=noreply) 260 | 261 | # read the response 262 | if not noreply: 263 | resp = self.stream.read_line() 264 | try: 265 | num = self.rx_inc_value.findall(resp)[0] 266 | return num 267 | except IndexError: 268 | raise create_exc(resp, 'Could not %s key %r' % (instr, key)) 269 | 270 | def quit(self): 271 | '''Tells the server to drop the connection.''' 272 | 273 | # prepare command 274 | command = 'quit\r\n' 275 | 276 | # execute command 277 | self.stream.write(command) 278 | 279 | def prepend(self, key, value, flags=0, exptime=0, noreply=False): 280 | return self._set_family( 281 | 'prepend', 282 | key=key, 283 | value=value, 284 | flags=flags, 285 | exptime=exptime, 286 | noreply=noreply, 287 | ) 288 | 289 | def replace(self, key, value, flags=0, exptime=0, noreply=False): 290 | return self._set_family( 291 | 'replace', 292 | key=key, 293 | value=value, 294 | flags=flags, 295 | exptime=exptime, 296 | noreply=noreply, 297 | ) 298 | 299 | def set(self, key, value, flags=0, exptime=0, noreply=False): 300 | return self._set_family( 301 | 'set', 302 | key=key, 303 | value=value, 304 | flags=flags, 305 | exptime=exptime, 306 | noreply=noreply, 307 | ) 308 | 309 | def _set_family(self, instr, key, value, flags=0, exptime=0, cas_unique=None, noreply=False): 310 | # prepare command 311 | header = '%(instr)s %(key)s %(flags)d %(exptime)d %(bytelen)d %(cas)s%(noreply)s\r\n' % { 312 | 'instr': instr, 313 | 'key': key, 314 | 'flags': flags, 315 | 'exptime': exptime, 316 | 'bytelen': len(value), 317 | 'cas': '%s ' % cas_unique if cas_unique else '', 318 | 'noreply': 'noreply' if noreply else '', 319 | } 320 | command = header + value + '\r\n' 321 | 322 | # execute command 323 | self.maybe_write_now(command, noreply=noreply) 324 | 325 | # check for success 326 | if not noreply: 327 | resp = self.stream.read_line() 328 | if not resp == 'STORED\r\n': 329 | raise create_exc(resp, 'Could not %s key %r to %r...' % 330 | (instr, key, value[:10])) 331 | 332 | def send_malformed_cmd(self): 333 | '''Sends an invalid command - causes the server to drop the 334 | connection''' 335 | 336 | self.stream.write('set 0 1\r\n') 337 | buf = self.stream.read(4096) 338 | return buf.strip() 339 | 340 | def touch(self, key, exptime=0, noreply=False): 341 | # prepare command 342 | command = 'touch %(key)s %(exptime)d %(noreply)s\r\n' % { 343 | 'key': key, 344 | 'exptime': exptime, 345 | 'noreply': 'noreply' if noreply else '', 346 | } 347 | 348 | # execute command 349 | self.maybe_write_now(command, noreply=noreply) 350 | 351 | # check for success 352 | if not noreply: 353 | resp = self.stream.read_line() 354 | if not resp == 'TOUCHED\r\n': 355 | raise create_exc(resp, 'Could not touch key %r' % key) 356 | 357 | def version(self): 358 | # prepare command 359 | command = 'version\r\n' 360 | 361 | # execute command 362 | self.stream.write(command) 363 | 364 | # check for success 365 | resp = self.stream.read_line() 366 | kw, version = resp.split(' ', 1) 367 | return version.strip() 368 | 369 | def maybe_write_now(self, command, noreply=False): 370 | if noreply and self.pipeline_mode: 371 | self.stream.write_pipelined(command) 372 | else: 373 | self.stream.write(command) 374 | 375 | def flush_pipeline(self): 376 | self.stream.flush_pipeline() 377 | -------------------------------------------------------------------------------- /pyemc/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import re 5 | import socket 6 | import sys 7 | import time 8 | 9 | from pyemc.abstractions.test_api import TestRunner 10 | from pyemc.client import MemcacheClient 11 | from pyemc.client import MemcacheClientParams 12 | from pyemc.task_filler import CacheFillerTask 13 | from pyemc.test_integration import TestApi 14 | from pyemc.test_stress import TestStress 15 | 16 | 17 | if __name__ == '__main__': 18 | from optparse import OptionParser 19 | parser = OptionParser(usage='%s [options] [test_filter]' % os.path.basename(sys.argv[0])) 20 | parser.add_option('-n', '', action='store', dest='host', 21 | help='Host to connect to') 22 | parser.add_option('-p', '', action='store', type='int', dest='port', 23 | help='Port to connect to') 24 | parser.add_option('', '--fill', action='store', type='float', dest='fill_cache', 25 | help='Fill the cache to a given percentage full') 26 | parser.add_option('', '--stress', action='store_true', dest='stress_test', 27 | help='Perform a stress test') 28 | parser.add_option('-w', '--workers', action='store', type='int', dest='worker_count', 29 | help='Use these many worker threads') 30 | (options, args) = parser.parse_args() 31 | 32 | 33 | host = options.host is not None and options.host or '127.0.0.1' 34 | port = options.port is not None and int(options.port) or 11311 35 | 36 | 37 | cli_params = MemcacheClientParams(host, port) 38 | 39 | if options.fill_cache: 40 | pct = float(options.fill_cache) 41 | filler = CacheFillerTask( 42 | client_params=cli_params, 43 | percentage=pct, 44 | jobs=options.worker_count, 45 | ) 46 | filler.launch() 47 | 48 | else: 49 | if options.stress_test: 50 | test_cases = [ 51 | TestStress, 52 | ] 53 | else: 54 | test_cases = [ 55 | TestApi, 56 | ] 57 | 58 | runner = TestRunner(cli_params, args) 59 | 60 | rv = runner.execute_all(test_cases) 61 | sys.exit(not rv) 62 | -------------------------------------------------------------------------------- /pyemc/socket_stream.py: -------------------------------------------------------------------------------- 1 | import socket 2 | try: 3 | import cStringIO as StringIO 4 | except ImportError: 5 | import StringIO 6 | 7 | 8 | def connected(func): 9 | def new_func(self, *args, **kwargs): 10 | if self.sock is None: 11 | self.connect() 12 | return func(self, *args, **kwargs) 13 | return new_func 14 | 15 | 16 | class BufferedSocketStream(object): 17 | def __init__(self, host, port, line_terminator='\r\n'): 18 | self.host = host 19 | self.port = port 20 | self.line_terminator = line_terminator 21 | 22 | self.sock = None 23 | self.std_read_size = 4096 24 | self.read_ahead = '' 25 | self.request_pipeline = StringIO.StringIO() 26 | 27 | def connect(self): 28 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 29 | self.sock.connect((self.host, self.port)) 30 | self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 31 | 32 | @connected 33 | def read(self, count): 34 | '''Attempts to satisfy the read from the read ahead buffer. Failing 35 | that it will recv() once from the socket and return whatever was 36 | there.''' 37 | 38 | if self.read_ahead: 39 | buf = self.read_ahead 40 | self.read_ahead = '' 41 | 42 | if len(buf) < count: 43 | left_to_read = count - len(buf) 44 | chunk = self.sock.recv(left_to_read) 45 | buf += chunk 46 | 47 | return buf 48 | 49 | buf = self.sock.recv(count) 50 | return buf 51 | 52 | @connected 53 | def read_exact(self, count): 54 | '''Reads the exact number of bytes from the socket, in as many recv() 55 | calls as necessary.''' 56 | 57 | buf = self.read_ahead 58 | 59 | while len(buf) < count: 60 | left_to_read = count - len(buf) 61 | chunk = self.sock.recv(left_to_read) 62 | buf += chunk 63 | 64 | if len(buf) >= count: 65 | self.read_ahead = buf[count:] 66 | buf = buf[:count] 67 | 68 | return buf 69 | 70 | @connected 71 | def read_line(self): 72 | '''Reads one line from the socket, in as many recv() calls as 73 | necessary. Stores any data left over in the read ahead buffer.''' 74 | 75 | line = self.read_ahead 76 | 77 | while not self.line_terminator in line: 78 | chunk = self.sock.recv(self.std_read_size) 79 | line += chunk 80 | 81 | idx = line.index(self.line_terminator) 82 | if idx + 2 <= len(line): 83 | self.read_ahead = line[idx + 2:] 84 | line = line[:idx + 2] 85 | 86 | return line 87 | 88 | def peek_contains(self, token, consume=False): 89 | '''Peeks at the stream for an expected value without consuming the 90 | stream. Tries to satisfy the peek from the read ahead buffer. Failing 91 | that, it calls read_exact() to read the number of bytes to match the 92 | length of the token.''' 93 | 94 | if len(self.read_ahead) < len(token): 95 | left_to_read = len(token) - len(self.read_ahead) 96 | self.read_ahead += self.read_exact(left_to_read) 97 | 98 | if self.read_ahead[:len(token)] == token: 99 | if consume: 100 | self.read_ahead = self.read_ahead[len(token):] 101 | 102 | return True 103 | 104 | return False 105 | 106 | @connected 107 | def write(self, buf): 108 | '''Writes the complete buffer to the socket.''' 109 | 110 | self.sock.sendall(buf) 111 | 112 | def write_pipelined(self, buf): 113 | self.request_pipeline.write(buf) 114 | 115 | @connected 116 | def flush_pipeline(self): 117 | self.sock.sendall(self.request_pipeline.getvalue()) 118 | self.request_pipeline.truncate(0) 119 | -------------------------------------------------------------------------------- /pyemc/task_filler.py: -------------------------------------------------------------------------------- 1 | from itertools import izip 2 | import time 3 | 4 | from pyemc.abstractions.task_api import Task 5 | from pyemc.abstractions.task_api import Tasklet 6 | from pyemc.util import generate_random_data 7 | from pyemc.util import generate_random_data_prng 8 | from pyemc.util import generate_random_key 9 | from pyemc.util import generate_random_key_uuid 10 | from pyemc.util import insert_number_commas 11 | 12 | 13 | class CacheFillerTask(Task): 14 | def __init__(self, percentage=None, jobs=None, *args, **kwargs): 15 | super(CacheFillerTask, self).__init__(*args, **kwargs) 16 | self.percentage = percentage or 0 17 | self.jobs = jobs or 4 18 | 19 | def create_tasklets(self, state): 20 | tasklets = [] 21 | 22 | for i in range(self.jobs): 23 | tasklet = CacheFillerTasklet( 24 | id=i + 1, 25 | client_params=self.client_params, 26 | percentage=self.percentage, 27 | ) 28 | tasklets.append(tasklet) 29 | 30 | return tasklets 31 | 32 | def pre_tasklets(self, client, state): 33 | stats = client.get_stats() 34 | capacity = stats['limit_maxbytes'] 35 | capacity_fmt = insert_number_commas(capacity) 36 | 37 | state.time_start = time.time() 38 | self.write("Filling to ~%s%% (of %s)" % (self.percentage, capacity_fmt)) 39 | 40 | def post_tasklets(self, client, state, metrics_list): 41 | state.time_stop = time.time() 42 | state.duration = state.time_stop - state.time_start 43 | 44 | time_cum = sum([m.time_cum for m in metrics_list]) 45 | time_total_cum = sum([m.time_total_cum for m in metrics_list]) 46 | overhead_pct = 100 * (time_total_cum - time_cum) / time_total_cum if time_total_cum else 0 47 | 48 | items_cum = sum([m.items_cum for m in metrics_list]) 49 | items_cum_str = insert_number_commas(str(items_cum)) 50 | rate_items_cum = float(items_cum) / float(time_cum) if time_cum > 0 else 0 51 | rate_items_str = insert_number_commas(str(int(rate_items_cum))) 52 | 53 | bytes_cum = sum([m.bytes_cum for m in metrics_list]) 54 | bytes_cum_str = insert_number_commas(str(bytes_cum)) 55 | rate_bytes_cum = float(bytes_cum) / float(time_cum) if time_cum > 0 else 0 56 | rate_bytes_str = insert_number_commas(str(int(rate_bytes_cum))) 57 | 58 | self.write("Done filling, took %.2fs to insert %s items" 59 | " (net avg rate: %s items/s - %s bytes/s)" % 60 | (state.duration, items_cum_str, rate_items_str, rate_bytes_str)) 61 | self.write("Spent %.2fs in network io, %.2fs in total (%.2f%% overhead - data gen, thread scheduling)" 62 | % (time_cum, time_total_cum, overhead_pct)) 63 | self.write("Wrote %s bytes in total" % bytes_cum_str) 64 | 65 | 66 | class CacheFillerTasklet(Tasklet): 67 | def __init__(self, percentage=0, *args, **kwargs): 68 | super(CacheFillerTasklet, self).__init__(*args, **kwargs) 69 | self.percentage = percentage 70 | 71 | def get_pct_full(self, client): 72 | stats = client.get_stats() 73 | capacity = stats['limit_maxbytes'] 74 | bytes = stats['bytes'] 75 | pct_full = 100 * float(bytes) / float(capacity) 76 | return pct_full 77 | 78 | def run(self, client, metrics): 79 | # use client.set + client.flush_pipeline pattern 80 | client.pipeline_mode = True 81 | 82 | stats = client.get_stats() 83 | capacity = stats['limit_maxbytes'] 84 | capacity_fmt = insert_number_commas(capacity) 85 | 86 | metrics.pct_full = self.get_pct_full(client) 87 | metrics.batch_size = 100 88 | metrics.bytes_cum = 0 89 | metrics.time_cum = 0 90 | metrics.time_total_cum = 0 91 | metrics.items_cum = 0 92 | rate = -1 93 | 94 | time_total_st = time.time() 95 | 96 | while metrics.pct_full < self.percentage: 97 | self.write("Cache is %.2f%% full of %s, inserting %s items (rate: %s items/s)" % 98 | (metrics.pct_full, capacity_fmt, metrics.batch_size, 99 | insert_number_commas(str(int(rate))))) 100 | 101 | # Pre-generate keys and values to avoid timing this work 102 | # TODO allow tuning the sizes of keys and values 103 | keys = [generate_random_key_uuid(10) 104 | for _ in xrange(metrics.batch_size)] 105 | values = [generate_random_data(100, 1000) 106 | for _ in xrange(metrics.batch_size)] 107 | 108 | for key, value in izip(keys, values): 109 | if not self._runnable: 110 | return 111 | 112 | client.set(key, value, noreply=True) 113 | 114 | metrics.bytes_cum += len(key) + len(value) 115 | 116 | time_st = time.time() 117 | client.flush_pipeline() 118 | duration = time.time() - time_st 119 | 120 | rate = metrics.batch_size / duration 121 | metrics.time_cum += duration 122 | metrics.items_cum += metrics.batch_size 123 | 124 | metrics.pct_full = self.get_pct_full(client) 125 | metrics.time_total_cum = time.time() - time_total_st 126 | -------------------------------------------------------------------------------- /pyemc/test_integration.py: -------------------------------------------------------------------------------- 1 | import math 2 | import time 3 | 4 | from pyemc.abstractions.test_api import TestCase 5 | from pyemc.client import ClientError 6 | from pyemc.client import DeleteFailedError 7 | from pyemc.client import ExistsError 8 | from pyemc.client import NotFoundError 9 | from pyemc.client import NotStoredError 10 | from pyemc.client import ServerError 11 | from pyemc.util import generate_random_data 12 | from pyemc.util import generate_random_key 13 | 14 | 15 | class TestApi(TestCase): 16 | 17 | ## Happy path 18 | 19 | # Add 20 | 21 | def test_add(self): 22 | key = generate_random_key(8) 23 | val = generate_random_data(10) 24 | 25 | self.client.add(key, val) 26 | item = self.client.get(key) 27 | assert val == item.value 28 | 29 | # try to add an existing key 30 | with self.assert_raises(NotStoredError): 31 | self.client.add(key, val) 32 | 33 | def test_add_noreply(self): 34 | key = generate_random_key(8) 35 | val = generate_random_data(10) 36 | 37 | self.client.add(key, val, noreply=True) 38 | item = self.client.get(key) 39 | assert val == item.value 40 | 41 | # Append 42 | 43 | def test_append(self): 44 | key = generate_random_key(8) 45 | val = generate_random_data(10) 46 | val2 = generate_random_data(10) 47 | 48 | # try to append to an invalid key 49 | with self.assert_raises(NotStoredError): 50 | self.client.append(key, val) 51 | 52 | self.client.set(key, val) 53 | self.client.append(key, val2) 54 | item = self.client.get(key) 55 | assert val + val2 == item.value 56 | 57 | def test_append_noreply(self): 58 | key = generate_random_key(8) 59 | val = generate_random_data(10) 60 | val2 = generate_random_data(10) 61 | 62 | self.client.set(key, val) 63 | self.client.append(key, val2, noreply=True) 64 | item = self.client.get(key) 65 | assert val + val2 == item.value 66 | 67 | # Cas 68 | 69 | def test_cas_ok(self): 70 | key = generate_random_key(8) 71 | val = generate_random_data(10) 72 | 73 | # try to update a key that doesn't exist 74 | with self.assert_raises(NotFoundError): 75 | self.client.cas(key, val, cas_unique='1') 76 | 77 | # set a key 78 | self.client.set(key, val) 79 | 80 | # get the cas_unique 81 | item = self.client.gets(key) 82 | cas_unique = item.cas_unique 83 | 84 | # conditional update 85 | val2 = generate_random_data(10) 86 | self.client.cas(key, val2, cas_unique=cas_unique) 87 | item = self.client.get(key) 88 | assert item.value == val2 89 | 90 | def test_cas_noreply(self): 91 | key = generate_random_key(8) 92 | val = generate_random_data(10) 93 | 94 | # set a key 95 | self.client.set(key, val) 96 | 97 | # get the cas_unique 98 | item = self.client.gets(key) 99 | cas_unique = item.cas_unique 100 | 101 | # conditional update 102 | val2 = generate_random_data(10) 103 | self.client.cas(key, val2, cas_unique=cas_unique, noreply=True) 104 | item = self.client.get(key) 105 | assert item.value == val2 106 | 107 | def test_cas_stale(self): 108 | key = generate_random_key(8) 109 | val = generate_random_data(10) 110 | 111 | # set a key 112 | self.client.set(key, val) 113 | 114 | # get the cas_unique 115 | item = self.client.gets(key) 116 | cas_unique = item.cas_unique 117 | 118 | # "another client" updates it 119 | val2 = generate_random_data(10) 120 | self.client.set(key, val2) 121 | 122 | # conditional update 123 | val3 = generate_random_data(10) 124 | with self.assert_raises(ExistsError): 125 | self.client.cas(key, val3, cas_unique=cas_unique) 126 | 127 | # Decr 128 | 129 | def test_decr(self): 130 | key = generate_random_key(10) 131 | val = '1' 132 | 133 | # try to decr an invalid key 134 | with self.assert_raises(NotFoundError): 135 | self.client.decr(key) 136 | 137 | self.client.set(key, val) 138 | val2 = self.client.decr(key, '1') 139 | assert int(val) - 1 == int(val2) 140 | 141 | def test_decr_noreply(self): 142 | key = generate_random_key(10) 143 | val = '1' 144 | 145 | self.client.set(key, val) 146 | self.client.decr(key, noreply=True) 147 | item = self.client.get(key) 148 | assert int(val) - 1 == int(item.value) 149 | 150 | # Delete 151 | 152 | def test_delete(self): 153 | key = generate_random_key(8) 154 | val = generate_random_data(10) 155 | 156 | # try to delete invalid key 157 | with self.assert_raises(NotFoundError): 158 | self.client.delete(key) 159 | 160 | self.client.set(key, val) 161 | self.client.delete(key) 162 | with self.assert_raises(NotFoundError): 163 | self.client.get(key) 164 | 165 | def test_delete_noreply(self): 166 | key = generate_random_key(8) 167 | val = generate_random_data(10) 168 | 169 | self.client.set(key, val) 170 | self.client.delete(key, noreply=True) 171 | with self.assert_raises(NotFoundError): 172 | self.client.get(key) 173 | 174 | # FlushAll 175 | 176 | def test_flush_all(self): 177 | key = generate_random_key(4) 178 | val = generate_random_data(5, 8) 179 | 180 | # key set before flush is expired 181 | self.client.set(key, val) 182 | self.client.flush_all() 183 | with self.assert_raises(NotFoundError): 184 | self.client.get(key) 185 | 186 | # sleep a bit to make sure we don't get any rounding errors on the 187 | # exact flush timestamp that might affect later stores 188 | time.sleep(1.5) 189 | 190 | # key set after flush works as expected 191 | key2 = generate_random_key(4) 192 | val2 = generate_random_data(5, 8) 193 | self.client.set(key2, val2) 194 | item = self.client.get(key2) 195 | assert item.value == val2 196 | 197 | # Get and Set 198 | 199 | def test_set_and_get_small_key(self): 200 | key = generate_random_key(4) 201 | val = generate_random_data(5, 8) 202 | 203 | self.client.set(key, val) 204 | 205 | item = self.client.get(key) 206 | val2 = item.value 207 | 208 | assert val == val2 209 | 210 | def test_set_and_get_large_value(self): 211 | key = generate_random_key(10) 212 | val = generate_random_data(1 << 19) # .5mb 213 | 214 | self.client.set(key, val) 215 | 216 | item = self.client.get(key) 217 | val2 = item.value 218 | 219 | assert val == val2 220 | 221 | def test_get_multiple(self): 222 | key1 = generate_random_key(10) 223 | val1 = generate_random_data(10) 224 | 225 | key2 = generate_random_key(10) 226 | 227 | key3 = generate_random_key(10) 228 | val3 = generate_random_data(10) 229 | 230 | self.client.set(key1, val1) 231 | 232 | self.client.set(key3, val3) 233 | 234 | keys = [key1, key2, key3] 235 | dct = self.client.get_multi(keys) 236 | 237 | assert val1 == dct[key1].value 238 | assert val3 == dct[key3].value 239 | 240 | def test_set_exptime_abs_2s(self): 241 | key = generate_random_key(10) 242 | val = generate_random_data(10) 243 | 244 | # we don't know if we have time sync with the server, so fetch the 245 | # server's time first 246 | stats = self.client.get_stats() 247 | now = int(stats['time']) 248 | 249 | self.client.set(key, val, exptime=now + 1) 250 | item = self.client.get(key) # still there 251 | 252 | time.sleep(2.3) 253 | with self.assert_raises(NotFoundError): 254 | item = self.client.get(key) # expired 255 | 256 | def test_set_exptime_rel_1s(self): 257 | key = generate_random_key(10) 258 | val = generate_random_data(10) 259 | 260 | self.client.set(key, val, exptime=1) 261 | item = self.client.get(key) # still there 262 | 263 | time.sleep(1.1) 264 | with self.assert_raises(NotFoundError): 265 | item = self.client.get(key) # expired 266 | 267 | def test_set_flags(self): 268 | key = generate_random_key(10) 269 | val = generate_random_data(10) 270 | flags = 15 271 | 272 | self.client.set(key, val, flags=flags) 273 | item = self.client.get(key) 274 | 275 | assert val == item.value 276 | assert flags == item.flags 277 | 278 | def test_set_noreply(self): 279 | key = generate_random_key(10) 280 | val = generate_random_data(10) 281 | 282 | # set without requesting confirmation 283 | self.client.set(key, val, noreply=True) 284 | 285 | # verify that it worked 286 | item = self.client.get(key) 287 | 288 | assert val == item.value 289 | 290 | # Gets 291 | 292 | def test_gets(self): 293 | key = generate_random_key(8) 294 | val = generate_random_data(10) 295 | 296 | # set a key, record cas_unique 297 | self.client.set(key, val) 298 | item = self.client.gets(key) 299 | 300 | # set again, cas_unique should have changed 301 | self.client.set(key, val) 302 | item2 = self.client.gets(key) 303 | assert item.cas_unique != item2.cas_unique 304 | 305 | def test_gets_multi(self): 306 | key1 = generate_random_key(8) 307 | val1 = generate_random_data(10) 308 | key2 = generate_random_key(8) 309 | val2 = generate_random_data(10) 310 | 311 | # i can fetch values as normal, cas_unique is set 312 | self.client.set(key1, val1) 313 | self.client.set(key2, val2) 314 | dct = self.client.gets_multi([key1, key2]) 315 | assert dct[key1].value == val1 316 | assert dct[key2].value == val2 317 | assert dct[key1].cas_unique is not None 318 | assert dct[key2].cas_unique is not None 319 | 320 | # Incr 321 | 322 | def test_incr(self): 323 | key = generate_random_key(10) 324 | val = '1' 325 | 326 | # try to incr an invalid key 327 | with self.assert_raises(NotFoundError): 328 | self.client.incr(key) 329 | 330 | self.client.set(key, val) 331 | val2 = self.client.incr(key, '40') 332 | assert int(val) + 40 == int(val2) 333 | 334 | def test_incr_noreply(self): 335 | key = generate_random_key(10) 336 | val = '1' 337 | 338 | self.client.set(key, val) 339 | self.client.incr(key, noreply=True) 340 | item = self.client.get(key) 341 | assert int(val) + 1 == int(item.value) 342 | 343 | # Quit 344 | 345 | def test_quit(self): 346 | self.client.quit() 347 | 348 | # Prepend 349 | 350 | def test_prepend(self): 351 | key = generate_random_key(8) 352 | val = generate_random_data(10) 353 | val2 = generate_random_data(10) 354 | 355 | # try to prepend to an invalid key 356 | with self.assert_raises(NotStoredError): 357 | self.client.prepend(key, val) 358 | 359 | self.client.set(key, val) 360 | self.client.prepend(key, val2) 361 | item = self.client.get(key) 362 | assert val2 + val == item.value 363 | 364 | def test_prepend_noreply(self): 365 | key = generate_random_key(8) 366 | val = generate_random_data(10) 367 | val2 = generate_random_data(10) 368 | 369 | self.client.set(key, val) 370 | self.client.prepend(key, val2, noreply=True) 371 | item = self.client.get(key) 372 | assert val2 + val == item.value 373 | 374 | # Replace 375 | 376 | def test_replace(self): 377 | key = generate_random_key(8) 378 | val = generate_random_data(10) 379 | val2 = generate_random_data(10) 380 | 381 | # try to replace an invalid key 382 | with self.assert_raises(NotStoredError): 383 | self.client.replace(key, val) 384 | 385 | self.client.set(key, val) 386 | self.client.replace(key, val2) 387 | item = self.client.get(key) 388 | assert val2 == item.value 389 | 390 | def test_replace_noreply(self): 391 | key = generate_random_key(8) 392 | val = generate_random_data(10) 393 | val2 = generate_random_data(10) 394 | 395 | self.client.set(key, val) 396 | self.client.replace(key, val2, noreply=True) 397 | item = self.client.get(key) 398 | assert val2 == item.value 399 | 400 | # Stats 401 | 402 | # name mangle to get it to run last so we see stats from other tests 403 | def test_z_get_stats(self): 404 | dct = self.client.get_stats() 405 | for (key, value) in dct.items(): 406 | self.write('%s: %s' % (key, value)) 407 | 408 | # Touch 409 | 410 | def test_touch(self): 411 | key = generate_random_key(8) 412 | val = generate_random_data(10) 413 | 414 | # try to touch an invalid key 415 | with self.assert_raises(NotFoundError): 416 | self.client.touch(key) 417 | 418 | # expires in 3s 419 | self.client.set(key, val, exptime=3) 420 | 421 | time.sleep(1.5) 422 | 423 | # keep it alive another 3s 424 | # TODO: what should happen if exptime is unset? 425 | self.client.touch(key, exptime=3) 426 | 427 | time.sleep(1.5) 428 | 429 | item = self.client.get(key) 430 | assert val == item.value 431 | 432 | def test_touch_noreply(self): 433 | key = generate_random_key(8) 434 | val = generate_random_data(10) 435 | 436 | # expires in 3s 437 | self.client.set(key, val, exptime=3) 438 | 439 | time.sleep(1.5) 440 | 441 | # keep it alive another 3s 442 | self.client.touch(key, exptime=3, noreply=True) 443 | 444 | time.sleep(1.5) 445 | 446 | item = self.client.get(key) 447 | assert val == item.value 448 | 449 | # Version 450 | 451 | def test_version(self): 452 | version = self.client.version() 453 | self.write(version) 454 | 455 | 456 | ## Failure cases 457 | 458 | def test_get_invalid_key(self): 459 | key = generate_random_key(10) 460 | 461 | self.client.delete(key, noreply=True) 462 | with self.assert_raises(NotFoundError): 463 | item = self.client.get(key) 464 | 465 | def test_decr_underflow(self): 466 | key = generate_random_key(10) 467 | val = '0' 468 | 469 | self.client.set(key, val) 470 | val2 = self.client.decr(key) 471 | assert int(val) == int(val2) 472 | 473 | def test_incr_overflow(self): 474 | key = generate_random_key(10) 475 | val = str((1 << 64) - 1) 476 | 477 | # set max unsigned 64bit value - overflows to 0 478 | self.client.set(key, val) 479 | val2 = self.client.incr(key) 480 | assert val2 == str(0) 481 | 482 | def test_incr_over_size(self): 483 | key = generate_random_key(10) 484 | val = str(1 << 64) # cannot store in 64 bits 485 | 486 | self.client.set(key, val) 487 | with self.assert_raises(ClientError): 488 | self.client.incr(key) # not treated as a number 489 | 490 | 491 | ## Exceed limits 492 | # TODO try key/val too large for each command? 493 | 494 | def test_set_too_large_key(self): 495 | key = generate_random_key(251) # limit is 250b 496 | val = generate_random_data(1) 497 | 498 | with self.assert_raises(ClientError): 499 | self.client.set(key, val) 500 | 501 | def test_set_too_large_value(self): 502 | key = generate_random_key(10) 503 | val = generate_random_data(1 << 21) # 2mb, limit is 1mb 504 | 505 | with self.assert_raises(ServerError): 506 | self.client.set(key, val) 507 | -------------------------------------------------------------------------------- /pyemc/test_stress.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from pyemc.abstractions.test_api import TestCase 4 | 5 | 6 | class TestStress(TestCase): 7 | def run_bench(self, func, loops, desc): 8 | # untimed warmup 9 | warmup_loops = loops / 3 10 | for _ in xrange(loops): 11 | func() 12 | 13 | # timed execution 14 | start_time = time.time() 15 | for _ in xrange(loops): 16 | func() 17 | end_time = time.time() 18 | interval = end_time - start_time 19 | 20 | rate = float(loops) / interval 21 | 22 | self.write("Made %d %s requests in %.2f seconds = %.2f requests/sec" % 23 | (loops, desc, interval, rate)) 24 | 25 | def test_set_const_key_noreply(self): 26 | def func(): 27 | self.client.set('x', 'abc', noreply=True) 28 | 29 | self.run_bench(func, 700000, 'constant key set+noreply') 30 | 31 | def test_set_const_key(self): 32 | def func(): 33 | self.client.set('x', 'abc') 34 | 35 | self.run_bench(func, 100000, 'constant key set') 36 | 37 | def test_get_const_key(self): 38 | self.client.set('x', 'abc') 39 | 40 | def func(): 41 | self.client.get('x') 42 | 43 | self.run_bench(func, 100000, 'constant key get') 44 | 45 | def test_version(self): 46 | '''Does not even touch the storage layer.''' 47 | def func(): 48 | self.client.version() 49 | 50 | self.run_bench(func, 100000, 'version') 51 | -------------------------------------------------------------------------------- /pyemc/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import uuid 4 | 5 | 6 | def generate_random_data(length_from, length_to=None): 7 | length = length_from 8 | if length_to is not None: 9 | length = random.randint(length_from, length_to) 10 | 11 | return os.urandom(length) 12 | 13 | def generate_random_data_prng(length_from, length_to=None): 14 | length = length_from 15 | if length_to is not None: 16 | length = random.randint(length_from, length_to) 17 | 18 | def get(length): 19 | for _ in xrange(length): 20 | yield random.getrandbits(8) 21 | 22 | num = bytearray(get(length)) 23 | return num 24 | 25 | def generate_random_key(length): 26 | data = '' 27 | while len(data) < length: 28 | bytes = generate_random_data(length) 29 | # filter out non printable chars 30 | bytes = [b for b in bytes 31 | if 65 <= ord(b) <= 90 or 97 <= ord(b) <= 122] 32 | bytes = ''.join(bytes) 33 | data += bytes 34 | data = data[:length] 35 | return data 36 | 37 | def generate_random_key_uuid(length_from, length_to=None): 38 | length = length_from 39 | if length_to is not None: 40 | length = random.randint(length_from, length_to) 41 | 42 | # XXX length will never be >32 43 | return uuid.uuid4().hex[:length] 44 | 45 | def insert_number_commas(number): 46 | chunks = [] 47 | 48 | num_iterations = (len(number) / 3) + 1 49 | idx = -3 50 | 51 | for i in range(1, num_iterations + 1): 52 | chunk = number[idx:] 53 | number = number[:idx] 54 | chunks.append(chunk) 55 | if i != num_iterations: 56 | chunks.append(',') 57 | 58 | chunks.reverse() 59 | number = ''.join(chunks) 60 | if number.startswith(','): 61 | number = number[1:] 62 | 63 | return number 64 | 65 | 66 | if __name__ == '__main__': 67 | import time 68 | 69 | def gen(num, num_chars, gen_func, type_name): 70 | time_start = time.time() 71 | 72 | for i in xrange(num): 73 | gen_func(num_chars) 74 | 75 | time_stop = time.time() 76 | 77 | duration = time_stop - time_start 78 | rate_strings = float(num) / duration 79 | rate_bytes = rate_strings * num_chars 80 | print("Generated %s %s-char %s in %.2fs (rate: %s strings/s - %s bytes/s)" % 81 | (insert_number_commas(str(num)), 82 | insert_number_commas(str(num_chars)), 83 | type_name, 84 | duration, 85 | insert_number_commas(str(int(rate_strings))), 86 | insert_number_commas(str(int(rate_bytes))) 87 | )) 88 | 89 | gen(100000, 100, generate_random_data, 'byte strings') 90 | gen(100000, 100, generate_random_data_prng, 'byte strings') 91 | gen(10000, 100, generate_random_key, 'alphanum strings') 92 | gen(10000, 100, generate_random_key_uuid, 'uuid strings') 93 | -------------------------------------------------------------------------------- /run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cargo build --release 4 | target/release/emcache $@ 5 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 79 2 | 3 | # Avoid nuking docopt spec for one 4 | format_strings = false 5 | -------------------------------------------------------------------------------- /src/common/consts.rs: -------------------------------------------------------------------------------- 1 | pub const APP_NAME: &'static str = "emcache"; 2 | pub const APP_VERSION: &'static str = "0.1.0-dev"; 3 | 4 | 5 | pub fn get_version_string() -> String { 6 | format!("{} {}", APP_NAME, APP_VERSION) 7 | } 8 | -------------------------------------------------------------------------------- /src/common/conversions.rs: -------------------------------------------------------------------------------- 1 | use std::str; 2 | 3 | 4 | pub fn string_to_str<'a>(st: &'a String) -> &'a str { 5 | // Not really unsafe because the String is already valid so.. 6 | unsafe { str::from_utf8_unchecked(st.as_bytes()) } 7 | } 8 | 9 | 10 | #[cfg(test)] 11 | mod tests { 12 | use super::string_to_str; 13 | 14 | 15 | #[test] 16 | fn test_string_to_str() { 17 | let st = "abc".to_string(); 18 | assert_eq!(&st, string_to_str(&st)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/common/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod consts; 3 | pub mod conversions; 4 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | // Benchmark testing primitives 2 | #![feature(test)] 3 | extern crate test; 4 | 5 | #[macro_use] 6 | extern crate maplit; 7 | extern crate bufstream; 8 | extern crate docopt; 9 | extern crate linked_hash_map; 10 | extern crate libc; 11 | extern crate net2; 12 | extern crate rand; 13 | extern crate rustc_serialize; 14 | extern crate time; 15 | 16 | mod common; 17 | mod metrics; 18 | mod options; 19 | mod orchestrator; 20 | mod platform; 21 | mod protocol; 22 | mod storage; 23 | mod tcp_transport; 24 | mod testlib; 25 | 26 | use common::consts; 27 | use options::parse_args; 28 | use orchestrator::ListenerTask; 29 | 30 | 31 | fn print_version() { 32 | println!("{} {}", consts::APP_NAME, consts::APP_VERSION); 33 | } 34 | 35 | fn main() { 36 | print_version(); 37 | 38 | let opts = parse_args(); 39 | if opts.flag_version { 40 | // We're done here :) 41 | return; 42 | } 43 | 44 | println!("Running tcp server on {} with {}mb capacity...", 45 | opts.get_bind_string(), 46 | opts.get_mem_limit()); 47 | let mut listener_task = ListenerTask::new(opts.clone()); 48 | listener_task.run(); 49 | } 50 | -------------------------------------------------------------------------------- /src/metrics/live_timers.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use platform::time::time_now; 4 | 5 | use super::StartTime; 6 | use super::Timing; 7 | 8 | 9 | #[derive(Debug, Clone, PartialEq)] 10 | pub struct LiveTimers { 11 | timers: HashMap, 12 | } 13 | 14 | impl LiveTimers { 15 | pub fn new() -> LiveTimers { 16 | LiveTimers { timers: HashMap::new() } 17 | } 18 | 19 | pub fn get_timers(&self) -> &HashMap { 20 | &self.timers 21 | } 22 | 23 | 24 | pub fn start(&mut self, name: &str) -> StartTime { 25 | let start_time = time_now(); 26 | self.timers.insert(name.to_string(), start_time.clone()); 27 | start_time 28 | } 29 | 30 | pub fn stop(&mut self, name: &str) -> Timing { 31 | let stop_time = time_now(); 32 | 33 | let opt = self.timers.remove(name); 34 | if opt.is_none() { 35 | panic!("Tried to stop non-live timer: {:?}", name); 36 | } 37 | 38 | let start_time = opt.unwrap(); 39 | let duration = stop_time - start_time; 40 | 41 | Timing::new(name, start_time, duration) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/metrics/metric.rs: -------------------------------------------------------------------------------- 1 | use super::Timing; 2 | 3 | 4 | #[derive(Debug, Clone, PartialEq)] 5 | pub enum Metric { 6 | Timing(Timing), 7 | } 8 | 9 | impl Metric { 10 | pub fn get_timing(&self) -> &Timing { 11 | match self { 12 | &Metric::Timing(ref timing) => &timing, 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/metrics/metrics.rs: -------------------------------------------------------------------------------- 1 | use super::Metric; 2 | 3 | 4 | #[derive(Debug, Clone, PartialEq)] 5 | pub struct Metrics { 6 | pub metrics: Vec, 7 | } 8 | 9 | impl Metrics { 10 | pub fn new() -> Metrics { 11 | Metrics { metrics: vec![] } 12 | } 13 | 14 | pub fn clear(&mut self) { 15 | self.metrics.clear(); 16 | } 17 | 18 | pub fn first(&self) -> &Metric { 19 | &self.metrics[0] 20 | } 21 | 22 | pub fn push(&mut self, item: Metric) { 23 | self.metrics.push(item); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/metrics/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod live_timers; 3 | pub mod metric; 4 | pub mod metrics; 5 | pub mod recorder; 6 | pub mod statistics; 7 | pub mod time_series; 8 | pub mod timer; 9 | pub mod timing; 10 | pub mod typedefs; 11 | 12 | // internal stuff 13 | mod tests; // needed to be part of the compilation unit in test mode 14 | 15 | 16 | // Export our public api 17 | pub use self::live_timers::LiveTimers; 18 | pub use self::metric::Metric; 19 | pub use self::metrics::Metrics; 20 | pub use self::recorder::MetricsRecorder; 21 | pub use self::time_series::TimeSeries; 22 | pub use self::timer::Timer; 23 | pub use self::timing::Timing; 24 | pub use self::typedefs::Duration; 25 | pub use self::typedefs::Second; 26 | pub use self::typedefs::StartTime; 27 | -------------------------------------------------------------------------------- /src/metrics/recorder.rs: -------------------------------------------------------------------------------- 1 | use orchestrator::MetricsSender; 2 | 3 | use super::LiveTimers; 4 | use super::Metric; 5 | use super::Metrics; 6 | 7 | 8 | pub struct MetricsRecorder { 9 | // If not enabled the recorder is just a shim - records nothing, transmits 10 | // nothing (no performance overhead) 11 | // TODO: Consider replacing this flag with a Collector trait - can implement 12 | // a real collector and a NullCollector 13 | enabled: bool, 14 | 15 | live_timers: LiveTimers, 16 | metrics: Metrics, 17 | 18 | met_tx: MetricsSender, 19 | } 20 | 21 | impl MetricsRecorder { 22 | pub fn new(met_tx: MetricsSender, enabled: bool) -> MetricsRecorder { 23 | MetricsRecorder { 24 | enabled: enabled, 25 | live_timers: LiveTimers::new(), 26 | metrics: Metrics::new(), 27 | met_tx: met_tx, 28 | } 29 | } 30 | 31 | pub fn start_timer(&mut self, name: &str) { 32 | if !self.enabled { 33 | return; 34 | } 35 | 36 | self.live_timers.start(name); 37 | } 38 | 39 | pub fn stop_timer(&mut self, name: &str) { 40 | if !self.enabled { 41 | return; 42 | } 43 | 44 | let timing = self.live_timers.stop(name); 45 | self.metrics.push(Metric::Timing(timing)); 46 | } 47 | 48 | pub fn flush_metrics(&mut self) { 49 | if !self.enabled { 50 | return; 51 | } 52 | 53 | // package up all our data into a metrics object 54 | let metrics = self.metrics.clone(); 55 | 56 | // transmit the metrics 57 | self.met_tx.send(metrics).unwrap(); 58 | 59 | // clear our counters 60 | self.metrics.clear(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/metrics/statistics/aggregate.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::Ordering; 2 | 3 | use super::AggregatedMetric; 4 | 5 | 6 | // f64 does not have total ordering hence this convenience function which 7 | // defaults to judging values equal if they cannot be compared 8 | pub fn sort_f64(samples: &mut Vec) -> &mut Vec { 9 | samples.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); 10 | samples 11 | } 12 | 13 | 14 | pub fn compute_average(samples: &Vec) -> Option { 15 | match samples.is_empty() { 16 | true => None, 17 | false => { 18 | let sum = samples.iter().fold(0.0, |acc, x| acc + x); 19 | let avg = sum / (samples.len() as f64); 20 | Some(avg) 21 | } 22 | } 23 | } 24 | 25 | 26 | pub fn compute_px(samples: &Vec, len: usize, pct: f64) -> Option { 27 | if samples.len() < len { 28 | return None; 29 | } 30 | 31 | let mut clone = samples.clone(); 32 | let sorted = sort_f64(&mut clone); 33 | let pos = ((samples.len() as f64) * pct) as usize; 34 | 35 | Some(sorted[pos]) 36 | } 37 | 38 | pub fn compute_p0(samples: &Vec) -> Option { 39 | compute_px(samples, 1, 0.0) 40 | } 41 | 42 | pub fn compute_p90(samples: &Vec) -> Option { 43 | compute_px(samples, 10, 0.9) 44 | } 45 | 46 | pub fn compute_p99(samples: &Vec) -> Option { 47 | compute_px(samples, 100, 0.99) 48 | } 49 | 50 | pub fn compute_p999(samples: &Vec) -> Option { 51 | compute_px(samples, 1000, 0.999) 52 | } 53 | 54 | 55 | pub fn aggregate_metric(name: &str, samples: &Vec) -> AggregatedMetric { 56 | AggregatedMetric { 57 | name: name.to_string(), 58 | n: samples.len() as u64, 59 | avg: compute_average(&samples), 60 | p0: compute_p0(&samples), 61 | p90: compute_p90(&samples), 62 | p99: compute_p99(&samples), 63 | p999: compute_p999(&samples), 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/metrics/statistics/aggregated_metric.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Clone, PartialEq)] 2 | pub struct AggregatedMetric { 3 | pub name: String, 4 | pub n: u64, 5 | pub avg: Option, 6 | pub p0: Option, 7 | pub p90: Option, 8 | pub p99: Option, 9 | pub p999: Option, 10 | } 11 | -------------------------------------------------------------------------------- /src/metrics/statistics/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod aggregate; 3 | pub mod aggregated_metric; 4 | 5 | // internal stuff 6 | mod tests; // needed to be part of the compilation unit in test mode 7 | 8 | 9 | // Export our public api 10 | pub use self::aggregate::aggregate_metric; 11 | pub use self::aggregate::compute_average; 12 | pub use self::aggregate::compute_p0; 13 | pub use self::aggregate::compute_p90; 14 | pub use self::aggregate::compute_p99; 15 | pub use self::aggregate::compute_p999; 16 | pub use self::aggregate::sort_f64; 17 | pub use self::aggregated_metric::AggregatedMetric; 18 | -------------------------------------------------------------------------------- /src/metrics/statistics/tests.rs: -------------------------------------------------------------------------------- 1 | use testlib::datagen::get_rand_f64_vec; 2 | 3 | use super::AggregatedMetric; 4 | use super::aggregate_metric; 5 | use super::compute_average; 6 | use super::compute_p0; 7 | use super::compute_p90; 8 | use super::compute_p999; 9 | use super::compute_p99; 10 | use super::sort_f64; 11 | 12 | 13 | #[test] 14 | fn test_get_rand_f64_vec() { 15 | let vals = get_rand_f64_vec(1, 100); 16 | assert_eq!(100, vals.len()); 17 | assert_eq!(5050.0, vals.iter().fold(0.0, |acc, x| acc + x)); 18 | } 19 | 20 | 21 | #[test] 22 | fn test_compute_average_empty() { 23 | assert_eq!(None, compute_average(&vec![])); 24 | } 25 | 26 | #[test] 27 | fn test_compute_average_ok() { 28 | assert_eq!(1.3, compute_average(&vec![1.1, 1.3, 1.5]).unwrap()); 29 | } 30 | 31 | 32 | #[test] 33 | fn test_sort_f64() { 34 | assert_eq!(&vec![1.2, 3.1, 9.1], sort_f64(&mut vec![9.1, 1.2, 3.1])); 35 | } 36 | 37 | 38 | #[test] 39 | fn test_compute_p0_too_short() { 40 | let vals = vec![]; 41 | assert_eq!(None, compute_p0(&vals)); 42 | } 43 | 44 | #[test] 45 | fn test_compute_p0_ok() { 46 | let vals = get_rand_f64_vec(1, 17); 47 | assert_eq!(1.0, compute_p0(&vals).unwrap()); 48 | } 49 | 50 | #[test] 51 | fn test_compute_p90_too_short() { 52 | let vals = get_rand_f64_vec(1, 9); 53 | assert_eq!(None, compute_p90(&vals)); 54 | } 55 | 56 | #[test] 57 | fn test_compute_p90_small() { 58 | let vals = get_rand_f64_vec(1, 10); 59 | assert_eq!(10.0, compute_p90(&vals).unwrap()); 60 | } 61 | 62 | #[test] 63 | fn test_compute_p90_large() { 64 | let vals = get_rand_f64_vec(1, 100); 65 | assert_eq!(91.0, compute_p90(&vals).unwrap()); 66 | } 67 | 68 | 69 | #[test] 70 | fn test_compute_p99_too_short() { 71 | let vals = get_rand_f64_vec(1, 99); 72 | assert_eq!(None, compute_p99(&vals)); 73 | } 74 | 75 | #[test] 76 | fn test_compute_p99_small() { 77 | let vals = get_rand_f64_vec(1, 100); 78 | assert_eq!(100.0, compute_p99(&vals).unwrap()); 79 | } 80 | 81 | #[test] 82 | fn test_compute_p99_large() { 83 | let vals = get_rand_f64_vec(1, 1000); 84 | assert_eq!(991.0, compute_p99(&vals).unwrap()); 85 | } 86 | 87 | 88 | #[test] 89 | fn test_compute_p999_too_short() { 90 | let vals = get_rand_f64_vec(1, 999); 91 | assert_eq!(None, compute_p999(&vals)); 92 | } 93 | 94 | #[test] 95 | fn test_compute_p999_small() { 96 | let vals = get_rand_f64_vec(1, 1000); 97 | assert_eq!(1000.0, compute_p999(&vals).unwrap()); 98 | } 99 | 100 | #[test] 101 | fn test_compute_p999_large() { 102 | let vals = get_rand_f64_vec(1, 10000); 103 | assert_eq!(9991.0, compute_p999(&vals).unwrap()); 104 | } 105 | 106 | 107 | #[test] 108 | fn test_compute_metric() { 109 | let vals = get_rand_f64_vec(1, 1000); 110 | let metric = aggregate_metric("latency", &vals); 111 | 112 | let expected = AggregatedMetric { 113 | name: "latency".to_string(), 114 | n: 1000, 115 | avg: Some(500.5), 116 | p0: Some(1.0), 117 | p90: Some(901.0), 118 | p99: Some(991.0), 119 | p999: Some(1000.0), 120 | }; 121 | assert_eq!(expected, metric); 122 | } 123 | -------------------------------------------------------------------------------- /src/metrics/tests.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc; 2 | 3 | use platform::time::sleep_secs; 4 | use platform::time::time_now; 5 | use testlib::cmp::eq_f64; 6 | 7 | use super::LiveTimers; 8 | use super::MetricsRecorder; 9 | use super::TimeSeries; 10 | use super::Timer; 11 | use super::Timing; 12 | 13 | 14 | #[test] 15 | #[should_panic(expected="Tried to stop non-live timer: \"resp\"")] 16 | fn test_live_timers_name_mismatch() { 17 | let mut lt = LiveTimers::new(); 18 | 19 | lt.start("cmd"); 20 | lt.stop("resp"); // panic expected 21 | } 22 | 23 | // this is a slow test that relies on sleeps 24 | #[ignore] 25 | #[test] 26 | fn test_live_timers_ok() { 27 | let mut lt = LiveTimers::new(); 28 | 29 | let t1 = time_now(); 30 | let start_time = lt.start("cmd"); 31 | // start_time is very close to *now* 32 | assert!(eq_f64(t1, start_time, 0.01)); 33 | // "cmd" -> start_time was added to the map 34 | assert_eq!(&start_time, lt.get_timers().get("cmd").unwrap()); 35 | 36 | sleep_secs(0.25); 37 | 38 | let timing = lt.stop("cmd"); 39 | // the returned start_time matches what we saw before 40 | assert_eq!(start_time, timing.start_time); 41 | // the duration is almost exactly the time we slept 42 | assert!(eq_f64(0.25, timing.duration, 0.01)); 43 | // "cmd" was removed from the map 44 | assert!(!lt.get_timers().contains_key("cmd")); 45 | } 46 | 47 | #[test] 48 | fn test_time_series_updates() { 49 | let mut ts = TimeSeries::new(); 50 | 51 | // add a timer 52 | ts.add_timing(&Timing::new("cmd", 1.1, 0.25)); 53 | // construct the expected value for comparison 54 | let expected = hashmap!{ 55 | "cmd".to_string() => hashmap!{ 56 | 1 => vec![0.25], 57 | }, 58 | }; 59 | // compare 60 | assert_eq!(&expected, ts.get_timers()); 61 | 62 | // add another timer 63 | ts.add_timing(&Timing::new("cmd", 1.9, 0.51)); 64 | // construct the expected value for comparison 65 | let expected = hashmap!{ 66 | "cmd".to_string() => hashmap!{ 67 | 1 => vec![0.25, 0.51], 68 | }, 69 | }; 70 | // compare 71 | assert_eq!(&expected, ts.get_timers()); 72 | 73 | // add another timer 74 | ts.add_timing(&Timing::new("cmd", 2.3, 8.8)); 75 | // construct the expected value for comparison 76 | let expected = hashmap!{ 77 | "cmd".to_string() => hashmap!{ 78 | 1 => vec![0.25, 0.51], 79 | 2 => vec![8.8], 80 | }, 81 | }; 82 | // compare 83 | assert_eq!(&expected, ts.get_timers()); 84 | 85 | // add another timer 86 | ts.add_timing(&Timing::new("resp", 4.1, 1.0)); 87 | // construct the expected value for comparison 88 | let expected = hashmap!{ 89 | "cmd".to_string() => hashmap!{ 90 | 1 => vec![0.25, 0.51], 91 | 2 => vec![8.8], 92 | }, 93 | "resp".to_string() => hashmap!{ 94 | 4 => vec![1.0], 95 | }, 96 | }; 97 | // compare 98 | assert_eq!(&expected, ts.get_timers()); 99 | 100 | // empty the series 101 | ts.clear(); 102 | // construct the expected value for comparison 103 | let expected = hashmap!{}; 104 | // compare 105 | assert_eq!(&expected, ts.get_timers()); 106 | } 107 | 108 | 109 | // this is a slow test that relies on sleeps 110 | #[ignore] 111 | #[test] 112 | fn test_timer_correct() { 113 | let (met_tx, met_rx) = mpsc::channel(); 114 | let mut rec = MetricsRecorder::new(met_tx, true); 115 | 116 | // use Timer to make one timing 117 | let _rv = { 118 | let _t = Timer::new(&mut rec, "cmd"); 119 | sleep_secs(0.25); 120 | () 121 | }; 122 | 123 | // flush the metrics so we can see them 124 | rec.flush_metrics(); 125 | 126 | // receive the metrics 127 | let metrics = met_rx.recv().unwrap(); 128 | // verify that the timing is correct 129 | let dur = metrics.first().get_timing().duration; 130 | assert!(eq_f64(0.25, dur, 0.03)); 131 | } 132 | 133 | // this is a slow test that relies on sleeps 134 | #[ignore] 135 | #[test] 136 | fn test_timer_wrong_binding() { 137 | let (met_tx, met_rx) = mpsc::channel(); 138 | let mut rec = MetricsRecorder::new(met_tx, true); 139 | 140 | // use Timer to make one timing 141 | let _rv = { 142 | // this binding discards the value right away! 143 | let _ = Timer::new(&mut rec, "cmd"); 144 | sleep_secs(0.25); 145 | () 146 | }; 147 | 148 | // flush the metrics so we can see them 149 | rec.flush_metrics(); 150 | 151 | // receive the metrics 152 | let metrics = met_rx.recv().unwrap(); 153 | // verify that the timing is correct 154 | let dur = metrics.first().get_timing().duration; 155 | assert!(eq_f64(0.0, dur, 0.01)); 156 | } 157 | 158 | // this is a slow test that relies on sleeps 159 | #[ignore] 160 | #[test] 161 | fn test_timer_no_binding() { 162 | let (met_tx, met_rx) = mpsc::channel(); 163 | let mut rec = MetricsRecorder::new(met_tx, true); 164 | 165 | // use Timer to make one timing 166 | let _rv = { 167 | // no binding means Timer does not live past the first line 168 | Timer::new(&mut rec, "cmd"); 169 | sleep_secs(0.25); 170 | () 171 | }; 172 | 173 | // flush the metrics so we can see them 174 | rec.flush_metrics(); 175 | 176 | // receive the metrics 177 | let metrics = met_rx.recv().unwrap(); 178 | // verify that the timing is correct 179 | let dur = metrics.first().get_timing().duration; 180 | assert!(eq_f64(0.0, dur, 0.01)); 181 | } 182 | -------------------------------------------------------------------------------- /src/metrics/time_series.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use super::Duration; 4 | use super::Second; 5 | use super::Timing; 6 | use super::statistics::AggregatedMetric; 7 | use super::statistics::aggregate_metric; 8 | 9 | 10 | #[derive(Debug, Clone, PartialEq)] 11 | pub struct TimeSeries { 12 | // name -> { 1 -> [0.13, 0.41], 2 -> [0.42, 0.6] } 13 | timers: HashMap>>, 14 | } 15 | 16 | impl TimeSeries { 17 | pub fn new() -> TimeSeries { 18 | TimeSeries { timers: HashMap::new() } 19 | } 20 | 21 | pub fn get_timers(&self) -> &HashMap>> { 22 | &self.timers 23 | } 24 | 25 | 26 | pub fn add_timing(&mut self, timing: &Timing) { 27 | // does the name series exist? 28 | if !self.timers.contains_key(&timing.name) { 29 | self.timers.insert(timing.name.to_string(), HashMap::new()); 30 | } 31 | 32 | // does the second series exist? 33 | let sec = timing.start_time as Second; 34 | if !self.timers.get(&timing.name).unwrap().contains_key(&sec) { 35 | self.timers.get_mut(&timing.name).unwrap().insert(sec, vec![]); 36 | } 37 | 38 | // insert the value 39 | self.timers 40 | .get_mut(&timing.name) 41 | .unwrap() 42 | .get_mut(&sec) 43 | .unwrap() 44 | .push(timing.duration); 45 | } 46 | 47 | pub fn aggregate_metrics(&self) -> HashMap { 48 | let mut agg_mets = HashMap::new(); 49 | 50 | for (name, seconds) in self.timers.iter() { 51 | for (_, samples) in seconds.iter() { 52 | let agg = aggregate_metric(name, samples); 53 | agg_mets.insert(name.to_string(), agg); 54 | } 55 | } 56 | 57 | agg_mets 58 | } 59 | 60 | pub fn clear(&mut self) { 61 | self.timers.clear(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/metrics/timer.rs: -------------------------------------------------------------------------------- 1 | use super::MetricsRecorder; 2 | 3 | 4 | pub struct Timer<'a> { 5 | recorder: &'a mut MetricsRecorder, 6 | name: &'a str, 7 | } 8 | 9 | impl<'a> Timer<'a> { 10 | pub fn new(recorder: &'a mut MetricsRecorder, name: &'a str) -> Timer<'a> { 11 | recorder.start_timer(name); 12 | 13 | Timer { 14 | name: name, 15 | recorder: recorder, 16 | } 17 | } 18 | } 19 | 20 | impl<'a> Drop for Timer<'a> { 21 | fn drop(&mut self) { 22 | self.recorder.stop_timer(self.name); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/metrics/timing.rs: -------------------------------------------------------------------------------- 1 | use super::Duration; 2 | use super::StartTime; 3 | 4 | 5 | #[derive(Debug, Clone, PartialEq)] 6 | pub struct Timing { 7 | pub name: String, 8 | pub start_time: StartTime, 9 | pub duration: Duration, 10 | } 11 | 12 | impl Timing { 13 | pub fn new(name: &str, 14 | start_time: StartTime, 15 | duration: Duration) 16 | -> Timing { 17 | Timing { 18 | name: name.to_string(), 19 | start_time: start_time, 20 | duration: duration, 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/metrics/typedefs.rs: -------------------------------------------------------------------------------- 1 | pub type Second = u64; 2 | pub type StartTime = f64; 3 | pub type Duration = f64; 4 | -------------------------------------------------------------------------------- /src/options.rs: -------------------------------------------------------------------------------- 1 | use docopt::Docopt; 2 | 3 | 4 | // Write the Docopt usage string. 5 | const USAGE: &'static str = " 6 | Usage: 7 | emcache [options] 8 | 9 | Options: 10 | --host HOST Interface to listen on (ie. ip hostname/ip). 11 | -p --port PORT Port to bind to. 12 | -m --mem MEMSIZE Max memory to use (in megabytes). 13 | --metrics Collect server performance metrics. 14 | -V --version Print version info and exit 15 | -h --help Show this screen. 16 | "; 17 | 18 | 19 | #[derive(Debug, Clone, RustcDecodable)] 20 | pub struct MemcacheOptions { 21 | pub flag_host: Option, 22 | pub flag_port: Option, 23 | pub flag_mem: Option, 24 | pub flag_metrics: bool, 25 | pub flag_version: bool, 26 | } 27 | 28 | impl MemcacheOptions { 29 | pub fn get_bind_params(&self) -> (String, u16) { 30 | let opts = self.clone(); 31 | (opts.flag_host.unwrap().clone(), 32 | opts.flag_port.unwrap().clone()) 33 | } 34 | 35 | pub fn get_bind_string(&self) -> String { 36 | let (host, port) = self.get_bind_params(); 37 | format!("{}:{}", host, port) 38 | } 39 | 40 | pub fn get_mem_limit(&self) -> u64 { 41 | self.flag_mem.unwrap() 42 | } 43 | 44 | pub fn get_mem_limit_bytes(&self) -> u64 { 45 | self.flag_mem.unwrap() << 20 46 | } 47 | 48 | pub fn get_metrics_enabled(&self) -> bool { 49 | self.flag_metrics 50 | } 51 | } 52 | 53 | 54 | pub fn parse_args() -> MemcacheOptions { 55 | let mut opts: MemcacheOptions = Docopt::new(USAGE) 56 | .and_then(|d| d.decode()) 57 | .unwrap_or_else(|e| e.exit()); 58 | 59 | if opts.flag_host.is_none() { 60 | opts.flag_host = Some("127.0.0.1".to_string()); 61 | } 62 | if opts.flag_port.is_none() { 63 | opts.flag_port = Some(11311); 64 | } 65 | 66 | if opts.flag_mem.is_none() { 67 | opts.flag_mem = Some(64); 68 | } 69 | 70 | opts 71 | } 72 | -------------------------------------------------------------------------------- /src/orchestrator/driver_task.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use metrics::MetricsRecorder; 4 | use metrics::Timer; 5 | use options::MemcacheOptions; 6 | use protocol::Driver; 7 | use storage::Cache; 8 | use tcp_transport::stats::TransportStats; 9 | 10 | use super::CmdReceiver; 11 | use super::MetricsSender; 12 | use super::TransportId; 13 | 14 | 15 | type StatsMap = HashMap; 16 | 17 | fn compute_stats_sums(map: &StatsMap) -> TransportStats { 18 | let mut total_stats = TransportStats::new(); 19 | 20 | for (_, value) in map { 21 | total_stats.bytes_read += value.bytes_read; 22 | total_stats.bytes_written += value.bytes_written; 23 | } 24 | 25 | total_stats 26 | } 27 | 28 | 29 | pub struct DriverTask { 30 | cmd_rx: CmdReceiver, 31 | met_tx: MetricsSender, 32 | options: MemcacheOptions, 33 | } 34 | 35 | impl DriverTask { 36 | pub fn new(cmd_rx: CmdReceiver, 37 | met_tx: MetricsSender, 38 | options: MemcacheOptions) 39 | -> DriverTask { 40 | DriverTask { 41 | cmd_rx: cmd_rx, 42 | met_tx: met_tx, 43 | options: options, 44 | } 45 | } 46 | 47 | pub fn run(&self) { 48 | let cache = Cache::new(self.options.get_mem_limit_bytes()); 49 | let mut driver = Driver::new(cache); 50 | 51 | // Here we store stats per transport 52 | let mut transport_stats: StatsMap = HashMap::new(); 53 | 54 | // For collecting server metrics 55 | let mut rec = MetricsRecorder::new(self.met_tx.clone(), 56 | self.options.get_metrics_enabled()); 57 | 58 | loop { 59 | // Time the whole loop 60 | rec.start_timer("DriverTask:loop"); 61 | 62 | // Receive command 63 | let (id, resp_tx, cmd, stats) = { 64 | let _t = Timer::new(&mut rec, "DriverTask:recv_cmd"); 65 | self.cmd_rx.recv().unwrap() 66 | }; 67 | 68 | // Update our stats store 69 | transport_stats.insert(id, stats); 70 | 71 | // Update the driver's view of all transport metrics 72 | let total_stats = compute_stats_sums(&transport_stats); 73 | driver.update_transport_stats(total_stats); 74 | 75 | // Execute the command 76 | let resp = { 77 | let _t = Timer::new(&mut rec, "DriverTask:exec_cmd"); 78 | driver.run(cmd) 79 | }; 80 | 81 | // Send response 82 | { 83 | let _t = Timer::new(&mut rec, "DriverTask:send_resp"); 84 | resp_tx.send(resp).unwrap(); 85 | } 86 | 87 | // Stop timing the loop 88 | rec.stop_timer("DriverTask:loop"); 89 | 90 | // Now flush metrics outside the request path 91 | rec.flush_metrics(); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/orchestrator/listener_task.rs: -------------------------------------------------------------------------------- 1 | use std::net::TcpListener; 2 | use std::sync::mpsc; 3 | use std::thread; 4 | 5 | use net2::TcpStreamExt; 6 | 7 | use common::conversions::string_to_str; 8 | use options::MemcacheOptions; 9 | 10 | use super::DriverTask; 11 | use super::MetricsTask; 12 | use super::TransportId; 13 | use super::TransportTask; 14 | 15 | 16 | pub struct ListenerTask { 17 | cur_transport_id: TransportId, 18 | options: MemcacheOptions, 19 | } 20 | 21 | impl ListenerTask { 22 | pub fn new(options: MemcacheOptions) -> ListenerTask { 23 | ListenerTask { 24 | cur_transport_id: TransportId(0), 25 | options: options, 26 | } 27 | } 28 | 29 | fn next_transport_id(&mut self) -> TransportId { 30 | let TransportId(id) = self.cur_transport_id; 31 | let next_id = id + 1; 32 | self.cur_transport_id = TransportId(next_id); 33 | TransportId(next_id) 34 | } 35 | 36 | pub fn run(&mut self) { 37 | // Initialize the metrics sink 38 | let (met_tx, met_rx) = mpsc::channel(); 39 | let metrics = MetricsTask::new(met_rx); 40 | 41 | thread::spawn(move || { 42 | metrics.run(); 43 | }); 44 | 45 | // Initialize the driver 46 | let (cmd_tx, cmd_rx) = mpsc::channel(); 47 | let driver = DriverTask::new(cmd_rx, 48 | met_tx.clone(), 49 | self.options.clone()); 50 | 51 | thread::spawn(move || { 52 | driver.run(); 53 | }); 54 | 55 | // Start up a tcp server 56 | let (host, port) = self.options.get_bind_params(); 57 | let tcp_listener = TcpListener::bind((string_to_str(&host), port)) 58 | .unwrap(); 59 | 60 | for stream in tcp_listener.incoming() { 61 | match stream { 62 | Ok(stream) => { 63 | // Make sure we don't delay on sending 64 | TcpStreamExt::set_nodelay(&stream, true).unwrap(); 65 | 66 | let id = self.next_transport_id(); 67 | let cmd_tx = cmd_tx.clone(); 68 | let met_tx = met_tx.clone(); 69 | let opts = self.options.clone(); 70 | let task = TransportTask::new(id, cmd_tx, met_tx, opts); 71 | 72 | thread::spawn(move || { 73 | task.run(stream); 74 | }); 75 | } 76 | Err(_) => { 77 | println!("Connection failed :("); 78 | } 79 | } 80 | } 81 | 82 | drop(tcp_listener); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/orchestrator/metrics_task.rs: -------------------------------------------------------------------------------- 1 | use metrics::Metric; 2 | use metrics::TimeSeries; 3 | use platform::time::time_now; 4 | 5 | use super::MetricsReceiver; 6 | 7 | 8 | pub struct MetricsTask { 9 | met_rx: MetricsReceiver, 10 | 11 | summary_interval: f64, 12 | } 13 | 14 | impl MetricsTask { 15 | pub fn new(met_rx: MetricsReceiver) -> MetricsTask { 16 | MetricsTask { 17 | met_rx: met_rx, 18 | 19 | summary_interval: 1.0, 20 | } 21 | } 22 | 23 | pub fn run(&self) { 24 | let mut ts = TimeSeries::new(); 25 | let mut last_summary_at = time_now(); 26 | 27 | loop { 28 | // Receive metrics 29 | let metrics = self.met_rx.recv().unwrap(); 30 | for metric in metrics.metrics { 31 | match metric { 32 | Metric::Timing(timing) => { 33 | ts.add_timing(&timing); 34 | } 35 | } 36 | } 37 | 38 | // Is is time to print a summary? 39 | if last_summary_at + self.summary_interval < time_now() { 40 | self.print_summary(&ts); 41 | ts.clear(); 42 | 43 | last_summary_at = time_now(); 44 | } 45 | } 46 | } 47 | 48 | pub fn print_summary(&self, ts: &TimeSeries) { 49 | let agg_mets = ts.aggregate_metrics(); 50 | let mut names: Vec<&String> = agg_mets.keys().collect(); 51 | names.sort(); 52 | 53 | println!("== Metrics {}s snapshot at {} ==", 54 | self.summary_interval, 55 | time_now() as u64); 56 | 57 | for name in names { 58 | let agg = agg_mets.get(name).unwrap(); 59 | 60 | let avg = agg.avg.unwrap_or(-1.0) * 1000.0; 61 | let p0 = agg.p0.unwrap_or(-1.0) * 1000.0; 62 | let p99 = agg.p99.unwrap_or(-1.0) * 1000.0; 63 | 64 | println!("{:30} n: {:5} p0: {:.3}ms avg: {:.3}ms p99: {:.3}ms", 65 | name, 66 | agg.n, 67 | p0, 68 | avg, 69 | p99); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/orchestrator/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod driver_task; 3 | pub mod metrics_task; 4 | pub mod listener_task; 5 | pub mod transport_task; 6 | pub mod typedefs; 7 | 8 | 9 | // Export our public api 10 | pub use self::driver_task::DriverTask; 11 | pub use self::listener_task::ListenerTask; 12 | pub use self::metrics_task::MetricsTask; 13 | pub use self::transport_task::TransportTask; 14 | pub use self::typedefs::CmdReceiver; 15 | pub use self::typedefs::CmdSender; 16 | pub use self::typedefs::MetricsReceiver; 17 | pub use self::typedefs::MetricsSender; 18 | pub use self::typedefs::RespReceiver; 19 | pub use self::typedefs::RespSender; 20 | pub use self::typedefs::TransportId; 21 | -------------------------------------------------------------------------------- /src/orchestrator/transport_task.rs: -------------------------------------------------------------------------------- 1 | use std::net::TcpStream; 2 | use std::sync::mpsc; 3 | 4 | use metrics::MetricsRecorder; 5 | use metrics::Timer; 6 | use options::MemcacheOptions; 7 | use protocol::cmd::Cmd; 8 | use protocol::cmd::Resp; 9 | use tcp_transport::TcpTransport; 10 | 11 | use super::CmdSender; 12 | use super::MetricsSender; 13 | use super::RespReceiver; 14 | use super::RespSender; 15 | use super::TransportId; 16 | 17 | 18 | pub struct TransportTask { 19 | id: TransportId, 20 | cmd_tx: CmdSender, 21 | met_tx: MetricsSender, 22 | options: MemcacheOptions, 23 | } 24 | 25 | impl TransportTask { 26 | pub fn new(id: TransportId, 27 | cmd_tx: CmdSender, 28 | met_tx: MetricsSender, 29 | options: MemcacheOptions) 30 | -> TransportTask { 31 | TransportTask { 32 | id: id, 33 | cmd_tx: cmd_tx, 34 | met_tx: met_tx, 35 | options: options, 36 | } 37 | } 38 | 39 | pub fn run(&self, stream: TcpStream) { 40 | let mut rec = MetricsRecorder::new(self.met_tx.clone(), 41 | self.options.get_metrics_enabled()); 42 | 43 | let mut transport = TcpTransport::new(stream); 44 | let (resp_tx, resp_rx): (RespSender, RespReceiver) = mpsc::channel(); 45 | 46 | loop { 47 | // Time the whole loop 48 | rec.start_timer("TransportTask:loop"); 49 | 50 | // println!("Ready to read command..."); 51 | let rv = { 52 | let _t = Timer::new(&mut rec, "TransportTask:read_cmd"); 53 | transport.read_cmd() 54 | }; 55 | 56 | // If we couldn't parse the command return an error 57 | if !rv.is_ok() { 58 | println!("Failed to read command: {:?}, \ 59 | returning error (client disconnected?)", 60 | rv.unwrap_err()); 61 | let _ = transport.write_resp(&Resp::Error); 62 | break; // Here we just drop the connection 63 | } 64 | 65 | let cmd = rv.unwrap(); 66 | 67 | // Special case commands handled directly by transport 68 | match cmd { 69 | Cmd::Quit => { 70 | println!("Client disconnected"); 71 | break; // Drop the connection 72 | } 73 | _ => (), 74 | } 75 | 76 | // Send the command to the driver 77 | let resp_tx_clone = resp_tx.clone(); 78 | let stats = transport.get_stats_clone(); 79 | { 80 | let _t = Timer::new(&mut rec, "TransportTask:send_cmd"); 81 | self.cmd_tx 82 | .send((self.id, resp_tx_clone, cmd, stats)) 83 | .unwrap(); 84 | } 85 | 86 | // Obtain a response 87 | let resp = { 88 | let _t = Timer::new(&mut rec, "TransportTask:recv_resp"); 89 | resp_rx.recv().unwrap() 90 | }; 91 | 92 | // Return a response 93 | // println!("Returning response: {:?}", &resp); 94 | let rv = { 95 | let _t = Timer::new(&mut rec, "TransportTask:write_resp"); 96 | transport.write_resp(&resp) 97 | }; 98 | if !rv.is_ok() { 99 | println!("Failed to write response :("); 100 | } 101 | 102 | // Stop timing the loop 103 | rec.stop_timer("TransportTask:loop"); 104 | 105 | // Now flush metrics outside the request path 106 | rec.flush_metrics(); 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/orchestrator/typedefs.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | use std::sync::mpsc::Receiver; 3 | 4 | use metrics::Metrics; 5 | use protocol::cmd::Cmd; 6 | use protocol::cmd::Resp; 7 | use tcp_transport::stats::TransportStats; 8 | 9 | 10 | // Cmd/Resp Protocol 11 | 12 | #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] 13 | pub struct TransportId(pub u64); 14 | 15 | pub type RespSender = Sender; 16 | pub type RespReceiver = Receiver; 17 | 18 | pub type CmdSender = Sender<(TransportId, RespSender, Cmd, TransportStats)>; 19 | pub type CmdReceiver = Receiver<(TransportId, 20 | RespSender, 21 | Cmd, 22 | TransportStats)>; 23 | 24 | // Metrics 25 | 26 | pub type MetricsSender = Sender; 27 | pub type MetricsReceiver = Receiver; 28 | -------------------------------------------------------------------------------- /src/platform/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod process; 3 | pub mod time; 4 | -------------------------------------------------------------------------------- /src/platform/process.rs: -------------------------------------------------------------------------------- 1 | use libc; 2 | 3 | 4 | pub fn get_pid() -> u32 { 5 | unsafe { libc::getpid() as u32 } 6 | } 7 | -------------------------------------------------------------------------------- /src/platform/time.rs: -------------------------------------------------------------------------------- 1 | use std; 2 | use std::time::Duration; 3 | 4 | use time; 5 | 6 | 7 | fn convert_secs_to_duration(duration: f64) -> Duration { 8 | // extract the seconds (before the decimal point) 9 | let secs: u64 = duration.floor() as u64; 10 | // obtain the rest (after the decimal point) 11 | let rest = duration - secs as f64; 12 | // convert the rest to nanoseconds 13 | let nanosecs: u32 = (1_000_000_000f64 * rest).round() as u32; 14 | 15 | Duration::new(secs, nanosecs) 16 | } 17 | 18 | fn convert_timespec_to_secs(ts: time::Timespec) -> f64 { 19 | ts.sec as f64 + (ts.nsec as f64 / 1_000_000_000f64) 20 | } 21 | 22 | pub fn time_now() -> f64 { 23 | let ts = time::get_time(); 24 | convert_timespec_to_secs(ts) 25 | } 26 | 27 | pub fn sleep_secs(secs: f64) { 28 | let dur = convert_secs_to_duration(secs); 29 | std::thread::sleep(dur); 30 | } 31 | 32 | 33 | #[cfg(test)] 34 | mod tests { 35 | use std::time::Duration; 36 | 37 | use time; 38 | 39 | use super::convert_secs_to_duration; 40 | use super::convert_timespec_to_secs; 41 | 42 | 43 | #[test] 44 | fn test_timespec_a_quarter_second() { 45 | let ts = time::Timespec::new(1i64, 250_000_000i32); 46 | assert_eq!(1.25f64, convert_timespec_to_secs(ts)); 47 | } 48 | 49 | #[test] 50 | fn test_timespec_a_half_second() { 51 | let ts = time::Timespec::new(1i64, 500_000_000i32); 52 | assert_eq!(1.5f64, convert_timespec_to_secs(ts)); 53 | } 54 | 55 | 56 | #[test] 57 | fn test_secs_a_quarter_second() { 58 | let dur = convert_secs_to_duration(1.25f64); 59 | assert_eq!(dur, Duration::new(1u64, 250_000_000u32)); 60 | } 61 | 62 | #[test] 63 | fn test_secs_a_half_second() { 64 | let dur = convert_secs_to_duration(1.5f64); 65 | assert_eq!(dur, Duration::new(1u64, 500_000_000u32)); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/protocol/cmd.rs: -------------------------------------------------------------------------------- 1 | // ref: https://github.com/memcached/memcached/blob/master/doc/protocol.txt 2 | 3 | 4 | // Request structs 5 | 6 | #[derive(Debug, PartialEq, Clone)] 7 | pub struct Delete { 8 | pub key: String, 9 | pub noreply: bool, // Indicates whether the server should reply to the delete 10 | } 11 | 12 | impl Delete { 13 | pub fn new(key: &str, noreply: bool) -> Delete { 14 | Delete { 15 | key: key.to_string(), 16 | noreply: noreply, 17 | } 18 | } 19 | } 20 | 21 | 22 | #[derive(Debug, PartialEq, Clone)] 23 | pub struct FlushAll { 24 | pub exptime: Option, // Relative (secs) or absolute (unixtime) expiry time 25 | pub noreply: bool, // Indicates whether the server should reply to the flush 26 | } 27 | 28 | impl FlushAll { 29 | pub fn new(exptime: Option, noreply: bool) -> FlushAll { 30 | FlushAll { 31 | exptime: exptime, 32 | noreply: noreply, 33 | } 34 | } 35 | } 36 | 37 | 38 | #[derive(Debug, PartialEq, Clone)] 39 | pub enum GetInstr { 40 | Get, // get blob + flags 41 | Gets, // same + cas_unique 42 | } 43 | 44 | 45 | #[derive(Debug, PartialEq, Clone)] 46 | pub struct Get { 47 | pub instr: GetInstr, // Instruction to perform 48 | pub keys: Vec, 49 | } 50 | 51 | impl Get { 52 | pub fn new(instr: GetInstr, keys: Vec) -> Get { 53 | Get { 54 | instr: instr, 55 | keys: keys, 56 | } 57 | } 58 | 59 | pub fn one(instr: GetInstr, key: &str) -> Get { 60 | Get { 61 | instr: instr, 62 | keys: vec![key.to_string()], 63 | } 64 | } 65 | } 66 | 67 | 68 | #[derive(Debug, PartialEq, Clone)] 69 | pub enum IncInstr { 70 | Incr, 71 | Decr, 72 | } 73 | 74 | 75 | #[derive(Debug, PartialEq, Clone)] 76 | pub struct Inc { 77 | pub instr: IncInstr, // Instruction to perform 78 | pub key: String, 79 | pub delta: u64, 80 | pub noreply: bool, 81 | } 82 | 83 | impl Inc { 84 | pub fn new(instr: IncInstr, key: &str, delta: u64, noreply: bool) -> Inc { 85 | Inc { 86 | instr: instr, 87 | key: key.to_string(), 88 | delta: delta, 89 | noreply: noreply, 90 | } 91 | } 92 | } 93 | 94 | 95 | #[derive(Debug, PartialEq, Clone)] 96 | pub enum SetInstr { 97 | Set, // Store an item 98 | Add, // Store only if the key does not yet exist 99 | Replace, // Store only if the key does already exist 100 | Append, // Append the data for an existing item 101 | Prepend, // Prepend the data for an existing item 102 | Cas, // Compare and swap 103 | } 104 | 105 | 106 | #[derive(Debug, PartialEq, Clone)] 107 | pub struct Set { 108 | pub instr: SetInstr, // Instruction to perform 109 | pub key: String, // Alphanumeric characters 110 | pub flags: u16, // Arbitrary bit pattern chosen by the client 111 | pub exptime: u32, // Relative (secs) or absolute (unixtime) expiry time 112 | pub data: Vec, // Binary data 113 | pub cas_unique: Option, // Client cookie used for conditional updates 114 | pub noreply: bool, // Indicates whether the server should reply to the set 115 | } 116 | 117 | impl Set { 118 | pub fn new(instr: SetInstr, 119 | key: &str, 120 | flags: u16, 121 | exptime: u32, 122 | data: Vec, 123 | noreply: bool) 124 | -> Set { 125 | Set { 126 | instr: instr, 127 | key: key.to_string(), 128 | flags: flags, 129 | exptime: exptime, 130 | data: data, 131 | cas_unique: None, 132 | noreply: noreply, 133 | } 134 | } 135 | 136 | pub fn with_cas_unique(&mut self, cas_unique: u64) -> &mut Self { 137 | self.cas_unique = Some(cas_unique); 138 | self 139 | } 140 | } 141 | 142 | 143 | #[derive(Debug, PartialEq, Clone)] 144 | pub struct Touch { 145 | pub key: String, 146 | pub exptime: u32, 147 | pub noreply: bool, 148 | } 149 | 150 | impl Touch { 151 | pub fn new(key: &str, exptime: u32, noreply: bool) -> Touch { 152 | Touch { 153 | key: key.to_string(), 154 | exptime: exptime, 155 | noreply: noreply, 156 | } 157 | } 158 | } 159 | 160 | 161 | // Response structs 162 | 163 | #[derive(Debug, PartialEq, Clone)] 164 | pub struct Stat { 165 | pub key: String, 166 | pub value: String, 167 | } 168 | 169 | impl Stat { 170 | pub fn new(key: &str, value: String) -> Stat { 171 | Stat { 172 | key: key.to_string(), 173 | value: value, 174 | } 175 | } 176 | } 177 | 178 | 179 | #[derive(Debug, PartialEq, Clone)] 180 | pub struct Value { 181 | pub key: String, 182 | pub flags: u16, 183 | pub cas_unique: Option, 184 | pub data: Vec, 185 | } 186 | 187 | impl Value { 188 | pub fn new(key: &str, flags: u16, data: Vec) -> Value { 189 | Value { 190 | key: key.to_string(), 191 | flags: flags, 192 | cas_unique: None, 193 | data: data, 194 | } 195 | } 196 | 197 | pub fn with_cas_unique(&mut self, cas_unique: u64) -> &mut Self { 198 | self.cas_unique = Some(cas_unique); 199 | self 200 | } 201 | } 202 | 203 | 204 | // High level groupings 205 | 206 | #[derive(Debug, PartialEq, Clone)] 207 | pub enum Cmd { 208 | Delete(Delete), 209 | FlushAll(FlushAll), 210 | Get(Get), 211 | Inc(Inc), 212 | Quit, 213 | Set(Set), 214 | Stats, 215 | Touch(Touch), 216 | Version, 217 | } 218 | 219 | #[derive(Debug, PartialEq, Clone)] 220 | pub enum Resp { 221 | // A sentinel value to indicate that there is nothing to return to the 222 | // client (in case of noreply) 223 | Empty, 224 | 225 | Error, 226 | ClientError(String), 227 | ServerError(String), 228 | 229 | Deleted, // The item was deleted successfully 230 | Exists, // The cas item has been modified 231 | Ok, // FlushAll succeeded 232 | NotFound, // The cas item does not exist 233 | NotStored, // Precondition not met 234 | Stored, // The item was stored successfully 235 | Touched, // The item was touched successfully 236 | 237 | IntValue(u64), // Result of an incr/decr 238 | Stats(Vec), 239 | Values(Vec), 240 | 241 | Version(String), 242 | } 243 | 244 | impl Resp { 245 | pub fn get_stats(&self) -> Option<&Vec> { 246 | match *self { 247 | Resp::Stats(ref stats) => Some(&stats), 248 | _ => None, 249 | } 250 | } 251 | 252 | pub fn get_values(&self) -> Option<&Vec> { 253 | match *self { 254 | Resp::Values(ref values) => Some(&values), 255 | _ => None, 256 | } 257 | } 258 | 259 | pub fn get_first_value(&self) -> Option<&Value> { 260 | match self.get_values() { 261 | Some(ref values) => Some(&values[0]), 262 | _ => None, 263 | } 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /src/protocol/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod cmd; 3 | pub mod driver; 4 | pub mod util; 5 | 6 | // internal stuff 7 | mod tests; // needed to be part of the compilation unit in test mode 8 | mod tests_bench; // needed to be part of the compilation unit in test mode 9 | 10 | 11 | // Export our public api 12 | pub use self::driver::Driver; 13 | -------------------------------------------------------------------------------- /src/protocol/tests_bench.rs: -------------------------------------------------------------------------------- 1 | use test::Bencher; 2 | 3 | use storage::Cache; 4 | 5 | use super::Driver; 6 | use super::cmd::Cmd; 7 | use super::cmd::Get; 8 | use super::cmd::GetInstr; 9 | use super::cmd::Resp; 10 | use super::cmd::Set; 11 | use super::cmd::SetInstr; 12 | 13 | 14 | #[bench] 15 | fn bench_cmd_set_key(b: &mut Bencher) { 16 | let cache = Cache::new(100); 17 | let mut driver = Driver::new(cache); 18 | 19 | let key_name = "x"; 20 | let blob = vec![1, 2, 3]; 21 | 22 | b.iter(|| { 23 | // Set a key 24 | let set = Set::new(SetInstr::Set, 25 | key_name, 26 | 15, 27 | 0, 28 | blob.clone(), 29 | false); 30 | let cmd = Cmd::Set(set); 31 | let resp = driver.run(cmd); 32 | assert_eq!(resp, Resp::Stored); 33 | }) 34 | } 35 | 36 | #[bench] 37 | fn bench_cmd_get_key(b: &mut Bencher) { 38 | let cache = Cache::new(100); 39 | let mut driver = Driver::new(cache); 40 | 41 | let key_name = "x"; 42 | let blob = vec![1, 2, 3]; 43 | 44 | // Set a key 45 | let set = Set::new(SetInstr::Set, key_name, 15, 0, blob.clone(), false); 46 | let cmd = Cmd::Set(set); 47 | let resp = driver.run(cmd); 48 | assert_eq!(resp, Resp::Stored); 49 | 50 | b.iter(|| { 51 | // Retrieve it 52 | let cmd = Cmd::Get(Get::one(GetInstr::Get, key_name)); 53 | let resp = driver.run(cmd); 54 | assert_eq!(15, resp.get_first_value().unwrap().flags); 55 | assert_eq!(blob, resp.get_first_value().unwrap().data); 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /src/protocol/util.rs: -------------------------------------------------------------------------------- 1 | use platform::time::time_now; 2 | use storage::CacheError; 3 | 4 | use super::cmd::Resp; 5 | 6 | 7 | pub fn bytes_to_u64(bytes: &Vec) -> Option { 8 | match String::from_utf8(bytes.clone()) { 9 | Ok(st) => { 10 | match st.parse::() { 11 | Ok(num) => Some(num), 12 | Err(_) => None, 13 | } 14 | } 15 | Err(_) => None, 16 | } 17 | } 18 | 19 | pub fn u64_to_bytes(num: &u64) -> Vec { 20 | let arr = num.to_string().into_bytes(); 21 | 22 | let mut bytes = vec![]; 23 | bytes.extend(arr); 24 | bytes 25 | } 26 | 27 | 28 | pub fn convert_exptime(exptime: u32) -> Option { 29 | // If exptime is greater than zero it means it's set, otherwise unset 30 | if exptime > 0 { 31 | let tm; 32 | 33 | // Is it an interval greater than 30 days? Then it's a timestamp 34 | if exptime > 60 * 60 * 24 * 30 { 35 | tm = exptime as f64; 36 | 37 | } else { 38 | // Otherwise it's relative from now 39 | tm = time_now() + exptime as f64; 40 | } 41 | 42 | return Some(tm); 43 | } 44 | 45 | None 46 | } 47 | 48 | pub fn from_cache_err(err: &CacheError) -> Resp { 49 | match *err { 50 | CacheError::KeyTooLong => { 51 | Resp::ClientError("bad command line format".to_string()) 52 | } 53 | CacheError::ValueTooLong => { 54 | Resp::ServerError("object too large for cache".to_string()) 55 | } 56 | _ => Resp::Error, 57 | } 58 | } 59 | 60 | 61 | #[cfg(test)] 62 | mod tests { 63 | use platform::time::time_now; 64 | use testlib::cmp::eq_f64; 65 | 66 | use super::bytes_to_u64; 67 | use super::convert_exptime; 68 | use super::u64_to_bytes; 69 | 70 | 71 | #[test] 72 | fn test_bytes_to_u64() { 73 | // whitespace 74 | assert_eq!(None, bytes_to_u64(&vec![b' ', b'2'])); 75 | 76 | // alpha 77 | assert_eq!(None, bytes_to_u64(&vec![b'0', b'x', b'2'])); 78 | 79 | // negative 80 | assert_eq!(None, bytes_to_u64(&vec![b'-', b'2'])); 81 | 82 | // too long for u64 83 | assert_eq!(None, bytes_to_u64(&vec![b'1'; 255])); 84 | 85 | // ok 86 | assert_eq!(12, bytes_to_u64(&vec![b'1', b'2']).unwrap()); 87 | } 88 | 89 | #[test] 90 | fn test_u64_to_bytes() { 91 | // any u64 is representable as bytes so there are no boundary conditions to 92 | // check 93 | assert_eq!(vec![b'1', b'2'], u64_to_bytes(&12)); 94 | } 95 | 96 | #[test] 97 | fn test_convert_exptime() { 98 | assert_eq!(None, convert_exptime(0)); 99 | 100 | // big enough to be a timestamp 101 | let val = (60 * 60 * 24 * 30) + 1; 102 | assert_eq!(Some(val as f64), convert_exptime(val)); 103 | 104 | // not big enough to be a timestamp - it's a relative time 105 | let val = 5; 106 | let expected = time_now() + val as f64; 107 | let actual = convert_exptime(val).unwrap(); 108 | assert!(eq_f64(expected, actual, 0.01)); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/storage/cache.rs: -------------------------------------------------------------------------------- 1 | use linked_hash_map::LinkedHashMap; 2 | 3 | use platform::time::time_now; 4 | 5 | use super::errors::CacheError; 6 | use super::key::Key; 7 | use super::typedefs::CacheResult; 8 | use super::value::Value; 9 | 10 | 11 | pub struct CacheStats { 12 | pub bytes: u64, // Bytes currently stored 13 | pub evictions: u64, // Number of items removed to make space for new items 14 | pub get_hits: u64, 15 | pub get_misses: u64, 16 | pub delete_misses: u64, 17 | pub delete_hits: u64, 18 | pub reclaimed: u64, // Number of times an entry was reclaimed to store a new entry 19 | pub total_items: u64, // Total items stored since server started 20 | } 21 | 22 | impl CacheStats { 23 | pub fn new() -> CacheStats { 24 | CacheStats { 25 | bytes: 0, 26 | delete_hits: 0, 27 | delete_misses: 0, 28 | evictions: 0, 29 | get_hits: 0, 30 | get_misses: 0, 31 | reclaimed: 0, 32 | total_items: 0, 33 | } 34 | } 35 | 36 | fn bytes_add(&mut self, key: &Key, value: &Value) { 37 | self.bytes += key.mem_size() as u64; 38 | self.bytes += value.mem_size() as u64; 39 | } 40 | 41 | fn bytes_subtract(&mut self, key: &Key, value: &Value) { 42 | self.bytes -= key.mem_size() as u64; 43 | self.bytes -= value.mem_size() as u64; 44 | } 45 | } 46 | 47 | 48 | pub struct Cache { 49 | pub capacity: u64, // in bytes 50 | storage: LinkedHashMap, 51 | item_lifetime: f64, // in seconds, <0 for unlimited 52 | global_exptime: f64, // unixtime, <0 for unset 53 | 54 | key_maxlen: u64, // in bytes 55 | value_maxlen: u64, // in bytes 56 | 57 | stats: CacheStats, 58 | } 59 | 60 | impl Cache { 61 | pub fn new(capacity: u64) -> Cache { 62 | Cache { 63 | capacity: capacity, 64 | global_exptime: -1.0, 65 | item_lifetime: -1.0, 66 | key_maxlen: 250, // 250b 67 | stats: CacheStats::new(), 68 | value_maxlen: 1048576, // 1mb 69 | storage: LinkedHashMap::new(), 70 | } 71 | } 72 | 73 | pub fn with_item_lifetime(&mut self, item_lifetime: f64) -> &mut Cache { 74 | self.item_lifetime = item_lifetime; 75 | self 76 | } 77 | 78 | pub fn with_key_maxlen(&mut self, key_maxlen: u64) -> &mut Cache { 79 | self.key_maxlen = key_maxlen; 80 | self 81 | } 82 | 83 | pub fn with_value_maxlen(&mut self, value_maxlen: u64) -> &mut Cache { 84 | self.value_maxlen = value_maxlen; 85 | self 86 | } 87 | 88 | 89 | pub fn get_stats(&self) -> &CacheStats { 90 | &self.stats 91 | } 92 | 93 | 94 | fn check_key_len(&self, key: &Key) -> bool { 95 | key.len() as u64 <= self.key_maxlen 96 | } 97 | 98 | fn check_value_len(&self, value: &Value) -> bool { 99 | value.len() as u64 <= self.value_maxlen 100 | } 101 | 102 | 103 | fn evict_oldest(&mut self) -> CacheResult<(Key, Value)> { 104 | let opt = self.storage.pop_back(); 105 | 106 | match opt { 107 | Some((key, value)) => { 108 | // Update stats 109 | self.stats.bytes_subtract(&key, &value); 110 | self.stats.evictions += 1; 111 | 112 | Ok((key, value)) 113 | } 114 | None => Err(CacheError::EvictionFailed), 115 | } 116 | } 117 | 118 | fn value_is_alive(&self, value: &Value) -> bool { 119 | // If we have a global exptime set, then any item touched before it is 120 | // dead 121 | if self.global_exptime > 0.0 { 122 | if *value.get_atime() < self.global_exptime { 123 | return false; 124 | } 125 | } 126 | 127 | // If the value has an exptime set, that determines lifetime 128 | // regardless of item_lifetime in the cache 129 | if *value.get_exptime() > 0.0 { 130 | if self.global_exptime > 0.0 { 131 | if *value.get_exptime() < self.global_exptime { 132 | return false; 133 | } 134 | } 135 | 136 | if *value.get_exptime() < time_now() { 137 | return false; 138 | } else { 139 | return true; 140 | } 141 | } 142 | 143 | // if we have no lifetime setting then values are always live 144 | if self.item_lifetime < 0.0 { 145 | return true; 146 | } 147 | 148 | // otherwise use lifetime to determine liveness 149 | *value.get_atime() + self.item_lifetime > time_now() 150 | } 151 | 152 | 153 | pub fn contains_key(&mut self, key: &Key) -> CacheResult { 154 | let result = self.get(key); 155 | 156 | match result { 157 | // We know how to interpret found and not found 158 | Ok(_) => Ok(true), 159 | Err(CacheError::KeyNotFound) => Ok(false), 160 | 161 | // Some other error 162 | Err(x) => Err(x), 163 | } 164 | } 165 | 166 | pub fn flush_all(&mut self, exptime: f64) -> CacheResult<()> { 167 | self.global_exptime = exptime; 168 | Ok(()) 169 | } 170 | 171 | pub fn get(&mut self, key: &Key) -> CacheResult<&Value> { 172 | // Check key size 173 | if !self.check_key_len(key) { 174 | return Err(CacheError::KeyTooLong); 175 | } 176 | 177 | // Pop the value first 178 | let opt = self.storage.remove(key); 179 | 180 | // We didn't find it 181 | if opt.is_none() { 182 | self.stats.get_misses += 1; 183 | return Err(CacheError::KeyNotFound); 184 | } 185 | 186 | // From here on we can assume we did find it 187 | let mut value = opt.unwrap(); 188 | 189 | // The value has been successfully removed - update stats 190 | self.stats.bytes_subtract(key, &value); 191 | 192 | // Now check if the value is still alive 193 | if !self.value_is_alive(&value) { 194 | self.stats.get_misses += 1; 195 | return Err(CacheError::KeyNotFound); 196 | } 197 | 198 | // Update the value to mark that it's been accessed just now 199 | value.touch(); 200 | 201 | // We are going to re-instate the key - update stats 202 | self.stats.bytes_add(key, &value); 203 | self.stats.get_hits += 1; 204 | 205 | // Now we reinsert the key to refresh it 206 | self.storage.insert(key.clone(), value); 207 | 208 | // Load since we need to return it 209 | let value = self.storage.get(key).unwrap(); 210 | 211 | // Return success 212 | Ok(value) 213 | } 214 | 215 | pub fn len(&self) -> usize { 216 | self.storage.len() 217 | } 218 | 219 | pub fn remove(&mut self, key: &Key) -> CacheResult { 220 | // Check key size 221 | if !self.check_key_len(key) { 222 | return Err(CacheError::KeyTooLong); 223 | } 224 | 225 | let opt = self.storage.remove(key); 226 | 227 | match opt { 228 | Some(value) => { 229 | // Update stats 230 | self.stats.delete_hits += 1; 231 | self.stats.bytes_subtract(key, &value); 232 | 233 | Ok((value)) 234 | } 235 | None => { 236 | // Update stats 237 | self.stats.delete_misses += 1; 238 | 239 | Err(CacheError::KeyNotFound) 240 | } 241 | } 242 | } 243 | 244 | pub fn set(&mut self, key: Key, mut value: Value) -> CacheResult<()> { 245 | // Check key & value sizes 246 | if !self.check_key_len(&key) { 247 | return Err(CacheError::KeyTooLong); 248 | } 249 | if !self.check_value_len(&value) { 250 | return Err(CacheError::ValueTooLong); 251 | } 252 | 253 | // Does this item even fit into our cache at all? 254 | if key.mem_size() as u64 + value.mem_size() as u64 > self.capacity { 255 | return Err(CacheError::CapacityExceeded); 256 | } 257 | 258 | // Do we already store this key? 259 | if self.storage.contains_key(&key) { 260 | let mut plus_delta = 0u64; 261 | { 262 | // Load the existing value 263 | let prev_value = self.storage.get(&key).unwrap(); 264 | 265 | // We're updating the key, possibly with a different size value 266 | self.stats.bytes_subtract(&key, &prev_value); 267 | 268 | // Figure out how much more space we need to store the new value 269 | if value.mem_size() > prev_value.mem_size() { 270 | plus_delta = value.mem_size() as u64 - 271 | prev_value.mem_size() as u64; 272 | } 273 | } 274 | 275 | // Would the new value exceed our capacity? Then we need to reclaim 276 | loop { 277 | if self.stats.bytes + key.mem_size() as u64 + plus_delta <= 278 | self.capacity { 279 | break; 280 | } 281 | 282 | try!(self.evict_oldest()); 283 | 284 | // Update stats 285 | self.stats.reclaimed += 1; 286 | } 287 | 288 | } else { 289 | // Do we have space for the new item? 290 | loop { 291 | if self.stats.bytes + key.mem_size() as u64 + 292 | value.mem_size() as u64 <= 293 | self.capacity { 294 | break; 295 | } 296 | 297 | try!(self.evict_oldest()); 298 | 299 | // Update stats 300 | self.stats.reclaimed += 1; 301 | } 302 | } 303 | 304 | // Update stats 305 | self.stats.bytes_add(&key, &value); 306 | self.stats.total_items += 1; 307 | 308 | // Update atime for value 309 | value.touch(); 310 | 311 | // Store the value 312 | self.storage.insert(key, value); 313 | 314 | // Return success 315 | Ok(()) 316 | } 317 | } 318 | -------------------------------------------------------------------------------- /src/storage/errors.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, PartialEq)] 2 | pub enum CacheError { 3 | CapacityExceeded, 4 | EvictionFailed, 5 | KeyNotFound, 6 | KeyTooLong, 7 | ValueTooLong, 8 | } 9 | -------------------------------------------------------------------------------- /src/storage/key.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | 3 | 4 | #[derive(PartialEq, Eq, Hash, Debug, Clone)] 5 | pub struct Key { 6 | pub item: Vec, 7 | } 8 | 9 | impl Key { 10 | pub fn new(item: Vec) -> Key { 11 | Key { item: item } 12 | } 13 | 14 | pub fn len(&self) -> usize { 15 | self.item.len() 16 | } 17 | 18 | pub fn mem_size(&self) -> usize { 19 | mem::size_of::() + self.item.len() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/storage/macros.rs: -------------------------------------------------------------------------------- 1 | #![macro_use] 2 | 3 | 4 | // key!(1, 2, 3) => Key { item: Vec = [1, 2, 3] } 5 | macro_rules! key { 6 | ( $( $x:expr ),* ) => { 7 | { 8 | let mut vec = Vec::new(); 9 | $( 10 | vec.push($x); 11 | )* 12 | Key::new(vec) 13 | } 14 | }; 15 | } 16 | 17 | // value!(1, 2, 3) => Value { item: Vec = [1, 2, 3] } 18 | macro_rules! value { 19 | ( $( $x:expr ),* ) => { 20 | { 21 | let mut vec = Vec::new(); 22 | $( 23 | vec.push($x); 24 | )* 25 | Value::new(vec) 26 | } 27 | }; 28 | } 29 | -------------------------------------------------------------------------------- /src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod macros; // must be listed first since macros are order dependent 3 | 4 | pub mod cache; 5 | pub mod errors; 6 | pub mod key; 7 | pub mod typedefs; 8 | pub mod value; 9 | 10 | // internal stuff 11 | mod tests; // needed to be part of the compilation unit in test mode 12 | mod tests_bench; // needed to be part of the compilation unit in test mode 13 | 14 | 15 | // Export our public api 16 | pub use self::cache::Cache; 17 | pub use self::errors::CacheError; 18 | pub use self::key::Key; 19 | pub use self::typedefs::CacheResult; 20 | pub use self::value::Value; 21 | -------------------------------------------------------------------------------- /src/storage/tests.rs: -------------------------------------------------------------------------------- 1 | use platform::time::sleep_secs; 2 | use platform::time::time_now; 3 | 4 | use super::Cache; 5 | use super::CacheError; 6 | use super::Key; 7 | use super::Value; 8 | 9 | 10 | #[test] 11 | fn test_cas_id() { 12 | let mut value = value!(1); 13 | assert_eq!(0, *value.get_cas_id()); 14 | 15 | value.set_item(vec![2]); 16 | assert_eq!(1, *value.get_cas_id()); 17 | 18 | value.set_flags(15); 19 | assert_eq!(2, *value.get_cas_id()); 20 | 21 | value.set_exptime(0.0); 22 | assert_eq!(3, *value.get_cas_id()); 23 | 24 | // Touch is never due to a client changing it, just us 25 | value.touch(); 26 | assert_eq!(3, *value.get_cas_id()); 27 | } 28 | 29 | #[test] 30 | fn test_set_one_key() { 31 | let mut cache = Cache::new(1024); 32 | 33 | let key = key!(1, 2, 3); 34 | let mut value = value!(4, 5, 6); 35 | value.set_flags(15); 36 | 37 | // First set it 38 | let rv = cache.set(key.clone(), value.clone()); 39 | assert!(rv.is_ok()); 40 | 41 | // Then test for it 42 | let rv = cache.contains_key(&key); 43 | assert_eq!(rv.unwrap(), true); 44 | 45 | // Check the size of the cache 46 | assert_eq!(1, cache.len()); 47 | 48 | // Test for a key that was not set 49 | let rv = cache.contains_key(&key!(9, 8)); 50 | assert_eq!(rv.unwrap(), false); 51 | 52 | // Now fetch it 53 | { 54 | let value_found = cache.get(&key).unwrap(); 55 | assert_eq!(value, *value_found); 56 | } 57 | 58 | // Now remove it 59 | let value_popped = cache.remove(&key).unwrap(); 60 | assert_eq!(value, value_popped); 61 | 62 | // Now test for it 63 | let rv = cache.contains_key(&key); 64 | assert_eq!(rv.unwrap(), false); 65 | 66 | // Check the size of the cache 67 | assert_eq!(0, cache.len()); 68 | } 69 | 70 | #[test] 71 | fn test_key_not_found() { 72 | let mut cache = Cache::new(1024); 73 | 74 | // Set a key 75 | let rv = cache.set(key!(1), value!(9)); 76 | assert!(rv.is_ok()); 77 | 78 | // Retrieve a different key 79 | let rv = cache.get(&key!(2)); 80 | assert_eq!(rv.unwrap_err(), CacheError::KeyNotFound); 81 | } 82 | 83 | #[test] 84 | fn test_store_beyond_capacity_lru() { 85 | let item_size = key!(1).mem_size() as u64 + value!(1).mem_size() as u64; 86 | let mut cache = Cache::new(item_size); 87 | 88 | // we've now reached capacity 89 | let rv = cache.set(key!(1), value!(8)); 90 | assert!(rv.is_ok()); 91 | assert_eq!(cache.len(), 1); 92 | 93 | // write another key 94 | let rv = cache.set(key!(2), value!(9)); 95 | assert!(rv.is_ok()); 96 | assert_eq!(cache.len(), 1); 97 | 98 | // the first key is gone 99 | { 100 | let rv = cache.contains_key(&key!(1)); 101 | assert_eq!(rv.unwrap(), false); 102 | } 103 | { 104 | let rv = cache.get(&key!(1)); 105 | assert!(rv.is_err()); 106 | } 107 | 108 | // the second key is present 109 | { 110 | let rv = cache.contains_key(&key!(2)); 111 | assert_eq!(rv.unwrap(), true); 112 | } 113 | { 114 | let rv = cache.get(&key!(2)); 115 | assert!(rv.is_ok()); 116 | } 117 | 118 | // try to set an item that's bigger than the whole cache 119 | let rv = cache.set(key!(2, 3), value!(9, 10, 11)); 120 | assert!(rv.is_err()); 121 | assert_eq!(cache.len(), 1); 122 | 123 | // make sure the previous set attempt didn't evict anything 124 | let rv = cache.contains_key(&key!(2)); 125 | assert_eq!(rv.unwrap(), true); 126 | 127 | } 128 | 129 | #[test] 130 | fn test_multiple_evictions() { 131 | // Get a cache just big enough to store two items with short key/val 132 | let item_size = key!(1).mem_size() as u64 + value!(1).mem_size() as u64; 133 | let mut cache = Cache::new(item_size * 2); 134 | 135 | // Set a key 136 | let rv = cache.set(key!(1), value!(8)); 137 | assert!(rv.is_ok()); 138 | assert_eq!(cache.len(), 1); 139 | assert_eq!(cache.get_stats().evictions, 0); 140 | 141 | // Set another key 142 | let rv = cache.set(key!(2), value!(9)); 143 | assert!(rv.is_ok()); 144 | assert_eq!(cache.len(), 2); 145 | assert_eq!(cache.get_stats().evictions, 0); 146 | 147 | // Set an item so big it forces everything else to be evicted 148 | let rv = cache.set(key!(3), value!(9, 10, 11)); 149 | assert!(rv.is_ok()); 150 | assert_eq!(cache.len(), 1); 151 | assert_eq!(cache.get_stats().evictions, 2); 152 | } 153 | 154 | #[test] 155 | fn test_exceed_item_size_limits() { 156 | let mut cache = Cache::new(1024); 157 | cache.with_key_maxlen(1) 158 | .with_value_maxlen(1); 159 | 160 | // contains_key: use a key that is too long 161 | { 162 | let rv = cache.contains_key(&key!(1, 2)); 163 | assert_eq!(rv.unwrap_err(), CacheError::KeyTooLong); 164 | } 165 | 166 | // get: use a key that is too long 167 | { 168 | let rv = cache.get(&key!(1, 2)); 169 | assert_eq!(rv.unwrap_err(), CacheError::KeyTooLong); 170 | } 171 | 172 | // remove: use a key that is too long 173 | { 174 | let rv = cache.remove(&key!(1, 2)); 175 | assert_eq!(rv.unwrap_err(), CacheError::KeyTooLong); 176 | } 177 | 178 | // set: use a key that is too long 179 | { 180 | let rv = cache.set(key!(1, 2), value!(9)); 181 | assert_eq!(rv.unwrap_err(), CacheError::KeyTooLong); 182 | } 183 | 184 | // set: use a value that is too long 185 | { 186 | let rv = cache.set(key!(1), value!(9, 8)); 187 | assert_eq!(rv.unwrap_err(), CacheError::ValueTooLong); 188 | } 189 | } 190 | 191 | #[test] 192 | fn test_key_expired_lifetime() { 193 | // our cache has a lifetime of 0 secs - all keys are dead on store 194 | let mut cache = Cache::new(1024); 195 | cache.with_item_lifetime(0.0); 196 | 197 | let key = key!(1); 198 | let value = value!(9); 199 | 200 | // set a key 201 | let rv = cache.set(key.clone(), value); 202 | assert!(rv.is_ok()); 203 | 204 | // try to retrieve it - it has expired 205 | let rv = cache.get(&key); 206 | assert_eq!(rv.unwrap_err(), CacheError::KeyNotFound); 207 | } 208 | 209 | #[test] 210 | fn test_key_explicit_exptime() { 211 | // our cache has infinite lifetime 212 | let mut cache = Cache::new(1024); 213 | 214 | let key = key!(1); 215 | let mut value = value!(9); 216 | // set exptime in the past 217 | value.set_exptime(time_now() - 1.0); 218 | 219 | // set a key 220 | let rv = cache.set(key.clone(), value); 221 | assert!(rv.is_ok()); 222 | 223 | // try to retrieve it - it has expired 224 | let rv = cache.get(&key); 225 | assert_eq!(rv.unwrap_err(), CacheError::KeyNotFound); 226 | } 227 | 228 | // this is a slow test that relies on sleeps 229 | #[ignore] 230 | #[test] 231 | fn test_key_kept_alive_on_access() { 232 | // our cache has a lifetime of 2 secs 233 | let mut cache = Cache::new(1024); 234 | cache.with_item_lifetime(2.0); 235 | 236 | let key = key!(1); 237 | let value = value!(9); 238 | 239 | let rv = cache.set(key.clone(), value.clone()); 240 | assert!(rv.is_ok()); 241 | 242 | // sleep 1.5 secs - not long enough to expire key 243 | sleep_secs(1.5); 244 | 245 | // access key - it's there 246 | assert!(cache.get(&key).is_ok()); 247 | 248 | // sleep 1 secs - not long enough to expire key 249 | sleep_secs(1.0); 250 | 251 | // access key - it's now been 2.5s since it was set, but it's been accessed 252 | // so we've kept it alive 253 | assert!(cache.get(&key).is_ok()); 254 | 255 | // sleep 2.5 secs - long enough to expire key 256 | sleep_secs(2.5); 257 | 258 | // access key - it's gone 259 | assert!(cache.get(&key).is_err()); 260 | } 261 | 262 | // this is a slow test that relies on sleeps 263 | #[ignore] 264 | #[test] 265 | fn test_flush_all() { 266 | // our cache has a lifetime of 2 secs 267 | let mut cache = Cache::new(1024); 268 | cache.with_item_lifetime(2.0); 269 | 270 | // this item lives for 3s 271 | let key1 = key!(1); 272 | let mut value1 = value!(9); 273 | value1.set_exptime(time_now() + 3.0); 274 | let rv = cache.set(key1.clone(), value1.clone()); 275 | assert!(rv.is_ok()); 276 | 277 | // this item lives until cache lifetime 278 | let key2 = key!(2); 279 | let value2 = value!(8); 280 | let rv = cache.set(key2.clone(), value2.clone()); 281 | assert!(rv.is_ok()); 282 | 283 | // make all items dead in one second 284 | cache.flush_all(time_now() + 1.0).unwrap(); 285 | 286 | // sleep until flush time kicks in 287 | sleep_secs(1.5); 288 | 289 | // access both keys - both have expired 290 | assert!(cache.get(&key1).is_err()); 291 | assert!(cache.get(&key2).is_err()); 292 | 293 | // set a new item that came after flush_all 294 | let key3 = key!(3); 295 | let value3 = value!(7); 296 | let rv = cache.set(key3.clone(), value3.clone()); 297 | assert!(rv.is_ok()); 298 | 299 | // it was not expired 300 | assert!(cache.get(&key3).is_ok()); 301 | } 302 | 303 | #[test] 304 | fn test_metrics() { 305 | // NOTE: The most crucial metric is bytes, so make sure to test every data 306 | // path that affects it. 307 | 308 | let item_size = key!(1).mem_size() as u64 + value!(1, 2).mem_size() as u64; 309 | let mut cache = Cache::new(item_size); 310 | assert_eq!(cache.get_stats().bytes, 0); 311 | assert_eq!(cache.get_stats().total_items, 0); 312 | 313 | // Set a key 314 | cache.set(key!(1), value!(2, 3)).unwrap(); 315 | assert_eq!(cache.get_stats().bytes, item_size); 316 | assert_eq!(cache.get_stats().evictions, 0); 317 | assert_eq!(cache.get_stats().get_hits, 0); 318 | assert_eq!(cache.get_stats().get_misses, 0); 319 | assert_eq!(cache.get_stats().delete_hits, 0); 320 | assert_eq!(cache.get_stats().delete_misses, 0); 321 | assert_eq!(cache.get_stats().total_items, 1); 322 | 323 | // Set a different key, evicting the first 324 | cache.set(key!(5), value!(6, 7)).unwrap(); 325 | assert_eq!(cache.get_stats().bytes, item_size); 326 | assert_eq!(cache.get_stats().evictions, 1); 327 | assert_eq!(cache.get_stats().get_hits, 0); 328 | assert_eq!(cache.get_stats().get_misses, 0); 329 | assert_eq!(cache.get_stats().delete_hits, 0); 330 | assert_eq!(cache.get_stats().delete_misses, 0); 331 | assert_eq!(cache.get_stats().total_items, 2); 332 | 333 | // Re-set the key with a different value 334 | cache.set(key!(5), value!(6, 8)).unwrap(); 335 | assert_eq!(cache.get_stats().bytes, item_size); 336 | assert_eq!(cache.get_stats().evictions, 1); 337 | assert_eq!(cache.get_stats().get_hits, 0); 338 | assert_eq!(cache.get_stats().get_misses, 0); 339 | assert_eq!(cache.get_stats().delete_hits, 0); 340 | assert_eq!(cache.get_stats().delete_misses, 0); 341 | assert_eq!(cache.get_stats().total_items, 3); 342 | 343 | // Retrieve the key successfully 344 | cache.get(&key!(5)).unwrap(); 345 | assert_eq!(cache.get_stats().bytes, item_size); 346 | assert_eq!(cache.get_stats().evictions, 1); 347 | assert_eq!(cache.get_stats().get_hits, 1); 348 | assert_eq!(cache.get_stats().get_misses, 0); 349 | assert_eq!(cache.get_stats().delete_hits, 0); 350 | assert_eq!(cache.get_stats().delete_misses, 0); 351 | assert_eq!(cache.get_stats().total_items, 3); 352 | 353 | // Test for the key successfully 354 | cache.contains_key(&key!(5)).unwrap(); 355 | assert_eq!(cache.get_stats().bytes, item_size); 356 | assert_eq!(cache.get_stats().evictions, 1); 357 | assert_eq!(cache.get_stats().get_hits, 2); 358 | assert_eq!(cache.get_stats().get_misses, 0); 359 | assert_eq!(cache.get_stats().delete_hits, 0); 360 | assert_eq!(cache.get_stats().delete_misses, 0); 361 | assert_eq!(cache.get_stats().total_items, 3); 362 | 363 | // Retrieve a key that doesn't exist 364 | cache.get(&key!(17)).unwrap_err(); 365 | assert_eq!(cache.get_stats().bytes, item_size); 366 | assert_eq!(cache.get_stats().evictions, 1); 367 | assert_eq!(cache.get_stats().get_hits, 2); 368 | assert_eq!(cache.get_stats().get_misses, 1); 369 | assert_eq!(cache.get_stats().delete_hits, 0); 370 | assert_eq!(cache.get_stats().delete_misses, 0); 371 | assert_eq!(cache.get_stats().total_items, 3); 372 | 373 | // Create an expired value 374 | let mut value = value!(11, 12); 375 | value.set_exptime(time_now() - 1.0); 376 | 377 | // Set a key that expires immediately 378 | cache.set(key!(9), value).unwrap(); 379 | assert_eq!(cache.get_stats().bytes, item_size); 380 | assert_eq!(cache.get_stats().evictions, 2); 381 | assert_eq!(cache.get_stats().get_hits, 2); 382 | assert_eq!(cache.get_stats().get_misses, 1); 383 | assert_eq!(cache.get_stats().delete_hits, 0); 384 | assert_eq!(cache.get_stats().delete_misses, 0); 385 | assert_eq!(cache.get_stats().total_items, 4); 386 | 387 | // Retrieve expired key 388 | cache.get(&key!(9)).unwrap_err(); 389 | assert_eq!(cache.get_stats().bytes, 0); 390 | assert_eq!(cache.get_stats().evictions, 2); 391 | assert_eq!(cache.get_stats().get_hits, 2); 392 | assert_eq!(cache.get_stats().get_misses, 2); 393 | assert_eq!(cache.get_stats().delete_hits, 0); 394 | assert_eq!(cache.get_stats().delete_misses, 0); 395 | assert_eq!(cache.get_stats().total_items, 4); 396 | 397 | // Set another key 398 | cache.set(key!(21), value!(12, 13)).unwrap(); 399 | assert_eq!(cache.get_stats().bytes, item_size); 400 | assert_eq!(cache.get_stats().evictions, 2); 401 | assert_eq!(cache.get_stats().get_hits, 2); 402 | assert_eq!(cache.get_stats().get_misses, 2); 403 | assert_eq!(cache.get_stats().delete_hits, 0); 404 | assert_eq!(cache.get_stats().delete_misses, 0); 405 | assert_eq!(cache.get_stats().total_items, 5); 406 | 407 | // Delete it 408 | cache.remove(&key!(21)).unwrap(); 409 | assert_eq!(cache.get_stats().bytes, 0); 410 | assert_eq!(cache.get_stats().evictions, 2); 411 | assert_eq!(cache.get_stats().get_hits, 2); 412 | assert_eq!(cache.get_stats().get_misses, 2); 413 | assert_eq!(cache.get_stats().delete_hits, 1); 414 | assert_eq!(cache.get_stats().delete_misses, 0); 415 | assert_eq!(cache.get_stats().total_items, 5); 416 | 417 | // Try to delete it again 418 | cache.remove(&key!(21)).unwrap_err(); 419 | assert_eq!(cache.get_stats().bytes, 0); 420 | assert_eq!(cache.get_stats().evictions, 2); 421 | assert_eq!(cache.get_stats().get_hits, 2); 422 | assert_eq!(cache.get_stats().get_misses, 2); 423 | assert_eq!(cache.get_stats().delete_hits, 1); 424 | assert_eq!(cache.get_stats().delete_misses, 1); 425 | assert_eq!(cache.get_stats().total_items, 5); 426 | } 427 | -------------------------------------------------------------------------------- /src/storage/tests_bench.rs: -------------------------------------------------------------------------------- 1 | use test::Bencher; 2 | 3 | use super::Cache; 4 | use super::Key; 5 | use super::Value; 6 | 7 | 8 | #[bench] 9 | fn bench_set_key(b: &mut Bencher) { 10 | let mut cache = Cache::new(1024); 11 | 12 | b.iter(|| { 13 | cache.set(key!(1), value!(9)).unwrap(); 14 | }) 15 | } 16 | 17 | #[bench] 18 | fn bench_get_key(b: &mut Bencher) { 19 | let mut cache = Cache::new(1024); 20 | 21 | cache.set(key!(1), value!(9)).unwrap(); 22 | b.iter(|| { 23 | cache.get(&key!(1)).unwrap(); 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /src/storage/typedefs.rs: -------------------------------------------------------------------------------- 1 | use super::errors::CacheError; 2 | 3 | 4 | pub type CacheResult = Result; 5 | -------------------------------------------------------------------------------- /src/storage/value.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | 3 | use platform::time::time_now; 4 | 5 | 6 | #[derive(Debug, Clone)] 7 | pub struct Value { 8 | // Settable/gettable 9 | item: Vec, 10 | flags: u16, // chosen by the client 11 | exptime: f64, // expiry time (unixtime), <0 for unset 12 | 13 | // Managed internally 14 | atime: f64, // last access time (unixtime) 15 | cas_id: u64, // Incremented every time the value is changed 16 | } 17 | 18 | impl PartialEq for Value { 19 | // Overload eq to make sure we only compare the fields that the client 20 | // stores explicitly 21 | fn eq(&self, other: &Value) -> bool { 22 | self.item == other.item && self.flags == other.flags 23 | } 24 | } 25 | 26 | impl Value { 27 | pub fn new(item: Vec) -> Value { 28 | Value { 29 | item: item, 30 | flags: 0, 31 | atime: -1.0, 32 | exptime: -1.0, 33 | cas_id: 0, 34 | } 35 | } 36 | 37 | pub fn empty() -> Value { 38 | Value { 39 | item: vec![], 40 | flags: 0, 41 | atime: -1.0, 42 | exptime: -1.0, 43 | cas_id: 0, 44 | } 45 | } 46 | 47 | 48 | pub fn get_item_mut(&mut self) -> &mut Vec { 49 | self.bump_cas_id(); 50 | &mut self.item 51 | } 52 | 53 | pub fn get_item(&self) -> &Vec { 54 | &self.item 55 | } 56 | 57 | pub fn set_item(&mut self, item: Vec) -> &mut Self { 58 | self.bump_cas_id(); 59 | self.item = item; 60 | self 61 | } 62 | 63 | pub fn get_flags(&self) -> &u16 { 64 | &self.flags 65 | } 66 | 67 | pub fn set_flags(&mut self, flags: u16) -> &mut Self { 68 | self.bump_cas_id(); 69 | self.flags = flags; 70 | self 71 | } 72 | 73 | pub fn get_exptime(&self) -> &f64 { 74 | &self.exptime 75 | } 76 | 77 | pub fn set_exptime(&mut self, exptime: f64) { 78 | self.bump_cas_id(); 79 | self.exptime = exptime; 80 | } 81 | 82 | pub fn get_atime(&self) -> &f64 { 83 | &self.atime 84 | } 85 | 86 | pub fn get_cas_id(&self) -> &u64 { 87 | &self.cas_id 88 | } 89 | 90 | fn bump_cas_id(&mut self) { 91 | self.cas_id += 1; 92 | } 93 | 94 | pub fn touch(&mut self) { 95 | self.atime = time_now(); 96 | } 97 | 98 | 99 | pub fn len(&self) -> usize { 100 | self.item.len() 101 | } 102 | 103 | pub fn mem_size(&self) -> usize { 104 | mem::size_of::() + self.item.len() 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/tcp_transport/conversions.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use super::errors::TcpTransportError; 4 | use super::typedefs::TcpTransportResult; 5 | 6 | 7 | pub fn as_string(bytes: Vec) -> TcpTransportResult { 8 | match String::from_utf8(bytes) { 9 | Ok(st) => Ok(st), 10 | Err(_) => Err(TcpTransportError::Utf8Error), 11 | } 12 | } 13 | 14 | pub fn as_number(bytes: Vec) -> TcpTransportResult { 15 | let string = try!(as_string(bytes)); 16 | match string.parse::() { 17 | Ok(num) => Ok(num), 18 | Err(_) => Err(TcpTransportError::NumberParseError), 19 | } 20 | } 21 | 22 | 23 | 24 | #[cfg(test)] 25 | mod tests { 26 | use tcp_transport::TcpTransportError; 27 | 28 | use super::as_number; 29 | use super::as_string; 30 | 31 | 32 | #[test] 33 | fn test_as_string() { 34 | // bytestring is utf8 35 | let st = as_string(vec![b'a', b'b']).unwrap(); 36 | assert_eq!(st, "ab".to_string()); 37 | 38 | // bytestring is not utf8 39 | let err = as_string(vec![b'a', 254, b'b']).unwrap_err(); 40 | assert_eq!(err, TcpTransportError::Utf8Error); 41 | } 42 | 43 | #[test] 44 | fn test_as_number() { 45 | // bytestring is a number 46 | let num = as_number::(vec![b'1', b'2']).unwrap(); 47 | assert_eq!(num, 12); 48 | 49 | // bytestring is not a number 50 | let err = as_number::(vec![b' ', b'1', b'2']).unwrap_err(); 51 | assert_eq!(err, TcpTransportError::NumberParseError); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/tcp_transport/errors.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, PartialEq)] 2 | pub enum TcpTransportError { 3 | CommandParseError, 4 | InvalidCmd, 5 | LineReadError, 6 | NumberParseError, 7 | StreamReadError, 8 | StreamWriteError, 9 | Utf8Error, 10 | } 11 | -------------------------------------------------------------------------------- /src/tcp_transport/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod conversions; 3 | pub mod errors; 4 | pub mod stats; 5 | pub mod transport; 6 | pub mod typedefs; 7 | 8 | // internal stuff 9 | mod tests; // needed to be part of the compilation unit in test mode 10 | mod tests_bench; // needed to be part of the compilation unit in test mode 11 | 12 | 13 | // Export our public api 14 | pub use self::errors::TcpTransportError; 15 | pub use self::stats::TransportStats; 16 | pub use self::transport::TcpTransport; 17 | pub use self::typedefs::TcpTransportResult; 18 | -------------------------------------------------------------------------------- /src/tcp_transport/stats.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Clone)] 2 | pub struct TransportStats { 3 | // These numbers are snapshots given that metrics are recorded concurrently 4 | // by each transport and transmitted to the protocol at regular intervals. 5 | pub bytes_read: u64, 6 | pub bytes_written: u64, 7 | } 8 | 9 | impl TransportStats { 10 | pub fn new() -> TransportStats { 11 | TransportStats { 12 | bytes_read: 0, 13 | bytes_written: 0, 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/tcp_transport/tests_bench.rs: -------------------------------------------------------------------------------- 1 | use test::Bencher; 2 | 3 | use protocol::cmd::Resp; 4 | use protocol::cmd::Value; 5 | use testlib::test_stream::TestStream; 6 | 7 | use super::TcpTransport; 8 | 9 | 10 | // Reading 11 | 12 | #[bench] 13 | fn bench_transport_read_cmd_get(b: &mut Bencher) { 14 | let cmd_str = "get variable1 variable2\r\n".to_string().into_bytes(); 15 | let mut ts = TestStream::new(vec![]); 16 | ts.set_incoming_rep(cmd_str); // set stream to repeating mode 17 | let mut transport = TcpTransport::new(ts); 18 | 19 | b.iter(|| transport.read_cmd().unwrap()) 20 | } 21 | 22 | #[bench] 23 | fn bench_transport_read_cmd_set(b: &mut Bencher) { 24 | let cmd_str = "set variable 13 1 10 noreply\r\n0123456789\r\n" 25 | .to_string() 26 | .into_bytes(); 27 | let mut ts = TestStream::new(vec![]); 28 | ts.set_incoming_rep(cmd_str); // set stream to repeating mode 29 | let mut transport = TcpTransport::new(ts); 30 | 31 | b.iter(|| { 32 | transport.read_cmd().unwrap(); 33 | }) 34 | } 35 | 36 | 37 | // Writing 38 | 39 | #[bench] 40 | fn bench_transport_write_resp_value(b: &mut Bencher) { 41 | let ts = TestStream::new(vec![]); 42 | let mut transport = TcpTransport::new(ts); 43 | 44 | b.iter(|| { 45 | let val = Value::new("x", 15, "abc".to_string().into_bytes()); 46 | let resp = Resp::Values(vec![val]); 47 | transport.write_resp(&resp).unwrap(); 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /src/tcp_transport/transport.rs: -------------------------------------------------------------------------------- 1 | use std::io::Read; 2 | use std::io::Write; 3 | 4 | use bufstream::BufStream; 5 | 6 | use protocol::cmd::Cmd; 7 | use protocol::cmd::Delete; 8 | use protocol::cmd::FlushAll; 9 | use protocol::cmd::Get; 10 | use protocol::cmd::GetInstr; 11 | use protocol::cmd::Inc; 12 | use protocol::cmd::IncInstr; 13 | use protocol::cmd::Resp; 14 | use protocol::cmd::Set; 15 | use protocol::cmd::SetInstr; 16 | use protocol::cmd::Touch; 17 | 18 | use super::conversions::as_number; 19 | use super::conversions::as_string; 20 | use super::errors::TcpTransportError; 21 | use super::stats::TransportStats; 22 | use super::typedefs::TcpTransportResult; 23 | 24 | 25 | // return_err_if!(end_of_line, TcpTransportError::StreamReadError) => 26 | // 27 | // if end_of_line { 28 | // return Err(TcpTransportError::StreamReadError); 29 | // } 30 | macro_rules! return_err_if { 31 | ( $cond:expr, $val:expr ) => { 32 | { 33 | if $cond { 34 | return Err($val); 35 | } 36 | } 37 | }; 38 | } 39 | 40 | 41 | pub struct TcpTransport { 42 | stream: BufStream, 43 | stats: TransportStats, 44 | } 45 | 46 | impl TcpTransport { 47 | pub fn new(stream: T) -> TcpTransport { 48 | TcpTransport { 49 | stats: TransportStats::new(), 50 | stream: BufStream::new(stream), 51 | } 52 | } 53 | 54 | 55 | pub fn get_stats_clone(&self) -> TransportStats { 56 | self.stats.clone() 57 | } 58 | 59 | pub fn get_stream(&self) -> &T { 60 | self.stream.get_ref() 61 | } 62 | 63 | // Basic bytes manipulation and reading from the stream 64 | 65 | pub fn read_bytes_exact(&mut self, 66 | len: u64) 67 | -> TcpTransportResult> { 68 | let mut bytes = vec![0; len as usize]; 69 | let mut cursor: usize = 0; 70 | let mut iteration = 0; 71 | 72 | loop { 73 | // Read as much as we can, hopefully the whole buffer 74 | let rv = self.stream.read(&mut bytes[cursor..]); 75 | 76 | // Something went wrong 77 | if rv.is_err() { 78 | return Err(TcpTransportError::StreamReadError); 79 | } 80 | 81 | // How much we actually read 82 | let bytes_cnt = rv.unwrap(); 83 | 84 | // Woops, there was nothing to read! 85 | if bytes_cnt == 0 { 86 | if iteration == 0 { 87 | // It's the first iteration, so there wasn't anything to 88 | // read in the first place, we were called in vain! 89 | return Err(TcpTransportError::StreamReadError); 90 | 91 | } else { 92 | // It turns out we read the very last byte on the last 93 | // iteration, so nothing more to do at this point 94 | break; 95 | } 96 | } 97 | 98 | // We advance the position in the buffer for next iteration 99 | cursor += bytes_cnt; 100 | 101 | // Update stats 102 | self.stats.bytes_read += bytes_cnt as u64; 103 | 104 | // We've read as much as was requested already 105 | if (bytes_cnt as u64) >= len { 106 | break; 107 | } 108 | 109 | iteration += 1; 110 | } 111 | 112 | if (cursor as u64) < len { 113 | bytes.truncate(cursor); 114 | } 115 | 116 | Ok(bytes) 117 | } 118 | 119 | pub fn read_word_in_line(&mut self) -> TcpTransportResult<(Vec, bool)> { 120 | let mut word = vec![]; 121 | let mut byte = [0; 1]; 122 | let mut end_of_line = false; 123 | 124 | loop { 125 | // Read a byte 126 | let rv = self.stream.read(&mut byte); 127 | 128 | // If there was an error or if there was nothing to read we bail 129 | if rv.is_err() || rv.unwrap() == 0 { 130 | return Err(TcpTransportError::StreamReadError); 131 | } 132 | 133 | // Update stats 134 | self.stats.bytes_read += 1; 135 | 136 | if byte[0] == b' ' { 137 | // We found a space 138 | 139 | if word.is_empty() { 140 | // If it's one or more leading space we ignore it 141 | continue; 142 | } 143 | 144 | // All good, we've found the end of the word 145 | break; 146 | 147 | } else if byte[0] == b'\r' { 148 | // We found \r, we think it's the end of the line 149 | 150 | // Try to read \n 151 | let rv = self.stream.read(&mut byte); 152 | 153 | // If there was an error or if there was nothing to read we bail 154 | if rv.is_err() || rv.unwrap() == 0 { 155 | return Err(TcpTransportError::StreamReadError); 156 | } 157 | 158 | // Update stats 159 | self.stats.bytes_read += 1; 160 | 161 | // If it's not a correct end of line we storm out in protest 162 | if byte[0] != b'\n' { 163 | return Err(TcpTransportError::LineReadError); 164 | } 165 | 166 | // Else it's all good, we've read the whole line including the 167 | // terminator 168 | end_of_line = true; 169 | break; 170 | 171 | } else { 172 | // It's not a special char, append to our word 173 | word.push(byte[0]); 174 | } 175 | } 176 | 177 | Ok((word, end_of_line)) 178 | } 179 | 180 | pub fn read_line_as_words(&mut self) -> TcpTransportResult>> { 181 | let mut words = vec![]; 182 | 183 | loop { 184 | let (word, end_of_line) = try!(self.read_word_in_line()); 185 | 186 | // Don't bother if it's an empty word (trailing space before \r\n) 187 | if !word.is_empty() { 188 | words.push(word); 189 | } 190 | 191 | if end_of_line { 192 | break; 193 | } 194 | } 195 | 196 | Ok(words) 197 | } 198 | 199 | // Writing to the stream 200 | 201 | pub fn flush_writes(&mut self) -> TcpTransportResult<()> { 202 | match self.stream.flush() { 203 | Ok(_) => Ok(()), 204 | Err(_) => Err(TcpTransportError::StreamWriteError), 205 | } 206 | } 207 | 208 | pub fn write_bytes(&mut self, 209 | bytes: &Vec) 210 | -> TcpTransportResult { 211 | match self.stream.write(bytes) { 212 | Ok(cnt_written) => { 213 | // Update stats 214 | self.stats.bytes_written += cnt_written as u64; 215 | 216 | Ok(cnt_written) 217 | } 218 | Err(_) => Err(TcpTransportError::StreamWriteError), 219 | } 220 | } 221 | 222 | pub fn write_string(&mut self, string: &str) -> TcpTransportResult { 223 | let bytes = string.to_string().into_bytes(); 224 | Ok(try!(self.write_bytes(&bytes))) 225 | } 226 | 227 | 228 | // Parse individual commands 229 | 230 | pub fn parse_cmd_delete(&mut self) -> TcpTransportResult { 231 | // parse the key 232 | let key_str = { 233 | let (key, end_of_line) = try!(self.read_word_in_line()); 234 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 235 | try!(as_string(key)) 236 | }; 237 | 238 | // parse noreply 239 | let noreply_flag = { 240 | let (noreply, end_of_line) = try!(self.read_word_in_line()); 241 | return_err_if!(!end_of_line, TcpTransportError::CommandParseError); 242 | let noreply_str = try!(as_string(noreply)); 243 | match noreply_str == "noreply" { 244 | true => true, 245 | false => false, 246 | } 247 | }; 248 | 249 | Ok(Cmd::Delete(Delete { 250 | key: key_str, 251 | noreply: noreply_flag, 252 | })) 253 | } 254 | 255 | pub fn parse_cmd_flush_all(&mut self) -> TcpTransportResult { 256 | // consume the line 257 | // try!(self.read_line_as_words()); 258 | 259 | // TODO hardcoded - need better parser primitives 260 | Ok(Cmd::FlushAll(FlushAll { 261 | exptime: None, 262 | noreply: false, 263 | })) 264 | } 265 | 266 | pub fn parse_cmd_get(&mut self, 267 | instr: GetInstr) 268 | -> TcpTransportResult { 269 | let mut keys = vec![]; 270 | 271 | loop { 272 | let (key, end_of_line) = try!(self.read_word_in_line()); 273 | let key_str = try!(as_string(key)); 274 | keys.push(key_str); 275 | 276 | if end_of_line { 277 | break; 278 | } 279 | } 280 | 281 | Ok(Cmd::Get(Get { 282 | instr: instr, 283 | keys: keys, 284 | })) 285 | } 286 | 287 | pub fn parse_cmd_inc(&mut self, 288 | instr: IncInstr) 289 | -> TcpTransportResult { 290 | // parse the key 291 | let key_str = { 292 | let (key, end_of_line) = try!(self.read_word_in_line()); 293 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 294 | try!(as_string(key)) 295 | }; 296 | 297 | // parse the delta 298 | let delta_num = { 299 | let (exptime, end_of_line) = try!(self.read_word_in_line()); 300 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 301 | try!(as_number::(exptime)) 302 | }; 303 | 304 | // parse noreply 305 | let noreply_flag = { 306 | let (noreply, end_of_line) = try!(self.read_word_in_line()); 307 | return_err_if!(!end_of_line, TcpTransportError::CommandParseError); 308 | let noreply_str = try!(as_string(noreply)); 309 | match noreply_str == "noreply" { 310 | true => true, 311 | false => false, 312 | } 313 | }; 314 | 315 | // We got all the values we expected and there is nothing left 316 | return Ok(Cmd::Inc(Inc { 317 | instr: instr, 318 | key: key_str, 319 | delta: delta_num, 320 | noreply: noreply_flag, 321 | })); 322 | } 323 | 324 | pub fn parse_cmd_set(&mut self, 325 | instr: SetInstr) 326 | -> TcpTransportResult { 327 | // parse the key 328 | let key_str = { 329 | let (key, end_of_line) = try!(self.read_word_in_line()); 330 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 331 | try!(as_string(key)) 332 | }; 333 | 334 | // parse the flags 335 | let flags_num = { 336 | let (flags, end_of_line) = try!(self.read_word_in_line()); 337 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 338 | try!(as_number::(flags)) 339 | }; 340 | 341 | // parse the exptime 342 | let exptime_num = { 343 | let (exptime, end_of_line) = try!(self.read_word_in_line()); 344 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 345 | try!(as_number::(exptime)) 346 | }; 347 | 348 | // parse the bytelen 349 | let bytelen_num = { 350 | let (bytelen, end_of_line) = try!(self.read_word_in_line()); 351 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 352 | try!(as_number::(bytelen)) 353 | }; 354 | 355 | // parse cas_unique 356 | let cas_unique_opt = { 357 | if instr == SetInstr::Cas { 358 | let (cas_unique, _) = try!(self.read_word_in_line()); 359 | let cas_unique = try!(as_number(cas_unique)); 360 | Some(cas_unique) 361 | } else { 362 | None 363 | } 364 | }; 365 | 366 | // parse noreply 367 | let noreply_flag = { 368 | let (noreply, end_of_line) = try!(self.read_word_in_line()); 369 | return_err_if!(!end_of_line, TcpTransportError::CommandParseError); 370 | let noreply_str = try!(as_string(noreply)); 371 | match noreply_str == "noreply" { 372 | true => true, 373 | false => false, 374 | } 375 | }; 376 | 377 | // We now know the byte length, so read the value 378 | let value = try!(self.read_bytes_exact(bytelen_num)); 379 | 380 | // The value is the wrong size 381 | if value.len() as u64 != bytelen_num { 382 | return Err(TcpTransportError::CommandParseError); 383 | } 384 | 385 | // Verify that we found the line terminator 386 | let terminator = try!(self.read_bytes_exact(2)); 387 | if !terminator.ends_with(&[b'\r', b'\n']) { 388 | return Err(TcpTransportError::CommandParseError); 389 | } 390 | 391 | // We got all the values we expected and there is nothing left 392 | return Ok(Cmd::Set(Set { 393 | instr: instr, 394 | key: key_str, 395 | flags: flags_num, 396 | exptime: exptime_num, 397 | data: value, 398 | cas_unique: cas_unique_opt, 399 | noreply: noreply_flag, 400 | })); 401 | } 402 | 403 | pub fn parse_cmd_touch(&mut self) -> TcpTransportResult { 404 | // parse the key 405 | let key_str = { 406 | let (key, end_of_line) = try!(self.read_word_in_line()); 407 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 408 | try!(as_string(key)) 409 | }; 410 | 411 | // parse the exptime 412 | let exptime_num = { 413 | let (exptime, end_of_line) = try!(self.read_word_in_line()); 414 | return_err_if!(end_of_line, TcpTransportError::CommandParseError); 415 | try!(as_number::(exptime)) 416 | }; 417 | 418 | // parse noreply 419 | let noreply_flag = { 420 | let (noreply, end_of_line) = try!(self.read_word_in_line()); 421 | return_err_if!(!end_of_line, TcpTransportError::CommandParseError); 422 | let noreply_str = try!(as_string(noreply)); 423 | match noreply_str == "noreply" { 424 | true => true, 425 | false => false, 426 | } 427 | }; 428 | 429 | // We got all the values we expected and there is nothing left 430 | return Ok(Cmd::Touch(Touch { 431 | key: key_str, 432 | exptime: exptime_num, 433 | noreply: noreply_flag, 434 | })); 435 | } 436 | 437 | // High level functions 438 | 439 | pub fn read_cmd(&mut self) -> TcpTransportResult { 440 | let keyword_str = { 441 | let (word, _) = try!(self.read_word_in_line()); 442 | try!(as_string(word)) 443 | }; 444 | 445 | // TODO replace if's with something nicer 446 | if keyword_str == "get" { 447 | return self.parse_cmd_get(GetInstr::Get); 448 | } else if keyword_str == "gets" { 449 | return self.parse_cmd_get(GetInstr::Gets); 450 | } else if keyword_str == "set" { 451 | return self.parse_cmd_set(SetInstr::Set); 452 | } else if keyword_str == "cas" { 453 | return self.parse_cmd_set(SetInstr::Cas); 454 | } else if keyword_str == "add" { 455 | return self.parse_cmd_set(SetInstr::Add); 456 | } else if keyword_str == "replace" { 457 | return self.parse_cmd_set(SetInstr::Replace); 458 | } else if keyword_str == "append" { 459 | return self.parse_cmd_set(SetInstr::Append); 460 | } else if keyword_str == "prepend" { 461 | return self.parse_cmd_set(SetInstr::Prepend); 462 | } else if keyword_str == "touch" { 463 | return self.parse_cmd_touch(); 464 | } else if keyword_str == "incr" { 465 | return self.parse_cmd_inc(IncInstr::Incr); 466 | } else if keyword_str == "decr" { 467 | return self.parse_cmd_inc(IncInstr::Decr); 468 | } else if keyword_str == "delete" { 469 | return self.parse_cmd_delete(); 470 | } else if keyword_str == "flush_all" { 471 | return self.parse_cmd_flush_all(); 472 | } else if keyword_str == "stats" { 473 | return Ok(Cmd::Stats); 474 | } else if keyword_str == "version" { 475 | return Ok(Cmd::Version); 476 | } else if keyword_str == "quit" { 477 | return Ok(Cmd::Quit); 478 | } 479 | 480 | Err(TcpTransportError::InvalidCmd) 481 | } 482 | 483 | pub fn write_resp(&mut self, resp: &Resp) -> TcpTransportResult<()> { 484 | match *resp { 485 | Resp::Empty => (), 486 | Resp::ClientError(ref err) => { 487 | try!(self.write_string("CLIENT_ERROR ")); 488 | try!(self.write_string(&err)); 489 | try!(self.write_string("\r\n")); 490 | } 491 | Resp::Deleted => { 492 | try!(self.write_string("DELETED\r\n")); 493 | } 494 | Resp::Error => { 495 | try!(self.write_string("ERROR\r\n")); 496 | } 497 | Resp::Exists => { 498 | try!(self.write_string("EXISTS\r\n")); 499 | } 500 | Resp::IntValue(ref val) => { 501 | try!(self.write_string(&val.to_string())); 502 | try!(self.write_string("\r\n")); 503 | } 504 | Resp::NotFound => { 505 | try!(self.write_string("NOT_FOUND\r\n")); 506 | } 507 | Resp::NotStored => { 508 | try!(self.write_string("NOT_STORED\r\n")); 509 | } 510 | Resp::Ok => { 511 | try!(self.write_string("OK\r\n")); 512 | } 513 | Resp::ServerError(ref err) => { 514 | try!(self.write_string("SERVER_ERROR ")); 515 | try!(self.write_string(&err)); 516 | try!(self.write_string("\r\n")); 517 | } 518 | Resp::Stats(ref stats) => { 519 | for stat in stats { 520 | try!(self.write_string("STAT ")); 521 | try!(self.write_string(&stat.key)); 522 | try!(self.write_string(" ")); 523 | try!(self.write_string(&stat.value)); 524 | try!(self.write_string("\r\n")); 525 | } 526 | try!(self.write_string("END\r\n")); 527 | } 528 | Resp::Stored => { 529 | try!(self.write_string("STORED\r\n")); 530 | } 531 | Resp::Touched => { 532 | try!(self.write_string("TOUCHED\r\n")); 533 | } 534 | Resp::Values(ref values) => { 535 | for value in values { 536 | try!(self.write_string("VALUE ")); // keyword 537 | try!(self.write_string(&value.key)); // key 538 | try!(self.write_string(" ")); // space 539 | try!(self.write_string(&value.flags.to_string())); // flags 540 | try!(self.write_string(" ")); // space 541 | try!(self.write_string(&value.data.len().to_string())); // bytelen 542 | if !value.cas_unique.is_none() { 543 | try!(self.write_string(" ")); // space 544 | try!(self.write_string(&value.cas_unique 545 | .unwrap() 546 | .to_string())); // flags 547 | } 548 | try!(self.write_string(&"\r\n".to_string())); // newline 549 | try!(self.write_bytes(&value.data)); // data block 550 | try!(self.write_string(&"\r\n".to_string())); // newline 551 | } 552 | try!(self.write_string(&"END\r\n".to_string())); // END + newline 553 | } 554 | Resp::Version(ref version) => { 555 | try!(self.write_string("VERSION ")); 556 | try!(self.write_string(&version)); // key 557 | try!(self.write_string(&"\r\n".to_string())); // newline 558 | } 559 | } 560 | 561 | // Make sure all bytes were actually sent 562 | self.flush_writes() 563 | } 564 | } 565 | -------------------------------------------------------------------------------- /src/tcp_transport/typedefs.rs: -------------------------------------------------------------------------------- 1 | use super::errors::TcpTransportError; 2 | 3 | 4 | pub type TcpTransportResult = Result; 5 | -------------------------------------------------------------------------------- /src/testlib/cmp.rs: -------------------------------------------------------------------------------- 1 | pub fn eq_f64(x: f64, y: f64, error: f64) -> bool { 2 | (x - y).abs() < error 3 | } 4 | 5 | 6 | #[cfg(test)] 7 | mod tests { 8 | use super::eq_f64; 9 | 10 | 11 | #[test] 12 | fn test_eq_f64() { 13 | // Actual equality (need to supply a >0 error though) 14 | assert!(eq_f64(1.1, 1.1, 0.000000001)); 15 | 16 | // The second is smaller 17 | assert!(eq_f64(1.1, 1.0, 1.000000001)); 18 | 19 | // The second is greater 20 | assert!(eq_f64(1.1, 1.2, 1.000000001)); 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/testlib/datagen.rs: -------------------------------------------------------------------------------- 1 | use rand::Rng; 2 | use rand::thread_rng; 3 | 4 | 5 | pub fn get_rand_f64_vec(lower: u64, upper: u64) -> Vec { 6 | // create the floats 7 | let mut items: Vec = (lower..upper + 1) 8 | .map(|x: u64| x as f64) 9 | .collect(); 10 | 11 | // now shuffle them 12 | thread_rng().shuffle(&mut items); 13 | 14 | items 15 | } 16 | 17 | 18 | #[cfg(test)] 19 | mod tests { 20 | use std::cmp::Ordering; 21 | 22 | use super::get_rand_f64_vec; 23 | 24 | 25 | #[test] 26 | fn test_get_rand_f64_vec() { 27 | // construct a sorted version of the same vector 28 | let expected: Vec = (1..101).map(|x: u64| x as f64).collect(); 29 | 30 | // get some random vectors 31 | let mut vec = get_rand_f64_vec(1, 100); 32 | let vec2 = get_rand_f64_vec(1, 100); 33 | 34 | // it isn't constant 35 | assert!(vec != vec2); 36 | 37 | // it isn't sorted 38 | assert!(vec != expected); 39 | 40 | // it has the right size 41 | assert_eq!(vec.len(), 100); 42 | 43 | // now sort the vector 44 | vec.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); 45 | 46 | // the sorted vector is now equal to the one we made ourselves 47 | assert_eq!(vec, expected); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/testlib/mod.rs: -------------------------------------------------------------------------------- 1 | // Declare sub modules 2 | pub mod cmp; 3 | pub mod datagen; 4 | pub mod test_stream; 5 | -------------------------------------------------------------------------------- /src/testlib/test_stream.rs: -------------------------------------------------------------------------------- 1 | use std::cmp; 2 | use std::io::Read; 3 | use std::io::Write; 4 | use std::io::Result; 5 | 6 | 7 | // A stream that is seeded with incoming data which can be consumed and 8 | // records data written to it (like a socket). 9 | // 10 | // This allows us to unit test a transport without using sockets. :) 11 | pub struct TestStream { 12 | pub incoming: Vec, 13 | pub incoming_cursor: usize, 14 | 15 | pub repeating_mode: bool, // Replay the same bytes over and over 16 | pub incoming_rep: Vec, 17 | pub incoming_rep_cursor: usize, 18 | 19 | pub outgoing: Vec, 20 | } 21 | 22 | impl TestStream { 23 | pub fn new(incoming: Vec) -> TestStream { 24 | TestStream { 25 | incoming: incoming, 26 | incoming_cursor: 0, 27 | 28 | repeating_mode: false, 29 | incoming_rep: vec![], 30 | incoming_rep_cursor: 0, 31 | 32 | // Should be a good fit for most test responses 33 | outgoing: Vec::with_capacity(200), 34 | } 35 | } 36 | 37 | pub fn set_incoming_rep(&mut self, buffer: Vec) -> &mut Self { 38 | self.incoming_rep_cursor = 0; 39 | self.incoming_rep = buffer; 40 | self.repeating_mode = true; 41 | self 42 | } 43 | 44 | 45 | fn read_repeating(&mut self, buf: &mut [u8]) -> Result { 46 | let read_len = buf.len(); 47 | 48 | let mut read_cnt: usize = 0; 49 | let mut buf_cur: usize = 0; 50 | 51 | while read_cnt < read_len { 52 | // We're going to read until the end of the buffer, then loop 53 | let left_to_read = cmp::min(read_len - read_cnt, 54 | self.incoming_rep.len() - 55 | self.incoming_rep_cursor); 56 | 57 | for i in 0..left_to_read { 58 | buf[buf_cur + i] = self.incoming_rep[self.incoming_rep_cursor + 59 | i]; 60 | } 61 | 62 | read_cnt += left_to_read; 63 | buf_cur += left_to_read; 64 | self.incoming_rep_cursor += left_to_read; 65 | 66 | // Have we reached the end? Reset the cursor 67 | if self.incoming_rep_cursor == self.incoming_rep.len() { 68 | self.incoming_rep_cursor = 0; 69 | } 70 | } 71 | 72 | Ok(read_cnt) 73 | } 74 | 75 | fn read_linear(&mut self, buf: &mut [u8]) -> Result { 76 | // We can only read either as much as we have in incoming, or as big as 77 | // the output buffer is. 78 | let read_len = cmp::min(buf.len(), 79 | self.incoming.len() - self.incoming_cursor); 80 | 81 | for i in 0..read_len { 82 | buf[i] = self.incoming[self.incoming_cursor + i]; 83 | } 84 | self.incoming_cursor += read_len; 85 | 86 | Ok(read_len) 87 | } 88 | } 89 | 90 | impl Read for TestStream { 91 | fn read(&mut self, buf: &mut [u8]) -> Result { 92 | if self.repeating_mode { 93 | return self.read_repeating(buf); 94 | } else { 95 | return self.read_linear(buf); 96 | } 97 | } 98 | } 99 | 100 | impl Write for TestStream { 101 | fn write(&mut self, buf: &[u8]) -> Result { 102 | self.outgoing.extend(buf.iter().cloned()); 103 | 104 | Ok(buf.len()) 105 | } 106 | 107 | fn flush(&mut self) -> Result<()> { 108 | Ok(()) 109 | } 110 | } 111 | 112 | 113 | #[cfg(test)] 114 | mod tests { 115 | use std::io::Read; 116 | use std::io::Write; 117 | 118 | use super::TestStream; 119 | 120 | 121 | #[test] 122 | fn test_stream_read_whole() { 123 | let mut ts = TestStream::new(vec![1, 2, 3]); 124 | 125 | let mut buf = [0; 4]; 126 | let read_cnt = ts.read(&mut buf).unwrap(); 127 | assert_eq!(buf, [1, 2, 3, 0]); 128 | assert_eq!(read_cnt, 3); 129 | assert_eq!(ts.incoming, [1, 2, 3]); 130 | assert_eq!(ts.incoming_cursor, 3); 131 | } 132 | 133 | #[test] 134 | fn test_stream_read_incremental() { 135 | let mut ts = TestStream::new(vec![1, 2, 3]); 136 | 137 | // Read once 138 | let mut buf = [0; 2]; 139 | let read_cnt = ts.read(&mut buf).unwrap(); 140 | assert_eq!(read_cnt, 2); 141 | assert_eq!(buf, [1, 2]); 142 | assert_eq!(ts.incoming, [1, 2, 3]); 143 | assert_eq!(ts.incoming_cursor, 2); 144 | 145 | // Read once more 146 | let mut buf = [0; 2]; 147 | let read_cnt = ts.read(&mut buf).unwrap(); 148 | assert_eq!(read_cnt, 1); 149 | assert_eq!(buf, [3, 0]); 150 | assert_eq!(ts.incoming, [1, 2, 3]); 151 | assert_eq!(ts.incoming_cursor, 3); 152 | } 153 | 154 | #[test] 155 | fn test_stream_read_repeating() { 156 | let mut ts = TestStream::new(vec![]); 157 | ts.set_incoming_rep(vec![1, 2, 3]); 158 | 159 | // Read once 160 | let mut buf = [0; 2]; 161 | let read_cnt = ts.read(&mut buf).unwrap(); 162 | assert_eq!(read_cnt, 2); 163 | assert_eq!(buf, [1, 2]); 164 | assert_eq!(ts.incoming_rep, [1, 2, 3]); 165 | assert_eq!(ts.incoming_rep_cursor, 2); 166 | 167 | // Read once more 168 | let mut buf = [0; 5]; 169 | let read_cnt = ts.read(&mut buf).unwrap(); 170 | assert_eq!(read_cnt, 5); 171 | assert_eq!(buf, [3, 1, 2, 3, 1]); 172 | assert_eq!(ts.incoming_rep, [1, 2, 3]); 173 | assert_eq!(ts.incoming_rep_cursor, 1); 174 | } 175 | 176 | #[test] 177 | fn test_stream_write() { 178 | let mut ts = TestStream::new(vec![]); 179 | 180 | // Write once 181 | let buf = [1, 2]; 182 | let write_cnt = ts.write(&buf).unwrap(); 183 | assert_eq!(write_cnt, 2); 184 | assert_eq!(ts.outgoing, [1, 2]); 185 | 186 | // Write once more 187 | let buf = [3, 4, 5]; 188 | let write_cnt = ts.write(&buf).unwrap(); 189 | assert_eq!(write_cnt, 3); 190 | assert_eq!(ts.outgoing, [1, 2, 3, 4, 5]); 191 | } 192 | } 193 | --------------------------------------------------------------------------------