├── chapter3 ├── test └── server │ ├── http │ ├── tcp │ ├── cache │ ├── stat.go │ ├── cache.go │ ├── inmemory.go │ ├── new.go │ ├── rocksdb.go │ ├── rocksdb_del.go │ ├── rocksdb_set.go │ ├── rocksdb_get.go │ ├── rocksdb_new.go │ └── rocksdb_getstat.go │ └── main.go ├── chapter4 ├── server └── test │ └── test.sh ├── chapter2 ├── server │ ├── cache │ ├── http │ ├── main.go │ └── tcp │ │ ├── new.go │ │ ├── utils.go │ │ ├── read_key.go │ │ └── process.go └── test │ └── test.sh ├── chapter5 ├── server │ ├── http │ ├── tcp │ ├── main.go │ └── cache │ │ ├── new.go │ │ ├── stat.go │ │ ├── cache.go │ │ ├── inmemory.go │ │ ├── rocksdb_del.go │ │ ├── rocksdb_get.go │ │ ├── rocksdb_getstat.go │ │ ├── pair.go │ │ ├── rocksdb.go │ │ ├── rocksdb_new.go │ │ └── rocksdb_set.go └── test │ └── test.sh ├── chapter6 ├── server │ ├── cache │ ├── http │ ├── main.go │ └── tcp │ │ ├── new.go │ │ ├── utils.go │ │ ├── read_key.go │ │ └── process.go └── test │ └── test.sh ├── chapter7 ├── server │ ├── cache │ ├── http │ │ ├── cache.go │ │ ├── status.go │ │ ├── server.go │ │ └── cluster.go │ ├── tcp │ │ ├── utils.go │ │ ├── process.go │ │ ├── new.go │ │ └── read_key.go │ ├── main.go │ └── cluster │ │ └── cluster.go └── test │ └── test.sh ├── chapter8 ├── server │ ├── tcp │ ├── cluster │ ├── main.go │ ├── cache │ │ ├── new.go │ │ ├── pair.go │ │ ├── stat.go │ │ ├── rocksdb.go │ │ ├── inmemory.go │ │ ├── rocksdb_del.go │ │ ├── rocksdb_get.go │ │ ├── rocksdb_new.go │ │ ├── rocksdb_set.go │ │ ├── rocksdb_getstat.go │ │ ├── scanner.go │ │ ├── cache.go │ │ ├── inmemory_scanner.go │ │ └── rocksdb_scanner.go │ └── http │ │ ├── cache.go │ │ ├── cluster.go │ │ ├── status.go │ │ ├── server.go │ │ └── rebalance.go └── test │ └── test.sh ├── chapter9 ├── server │ ├── http │ ├── tcp │ ├── cluster │ ├── cache │ │ ├── pair.go │ │ ├── stat.go │ │ ├── cache.go │ │ ├── rocksdb.go │ │ ├── scanner.go │ │ ├── rocksdb_del.go │ │ ├── rocksdb_get.go │ │ ├── rocksdb_set.go │ │ ├── rocksdb_getstat.go │ │ ├── rocksdb_scanner.go │ │ ├── new.go │ │ ├── rocksdb_new.go │ │ ├── inmemory_scanner.go │ │ └── inmemory.go │ └── main.go └── test │ └── test.sh ├── .gitmodules ├── chapter1 ├── server │ ├── cache │ │ ├── cache.go │ │ ├── new.go │ │ ├── stat.go │ │ └── inmemory.go │ ├── main.go │ └── http │ │ ├── server.go │ │ ├── status.go │ │ └── cache.go └── test │ └── test.sh ├── .gitignore ├── README ├── cache-benchmark ├── cacheClient │ ├── client.go │ ├── http.go │ ├── redis.go │ └── tcp.go └── main.go ├── client └── main.go ├── rocksdb_performance ├── Makefile ├── ingest_data.cpp ├── test_basic.cpp ├── test_batch_write.cpp └── common.h └── LICENSE /chapter3/test: -------------------------------------------------------------------------------- 1 | ../chapter2/test -------------------------------------------------------------------------------- /chapter4/server: -------------------------------------------------------------------------------- 1 | ../chapter3/server -------------------------------------------------------------------------------- /chapter2/server/cache: -------------------------------------------------------------------------------- 1 | ../../chapter1/server/cache -------------------------------------------------------------------------------- /chapter2/server/http: -------------------------------------------------------------------------------- 1 | ../../chapter1/server/http -------------------------------------------------------------------------------- /chapter3/server/http: -------------------------------------------------------------------------------- 1 | ../../chapter1/server/http -------------------------------------------------------------------------------- /chapter3/server/tcp: -------------------------------------------------------------------------------- 1 | ../../chapter2/server/tcp -------------------------------------------------------------------------------- /chapter5/server/http: -------------------------------------------------------------------------------- 1 | ../../chapter1/server/http -------------------------------------------------------------------------------- /chapter5/server/tcp: -------------------------------------------------------------------------------- 1 | ../../chapter2/server/tcp -------------------------------------------------------------------------------- /chapter6/server/cache: -------------------------------------------------------------------------------- 1 | ../../chapter5/server/cache -------------------------------------------------------------------------------- /chapter6/server/http: -------------------------------------------------------------------------------- 1 | ../../chapter1/server/http -------------------------------------------------------------------------------- /chapter7/server/cache: -------------------------------------------------------------------------------- 1 | ../../chapter5/server/cache -------------------------------------------------------------------------------- /chapter8/server/tcp: -------------------------------------------------------------------------------- 1 | ../../chapter7/server/tcp -------------------------------------------------------------------------------- /chapter9/server/http: -------------------------------------------------------------------------------- 1 | ../../chapter8/server/http -------------------------------------------------------------------------------- /chapter9/server/tcp: -------------------------------------------------------------------------------- 1 | ../../chapter7/server/tcp -------------------------------------------------------------------------------- /chapter5/server/main.go: -------------------------------------------------------------------------------- 1 | ../../chapter3/server/main.go -------------------------------------------------------------------------------- /chapter6/server/main.go: -------------------------------------------------------------------------------- 1 | ../../chapter3/server/main.go -------------------------------------------------------------------------------- /chapter8/server/cluster: -------------------------------------------------------------------------------- 1 | ../../chapter7/server/cluster -------------------------------------------------------------------------------- /chapter8/server/main.go: -------------------------------------------------------------------------------- 1 | ../../chapter7/server/main.go -------------------------------------------------------------------------------- /chapter9/server/cluster: -------------------------------------------------------------------------------- 1 | ../../chapter7/server/cluster -------------------------------------------------------------------------------- /chapter6/server/tcp/new.go: -------------------------------------------------------------------------------- 1 | ../../../chapter2/server/tcp/new.go -------------------------------------------------------------------------------- /chapter3/server/cache/stat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/stat.go -------------------------------------------------------------------------------- /chapter5/server/cache/new.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/new.go -------------------------------------------------------------------------------- /chapter5/server/cache/stat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/stat.go -------------------------------------------------------------------------------- /chapter6/server/tcp/utils.go: -------------------------------------------------------------------------------- 1 | ../../../chapter2/server/tcp/utils.go -------------------------------------------------------------------------------- /chapter7/server/http/cache.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/http/cache.go -------------------------------------------------------------------------------- /chapter7/server/tcp/utils.go: -------------------------------------------------------------------------------- 1 | ../../../chapter6/server/tcp/utils.go -------------------------------------------------------------------------------- /chapter8/server/cache/new.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/new.go -------------------------------------------------------------------------------- /chapter8/server/cache/pair.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/pair.go -------------------------------------------------------------------------------- /chapter8/server/cache/stat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/stat.go -------------------------------------------------------------------------------- /chapter8/server/http/cache.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/http/cache.go -------------------------------------------------------------------------------- /chapter9/server/cache/pair.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/pair.go -------------------------------------------------------------------------------- /chapter9/server/cache/stat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/stat.go -------------------------------------------------------------------------------- /chapter3/server/cache/cache.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/cache.go -------------------------------------------------------------------------------- /chapter5/server/cache/cache.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/cache.go -------------------------------------------------------------------------------- /chapter6/server/tcp/read_key.go: -------------------------------------------------------------------------------- 1 | ../../../chapter2/server/tcp/read_key.go -------------------------------------------------------------------------------- /chapter7/server/http/status.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/http/status.go -------------------------------------------------------------------------------- /chapter7/server/tcp/process.go: -------------------------------------------------------------------------------- 1 | ../../../chapter6/server/tcp/process.go -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/rocksdb.go -------------------------------------------------------------------------------- /chapter8/server/http/cluster.go: -------------------------------------------------------------------------------- 1 | ../../../chapter7/server/http/cluster.go -------------------------------------------------------------------------------- /chapter8/server/http/status.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/http/status.go -------------------------------------------------------------------------------- /chapter9/server/cache/cache.go: -------------------------------------------------------------------------------- 1 | ../../../chapter8/server/cache/cache.go -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/rocksdb.go -------------------------------------------------------------------------------- /chapter9/server/cache/scanner.go: -------------------------------------------------------------------------------- 1 | ../../../chapter8/server/cache/scanner.go -------------------------------------------------------------------------------- /chapter3/server/cache/inmemory.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/inmemory.go -------------------------------------------------------------------------------- /chapter5/server/cache/inmemory.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/inmemory.go -------------------------------------------------------------------------------- /chapter8/server/cache/inmemory.go: -------------------------------------------------------------------------------- 1 | ../../../chapter1/server/cache/inmemory.go -------------------------------------------------------------------------------- /chapter5/server/cache/rocksdb_del.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_del.go -------------------------------------------------------------------------------- /chapter5/server/cache/rocksdb_get.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_get.go -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb_del.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_del.go -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb_get.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_get.go -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb_new.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/rocksdb_new.go -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb_set.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/rocksdb_set.go -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb_del.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_del.go -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb_get.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_get.go -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb_set.go: -------------------------------------------------------------------------------- 1 | ../../../chapter5/server/cache/rocksdb_set.go -------------------------------------------------------------------------------- /chapter5/server/cache/rocksdb_getstat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_getstat.go -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb_getstat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_getstat.go -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb_getstat.go: -------------------------------------------------------------------------------- 1 | ../../../chapter3/server/cache/rocksdb_getstat.go -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb_scanner.go: -------------------------------------------------------------------------------- 1 | ../../../chapter8/server/cache/rocksdb_scanner.go -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "rocksdb"] 2 | path = rocksdb 3 | url = https://github.com/facebook/rocksdb 4 | -------------------------------------------------------------------------------- /chapter5/server/cache/pair.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type pair struct { 4 | k string 5 | v []byte 6 | } 7 | -------------------------------------------------------------------------------- /chapter4/test/test.sh: -------------------------------------------------------------------------------- 1 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t set -P 3 2 | 3 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t get -P 3 4 | -------------------------------------------------------------------------------- /chapter8/server/cache/scanner.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Scanner interface { 4 | Scan() bool 5 | Key() string 6 | Value() []byte 7 | Close() 8 | } 9 | -------------------------------------------------------------------------------- /chapter1/server/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Cache interface { 4 | Set(string, []byte) error 5 | Get(string) ([]byte, error) 6 | Del(string) error 7 | GetStat() Stat 8 | } 9 | -------------------------------------------------------------------------------- /chapter1/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./cache" 5 | "./http" 6 | ) 7 | 8 | func main() { 9 | c := cache.New("inmemory") 10 | http.New(c).Listen() 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pkg/ 2 | src/ 3 | */server/server 4 | cache-benchmark/cache-benchmark 5 | client/client 6 | rocksdb_performance/test_basic 7 | rocksdb_performance/test_batch_write 8 | rocksdb_performance/ingest_data 9 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | export GOPATH=`pwd` 2 | 3 | go get github.com/go-redis/redis 4 | go get github.com/hashicorp/memberlist 5 | go get stathat.com/c/consistent 6 | 7 | git submodule update --init 8 | cd rocksdb && make static_lib 9 | -------------------------------------------------------------------------------- /chapter8/server/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Cache interface { 4 | Set(string, []byte) error 5 | Get(string) ([]byte, error) 6 | Del(string) error 7 | GetStat() Stat 8 | NewScanner() Scanner 9 | } 10 | -------------------------------------------------------------------------------- /chapter2/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./cache" 5 | "./http" 6 | "./tcp" 7 | ) 8 | 9 | func main() { 10 | ca := cache.New("inmemory") 11 | go tcp.New(ca).Listen() 12 | http.New(ca).Listen() 13 | } 14 | -------------------------------------------------------------------------------- /chapter9/test/test.sh: -------------------------------------------------------------------------------- 1 | # ./server 2 | 3 | curl 127.0.0.1:12345/cache/a -XPUT -daa 4 | 5 | curl 127.0.0.1:12345/cache/a 6 | 7 | curl 127.0.0.1:12345/status 8 | 9 | # wait 30 seconds 10 | 11 | curl 127.0.0.1:12345/cache/a 12 | 13 | curl 127.0.0.1:12345/status 14 | -------------------------------------------------------------------------------- /chapter5/test/test.sh: -------------------------------------------------------------------------------- 1 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t set 2 | 3 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t set -P 3 4 | 5 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t set -c 50 6 | 7 | redis-benchmark -c 50 -n 100000 -d 1000 -t set -r 100000 -P 1 8 | -------------------------------------------------------------------------------- /chapter6/test/test.sh: -------------------------------------------------------------------------------- 1 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t get -P 10 2 | 3 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t get -P 100 4 | 5 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t get -c 50 6 | 7 | redis-benchmark -c 50 -n 100000 -d 1000 -t get -r 100000 -P 1 8 | -------------------------------------------------------------------------------- /chapter1/server/cache/new.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "log" 4 | 5 | func New(typ string) Cache { 6 | var c Cache 7 | if typ == "inmemory" { 8 | c = newInMemoryCache() 9 | } 10 | if c == nil { 11 | panic("unknown cache type " + typ) 12 | } 13 | log.Println(typ, "ready to serve") 14 | return c 15 | } 16 | -------------------------------------------------------------------------------- /chapter2/test/test.sh: -------------------------------------------------------------------------------- 1 | ./client -c set -k testkey -v testvalue 2 | 3 | ./client -c get -k testkey 4 | 5 | curl 127.0.0.1:12345/status 6 | 7 | ./client -c del -k testkey 8 | 9 | curl 127.0.0.1:12345/status 10 | 11 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t set 12 | 13 | ./cache-benchmark -type tcp -n 100000 -r 100000 -t get 14 | -------------------------------------------------------------------------------- /chapter3/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./cache" 5 | "./http" 6 | "./tcp" 7 | "flag" 8 | "log" 9 | ) 10 | 11 | func main() { 12 | typ := flag.String("type", "inmemory", "cache type") 13 | flag.Parse() 14 | log.Println("type is", *typ) 15 | c := cache.New(*typ) 16 | go tcp.New(c).Listen() 17 | http.New(c).Listen() 18 | } 19 | -------------------------------------------------------------------------------- /chapter3/server/cache/new.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "log" 4 | 5 | func New(typ string) Cache { 6 | var c Cache 7 | if typ == "inmemory" { 8 | c = newInMemoryCache() 9 | } 10 | if typ == "rocksdb" { 11 | c = newRocksdbCache() 12 | } 13 | if c == nil { 14 | panic("unknown cache type " + typ) 15 | } 16 | log.Println(typ, "ready to serve") 17 | return c 18 | } 19 | -------------------------------------------------------------------------------- /chapter9/server/cache/new.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "log" 4 | 5 | func New(typ string, ttl int) Cache { 6 | var c Cache 7 | if typ == "inmemory" { 8 | c = newInMemoryCache(ttl) 9 | } 10 | if typ == "rocksdb" { 11 | c = newRocksdbCache(ttl) 12 | } 13 | if c == nil { 14 | panic("unknown cache type " + typ) 15 | } 16 | log.Println(typ, "ready to serve") 17 | return c 18 | } 19 | -------------------------------------------------------------------------------- /chapter3/server/cache/rocksdb.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include "rocksdb/c.h" 4 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 5 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 6 | import "C" 7 | 8 | type rocksdbCache struct { 9 | db *C.rocksdb_t 10 | ro *C.rocksdb_readoptions_t 11 | wo *C.rocksdb_writeoptions_t 12 | e *C.char 13 | } 14 | -------------------------------------------------------------------------------- /chapter1/server/http/server.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "../cache" 5 | "net/http" 6 | ) 7 | 8 | type Server struct { 9 | cache.Cache 10 | } 11 | 12 | func (s *Server) Listen() { 13 | http.Handle("/cache/", s.cacheHandler()) 14 | http.Handle("/status", s.statusHandler()) 15 | http.ListenAndServe(":12345", nil) 16 | } 17 | 18 | func New(c cache.Cache) *Server { 19 | return &Server{c} 20 | } 21 | -------------------------------------------------------------------------------- /chapter1/server/cache/stat.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Stat struct { 4 | Count int64 5 | KeySize int64 6 | ValueSize int64 7 | } 8 | 9 | func (s *Stat) add(k string, v []byte) { 10 | s.Count += 1 11 | s.KeySize += int64(len(k)) 12 | s.ValueSize += int64(len(v)) 13 | } 14 | 15 | func (s *Stat) del(k string, v []byte) { 16 | s.Count -= 1 17 | s.KeySize -= int64(len(k)) 18 | s.ValueSize -= int64(len(v)) 19 | } 20 | -------------------------------------------------------------------------------- /chapter5/server/cache/rocksdb.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 6 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 7 | import "C" 8 | 9 | type rocksdbCache struct { 10 | db *C.rocksdb_t 11 | ro *C.rocksdb_readoptions_t 12 | wo *C.rocksdb_writeoptions_t 13 | e *C.char 14 | ch chan *pair 15 | } 16 | -------------------------------------------------------------------------------- /chapter2/server/tcp/new.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "../cache" 5 | "net" 6 | ) 7 | 8 | type Server struct { 9 | cache.Cache 10 | } 11 | 12 | func (s *Server) Listen() { 13 | l, e := net.Listen("tcp", ":12346") 14 | if e != nil { 15 | panic(e) 16 | } 17 | for { 18 | c, e := l.Accept() 19 | if e != nil { 20 | panic(e) 21 | } 22 | go s.process(c) 23 | } 24 | } 25 | 26 | func New(c cache.Cache) *Server { 27 | return &Server{c} 28 | } 29 | -------------------------------------------------------------------------------- /chapter1/test/test.sh: -------------------------------------------------------------------------------- 1 | curl 127.0.0.1:12345/status 2 | 3 | curl -v 127.0.0.1:12345/cache/testkey -XPUT -dtestvalue 4 | 5 | curl 127.0.0.1:12345/cache/testkey 6 | 7 | curl 127.0.0.1:12345/status 8 | 9 | curl 127.0.0.1:12345/cache/testkey -XDELETE 10 | 11 | curl 127.0.0.1:12345/status 12 | 13 | ./cache-benchmark -type http -n 100000 -r 100000 -t set 14 | 15 | ./cache-benchmark -type http -n 100000 -r 100000 -t get 16 | 17 | redis-benchmark -c 1 -n 100000 -d 1000 -t set,get -r 100000 18 | -------------------------------------------------------------------------------- /chapter7/server/http/server.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "../cache" 5 | "../cluster" 6 | "net/http" 7 | ) 8 | 9 | type Server struct { 10 | cache.Cache 11 | cluster.Node 12 | } 13 | 14 | func (s *Server) Listen() { 15 | http.Handle("/cache/", s.cacheHandler()) 16 | http.Handle("/status", s.statusHandler()) 17 | http.Handle("/cluster", s.clusterHandler()) 18 | http.ListenAndServe(s.Addr()+":12345", nil) 19 | } 20 | 21 | func New(c cache.Cache, n cluster.Node) *Server { 22 | return &Server{c, n} 23 | } 24 | -------------------------------------------------------------------------------- /chapter7/server/tcp/new.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "../cache" 5 | "../cluster" 6 | "net" 7 | ) 8 | 9 | type Server struct { 10 | cache.Cache 11 | cluster.Node 12 | } 13 | 14 | func (s *Server) Listen() { 15 | l, e := net.Listen("tcp", s.Addr()+":12346") 16 | if e != nil { 17 | panic(e) 18 | } 19 | for { 20 | c, e := l.Accept() 21 | if e != nil { 22 | panic(e) 23 | } 24 | go s.process(c) 25 | } 26 | } 27 | 28 | func New(c cache.Cache, n cluster.Node) *Server { 29 | return &Server{c, n} 30 | } 31 | -------------------------------------------------------------------------------- /cache-benchmark/cacheClient/client.go: -------------------------------------------------------------------------------- 1 | package cacheClient 2 | 3 | type Cmd struct { 4 | Name string 5 | Key string 6 | Value string 7 | Error error 8 | } 9 | 10 | type Client interface { 11 | Run(*Cmd) 12 | PipelinedRun([]*Cmd) 13 | } 14 | 15 | func New(typ, server string) Client { 16 | if typ == "redis" { 17 | return newRedisClient(server) 18 | } 19 | if typ == "http" { 20 | return newHTTPClient(server) 21 | } 22 | if typ == "tcp" { 23 | return newTCPClient(server) 24 | } 25 | panic("unknown client type " + typ) 26 | } 27 | -------------------------------------------------------------------------------- /chapter8/server/http/server.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "../cache" 5 | "../cluster" 6 | "net/http" 7 | ) 8 | 9 | type Server struct { 10 | cache.Cache 11 | cluster.Node 12 | } 13 | 14 | func (s *Server) Listen() { 15 | http.Handle("/cache/", s.cacheHandler()) 16 | http.Handle("/status", s.statusHandler()) 17 | http.Handle("/cluster", s.clusterHandler()) 18 | http.Handle("/rebalance", s.rebalanceHandler()) 19 | http.ListenAndServe(s.Addr()+":12345", nil) 20 | } 21 | 22 | func New(c cache.Cache, n cluster.Node) *Server { 23 | return &Server{c, n} 24 | } 25 | -------------------------------------------------------------------------------- /chapter8/test/test.sh: -------------------------------------------------------------------------------- 1 | # ./server -node 1.1.1.1 2 | 3 | ./cache-benchmark -type tcp -n 10000 -d 1 -h 1.1.1.1 4 | 5 | curl 1.1.1.1:12345/status 6 | 7 | # ./server -node 1.1.1.2 -cluster 1.1.1.1 8 | 9 | curl 1.1.1.2:12345/status 10 | 11 | curl 1.1.1.1:12345/rebalance -XPOST 12 | 13 | curl 1.1.1.1:12345/status 14 | 15 | curl 1.1.1.2:12345/status 16 | 17 | # ./server -node 1.1.1.3 -cluster 1.1.1.2 18 | 19 | curl 1.1.1.1:12345/rebalance -XPOST 20 | 21 | curl 1.1.1.2:12345/rebalance -XPOST 22 | 23 | curl 1.1.1.1:12345/status 24 | 25 | curl 1.1.1.2:12345/status 26 | 27 | curl 1.1.1.3:12345/status 28 | -------------------------------------------------------------------------------- /chapter3/server/cache/rocksdb_del.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 6 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 7 | import "C" 8 | import ( 9 | "errors" 10 | "unsafe" 11 | ) 12 | 13 | func (c *rocksdbCache) Del(key string) error { 14 | k := C.CString(key) 15 | defer C.free(unsafe.Pointer(k)) 16 | C.rocksdb_delete(c.db, c.wo, k, C.size_t(len(key)), &c.e) 17 | if c.e != nil { 18 | return errors.New(C.GoString(c.e)) 19 | } 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "../cache-benchmark/cacheClient" 5 | "flag" 6 | "fmt" 7 | ) 8 | 9 | func main() { 10 | server := flag.String("h", "localhost", "cache server address") 11 | op := flag.String("c", "get", "command, could be get/set/del") 12 | key := flag.String("k", "", "key") 13 | value := flag.String("v", "", "value") 14 | flag.Parse() 15 | client := cacheClient.New("tcp", *server) 16 | cmd := &cacheClient.Cmd{*op, *key, *value, nil} 17 | client.Run(cmd) 18 | if cmd.Error != nil { 19 | fmt.Println("error:", cmd.Error) 20 | } else { 21 | fmt.Println(cmd.Value) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /chapter1/server/http/status.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | ) 8 | 9 | type statusHandler struct { 10 | *Server 11 | } 12 | 13 | func (h *statusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 14 | if r.Method != http.MethodGet { 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | return 17 | } 18 | b, e := json.Marshal(h.GetStat()) 19 | if e != nil { 20 | log.Println(e) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | w.Write(b) 25 | } 26 | 27 | func (s *Server) statusHandler() http.Handler { 28 | return &statusHandler{s} 29 | } 30 | -------------------------------------------------------------------------------- /chapter7/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./cache" 5 | "./cluster" 6 | "./http" 7 | "./tcp" 8 | "flag" 9 | "log" 10 | ) 11 | 12 | func main() { 13 | typ := flag.String("type", "inmemory", "cache type") 14 | node := flag.String("node", "127.0.0.1", "node address") 15 | clus := flag.String("cluster", "", "cluster address") 16 | flag.Parse() 17 | log.Println("type is", *typ) 18 | log.Println("node is", *node) 19 | log.Println("cluster is", *clus) 20 | c := cache.New(*typ) 21 | n, e := cluster.New(*node, *clus) 22 | if e != nil { 23 | panic(e) 24 | } 25 | go tcp.New(c, n).Listen() 26 | http.New(c, n).Listen() 27 | } 28 | -------------------------------------------------------------------------------- /chapter7/server/http/cluster.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | ) 8 | 9 | type clusterHandler struct { 10 | *Server 11 | } 12 | 13 | func (h *clusterHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 14 | if r.Method != http.MethodGet { 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | return 17 | } 18 | m := h.Members() 19 | b, e := json.Marshal(m) 20 | if e != nil { 21 | log.Println(e) 22 | w.WriteHeader(http.StatusInternalServerError) 23 | return 24 | } 25 | w.Write(b) 26 | } 27 | 28 | func (s *Server) clusterHandler() http.Handler { 29 | return &clusterHandler{s} 30 | } 31 | -------------------------------------------------------------------------------- /rocksdb_performance/Makefile: -------------------------------------------------------------------------------- 1 | BIN = test_basic test_batch_write ingest_data 2 | CXXFLAGS = -I ../rocksdb/include/ -std=gnu++14 -L ../rocksdb -lrocksdb -lpthread -lz -lsnappy -lboost_program_options 3 | 4 | ifeq ($(MAKECMDGOALS), debug) 5 | CXXFLAGS += -g -ggdb 6 | else 7 | CXXFLAGS += -O3 8 | endif 9 | 10 | all: release 11 | 12 | debug: $(BIN) 13 | 14 | release: $(BIN) 15 | 16 | test_basic: test_basic.cpp common.h 17 | g++ -o $@ $< $(CXXFLAGS) 18 | 19 | test_batch_write: test_batch_write.cpp common.h 20 | g++ -o $@ $< $(CXXFLAGS) 21 | 22 | ingest_data: ingest_data.cpp common.h 23 | g++ -o $@ $< $(CXXFLAGS) 24 | 25 | clean: 26 | rm -f $(BIN) 27 | -------------------------------------------------------------------------------- /chapter3/server/cache/rocksdb_set.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 6 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 7 | import "C" 8 | import ( 9 | "errors" 10 | "unsafe" 11 | ) 12 | 13 | func (c *rocksdbCache) Set(key string, value []byte) error { 14 | k := C.CString(key) 15 | defer C.free(unsafe.Pointer(k)) 16 | v := C.CBytes(value) 17 | defer C.free(v) 18 | C.rocksdb_put(c.db, c.wo, k, C.size_t(len(key)), (*C.char)(v), C.size_t(len(value)), &c.e) 19 | if c.e != nil { 20 | return errors.New(C.GoString(c.e)) 21 | } 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /chapter3/server/cache/rocksdb_get.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 6 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 7 | import "C" 8 | import ( 9 | "errors" 10 | "unsafe" 11 | ) 12 | 13 | func (c *rocksdbCache) Get(key string) ([]byte, error) { 14 | k := C.CString(key) 15 | defer C.free(unsafe.Pointer(k)) 16 | var length C.size_t 17 | v := C.rocksdb_get(c.db, c.ro, k, C.size_t(len(key)), &length, &c.e) 18 | if c.e != nil { 19 | return nil, errors.New(C.GoString(c.e)) 20 | } 21 | defer C.free(unsafe.Pointer(v)) 22 | return C.GoBytes(unsafe.Pointer(v), C.int(length)), nil 23 | } 24 | -------------------------------------------------------------------------------- /chapter9/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./cache" 5 | "./cluster" 6 | "./http" 7 | "./tcp" 8 | "flag" 9 | "log" 10 | ) 11 | 12 | func main() { 13 | typ := flag.String("type", "inmemory", "cache type") 14 | ttl := flag.Int("ttl", 30, "cache time to live") 15 | node := flag.String("node", "127.0.0.1", "node address") 16 | clus := flag.String("cluster", "", "cluster address") 17 | flag.Parse() 18 | log.Println("type is", *typ) 19 | log.Println("ttl is", *ttl) 20 | log.Println("node is", *node) 21 | log.Println("cluster is", *clus) 22 | c := cache.New(*typ, *ttl) 23 | n, e := cluster.New(*node, *clus) 24 | if e != nil { 25 | panic(e) 26 | } 27 | go tcp.New(c, n).Listen() 28 | http.New(c, n).Listen() 29 | } 30 | -------------------------------------------------------------------------------- /chapter2/server/tcp/utils.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | func readLen(r *bufio.Reader) (int, error) { 12 | tmp, e := r.ReadString(' ') 13 | if e != nil { 14 | return 0, e 15 | } 16 | l, e := strconv.Atoi(strings.TrimSpace(tmp)) 17 | if e != nil { 18 | return 0, e 19 | } 20 | return l, nil 21 | } 22 | 23 | func sendResponse(value []byte, err error, conn net.Conn) error { 24 | if err != nil { 25 | errString := err.Error() 26 | tmp := fmt.Sprintf("-%d ", len(errString)) + errString 27 | _, e := conn.Write([]byte(tmp)) 28 | return e 29 | } 30 | vlen := fmt.Sprintf("%d ", len(value)) 31 | _, e := conn.Write(append([]byte(vlen), value...)) 32 | return e 33 | } 34 | -------------------------------------------------------------------------------- /chapter3/server/cache/rocksdb_new.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include "rocksdb/c.h" 4 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 5 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 6 | import "C" 7 | import "runtime" 8 | 9 | func newRocksdbCache() *rocksdbCache { 10 | options := C.rocksdb_options_create() 11 | C.rocksdb_options_increase_parallelism(options, C.int(runtime.NumCPU())) 12 | C.rocksdb_options_set_create_if_missing(options, 1) 13 | var e *C.char 14 | db := C.rocksdb_open(options, C.CString("/mnt/rocksdb"), &e) 15 | if e != nil { 16 | panic(C.GoString(e)) 17 | } 18 | C.rocksdb_options_destroy(options) 19 | return &rocksdbCache{db, C.rocksdb_readoptions_create(), C.rocksdb_writeoptions_create(), e} 20 | } 21 | -------------------------------------------------------------------------------- /chapter8/server/http/rebalance.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "net/http" 6 | ) 7 | 8 | type rebalanceHandler struct { 9 | *Server 10 | } 11 | 12 | func (h *rebalanceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 13 | if r.Method != http.MethodPost { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | } 16 | go h.rebalance() 17 | } 18 | 19 | func (h *rebalanceHandler) rebalance() { 20 | s := h.NewScanner() 21 | defer s.Close() 22 | c := &http.Client{} 23 | for s.Scan() { 24 | k := s.Key() 25 | n, ok := h.ShouldProcess(k) 26 | if !ok { 27 | r, _ := http.NewRequest(http.MethodPut, "http://"+n+":12345/cache/"+k, bytes.NewReader(s.Value())) 28 | c.Do(r) 29 | h.Del(k) 30 | } 31 | } 32 | } 33 | 34 | func (s *Server) rebalanceHandler() http.Handler { 35 | return &rebalanceHandler{s} 36 | } 37 | -------------------------------------------------------------------------------- /chapter5/server/cache/rocksdb_new.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include "rocksdb/c.h" 4 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 5 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 6 | import "C" 7 | import "runtime" 8 | 9 | func newRocksdbCache() *rocksdbCache { 10 | options := C.rocksdb_options_create() 11 | C.rocksdb_options_increase_parallelism(options, C.int(runtime.NumCPU())) 12 | C.rocksdb_options_set_create_if_missing(options, 1) 13 | var e *C.char 14 | db := C.rocksdb_open(options, C.CString("/mnt/rocksdb"), &e) 15 | if e != nil { 16 | panic(C.GoString(e)) 17 | } 18 | C.rocksdb_options_destroy(options) 19 | c := make(chan *pair, 5000) 20 | wo := C.rocksdb_writeoptions_create() 21 | go write_func(db, c, wo) 22 | return &rocksdbCache{db, C.rocksdb_readoptions_create(), wo, e, c} 23 | } 24 | -------------------------------------------------------------------------------- /chapter2/server/tcp/read_key.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | ) 7 | 8 | func (s *Server) readKey(r *bufio.Reader) (string, error) { 9 | klen, e := readLen(r) 10 | if e != nil { 11 | return "", e 12 | } 13 | k := make([]byte, klen) 14 | _, e = io.ReadFull(r, k) 15 | if e != nil { 16 | return "", e 17 | } 18 | return string(k), nil 19 | } 20 | 21 | func (s *Server) readKeyAndValue(r *bufio.Reader) (string, []byte, error) { 22 | klen, e := readLen(r) 23 | if e != nil { 24 | return "", nil, e 25 | } 26 | vlen, e := readLen(r) 27 | if e != nil { 28 | return "", nil, e 29 | } 30 | k := make([]byte, klen) 31 | _, e = io.ReadFull(r, k) 32 | if e != nil { 33 | return "", nil, e 34 | } 35 | v := make([]byte, vlen) 36 | _, e = io.ReadFull(r, v) 37 | if e != nil { 38 | return "", nil, e 39 | } 40 | return string(k), v, nil 41 | } 42 | -------------------------------------------------------------------------------- /chapter9/server/cache/rocksdb_new.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include "rocksdb/c.h" 4 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 5 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 6 | import "C" 7 | import "runtime" 8 | 9 | func newRocksdbCache(ttl int) *rocksdbCache { 10 | options := C.rocksdb_options_create() 11 | C.rocksdb_options_increase_parallelism(options, C.int(runtime.NumCPU())) 12 | C.rocksdb_options_set_create_if_missing(options, 1) 13 | var e *C.char 14 | db := C.rocksdb_open_with_ttl(options, C.CString("/mnt/rocksdb"), C.int(ttl), &e) 15 | if e != nil { 16 | panic(C.GoString(e)) 17 | } 18 | C.rocksdb_options_destroy(options) 19 | c := make(chan *pair, 5000) 20 | wo := C.rocksdb_writeoptions_create() 21 | go write_func(db, c, wo) 22 | return &rocksdbCache{db, C.rocksdb_readoptions_create(), wo, e, c} 23 | } 24 | -------------------------------------------------------------------------------- /chapter7/test/test.sh: -------------------------------------------------------------------------------- 1 | # ./server -node 1.1.1.1 2 | 3 | # ./server -node 1.1.1.2 -cluster 1.1.1.1 4 | 5 | ./client -h 1.1.1.1 -c set -k keya -v a 6 | 7 | ./client -h 1.1.1.1 -c set -k keyb -v b 8 | 9 | ./client -h 1.1.1.1 -c set -k keyc -v c 10 | 11 | ./client -h 1.1.1.1 -c set -k keyd -v d 12 | 13 | ./client -h 1.1.1.1 -c set -k keye -v e 14 | 15 | # ./server -node 1.1.1.3 -cluster 1.1.1.2 16 | 17 | ./client -h 1.1.1.1 -c set -k keya -v a 18 | 19 | ./client -h 1.1.1.1 -c set -k keyb -v b 20 | 21 | ./client -h 1.1.1.1 -c set -k keyc -v c 22 | 23 | ./client -h 1.1.1.1 -c set -k keyd -v d 24 | 25 | ./client -h 1.1.1.1 -c set -k keye -v e 26 | 27 | # stop 1.1.1.1 28 | 29 | ./client -h 1.1.1.2 -c set -k keya -v a 30 | 31 | ./client -h 1.1.1.2 -c set -k keyb -v b 32 | 33 | ./client -h 1.1.1.2 -c set -k keyc -v c 34 | 35 | ./client -h 1.1.1.2 -c set -k keyd -v d 36 | 37 | ./client -h 1.1.1.2 -c set -k keye -v e 38 | -------------------------------------------------------------------------------- /chapter1/server/cache/inmemory.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "sync" 4 | 5 | type inMemoryCache struct { 6 | c map[string][]byte 7 | mutex sync.RWMutex 8 | Stat 9 | } 10 | 11 | func (c *inMemoryCache) Set(k string, v []byte) error { 12 | c.mutex.Lock() 13 | defer c.mutex.Unlock() 14 | tmp, exist := c.c[k] 15 | if exist { 16 | c.del(k, tmp) 17 | } 18 | c.c[k] = v 19 | c.add(k, v) 20 | return nil 21 | } 22 | 23 | func (c *inMemoryCache) Get(k string) ([]byte, error) { 24 | c.mutex.RLock() 25 | defer c.mutex.RUnlock() 26 | return c.c[k], nil 27 | } 28 | 29 | func (c *inMemoryCache) Del(k string) error { 30 | c.mutex.Lock() 31 | defer c.mutex.Unlock() 32 | v, exist := c.c[k] 33 | if exist { 34 | delete(c.c, k) 35 | c.del(k, v) 36 | } 37 | return nil 38 | } 39 | 40 | func (c *inMemoryCache) GetStat() Stat { 41 | return c.Stat 42 | } 43 | 44 | func newInMemoryCache() *inMemoryCache { 45 | return &inMemoryCache{make(map[string][]byte), sync.RWMutex{}, Stat{}} 46 | } 47 | -------------------------------------------------------------------------------- /chapter8/server/cache/inmemory_scanner.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type inMemoryScanner struct { 4 | pair 5 | pairCh chan *pair 6 | closeCh chan struct{} 7 | } 8 | 9 | func (s *inMemoryScanner) Close() { 10 | close(s.closeCh) 11 | } 12 | 13 | func (s *inMemoryScanner) Scan() bool { 14 | p, ok := <-s.pairCh 15 | if ok { 16 | s.k, s.v = p.k, p.v 17 | } 18 | return ok 19 | } 20 | 21 | func (s *inMemoryScanner) Key() string { 22 | return s.k 23 | } 24 | 25 | func (s *inMemoryScanner) Value() []byte { 26 | return s.v 27 | } 28 | 29 | func (c *inMemoryCache) NewScanner() Scanner { 30 | pairCh := make(chan *pair) 31 | closeCh := make(chan struct{}) 32 | go func() { 33 | defer close(pairCh) 34 | c.mutex.RLock() 35 | for k, v := range c.c { 36 | c.mutex.RUnlock() 37 | select { 38 | case <-closeCh: 39 | return 40 | case pairCh <- &pair{k, v}: 41 | } 42 | c.mutex.RLock() 43 | } 44 | c.mutex.RUnlock() 45 | }() 46 | return &inMemoryScanner{pair{}, pairCh, closeCh} 47 | } 48 | -------------------------------------------------------------------------------- /chapter9/server/cache/inmemory_scanner.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type inMemoryScanner struct { 4 | pair 5 | pairCh chan *pair 6 | closeCh chan struct{} 7 | } 8 | 9 | func (s *inMemoryScanner) Close() { 10 | close(s.closeCh) 11 | } 12 | 13 | func (s *inMemoryScanner) Scan() bool { 14 | p, ok := <-s.pairCh 15 | if ok { 16 | s.k, s.v = p.k, p.v 17 | } 18 | return ok 19 | } 20 | 21 | func (s *inMemoryScanner) Key() string { 22 | return s.k 23 | } 24 | 25 | func (s *inMemoryScanner) Value() []byte { 26 | return s.v 27 | } 28 | 29 | func (c *inMemoryCache) NewScanner() Scanner { 30 | pairCh := make(chan *pair) 31 | closeCh := make(chan struct{}) 32 | go func() { 33 | defer close(pairCh) 34 | c.mutex.RLock() 35 | for k, v := range c.c { 36 | c.mutex.RUnlock() 37 | select { 38 | case <-closeCh: 39 | return 40 | case pairCh <- &pair{k, v.v}: 41 | } 42 | c.mutex.RLock() 43 | } 44 | c.mutex.RUnlock() 45 | }() 46 | return &inMemoryScanner{pair{}, pairCh, closeCh} 47 | } 48 | -------------------------------------------------------------------------------- /chapter3/server/cache/rocksdb_getstat.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 6 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 7 | import "C" 8 | import ( 9 | "regexp" 10 | "strconv" 11 | "unsafe" 12 | ) 13 | 14 | func (c *rocksdbCache) GetStat() Stat { 15 | k := C.CString("rocksdb.aggregated-table-properties") 16 | defer C.free(unsafe.Pointer(k)) 17 | v := C.rocksdb_property_value(c.db, k) 18 | defer C.free(unsafe.Pointer(v)) 19 | p := C.GoString(v) 20 | r := regexp.MustCompile(`([^;]+)=([^;]+);`) 21 | s := Stat{} 22 | for _, submatches := range r.FindAllStringSubmatch(p, -1) { 23 | if submatches[1] == " # entries" { 24 | s.Count, _ = strconv.ParseInt(submatches[2], 10, 64) 25 | } else if submatches[1] == " raw key size" { 26 | s.KeySize, _ = strconv.ParseInt(submatches[2], 10, 64) 27 | } else if submatches[1] == " raw value size" { 28 | s.ValueSize, _ = strconv.ParseInt(submatches[2], 10, 64) 29 | } 30 | } 31 | return s 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [2018] [Stuart Hu] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /chapter7/server/tcp/read_key.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "io" 7 | ) 8 | 9 | func (s *Server) readKey(r *bufio.Reader) (string, error) { 10 | klen, e := readLen(r) 11 | if e != nil { 12 | return "", e 13 | } 14 | k := make([]byte, klen) 15 | _, e = io.ReadFull(r, k) 16 | if e != nil { 17 | return "", e 18 | } 19 | key := string(k) 20 | addr, ok := s.ShouldProcess(key) 21 | if !ok { 22 | return "", errors.New("redirect " + addr) 23 | } 24 | return key, nil 25 | } 26 | 27 | func (s *Server) readKeyAndValue(r *bufio.Reader) (string, []byte, error) { 28 | klen, e := readLen(r) 29 | if e != nil { 30 | return "", nil, e 31 | } 32 | vlen, e := readLen(r) 33 | if e != nil { 34 | return "", nil, e 35 | } 36 | k := make([]byte, klen) 37 | _, e = io.ReadFull(r, k) 38 | if e != nil { 39 | return "", nil, e 40 | } 41 | key := string(k) 42 | addr, ok := s.ShouldProcess(key) 43 | if !ok { 44 | return "", nil, errors.New("redirect " + addr) 45 | } 46 | v := make([]byte, vlen) 47 | _, e = io.ReadFull(r, v) 48 | if e != nil { 49 | return "", nil, e 50 | } 51 | return key, v, nil 52 | } 53 | -------------------------------------------------------------------------------- /rocksdb_performance/ingest_data.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include 4 | 5 | int main(int argc, char** argv) 6 | { 7 | // ingest total * valueSize = 10GB data 8 | variables_map vm = parse(argc, argv, 0); 9 | int total = vm["total"].as(); 10 | int valueSize = vm["size"].as(); 11 | 12 | DB* db = opendb(); 13 | 14 | struct timeval tv; 15 | gettimeofday(&tv, 0); 16 | long start = tv.tv_sec * 1000000 + tv.tv_usec; 17 | 18 | std::string valuePrefix = std::string(valueSize, 'a'); 19 | for (int i = 0; i < total; i++) { 20 | std::string key = "ingest" + std::to_string(i); 21 | std::string value = valuePrefix + std::to_string(i); 22 | db->Put(WriteOptions(), key, value); 23 | } 24 | 25 | gettimeofday(&tv, 0); 26 | long end = tv.tv_sec * 1000000 + tv.tv_usec; 27 | std::cout << total << " total records set in " << end - start << " usec," 28 | << double(end - start) / total << " usec average, throughput " 29 | << double(total) / (end - start) * valueSize << " MB/s, rps is " 30 | << double(total) / (end - start) * 1000000 << std::endl; 31 | } 32 | -------------------------------------------------------------------------------- /chapter8/server/cache/rocksdb_scanner.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include "rocksdb/c.h" 4 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 5 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 6 | import "C" 7 | import "unsafe" 8 | 9 | type rocksdbScanner struct { 10 | i *C.rocksdb_iterator_t 11 | initialized bool 12 | } 13 | 14 | func (s *rocksdbScanner) Close() { 15 | C.rocksdb_iter_destroy(s.i) 16 | } 17 | 18 | func (s *rocksdbScanner) Scan() bool { 19 | if !s.initialized { 20 | C.rocksdb_iter_seek_to_first(s.i) 21 | s.initialized = true 22 | } else { 23 | C.rocksdb_iter_next(s.i) 24 | } 25 | return C.rocksdb_iter_valid(s.i) != 0 26 | } 27 | 28 | func (s *rocksdbScanner) Key() string { 29 | var length C.size_t 30 | k := C.rocksdb_iter_key(s.i, &length) 31 | return C.GoString(k) 32 | } 33 | 34 | func (s *rocksdbScanner) Value() []byte { 35 | var length C.size_t 36 | v := C.rocksdb_iter_value(s.i, &length) 37 | return C.GoBytes(unsafe.Pointer(v), C.int(length)) 38 | } 39 | 40 | func (c *rocksdbCache) NewScanner() Scanner { 41 | return &rocksdbScanner{C.rocksdb_create_iterator(c.db, c.ro), false} 42 | } 43 | -------------------------------------------------------------------------------- /chapter1/server/http/cache.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | type cacheHandler struct { 11 | *Server 12 | } 13 | 14 | func (h *cacheHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 15 | key := strings.Split(r.URL.EscapedPath(), "/")[2] 16 | if len(key) == 0 { 17 | w.WriteHeader(http.StatusBadRequest) 18 | return 19 | } 20 | m := r.Method 21 | if m == http.MethodPut { 22 | b, _ := ioutil.ReadAll(r.Body) 23 | if len(b) != 0 { 24 | e := h.Set(key, b) 25 | if e != nil { 26 | log.Println(e) 27 | w.WriteHeader(http.StatusInternalServerError) 28 | } 29 | } 30 | return 31 | } 32 | if m == http.MethodGet { 33 | b, e := h.Get(key) 34 | if e != nil { 35 | log.Println(e) 36 | w.WriteHeader(http.StatusInternalServerError) 37 | return 38 | } 39 | if len(b) == 0 { 40 | w.WriteHeader(http.StatusNotFound) 41 | return 42 | } 43 | w.Write(b) 44 | return 45 | } 46 | if m == http.MethodDelete { 47 | e := h.Del(key) 48 | if e != nil { 49 | log.Println(e) 50 | w.WriteHeader(http.StatusInternalServerError) 51 | } 52 | return 53 | } 54 | w.WriteHeader(http.StatusMethodNotAllowed) 55 | } 56 | 57 | func (s *Server) cacheHandler() http.Handler { 58 | return &cacheHandler{s} 59 | } 60 | -------------------------------------------------------------------------------- /chapter7/server/cluster/cluster.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "github.com/hashicorp/memberlist" 5 | "io/ioutil" 6 | "stathat.com/c/consistent" 7 | "time" 8 | ) 9 | 10 | type Node interface { 11 | ShouldProcess(key string) (string, bool) 12 | Members() []string 13 | Addr() string 14 | } 15 | 16 | type node struct { 17 | *consistent.Consistent 18 | addr string 19 | } 20 | 21 | func (n *node) Addr() string { 22 | return n.addr 23 | } 24 | 25 | func New(addr, cluster string) (Node, error) { 26 | conf := memberlist.DefaultLANConfig() 27 | conf.Name = addr 28 | conf.BindAddr = addr 29 | conf.LogOutput = ioutil.Discard 30 | l, e := memberlist.Create(conf) 31 | if e != nil { 32 | return nil, e 33 | } 34 | if cluster == "" { 35 | cluster = addr 36 | } 37 | clu := []string{cluster} 38 | _, e = l.Join(clu) 39 | if e != nil { 40 | return nil, e 41 | } 42 | circle := consistent.New() 43 | circle.NumberOfReplicas = 256 44 | go func() { 45 | for { 46 | m := l.Members() 47 | nodes := make([]string, len(m)) 48 | for i, n := range m { 49 | nodes[i] = n.Name 50 | } 51 | circle.Set(nodes) 52 | time.Sleep(time.Second) 53 | } 54 | }() 55 | return &node{circle, addr}, nil 56 | } 57 | 58 | func (n *node) ShouldProcess(key string) (string, bool) { 59 | addr, _ := n.Get(key) 60 | return addr, addr == n.addr 61 | } 62 | -------------------------------------------------------------------------------- /chapter2/server/tcp/process.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log" 7 | "net" 8 | ) 9 | 10 | func (s *Server) get(conn net.Conn, r *bufio.Reader) error { 11 | k, e := s.readKey(r) 12 | if e != nil { 13 | return e 14 | } 15 | v, e := s.Get(k) 16 | return sendResponse(v, e, conn) 17 | } 18 | 19 | func (s *Server) set(conn net.Conn, r *bufio.Reader) error { 20 | k, v, e := s.readKeyAndValue(r) 21 | if e != nil { 22 | return e 23 | } 24 | return sendResponse(nil, s.Set(k, v), conn) 25 | } 26 | 27 | func (s *Server) del(conn net.Conn, r *bufio.Reader) error { 28 | k, e := s.readKey(r) 29 | if e != nil { 30 | return e 31 | } 32 | return sendResponse(nil, s.Del(k), conn) 33 | } 34 | 35 | func (s *Server) process(conn net.Conn) { 36 | defer conn.Close() 37 | r := bufio.NewReader(conn) 38 | for { 39 | op, e := r.ReadByte() 40 | if e != nil { 41 | if e != io.EOF { 42 | log.Println("close connection due to error:", e) 43 | } 44 | return 45 | } 46 | if op == 'S' { 47 | e = s.set(conn, r) 48 | } else if op == 'G' { 49 | e = s.get(conn, r) 50 | } else if op == 'D' { 51 | e = s.del(conn, r) 52 | } else { 53 | log.Println("close connection due to invalid operation:", op) 54 | return 55 | } 56 | if e != nil { 57 | log.Println("close connection due to error:", e) 58 | return 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /chapter9/server/cache/inmemory.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type value struct { 9 | v []byte 10 | created time.Time 11 | } 12 | 13 | type inMemoryCache struct { 14 | c map[string]value 15 | mutex sync.RWMutex 16 | Stat 17 | ttl time.Duration 18 | } 19 | 20 | func (c *inMemoryCache) Set(k string, v []byte) error { 21 | c.mutex.Lock() 22 | defer c.mutex.Unlock() 23 | c.c[k] = value{v, time.Now()} 24 | c.add(k, v) 25 | return nil 26 | } 27 | 28 | func (c *inMemoryCache) Get(k string) ([]byte, error) { 29 | c.mutex.RLock() 30 | defer c.mutex.RUnlock() 31 | return c.c[k].v, nil 32 | } 33 | 34 | func (c *inMemoryCache) Del(k string) error { 35 | c.mutex.Lock() 36 | defer c.mutex.Unlock() 37 | v, exist := c.c[k] 38 | if exist { 39 | delete(c.c, k) 40 | c.del(k, v.v) 41 | } 42 | return nil 43 | } 44 | 45 | func (c *inMemoryCache) GetStat() Stat { 46 | return c.Stat 47 | } 48 | 49 | func newInMemoryCache(ttl int) *inMemoryCache { 50 | c := &inMemoryCache{make(map[string]value), sync.RWMutex{}, Stat{}, time.Duration(ttl) * time.Second} 51 | if ttl > 0 { 52 | go c.expirer() 53 | } 54 | return c 55 | } 56 | 57 | func (c *inMemoryCache) expirer() { 58 | for { 59 | time.Sleep(c.ttl) 60 | c.mutex.RLock() 61 | for k, v := range c.c { 62 | c.mutex.RUnlock() 63 | if v.created.Add(c.ttl).Before(time.Now()) { 64 | c.Del(k) 65 | } 66 | c.mutex.RLock() 67 | } 68 | c.mutex.RUnlock() 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /chapter5/server/cache/rocksdb_set.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | // #cgo CFLAGS: -I${SRCDIR}/../../../rocksdb/include 6 | // #cgo LDFLAGS: -L${SRCDIR}/../../../rocksdb -lrocksdb -lz -lpthread -lsnappy -lstdc++ -lm -O3 7 | import "C" 8 | import ( 9 | "time" 10 | "unsafe" 11 | ) 12 | 13 | const BATCH_SIZE = 100 14 | 15 | func flush_batch(db *C.rocksdb_t, b *C.rocksdb_writebatch_t, o *C.rocksdb_writeoptions_t) { 16 | var e *C.char 17 | C.rocksdb_write(db, o, b, &e) 18 | if e != nil { 19 | panic(C.GoString(e)) 20 | } 21 | C.rocksdb_writebatch_clear(b) 22 | } 23 | 24 | func write_func(db *C.rocksdb_t, c chan *pair, o *C.rocksdb_writeoptions_t) { 25 | count := 0 26 | t := time.NewTimer(time.Second) 27 | b := C.rocksdb_writebatch_create() 28 | for { 29 | select { 30 | case p := <-c: 31 | count++ 32 | key := C.CString(p.k) 33 | value := C.CBytes(p.v) 34 | C.rocksdb_writebatch_put(b, key, C.size_t(len(p.k)), (*C.char)(value), C.size_t(len(p.v))) 35 | C.free(unsafe.Pointer(key)) 36 | C.free(value) 37 | if count == BATCH_SIZE { 38 | flush_batch(db, b, o) 39 | count = 0 40 | } 41 | if !t.Stop() { 42 | <-t.C 43 | } 44 | t.Reset(time.Second) 45 | case <-t.C: 46 | if count != 0 { 47 | flush_batch(db, b, o) 48 | count = 0 49 | } 50 | t.Reset(time.Second) 51 | } 52 | } 53 | } 54 | 55 | func (c *rocksdbCache) Set(key string, value []byte) error { 56 | c.ch <- &pair{key, value} 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /rocksdb_performance/test_basic.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include 4 | 5 | int main(int argc, char** argv) 6 | { 7 | variables_map vm = parse(argc, argv, 0); 8 | int total = vm["total"].as(); 9 | int valueSize = vm["size"].as(); 10 | 11 | DB* db = opendb(); 12 | 13 | struct timeval tv; 14 | gettimeofday(&tv, 0); 15 | long start = tv.tv_sec * 1000000 + tv.tv_usec; 16 | std::string valuePrefix = std::string(valueSize, 'a'); 17 | for (int i = 0; i < total; i++) { 18 | std::string key = keyPrefix + std::to_string(i); 19 | std::string value = valuePrefix + std::to_string(i); 20 | db->Put(WriteOptions(), key, value); 21 | } 22 | gettimeofday(&tv, 0); 23 | long end = tv.tv_sec * 1000000 + tv.tv_usec; 24 | std::cout << total << " records put in " << end - start << " usec, " 25 | << double(end - start) / total << " usec average" << std::endl; 26 | 27 | gettimeofday(&tv, 0); 28 | std::string value; 29 | start = tv.tv_sec * 1000000 + tv.tv_usec; 30 | for (int i = 0; i < total; i++) { 31 | std::string tmp = std::to_string(std::rand() % total); 32 | std::string key = keyPrefix + tmp; 33 | db->Get(ReadOptions(), key, &value); 34 | assert(value == valuePrefix + tmp); 35 | } 36 | gettimeofday(&tv, 0); 37 | end = tv.tv_sec * 1000000 + tv.tv_usec; 38 | std::cout << total << " records get in " << end - start << " usec, " 39 | << double(end - start) / total << " usec average" << std::endl; 40 | 41 | delete db; 42 | } 43 | -------------------------------------------------------------------------------- /cache-benchmark/cacheClient/http.go: -------------------------------------------------------------------------------- 1 | package cacheClient 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | type httpClient struct { 11 | *http.Client 12 | server string 13 | } 14 | 15 | func (c *httpClient) get(key string) string { 16 | resp, e := c.Get(c.server + key) 17 | if e != nil { 18 | log.Println(key) 19 | panic(e) 20 | } 21 | if resp.StatusCode == http.StatusNotFound { 22 | return "" 23 | } 24 | if resp.StatusCode != http.StatusOK { 25 | panic(resp.Status) 26 | } 27 | b, e := ioutil.ReadAll(resp.Body) 28 | if e != nil { 29 | panic(e) 30 | } 31 | return string(b) 32 | } 33 | 34 | func (c *httpClient) set(key, value string) { 35 | req, e := http.NewRequest(http.MethodPut, 36 | c.server+key, strings.NewReader(value)) 37 | if e != nil { 38 | log.Println(key) 39 | panic(e) 40 | } 41 | resp, e := c.Do(req) 42 | if e != nil { 43 | log.Println(key) 44 | panic(e) 45 | } 46 | if resp.StatusCode != http.StatusOK { 47 | panic(resp.Status) 48 | } 49 | } 50 | 51 | func (c *httpClient) Run(cmd *Cmd) { 52 | if cmd.Name == "get" { 53 | cmd.Value = c.get(cmd.Key) 54 | return 55 | } 56 | if cmd.Name == "set" { 57 | c.set(cmd.Key, cmd.Value) 58 | return 59 | } 60 | panic("unknown cmd name " + cmd.Name) 61 | } 62 | 63 | func newHTTPClient(server string) *httpClient { 64 | client := &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 1}} 65 | return &httpClient{client, "http://" + server + ":12345/cache/"} 66 | } 67 | 68 | func (c *httpClient) PipelinedRun([]*Cmd) { 69 | panic("httpClient pipelined run not implement") 70 | } 71 | -------------------------------------------------------------------------------- /cache-benchmark/cacheClient/redis.go: -------------------------------------------------------------------------------- 1 | package cacheClient 2 | 3 | import ( 4 | "github.com/go-redis/redis" 5 | ) 6 | 7 | type redisClient struct { 8 | *redis.Client 9 | } 10 | 11 | func (r *redisClient) get(key string) (string, error) { 12 | res, e := r.Get(key).Result() 13 | if e == redis.Nil { 14 | return "", nil 15 | } 16 | return res, e 17 | } 18 | 19 | func (r *redisClient) set(key, value string) error { 20 | return r.Set(key, value, 0).Err() 21 | } 22 | 23 | func (r *redisClient) del(key string) error { 24 | return r.Del(key).Err() 25 | } 26 | 27 | func (r *redisClient) Run(c *Cmd) { 28 | if c.Name == "get" { 29 | c.Value, c.Error = r.get(c.Key) 30 | return 31 | } 32 | if c.Name == "set" { 33 | c.Error = r.set(c.Key, c.Value) 34 | return 35 | } 36 | if c.Name == "del" { 37 | c.Error = r.del(c.Key) 38 | return 39 | } 40 | panic("unknown cmd name " + c.Name) 41 | } 42 | 43 | func (r *redisClient) PipelinedRun(cmds []*Cmd) { 44 | if len(cmds) == 0 { 45 | return 46 | } 47 | pipe := r.Pipeline() 48 | cmders := make([]redis.Cmder, len(cmds)) 49 | for i, c := range cmds { 50 | if c.Name == "get" { 51 | cmders[i] = pipe.Get(c.Key) 52 | } else if c.Name == "set" { 53 | cmders[i] = pipe.Set(c.Key, c.Value, 0) 54 | } else if c.Name == "del" { 55 | cmders[i] = pipe.Del(c.Key) 56 | } else { 57 | panic("unknown cmd name " + c.Name) 58 | } 59 | } 60 | _, e := pipe.Exec() 61 | if e != nil && e != redis.Nil { 62 | panic(e) 63 | } 64 | for i, c := range cmds { 65 | if c.Name == "get" { 66 | value, e := cmders[i].(*redis.StringCmd).Result() 67 | if e == redis.Nil { 68 | value, e = "", nil 69 | } 70 | c.Value, c.Error = value, e 71 | } else { 72 | c.Error = cmders[i].Err() 73 | } 74 | } 75 | } 76 | 77 | func newRedisClient(server string) *redisClient { 78 | return &redisClient{redis.NewClient(&redis.Options{Addr: server + ":6379", ReadTimeout: -1})} 79 | } 80 | -------------------------------------------------------------------------------- /rocksdb_performance/test_batch_write.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include 4 | 5 | int main(int argc, char** argv) 6 | { 7 | int batchSize; 8 | options_description batchOption("batch_write option"); 9 | batchOption.add_options()("batch_size,b", value(&batchSize)->default_value(1), "batch size"); 10 | 11 | variables_map vm = parse(argc, argv, &batchOption); 12 | int total = vm["total"].as(); 13 | int valueSize = vm["size"].as(); 14 | std::cout << "batch size is " << batchSize << std::endl; 15 | 16 | DB* db = opendb(); 17 | 18 | struct timeval tv; 19 | gettimeofday(&tv, 0); 20 | long start = tv.tv_sec * 1000000 + tv.tv_usec; 21 | int count = 0; 22 | std::string valuePrefix = std::string(valueSize, 'a'); 23 | while (count != total) { 24 | WriteBatch batch; 25 | for (int j = 0; j < batchSize; j++, count++) { 26 | if (count == total) 27 | break; 28 | std::string tmp = std::to_string(count); 29 | std::string key = keyPrefix + tmp; 30 | std::string value = valuePrefix + tmp; 31 | auto s = batch.Put(key, value); 32 | if (!s.ok()) 33 | std::cerr << "batch.Put():" << s.ToString() << std::endl; 34 | assert(s.ok()); 35 | } 36 | auto s = db->Write(WriteOptions(), &batch); 37 | if (!s.ok()) 38 | std::cerr << "db->Write():" << s.ToString() << std::endl; 39 | assert(s.ok()); 40 | } 41 | gettimeofday(&tv, 0); 42 | long end = tv.tv_sec * 1000000 + tv.tv_usec; 43 | std::cout << total << " records batch put in " << end - start << " usec, " 44 | << double(end - start) / total << " usec average, throughput is " 45 | << (double)total * valueSize / (end - start) << " MB/s, rps is " 46 | << (double)1000000 * total / (end - start) << std::endl; 47 | 48 | delete db; 49 | } 50 | -------------------------------------------------------------------------------- /chapter6/server/tcp/process.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log" 7 | "net" 8 | ) 9 | 10 | type result struct { 11 | v []byte 12 | e error 13 | } 14 | 15 | func (s *Server) get(ch chan chan *result, r *bufio.Reader) { 16 | c := make(chan *result) 17 | ch <- c 18 | k, e := s.readKey(r) 19 | if e != nil { 20 | c <- &result{nil, e} 21 | return 22 | } 23 | go func() { 24 | v, e := s.Get(k) 25 | c <- &result{v, e} 26 | }() 27 | } 28 | 29 | func (s *Server) set(ch chan chan *result, r *bufio.Reader) { 30 | c := make(chan *result) 31 | ch <- c 32 | k, v, e := s.readKeyAndValue(r) 33 | if e != nil { 34 | c <- &result{nil, e} 35 | return 36 | } 37 | go func() { 38 | c <- &result{nil, s.Set(k, v)} 39 | }() 40 | } 41 | 42 | func (s *Server) del(ch chan chan *result, r *bufio.Reader) { 43 | c := make(chan *result) 44 | ch <- c 45 | k, e := s.readKey(r) 46 | if e != nil { 47 | c <- &result{nil, e} 48 | return 49 | } 50 | go func() { 51 | c <- &result{nil, s.Del(k)} 52 | }() 53 | } 54 | 55 | func reply(conn net.Conn, resultCh chan chan *result) { 56 | defer conn.Close() 57 | for { 58 | c, open := <-resultCh 59 | if !open { 60 | return 61 | } 62 | r := <-c 63 | e := sendResponse(r.v, r.e, conn) 64 | if e != nil { 65 | log.Println("close connection due to error:", e) 66 | return 67 | } 68 | } 69 | } 70 | 71 | func (s *Server) process(conn net.Conn) { 72 | r := bufio.NewReader(conn) 73 | resultCh := make(chan chan *result, 5000) 74 | defer close(resultCh) 75 | go reply(conn, resultCh) 76 | for { 77 | op, e := r.ReadByte() 78 | if e != nil { 79 | if e != io.EOF { 80 | log.Println("close connection due to error:", e) 81 | } 82 | return 83 | } 84 | if op == 'S' { 85 | s.set(resultCh, r) 86 | } else if op == 'G' { 87 | s.get(resultCh, r) 88 | } else if op == 'D' { 89 | s.del(resultCh, r) 90 | } else { 91 | log.Println("close connection due to invalid operation:", op) 92 | return 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /rocksdb_performance/common.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include "rocksdb/db.h" 6 | #include "rocksdb/utilities/db_ttl.h" 7 | 8 | using namespace rocksdb; 9 | using namespace boost::program_options; 10 | 11 | const std::string keyPrefix = std::string(99, 'a'); 12 | 13 | variables_map parse(int argc, char** argv, options_description* additional) 14 | { 15 | int total, valueSize; 16 | options_description desc("Allowed options"); 17 | desc.add_options()("help,h", "produce help message")("total,t", value(&total)->default_value(10000), "total record number")("size,s", value(&valueSize)->default_value(1000), "value size"); 18 | 19 | if (additional) { 20 | desc.add(*additional); 21 | } 22 | 23 | variables_map vm; 24 | store(parse_command_line(argc, argv, desc), vm); 25 | notify(vm); 26 | 27 | if (vm.count("help")) { 28 | std::cout << desc << std::endl; 29 | exit(1); 30 | } 31 | 32 | std::cout << "total record number is " << total << std::endl; 33 | std::cout << "value size is " << valueSize << std::endl; 34 | 35 | return vm; 36 | } 37 | 38 | DB* opendb(bool readonly = false, const std::string& dir = "/mnt/rocksdb", int ttl = 0) 39 | { 40 | DB* db; 41 | Options options; 42 | // Optimize RocksDB. This is the easiest way to get RocksDB to perform well 43 | options.IncreaseParallelism(); 44 | options.OptimizeLevelStyleCompaction(); 45 | // create the DB if it's not already present 46 | options.create_if_missing = true; 47 | 48 | // open DB 49 | Status s; 50 | if (readonly) 51 | s = DB::OpenForReadOnly(options, dir, &db); 52 | else if (ttl) { 53 | rocksdb::DBWithTTL* db_ttl; 54 | s = rocksdb::DBWithTTL::Open(options, dir, &db_ttl, ttl); 55 | db = db_ttl; 56 | } 57 | else 58 | s = DB::Open(options, dir, &db); 59 | 60 | if (!s.ok()) { 61 | std::cout << "open " << dir << ":" << s.ToString() << std::endl; 62 | } 63 | assert(db); 64 | return db; 65 | } 66 | -------------------------------------------------------------------------------- /cache-benchmark/cacheClient/tcp.go: -------------------------------------------------------------------------------- 1 | package cacheClient 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | type tcpClient struct { 15 | net.Conn 16 | r *bufio.Reader 17 | } 18 | 19 | func (c *tcpClient) sendGet(key string) { 20 | klen := len(key) 21 | c.Write([]byte(fmt.Sprintf("G%d %s", klen, key))) 22 | } 23 | 24 | func (c *tcpClient) sendSet(key, value string) { 25 | klen := len(key) 26 | vlen := len(value) 27 | c.Write([]byte(fmt.Sprintf("S%d %d %s%s", klen, vlen, key, value))) 28 | } 29 | 30 | func (c *tcpClient) sendDel(key string) { 31 | klen := len(key) 32 | c.Write([]byte(fmt.Sprintf("D%d %s", klen, key))) 33 | } 34 | 35 | func readLen(r *bufio.Reader) int { 36 | tmp, e := r.ReadString(' ') 37 | if e != nil { 38 | log.Println(e) 39 | return 0 40 | } 41 | l, e := strconv.Atoi(strings.TrimSpace(tmp)) 42 | if e != nil { 43 | log.Println(tmp, e) 44 | return 0 45 | } 46 | return l 47 | } 48 | 49 | func (c *tcpClient) recvResponse() (string, error) { 50 | vlen := readLen(c.r) 51 | if vlen == 0 { 52 | return "", nil 53 | } 54 | if vlen < 0 { 55 | err := make([]byte, -vlen) 56 | _, e := io.ReadFull(c.r, err) 57 | if e != nil { 58 | return "", e 59 | } 60 | return "", errors.New(string(err)) 61 | } 62 | value := make([]byte, vlen) 63 | _, e := io.ReadFull(c.r, value) 64 | if e != nil { 65 | return "", e 66 | } 67 | return string(value), nil 68 | } 69 | 70 | func (c *tcpClient) Run(cmd *Cmd) { 71 | if cmd.Name == "get" { 72 | c.sendGet(cmd.Key) 73 | cmd.Value, cmd.Error = c.recvResponse() 74 | return 75 | } 76 | if cmd.Name == "set" { 77 | c.sendSet(cmd.Key, cmd.Value) 78 | _, cmd.Error = c.recvResponse() 79 | return 80 | } 81 | if cmd.Name == "del" { 82 | c.sendDel(cmd.Key) 83 | _, cmd.Error = c.recvResponse() 84 | return 85 | } 86 | panic("unknown cmd name " + cmd.Name) 87 | } 88 | 89 | func (c *tcpClient) PipelinedRun(cmds []*Cmd) { 90 | if len(cmds) == 0 { 91 | return 92 | } 93 | for _, cmd := range cmds { 94 | if cmd.Name == "get" { 95 | c.sendGet(cmd.Key) 96 | } 97 | if cmd.Name == "set" { 98 | c.sendSet(cmd.Key, cmd.Value) 99 | } 100 | if cmd.Name == "del" { 101 | c.sendDel(cmd.Key) 102 | } 103 | } 104 | for _, cmd := range cmds { 105 | cmd.Value, cmd.Error = c.recvResponse() 106 | } 107 | } 108 | 109 | func newTCPClient(server string) *tcpClient { 110 | c, e := net.Dial("tcp", server+":12346") 111 | if e != nil { 112 | panic(e) 113 | } 114 | r := bufio.NewReader(c) 115 | return &tcpClient{c, r} 116 | } 117 | -------------------------------------------------------------------------------- /cache-benchmark/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./cacheClient" 5 | "flag" 6 | "fmt" 7 | "math/rand" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | type statistic struct { 13 | count int 14 | time time.Duration 15 | } 16 | 17 | type result struct { 18 | getCount int 19 | missCount int 20 | setCount int 21 | statBuckets []statistic 22 | } 23 | 24 | func (r *result) addStatistic(bucket int, stat statistic) { 25 | if bucket > len(r.statBuckets)-1 { 26 | newStatBuckets := make([]statistic, bucket+1) 27 | copy(newStatBuckets, r.statBuckets) 28 | r.statBuckets = newStatBuckets 29 | } 30 | s := r.statBuckets[bucket] 31 | s.count += stat.count 32 | s.time += stat.time 33 | r.statBuckets[bucket] = s 34 | } 35 | 36 | func (r *result) addDuration(d time.Duration, typ string) { 37 | bucket := int(d / time.Millisecond) 38 | r.addStatistic(bucket, statistic{1, d}) 39 | if typ == "get" { 40 | r.getCount++ 41 | } else if typ == "set" { 42 | r.setCount++ 43 | } else { 44 | r.missCount++ 45 | } 46 | } 47 | 48 | func (r *result) addResult(src *result) { 49 | for b, s := range src.statBuckets { 50 | r.addStatistic(b, s) 51 | } 52 | r.getCount += src.getCount 53 | r.missCount += src.missCount 54 | r.setCount += src.setCount 55 | } 56 | 57 | func run(client cacheClient.Client, c *cacheClient.Cmd, r *result) { 58 | expect := c.Value 59 | start := time.Now() 60 | client.Run(c) 61 | d := time.Now().Sub(start) 62 | resultType := c.Name 63 | if resultType == "get" { 64 | if c.Value == "" { 65 | resultType = "miss" 66 | } else if c.Value != expect { 67 | panic(c) 68 | } 69 | } 70 | r.addDuration(d, resultType) 71 | } 72 | 73 | func pipeline(client cacheClient.Client, cmds []*cacheClient.Cmd, r *result) { 74 | expect := make([]string, len(cmds)) 75 | for i, c := range cmds { 76 | if c.Name == "get" { 77 | expect[i] = c.Value 78 | } 79 | } 80 | start := time.Now() 81 | client.PipelinedRun(cmds) 82 | d := time.Now().Sub(start) 83 | for i, c := range cmds { 84 | resultType := c.Name 85 | if resultType == "get" { 86 | if c.Value == "" { 87 | resultType = "miss" 88 | } else if c.Value != expect[i] { 89 | fmt.Println(expect[i]) 90 | panic(c.Value) 91 | } 92 | } 93 | r.addDuration(d, resultType) 94 | } 95 | } 96 | 97 | func operate(id, count int, ch chan *result) { 98 | client := cacheClient.New(typ, server) 99 | cmds := make([]*cacheClient.Cmd, 0) 100 | valuePrefix := strings.Repeat("a", valueSize) 101 | r := &result{0, 0, 0, make([]statistic, 0)} 102 | for i := 0; i < count; i++ { 103 | var tmp int 104 | if keyspacelen > 0 { 105 | tmp = rand.Intn(keyspacelen) 106 | } else { 107 | tmp = id*count + i 108 | } 109 | key := fmt.Sprintf("%d", tmp) 110 | value := fmt.Sprintf("%s%d", valuePrefix, tmp) 111 | name := operation 112 | if operation == "mixed" { 113 | if rand.Intn(2) == 1 { 114 | name = "set" 115 | } else { 116 | name = "get" 117 | } 118 | } 119 | c := &cacheClient.Cmd{name, key, value, nil} 120 | if pipelen > 1 { 121 | cmds = append(cmds, c) 122 | if len(cmds) == pipelen { 123 | pipeline(client, cmds, r) 124 | cmds = make([]*cacheClient.Cmd, 0) 125 | } 126 | } else { 127 | run(client, c, r) 128 | } 129 | } 130 | if len(cmds) != 0 { 131 | pipeline(client, cmds, r) 132 | } 133 | ch <- r 134 | } 135 | 136 | var typ, server, operation string 137 | var total, valueSize, threads, keyspacelen, pipelen int 138 | 139 | func init() { 140 | flag.StringVar(&typ, "type", "redis", "cache server type") 141 | flag.StringVar(&server, "h", "localhost", "cache server address") 142 | flag.IntVar(&total, "n", 1000, "total number of requests") 143 | flag.IntVar(&valueSize, "d", 1000, "data size of SET/GET value in bytes") 144 | flag.IntVar(&threads, "c", 1, "number of parallel connections") 145 | flag.StringVar(&operation, "t", "set", "test set, could be get/set/mixed") 146 | flag.IntVar(&keyspacelen, "r", 0, "keyspacelen, use random keys from 0 to keyspacelen-1") 147 | flag.IntVar(&pipelen, "P", 1, "pipeline length") 148 | flag.Parse() 149 | fmt.Println("type is", typ) 150 | fmt.Println("server is", server) 151 | fmt.Println("total", total, "requests") 152 | fmt.Println("data size is", valueSize) 153 | fmt.Println("we have", threads, "connections") 154 | fmt.Println("operation is", operation) 155 | fmt.Println("keyspacelen is", keyspacelen) 156 | fmt.Println("pipeline length is", pipelen) 157 | 158 | rand.Seed(time.Now().UnixNano()) 159 | } 160 | 161 | func main() { 162 | ch := make(chan *result, threads) 163 | res := &result{0, 0, 0, make([]statistic, 0)} 164 | start := time.Now() 165 | for i := 0; i < threads; i++ { 166 | go operate(i, total/threads, ch) 167 | } 168 | for i := 0; i < threads; i++ { 169 | res.addResult(<-ch) 170 | } 171 | d := time.Now().Sub(start) 172 | totalCount := res.getCount + res.missCount + res.setCount 173 | fmt.Printf("%d records get\n", res.getCount) 174 | fmt.Printf("%d records miss\n", res.missCount) 175 | fmt.Printf("%d records set\n", res.setCount) 176 | fmt.Printf("%f seconds total\n", d.Seconds()) 177 | statCountSum := 0 178 | statTimeSum := time.Duration(0) 179 | for b, s := range res.statBuckets { 180 | if s.count == 0 { 181 | continue 182 | } 183 | statCountSum += s.count 184 | statTimeSum += s.time 185 | fmt.Printf("%d%% requests < %d ms\n", statCountSum*100/totalCount, b+1) 186 | } 187 | fmt.Printf("%d usec average for each request\n", int64(statTimeSum/time.Microsecond)/int64(statCountSum)) 188 | fmt.Printf("throughput is %f MB/s\n", float64((res.getCount+res.setCount)*valueSize)/1e6/d.Seconds()) 189 | fmt.Printf("rps is %f\n", float64(totalCount)/float64(d.Seconds())) 190 | } 191 | --------------------------------------------------------------------------------