├── .idea
    └── encodings.xml
├── .gitignore
├── .travis.yml
├── mapstore.go
├── LICENSE
├── README.md
├── mapstore_single.go
├── mapstore_test.go
└── mapstore_shard.go


/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="Encoding">
4 |     <file url="PROJECT" charset="UTF-8" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
 2 | *.o
 3 | *.a
 4 | *.so
 5 | 
 6 | # Folders
 7 | _obj
 8 | _test
 9 | 
10 | # Architecture specific extensions/prefixes
11 | *.[568vq]
12 | [568vq].out
13 | 
14 | *.cgo1.go
15 | *.cgo2.c
16 | _cgo_defun.c
17 | _cgo_gotypes.go
18 | _cgo_export.*
19 | 
20 | _testmain.go
21 | 
22 | *.exe
23 | *.test
24 | *.prof
25 | 
26 | .idea
27 | *.iml


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: go
 2 | 
 3 | # The feature/go15 branch uses new features in go15. Only testing this branch
 4 | # against tip which has the features.
 5 | go:
 6 |   - 1.5
 7 | 
 8 | # Setting sudo access to false will let Travis CI use containers rather than
 9 | # VMs to run the tests. For more details see:
10 | # - http://docs.travis-ci.com/user/workers/container-based-infrastructure/
11 | # - http://docs.travis-ci.com/user/workers/standard-infrastructure/
12 | sudo: false
13 | 
14 | install: 
15 | 
16 | # The default script is go test -v ./... which will test everything
17 | # in the vendor directory. We don't need to test all dependent packages.
18 | # Only testing this project.
19 | script:
20 |   - GO15VENDOREXPERIMENT=1 go test -v .
21 | 
22 | notifications:


--------------------------------------------------------------------------------
/mapstore.go:
--------------------------------------------------------------------------------
 1 | package mapstore
 2 | 
 3 | const StoreDefaultSize = 1024
 4 | 
 5 | type Entry struct {
 6 | 	Key   string
 7 | 	Value interface{}
 8 | }
 9 | 
10 | type Store interface {
11 | 	Set(string, interface{})
12 | 	Get(string, interface{}) (interface{}, bool)
13 | 	GetOrSet(string, interface{}) (interface{}, bool)
14 | 	GetOrSetFunc(string, func() interface{}) (interface{}, bool)
15 | 	Delete(string) bool
16 | 	Update(string, func(interface{}) interface{}) bool
17 | 	UpdateIfExists(string, func(interface{}) interface{}) bool
18 | 
19 | 	Load(chan Entry)
20 | 	Save(chan<- Entry)
21 | 
22 | 	Len() int
23 | 	ShardStats() []int
24 | }
25 | 
26 | func NewWithSize(shardsCount int) Store {
27 | 	if shardsCount > 1 {
28 | 		return newStoreShard(shardsCount)
29 | 	}
30 | 	return newStoreSingle()
31 | }
32 | 
33 | func New() Store {
34 | 	return NewWithSize(StoreDefaultSize)
35 | }
36 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # mapstore
 2 | Multicore optimized in-memory data store in Go
 3 | 
 4 | [![Build Status](https://travis-ci.org/miolini/mapstore.svg)](https://travis-ci.org/miolini/mapstore.svg) [![Go Report Card](http://goreportcard.com/badge/miolini/mapstore)](http://goreportcard.com/report/miolini/mapstore)
 5 | 
 6 | ## Benchmark
 7 | 
 8 | ```bash
 9 | $ go test -v -benchmem -bench .
10 | === RUN   TestShardsStat
11 | --- PASS: TestShardsStat (5.69s)
12 | 	mapstore_test.go:67: stats: [249597 249954 249518 250931]
13 | PASS
14 | BenchmarkShard1Read4Write4-8   	  300000	      5507 ns/op	     403 B/op	       8 allocs/op
15 | BenchmarkShard10Read4Write4-8  	  300000	      4131 ns/op	     591 B/op	      16 allocs/op
16 | BenchmarkShard100Read4Write4-8 	 1000000	      2316 ns/op	     384 B/op	      16 allocs/op
17 | BenchmarkShard1000Read4Write4-8	 1000000	      2199 ns/op	     422 B/op	      16 allocs/op
18 | BenchmarkShard1Read8Write2-8   	  300000	      5670 ns/op	     425 B/op	      10 allocs/op
19 | BenchmarkShard10Read8Write2-8  	  300000	      4490 ns/op	     494 B/op	      20 allocs/op
20 | BenchmarkShard100Read8Write2-8 	 1000000	      2575 ns/op	     446 B/op	      20 allocs/op
21 | BenchmarkShard1000Read8Write2-8	 1000000	      2658 ns/op	     476 B/op	      20 allocs/op
22 | BenchmarkShard1Read2Write8-8   	  200000	      6520 ns/op	     586 B/op	      10 allocs/op
23 | BenchmarkShard10Read2Write8-8  	  300000	      4607 ns/op	     668 B/op	      20 allocs/op
24 | BenchmarkShard100Read2Write8-8 	 1000000	      2615 ns/op	     448 B/op	      20 allocs/op
25 | BenchmarkShard1000Read2Write8-8	 1000000	      2725 ns/op	     487 B/op	      20 allocs/op
26 | BenchmarkShard1Read8Write0-8   	 1000000	      1860 ns/op	     128 B/op	       8 allocs/op
27 | BenchmarkShard10Read8Write0-8  	 1000000	      1987 ns/op	     257 B/op	      16 allocs/op
28 | BenchmarkShard100Read8Write0-8 	 1000000	      1981 ns/op	     257 B/op	      16 allocs/op
29 | BenchmarkShard1000Read8Write0-8	 1000000	      2036 ns/op	     257 B/op	      16 allocs/op
30 | ok  	github.com/miolini/mapstore	54.959s
31 | ```


--------------------------------------------------------------------------------
/mapstore_single.go:
--------------------------------------------------------------------------------
  1 | package mapstore
  2 | 
  3 | import (
  4 | 	"sync"
  5 | )
  6 | 
  7 | var _ Store = (*StoreSingle)(nil)
  8 | 
  9 | type StoreSingle struct {
 10 | 	m sync.RWMutex
 11 | 	s map[string]interface{}
 12 | }
 13 | 
 14 | func newStoreSingle() *StoreSingle {
 15 | 	return &StoreSingle{
 16 | 		s: make(map[string]interface{}),
 17 | 	}
 18 | }
 19 | 
 20 | func (s *StoreSingle) Get(key string, defaultValue interface{}) (interface{}, bool) {
 21 | 	s.m.RLock()
 22 | 	value, ok := s.s[key]
 23 | 	s.m.RUnlock()
 24 | 	if !ok {
 25 | 		return defaultValue, false
 26 | 	}
 27 | 	return value, true
 28 | }
 29 | 
 30 | func (s *StoreSingle) Set(key string, value interface{}) {
 31 | 	s.m.Lock()
 32 | 	s.s[key] = value
 33 | 	s.m.Unlock()
 34 | }
 35 | 
 36 | func (s *StoreSingle) GetOrSet(key string, defaultValue interface{}) (interface{}, bool) {
 37 | 	s.m.Lock()
 38 | 	value, ok := s.s[key]
 39 | 	if !ok {
 40 | 		s.s[key] = defaultValue
 41 | 		value = defaultValue
 42 | 	}
 43 | 	s.m.Unlock()
 44 | 	return value, ok
 45 | }
 46 | 
 47 | func (s *StoreSingle) GetOrSetFunc(key string, newFn func() interface{}) (interface{}, bool) {
 48 | 	s.m.Lock()
 49 | 	value, ok := s.s[key]
 50 | 	if !ok {
 51 | 		value = newFn()
 52 | 		s.s[key] = value
 53 | 	}
 54 | 	s.m.Unlock()
 55 | 	return value, ok
 56 | }
 57 | 
 58 | func (s *StoreSingle) Delete(key string) bool {
 59 | 	s.m.Lock()
 60 | 	_, ok := s.s[key]
 61 | 	delete(s.s, key)
 62 | 	s.m.Unlock()
 63 | 	return ok
 64 | }
 65 | 
 66 | func (s *StoreSingle) Load(entries chan Entry) {
 67 | 	for entry := range entries {
 68 | 		s.m.Lock()
 69 | 		s.s[entry.Key] = entry.Value
 70 | 		s.m.Unlock()
 71 | 	}
 72 | }
 73 | 
 74 | func (s *StoreSingle) Save(entries chan<- Entry) {
 75 | 	s.m.RLock()
 76 | 	defer s.m.Unlock()
 77 | 	for k, v := range s.s {
 78 | 		entries <- Entry{k, v}
 79 | 	}
 80 | }
 81 | 
 82 | func (s *StoreSingle) ShardStats() []int {
 83 | 	return []int{s.Len()}
 84 | }
 85 | 
 86 | func (s *StoreSingle) Len() int {
 87 | 	s.m.RLock()
 88 | 	storeLen := len(s.s)
 89 | 	s.m.RUnlock()
 90 | 	return storeLen
 91 | }
 92 | 
 93 | func (s *StoreSingle) Update(key string, fn func(interface{}) interface{}) bool {
 94 | 	s.m.Lock()
 95 | 	defer s.m.Unlock()
 96 | 	v, ok := s.s[key]
 97 | 	s.s[key] = fn(v)
 98 | 	return ok
 99 | }
100 | 
101 | func (s *StoreSingle) UpdateIfExists(key string, fn func(interface{}) interface{}) bool {
102 | 	s.m.Lock()
103 | 	defer s.m.Unlock()
104 | 	v, ok := s.s[key]
105 | 	if !ok {
106 | 		return false
107 | 	}
108 | 	s.s[key] = fn(v)
109 | 	return ok
110 | }
111 | 


--------------------------------------------------------------------------------
/mapstore_test.go:
--------------------------------------------------------------------------------
  1 | package mapstore
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"math/rand"
  6 | 	"sync"
  7 | 	"testing"
  8 | )
  9 | 
 10 | func BenchmarkShard1Read4Write4(b *testing.B) {
 11 | 	bench(1, 4, 4, 1000000, b)
 12 | }
 13 | func BenchmarkShard10Read4Write4(b *testing.B) {
 14 | 	bench(10, 4, 4, 1000000, b)
 15 | }
 16 | func BenchmarkShard100Read4Write4(b *testing.B) {
 17 | 	bench(100, 4, 4, 1000000, b)
 18 | }
 19 | func BenchmarkShard1000Read4Write4(b *testing.B) {
 20 | 	bench(1000, 4, 4, 1000000, b)
 21 | }
 22 | func BenchmarkShard1Read8Write2(b *testing.B) {
 23 | 	bench(1, 8, 2, 1000000, b)
 24 | }
 25 | func BenchmarkShard10Read8Write2(b *testing.B) {
 26 | 	bench(10, 8, 2, 1000000, b)
 27 | }
 28 | func BenchmarkShard100Read8Write2(b *testing.B) {
 29 | 	bench(100, 8, 2, 1000000, b)
 30 | }
 31 | func BenchmarkShard1000Read8Write2(b *testing.B) {
 32 | 	bench(1000, 8, 2, 1000000, b)
 33 | }
 34 | func BenchmarkShard1Read2Write8(b *testing.B) {
 35 | 	bench(1, 2, 8, 1000000, b)
 36 | }
 37 | func BenchmarkShard10Read2Write8(b *testing.B) {
 38 | 	bench(10, 2, 8, 1000000, b)
 39 | }
 40 | func BenchmarkShard100Read2Write8(b *testing.B) {
 41 | 	bench(100, 2, 8, 1000000, b)
 42 | }
 43 | func BenchmarkShard1000Read2Write8(b *testing.B) {
 44 | 	bench(1000, 2, 8, 1000000, b)
 45 | }
 46 | func BenchmarkShard1Read8Write0(b *testing.B) {
 47 | 	bench(1, 8, 0, 1000000, b)
 48 | }
 49 | func BenchmarkShard10Read8Write0(b *testing.B) {
 50 | 	bench(10, 8, 0, 1000000, b)
 51 | }
 52 | func BenchmarkShard100Read8Write0(b *testing.B) {
 53 | 	bench(100, 8, 0, 1000000, b)
 54 | }
 55 | func BenchmarkShard1000Read8Write0(b *testing.B) {
 56 | 	bench(1000, 8, 0, 1000000, b)
 57 | }
 58 | 
 59 | func TestShardsStat(t *testing.T) {
 60 | 	keys := genKeys(1e6)
 61 | 	shards := 4
 62 | 	store := NewWithSize(shards)
 63 | 	for i := 0; i < 1e7; i++ {
 64 | 		key := keys[i%len(keys)]
 65 | 		store.Set(key, key)
 66 | 	}
 67 | 	t.Logf("stats: %v", store.ShardStats())
 68 | }
 69 | 
 70 | func genKeys(count int) []string {
 71 | 	keys := make([]string, count)
 72 | 	for i := 0; i < len(keys); i++ {
 73 | 		keys[i] = fmt.Sprintf("%d", i)
 74 | 	}
 75 | 	return keys
 76 | }
 77 | 
 78 | func bench(shards, readThreads, writeThreads int, keysCount int, b *testing.B) {
 79 | 	keys := genKeys(keysCount)
 80 | 	store := NewWithSize(shards)
 81 | 	wg := sync.WaitGroup{}
 82 | 	b.ResetTimer()
 83 | 	wg.Add(readThreads + writeThreads)
 84 | 	for i := 0; i < writeThreads; i++ {
 85 | 		go testWrites(store, keys, b.N, &wg)
 86 | 	}
 87 | 	for i := 0; i < readThreads; i++ {
 88 | 		go testReads(store, keys, b.N, &wg)
 89 | 	}
 90 | 	wg.Wait()
 91 | }
 92 | 
 93 | func testWrites(s Store, keys []string, num int, wg *sync.WaitGroup) {
 94 | 	defer wg.Done()
 95 | 	lenKeys := len(keys)
 96 | 	for i := 0; i < num; i++ {
 97 | 		key := keys[rand.Int()%lenKeys]
 98 | 		s.Set(key, key)
 99 | 	}
100 | }
101 | 
102 | func testReads(s Store, keys []string, num int, wg *sync.WaitGroup) {
103 | 	defer wg.Done()
104 | 	lenKeys := len(keys)
105 | 	for i := 0; i < num; i++ {
106 | 		key := keys[rand.Int()%lenKeys]
107 | 		s.Get(key, key)
108 | 	}
109 | }
110 | 


--------------------------------------------------------------------------------
/mapstore_shard.go:
--------------------------------------------------------------------------------
  1 | package mapstore
  2 | 
  3 | import (
  4 | 	"sync"
  5 | 
  6 | 	"github.com/spaolacci/murmur3"
  7 | )
  8 | 
  9 | var _ Store = (*StoreShard)(nil)
 10 | 
 11 | type shard struct {
 12 | 	m sync.RWMutex
 13 | 	s map[string]interface{}
 14 | }
 15 | 
 16 | type StoreShard struct {
 17 | 	m           sync.RWMutex
 18 | 	shards      []*shard
 19 | 	shardsCount int
 20 | }
 21 | 
 22 | func newShard() *shard {
 23 | 	return &shard{
 24 | 		s: make(map[string]interface{}),
 25 | 	}
 26 | }
 27 | 
 28 | func newStoreShard(shardsCount int) *StoreShard {
 29 | 	store := &StoreShard{}
 30 | 
 31 | 	store.shardsCount = shardsCount
 32 | 	store.shards = make([]*shard, shardsCount)
 33 | 
 34 | 	for k := range store.shards {
 35 | 		store.shards[k] = newShard()
 36 | 	}
 37 | 
 38 | 	return store
 39 | }
 40 | 
 41 | func (s *StoreShard) getShard(key string) *shard {
 42 | 	s.m.RLock()
 43 | 	defer s.m.RUnlock()
 44 | 	num := int(murmur3.Sum64([]byte(key))>>1) % s.shardsCount
 45 | 	return s.shards[num]
 46 | }
 47 | 
 48 | func (s *StoreShard) Get(key string, defaultValue interface{}) (interface{}, bool) {
 49 | 	shard := s.getShard(key)
 50 | 	shard.m.RLock()
 51 | 	value, ok := shard.s[key]
 52 | 	shard.m.RUnlock()
 53 | 
 54 | 	if !ok {
 55 | 		return defaultValue, false
 56 | 	}
 57 | 	return value, true
 58 | }
 59 | 
 60 | func (s *StoreShard) GetOrSet(key string, defaultValue interface{}) (interface{}, bool) {
 61 | 	shard := s.getShard(key)
 62 | 	shard.m.Lock()
 63 | 	value, ok := shard.s[key]
 64 | 	if !ok {
 65 | 		shard.s[key] = defaultValue
 66 | 		value = defaultValue
 67 | 	}
 68 | 	shard.m.Unlock()
 69 | 	return value, ok
 70 | }
 71 | 
 72 | func (s *StoreShard) GetOrSetFunc(key string, newFn func() interface{}) (interface{}, bool) {
 73 | 	shard := s.getShard(key)
 74 | 	shard.m.Lock()
 75 | 	value, ok := shard.s[key]
 76 | 	if !ok {
 77 | 		value = newFn()
 78 | 		shard.s[key] = value
 79 | 	}
 80 | 	shard.m.Unlock()
 81 | 	return value, ok
 82 | }
 83 | 
 84 | func (s *StoreShard) Delete(key string) bool {
 85 | 	shard := s.getShard(key)
 86 | 	shard.m.Lock()
 87 | 	_, ok := shard.s[key]
 88 | 	delete(shard.s, key)
 89 | 	shard.m.Unlock()
 90 | 	return ok
 91 | }
 92 | 
 93 | func (s *StoreShard) Set(key string, value interface{}) {
 94 | 	shard := s.getShard(key)
 95 | 	shard.m.Lock()
 96 | 	shard.s[key] = value
 97 | 	shard.m.Unlock()
 98 | }
 99 | 
100 | func (s *StoreShard) Load(entries chan Entry) {
101 | 	s.m.Lock()
102 | 	defer s.m.Unlock()
103 | 	for entry := range entries {
104 | 		shard := s.getShard(entry.Key)
105 | 		shard.s[entry.Key] = entry.Value
106 | 	}
107 | }
108 | 
109 | func (s *StoreShard) Save(entries chan<- Entry) {
110 | 	s.m.RLock()
111 | 	defer s.m.RUnlock()
112 | 	for _, shard := range s.shards {
113 | 		for key, value := range shard.s {
114 | 			entries <- Entry{key, value}
115 | 		}
116 | 	}
117 | }
118 | 
119 | func (s *StoreShard) ShardStats() []int {
120 | 	result := make([]int, s.shardsCount)
121 | 	for i := 0; i < s.shardsCount; i++ {
122 | 		shard := s.shards[i]
123 | 		shard.m.RLock()
124 | 		result[i] = len(shard.s)
125 | 		shard.m.RUnlock()
126 | 	}
127 | 	return result
128 | }
129 | 
130 | func (s *StoreShard) Len() int {
131 | 	var len int
132 | 	stats := s.ShardStats()
133 | 	for _, shardLen := range stats {
134 | 		len += shardLen
135 | 	}
136 | 	return len
137 | }
138 | 
139 | func (s *StoreShard) Update(key string, f func(v interface{}) interface{}) bool {
140 | 	shard := s.getShard(key)
141 | 	shard.m.Lock()
142 | 	defer shard.m.Unlock()
143 | 	v, ok := shard.s[key]
144 | 	shard.s[key] = f(v)
145 | 	return ok
146 | }
147 | 
148 | func (s *StoreShard) UpdateIfExists(key string, f func(v interface{}) interface{}) bool {
149 | 	shard := s.getShard(key)
150 | 	shard.m.Lock()
151 | 	defer shard.m.Unlock()
152 | 	v, ok := shard.s[key]
153 | 	if !ok {
154 | 		return false
155 | 	}
156 | 	shard.s[key] = f(v)
157 | 	return ok
158 | }
159 | 


--------------------------------------------------------------------------------