├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── docs
    └── nuummite.png
├── shard.yml
├── spec
    ├── benchmark.cr
    ├── nuummite_spec.cr
    └── spec_helper.cr
└── src
    ├── locking
        └── locking.cr
    └── nuummite.cr


/.gitignore:
--------------------------------------------------------------------------------
 1 | /doc/
 2 | /libs/
 3 | /.crystal/
 4 | /.shards/
 5 | 
 6 | 
 7 | # Libraries don't need dependency lock
 8 | # Dependencies will be locked in application that uses them
 9 | /shard.lock
10 | 
11 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: crystal
2 | crystal:
3 |   - latest
4 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 CodeSteak
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <img src="https://raw.githubusercontent.com/codesteak/nuummite/master/docs/nuummite.png" width="25%" align="right">
  2 | 
  3 | # Nuummite [![Build Status](https://travis-ci.org/CodeSteak/Nuummite.svg?branch=master)](https://travis-ci.org/CodeSteak/Nuummite)
  4 | 
  5 | Nuummite is a tiny persistent embedded key-value store. All data is kept
  6 | in RAM (in a Crystal Hash) and is also written to disk.
  7 | So don't use Nuummite to handle big chunks of data.
  8 | Keys and Values are always Strings.
  9 | It just comes with the most basic operations.
 10 | 
 11 | 
 12 | ## Installation
 13 | 
 14 | Add this to your application's `shard.yml`:
 15 | 
 16 | ```yaml
 17 | dependencies:
 18 |   nuummite:
 19 |     github: codesteak/nuummite
 20 |     version: ~> 0.1.5
 21 | ```
 22 | 
 23 | 
 24 | ## Usage
 25 | 
 26 | ```crystal
 27 | require "nuummite"
 28 | ```
 29 | 
 30 | #### Open the database
 31 | ```crystal
 32 | db = Nuummite.new("path/to/folder", "optional-filename.db")
 33 | 
 34 | # You can also have multiple:
 35 | flowers  = Nuummite.new("flowers")
 36 | minerals = Nuummite.new("minerals")
 37 | ```
 38 | 
 39 | #### Put some values in
 40 | ```crystal
 41 | db["hello"] = "world"
 42 | db["42"] = "Answer to the Ultimate Question of Life, The Universe, and Everything"
 43 | db["whitespace"] = "Hey\n\t\t :D"
 44 | ```
 45 | 
 46 | #### Read values
 47 | ```crystal
 48 | db["hello"]? # => "world"
 49 | db["ehhhh"]? # => nil
 50 | 
 51 | #Note: db is locked while reading, so don't write to db!
 52 | db.each do |key,value|
 53 |   # reads everything
 54 | end
 55 | 
 56 | db["crystals/ruby"] = "9.0"
 57 | db["crystals/quartz"] = "~ 7.0"
 58 | db["crystals/nuummite"] = "5.5 - 6.0"
 59 | 
 60 | db.each("crystals/") do |key,value|
 61 |   # only crystals in here
 62 | end
 63 | ```
 64 | 
 65 | #### Delete
 66 | ```crystal
 67 | db.delete "hello"
 68 | ```
 69 | 
 70 | #### Garbage collect
 71 | Since values are saved to disk in a log style, file sizes grow,
 72 | your key-value store needs to rewrite all data at some point:
 73 | ```crystal
 74 | db.garbage_collect
 75 | ```
 76 | By default it auto garbage collects after 10_000_000 writes.
 77 | To modify this behavior you can:
 78 | ```crystal
 79 | # garbage collects after 1000 writes or deletes
 80 | db.auto_garbage_collect_after_writes = 1000
 81 | 
 82 | # does not auto garbage collect
 83 | db.auto_garbage_collect_after_writes = nil  
 84 | ```
 85 | 
 86 | #### Shutdown
 87 | You can also shutdown Nuummite by
 88 | ```crystal
 89 | db.shutdown
 90 | ```
 91 | 
 92 | That's all you need to know :smile:
 93 | 
 94 | ## Contributing
 95 | 
 96 | 1. Fork it ( https://github.com/codesteak/nuummite/fork )
 97 | 2. Create your feature branch (git checkout -b my-new-feature)
 98 | 3. Commit your changes (git commit -am 'Add some feature')
 99 | 4. Push to the branch (git push origin my-new-feature)
100 | 5. Create a new Pull Request
101 | 
102 | ## Contributors
103 | 
104 | - [CodeSteak](https://github.com/CodeSteak) - creator, maintainer
105 | 


--------------------------------------------------------------------------------
/docs/nuummite.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CodeSteak/Nuummite/fbbdb6f3e7817a38e218aa2954ceb6fe0f19013d/docs/nuummite.png


--------------------------------------------------------------------------------
/shard.yml:
--------------------------------------------------------------------------------
1 | name: nuummite
2 | version: 0.1.5
3 | 
4 | authors:
5 |   - CodeSteak <CodeSteak@users.noreply.github.com>
6 | 
7 | license: MIT
8 | 


--------------------------------------------------------------------------------
/spec/benchmark.cr:
--------------------------------------------------------------------------------
 1 | require "benchmark"
 2 | require "../src/nuummite"
 3 | 
 4 | puts
 5 | puts
 6 | puts "      user     system      total        real"
 7 | puts
 8 | 
 9 | db = Nuummite.new("benchmark_db")
10 | 
11 | db.auto_garbage_collect_after_writes = nil
12 | db.sync = false
13 | 
14 | puts "1000_000 small writes"
15 | puts Benchmark.measure {
16 |   1000_000.times do |i|
17 |     db["#{i}"] = "I <3 DATA"
18 |   end
19 | }
20 | puts
21 | 
22 | puts "1000_000 small deletes"
23 | puts Benchmark.measure {
24 |   1000_000.times do |i|
25 |     db.delete "#{i}"
26 |   end
27 | }
28 | puts
29 | 
30 | db.shutdown
31 | sleep 0.01
32 | 
33 | puts "reopen after 2000_000 operations"
34 | puts Benchmark.measure {
35 |   db = Nuummite.new("benchmark_db")
36 | }
37 | puts
38 | 
39 | 1000_000.times do |i|
40 |   db["#{i}/lol"] = "I <3 DATA !!!!"
41 | end
42 | 
43 | puts "garbage collect with 1000_000 entries"
44 | puts Benchmark.measure {
45 |   db.garbage_collect
46 | }
47 | puts
48 | 
49 | db.shutdown
50 | clean("benchmark_db")
51 | 
52 | def clean(dir_name = "tmpdb")
53 |   Dir.foreach(dir_name) do |filename|
54 |     path = "#{dir_name}#{File::SEPARATOR}#{filename}"
55 |     File.delete(path) if File.file?(path)
56 |   end
57 |   Dir.delete(dir_name)
58 | end
59 | 


--------------------------------------------------------------------------------
/spec/nuummite_spec.cr:
--------------------------------------------------------------------------------
  1 | require "./spec_helper"
  2 | 
  3 | def with_db(name)
  4 |   db = Nuummite.new("tmpdb", name)
  5 |   yield db
  6 |   db.shutdown
  7 | end
  8 | 
  9 | describe Nuummite do
 10 |   it "make new db and save state, do operations" do
 11 |     with_db("one") do |db|
 12 |       db["a"] = "aaa"
 13 |       db["b"] = "bbb"
 14 |       db["c"] = "ccc"
 15 |       db["d"] = "nope"
 16 |       db["e"] = "eee"
 17 | 
 18 |       db.delete("e")
 19 |       db["d"] = "ddd"
 20 |     end
 21 | 
 22 |     with_db("one") do |db|
 23 |       db["a"]?.should eq("aaa")
 24 |       db["b"]?.should eq("bbb")
 25 |       db["c"]?.should eq("ccc")
 26 |       db["d"]?.should eq("ddd")
 27 |       db["e"]?.should eq(nil)
 28 |     end
 29 |   end
 30 | 
 31 |   it "open empty file" do
 32 |     #create empty file
 33 |     folder = "tmpdb";
 34 |     Dir.mkdir(folder) unless Dir.exists?(folder)
 35 | 
 36 |     path = "#{folder}#{File::SEPARATOR}empty"
 37 | 
 38 |     file = File.new(path, "a")
 39 |     file.close()
 40 |     # and reopen it
 41 |     with_db("empty") do |db|
 42 |       db["a"] = "aaa"
 43 |       db["a"]?.should eq("aaa")
 44 |     end
 45 |   end
 46 | 
 47 |   it "open empty alt file" do
 48 |     #create empty alt file
 49 |     folder = "tmpdb";
 50 |     Dir.mkdir(folder) unless Dir.exists?(folder)
 51 | 
 52 |     path = "#{folder}#{File::SEPARATOR}empty2.1"
 53 | 
 54 |     file = File.new(path, "a")
 55 |     file.close()
 56 |     # and reopen it
 57 |     with_db("empty2") do |db|
 58 |       db["a"] = "aaa"
 59 |       db["a"]?.should eq("aaa")
 60 |     end
 61 |   end
 62 | 
 63 |   it "make new db and save state, do operations and garbage_collect" do
 64 |     with_db("one") do |db|
 65 |       db["a"] = "a✌a"
 66 |       db["b"] = "bbb"
 67 |       db["c"] = "ccc"
 68 |       db["d"] = "nope"
 69 |       db["e"] = "eee"
 70 | 
 71 |       db.delete("e")
 72 |       db["d"] = "ddd"
 73 | 
 74 |       db.garbage_collect
 75 |     end
 76 | 
 77 |     with_db("one") do |db|
 78 |       db["a"]?.should eq("a✌a")
 79 |       db["b"]?.should eq("bbb")
 80 |       db["c"]?.should eq("ccc")
 81 |       db["d"]?.should eq("ddd")
 82 |       db["e"]?.should eq(nil)
 83 |     end
 84 |   end
 85 | 
 86 |   it "each" do
 87 |     with_db("each") do |db|
 88 |       db["a"] = ">D"
 89 |       db["crystals/ruby"] = ""
 90 |       db["crystals/quartz"] = ""
 91 |       db["crystals/nuummite"] = ""
 92 | 
 93 |       i = 0
 94 |       db.each do
 95 |         i += 1
 96 |       end
 97 |       i.should eq(4)
 98 | 
 99 |       i = 0
100 |       db.each("crystals/") do
101 |         i += 1
102 |       end
103 |       i.should eq(3)
104 | 
105 |       db["a"]?.should eq(">D")
106 |     end
107 |   end
108 | 
109 |   it "garbage collect db" do
110 |     with_db("two") do |db|
111 |       1000.times do |i|
112 |         db["key"] = "#{i}"
113 |       end
114 |       db.garbage_collect
115 |     end
116 |     file_size = File.size("tmpdb/two")
117 |     (file_size < 100).should be_true
118 |   end
119 | 
120 |   it "auto garbage collect db" do
121 |     with_db("three") do |db|
122 |       db.auto_garbage_collect_after_writes = 10000
123 |       10001.times do |i|
124 |         db["#{i}"] = "data"*5
125 |         db.delete "#{i}"
126 |       end
127 |       sleep 0.01
128 |       file_size = File.size("tmpdb/three")
129 |       (file_size < 100).should be_true
130 |     end
131 |   end
132 | 
133 |   clean()
134 | end
135 | 
136 | def clean(dir_name = "tmpdb")
137 |   Dir.new(dir_name).each do |filename|
138 |     path = "#{dir_name}#{File::SEPARATOR}#{filename}"
139 |     File.delete(path) if File.file?(path)
140 |   end
141 |   Dir.delete(dir_name)
142 | end
143 | 


--------------------------------------------------------------------------------
/spec/spec_helper.cr:
--------------------------------------------------------------------------------
1 | require "spec"
2 | require "../src/nuummite"
3 | require "./benchmark"
4 | 


--------------------------------------------------------------------------------
/src/locking/locking.cr:
--------------------------------------------------------------------------------
 1 | # Allows basic locking.
 2 | module Locking
 3 |   @running = true
 4 |   @channel_lock = Channel(Nil).new
 5 |   @channel_unlock = Channel(Nil).new
 6 | 
 7 |   private def lock
 8 |     @channel_lock.receive
 9 |   end
10 | 
11 |   private def unlock
12 |     @channel_unlock.send nil
13 |   end
14 | 
15 |   private def save
16 |     lock
17 |     yield
18 |   ensure
19 |     unlock
20 |   end
21 | 
22 |   private def enable_locking
23 |     spawn do
24 |       manage_locks
25 |     end
26 |   end
27 | 
28 |   private def disable_locking
29 |     save do
30 |       @running = false
31 |     end
32 |   end
33 | 
34 |   private def manage_locks
35 |     while @running
36 |       @channel_lock.send nil
37 |       @channel_unlock.receive
38 |     end
39 |   end
40 | end
41 | 


--------------------------------------------------------------------------------
/src/nuummite.cr:
--------------------------------------------------------------------------------
  1 | require "./locking/*"
  2 | 
  3 | # Nuummite is a minimalistic persistent key-value store.
  4 | class Nuummite
  5 |   include Locking
  6 | 
  7 |   VERSION = 1
  8 | 
  9 |   # Number of writes or deletes before automatic garbage collection happens.
 10 |   #
 11 |   # Set it to nil to disable automatic garbage collection.
 12 |   property auto_garbage_collect_after_writes : Int32? = 10_000_000
 13 | 
 14 |   # If true the logfile is flushed on every write.
 15 |   property sync : Bool
 16 | 
 17 |   @log : File
 18 |   @kv : Hash(String, String)
 19 | 
 20 |   enum Opcode : UInt8
 21 |     RENAME = 3
 22 |     REMOVE = 2
 23 |     WRITE  = 1
 24 |   end
 25 | 
 26 |   def initialize(folder : String, @filename = "db.nuummite", @sync = true)
 27 |     @need_gc = false
 28 |     @log, @kv = open_folder(folder, @filename)
 29 | 
 30 |     enable_locking
 31 |     garbage_collect if @need_gc
 32 |   end
 33 | 
 34 |   private def open_folder(folder, filename) : {File, Hash(String, String)}
 35 |     Dir.mkdir(folder) unless Dir.exists?(folder)
 36 | 
 37 |     path = "#{folder}#{File::SEPARATOR}#{filename}"
 38 |     alt_path = "#{folder}#{File::SEPARATOR}#{filename}.1"
 39 | 
 40 |     new_file = false
 41 | 
 42 |     kv = if File.exists?(path) && File.size(path) > 0
 43 |            read_file_to_kv path
 44 |          elsif File.exists?(alt_path) && File.size(alt_path) > 0
 45 |            File.rename(alt_path, path)
 46 |            read_file_to_kv path
 47 |          else
 48 |            new_file = true
 49 |            Hash(String, String).new
 50 |          end
 51 | 
 52 |     file = File.new(path, "a")
 53 |     if new_file
 54 |       file.write_byte(VERSION.to_u8)
 55 |       file.flush
 56 |     end
 57 |     {file, kv}
 58 |   end
 59 | 
 60 |   @writes = 0
 61 |   private def check_autogc
 62 |     if autogc = @auto_garbage_collect_after_writes
 63 |       @writes += 1
 64 |       if @writes > autogc
 65 |         @writes = 0
 66 |         spawn do
 67 |           garbage_collect
 68 |         end
 69 |       end
 70 |     end
 71 |   end
 72 | 
 73 |   # Shuts down this instance of Nuummite.
 74 |   def shutdown
 75 |     disable_locking
 76 |     @log.flush
 77 |     @log.close
 78 |   end
 79 | 
 80 |   # Delets a key. Returns its value.
 81 |   def delete(key)
 82 |     save do
 83 |       log_remove(key)
 84 |       @kv.delete(key)
 85 |     end
 86 |   ensure
 87 |     check_autogc
 88 |   end
 89 | 
 90 |   # Set key to value.
 91 |   def []=(key, value)
 92 |     save do
 93 |       log_write(key, value)
 94 |       @kv[key] = value
 95 |     end
 96 |   ensure
 97 |     check_autogc
 98 |   end
 99 | 
100 |   # Reads value to given key.
101 |   def [](key)
102 |     @kv[key]
103 |   end
104 | 
105 |   # Reads value to given key. Returns `nil` if key is not avilable.
106 |   def []?(key)
107 |     @kv[key]?
108 |   end
109 | 
110 |   # Yields every key-value pair where the key starts with `starts_with`.
111 |   # ```
112 |   # db["crystals/ruby"] = "9.0"
113 |   # db["crystals/quartz"] = "~ 7.0"
114 |   # db["crystals/nuummite"] = "5.5 - 6.0"
115 |   #
116 |   # db.each("crystals/") do |key, value|
117 |   #   # only crystals in here
118 |   # end
119 |   # ```
120 |   # `starts_with` defaults to `""`. Then every key-value pair is yield.
121 |   def each(starts_with : String = "")
122 |     save do
123 |       @kv.each do |key, value|
124 |         if key.starts_with?(starts_with)
125 |           yield key, value
126 |         end
127 |       end
128 |     end
129 |   end
130 | 
131 |   # Rewrites current state to logfile.
132 |   def garbage_collect
133 |     save do
134 |       path = @log.path
135 |       alt_path = "#{path}.1"
136 |       File.delete(alt_path) if File.exists?(alt_path)
137 | 
138 |       @log.flush
139 |       @log.close
140 | 
141 |       sync = @sync
142 | 
143 |       @log = File.new(alt_path, "w")
144 |       @sync = false
145 | 
146 |       @log.write_byte(VERSION.to_u8)
147 |       @kv.each do |key, value|
148 |         log_write(key, value)
149 |       end
150 |       @log.flush
151 |       @log.close
152 | 
153 |       @sync = sync
154 | 
155 |       File.delete(path)
156 |       File.rename(alt_path, path)
157 | 
158 |       @log = File.new(path, "a")
159 |     end
160 |   end
161 | 
162 |   private def log_write(key, value)
163 |     log Opcode::WRITE, key, value
164 |   end
165 | 
166 |   private def log_remove(key)
167 |     log Opcode::REMOVE, key
168 |   end
169 | 
170 |   private def log_rename(key_old, key_new)
171 |     log Opcode::RENAME, key_old, key_new
172 |   end
173 | 
174 |   private def log(opcode, arg0, arg1 = nil)
175 |     @log.write_byte(opcode.value.to_u8)
176 | 
177 |     write_string_arg(@log, arg0)
178 |     write_string_arg(@log, arg1) if arg1
179 | 
180 |     @log.flush if @sync
181 |   end
182 | 
183 |   private def write_string_arg(io, value)
184 |     data = value.bytes
185 | 
186 |     io.write_bytes(data.size.to_i32, IO::ByteFormat::NetworkEndian)
187 |     data.each do |b|
188 |       io.write_byte(b)
189 |     end
190 |   end
191 | 
192 |   private def read_file_to_kv(path)
193 |     kv = Hash(String, String).new
194 | 
195 |     file = File.new(path)
196 |     version = file.read_byte.not_nil!
197 |     raise Exception.new("Unsupported Version #{version}") if version != 1
198 | 
199 |     begin
200 |       while opcode = file.read_byte
201 |         case Opcode.new opcode
202 |         when Opcode::WRITE
203 |           key = read_string_arg file
204 |           value = read_string_arg file
205 | 
206 |           kv[key] = value
207 |         when Opcode::REMOVE
208 |           key = read_string_arg file
209 | 
210 |           kv.delete key
211 |         when Opcode::RENAME
212 |           key_old = read_string_arg file
213 |           key_new = read_string_arg file
214 | 
215 |           kv[key_new] = kv[key_old]
216 |           kv.delete key_old
217 |         else
218 |           raise Exception.new("Invalid format: Opcode #{opcode}")
219 |         end
220 |       end
221 |     rescue ex : IO::EOFError
222 |       puts "Data is incomplete. \
223 |             Please set sync to true to prevent this type of data corruption"
224 |       puts "Continue..."
225 |       @need_gc = true
226 |     end
227 |     file.close
228 |     kv
229 |   end
230 | 
231 |   private def read_string_arg(io)
232 |     size = read_int(io)
233 |     read_string(io, size)
234 |   end
235 | 
236 |   private def read_int(io)
237 |     value = 0
238 |     4.times { value <<= 8; value += io.read_byte.not_nil! }
239 |     value
240 |   end
241 | 
242 |   private def read_string(io, size)
243 |     data = Slice(UInt8).new(size)
244 |     io.read_fully(data)
245 |     String.new(data)
246 |   end
247 | end
248 | 


--------------------------------------------------------------------------------