├── .gitignore ├── README.md ├── project.clj ├── src └── clj_leveldb.clj └── test └── clj_leveldb_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /lib 3 | /classes 4 | /checkouts 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | .lein-deps-sum 10 | .lein-failures 11 | .lein-plugins 12 | .lein-repl-history 13 | /doc 14 | push 15 | .nrepl* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a self-contained wrapper around [LevelDB](https://code.google.com/p/leveldb/), which provides all the necessary binaries via [leveldbjni](https://github.com/fusesource/leveldbjni). 2 | 3 | ### basic usage 4 | 5 | ```clj 6 | [factual/clj-leveldb "0.1.2"] 7 | ``` 8 | 9 | To create or access a database, use `clj-leveldb/create-db`: 10 | 11 | ```clj 12 | clj-leveldb> (def db (create-db "/tmp/leveldb" {})) 13 | #'clj-leveldb/db 14 | ``` 15 | 16 | This database object can now be used with `clj-leveldb/get`, `put`, `delete`, `batch`, and `iterator`. 17 | 18 | ```clj 19 | clj-leveldb> (put db "a" "b") 20 | nil 21 | clj-leveldb> (get db "a") 22 | # 23 | ``` 24 | 25 | Notice that the value returned is a byte-array. This is because byte arrays are the native storage format for LevelDB, and we haven't defined custom encoders and decoders. This can be done in `create-db`: 26 | 27 | ```clj 28 | clj-leveldb> (def db (create-db "/tmp/leveldb" 29 | {:key-decoder byte-streams/to-string 30 | :val-decoder byte-streams/to-string})) 31 | #'clj-leveldb/db 32 | clj-leveldb> (get db "a") 33 | "b" 34 | ``` 35 | 36 | Notice that we haven't defined `key-encoder` or `val-encoder`; this is because there's a default transformation between strings and byte-arrays, which assumes a utf-8 encoding. If we wanted to support keywords, or use a different encoding, we'd have to explicitly specify encoders. 37 | 38 | Both `put` and `delete` can take multiple values, which will be written in batch: 39 | 40 | ```clj 41 | clj-leveldb> (put db "a" "b" "c" "d" "e" "f") 42 | nil 43 | clj-leveldb> (delete db "a" "c" "e") 44 | nil 45 | ``` 46 | 47 | If you need to batch a collection of puts and deletes, use `batch`: 48 | 49 | ```clj 50 | clj-leveldb> (batch db {:put ["a" "b" "c" "d"] :delete ["j" "k" "l"]}) 51 | ``` 52 | 53 | We can also get a sequence of all key/value pairs, either in the entire database or within a given range using `iterator`: 54 | 55 | ```clj 56 | clj-leveldb> (put db "a" "b" "c" "d" "e" "f") 57 | nil 58 | clj-leveldb> (iterator db) 59 | (["a" "b"] ["c" "d"] ["e" "f"]) 60 | clj-leveldb> (iterator db "c" nil) 61 | (["c" "d"] ["e" "f"]) 62 | clj-leveldb> (iterator db nil "c") 63 | (["a" "b"] ["c" "d"]) 64 | ``` 65 | 66 | Syncing writes to disk can be forced via `sync`, and compaction can be forced via `compact`. 67 | 68 | Full documentation can be found [here](http://factual.github.io/clj-leveldb/). 69 | 70 | ### license 71 | 72 | Copyright © 2013 Factual, Inc. 73 | 74 | Distributed under the Eclipse Public License, the same as Clojure. 75 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject factual/clj-leveldb "0.1.2" 2 | :description "an idiomatic wrapper for LevelDB" 3 | :license {:name "Eclipse Public License" 4 | :url "http://www.eclipse.org/legal/epl-v10.html"} 5 | :dependencies [[org.fusesource.leveldbjni/leveldbjni-all "1.8"] 6 | [org.iq80.leveldb/leveldb-api "0.7"] 7 | [byte-streams "0.1.13"]] 8 | :profiles {:dev {:dependencies [[org.clojure/clojure "1.5.1"] 9 | [criterium "0.4.3"] 10 | [codox-md "0.2.0" :exclusions [org.clojure/clojure]]]}} 11 | :plugins [[codox "0.6.4"]] 12 | :codox {:writer codox-md.writer/write-docs 13 | :include [clj-leveldb]}) 14 | -------------------------------------------------------------------------------- /src/clj_leveldb.clj: -------------------------------------------------------------------------------- 1 | (ns clj-leveldb 2 | (:refer-clojure :exclude [get sync]) 3 | (:require 4 | [clojure.java.io :as io] 5 | [byte-streams :as bs]) 6 | (:import 7 | [java.io 8 | Closeable 9 | File])) 10 | 11 | ;; HawtJNI tends to leave trash in /tmp, repeated loading of this 12 | ;; namespace can add up. We allow the 10 newest files to stick 13 | ;; around in case there's some sort of contention, since we're only 14 | ;; trying to put an upper bound on how many of these files stick around. 15 | (let [tmp-dir (str 16 | (System/getProperty "java.io.tmpdir") 17 | (System/getProperty "file.separator") 18 | "com.factual.clj-leveldb")] 19 | (let [d (io/file tmp-dir)] 20 | (doseq [f (->> (.listFiles d) 21 | (sort-by #(.lastModified ^File %)) 22 | reverse 23 | (drop 10))] 24 | (.delete ^File f))) 25 | (System/setProperty "library.leveldbjni.path" tmp-dir)) 26 | 27 | (import 28 | '[org.fusesource.leveldbjni 29 | JniDBFactory] 30 | '[org.iq80.leveldb 31 | WriteBatch 32 | DBIterator 33 | Options 34 | ReadOptions 35 | WriteOptions 36 | CompressionType 37 | DB 38 | Range]) 39 | 40 | ;;; 41 | 42 | (defn- closeable-seq 43 | "Creates a seq which can be closed, given a latch which can be closed 44 | and dereferenced to check whether it's already been closed." 45 | [s close-fn] 46 | (if (empty? s) 47 | 48 | ;; if we've exhausted the seq, just close it 49 | (do 50 | (close-fn) 51 | nil) 52 | 53 | (reify 54 | 55 | Closeable 56 | (close [this] 57 | (close-fn)) 58 | 59 | clojure.lang.Sequential 60 | clojure.lang.ISeq 61 | clojure.lang.Seqable 62 | clojure.lang.IPersistentCollection 63 | (equiv [this x] 64 | (loop [a this, b x] 65 | (if (or (empty? a) (empty? b)) 66 | (and (empty? a) (empty? b)) 67 | (if (= (first x) (first b)) 68 | (recur (rest a) (rest b)) 69 | false)))) 70 | (empty [_] 71 | []) 72 | (count [this] 73 | (count (seq this))) 74 | (cons [_ a] 75 | (cons a s)) 76 | (next [this] 77 | (closeable-seq (next s) close-fn)) 78 | (more [this] 79 | (let [rst (next this)] 80 | (if (empty? rst) 81 | '() 82 | rst))) 83 | (first [_] 84 | (first s)) 85 | (seq [this] 86 | this)))) 87 | 88 | (defn- iterator-seq- [^DBIterator iterator start end key-decoder key-encoder val-decoder] 89 | (if start 90 | (.seek ^DBIterator iterator (bs/to-byte-array (key-encoder start))) 91 | (.seekToFirst ^DBIterator iterator)) 92 | 93 | (let [s (iterator-seq iterator) 94 | s (if end 95 | (let [end (bs/to-byte-array (key-encoder end))] 96 | (take-while 97 | #(not (pos? (bs/compare-bytes (key %) end))) 98 | s)) 99 | s)] 100 | (closeable-seq 101 | (map 102 | #(vector 103 | (key-decoder (key %)) 104 | (val-decoder (val %))) 105 | s) 106 | (reify 107 | Object 108 | (finalize [_] (.close iterator)) 109 | clojure.lang.IFn 110 | (invoke [_] (.close iterator)))))) 111 | 112 | ;;; 113 | 114 | (defprotocol ILevelDB 115 | (^:private ^DB db- [_]) 116 | (^:private batch- [_] [_ options]) 117 | (^:private iterator- [_] [_ start end]) 118 | (^:private get- [_ k]) 119 | (^:private put- [_ k v options]) 120 | (^:private del- [_ k options]) 121 | (^:private snapshot- [_])) 122 | 123 | (defrecord Snapshot 124 | [db 125 | key-decoder 126 | key-encoder 127 | val-decoder 128 | ^ReadOptions read-options] 129 | ILevelDB 130 | (snapshot- [this] this) 131 | (db- [_] (db- db)) 132 | (get- [_ k] 133 | (val-decoder (.get (db- db) (bs/to-byte-array (key-encoder k)) read-options))) 134 | (iterator- [_ start end] 135 | (iterator-seq- 136 | (.iterator (db- db) read-options) 137 | start 138 | end 139 | key-decoder 140 | key-encoder 141 | val-decoder)) 142 | Closeable 143 | (close [_] 144 | (-> read-options .snapshot .close)) 145 | (finalize [this] (.close this))) 146 | 147 | (defrecord Batch 148 | [^DB db 149 | ^WriteBatch batch 150 | key-encoder 151 | val-encoder 152 | ^WriteOptions options] 153 | ILevelDB 154 | (db- [_] db) 155 | (batch- [this _] this) 156 | (put- [_ k v _] 157 | (.put batch 158 | (bs/to-byte-array (key-encoder k)) 159 | (bs/to-byte-array (val-encoder v)))) 160 | (del- [_ k _] 161 | (.delete batch (bs/to-byte-array (key-encoder k)))) 162 | Closeable 163 | (close [_] 164 | (if options 165 | (.write db batch options) 166 | (.write db batch)) 167 | (.close batch))) 168 | 169 | (defrecord LevelDB 170 | [^DB db 171 | key-decoder 172 | key-encoder 173 | val-decoder 174 | val-encoder] 175 | Closeable 176 | (close [_] (.close db)) 177 | ILevelDB 178 | (db- [_] 179 | db) 180 | (get- [_ k] 181 | (let [k (bs/to-byte-array (key-encoder k))] 182 | (val-decoder (.get db k)))) 183 | (put- [_ k v options] 184 | (let [k (bs/to-byte-array (key-encoder k)) 185 | v (bs/to-byte-array (val-encoder v))] 186 | (if options 187 | (.put db k v options) 188 | (.put db k v)))) 189 | (del- [_ k options] 190 | (let [k (bs/to-byte-array (key-encoder k))] 191 | (if options 192 | (.delete db k options) 193 | (.delete db k)))) 194 | (snapshot- [this] 195 | (->Snapshot 196 | this 197 | key-decoder 198 | key-encoder 199 | val-decoder 200 | (doto (ReadOptions.) 201 | (.snapshot (.getSnapshot db))))) 202 | (batch- [this options] 203 | (->Batch 204 | db 205 | (.createWriteBatch db) 206 | key-encoder 207 | val-encoder 208 | options)) 209 | (iterator- [_ start end] 210 | (iterator-seq- 211 | (.iterator db) 212 | start 213 | end 214 | key-decoder 215 | key-encoder 216 | val-decoder))) 217 | 218 | ;;; 219 | 220 | (def ^:private option-setters 221 | {:create-if-missing? #(.createIfMissing ^Options %1 %2) 222 | :error-if-exists? #(.errorIfExists ^Options %1 %2) 223 | :write-buffer-size #(.writeBufferSize ^Options %1 %2) 224 | :block-size #(.blockSize ^Options %1 %2) 225 | :block-restart-interval #(.blockRestartInterval ^Options %1 %2) 226 | :max-open-files #(.maxOpenFiles ^Options %1 %2) 227 | :cache-size #(.cacheSize ^Options %1 %2) 228 | :comparator #(.comparator ^Options %1 %2) 229 | :paranoid-checks? #(.paranoidChecks ^Options %1 %2) 230 | :compress? #(.compressionType ^Options %1 (if % CompressionType/SNAPPY CompressionType/NONE)) 231 | :logger #(.logger ^Options %1 %2)}) 232 | 233 | (defn create-db 234 | "Creates a closeable database object, which takes a directory and zero or more options. 235 | 236 | The key and val encoder/decoders are functions for transforming to and from byte-arrays." 237 | [directory 238 | {:keys [key-decoder 239 | key-encoder 240 | val-decoder 241 | val-encoder 242 | create-if-missing? 243 | error-if-exists? 244 | write-buffer-size 245 | block-size 246 | max-open-files 247 | cache-size 248 | comparator 249 | compress? 250 | paranoid-checks? 251 | block-restart-interval 252 | logger] 253 | :or {key-decoder identity 254 | key-encoder identity 255 | val-decoder identity 256 | val-encoder identity 257 | compress? true 258 | cache-size (* 32 1024 1024) 259 | block-size (* 16 1024) 260 | write-buffer-size (* 32 1024 1024) 261 | create-if-missing? true 262 | error-if-exists? false} 263 | :as options}] 264 | (->LevelDB 265 | (.open JniDBFactory/factory 266 | (io/file directory) 267 | (let [opts (Options.)] 268 | (doseq [[k v] options] 269 | (when (and v (contains? option-setters k)) 270 | ((option-setters k) opts v))) 271 | opts)) 272 | key-decoder 273 | key-encoder 274 | val-decoder 275 | val-encoder)) 276 | 277 | (defn destroy-db 278 | "Destroys the database at the specified `directory`." 279 | [directory] 280 | (.destroy JniDBFactory/factory 281 | (io/file directory) 282 | (Options.))) 283 | 284 | (defn repair-db 285 | "Repairs the database at the specified `directory`." 286 | [directory] 287 | (.repair JniDBFactory/factory 288 | (io/file directory) 289 | (Options.))) 290 | 291 | ;;; 292 | 293 | (defn get 294 | "Returns the value of `key` for the given database or snapshot. If the key doesn't exist, returns 295 | `default-value` or nil." 296 | ([db key] 297 | (get db key nil)) 298 | ([db key default-value] 299 | (let [v (get- db key)] 300 | (if (nil? v) 301 | default-value 302 | v)))) 303 | 304 | (defn snapshot 305 | "Returns a snapshot of the database that can be used with `get` and `iterator`. This implements 306 | java.io.Closeable, and can leak space in the database if not closed." 307 | [db] 308 | (snapshot- db)) 309 | 310 | (defn iterator 311 | "Returns a closeable sequence of map entries (accessed with `key` and `val`) that is the inclusive 312 | range from `start `to `end`. If exhausted, the sequence is automatically closed." 313 | ([db] 314 | (iterator db nil nil)) 315 | ([db start] 316 | (iterator db start nil)) 317 | ([db start end] 318 | (iterator- db start end))) 319 | 320 | (defn put 321 | "Puts one or more key/value pairs into the given `db`." 322 | ([db] 323 | ) 324 | ([db key val] 325 | (put- db key val nil)) 326 | ([db key val & key-vals] 327 | (with-open [^Batch batch (batch- db nil)] 328 | (put- batch key val nil) 329 | (doseq [[k v] (partition 2 key-vals)] 330 | (put- batch k v nil))))) 331 | 332 | (defn delete 333 | "Deletes one or more keys in the given `db`." 334 | ([db] 335 | ) 336 | ([db key] 337 | (del- db key nil)) 338 | ([db key & keys] 339 | (with-open [^Batch batch (batch- db nil)] 340 | (del- batch key nil) 341 | (doseq [k keys] 342 | (del- batch k nil))))) 343 | 344 | (defn sync 345 | "Forces the database to fsync." 346 | [db] 347 | (with-open [^Batch batch (batch- db (doto (WriteOptions.) (.sync true)))] 348 | )) 349 | 350 | (defn stats 351 | "Returns statistics for the database." 352 | [db property] 353 | (.getProperty (db- db) "leveldb.stats")) 354 | 355 | (defn bounds 356 | "Returns a tuple of the lower and upper keys in the database or snapshot." 357 | [db] 358 | (let [key-decoder (:key-decoder db)] 359 | (with-open [^DBIterator iterator (condp instance? db 360 | LevelDB (.iterator (db- db)) 361 | Snapshot (.iterator (db- db) (:read-options db)))] 362 | (when (.hasNext (doto iterator .seekToFirst)) 363 | [(-> (doto iterator .seekToFirst) .peekNext key key-decoder) 364 | (-> (doto iterator .seekToLast) .peekNext key key-decoder)])))) 365 | 366 | (defn approximate-size 367 | "Returns an estimate of the size of entries, in bytes, inclusively between `start` and `end`." 368 | ([db] 369 | (apply approximate-size db (bounds db))) 370 | ([db start end] 371 | (let [key-encoder (:key-encoder db)] 372 | (first 373 | (.getApproximateSizes (db- db) 374 | (into-array 375 | [(Range. 376 | (bs/to-byte-array (key-encoder start)) 377 | (bs/to-byte-array (key-encoder end)))])))))) 378 | 379 | (defn compact 380 | "Forces compaction of database over the given range. If `start` or `end` are nil, they default to 381 | the full range of the database." 382 | ([db] 383 | (compact db nil nil)) 384 | ([db start] 385 | (compact db start nil)) 386 | ([db start end] 387 | (let [encoder (:key-encoder db) 388 | [start' end'] (bounds db) 389 | start (or start start') 390 | end (or end end')] 391 | (when (and start end) 392 | (.compactRange (db- db) 393 | (bs/to-byte-array (encoder start)) 394 | (bs/to-byte-array (encoder end))))))) 395 | 396 | (defn batch 397 | "Batch a collection of put and/or delete operations into the supplied `db`. 398 | Takes a map of the form `{:put [key1 value1 key2 value2] :delete [key3 key4]}`. 399 | If `:put` key is provided, it must contain an even-length sequence of alternating keys and values." 400 | ([db] ) 401 | ([db {puts :put deletes :delete}] 402 | (assert (even? (count puts)) ":put option requires even number of keys and values.") 403 | (with-open [^Batch batch (batch- db nil)] 404 | (doseq [[k v] (partition 2 puts)] 405 | (put- batch k v nil)) 406 | (doseq [k deletes] 407 | (del- batch k nil))))) 408 | -------------------------------------------------------------------------------- /test/clj_leveldb_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-leveldb-test 2 | (:require 3 | [clojure.test :refer :all] 4 | [clj-leveldb :as l] 5 | [byte-streams :as bs] 6 | [clojure.edn :as edn]) 7 | (:import 8 | [java.util 9 | UUID] 10 | [java.io 11 | File])) 12 | 13 | 14 | (def db 15 | (l/create-db 16 | (doto (File. (str "/tmp/" (UUID/randomUUID))) 17 | .deleteOnExit) 18 | {:key-encoder name 19 | :key-decoder (comp keyword bs/to-string) 20 | :val-decoder (comp edn/read-string bs/to-char-sequence) 21 | :val-encoder pr-str})) 22 | 23 | (deftest test-basic-operations 24 | (l/put db :a :b) 25 | (is (= :b 26 | (l/get db :a) 27 | (l/get db :a ::foo))) 28 | (is (= [[:a :b]] 29 | (l/iterator db) 30 | (l/iterator db :a) 31 | (l/iterator db :a :a) 32 | (l/iterator db :a :c))) 33 | (is (= nil 34 | (l/iterator db :b) 35 | (l/iterator db :b :d))) 36 | (l/delete db :a) 37 | (is (= nil (l/get db :a))) 38 | (is (= ::foo (l/get db :a ::foo))) 39 | 40 | (l/put db :a :b :z :y) 41 | (is (= :b (l/get db :a))) 42 | (is (= :y (l/get db :z))) 43 | 44 | (is (= [[:a :b] [:z :y]] 45 | (l/iterator db))) 46 | (is (= [[:a :b]] 47 | (l/iterator db :a :x) 48 | (l/iterator db nil :x))) 49 | (is (= [[:z :y]] 50 | (l/iterator db :b) 51 | (l/iterator db :b :z))) 52 | 53 | (is (= [:a :z] (l/bounds db))) 54 | 55 | (l/compact db) 56 | 57 | (with-open [snapshot (l/snapshot db)] 58 | (l/delete db :a :z) 59 | (is (= nil (l/get db :a))) 60 | (is (= :b (l/get snapshot :a)))) 61 | 62 | (l/compact db) 63 | 64 | (l/delete db :a :b :z :y) 65 | 66 | (l/put db :j :k :l :m) 67 | (is (= :k (l/get db :j))) 68 | (is (= :m (l/get db :l))) 69 | 70 | (l/batch db) 71 | (is (= :k (l/get db :j))) 72 | (is (= :m (l/get db :l))) 73 | (l/batch db {:put [:r :s :t :u]}) 74 | (is (= :s (l/get db :r))) 75 | (is (= :u (l/get db :t))) 76 | (l/batch db {:delete [:r :t]}) 77 | (is (= nil (l/get db :r))) 78 | (is (= nil (l/get db :t))) 79 | 80 | (l/batch db {:put [:a :b :c :d] 81 | :delete [:j :l]}) 82 | (is (= :b (l/get db :a))) 83 | (is (= :d (l/get db :c))) 84 | (is (= nil (l/get db :j))) 85 | (is (= nil (l/get db :l))) 86 | (is (thrown? AssertionError (l/batch db {:put [:a]})))) 87 | --------------------------------------------------------------------------------