├── .gitignore ├── README ├── project.clj ├── src └── persister │ └── core.clj └── test └── persister └── test └── core.clj /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | *jar 3 | /lib/ 4 | /classes/ 5 | .lein-failures 6 | .lein-deps-sum 7 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Simple Persistence for Clojure is a journal-based persistence library for Clojure programs. It follows "Prevalent system" design pattern. 2 | 3 | The intended usage is assist you in making a prevalent system. Thus you work with your in-memory data and wrap every writing call into one of (apply-transaction*) macros. 4 | 5 | Also it is possible to use it just as the journaling layer to record the (writing) transactions that happened to your system and replay them later. 6 | 7 | Transactions are logged as a valid Clojure code, so they are easy to read and run separately. (Just wrap them into dosync) 8 | 9 | Probably under certain conditions it can be a good fit for the prototyping stage of a project development. 10 | 11 | However this pattern is used in production by some teams, see Prevayler mail list for details. Of course this implementation can contain bugs and must be tested well before doing that. Though it is very simple and I made some tests. 12 | 13 | The disadvantage of the pattern is that your data structures must fit in memory (unless you implement ad-hoc paging solution). However the journaling nature lets you easily switch to other databases. For that you just re-implement your transaction functions (subjects of apply-transaction* ) to write/read from another DB and replay transactions one time (init-db). 14 | 15 | Snapshotting is not implemented. It can solve another pattern problem - growing startup time. 16 | 17 | In comparison with Prevayler, this library does not block the reads, because it relies on Clojure STM. However it blocks the writes as Prevayler. To avoid this, atoms can be used to generate a transaction id without locking, but that will make reading and backup logic much more complex. 18 | 19 | Probably blocking of writes is not important now, when the most of today computers have <=8 cores and the typical usage pattern is that there are an order of magnitude more reads than writes. 20 | 21 | Usage examples: 22 | 23 | 1. first run 24 | 25 | (use 'persister.core) 26 | 27 | (def refx (ref 0)) 28 | (def refy (ref 0)) 29 | (def refs (ref {})) 30 | 31 | (defn tr-fn [x y] 32 | (do 33 | (alter refx + x) 34 | (alter refy + y) )) 35 | 36 | (defn tr-fn-swap [] 37 | (let [tmp @refx] 38 | (ref-set refx @refy) 39 | (ref-set refy tmp))) 40 | 41 | (defn tr-inc [] 42 | (ref-set refx (inc @refx)) 43 | (ref-set refy (inc @refy)) ) 44 | 45 | (defn tr-set-s [new-s] 46 | (ref-set refs new-s) ) 47 | 48 | (init-db) 49 | (apply-transaction tr-fn 1 2) 50 | (apply-transaction tr-fn 10 20) 51 | (apply-transaction tr-fn-swap) 52 | (apply-transaction tr-inc) 53 | (apply-transaction tr-set-s {:a :bb "key" #{"val1" :val2}}) 54 | [refx refy refs] 55 | 56 | [# # #] 57 | 58 | 2. the second run 59 | 60 | (use 'persister.core) 61 | 62 | (def refx (ref 0)) 63 | (def refy (ref 0)) 64 | (def refs (ref {})) 65 | 66 | (defn tr-fn [x y] 67 | (do 68 | (alter refx + x) 69 | (alter refy + y) )) 70 | 71 | (defn tr-fn-swap [] 72 | (let [tmp @refx] 73 | (ref-set refx @refy) 74 | (ref-set refy tmp))) 75 | 76 | (defn tr-inc [] 77 | (ref-set refx (inc @refx)) 78 | (ref-set refy (inc @refy)) ) 79 | 80 | (defn tr-set-s [new-s] 81 | (ref-set refs new-s) ) 82 | 83 | (init-db) 84 | [refx refy refs] 85 | 86 | [# # #] 87 | 88 | 89 | Note that journaled functions must be accessible in the current namespace when you replay transactions. 90 | 91 | See inline doc for details. 92 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject persister "1.0.1" 2 | :description "Simple Persistence for Clojure" 3 | :dependencies [[org.clojure/clojure "1.3.0"]] 4 | :local-repo-classpath true) 5 | -------------------------------------------------------------------------------- /src/persister/core.clj: -------------------------------------------------------------------------------- 1 | ;;;; Simple journal-based persistence for Clojure 2 | 3 | ;;;; by Sergey Didenko 4 | ;;;; last updated Sep 11, 2011 5 | 6 | ;;;; Copyright (c) Sergey Didenko, 2011. All rights reserved. The use 7 | ;;;; and distribution terms for this software are covered by the Eclipse 8 | ;;;; Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 9 | ;;;; which can be found in the file epl-v10.html at the root of this 10 | ;;;; distribution. By using this software in any fashion, you are 11 | ;;;; agreeing to be bound by the terms of this license. You must not 12 | ;;;; remove this notice, or any other, from this software. 13 | 14 | (ns #^{ 15 | :author "Sergey Didenko", 16 | :doc "Simple journal-based persistence for Clojure 17 | 18 | Basics: 19 | 20 | See README. 21 | 22 | WARNING! Do not use atoms inside transaction handlers. 23 | Atom actions are not rollbacked in a failing dosync block. 24 | 25 | Apply-transaction macro uses the smart buffer, 26 | apply-transaction-and-block writes immediately, see their docs. 27 | 28 | The module switches the current journal file with the given interval, 29 | to make it easy to backup the live system. 30 | 31 | Usage: 32 | 33 | (init-db) 34 | ... 35 | (apply-transaction transaction-wo-dosync1 param1 param2 param3) 36 | (apply-transaction transaction-wo-dosync2) 37 | (apply-transaction transaction-wo-dosync3 param1) 38 | ... 39 | (shutdown-agents) 40 | 41 | OR 42 | 43 | (init-db) 44 | ... 45 | (apply-transaction-and-block transaction-wo-dosync1 param1 param2 param3) 46 | (apply-transaction-and-block transaction-wo-dosync2) 47 | (apply-transaction-and-block transaction-wo-dosync3 param1) 48 | ... 49 | (shutdown-agents) 50 | 51 | 52 | Notes: 53 | 54 | - Snapshotting is not yet implemented. 55 | 56 | - Currently str function is used to log transaction parameters in a readable way 57 | 58 | - Relies on the assumption that messages sent to an agent from a locked area 59 | will save their order when sent from different threads. This concludes from: 60 | 61 | 1. the agent contract ( messages sent from the same thread are not reordered) 62 | 2. all transactions are applied and messages sent in the single locking area 63 | 3. the Clojure code does not make any aditional messages reordering 64 | after they are put into agent queues 65 | 66 | "} 67 | persister.core 68 | 69 | (:import (java.io FileOutputStream File PrintWriter OutputStreamWriter 70 | BufferedReader InputStreamReader FileInputStream)) 71 | (:require [clojure.string :as string])) 72 | 73 | ;;; Temporaly adapted from deprecated clojure.contrib.duck-streams 74 | (defn read-lines 75 | "Like clojure.core/line-seq but opens f with reader. Automatically 76 | closes the reader AFTER YOU CONSUME THE ENTIRE SEQUENCE." 77 | [^String f] 78 | (let [read-line (fn this [^BufferedReader rdr] 79 | (lazy-seq 80 | (if-let [line (.readLine rdr)] 81 | (cons line (this rdr)) 82 | (.close rdr))))] 83 | ;; 84 | (read-line 85 | (BufferedReader. (InputStreamReader. (FileInputStream. (File. f))))))) 86 | ;; (read-line (reader f)))) 87 | 88 | (def buffering-agent 89 | (agent { 90 | :pending-transactions [] 91 | :buffer-first-transaction-id 0}) ) 92 | 93 | (def writing-agent (agent { 94 | :fos nil, 95 | :writer nil, 96 | :journal-creation-time nil, 97 | :directory "database" 98 | :file-change-interval 1000 })) 99 | 100 | ;;; Used to check if there is an ongoing write operation (in this module) 101 | (def io-indicator-lock (java.util.concurrent.locks.ReentrantLock.) ) 102 | 103 | (def transaction-lock (java.util.concurrent.locks.ReentrantLock.) ) 104 | 105 | (def transaction-counter (atom 0M)) 106 | 107 | ;;; change journal file regularly 108 | (defn- time-to-change-journal-file 109 | [journal-creation-time interval] 110 | (not (when journal-creation-time 111 | (< (- (System/currentTimeMillis) journal-creation-time) interval) ))) 112 | 113 | (defn- change-journal-file-on-time [agent-state first-transaction-id] 114 | (let [ 115 | journal-creation-time (:journal-creation-time agent-state) 116 | writer (:writer agent-state) 117 | ] 118 | (if (time-to-change-journal-file journal-creation-time (:file-change-interval agent-state)) 119 | (do 120 | ;; close the old file 121 | (when journal-creation-time (.close writer)) 122 | ;; open the new file 123 | (let [new-creation-time (System/currentTimeMillis) 124 | filename (str (:directory agent-state) "/" first-transaction-id ".journal") 125 | fos (FileOutputStream. filename) 126 | writer (PrintWriter. (OutputStreamWriter. fos "UTF-8")) 127 | ] 128 | (assoc agent-state :fos fos :writer writer :journal-creation-time new-creation-time) 129 | ) 130 | ) 131 | agent-state ))) 132 | 133 | (defn serialized-transaction 134 | [transaction-id & transaction-params] 135 | (str "(" (string/join " " (map pr-str transaction-params)) ") ;" transaction-id) ) 136 | 137 | (declare try-flushing-smart-buffer) 138 | 139 | (defn- log-to-file [agent-state serialized-transaction first-transaction-id] 140 | ;; using lock only to indicate that there is an ongoing file operation 141 | (.lock io-indicator-lock) 142 | (try 143 | (let [ 144 | new-agent-state (change-journal-file-on-time agent-state first-transaction-id) 145 | filename (:filename new-agent-state) 146 | fos (:fos new-agent-state) 147 | writer (:writer new-agent-state) 148 | ] 149 | (.print writer (str serialized-transaction "\n")) 150 | (.flush writer) 151 | (.. fos getFD sync) 152 | new-agent-state ) 153 | 154 | (finally 155 | (.unlock io-indicator-lock) 156 | (try-flushing-smart-buffer)))) 157 | 158 | (defn persist-string [serialized-transaction first-transaction-id] 159 | (send writing-agent log-to-file serialized-transaction first-transaction-id) ) 160 | 161 | (defn- log-to-smart-buffer 162 | [agent-state serialized-transaction first-transaction-id] 163 | (let [ 164 | ongoing-transaction (.isLocked io-indicator-lock) 165 | pending-transactions (:pending-transactions agent-state) 166 | 167 | new-buffer-first-transaction-id 168 | (if (and serialized-transaction (empty? pending-transactions)) 169 | first-transaction-id 170 | (:buffer-first-transaction-id agent-state)) 171 | 172 | new-pending-transactions 173 | (if serialized-transaction 174 | (conj pending-transactions serialized-transaction) 175 | pending-transactions) 176 | ] 177 | (if (or ongoing-transaction (empty? new-pending-transactions ) ) 178 | (assoc agent-state 179 | :pending-transactions new-pending-transactions 180 | :buffer-first-transaction-id new-buffer-first-transaction-id) 181 | (do 182 | (persist-string (string/join "\n" new-pending-transactions ) new-buffer-first-transaction-id) 183 | (assoc agent-state :pending-transactions []) )))) 184 | 185 | (defn persist-string-in-smart-buffer 186 | [serialized-transaction first-transaction-id] 187 | (send buffering-agent log-to-smart-buffer serialized-transaction first-transaction-id) ) 188 | 189 | (def try-flushing-smart-buffer 190 | (partial persist-string-in-smart-buffer nil nil)) 191 | 192 | (defmacro apply-transaction 193 | "Apply transaction to the root object and write it to disk unless 194 | the transaction fails. Disk writes are made through the buffering 195 | agent, so there is a small chance to lose the latest succesfully applied 196 | transactions on account of disk failure. 197 | Use apply-transaction-and-block if you want to further reduce the chance of 198 | the possible loss at the expense of reduced throughput. 199 | Warning: do not mix the both macros in the same workflow!" 200 | [transaction-fn & transaction-fn-arg] 201 | `(locking transaction-lock 202 | (let [ 203 | res# (dosync (~transaction-fn ~@transaction-fn-arg)) 204 | transaction-id# (swap! transaction-counter inc) 205 | ] 206 | (persist-string-in-smart-buffer 207 | (serialized-transaction transaction-id# '~transaction-fn ~@transaction-fn-arg) 208 | transaction-id#) 209 | res# ))) 210 | 211 | (defmacro apply-transaction-and-block 212 | "Apply transaction to the root object and block until it is flushed to disk. 213 | Blocking happens outside the transaction, so there is a really small chance 214 | that in-memory changes will be visible from other threads considerably 215 | earlier than disk flush happens. 216 | Use apply-transaction if you want better throughput at the expense of losing 217 | more transactions on account of disk failure. 218 | Warning: do not mix the both macros in the same workflow!" 219 | [transaction-fn & transaction-fn-arg] 220 | `(locking transaction-lock 221 | (let [ 222 | res# (dosync (~transaction-fn ~@transaction-fn-arg)) 223 | transaction-id# (swap! transaction-counter inc) 224 | ] 225 | (persist-string 226 | (serialized-transaction transaction-id# '~transaction-fn ~@transaction-fn-arg) 227 | transaction-id#) 228 | (await writing-agent) 229 | res# ))) 230 | 231 | (defn- initialize-wr-agent [agent-state data-directory file-change-interval-in-seconds] 232 | (assoc agent-state 233 | :directory data-directory 234 | :file-change-interval (* 1000 file-change-interval-in-seconds) )) 235 | 236 | (defn- db-file-names [data-directory re] 237 | (map #(BigDecimal. %) 238 | (filter #(not( nil? %)) 239 | (map #(second (re-matches re (.getName %))) 240 | (seq (.listFiles (java.io.File. data-directory))) )))) 241 | 242 | (defn- journal-numbers [data-directory] 243 | (db-file-names data-directory #"(\d+)\.journal$")) 244 | 245 | (defn- snapshot-numbers [data-directory] 246 | (db-file-names data-directory #"(\d+)\.snapshot$")) 247 | 248 | ;;; 249 | ;;; 250 | (defn- make-str-join-n 251 | "returns the function that joins consecutive items into string, decorates it, 252 | and returns sequence ([processed-number-accumulator joined-items-chunk]...) 253 | " 254 | [n start-str join-str end-str] 255 | (fn joinn [coll acc-size] 256 | (lazy-seq 257 | (when-let [s (seq coll) ] 258 | (let [tr-list (take n s) 259 | new-acc-size (+ acc-size (count tr-list))] 260 | (cons 261 | [new-acc-size 262 | (str start-str (apply string/join (cons join-str (list tr-list))) end-str) 263 | ] 264 | (joinn (drop n s) new-acc-size ) )))))) 265 | 266 | (defn init-db 267 | "Make sure to call it before any apply-transaction* call" 268 | ([] 269 | ;; set default change file time to 15 minutes, and transaction chunk size to 1000 270 | (init-db "database" (* 60 15) 1000)) 271 | 272 | ([data-directory file-change-interval transaction-chunk-size] 273 | ;; create data directory if it does not exist 274 | (let [data-dir (File. data-directory )] 275 | (if (.exists data-dir) 276 | (when-not 277 | (.isDirectory data-dir) 278 | (throw (RuntimeException. (str "\"" data-dir "\" must be a directory"))) ) 279 | (when-not 280 | (.mkdir data-dir) 281 | (throw (RuntimeException. (str "Can't create database directory \"" data-dir "\""))) ))) 282 | 283 | ;; initialize agent 284 | (send writing-agent initialize-wr-agent data-directory file-change-interval) 285 | ;; load transactions 286 | (let [str-join-dosync (make-str-join-n transaction-chunk-size "(dosync\n" "\n" "\n)")] 287 | (doseq [ 288 | journal-number (sort (journal-numbers data-directory)) 289 | [last-transaction-id chunk-to-load] 290 | (str-join-dosync 291 | (read-lines (str data-directory "/" journal-number ".journal")) 292 | (dec journal-number)) 293 | ] 294 | (load-string chunk-to-load) 295 | (reset! transaction-counter last-transaction-id) )))) 296 | -------------------------------------------------------------------------------- /test/persister/test/core.clj: -------------------------------------------------------------------------------- 1 | (ns persister.test.core 2 | (:use [persister.core]) 3 | (:use [clojure.test])) 4 | 5 | (deftest test-serialized-transaction 6 | (is (= 7 | "(5 \"param\") ;1" 8 | (serialized-transaction 1 5 "param") ))) 9 | 10 | (deftest test-make-str-join-n 11 | (let [str-join-dosync (@#'persister.core/make-str-join-n 3 "(" "-" ")")] 12 | (doseq [chunk (str-join-dosync (take 7 (iterate inc 1)) 0)] 13 | (condp = (first chunk) 14 | 3 (is (= (second chunk) "(1-2-3)")) 15 | 6 (is (= (second chunk) "(4-5-6)")) 16 | 7 (is (= (second chunk) "(7)")) 17 | (throw (RuntimeException. "wrong accumulator value test-make-str-join-n")) )))) 18 | --------------------------------------------------------------------------------