├── .gitignore ├── project.clj ├── test ├── logback.xml └── com │ └── manigfeald │ ├── single_test.clj │ ├── knossos.clj │ ├── raft_test.clj │ └── raft │ └── rules_test.clj ├── README.md ├── src └── com │ └── manigfeald │ ├── raft.clj │ └── raft │ ├── log.clj │ ├── core.clj │ └── rules.clj └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | logs/ 11 | doc/ 12 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject com.manigfeald/raft "0.2.0-SNAPSHOT" 2 | :description "abstract raft algorithm written in clojure" 3 | :url "http://example.com/FIXME" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.6.0"]] 7 | :profiles {:dev {:dependencies [[org.clojure/core.async "0.1.303.0-886421-alpha"] 8 | [org.clojure/tools.logging "0.2.6"] 9 | ;;[org.slf4j/slf4j-nop "1.7.2"] 10 | [robert/bruce "0.7.1"] 11 | [ch.qos.logback/logback-classic "1.0.9"] 12 | [ch.qos.logback/logback-core "1.0.9"] 13 | [org.slf4j/jcl-over-slf4j "1.7.2"] 14 | [org.clojure/test.check "0.5.8"] 15 | [knossos "0.2" 16 | :exclusions [org.slf4j/slf4j-log4j12]]]}}) 17 | -------------------------------------------------------------------------------- /test/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ␂%r჻%thread჻%logger{36}჻%msg␃%n 10 | 11 | 12 | 13 | 14 | logs/raft-%d{yyyy-MM-dd}.%i.log 15 | 17 | 18 | 64 MB 19 | 20 | 21 | 22 | 23 | true 24 | 25 | 26 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # raft 2 | 3 | A implementation of the Raft consensus algorithm written in 4 | Clojure. This implementation has a heavy emphasis on providing an 5 | algorithm disentangled as much as possible from concerns like network 6 | transport. 7 | 8 | This implementation provides a function `com.manigfeald.raft/raft` 9 | that creates a value that describes the state of the algorithm, and a 10 | function `com.manigfeald.raft/run-one` that steps a given state of the 11 | algorithm to the next state. 12 | 13 | The state of the raft algorithm is described in five records: 14 | 15 | 1. ImplementationState 16 | - The overall container of the rest of the state 17 | 2. Timer 18 | - a number representing now 19 | - a number representing the next timeout 20 | - the period between timeouts 21 | 3. IO 22 | - a possible message received 23 | - a PersistentQueue of messages to send out 24 | 4. RaftState 25 | - raft state common to all nodes 26 | 5. RaftLeaderState 27 | - raft state used by the leader node 28 | 29 | ## Why is this so weird? 30 | 31 | You can think of it more as a model of a Raft implementation, but the 32 | model is in clojure so it is easy enough to just wire the model in to 33 | some transports and run it. Like wiring in to a spreadsheet model, but 34 | without the instinctive gag reflex of wiring in to a spreadsheet. 35 | 36 | ## Usage 37 | 38 | `[com.manigfeald/raft 0.1.0]` 39 | 40 | call `com.manigfeald.raft/raft` to get an initial state, call 41 | `com.manigfeald.raft/run-one` to advance the state. if you want the 42 | node you are modeling to receive a message, assoc the message in to 43 | `[:io :message]` in the state, `[:io :out-queue]` will contain messages 44 | the node you are modeling wants to send out. Update the value of 45 | `[:timer :now]` to model the passage of time. 46 | 47 | the tests have an example using core.async channels as the transport 48 | 49 | ## Generated Docs 50 | 51 | http://ce2144dc-f7c9-4f54-8fb6-7321a4c318db.s3.amazonaws.com/raft/index.html 52 | 53 | ## See also 54 | 55 | https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf 56 | 57 | ## What is missing 58 | 59 | The raft paper also has snapshot generation to avoid logs growing too 60 | large and an interesting scheme for changing cluster membership, both 61 | of which are unimplemented. 62 | 63 | The raft paper also talks about providing a fast path for reads that 64 | avoids the entire log and commit process, I have left that out over 65 | fears of screwing it up, so reads and writes both go through the 66 | entire consensus process 67 | 68 | ## License 69 | 70 | Copyright © 2014 Kevin Downey 71 | 72 | Distributed under the Eclipse Public License either version 1.0 or (at 73 | your option) any later version. 74 | -------------------------------------------------------------------------------- /test/com/manigfeald/single_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.single-test 2 | (:require [clojure.test :refer :all] 3 | [com.manigfeald.raft :refer :all] 4 | [clojure.test.check :as tc] 5 | [clojure.test.check.generators :as gen] 6 | [clojure.test.check.properties :as prop] 7 | [clojure.test.check.clojure-test :refer [defspec]])) 8 | 9 | (defn n-rafts [n delays] 10 | (for [i (range n)] 11 | {:id i 12 | :in-queue clojure.lang.PersistentQueue/EMPTY 13 | :raft (raft i (set (range n)) (->Timer 0 0 (get delays i)))})) 14 | 15 | (defn step-n-rafts [rafts instructions] 16 | (let [messages (group-by :target (mapcat (comp :out-queue :io :raft) rafts)) 17 | rafts (for [{:keys [id in-queue raft]} rafts] 18 | {:id id 19 | :raft (update-in raft [:io :out-queue] empty) 20 | :in-queue (into in-queue (get messages id))})] 21 | (for [{:keys [id in-queue raft]} rafts 22 | :let [inst (get instructions id) 23 | raft (update-in raft [:timer :now] inc)]] 24 | (case inst 25 | :drop-message {:id id 26 | :raft (run-one (assoc-in raft [:io :message] nil)) 27 | :in-queue (if (empty? in-queue) 28 | in-queue 29 | (pop in-queue))} 30 | :receive-message (let [message (when-not (empty? in-queue) 31 | (peek in-queue)) 32 | in-queue (if (empty? in-queue) 33 | in-queue 34 | (pop in-queue))] 35 | {:id id 36 | :raft (run-one (assoc-in raft 37 | [:io :message] 38 | message)) 39 | :in-queue in-queue}) 40 | :stall {:id id 41 | :raft raft 42 | :in-queue in-queue} 43 | :no-message {:id id 44 | :raft (run-one (assoc-in raft [:io :message] nil)) 45 | :in-queue in-queue})))) 46 | 47 | (def instructions 48 | (gen/elements [:drop-message :receive-message :stall :no-message])) 49 | 50 | (defn programs [machines] 51 | (fn [program-size] 52 | (gen/vector (apply gen/tuple (repeat machines instructions)) 53 | program-size))) 54 | 55 | (def zero-or-one-leader-per-term 56 | (prop/for-all 57 | [delays (gen/tuple gen/s-pos-int 58 | gen/s-pos-int 59 | gen/s-pos-int) 60 | program (gen/bind gen/nat (programs 3))] 61 | (every? 62 | #(or (= % 0) (= % 1)) 63 | (for [{:keys [rafts]} (take-while 64 | identity 65 | (iterate 66 | (fn [{:keys [rafts program]}] 67 | (when (seq program) 68 | {:program (rest program) 69 | :rafts (step-n-rafts rafts 70 | (first program))})) 71 | {:program program 72 | :rafts (n-rafts 3 delays)})) 73 | n (vals (frequencies 74 | (for [{{{:keys [node-type current-term]} :raft-state} :raft} 75 | rafts 76 | :let [_ (assert node-type node-type)] 77 | :when (= :leader node-type)] 78 | current-term)))] 79 | n)))) 80 | 81 | (defspec zero-or-one-leader-per-term-spec 82 | 10000 83 | zero-or-one-leader-per-term) 84 | 85 | (comment 86 | 87 | 88 | (tc/quick-check 10 zero-or-one-leader-per-term) 89 | 90 | 91 | ) 92 | 93 | (defmacro foo [exp & body] 94 | (when-not (eval exp) 95 | `(do ~@body))) 96 | 97 | (foo (resolve 'clojure.test/original-test-var) 98 | (in-ns 'clojure.test) 99 | 100 | (def original-test-var test-var) 101 | 102 | (defn test-var [v] 103 | (clojure.tools.logging/trace "testing" v) 104 | (let [start (System/currentTimeMillis) 105 | result (original-test-var v)] 106 | (clojure.tools.logging/trace 107 | "testing" v "took" 108 | (/ (- (System/currentTimeMillis) start) 1000.0) 109 | "seconds") 110 | result)) 111 | 112 | (def original-test-ns test-ns) 113 | 114 | (defn test-ns [ns] 115 | (clojure.tools.logging/trace "testing namespace" ns) 116 | (let [start (System/currentTimeMillis) 117 | result (original-test-ns ns)] 118 | (clojure.tools.logging/trace 119 | "testing namespace" ns "took" 120 | (/ (- (System/currentTimeMillis) start) 1000.0) 121 | "seconds") 122 | result)) 123 | 124 | (in-ns 'com.manigfeald.raft-test)) 125 | -------------------------------------------------------------------------------- /src/com/manigfeald/raft.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.raft 2 | "runs the raft algorithm one step at a time." 3 | (:require [com.manigfeald.raft.core :refer :all] 4 | [com.manigfeald.raft.rules :refer [rules-of-raft]]) 5 | (:import (clojure.lang PersistentQueue))) 6 | 7 | ;; TODO: why do the tests still need with-pings with leaders sending noops? 8 | ;; TODO: document message formats 9 | ;; defrecords mainly just to document the expected fields 10 | (defrecord RaftLeaderState [next-index match-index]) 11 | (defrecord RaftState [current-term voted-for log commit-index last-applied 12 | node-type value votes leader-id node-set]) 13 | (defrecord IO [message out-queue]) 14 | (defrecord Timer [now next-timeout period]) 15 | (defrecord ImplementationState [io raft-state raft-leader-state id running-log 16 | timer]) 17 | 18 | (alter-meta! #'map->RaftLeaderState assoc :no-doc true) 19 | (alter-meta! #'map->RaftState assoc :no-doc true) 20 | (alter-meta! #'map->IO assoc :no-doc true) 21 | (alter-meta! #'map->Timer assoc :no-doc true) 22 | (alter-meta! #'map->ImplementationState assoc :no-doc true) 23 | (alter-meta! #'->RaftLeaderState assoc :no-doc true) 24 | (alter-meta! #'->RaftState assoc :no-doc true) 25 | (alter-meta! #'->IO assoc :no-doc true) 26 | (alter-meta! #'->Timer assoc :no-doc true) 27 | (alter-meta! #'->ImplementationState assoc :no-doc true) 28 | 29 | (defn raft 30 | "return an init state when given a node id and a node-set, id is the 31 | id of the node this state represents, node-set is a set of the ids 32 | of other nodes in the cluster" 33 | [id node-set timer] 34 | (->ImplementationState 35 | (->IO nil PersistentQueue/EMPTY) 36 | (->RaftState 0N 37 | nil 38 | (empty-log) 39 | 0N 40 | 0N 41 | :follower 42 | (->MapValue) 43 | 0N 44 | nil 45 | (set node-set)) 46 | (->RaftLeaderState {} {}) 47 | id 48 | PersistentQueue/EMPTY 49 | timer)) 50 | 51 | (defn run-one 52 | "given a state, advance it to the next state" 53 | [raft-state] 54 | {:post [(not (seq (for [message (:out-queue (:io %)) 55 | :when (= (:type message) :request-vote-response) 56 | :when (:success? message) 57 | :when (not= (:voted-for (:raft-state %)) 58 | (:target message))] 59 | message))) 60 | (or (zero? (:last-applied (:raft-state %))) 61 | (contains? (log-entry-of (:raft-state %) 62 | (:last-applied (:raft-state %))) 63 | :return))]} 64 | (let [[applied? new-state] (rules-of-raft raft-state) 65 | r (as-> new-state new-state 66 | (cond-> new-state 67 | (and (not (zero? (:last-applied 68 | (:raft-state new-state)))) 69 | (not (contains? (log-entry-of 70 | (:raft-state new-state) 71 | (:last-applied 72 | (:raft-state new-state))) 73 | :return))) 74 | ((fn [x] 75 | (locking #'*out* 76 | (prn x)) 77 | x)) 78 | (not= (:node-type (:raft-state new-state)) 79 | (:node-type (:raft-state raft-state))) 80 | (log-trace 81 | (:node-type (:raft-state raft-state)) 82 | "=>" 83 | (:node-type (:raft-state new-state)) 84 | (:run-count new-state)) 85 | ;; (not= (:votes (:raft-state new-state)) 86 | ;; (:votes (:raft-state raft-state))) 87 | ;; (log-trace "votes" 88 | ;; (:votes (:raft-state new-state)) 89 | ;; (:run-count new-state)) 90 | (not= (:current-term (:raft-state new-state)) 91 | (:current-term (:raft-state raft-state))) 92 | (log-trace "current-term" 93 | (:current-term (:raft-state new-state)) 94 | (:run-count new-state)) 95 | (not= (:commit-index (:raft-state new-state)) 96 | (:commit-index (:raft-state raft-state))) 97 | (log-trace "commit index" 98 | (:commit-index (:raft-state new-state)) 99 | (:run-count new-state))) 100 | (update-in new-state [:run-count] (fnil inc 0N)))] 101 | (assert (not (seq (for [message (:out-queue (:io r)) 102 | :when (= (:type message) :request-vote-response) 103 | :when (:success? message) 104 | :when (not= (:voted-for (:raft-state r)) 105 | (:target message))] 106 | message))) 107 | (pr-str 108 | (update-in r [:io :out-queue] seq))) 109 | r)) 110 | -------------------------------------------------------------------------------- /src/com/manigfeald/raft/log.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.raft.log 2 | "extends RaftLog and Counted to ISeq and IPersistentMap because you 3 | need at least two implementations for an abstraction" 4 | (:require [clojure.set :as set])) 5 | 6 | (defprotocol RaftLog 7 | (log-contains? [log log-term log-index] 8 | "does the log contain an entry with the given term and index") 9 | (last-log-index [log] 10 | "what is the index of the last entry in the log") 11 | (last-log-term [log] 12 | "what is the term of the last entry in the log") 13 | (indices-and-terms [log] 14 | "a seq of [index term] pairs from the log") 15 | (add-to-log [log index entry] 16 | "add an entry to the log with a given index") 17 | (log-entry-of [log index] 18 | "return the log entry with the given index or nil") 19 | (entry-with-serial [log serial] 20 | "return the log entry associated with the given serial or nil") 21 | (delete-from [log index] 22 | "delete log entries from the given index onwards")) 23 | 24 | (defprotocol Counted 25 | (log-count [log] 26 | "return the count of entries in the log")) 27 | 28 | (extend-type clojure.lang.ISeq 29 | Counted 30 | (log-count [log] 31 | (count log)) 32 | RaftLog 33 | (log-contains? [log log-term log-index] 34 | (loop [[head & tail] log] 35 | (cond 36 | (nil? head) 37 | false 38 | (and (= log-term (:term head)) 39 | (= log-index (:index head))) 40 | true 41 | :else 42 | (recur tail)))) 43 | (last-log-index [log] 44 | (or (:index (first log)) 0)) 45 | (last-log-term [log] 46 | (or (:term (first log)) 0)) 47 | (indices-and-terms [log] 48 | (for [entry log] 49 | [(:index entry) (:term entry)])) 50 | (add-to-log [log index entry] 51 | (let [r (sort-by #(- 0 (:index %)) 52 | (conj (for [entry log 53 | :when (not= index (:index entry))] 54 | entry) 55 | (assoc entry 56 | :index index)))] 57 | (assert (some #{(assoc entry 58 | :index index)} 59 | r)) 60 | r)) 61 | (log-entry-of [log needle-index] 62 | (first 63 | (for [{:keys [index] :as entry} log 64 | :when (= index needle-index)] 65 | entry))) 66 | (entry-with-serial [log needle-serial] 67 | (first 68 | (for [{:keys [serial] :as entry} log 69 | :when (= serial needle-serial)] 70 | entry))) 71 | (delete-from [log index] 72 | (doall (for [item log 73 | :when (not (>= (:index item) index))] 74 | item)))) 75 | 76 | (extend-type clojure.lang.IPersistentMap 77 | Counted 78 | (log-count [this] 79 | (count this)) 80 | RaftLog 81 | (log-contains? [log log-term log-index] 82 | (boolean (seq (for [[index {:keys [term]}] log 83 | :when (= index log-index) 84 | :when (= term log-term)] 85 | index)))) 86 | (last-log-index [log] 87 | (apply max 0 (keys log))) 88 | (last-log-term [log] 89 | (or (first (for [[index {:keys [term]}] log 90 | :when (= index (last-log-index log))] 91 | term)) 92 | 0)) 93 | (indices-and-terms [log] 94 | (for [[index {:keys [term]}] log] 95 | [index term])) 96 | (add-to-log [log index entry] 97 | (assert (map? log)) 98 | (assert (number? index)) 99 | (assert (map? entry)) 100 | (let [l (assoc log 101 | index (assoc entry :index index))] 102 | (assert (every? number? (keys log))) 103 | l)) 104 | (log-entry-of [log index] 105 | (get log index)) 106 | (entry-with-serial [log needle-serial] 107 | (first (for [[index {:keys [serial] :as entry}] log 108 | :when (= serial needle-serial)] 109 | entry))) 110 | (delete-from [log index] 111 | (loop [log log 112 | index index] 113 | (if (contains? log index) 114 | (recur (dissoc log index) (inc index)) 115 | log)))) 116 | 117 | (deftype LogChecker [log1 log2] 118 | Counted 119 | (log-count [_] 120 | (let [r1 (log-count log1) 121 | r2 (log-count log2)] 122 | (assert (= r1 r2) ["count" r1 r2]) 123 | r1)) 124 | RaftLog 125 | (log-contains? [log log-term log-index] 126 | (let [r1 (log-contains? log1 log-term log-index) 127 | r2 (log-contains? log2 log-term log-index)] 128 | (assert (= r1 r2) ["log-contains?" r1 r2]) 129 | r1)) 130 | (last-log-index [log] 131 | (let [r1 (last-log-index log1) 132 | r2 (last-log-index log2)] 133 | (assert (= r1 r2) ["last-log-index" r1 r2]) 134 | r1)) 135 | (last-log-term [log] 136 | (let [r1 (last-log-term log1) 137 | r2 (last-log-term log2)] 138 | (assert (= r1 r2) ["last-log-term" r1 r2]) 139 | r1)) 140 | (indices-and-terms [log] 141 | (let [r1 (indices-and-terms log1) 142 | r2 (indices-and-terms log2)] 143 | (assert (= (set r1) (set r2)) ["indices-and-terms" r1 r2]) 144 | r1)) 145 | (add-to-log [log index entry] 146 | (assert (map? entry) entry) 147 | (assert (number? index) index) 148 | (let [r1 (add-to-log log1 index entry) 149 | r2 (add-to-log log2 index entry)] 150 | (assert (= (set (indices-and-terms r1)) 151 | (set (indices-and-terms r2))) 152 | ["add-to-log" 153 | (set (indices-and-terms r1)) 154 | (set (indices-and-terms r2))]) 155 | (LogChecker. r1 r2))) 156 | (log-entry-of [log index] 157 | (let [r1 (log-entry-of log1 index) 158 | r2 (log-entry-of log2 index)] 159 | (assert (= r1 r2) ["log-entry-of" r1 r2]) 160 | r1)) 161 | (entry-with-serial [log serial] 162 | (let [r1 (entry-with-serial log1 serial) 163 | r2 (entry-with-serial log2 serial)] 164 | (assert (= r1 r2) ["entry-with-serial" r1 r2]) 165 | r1)) 166 | (delete-from [log index] 167 | (let [r1 (delete-from log1 index) 168 | r2 (delete-from log2 index)] 169 | (assert (= (set (indices-and-terms r1)) 170 | (set (indices-and-terms r2))) 171 | ["delete-from" 172 | (set/difference (set (indices-and-terms r1)) 173 | (set (indices-and-terms r2))) 174 | (set/difference (set (indices-and-terms r2)) 175 | (set (indices-and-terms r1)))]) 176 | (LogChecker. r1 r2)))) 177 | 178 | (alter-meta! #'->LogChecker assoc :doc 179 | "satisfies RaftLog, takes two things that satisfy RaftLog 180 | and runs all operations against both, checking one 181 | against the other") 182 | 183 | ;; document log entry format 184 | (defrecord LogEntry [return index term payload operation-type serial]) 185 | -------------------------------------------------------------------------------- /src/com/manigfeald/raft/core.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.raft.core 2 | "clojure.core contains lots of functions used in most(all?) clojure 3 | namespaces, com.manigfeald.raft.core contains functions used in 4 | most(all?) com.manigfeald.raft* namespaces. 5 | 6 | com.manigfeald.raft.rules may be more interesting to you" 7 | (:require [com.manigfeald.raft.log :as log]) 8 | (:import (clojure.lang PersistentQueue))) 9 | 10 | (defprotocol RaftOperations 11 | "The value that you want raft to maintain implements this protocol" 12 | (apply-operation [value operation] 13 | "apply-operation returns a tuple of 14 | [logical-operation-return-value possibly-updated-value]")) 15 | 16 | (defrecord MapValue [] 17 | RaftOperations 18 | (apply-operation [value operation] 19 | (case (:op operation) 20 | :read [(get value (:key operation)) value] 21 | :write [nil (assoc value 22 | (:key operation) (:value operation))] 23 | :write-if (if (contains? value (:key operation)) 24 | [false value] 25 | [true (assoc value 26 | (:key operation) (:value operation))]) 27 | :delete [nil (dissoc value (:key operation))] 28 | (assert nil operation)))) 29 | 30 | (alter-meta! #'map->MapValue assoc :no-doc true) 31 | 32 | (declare log-entry-of) 33 | 34 | (defn set-return-value 35 | "set the return value of an operation in the log" 36 | [raft-state index value] 37 | {:post [(contains? (log-entry-of % index) :return)]} 38 | (let [entry (log-entry-of raft-state index)] 39 | (update-in raft-state [:log] log/add-to-log (:index entry) 40 | (assoc entry :return value)))) 41 | 42 | ;; TODO: move add and remove node in to its own code 43 | (defn advance-applied-to-commit 44 | "given a RaftState, ensure all commited operations have been applied 45 | to the value" 46 | [raft-state] 47 | (if (> (:commit-index raft-state) 48 | (:last-applied raft-state)) 49 | (let [new-last (inc (:last-applied raft-state)) 50 | op (log-entry-of raft-state new-last)] 51 | (assert op "op is in the log") 52 | (case (:operation-type op) 53 | :noop (advance-applied-to-commit 54 | (assoc (set-return-value raft-state (:index op) nil) :last-applied new-last)) 55 | :add-node (advance-applied-to-commit 56 | (-> raft-state 57 | (assoc :last-applied new-last) 58 | (update-in [:node-set] conj (:node op)))) 59 | :remove-node (advance-applied-to-commit 60 | (-> raft-state 61 | (assoc :last-applied new-last) 62 | (update-in [:node-set] disj (:node op)))) 63 | (let [[return new-value] (apply-operation (:value raft-state) 64 | (:payload op)) 65 | new-state (set-return-value raft-state (:index op) return)] 66 | (assert (:index op)) 67 | (assert (= return (:return (log-entry-of new-state (:index op)))) 68 | [op (log-entry-of new-state (:index op))]) 69 | ;; post condition means it cannot be a recur 70 | (advance-applied-to-commit 71 | (assoc new-state 72 | :value new-value 73 | :last-applied new-last))))) 74 | raft-state)) 75 | 76 | (defn consume-message 77 | "remove a message from the state" 78 | [state] 79 | (assoc-in state [:io :message] nil)) 80 | 81 | (defn publish 82 | "add messages to the out-queue in the state" 83 | [state messages] 84 | {:pre [(not (map? messages)) 85 | (every? :from messages)]} 86 | (update-in state [:io :out-queue] into 87 | (for [message messages 88 | :when (not= (:target message) (:id state))] 89 | message))) 90 | 91 | (defn log-contains? 92 | "does the log in this raft-state contain an entry with the given 93 | term and index" 94 | [raft-state log-term log-index] 95 | (or (and (zero? log-term) 96 | (zero? log-index)) 97 | (log/log-contains? (:log raft-state) log-term log-index))) 98 | 99 | (defn last-log-index 100 | "what is the index of the latest log entry in the raft-state" 101 | [raft-state] 102 | (biginteger (log/last-log-index (:log raft-state)))) 103 | 104 | (defn last-log-term 105 | "return the term of the latest log entry" 106 | [raft-state] 107 | {:post [(number? %) 108 | (not (neg? %))]} 109 | (biginteger (log/last-log-term (:log raft-state)))) 110 | 111 | (defn broadcast 112 | "replicate the given message once for every node in the list of 113 | nodes" 114 | [node-set msg] 115 | (for [node node-set] 116 | (assoc msg :target node))) 117 | 118 | (defn enough-votes? 119 | "given a total number of nodes is the number of votes sufficient to 120 | elect a leader" 121 | [total votes] 122 | (>= votes (inc (Math/floor (/ total 2.0))))) 123 | 124 | (defn possible-new-commit 125 | "is there a log index that is greater than the current commit-index 126 | and a majority of nodes have a copy of it" 127 | [commit-index raft-state match-index node-set current-term] 128 | (first (sort (for [[n c] (frequencies 129 | (for [[index term] (log/indices-and-terms 130 | (:log raft-state)) 131 | [node match-index] match-index 132 | :when (>= match-index index) 133 | :when (= current-term term) 134 | :when (> index commit-index)] 135 | index)) 136 | :when (>= c (inc (Math/floor (/ (count node-set) 2))))] 137 | n)))) 138 | 139 | (def ^:dynamic *log-context* 140 | "a dynamic context used for logging" 141 | nil) 142 | 143 | (defn log-trace 144 | "given a state and a log message (as a seq of strings) append the 145 | message to the log at the trace level" 146 | [state & message] 147 | (update-in state [:running-log] 148 | (fnil conj PersistentQueue/EMPTY) 149 | {:level :trace 150 | :context *log-context* 151 | :message (apply print-str (:id state) message)})) 152 | 153 | (defn serial-exists? 154 | "does an entry with the given serial exist in the log in the 155 | raft-state?" 156 | [raft-state serial] 157 | (boolean (log/entry-with-serial (:log raft-state) serial))) 158 | 159 | (defn add-to-log 160 | "add the given operation to the log in the raft-state with the next 161 | index, as long as an entry with that serial number doesn't already 162 | exist in the log" 163 | [raft-state operation] 164 | {:pre [(contains? operation :operation-type) 165 | (contains? operation :payload) 166 | (contains? operation :term) 167 | (number? (:term operation)) 168 | (not (neg? (:term operation)))]} 169 | (if (serial-exists? raft-state (:serial operation)) 170 | raft-state 171 | (update-in raft-state [:log] 172 | log/add-to-log 173 | (biginteger (inc (last-log-index raft-state))) 174 | operation))) 175 | 176 | (defn insert-entries 177 | "given a collection of log entries, insert them in to the log" 178 | [raft-state entries] 179 | {:pre [(every? map? entries)] 180 | :post [#_(every? 181 | (fn [entry] 182 | (= (:operation (log-entry-of % (:index entry))) 183 | (:operation entry))) 184 | entries) 185 | (not (seq (for [[idx term] (log/indices-and-terms (:log %)) 186 | :when (>= (:last-applied %) idx) 187 | :when (not (contains? (log/log-entry-of (:log %) idx) 188 | :return))] 189 | true)))]} 190 | #_(doseq [entry entries 191 | :let [e (log/log-entry-of (:log raft-state) (:index entry))] 192 | :when e 193 | :when (contains? e :return)] 194 | (assert (= (:term entry) (:term e)) 195 | [(meta raft-state) entry e])) 196 | (assoc raft-state 197 | :log (reduce 198 | (fn [log entry] 199 | (if (log/log-contains? log (:term entry) (:index entry)) 200 | log 201 | (let [log (if (log/log-entry-of log (:index entry)) 202 | (log/delete-from log (:index entry)) 203 | log)] 204 | (log/add-to-log log (:index entry) entry)))) 205 | (:log raft-state) 206 | entries))) 207 | 208 | (defn log-entry-of 209 | "return the log entry for a given index" 210 | [raft-state index] 211 | (log/log-entry-of (:log raft-state) index)) 212 | 213 | (defn log-count 214 | "return the count of the log" 215 | [raft-state] 216 | (log/log-count (:log raft-state))) 217 | 218 | (defn empty-log 219 | "create an empty thing that satisfies the RaftLog protocol" 220 | [] 221 | #_{} 222 | #_(com.manigfeald.raft.log.LogChecker. () {}) 223 | ()) 224 | 225 | (defn reject-append-entries 226 | "update the state so the append entries message has been rejected" 227 | [state leader-id current-term id] 228 | (-> state 229 | (consume-message) 230 | (publish [{:type :append-entries-response 231 | :term current-term 232 | :success? false 233 | :from id 234 | :target leader-id}]) 235 | (assoc-in [:timer :next-timeout] (+ (-> state :timer :period) 236 | (-> state :timer :now))))) 237 | 238 | (defn accept-append-entries 239 | "update the state so the append entries message has been accepted" 240 | [state leader-id current-term id] 241 | (-> state 242 | (consume-message) 243 | (publish [{:type :append-entries-response 244 | :target leader-id 245 | :term current-term 246 | :success? true 247 | :from id 248 | :last-log-index (last-log-index (:raft-state state))}]) 249 | (assoc-in [:raft-state :leader-id] leader-id) 250 | (assoc-in [:timer :next-timeout] (+ (-> state :timer :period) 251 | (-> state :timer :now))))) 252 | 253 | (defn log-entries-this-term-and-committed? 254 | "are there any log entries from this current term that have been 255 | committed?" 256 | [raft-state] 257 | (first (for [[index term] (log/indices-and-terms (:log raft-state)) 258 | :when (= term (:current-term raft-state)) 259 | :when (>= (:commit-index raft-state) index)] 260 | true))) 261 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of Washington and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /test/com/manigfeald/knossos.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.knossos 2 | (:require [clojure.test :refer :all] 3 | [com.manigfeald.raft :refer :all] 4 | [clojure.core.async :refer [alt!! timeout !! chan 5 | sliding-buffer dropping-buffer 6 | close!]] 7 | [clojure.tools.logging :as log] 8 | [clojure.pprint :as pp] 9 | [robert.bruce :refer [try-try-again]] 10 | [com.manigfeald.raft.log :as rlog] 11 | [knossos.op :as kop] 12 | [knossos.core :as kn]) 13 | (:import (knossos.core Model))) 14 | 15 | ;; stop sf4j or whatever from printing out nonsense when you run tests 16 | (log/info "logging is terrible") 17 | 18 | (defprotocol Cluster 19 | (add-node [cluster node arg1]) 20 | (remove-node [cluster node]) 21 | (list-nodes [cluster]) 22 | (send-to [cluster node msg])) 23 | 24 | (defrecord ChannelCluster [] 25 | Cluster 26 | (add-node [cluster node arg1] 27 | (assoc cluster node arg1)) 28 | (remove-node [cluster node] 29 | (dissoc cluster node)) 30 | (list-nodes [cluster] 31 | (keys cluster)) 32 | (send-to [cluster node msg] 33 | (assert (contains? cluster node) [node msg]) 34 | (>!! (get cluster node) msg))) 35 | 36 | (defn shut-it-down! [nodes] 37 | (doseq [i nodes] 38 | (future-cancel (:future i)))) 39 | 40 | (defn raft-obj [in id cluster] 41 | (let [s (atom nil) 42 | lock (java.util.concurrent.Semaphore. 1) 43 | commands (chan (sliding-buffer 2)) 44 | f (future 45 | (try 46 | (loop [state (raft id (conj (set (list-nodes cluster)) id) 47 | (->Timer (System/currentTimeMillis) 48 | (System/currentTimeMillis) 49 | 1500))] 50 | (.acquire lock) 51 | (let [state (cond-> state 52 | (= :leader (:node-type (:raft-state state))) 53 | (assoc-in [:timer :period] 300) 54 | (not= :leader (:node-type 55 | (:raft-state state))) 56 | (assoc-in [:timer :period] 57 | (+ 500 (* 100 (rand-int 10))))) 58 | message (alt!! 59 | commands 60 | ([message] message) 61 | in 62 | ([message] message) 63 | ;; (timeout 100) ([_] nil) 64 | :default nil 65 | :priority true)] 66 | (let [new-state (try 67 | (-> state 68 | (assoc-in [:timer :now] 69 | (System/currentTimeMillis)) 70 | (assoc-in [:io :message] message) 71 | (run-one)) 72 | (finally 73 | (.release lock))) 74 | ;; _ (when (= :operation (:type message)) 75 | ;; (log/trace (:applied-rules new-state))) 76 | _ (doseq [msg (:out-queue (:io new-state))] 77 | (assert (not= :broadcast (:target msg))) 78 | (send-to cluster (:target msg) msg)) 79 | _ (doseq [{:keys [level message context] :as m} 80 | (:running-log new-state)] 81 | (case level 82 | :trace (log/trace message 83 | :context context))) 84 | new-state (update-in new-state [:io :out-queue] empty) 85 | new-state (update-in new-state [:running-log] empty) 86 | new-state (update-in new-state [:applied-rules] empty)] 87 | (reset! s new-state) 88 | (if (:stopped? new-state) 89 | new-state 90 | (recur new-state))))) 91 | (catch InterruptedException _) 92 | (catch Throwable t 93 | (locking #'*out* 94 | (.printStackTrace t)) 95 | (log/error t "whoops"))))] 96 | {:in in 97 | :id id 98 | :cluster cluster 99 | :raft s 100 | :lock lock 101 | :commands commands 102 | :future f})) 103 | 104 | (defn with-leader [nodes action] 105 | (let [[[_ leader]] (sort-by 106 | first 107 | (for [node nodes 108 | :let [{:keys [raft]} node 109 | v (deref raft) 110 | {{:keys [leader-id]} :raft-state} v] 111 | :when leader-id 112 | leader nodes 113 | :when (= leader-id (:id leader))] 114 | [(* -1 (:current-term (:raft-state v))) 115 | leader]))] 116 | (when leader 117 | (action leader)) 118 | nil)) 119 | 120 | (defn applied [nodes serial-w else] 121 | (let [greatest-term (apply 122 | max 123 | (for [node nodes] 124 | (-> node :raft deref :raft-state :current-term)))] 125 | (or (first (for [node nodes 126 | :let [{:keys [raft]} node 127 | v (deref raft) 128 | {{:keys [last-applied log]} :raft-state} v] 129 | :when (= (:current-term (:raft-state v)) greatest-term) 130 | :when log 131 | :let [entry (rlog/entry-with-serial log serial-w)] 132 | :when entry 133 | :when (>= last-applied (:index entry))] 134 | entry)) 135 | else))) 136 | 137 | (defn raft-write-and-forget 138 | ([nodes key value] 139 | (raft-write-and-forget nodes key value (java.util.UUID/randomUUID))) 140 | ([nodes key value id] 141 | (with-leader nodes 142 | (fn [leader] 143 | (>!! (:commands leader) {:type :operation 144 | :payload {:op :write 145 | :key key 146 | :value value} 147 | :operation-type ::bogon 148 | :serial id}))))) 149 | 150 | (defn raft-write [nodes key value] 151 | (let [id (java.util.UUID/randomUUID)] 152 | (try-try-again 153 | {:sleep 100 154 | :tries 300} 155 | (fn [] 156 | (raft-write-and-forget nodes key value id) 157 | (when (= :dunno (applied nodes id :dunno)) 158 | (throw (Exception. 159 | (with-out-str 160 | (println "write failed?") 161 | (println "key" key "value" value) 162 | (println) 163 | (doseq [node nodes] 164 | (-> node :raft deref prn) 165 | (println) 166 | (println)))))))))) 167 | 168 | (defn raft-read [nodes key] 169 | (let [id (java.util.UUID/randomUUID)] 170 | (try 171 | (try-try-again 172 | {:sleep 100 173 | :tries 600} 174 | (fn [] 175 | (with-leader nodes 176 | (fn [leader] 177 | (>!! (:commands leader) {:type :operation 178 | :payload {:op :read 179 | :key key} 180 | :operation-type ::bogon 181 | :serial id}))) 182 | (let [r (applied nodes id :dunno)] 183 | (if (= :dunno r) 184 | (throw (Exception. "read failed?")) 185 | (:return r))))) 186 | (catch Exception e 187 | (doseq [node nodes] 188 | (log/trace (-> node :raft deref))) 189 | (throw e))))) 190 | 191 | (defrecord MapRegister [m] 192 | Model 193 | (step [r op] 194 | (condp = (:f op) 195 | :write (MapRegister. 196 | (let [[k v] (:value op)] 197 | (assoc m k v))) 198 | :read (if (or (nil? (second (:value op))) 199 | (= (get m (first (:value op))) 200 | (second (:value op)))) 201 | r 202 | (kn/inconsistent 203 | (str "read " (pr-str (:value op)) 204 | "from" m)))))) 205 | 206 | (deftest test-linear 207 | (let [node-ids-and-channels (into {} (for [i (range 5)] 208 | [i (chan (sliding-buffer 5))])) 209 | cluster (reduce #(add-node % (key %2) (val %2)) (->ChannelCluster) 210 | node-ids-and-channels) 211 | nodes (doall (for [[node-id in] node-ids-and-channels] 212 | (raft-obj in node-id cluster))) 213 | history (atom []) 214 | keys (repeatedly 5 gensym) 215 | stop? (atom false) 216 | clients (for [client-id (range 10)] 217 | (future 218 | (while (not @stop?) 219 | (let [k (rand-nth keys)] 220 | (if (zero? (rand-int 2)) 221 | (try 222 | (swap! history conj (kop/invoke client-id :read [k nil])) 223 | (let [r (raft-read nodes k)] 224 | (swap! history conj (kop/ok client-id :read [k r]))) 225 | (catch Exception _ 226 | (swap! history conj (kop/fail client-id :read [k nil])))) 227 | (let [v (gensym) 228 | _ (swap! history conj (kop/invoke client-id :write [k v]))] 229 | (try 230 | (raft-write nodes k v) 231 | (swap! history conj (kop/ok client-id :write [k v])) 232 | (catch Exception _ 233 | (swap! history conj (kop/fail client-id :write [k v])))))))))) 234 | killer (future 235 | (while (not @stop?) 236 | (let [n (rand-nth nodes)] 237 | (try 238 | (.acquire (:lock n)) 239 | (Thread/sleep 500) 240 | (finally 241 | (.release (:lock n)))))))] 242 | (doall clients) 243 | (Thread/sleep (* 1000 (inc (rand-int 60)))) 244 | (reset! stop? true) 245 | (doseq [f clients] 246 | (deref f)) 247 | (deref killer) 248 | (is (:valid? (kn/analysis (->MapRegister {}) @history))))) 249 | 250 | (defmacro foo [exp & body] 251 | (when-not (eval exp) 252 | `(do ~@body))) 253 | 254 | (foo (resolve 'clojure.test/original-test-var) 255 | (in-ns 'clojure.test) 256 | 257 | (def original-test-var test-var) 258 | 259 | (defn test-var [v] 260 | (clojure.tools.logging/trace "testing" v) 261 | (let [start (System/currentTimeMillis) 262 | result (original-test-var v)] 263 | (clojure.tools.logging/trace 264 | "testing" v "took" 265 | (/ (- (System/currentTimeMillis) start) 1000.0) 266 | "seconds") 267 | result)) 268 | 269 | (def original-test-ns test-ns) 270 | 271 | (defn test-ns [ns] 272 | (clojure.tools.logging/trace "testing namespace" ns) 273 | (let [start (System/currentTimeMillis) 274 | result (original-test-ns ns)] 275 | (clojure.tools.logging/trace 276 | "testing namespace" ns "took" 277 | (/ (- (System/currentTimeMillis) start) 1000.0) 278 | "seconds") 279 | result)) 280 | 281 | (in-ns 'com.manigfeald.raft-test)) 282 | -------------------------------------------------------------------------------- /test/com/manigfeald/raft_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.raft-test 2 | (:require [clojure.test :refer :all] 3 | [com.manigfeald.raft :refer :all] 4 | [clojure.core.async :refer [alt!! timeout !! chan 5 | sliding-buffer dropping-buffer 6 | close!]] 7 | [clojure.tools.logging :as log] 8 | [clojure.pprint :as pp] 9 | [robert.bruce :refer [try-try-again]] 10 | [com.manigfeald.raft.log :as rlog])) 11 | 12 | ;; stop sf4j or whatever from printing out nonsense when you run tests 13 | (log/info "logging is terrible") 14 | 15 | (defprotocol Cluster 16 | (add-node [cluster node arg1]) 17 | (remove-node [cluster node]) 18 | (list-nodes [cluster]) 19 | (send-to [cluster node msg])) 20 | 21 | (defrecord ChannelCluster [] 22 | Cluster 23 | (add-node [cluster node arg1] 24 | (assoc cluster node arg1)) 25 | (remove-node [cluster node] 26 | (dissoc cluster node)) 27 | (list-nodes [cluster] 28 | (keys cluster)) 29 | (send-to [cluster node msg] 30 | (assert (contains? cluster node) [node msg]) 31 | (>!! (get cluster node) msg))) 32 | 33 | (defn shut-it-down! [nodes] 34 | (doseq [i nodes] 35 | (future-cancel (:future i)))) 36 | 37 | (defn raft-obj [in id cluster] 38 | (let [s (atom nil) 39 | lock (java.util.concurrent.Semaphore. 1) 40 | commands (chan (sliding-buffer 2)) 41 | f (future 42 | (try 43 | (loop [state (raft id (conj (set (list-nodes cluster)) id) 44 | (->Timer (System/currentTimeMillis) 45 | (System/currentTimeMillis) 46 | 1500))] 47 | (.acquire lock) 48 | (let [state (cond-> state 49 | (= :leader (:node-type (:raft-state state))) 50 | (assoc-in [:timer :period] 300) 51 | (not= :leader (:node-type 52 | (:raft-state state))) 53 | (assoc-in [:timer :period] 54 | (+ 500 (* 100 (rand-int 10))))) 55 | message (alt!! 56 | commands 57 | ([message] message) 58 | in 59 | ([message] message) 60 | ;; (timeout 100) ([_] nil) 61 | :default nil 62 | :priority true)] 63 | (let [new-state (try 64 | (-> state 65 | (assoc-in [:timer :now] 66 | (System/currentTimeMillis)) 67 | (assoc-in [:io :message] message) 68 | (run-one)) 69 | (finally 70 | (.release lock))) 71 | ;; _ (when (= :operation (:type message)) 72 | ;; (log/trace (:applied-rules new-state))) 73 | _ (doseq [msg (:out-queue (:io new-state))] 74 | (assert (not= :broadcast (:target msg))) 75 | (send-to cluster (:target msg) msg)) 76 | _ (doseq [{:keys [level message context] :as m} 77 | (:running-log new-state)] 78 | (case level 79 | :trace (log/trace message 80 | :context context))) 81 | new-state (update-in new-state [:io :out-queue] empty) 82 | new-state (update-in new-state [:running-log] empty) 83 | new-state (update-in new-state [:applied-rules] empty)] 84 | (reset! s new-state) 85 | (if (:stopped? new-state) 86 | new-state 87 | (recur new-state))))) 88 | (catch InterruptedException _) 89 | (catch Throwable t 90 | (locking #'*out* 91 | (.printStackTrace t)) 92 | (log/error t "whoops"))))] 93 | {:in in 94 | :id id 95 | :cluster cluster 96 | :raft s 97 | :lock lock 98 | :commands commands 99 | :future f})) 100 | 101 | (defn stable-leader? [nodes n] 102 | (try-try-again 103 | {:decay :exponential 104 | :sleep 10 105 | :tries 20} 106 | (fn [] 107 | (let [lead (for [node nodes 108 | :let [{:keys [raft]} node 109 | v (deref raft) 110 | {{:keys [leader-id]} :raft-state} v] 111 | :when leader-id 112 | leader nodes 113 | :when (= leader-id (:id leader))] 114 | leader) 115 | f (frequencies lead) 116 | [lead c] (last (sort-by second f))] 117 | (if (and lead 118 | (>= c n)) 119 | lead 120 | (throw (Exception. "failed to get a leader"))))))) 121 | 122 | (defn with-leader [nodes action] 123 | (let [[[_ leader]] (sort-by 124 | first 125 | (for [node nodes 126 | :let [{:keys [raft]} node 127 | v (deref raft) 128 | {{:keys [leader-id]} :raft-state} v] 129 | :when leader-id 130 | leader nodes 131 | :when (= leader-id (:id leader))] 132 | [(* -1 (:current-term (:raft-state v))) 133 | leader]))] 134 | (when leader 135 | (action leader)) 136 | nil)) 137 | 138 | (defn applied [nodes serial-w else] 139 | (let [greatest-term (apply 140 | max 141 | (for [node nodes] 142 | (-> node :raft deref :raft-state :current-term)))] 143 | (or (first (for [node nodes 144 | :let [{:keys [raft]} node 145 | v (deref raft) 146 | {{:keys [last-applied log]} :raft-state} v] 147 | :when (= (:current-term (:raft-state v)) greatest-term) 148 | :when log 149 | :let [entry (rlog/entry-with-serial log serial-w)] 150 | :when entry 151 | :when (>= last-applied (:index entry))] 152 | entry)) 153 | else))) 154 | 155 | (defn raft-write-and-forget 156 | ([nodes key value] 157 | (raft-write-and-forget nodes key value (java.util.UUID/randomUUID))) 158 | ([nodes key value id] 159 | (with-leader nodes 160 | (fn [leader] 161 | (>!! (:commands leader) {:type :operation 162 | :payload {:op :write 163 | :key key 164 | :value value} 165 | :operation-type ::bogon 166 | :serial id}))))) 167 | 168 | (defn raft-write [nodes key value] 169 | (let [id (java.util.UUID/randomUUID)] 170 | (try-try-again 171 | {:sleep 100 172 | :tries 300} 173 | (fn [] 174 | (raft-write-and-forget nodes key value id) 175 | (when (= :dunno (applied nodes id :dunno)) 176 | (throw (Exception. 177 | (with-out-str 178 | (println "write failed?") 179 | (println "key" key "value" value) 180 | (println) 181 | (doseq [node nodes] 182 | (-> node :raft deref prn) 183 | (println) 184 | (println)))))))))) 185 | 186 | (defn raft-read [nodes key] 187 | (let [id (java.util.UUID/randomUUID)] 188 | (try 189 | (try-try-again 190 | {:sleep 100 191 | :tries 600} 192 | (fn [] 193 | (with-leader nodes 194 | (fn [leader] 195 | (>!! (:commands leader) {:type :operation 196 | :payload {:op :read 197 | :key key} 198 | :operation-type ::bogon 199 | :serial id}))) 200 | (let [r (applied nodes id :dunno)] 201 | (if (= :dunno r) 202 | (throw (Exception. "read failed?")) 203 | (:return r))))) 204 | (catch Exception e 205 | (doseq [node nodes] 206 | (log/trace (-> node :raft deref))) 207 | (throw e))))) 208 | 209 | (defmacro with-pings [nodes & body] 210 | `(let [nodes# ~nodes 211 | fut# (future 212 | (while true 213 | (Thread/sleep 15000) 214 | (future 215 | (raft-write-and-forget nodes# "ping" "pong"))))] 216 | (try 217 | ~@body 218 | (finally 219 | (future-cancel fut#))))) 220 | 221 | (deftest test-leader-election 222 | (let [node-ids-and-channels (into {} (for [i (range 3)] 223 | [i (chan (sliding-buffer 10))])) 224 | cluster (reduce #(add-node % (key %2) (val %2)) (->ChannelCluster) 225 | node-ids-and-channels) 226 | nodes (doall (for [[node-id in] node-ids-and-channels] 227 | (raft-obj in node-id cluster)))] 228 | (try 229 | (is (stable-leader? nodes 3)) 230 | (finally 231 | (shut-it-down! nodes))))) 232 | 233 | (deftest test-remove-node 234 | (let [node-ids-and-channels (into {} (for [i (range 5)] 235 | [i (chan (sliding-buffer 10))])) 236 | cluster (reduce #(add-node % (key %2) (val %2)) (->ChannelCluster) 237 | node-ids-and-channels) 238 | nodes (doall (for [[node-id in] node-ids-and-channels] 239 | (raft-obj in node-id cluster)))] 240 | (try 241 | (testing "elect leader" 242 | (is (stable-leader? nodes 5))) 243 | (testing "kill leader and elect a new one" 244 | (future-cancel (:future (stable-leader? nodes 5))) 245 | (is (stable-leader? nodes 4)) 246 | (future-cancel (:future (stable-leader? nodes 4))) 247 | (is (stable-leader? nodes 3))) 248 | (finally 249 | (shut-it-down! nodes))))) 250 | 251 | ;; (deftest test-operations 252 | ;; (let [node-ids-and-channels (into {} (for [i (range 5)] 253 | ;; [i (chan (sliding-buffer 10))])) 254 | ;; cluster (reduce #(add-node % (key %2) (val %2)) 255 | ;; (->ChannelCluster) node-ids-and-channels) 256 | ;; nodes (doall (for [[node-id in] node-ids-and-channels] 257 | ;; (raft-obj in node-id cluster)))] 258 | ;; (with-pings nodes 259 | ;; (try 260 | ;; (testing "elect leader" 261 | ;; (is (stable-leader? nodes 5))) 262 | ;; (raft-write nodes "hello" "world") 263 | ;; (doseq [node nodes 264 | ;; :let [{:keys [raft]} node 265 | ;; {{{:strs [hello]} :value} :raft-state} (deref raft)]] 266 | ;; (is (= hello "world") (deref (:raft (stable-leader? nodes 5))))) 267 | ;; (finally 268 | ;; (shut-it-down! nodes)))))) 269 | 270 | (deftest test-read-operations 271 | (let [node-ids-and-channels (into {} (for [i (range 5)] 272 | [i (chan (sliding-buffer 10))])) 273 | cluster (reduce #(add-node % (key %2) (val %2)) 274 | (->ChannelCluster) node-ids-and-channels) 275 | nodes (doall (for [[node-id in] node-ids-and-channels] 276 | (raft-obj in node-id cluster)))] 277 | (with-pings nodes 278 | (try 279 | (testing "elect leader" 280 | (is (stable-leader? nodes 5))) 281 | (raft-write nodes "hello" "world") 282 | (let [x (raft-read nodes "hello")] 283 | (is (= "world" x) 284 | (with-out-str 285 | (prn x) 286 | (println) 287 | (doseq [node nodes] 288 | (prn (-> node :raft deref)) 289 | (println) 290 | (println))))) 291 | (finally 292 | (shut-it-down! nodes)))))) 293 | 294 | (deftest test-stress 295 | (let [node-ids-and-channels (into {} (for [i (range 3)] 296 | [i (chan (sliding-buffer 5))])) 297 | cluster (reduce #(add-node % (key %2) (val %2)) (->ChannelCluster) 298 | node-ids-and-channels) 299 | nodes (doall (for [[node-id in] node-ids-and-channels] 300 | (raft-obj in node-id cluster))) 301 | a (atom 0)] 302 | (with-pings nodes 303 | (try 304 | (raft-write nodes :key 0) 305 | (dotimes [i 10] 306 | (log/trace "start") 307 | (let [victim (rand-nth nodes)] 308 | (log/trace "victim" (:id victim)) 309 | (.acquire (:lock victim)) 310 | (try 311 | (let [rv (raft-read nodes :key)] 312 | (log/trace "read" rv) 313 | (when-not (= rv @a) 314 | (log/trace "FAILLLLLLL") 315 | (doseq [node (sort-by :id nodes) 316 | :let [_ (log/trace "node" (:id node)) 317 | {:keys [raft id]} node 318 | v (deref raft)] 319 | entry (sort-by :index (vals (:log (:raft-state v))))] 320 | (log/trace id "log entry" entry)) 321 | (assert nil)) 322 | (is (= @a (raft-read nodes :key)) 323 | (with-out-str 324 | (pp/pprint nodes)))) 325 | (swap! a inc) 326 | (log/trace "writing" @a) 327 | (raft-write nodes :key @a) 328 | (doseq [node nodes 329 | :when (future-done? (:future node))] 330 | (deref (:future node))) 331 | (finally 332 | (.release (:lock victim)))))) 333 | (finally 334 | (shut-it-down! nodes)))))) 335 | 336 | (defn rand-lock [nodes n] 337 | (let [to-lock (take n (shuffle nodes))] 338 | (doseq [node to-lock] 339 | (.acquire (:lock node))) 340 | to-lock)) 341 | 342 | (defn unlock [locked-nodes] 343 | (doseq [node locked-nodes] 344 | (.release (:lock node)))) 345 | 346 | (deftest test-stress-down-down-down 347 | (let [node-ids-and-channels (into {} (for [i (range 5)] 348 | [i (chan (sliding-buffer 1000))])) 349 | cluster (reduce #(add-node % (key %2) (val %2)) (->ChannelCluster) 350 | node-ids-and-channels) 351 | nodes (doall (for [[node-id in] node-ids-and-channels] 352 | (raft-obj in node-id cluster))) 353 | a (atom {}) 354 | keys (repeatedly 10 gensym)] 355 | (with-pings nodes 356 | (try 357 | (let [v (gensym) 358 | k (rand-nth keys)] 359 | (swap! a assoc k v) 360 | (raft-write nodes k v)) 361 | (dotimes [i 10] 362 | (let [locked-nodes (rand-lock nodes (min 2 (inc i)))] 363 | (log/trace "victims" (pr-str (map :id locked-nodes))) 364 | (try 365 | (doseq [[k v] @a] 366 | (is (= v (raft-read nodes k)) 367 | [k v])) 368 | (let [v (gensym) 369 | k (rand-nth keys)] 370 | (swap! a assoc k v) 371 | (raft-write nodes k v)) 372 | (doseq [node nodes 373 | :when (future-done? (:future node))] 374 | (deref (:future node))) 375 | (finally 376 | (unlock locked-nodes))))) 377 | (finally 378 | (shut-it-down! nodes)))))) 379 | 380 | (defmacro foo [exp & body] 381 | (when-not (eval exp) 382 | `(do ~@body))) 383 | 384 | (foo (resolve 'clojure.test/original-test-var) 385 | (in-ns 'clojure.test) 386 | 387 | (def original-test-var test-var) 388 | 389 | (defn test-var [v] 390 | (clojure.tools.logging/trace "testing" v) 391 | (let [start (System/currentTimeMillis) 392 | result (original-test-var v)] 393 | (clojure.tools.logging/trace 394 | "testing" v "took" 395 | (/ (- (System/currentTimeMillis) start) 1000.0) 396 | "seconds") 397 | result)) 398 | 399 | (def original-test-ns test-ns) 400 | 401 | (defn test-ns [ns] 402 | (clojure.tools.logging/trace "testing namespace" ns) 403 | (let [start (System/currentTimeMillis) 404 | result (original-test-ns ns)] 405 | (clojure.tools.logging/trace 406 | "testing namespace" ns "took" 407 | (/ (- (System/currentTimeMillis) start) 1000.0) 408 | "seconds") 409 | result)) 410 | 411 | (in-ns 'com.manigfeald.raft-test)) 412 | -------------------------------------------------------------------------------- /src/com/manigfeald/raft/rules.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.raft.rules 2 | "rules are a combination of a head which is a predicate and a body 3 | that updates and returns the passed in state. when a rule is applied 4 | to a value, if the head returns true for a given value that value is 5 | give to the body, and the updated value is returned, if the head 6 | returns false the passed in value is returned unmodified. 7 | 8 | syntactically rules have a third component, which is the name/binding 9 | form that the value will be bound to in both the head and the body." 10 | (:require [com.manigfeald.raft.core :refer :all])) 11 | 12 | (defrecord Rule [head body] 13 | clojure.lang.IFn 14 | (invoke [this arg] 15 | (if (head arg) 16 | (let [r (binding [*log-context* (:name this)] 17 | (body arg))] 18 | #_(assert (or (zero? (:last-applied (:raft-state r))) 19 | (contains? (get (:log (:raft-state r)) 20 | (:last-applied (:raft-state r))) 21 | :return)) 22 | (with-out-str 23 | (prn "rule" this) 24 | (println "before:") 25 | (prn arg) 26 | (println "after:") 27 | (prn r) 28 | (println))) 29 | (assert (>= (:commit-index (:raft-state r)) 30 | (:last-applied (:raft-state r))) 31 | (with-out-str 32 | (prn "rule" this) 33 | (println "before:") 34 | (prn arg) 35 | (println "after:") 36 | (prn r) 37 | (println))) 38 | [true r #_(update-in r [:applied-rules] conj this)]) 39 | [false arg]))) 40 | 41 | (defrecord CompoundRule [head subrules] 42 | clojure.lang.IFn 43 | (invoke [_ arg] 44 | (reduce 45 | (fn [[applied? arg] rule] 46 | (if (head arg) 47 | (let [[new-applied? new-arg] (rule arg)] 48 | [(or applied? new-applied?) new-arg]) 49 | (reduced [applied? arg]))) 50 | [false arg] 51 | subrules))) 52 | 53 | (alter-meta! #'->CompoundRule assoc :no-doc true) 54 | (alter-meta! #'->Rule assoc :no-doc true) 55 | (alter-meta! #'map->CompoundRule assoc :no-doc true) 56 | (alter-meta! #'map->Rule assoc :no-doc true) 57 | 58 | (defmacro rule 59 | "a rule is a combination of some way to decide if the rule 60 | applies (the head) and some way to apply the rule (the body)" 61 | ([head body bindings] 62 | (if-let [line (:line (meta &form))] 63 | `(rule ~(str "line-" line) 64 | ~head ~body ~bindings) 65 | `(->Rule (fn [~bindings] 66 | ~head) 67 | (fn [~bindings] 68 | ~body)))) 69 | ([name head body bindings] 70 | `(assoc (->Rule (fn ~(symbol (str (clojure.core/name name) "-head")) 71 | [~bindings] 72 | ~head) 73 | (fn ~(symbol (str (clojure.core/name name) "-body")) 74 | [~bindings] 75 | ~body)) 76 | :name ~name))) 77 | 78 | (defmacro guard 79 | "a guard is a rule with a normal rule head, but the body is a 80 | composition of other rules" 81 | [head & things] 82 | (let [rules (butlast things) 83 | bindings (last things)] 84 | `(->CompoundRule (fn [~bindings] 85 | ~head) 86 | ~(vec rules)))) 87 | 88 | ;; 89 | (def keep-up-apply 90 | (rule 91 | (and (> commit-index last-applied) 92 | ;; only apply when the latest thing to apply is from the 93 | ;; current term, this deals with split writes 94 | (log-entries-this-term-and-committed? raft-state)) 95 | (-> state 96 | (update-in [:raft-state] advance-applied-to-commit)) 97 | {:as state 98 | {:keys [commit-index last-applied current-term] 99 | :as raft-state} :raft-state})) 100 | 101 | (def jump-to-newer-term 102 | (rule 103 | (and message-term 104 | (> message-term current-term)) 105 | (-> state 106 | (log-trace "jump-to-newer-term" message-term) 107 | (update-in [:raft-state] merge {:node-type :follower 108 | :current-term message-term 109 | :leader-id nil 110 | :votes 0N 111 | :voted-for nil})) 112 | {:as state 113 | :keys [id] 114 | {{message-term :term} :message} :io 115 | {:keys [current-term]} :raft-state})) 116 | ;; 117 | 118 | (def follower-respond-to-append-entries-with-wrong-term 119 | (rule 120 | :fail-old-term 121 | (> current-term message-term) 122 | (-> state 123 | (reject-append-entries leader-id current-term id) 124 | (log-trace "rejecting append entries from" leader-id)) 125 | {:as state 126 | :keys [id] 127 | {{leader-id :leader-id 128 | message-term :term} :message} :io 129 | {:keys [current-term node-type]} :raft-state})) 130 | 131 | (def follower-respond-to-append-entries-with-unknown-prev 132 | (rule 133 | :fail-missing-prevs 134 | (and (= current-term message-term) 135 | (not (log-contains? raft-state prev-log-term prev-log-index))) 136 | (-> state 137 | (log-trace "rejecting append entries from" leader-id) 138 | (reject-append-entries leader-id current-term id)) 139 | {:as state 140 | :keys [id] 141 | {{prev-log-index :prev-log-index 142 | prev-log-term :prev-log-term 143 | leader-id :leader-id 144 | message-term :term} :message} :io 145 | {:keys [current-term node-type] :as raft-state} :raft-state})) 146 | 147 | (def follower-respond-to-append-entries 148 | (guard 149 | (and (= :follower node-type) 150 | (= message-type :append-entries)) 151 | #'follower-respond-to-append-entries-with-wrong-term 152 | #'follower-respond-to-append-entries-with-unknown-prev 153 | (rule 154 | :good 155 | (and (= message-term current-term) 156 | (log-contains? raft-state prev-log-term prev-log-index)) 157 | (-> state 158 | (update-in [:raft-state] insert-entries entries) 159 | (accept-append-entries leader-id current-term id) 160 | (as-> n 161 | (if (> leader-commit commit-index) 162 | (assoc-in n [:raft-state :commit-index] 163 | (min leader-commit 164 | (last-log-index (:raft-state n)))) 165 | n))) 166 | {:as state 167 | :keys [id] 168 | {{prev-log-index :prev-log-index 169 | prev-log-term :prev-log-term 170 | message-term :term 171 | entries :entries 172 | leader-id :leader-id 173 | leader-commit :leader-commit} :message} :io 174 | {:keys [current-term node-type commit-index] :as raft-state} :raft-state}) 175 | {{{message-type :type 176 | :as message} :message} :io 177 | {:keys [node-type]} :raft-state})) 178 | 179 | (def follower-respond-to-request-vote 180 | (guard 181 | (and (= :follower node-type) 182 | (= message-type :request-vote)) 183 | (rule 184 | :success 185 | (and (or (= voted-for candidate-id) 186 | (nil? voted-for)) 187 | (>= last-log-index (com.manigfeald.raft.core/last-log-index 188 | raft-state)) 189 | (>= last-log-term (com.manigfeald.raft.core/last-log-term raft-state)) 190 | (or (log-contains? raft-state last-log-term last-log-index) 191 | (nil? (log-entry-of raft-state last-log-index)))) 192 | (-> state 193 | (log-trace "votes for" candidate-id "in" current-term) 194 | (consume-message) 195 | (assoc-in [:raft-state :voted-for] candidate-id) 196 | (publish [{:type :request-vote-response 197 | :target candidate-id 198 | :term current-term 199 | :success? true 200 | :from id}])) 201 | {:as state 202 | :keys [id] 203 | {{message-type :type 204 | candidate-id :candidate-id 205 | :keys [last-log-index last-log-term] :as message} :message} :io 206 | {:keys [current-term node-type voted-for] :as raft-state} :raft-state}) 207 | (rule 208 | (not (and (or (= voted-for candidate-id) 209 | (nil? voted-for)) 210 | (or (log-contains? raft-state last-log-term last-log-index) 211 | (and (:term (log-entry-of raft-state last-log-index)) 212 | (> last-log-term 213 | (:term (log-entry-of raft-state last-log-index))))))) 214 | (-> state 215 | (log-trace "doesn't vote for" candidate-id "in" current-term 216 | ;; {:last-log-index last-log-index 217 | ;; :last-log-term last-log-term 218 | ;; :voted-for voted-for} 219 | ;; (com.manigfeald.raft.log/indices-and-terms 220 | ;; (:log (:raft-state state))) 221 | ) 222 | (consume-message) 223 | (publish [{:type :request-vote-response 224 | :target candidate-id 225 | :term current-term 226 | :success? false 227 | :from id}])) 228 | {:as state 229 | :keys [id] 230 | {{message-type :type 231 | candidate-id :candidate-id 232 | :keys [last-log-index last-log-term] 233 | :as message} :message} :io 234 | {:keys [current-term node-type voted-for] :as raft-state} :raft-state}) 235 | {{{message-type :type} :message} :io 236 | {:keys [node-type]} :raft-state})) 237 | 238 | (def become-candidate-and-call-for-election 239 | (rule 240 | (and (= node-type :follower) 241 | (>= now next-timeout)) 242 | (-> state 243 | (log-trace "call for election") 244 | (consume-message) 245 | (update-in [:raft-state] merge 246 | {:node-type :candidate 247 | :votes 1 248 | :voted-for id 249 | :current-term (inc current-term) 250 | :leader-id nil}) 251 | (assoc-in [:timer :next-timeout] 252 | (+ now period)) 253 | (update-in [:io :out-queue] empty) 254 | (publish (broadcast 255 | node-set 256 | {:type :request-vote 257 | :candidate-id id 258 | :last-log-index (last-log-index raft-state) 259 | :last-log-term (last-log-term raft-state) 260 | :term (inc current-term) 261 | :from id}))) 262 | {:as state 263 | :keys [id] 264 | {:keys [now period next-timeout]} :timer 265 | {:keys [current-term node-set node-type] :as raft-state} :raft-state})) 266 | ;; 267 | 268 | (def candidate-receive-votes 269 | (guard 270 | (and (= node-type :candidate) 271 | (= message-type :request-vote-response) 272 | success?) 273 | (rule 274 | (enough-votes? (count node-set) (inc votes)) 275 | (-> state 276 | (consume-message) 277 | (log-trace "received vote from" from "in" current-term) 278 | (log-trace "becoming leader with" (inc votes) "in" current-term) 279 | (update-in [:raft-state] merge {:node-type :leader 280 | :votes 0N 281 | :voted-for nil 282 | :leader-id id}) 283 | (update-in [:raft-leader-state] merge 284 | {:match-index 285 | (into {} (for [node node-set] 286 | [node 0N])) 287 | :next-index 288 | (into {} (for [node node-set] 289 | [node (inc (last-log-index raft-state))]))}) 290 | (assoc-in [:timer :next-timeout] (+ now period)) 291 | (update-in [:raft-state] add-to-log 292 | {:term current-term 293 | :payload nil 294 | :operation-type :noop}) 295 | (publish (broadcast node-set 296 | {:type :append-entries 297 | :term current-term 298 | :leader-id id 299 | :prev-log-index (last-log-index raft-state) 300 | :prev-log-term (last-log-term raft-state) 301 | :entries [] 302 | :leader-commit commit-index 303 | :from id}))) 304 | {:as state 305 | :keys [id] 306 | {{:keys [success? from]} :message} :io 307 | {:keys [now period]} :timer 308 | {:keys [votes node-set current-term commit-index] 309 | :as raft-state} :raft-state}) 310 | (rule 311 | (not (enough-votes? (count node-set) (inc votes))) 312 | (-> state 313 | (consume-message) 314 | (log-trace "received vote from" from "in" current-term) 315 | (update-in [:raft-state :votes] inc)) 316 | {:as state 317 | :keys [id] 318 | {{:keys [from]} :message} :io 319 | {:keys [votes node-set current-term]} :raft-state}) 320 | {{{:keys [success?] 321 | message-type :type} :message} :io 322 | {:keys [node-type]} :raft-state})) 323 | 324 | (def candidate-respond-to-append-entries 325 | (rule 326 | (and (= node-type :candidate) 327 | (= message-type :append-entries)) 328 | (-> state 329 | (log-trace "candidate-respond-to-append-entries") 330 | (consume-message) 331 | (update-in [:raft-state] merge {:node-type :follower 332 | :voted-for leader-id 333 | :votes 0N}) 334 | (assoc-in [:timer :next-timeout] (+ period now))) 335 | {:as state 336 | {{message-type :type 337 | leader-id :leader-id} :message} :io 338 | {:keys [node-type]} :raft-state 339 | {:keys [period now]} :timer})) 340 | 341 | (def candidate-election-timeout 342 | (rule 343 | (and (= node-type :candidate) 344 | (>= now next-timeout)) 345 | (-> state 346 | (log-trace "candidate-election-timeout") 347 | (consume-message) 348 | (update-in [:raft-state] merge 349 | {:node-type :candidate 350 | :votes 1 351 | :voted-for id 352 | :current-term (inc current-term)}) 353 | (assoc-in [:timer :next-timeout] 354 | (+ now period)) 355 | (publish (broadcast 356 | node-set 357 | {:type :request-vote 358 | :candidate-id id 359 | :last-log-index (last-log-index raft-state) 360 | :last-log-term (last-log-term raft-state) 361 | :term (inc current-term) 362 | :from id}))) 363 | {:as state 364 | :keys [id] 365 | {:keys [now next-timeout period]} :timer 366 | {:keys [node-type current-term node-set] :as raft-state} :raft-state})) 367 | ;; 368 | (def heart-beat 369 | (rule 370 | (and (= node-type :leader) 371 | (>= now next-timeout)) 372 | (-> state 373 | (publish (broadcast node-set 374 | {:type :append-entries 375 | :term current-term 376 | :leader-id id 377 | :prev-log-index (last-log-index raft-state) 378 | :prev-log-term (last-log-term raft-state) 379 | :entries [] 380 | :leader-commit commit-index 381 | :from id}))) 382 | {:as state 383 | :keys [id] 384 | {:keys [now next-timeout]} :timer 385 | {:keys [node-type node-set current-term commit-index] :as raft-state} 386 | :raft-state})) 387 | 388 | ;; TODO: need a test to ensure that the leaders last-log-index is 389 | ;; considered when deciding to advance the commit index 390 | (def leader-receive-command 391 | (rule 392 | (and (= node-type :leader) 393 | (= message-type :operation)) 394 | (-> state 395 | (log-trace "received command serial" (:serial message)) 396 | (consume-message) 397 | (update-in [:raft-state] add-to-log (assoc (dissoc message :type) :term current-term)) 398 | (as-> state 399 | (assoc-in state [:raft-leader-state :match-index id] 400 | (last-log-index (:raft-state state))))) 401 | {:as state 402 | :keys [id] 403 | {{:as message message-type :type} :message} :io 404 | {:keys [match-index]} :raft-leader-state 405 | {:keys [current-term node-type commit-index node-set] 406 | :as raft-state} :raft-state})) 407 | 408 | ;; TODO: rate limit this rule 409 | (def update-followers 410 | (rule 411 | (and (= :leader node-type) 412 | (seq (for [[node next-index] next-index 413 | :when (>= (last-log-index raft-state) next-index)] 414 | node))) 415 | (-> state 416 | ;; (log-trace "update followers") 417 | (publish (for [[node next-index] next-index 418 | :when (not= node id) 419 | :when (>= (last-log-index raft-state) next-index)] 420 | {:type :append-entries 421 | :target node 422 | :term current-term 423 | :leader-id id 424 | :prev-log-index (max 0N (dec next-index)) 425 | :prev-log-term (or (:term (log-entry-of raft-state 426 | (dec next-index))) 427 | 0N) 428 | :entries (for [index (range (max 0N (dec next-index)) 429 | (inc next-index)) 430 | :when (not (zero? index))] 431 | (dissoc (log-entry-of raft-state index) :return)) 432 | :leader-commit commit-index 433 | :from id}))) 434 | {:as state 435 | :keys [id] 436 | {:keys [next-index]} :raft-leader-state 437 | {:keys [now next-timeout period]} :timer 438 | {:as raft-state 439 | :keys [commit-index current-term node-type]} :raft-state})) 440 | 441 | (def update-commit 442 | (rule 443 | :update-commit 444 | (and (= :leader node-type) 445 | (possible-new-commit commit-index raft-state match-index node-set 446 | current-term)) 447 | (-> state 448 | (log-trace "update-commit" 449 | (possible-new-commit 450 | commit-index raft-state match-index node-set current-term) 451 | "match-index value frequencies" 452 | (frequencies 453 | (vals (:match-index (:raft-leader-state state))))) 454 | (assoc-in [:raft-state :commit-index] 455 | (possible-new-commit 456 | commit-index raft-state match-index node-set current-term))) 457 | {:as state 458 | {:keys [match-index]} :raft-leader-state 459 | {:keys [commit-index node-set current-term node-type] 460 | :as raft-state} :raft-state})) 461 | 462 | (def leader-handle-append-entries-response 463 | (guard 464 | (and (= node-type :leader) 465 | (= message-type :append-entries-response)) 466 | (rule 467 | (not success?) 468 | (-> state 469 | (consume-message) 470 | (update-in [:raft-leader-state :next-index from] dec) 471 | (update-in [:raft-leader-state :next-index from] max 0N)) 472 | {:as state 473 | {{:keys [success? from]} :message} :io}) 474 | (rule 475 | success? 476 | (-> state 477 | (consume-message) 478 | (assoc-in [:raft-leader-state :next-index from] (inc last-log-index)) 479 | (assoc-in [:raft-leader-state :match-index from] last-log-index)) 480 | {:as state 481 | {{:keys [success? from last-log-index]} :message} :io}) 482 | {:as state 483 | {{message-type :type} :message} :io 484 | {:keys [node-type]} :raft-state})) 485 | 486 | (def ignore-messages-from-old-terms 487 | (rule 488 | (and message-term 489 | (> current-term message-term)) 490 | (-> state 491 | (consume-message)) 492 | {:as state 493 | {{message-term :term} :message} :io 494 | {:keys [current-term]} :raft-state})) 495 | 496 | (def rules-of-raft 497 | (guard 498 | true 499 | #'ignore-messages-from-old-terms 500 | #'keep-up-apply 501 | #'jump-to-newer-term 502 | #'follower-respond-to-append-entries 503 | #'follower-respond-to-request-vote 504 | #'become-candidate-and-call-for-election 505 | #'candidate-receive-votes 506 | #'candidate-respond-to-append-entries 507 | #'candidate-election-timeout 508 | #'heart-beat 509 | #'leader-receive-command 510 | #'update-followers 511 | #'update-commit 512 | #'leader-handle-append-entries-response 513 | _)) 514 | -------------------------------------------------------------------------------- /test/com/manigfeald/raft/rules_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.manigfeald.raft.rules-test 2 | (:require [clojure.test :refer :all] 3 | [com.manigfeald.raft.rules :refer :all] 4 | [clojure.tools.logging :as log] 5 | [com.manigfeald.raft.core :refer :all]) 6 | (:import (clojure.lang PersistentQueue))) 7 | 8 | ;; TODO: when applied always have a :return 9 | 10 | ;; TODO: these tests really suck 11 | 12 | (deftest t-keep-up-apply 13 | (is (= [false {:raft-state {:commit-index 0 :last-applied 0}}] 14 | (keep-up-apply {:raft-state {:commit-index 0 :last-applied 0}}))) 15 | (is (= [true {:raft-state {:commit-index 1 16 | :last-applied 1 17 | :log {1 {:payload {:op :write 18 | :key :foo 19 | :value :bar} 20 | :index 1 21 | :return nil}} 22 | :value (assoc (->MapValue) 23 | :foo :bar)}}] 24 | (keep-up-apply {:raft-state {:commit-index 1 25 | :last-applied 0 26 | :log {1 {:payload {:op :write 27 | :key :foo 28 | :value :bar} 29 | :index 1}} 30 | :value (->MapValue)}})))) 31 | 32 | (deftest t-jump-to-newer-term 33 | (is (= [false {:io {:message {:term 1}} 34 | :running-log PersistentQueue/EMPTY 35 | :raft-state {:current-term 1}}] 36 | (jump-to-newer-term {:io {:message {:term 1}} 37 | :running-log PersistentQueue/EMPTY 38 | :raft-state {:current-term 1}}))) 39 | (is (= [true {:io {:message {:term 2}} 40 | :raft-state {:current-term 2 41 | :node-type :follower 42 | :votes 0 43 | :last-applied 0 44 | :commit-index 0 45 | :log {} 46 | :leader-id nil 47 | :voted-for nil}}] 48 | (update-in (jump-to-newer-term 49 | {:io {:message {:term 2}} 50 | :running-log PersistentQueue/EMPTY 51 | :raft-state {:current-term 1 52 | :last-applied 0 53 | :commit-index 0 54 | :log {} 55 | :node-type :candidate-id}}) 56 | [1] dissoc :running-log)))) 57 | 58 | (deftest t-follower-respond-to-append-entries 59 | (testing "failing old term" 60 | (is (= [true {:id :com.manigfeald.raft.rules-test/me, 61 | :raft-state {:last-applied 1, 62 | :node-type :follower, 63 | :commit-index 1, 64 | :current-term 1}, 65 | :timer {:next-timeout 8, 66 | :now 5, 67 | :period 3}, 68 | :io {:out-queue [{:type :append-entries-response, 69 | :term 1, 70 | :success? false, 71 | :from ::me, 72 | :target ::bob}], 73 | :message nil}}] 74 | (update-in (follower-respond-to-append-entries 75 | {:id ::me 76 | :running-log PersistentQueue/EMPTY 77 | :raft-state {:node-type :follower 78 | :current-term 1 79 | :commit-index 1 80 | :last-applied 1} 81 | :timer {:period 3 82 | :now 5} 83 | :io {:message {:type :append-entries 84 | :leader-id ::bob 85 | :term 0}}}) 86 | [1] dissoc :running-log)))) 87 | (testing "entries not in log" 88 | (let [[matched? result] (follower-respond-to-append-entries 89 | {:id ::me 90 | :raft-state {:node-type :follower 91 | :current-term 1 92 | :commit-index 1 93 | :last-applied 1 94 | :log {}} 95 | :timer {:period 3 96 | :now 5} 97 | :running-log PersistentQueue/EMPTY 98 | :io {:message {:type :append-entries 99 | :term 1 100 | :prev-log-index 1 101 | :prev-log-term 1}}}) 102 | {{messages :out-queue} :io} result] 103 | (is matched? result) 104 | (doseq [{:keys [type term success? from]} messages] 105 | (is (= type :append-entries-response)) 106 | (is (= term 1)) 107 | (is (not success?)) 108 | (is (= from ::me)))))) 109 | (testing "success" 110 | (let [[matched? result] (follower-respond-to-append-entries 111 | {:id ::me 112 | :raft-state {:node-type :follower 113 | :current-term 1 114 | :commit-index 0 115 | :last-applied 0 116 | :log {1 {:term 1}}} 117 | :timer {:period 3 118 | :now 5} 119 | :running-log PersistentQueue/EMPTY 120 | :io {:message {:type :append-entries 121 | :term 1 122 | :leader-commit 0 123 | :leader-id ::bob 124 | :prev-log-index 1 125 | :prev-log-term 1}}}) 126 | {{messages :out-queue} :io} result] 127 | (is matched? result) 128 | (is (= ::bob (:leader-id (:raft-state result)))) 129 | (doseq [{:keys [type term success? from last-log-index]} 130 | messages] 131 | (is (= type :append-entries-response)) 132 | (is (= term 1)) 133 | (is success?) 134 | (is (= from ::me)) 135 | (is (= 1 last-log-index))))) 136 | 137 | (deftest t-follower-respond-to-request-vote 138 | (testing "not voted zero last term and index" 139 | (let [[applied? result] (follower-respond-to-request-vote 140 | {:id ::me 141 | :raft-state {:node-type :follower 142 | :current-term 1 143 | :voted-for nil 144 | :commit-index 0 145 | :last-applied 0 146 | :log {}} 147 | :timer {:period 3 148 | :now 5} 149 | :running-log PersistentQueue/EMPTY 150 | :io {:message {:type :request-vote 151 | :term 1 152 | :candidate-id ::bob 153 | :last-log-index 0 154 | :last-log-term 0}}}) 155 | messages (:out-queue (:io result))] 156 | (is applied? result) 157 | (is (= 1 (count messages)) messages) 158 | (doseq [{:keys [type target from success?]} messages] 159 | (is success?) 160 | (is (= from ::me)) 161 | (is (= target ::bob)) 162 | (is (= type :request-vote-response))))) 163 | (testing "not voted and last term and index match" 164 | (let [[applied? result] (follower-respond-to-request-vote 165 | {:id ::me 166 | :raft-state {:node-type :follower 167 | :current-term 1 168 | :voted-for nil 169 | :commit-index 0 170 | :last-applied 0 171 | :log {1 {:term 1}}} 172 | :timer {:period 3 173 | :now 5} 174 | :running-log PersistentQueue/EMPTY 175 | :io {:message {:type :request-vote 176 | :term 1 177 | :candidate-id ::bob 178 | :last-log-index 1 179 | :last-log-term 1}}}) 180 | messages (:out-queue (:io result))] 181 | (is applied? result) 182 | (is (= 1 (count messages)) messages) 183 | (doseq [{:keys [type target from success?]} messages] 184 | (is success?) 185 | (is (= from ::me)) 186 | (is (= target ::bob)) 187 | (is (= type :request-vote-response))))) 188 | (testing "not voted and last term and last index don't match" 189 | (let [[applied? result] (follower-respond-to-request-vote 190 | {:id ::me 191 | :raft-state {:node-type :follower 192 | :current-term 1 193 | :voted-for nil 194 | :commit-index 0 195 | :last-applied 0 196 | :log {}} 197 | :timer {:period 3 198 | :now 5} 199 | :running-log PersistentQueue/EMPTY 200 | :io {:message {:type :request-vote 201 | :term 1 202 | :candidate-id ::bob 203 | :last-log-index -1 204 | :last-log-term -11}}}) 205 | messages (:out-queue (:io result))] 206 | (is applied? result) 207 | (is (= 1 (count messages)) messages) 208 | (doseq [{:keys [type target from success?]} messages] 209 | (is (not success?)) 210 | (is (= from ::me)) 211 | (is (= target ::bob)) 212 | (is (= type :request-vote-response))))) 213 | (testing "already voted" 214 | (let [[applied? result] (follower-respond-to-request-vote 215 | {:id ::me 216 | :raft-state {:node-type :follower 217 | :current-term 1 218 | :voted-for ::alice 219 | :commit-index 1 220 | :last-applied 1 221 | :log {}} 222 | :timer {:period 3 223 | :now 5} 224 | :running-log PersistentQueue/EMPTY 225 | :io {:message {:type :request-vote 226 | :term 1 227 | :candidate-id ::bob 228 | :last-log-index 1 229 | :last-log-term 1}}}) 230 | messages (:out-queue (:io result))] 231 | (is applied? result) 232 | (is (= 1 (count messages)) messages) 233 | (doseq [{:keys [type target from success?]} messages] 234 | (is (not success?)) 235 | (is (= from ::me)) 236 | (is (= target ::bob)) 237 | (is (= type :request-vote-response)))))) 238 | 239 | (deftest t-become-candidate-and-call-for-election 240 | (let [[applied? result] (become-candidate-and-call-for-election 241 | {:id ::me 242 | :raft-state {:node-type :follower 243 | :current-term 1 244 | :commit-index 0 245 | :last-applied 0 246 | :voted-for nil 247 | :node-set #{::a ::b ::c} 248 | :log {1 {:index 1 249 | :term 1}}} 250 | :running-log PersistentQueue/EMPTY 251 | :timer {:period 3 252 | :next-timeout 3 253 | :now 5} 254 | :io {:message nil}}) 255 | messages (:out-queue (:io result))] 256 | (is (= 8 (:next-timeout (:timer result))) result) 257 | (is applied? result) 258 | (is (= 3 (count messages)) messages) 259 | (is (= #{::a ::b ::c} (set (map :target messages)))) 260 | (is (= 2 (:current-term (:raft-state result)))) 261 | (doseq [{:keys [type target candidate-id from last-log-term 262 | last-log-index term]} 263 | messages] 264 | (is (= term 2)) 265 | (is (= last-log-index 1)) 266 | (is (= last-log-term 1)) 267 | (is (= from ::me)) 268 | (is (= candidate-id ::me)) 269 | (is (= type :request-vote))))) 270 | 271 | (deftest t-candidate-receive-votes 272 | (testing "receive vote, become master" 273 | (let [[applied? result] (candidate-receive-votes 274 | {:id ::me 275 | :raft-state {:node-type :candidate 276 | :current-term 1 277 | :last-applied 0 278 | :commit-index 0 279 | :log {} 280 | :node-set #{1 2 3} 281 | :votes 2} 282 | :timer {:now 10 283 | :period 3} 284 | :running-log PersistentQueue/EMPTY 285 | :io {:message {:type :request-vote-response 286 | :success? true}}}) 287 | {{messages :in-queue} :io} result] 288 | (is applied? result) 289 | (is (= :leader (:node-type (:raft-state result)))) 290 | (is (= 13 (:next-timeout (:timer result)))))) 291 | (testing "receive vote, count it" 292 | (let [[applied? result] (candidate-receive-votes 293 | {:id ::me 294 | :raft-state {:node-type :candidate 295 | :current-term 1 296 | :last-applied 0 297 | :commit-index 0 298 | :log {} 299 | :node-set #{1 2 3 4 5} 300 | :votes 1} 301 | :running-log PersistentQueue/EMPTY 302 | :timer {:now 10 303 | :period 3} 304 | :io {:message {:type :request-vote-response 305 | :success? true}}}) 306 | {{messages :in-queue} :io} result] 307 | (is applied? result) 308 | (is (= :candidate (:node-type (:raft-state result)))) 309 | (is (= 2 (:votes (:raft-state result)))))) 310 | (testing "don't receive vote" 311 | (let [[applied? result] (candidate-receive-votes 312 | {:id ::me 313 | :raft-state {:node-type :candidate 314 | :current-term 1 315 | :node-set #{1 2 3 4 5} 316 | :votes 1} 317 | :running-log PersistentQueue/EMPTY 318 | :timer {:now 10 319 | :period 3} 320 | :io {:message {:type :request-vote-response 321 | :success? false}}}) 322 | {{messages :in-queue} :io} result] 323 | (is (not applied?) result)))) 324 | 325 | (deftest t-candidate-respond-to-append-entries 326 | (let [[applied? result] (candidate-respond-to-append-entries 327 | {:id ::me 328 | :raft-state {:node-type :candidate 329 | :current-term 1 330 | :commit-index 0 331 | :last-applied 0 332 | :node-set #{1 2 3} 333 | :votes 2} 334 | :timer {:now 10 335 | :period 3} 336 | :running-log PersistentQueue/EMPTY 337 | :io {:message {:type :append-entries 338 | :term 1 339 | :leader-id ::bob 340 | :prev-log-index 0 341 | :prev-log-term 0 342 | :entries [] 343 | :leader-commit 0}}}) 344 | {{messages :in-queue} :io} result] 345 | (is applied? result) 346 | (is (= :follower (:node-type (:raft-state result)))) 347 | (is (= 13 (:next-timeout (:timer result)))))) 348 | 349 | (deftest t-candidate-election-timeout 350 | (let [[applied? result] (candidate-election-timeout 351 | {:id ::me 352 | :raft-state {:node-type :candidate 353 | :current-term 1 354 | :node-set #{::a ::b ::c} 355 | :commit-index 0 356 | :last-applied 0 357 | :log {}} 358 | :running-log PersistentQueue/EMPTY 359 | :timer {:now 10 360 | :next-timeout 9 361 | :period 3} 362 | :io {:message nil}}) 363 | {{messages :out-queue} :io} result] 364 | (is applied? result) 365 | (is (= #{::a ::b ::c} (set (map :target messages)))) 366 | (is (= 13 (:next-timeout (:timer result)))) 367 | (is (= 2 (:current-term (:raft-state result)))) 368 | (doseq [{:keys [type candidate-id term]} messages] 369 | (is (= type :request-vote)) 370 | (is (= candidate-id ::me)) 371 | (is (= 2 term))))) 372 | 373 | (deftest t-heart-beat 374 | (let [[applied? result] (heart-beat 375 | {:id ::me 376 | :raft-state {:node-type :leader 377 | :current-term 1 378 | :commit-index 0 379 | :last-applied 0 380 | :log {} 381 | :node-set #{::a ::b ::me}} 382 | :timer {:now 10 383 | :next-timeout 9 384 | :period 3} 385 | :io {:message nil}}) 386 | {{messages :out-queue} :io} result] 387 | (is applied? result) 388 | (is (= 2 (count messages))) 389 | (is (= #{::a ::b} (set (map :target messages)))) 390 | (is (= #{:append-entries} (set (map :type messages)))))) 391 | 392 | (deftest t-update-followers 393 | (let [[applied? result] (update-followers 394 | {:id ::me 395 | :raft-state {:node-type :leader 396 | :current-term 1 397 | :commit-index 1 398 | :last-applied 1 399 | :log {1 {:index 1 :term 1}} 400 | :node-set #{::a ::b ::me}} 401 | :raft-leader-state {:next-index {::a 1 402 | ::b 1}} 403 | :io {:message nil}}) 404 | {{messages :out-queue} :io} result] 405 | (is (= 2 (count messages))) 406 | (is applied? result) 407 | (doseq [{:keys [type entries prev-log-term prev-log-index]} messages] 408 | (is (zero? prev-log-term)) 409 | (is (zero? prev-log-index)) 410 | (is (= type :append-entries)) 411 | (is (= [{:index 1 :term 1}] entries)))) 412 | (let [[applied? result] (update-followers 413 | {:id ::me 414 | :raft-state {:node-type :leader 415 | :current-term 1 416 | :log {1 {:index 1 :term 1}} 417 | :node-set #{::a ::b ::me}} 418 | :raft-leader-state {:next-index {::a 2 419 | ::b 2}} 420 | :io {:message nil}}) 421 | {{messages :out-queue} :io} result] 422 | (is (not applied?) result))) 423 | 424 | (deftest t-update-commit 425 | (let [[applied? result] (update-commit 426 | {:id ::me 427 | :running-log PersistentQueue/EMPTY 428 | :raft-state {:node-type :leader 429 | :current-term 1 430 | :last-applied 0 431 | :log {1 {:index 1 :term 1}} 432 | :node-set #{::a ::b ::me} 433 | :commit-index 0} 434 | :raft-leader-state {:match-index {::a 1 435 | ::b 1}} 436 | :io {:message nil}}) 437 | {{messages :out-queue} :io} result] 438 | (is applied? result) 439 | (is (= 1 (:commit-index (:raft-state result)))))) 440 | 441 | (deftest t-leader-handle-append-entries-response 442 | (let [[applied? result] (leader-handle-append-entries-response 443 | {:id ::me 444 | :raft-state {:node-type :leader 445 | :current-term 1 446 | :last-applied 0 447 | :log {1 {:index 1 :term 1}} 448 | :node-set #{::a ::b ::me} 449 | :commit-index 0} 450 | :raft-leader-state {:match-index {::a 0} 451 | :next-index {::a 1}} 452 | :io {:message {:type :append-entries-response 453 | :from ::a 454 | :success? false}}}) 455 | {{messages :out-queue} :io} result] 456 | (is applied? result) 457 | (is (= 0 (::a (:next-index (:raft-leader-state result)))))) 458 | (let [[applied? result] (leader-handle-append-entries-response 459 | {:id ::me 460 | :raft-state {:node-type :leader 461 | :current-term 1 462 | :last-applied 0 463 | :log {1 {:index 1 :term 1}} 464 | :node-set #{::a ::b ::me} 465 | :commit-index 0} 466 | :raft-leader-state {:match-index {::a 0} 467 | :next-index {::a 1}} 468 | :io {:message {:type :append-entries-response 469 | :from ::a 470 | :success? true 471 | :last-log-index 1}}}) 472 | {{messages :out-queue} :io} result] 473 | (is applied? result) 474 | (is (= 2 (::a (:next-index (:raft-leader-state result))))) 475 | (is (= 1 (::a (:match-index (:raft-leader-state result))))))) 476 | 477 | (deftest t-leader-receive-command 478 | (let [[applied? result] 479 | (leader-receive-command 480 | {:io {:message {:type :operation, 481 | :payload {:op :write, 482 | :key :hello, 483 | :value :world}, 484 | :operation-type :com.manigfeald.raft-test/bogon, 485 | :serial #uuid "73315024-7517-4506-8428-275a8d1a6d84"}, 486 | :out-queue PersistentQueue/EMPTY}, 487 | :raft-state {:current-term 2N, 488 | :voted-for 1, 489 | :log {}, 490 | :commit-index 0N, 491 | :last-applied 0N, 492 | :node-type :leader, 493 | :value #com.manigfeald.raft.core.MapValue{}, 494 | :votes 0, :leader-id 1, :node-set #{0 1 4 3 2}}, 495 | :raft-leader-state {:next-index {}, 496 | :match-index {}}, 497 | :id 2, 498 | :running-log PersistentQueue/EMPTY, 499 | :timer {:now 1401418884457, :next-timeout 1401418884545, :period 1005} 500 | :run-count 16530N})] 501 | (is applied?) 502 | (is (contains? (:log (:raft-state result)) 1N)))) 503 | 504 | (defmacro foo [exp & body] 505 | (when-not (eval exp) 506 | `(do ~@body))) 507 | 508 | (foo (resolve 'clojure.test/original-test-var) 509 | (in-ns 'clojure.test) 510 | 511 | (def original-test-var test-var) 512 | 513 | (defn test-var [v] 514 | (clojure.tools.logging/trace "testing" v) 515 | (let [start (System/currentTimeMillis) 516 | result (original-test-var v)] 517 | (clojure.tools.logging/trace 518 | "testing" v "took" 519 | (/ (- (System/currentTimeMillis) start) 1000.0) 520 | "seconds") 521 | result)) 522 | 523 | (def original-test-ns test-ns) 524 | 525 | (defn test-ns [ns] 526 | (clojure.tools.logging/trace "testing namespace" ns) 527 | (let [start (System/currentTimeMillis) 528 | result (original-test-ns ns)] 529 | (clojure.tools.logging/trace 530 | "testing namespace" ns "took" 531 | (/ (- (System/currentTimeMillis) start) 1000.0) 532 | "seconds") 533 | result)) 534 | 535 | (in-ns 'com.manigfeald.raft.rules-test)) 536 | --------------------------------------------------------------------------------