├── deps.edn ├── src └── lmalob.clj └── readme.md /deps.edn: -------------------------------------------------------------------------------- 1 | {:deps 2 | { 3 | org.clojure/clojure {:mvn/version "1.10.2-rc1"} 4 | org.clojure/data.json {:mvn/version "1.0.0"} 5 | java-http-clj/java-http-clj {:mvn/version "0.4.1"} 6 | org.clojure/core.async {:mvn/version "1.2.603"} 7 | dev.jt/lob {:git/url "https://github.com/jjttjj/lob.git" 8 | :sha "f616f3c0728f0a9cd016093d552e1c9aa0fb72a8"} 9 | 10 | }} 11 | -------------------------------------------------------------------------------- /src/lmalob.clj: -------------------------------------------------------------------------------- 1 | (ns lmalob 2 | (:require [java-http-clj.core :as http] 3 | [java-http-clj.websocket :as ws] 4 | [clojure.data.json :as json] 5 | [clojure.core.async :as a :refer 6 | [! >!! alt! alts! chan go go-loop poll!]] 7 | [dev.jt.lob :as lob])) 8 | 9 | 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | ;;; util 12 | 13 | (defn info [& xs] (apply println xs)) 14 | 15 | (defn uuid [s] (java.util.UUID/fromString s)) 16 | 17 | (defn cf->ch [^java.util.concurrent.CompletableFuture cf ch] 18 | (.whenCompleteAsync cf 19 | (reify 20 | java.util.function.BiConsumer 21 | (accept [_ result exception] 22 | (a/put! ch (or result exception))))) 23 | ch) 24 | 25 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 26 | ;;; rest api 27 | 28 | (def rest-url "https://api.pro.coinbase.com") 29 | (def sandbox-rest-url "https://api-public.sandbox.pro.coinbase.com") 30 | 31 | (defn request-ch [req-map ch] 32 | (-> (http/send-async req-map) 33 | (cf->ch ch))) 34 | 35 | (defn parse-level3-val [k v] 36 | (case k 37 | (:asks :bids) (mapv (fn [[px sz id]] [(bigdec px) (bigdec sz) (uuid id)]) v) 38 | ;;else return unchanged string 39 | v)) 40 | 41 | (defn parse-level3 [s] 42 | (json/read-str s 43 | :key-fn keyword 44 | :value-fn parse-level3-val)) 45 | 46 | (defn req-level3 [sandbox? ch] 47 | (let [base (if sandbox? sandbox-rest-url rest-url) 48 | result (chan 1 (map (fn [response] (parse-level3 (:body response)))))] 49 | (request-ch 50 | {:method :get 51 | :uri (str base "/products/BTC-USD/book?level=3")} 52 | result) 53 | (a/pipe result ch))) 54 | 55 | (defn cb-level3->lob [{:keys [asks bids sequence] :as cb-lob}] 56 | (as-> (lob/empty-lob) lob 57 | (reduce (fn [lob [px sz id time]] (lob/insert lob ::lob/asks px id nil sz)) lob asks) 58 | (reduce (fn [lob [px sz id time]] (lob/insert lob ::lob/bids px id nil sz)) lob bids) 59 | (assoc lob :sequence sequence))) 60 | 61 | (comment 62 | (cb-level3->lob (ch [url open-ch recv-ch] 71 | (ws/build-websocket url 72 | {:on-text (let [sa (atom "")] 73 | (fn [ws s last?] 74 | (let [s (swap! sa str s)] 75 | (when last? 76 | (a/put! recv-ch s) 77 | (reset! sa ""))))) 78 | :on-open (fn [ws] 79 | (info "ws opened") 80 | (a/put! open-ch ws)) 81 | :on-close (fn [ws status reason] (info "ws closed. status:" status "reason:" reason)) 82 | :on-error (fn [ws throwable] (info throwable "ws error"))})) 83 | 84 | (defn parse-websocket-value [k v] 85 | (case k 86 | :time (java.time.Instant/parse v) 87 | (:size :price :remaining_size :funds) (bigdec v) 88 | (:type :product_id :side :order_type :reason) (keyword v) 89 | (:order_id :maker_order_id :taker_order_id) (uuid v) 90 | ;;else 91 | (when (not= "" v) 92 | v))) 93 | 94 | (defn parse-ws-msg [s] 95 | (json/read-str s :key-fn keyword :value-fn parse-websocket-value)) 96 | 97 | (defn lob-msg? [{:keys [type]}] 98 | (or (identical? type :open) 99 | (identical? type :done))) 100 | 101 | (defn with-msg [lob {:keys [type time price order_id remaining_size side sequence]}] 102 | (when-let [k (#{:done :open} type)] 103 | (let [lob-side (case side :buy ::lob/bids :sell ::lob/asks)] 104 | (-> 105 | (case k 106 | :open (lob/insert lob lob-side price order_id time remaining_size) 107 | ;; presumes that delete is noop for orders not in book 108 | :done (lob/delete lob lob-side price order_id)) 109 | (assoc :time time :sequence sequence))))) 110 | 111 | (defn with-msgs [lob msgs] 112 | (reduce with-msg lob msgs)) 113 | 114 | (defn get-initial-lob 115 | "Takes a map with the following keys: 116 | :out - a channel on which the resulting initial LOB will be put. 117 | :in - a channel of :open and :done messages from coinbase's full feed. Values will be consumed from this feed as necessary until the :sequence number of the messages surpass that of the initial LOB. 118 | :init-delay - the delay in milliseconds to wait before initially requesting the Coinbase level 3 LOB. 119 | :retry-delay - the delay in milliseconds to wait after an invalid Coinbase LOB 120 | was received before trying again." 121 | [{:keys [in out init-delay retry-delay sandbox?] 122 | :or {init-delay 5000 123 | retry-delay 3000 124 | out (chan 1)} 125 | :as opt}] 126 | (go 127 | (info "waiting for first input msg...") 128 | (let [cb-level3-ch (chan 1) 129 | msg1 (= (:sequence cb-lob) (:sequence msg1)) 142 | (do (info "lob initialized") 143 | (->> msgs 144 | (drop-while (fn [msg] (< (:sequence msg) (:sequence cb-lob)))) 145 | (with-msgs (cb-level3->lob cb-lob)) 146 | (>! out))) 147 | ;; else, request another lob and try again 148 | (do 149 | (info "lob snapshot occured before first collected feed message, invalid" 150 | "waiting to retry...") 151 | (! out new-acc) 170 | (recur new-acc [] (a/timeout batch-ms))))))) 171 | out)) 172 | 173 | 174 | (comment 175 | 176 | (def sandbox? false) 177 | (def ws-prom (a/promise-chan)) 178 | (def lob-input-ch (chan 20000 (comp (map parse-ws-msg) (filter lob-msg?)))) 179 | (def init-ws-msg (json/write-str 180 | {:type :subscribe 181 | :product_ids ["BTC-USD"] 182 | :channels ["full"]})) 183 | 184 | 185 | (ws->ch (if sandbox? sandbox-websocket-url websocket-url) ws-prom lob-input-ch) 186 | 187 | (def init-lob (a/promise-chan)) 188 | 189 | 190 | (def lobs (chan (a/sliding-buffer 1))) 191 | 192 | (ws/send ( total size) 208 | (->> (select-keys (a/poll! lobs) [::lob/asks ::lob/bids]) 209 | (map (fn [[side-key px->level]] 210 | [side-key 211 | (reduce-kv 212 | (fn [m px level] 213 | (assoc m px (lob/level-size level))) 214 | (empty px->level) 215 | px->level)])) 216 | (into {})) 217 | 218 | (ws/close (! >!! alt! alts! chan go go-loop poll!]] 48 | [dev.jt.lob :as lob])) 49 | ``` 50 | 51 | ### Util 52 | 53 | ```clojure 54 | (defn info [& xs] (apply println xs)) 55 | 56 | (defn uuid [s] (java.util.UUID/fromString s)) 57 | 58 | (defn cf->ch [^java.util.concurrent.CompletableFuture cf ch] 59 | (.whenCompleteAsync cf 60 | (reify 61 | java.util.function.BiConsumer 62 | (accept [_ result exception] 63 | (a/put! ch (or result exception))))) 64 | ch) 65 | 66 | ``` 67 | 68 | I'll use `info` as a placeholder for some real logging I may eventually want to do. `uuid` parses uuids strings. 69 | `cf->ch` puts the result of a java `CompletableFuture` onto a core.async `chan`. The `java-http-clj` library gives us the option to have results returned on a `CompletableFuture` to use them asynchronously. This helper function lets me use core.async instead. 70 | 71 | First we will need a way to interact with both the REST and websocket APIs coinbase offers. There are many options for these things today in clojure land. Recently I've been using [java-http-clj](https://github.com/schmee/java-http-clj) which offers a minimal wrapper over the built in `java.net.http` HTTP and websocket clients, and uses no dependencies. 72 | 73 | 74 | ### API notes 75 | 76 | Coinbase is kind enough to offer their real data publicly. We don't technically need to use the sandbox URLs, since we're not using any endpoints that are potentially dangerous (such as placing orders). However, the full LOB data we're requesting is huge (~20MB per request), and the sandbox versions are significantly smaller (1-2MB) so it is polite of us (and prevents our IP from being potentially banned for abuse) to use the sandbox endpoints until we ready to try things out with the real feed. 77 | 78 | ```clojure 79 | (def rest-url "https://api.pro.coinbase.com") 80 | (def sandbox-rest-url "https://api-public.sandbox.pro.coinbase.com") 81 | ``` 82 | 83 | For this demo we used Bitcoin, but any of the currencies Coinbase offers works just as well, just replace "BTC-USD" with the appropriate product id. You can see all products with the following code. Note that the sandbox API provides much fewer products than the "real" one. 84 | 85 | ```clojure 86 | 87 | (-> (http/get (str sandbox-rest-url "/products")) 88 | :body 89 | (json/read-str :key-fn keyword) 90 | (->> (map :id))) 91 | ``` 92 | 93 | 94 | ## The LOB snapshot 95 | 96 | We'll need a snapshot to use as a starting point to build the current LOB from. 97 | 98 | The [Order Book](https://docs.pro.coinbase.com/#get-product-order-book) endpoint, with the `level` parameter set to 3, will give us a good starting point. 99 | 100 | ``` 101 | (http/get (str sandbox-rest-url "/products/BTC-USD/book?level=3")) 102 | 103 | ``` 104 | 105 | The returned JSON looks like: 106 | 107 | ```javascript 108 | { 109 | "sequence": "3", 110 | "bids": [ 111 | [ "295.96","0.05088265","3b0f1225-7f84-490b-a29f-0faef9de823a" ], 112 | ... 113 | ], 114 | "asks": [ 115 | [ "295.97","5.72036512","da863862-25f4-4868-ac41-005d11ab0a5f" ], 116 | ... 117 | ] 118 | } 119 | ``` 120 | 121 | It is important that the we parse the prices as `bigdec` because they will serve as keys in our lob, and doubles make for bad map keys because of their [rounding errors](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html). We'll make `size` a bigdec as well. 122 | 123 | The `:sequence` value represents a number that is guaranteed to be increasing with time. When comparing two messages, a higher sequence number will mean that message definitely occurred later. This will be important for us later. 124 | 125 | We also want to make our requests asynchronously because these are big/slow requests and we'll have stuff to do while waiting for the response 126 | 127 | ```clojure 128 | (defn request-ch [req-map ch] 129 | (-> (http/send-async req-map) 130 | (cf->ch ch))) 131 | 132 | (defn parse-level3-val [k v] 133 | (case k 134 | (:asks :bids) (mapv (fn [[px sz id]] [(bigdec px) (bigdec sz) (uuid id)]) v) 135 | ;;else return unchanged string 136 | v) 137 | ) 138 | (defn parse-level3 [s] 139 | (json/read-str s 140 | :key-fn keyword 141 | :value-fn parse-level3-val)) 142 | 143 | (defn req-level3 [sandbox? ch] 144 | (let [base (if sandbox? sandbox-websocket-url rest-url) 145 | result (chan 1 (map (fn [response] (parse-level3 (:body response)))))] 146 | (request-ch 147 | {:method :get 148 | :uri (str base "/products/BTC-USD/book?level=3")} 149 | result) 150 | (a/pipe result ch))) 151 | ``` 152 | 153 | Now we can conveniently request the level3 data be parsed into nice clojure types and put onto a channel we provide: 154 | 155 | ```clojure 156 | (lob [{:keys [asks bids sequence] :as cb-lob}] 175 | (as-> (lob/empty-lob) lob 176 | (reduce (fn [lob [px sz id time]] (lob/insert lob ::lob/asks px id nil sz)) lob asks) 177 | (reduce (fn [lob [px sz id time]] (lob/insert lob ::lob/bids px id nil sz)) lob bids) 178 | (assoc lob :sequence sequence))) 179 | ``` 180 | 181 | One thing to note is that the Coinbase level 3 lob gives us order ids for each order but not the time they were placed. So we `lob/insert` them with a time of `nil`. In effect this will cause all the orders initialized in this lob to behave as if they were added before any orders which ARE `lob/insert`ed with a time value. This is good enough for our usage. 182 | 183 | ```clojure 184 | (cb-level3->lob ( 187 | {:dev.jt.lob/asks {31089M {#uuid "0d4982a1..." [nil 3009.99903649M]}, 188 | 31089.01M {#uuid "7c517469..." [nil 4771M]}, 189 | 31089.02M {#uuid "e9dba70f..." [nil 6020M]}, 190 | ...}, 191 | :dev.jt.lob/bids {31088.98M {#uuid "18a0459a..." [nil 3009.9993567M]}, 192 | 31088.97M {#uuid "fd7aa4f0..." [nil 4771M]}, 193 | 31088.96M {#uuid "b865e258..." [nil 6020M]}, 194 | ...}, 195 | :sequence 243573659} 196 | ``` 197 | 198 | ## The Websocket Feed 199 | 200 | Now we have a base LOB to work off of, it's time to update it with a websocket connection. 201 | 202 | ```clojure 203 | (def websocket-url "wss://ws-feed.pro.coinbase.com") 204 | (def sandbox-websocket-url "wss://ws-feed-public.sandbox.pro.coinbase.com") 205 | 206 | ``` 207 | 208 | `ws->clj` is a function to establish a websocket connection, puts the websocket result object on a core.async channel and puts every text message received on the socket onto a seperate channel. We'll also report the status changes with our `info` function. 209 | 210 | ```clojure 211 | (defn ws->ch [url open-ch recv-ch] 212 | (ws/build-websocket url 213 | {:on-text (let [sa (atom "")] 214 | (fn [ws s last?] 215 | (let [s (swap! sa str s)] 216 | (when last? 217 | (a/put! recv-ch s) 218 | (reset! sa ""))))) 219 | :on-open (fn [ws] 220 | (info "ws opened") 221 | (a/put! open-ch ws)) 222 | :on-close (fn [ws status reason] (info "ws closed. status:" status "reason:" reason)) 223 | :on-error (fn [ws throwable] (info throwable "ws error"))})) 224 | ``` 225 | 226 | Let's give this a spin: 227 | 228 | ```clojure 229 | (def recv1 (a/chan (a/sliding-buffer 10))) 230 | (def ws1 (a/promise-chan)) 231 | 232 | (ws->ch websocket-url ws1 recv1) 233 | 234 | (ws/send ( 242 | "{\"type\":\"done\",\"side\":\"sell\",\"product_id\":\"BTC-USD\",\"time\":\"2021-01-04T17:51:47.560057Z\",\"sequence\":19370793361,\"order_id\":\"10fe8869-7837-4180-867a-393f77b8b93d\",\"reason\":\"canceled\",\"price\":\"30734.96\",\"remaining_size\":\"0.38\"}" 243 | 244 | (ws/close (ch websocket-url ws2 recv2) 276 | (ws/send ( 314 | (case k 315 | :open (lob/insert lob lob-side price order_id time remaining_size) 316 | ;; presumes that delete is noop for orders not in book 317 | :done (lob/delete lob lob-side price order_id)) 318 | (assoc :time time :sequence sequence))))) 319 | 320 | (defn with-msgs [lob msgs] 321 | (reduce with-msg lob msgs)) 322 | ``` 323 | 324 | ## Syncing up the feed and initial LOB 325 | 326 | Now for the tricky part. Remember the LOB we got from the rest api has a `:sequence` number? So do all the messages we receive on the websocket feed. The sequence number represents the order of these events occuring in time. In practice I've found that often when you request a LOB via the rest api and start a websocket full feed at roughly the same time, the sequence number of the lob result will be behind the sequence number of the first websocket feed message. This is not good, because it means that there are messages that have occured between the LOB being generated and the messages we receive from the feed, meaning our LOB would not be fully accurate. It seems that the LOB is generally a few seconds behind the websocket feed. To fix this we need to 327 | 328 | 1. Collect all messages from the "full" feed, making note of the first sequence number 329 | 2. Wait a few seconds, then request the level 3 LOB, while continuing to collect messages from the feed. 330 | 3. When we get the level 3 LOB, compare its sequence number to the first feed message. 331 | - If the level 3 LOB's sequence number is greater than or equal to the sequence of the first feed message, then drop all messages from the feed with a sequence number less than the level 3 LOB, merge those into the lob with `with-messages` and this will be our initial LOB. We also need to immediately stop taking messages from the feed so every message occuring after the Level 3 LOB sequence number will be added to our LOB. 332 | - If the level 3 LOB's sequence number is less than the first feed messages, we need to try again. `GOTO 2.` 333 | 4. Now we have an initial LOB and a channel which contains all the "full" feed messages occuring after this initial LOB. 334 | 335 | 336 | ```clojure 337 | (defn get-initial-lob 338 | "Takes a map with the following keys: 339 | :out - a channel on which the resulting initial LOB will be put. 340 | :in - a channel of :open and :done messages from coinbase's full feed. Values will be consumed from this feed as necessary until the :sequence number of the messages surpass that of the initial LOB. 341 | :init-delay - the delay in milliseconds to wait before initially requesting the Coinbase level 3 LOB. 342 | :retry-delay - the delay in milliseconds to wait after an invalid Coinbase LOB 343 | was received before trying again." 344 | [{:keys [in out init-delay retry-delay sandbox?] 345 | :or {init-delay 5000 346 | retry-delay 3000 347 | out (chan 1)} 348 | :as opt}] 349 | (go 350 | (info "waiting for first input msg...") 351 | (let [cb-level3-ch (chan 1) 352 | msg1 (= (:sequence cb-lob) (:sequence msg1)) 365 | (do (info "lob initialized") 366 | (->> msgs 367 | (drop-while (fn [msg] (< (:sequence msg) (:sequence cb-lob)))) 368 | (with-msgs (cb-level3->lob cb-lob)) 369 | (>! out))) 370 | ;; else, request another lob and try again 371 | (do 372 | (info "lob snapshot occured before first collected feed message, invalid" 373 | "waiting to retry...") 374 | (! out new-acc) 396 | (recur new-acc [] (a/timeout batch-ms))))))) 397 | out)) 398 | ``` 399 | 400 | ## Putting it all together. 401 | 402 | We can now finally get a real time LOB! 403 | 404 | ```clojure 405 | (def sandbox? false) ;;We're now ready for non-sandbox data 406 | (def ws-prom (a/promise-chan)) 407 | (def lob-input-ch (chan 20000 (comp (map parse-ws-msg) (filter lob-msg?)))) 408 | (def init-ws-msg (json/write-str 409 | {:type :subscribe 410 | :product_ids ["BTC-USD"] 411 | :channels ["full"]})) 412 | 413 | 414 | (ws->ch (if sandbox? sandbox-websocket-url websocket-url) ws-prom lob-input-ch) 415 | ;;(ws/close (> (select-keys (a/poll! lobs) [::lob/asks ::lob/bids]) 472 | (map (fn [[side-key px->levels]] 473 | [side-key 474 | (reduce-kv 475 | (fn [m px level] 476 | (assoc m px (lob/level-size level))) 477 | (empty px->levels) 478 | px->levels)])) 479 | (into {})) 480 | ``` 481 | 482 | 483 | ## Conclusion 484 | 485 | We now have a non-aggregated bitcoin order book. This is cool but it's not the funnest thing to stare at for hours on end. However it is a necessary foundation on which to build things that are more fun, such as visualizations and bots for executing trading strategies. We will explore these in future posts, stay tuned! 486 | 487 | 488 | ### About Me 489 | 490 | My name is Justin. If you're interested in Clojure and trading and want to chat feel free to reach out to me at jjttjj@gmail.com. I'm beginning to try to find people with similar interests to collaborate with. 491 | 492 | 493 | Copyright © 2021 Justin Tirrell 494 | --------------------------------------------------------------------------------