├── .dockerignore ├── .github └── workflows │ └── build.yml ├── .gitignore ├── @types └── observablehq__plot.d.ts ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── Makefile ├── common ├── Cargo.toml ├── src │ ├── backoff.rs │ ├── http.rs │ ├── lib.rs │ ├── logging.rs │ ├── orders.rs │ ├── worker.rs │ └── ws.rs └── tests │ ├── test_context.rs │ └── test_orders.rs ├── devops ├── common ├── docker-compose.yml ├── docker-login ├── kollector.dockerfile ├── kong.yml ├── lint-py ├── pyservice.dockerfile └── web.dockerfile ├── gateways ├── Cargo.toml └── src │ ├── binance │ ├── gateway.rs │ ├── mod.rs │ ├── models.rs │ └── requests.rs │ ├── bitstamp │ ├── gateway.rs │ ├── mod.rs │ └── models.rs │ ├── gateway.rs │ └── lib.rs ├── main.py ├── package.json ├── poetry.lock ├── pyproject.toml ├── pyservice ├── __init__.py ├── app.py ├── binance.py ├── book.py ├── config.py ├── console.py ├── gateway.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_app.py │ ├── test_book.py │ └── ws.py └── workers.py ├── readme.md ├── service ├── Cargo.toml ├── build.rs ├── proto │ └── orderbook.proto └── src │ ├── grpc.rs │ ├── http.rs │ ├── kollector.rs │ ├── lib.rs │ ├── main.rs │ └── metrics.rs ├── setup.cfg ├── tsconfig.json ├── web ├── Index.tsx ├── Main.tsx ├── Viz │ ├── Chart.tsx │ ├── PlotReact.tsx │ └── index.ts ├── index.html ├── proto │ ├── orderbook_grpc_web_pb.js │ └── orderbook_pb.js └── service.ts ├── webpack.config.js └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | target 2 | node_modules 3 | dist 4 | .venv 5 | .env 6 | 7 | *.log 8 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: push 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | env: 9 | GITHUB_TOKEN: ${{ secrets.QMBOT_GITHUB_TOKEN }} 10 | DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} 11 | AWS_DEFAULT_REGION: eu-west-1 12 | AWS_ACCESS_KEY_ID: AKIAU5LDMRBGMQEI4JXP 13 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 14 | 15 | steps: 16 | - name: checkout code 17 | uses: actions/checkout@v2 18 | - name: login to docker repos 19 | run: make docker-login 20 | - name: build image 21 | run: make image 22 | - name: build web image 23 | run: make image-web 24 | - name: build py image 25 | run: make image-py 26 | - name: lint 27 | run: make test-lint 28 | - name: tests 29 | run: make test 30 | - name: push image docker repos 31 | run: make image-push 32 | - name: build documentation 33 | run: make doc-ci 34 | - name: publish documentation 35 | if: ${{ github.ref == 'refs/heads/main' }} 36 | uses: JamesIves/github-pages-deploy-action@v4.3.3 37 | with: 38 | branch: gh-pages 39 | folder: target/doc 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | node_modules 3 | dist 4 | devops/helm 5 | .env 6 | .venv 7 | __pycache__ 8 | *.pyc 9 | 10 | *.log 11 | -------------------------------------------------------------------------------- /@types/observablehq__plot.d.ts: -------------------------------------------------------------------------------- 1 | // plot does not have @types yet, this is just an hack to make typescript happy 2 | declare module "@observablehq/plot" { 3 | let plot: any; 4 | let areaX: any; 5 | let lineX: any; 6 | let legend: any; 7 | let barX: any; 8 | let barY: any; 9 | let rectX: any; 10 | let rectY: any; 11 | let ruleX: any; 12 | let ruleY: any; 13 | let dot: any; 14 | 15 | export { plot, areaX, lineX, dot, legend, barX, barY, ruleX, ruleY, rectX, rectY }; 16 | } 17 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "common", 4 | "gateways", 5 | "service" 6 | ] 7 | 8 | 9 | [profile.release] 10 | opt-level = 3 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Quantmind 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # import local .env with local env variables 2 | $(shell touch .env) 3 | include .env 4 | export $(shell sed 's/=.*//' .env) 5 | RUST_VERSION = 1.60 6 | 7 | .PHONY: help build cloc doc-ci doc docker-login image image-push web lint test test-lint 8 | 9 | help: 10 | @echo ================================================================================ 11 | @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' 12 | @echo ================================================================================ 13 | 14 | 15 | build: ## build and test 16 | cargo build 17 | cargo test 18 | 19 | cloc: ## Count lines of code - requires cloc 20 | cloc --exclude-dir=target,.venv,node_modules,dist,.mypy_cache . 21 | 22 | doc-ci: ## build documentation 23 | @cargo doc --workspace --no-deps 24 | 25 | doc: ## build documentation and open web page 26 | @cargo doc --workspace --no-deps --open 27 | 28 | docker-login: ## login to docker repos - this is for admins only 29 | @./devops/docker-login 30 | 31 | image: ## build docker image 32 | docker build . -f devops/kollector.dockerfile -t kollector 33 | 34 | image-web: ## build docker image 35 | docker build . -f devops/web.dockerfile -t kollector-web 36 | 37 | image-py: ## build pyservice image 38 | docker build . -f devops/pyservice.dockerfile -t kollector-py 39 | 40 | image-push: ## push image to repo 41 | @echo skip 42 | 43 | web: ## build web interface 44 | protoc -I ./service/proto orderbook.proto --js_out=import_style=commonjs:web/proto 45 | protoc -I ./service/proto orderbook.proto --grpc-web_out=import_style=commonjs,mode=grpcwebtext:web/proto 46 | 47 | lint: ## lint code 48 | @./devops/lint-py 49 | @cargo fmt 50 | @cargo clippy 51 | 52 | lint-py: 53 | @./devops/lint-py 54 | 55 | service-py: ## start python service with console UI 56 | @poetry run python main.py --console 57 | 58 | test-py: ## test python service 59 | @poetry run pytest -v 60 | 61 | start: ## start dev services 62 | @docker-compose -f devops/docker-compose.yml up 63 | 64 | test: ## run tests 65 | @echo skip 66 | 67 | test-lint: ## lint 68 | @cargo fmt --check 69 | -------------------------------------------------------------------------------- /common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "common" 3 | version = "0.1.0" 4 | edition = "2021" 5 | homepage = "https://github.com/quantmind/kollector" 6 | repository = "https://github.com/quantmind/kollector" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | anyhow = "1.0.57" 12 | async-channel = "1.6.1" 13 | config = "0.13.1" 14 | futures-util = "0.3.21" 15 | reqwest = "0.11.10" 16 | rust_decimal = "1.23.1" 17 | rust_decimal_macros = "1.23.1" 18 | serde = "1.0.137" 19 | serde_json = "1.0.81" 20 | serde_urlencoded = "0.7.1" 21 | slog = "2.7.0" 22 | slog-async = "2.7.0" 23 | slog-json = "2.6.1" 24 | slog-term = "2.9.0" 25 | tokio = { version = "^1.18.2", features = ["full"] } 26 | tokio-tungstenite = { version="^0.17.1", features = ["native-tls"] } 27 | url = "2.2.2" 28 | -------------------------------------------------------------------------------- /common/src/backoff.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::min; 2 | use tokio::time::{sleep, Duration}; 3 | 4 | pub async fn millis_sleep(millis: u64) { 5 | let duration = Duration::from_millis(millis); 6 | sleep(duration).await; 7 | } 8 | 9 | /// Exponential backoff type. 10 | #[derive(Debug, Clone)] 11 | pub struct Backoff { 12 | // 13 | retries: u32, 14 | min: u32, 15 | max: u32, 16 | factor: u32, 17 | counter: u32, 18 | value: u32, 19 | } 20 | 21 | impl Backoff { 22 | pub fn new(retries: u32, min: u32, max: u32, factor: u32) -> Self { 23 | Self { 24 | retries, 25 | min, 26 | max, 27 | factor, 28 | counter: 0, 29 | value: 0, 30 | } 31 | } 32 | 33 | /// Get the next value for the retry count. 34 | pub fn next(&mut self) -> Option { 35 | if self.counter >= self.retries { 36 | return None; 37 | } 38 | let value = self.value; 39 | self.counter += 1; 40 | self.value = match self.counter { 41 | 1 => self.min, 42 | _ => min(self.value * self.factor, self.max), 43 | }; 44 | Some(value) 45 | } 46 | 47 | pub fn reset(&mut self) { 48 | self.value = self.min; 49 | self.counter = 0; 50 | } 51 | 52 | pub fn count(&self) -> u32 { 53 | self.counter 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /common/src/http.rs: -------------------------------------------------------------------------------- 1 | use reqwest::{Client, Method, RequestBuilder, Response, StatusCode}; 2 | use serde::de::DeserializeOwned; 3 | use serde::{Deserialize, Serialize}; 4 | use std::fmt; 5 | use url::Url; 6 | 7 | /// An HTTP Request 8 | pub trait Request: Serialize + Send { 9 | const METHOD: Method; 10 | const PATH: &'static str; 11 | const HAS_PAYLOAD: bool = true; 12 | type Response: DeserializeOwned; 13 | } 14 | 15 | #[derive(Debug, Deserialize, Clone, Copy)] 16 | pub enum ErrorKind { 17 | Network, 18 | InvalidContentMatch, 19 | BadContent, 20 | BadStatus, 21 | } 22 | 23 | #[derive(Debug)] 24 | pub struct Error { 25 | method: Method, 26 | url: Url, 27 | status: String, 28 | err: String, 29 | content: String, 30 | error_kind: ErrorKind, 31 | } 32 | 33 | type Result = std::result::Result; 34 | 35 | /// An Http client for Rest APIs 36 | pub struct HttpClient { 37 | base_url: String, 38 | client: Client, 39 | } 40 | 41 | impl HttpClient { 42 | /// Create a new HttpClient with a base url 43 | pub fn new(url: &str) -> Self { 44 | Self { 45 | base_url: url.to_owned(), 46 | client: Client::new(), 47 | } 48 | } 49 | 50 | /// full url given a path 51 | pub fn url(&self, path: &str) -> String { 52 | format!("{}{}", self.base_url, path) 53 | } 54 | 55 | /// create an unsigned request builder 56 | pub fn unsigned(&self, _req: &R, url: &Url, body: &str) -> RequestBuilder { 57 | let request = self.client.request(R::METHOD, url.clone()); 58 | match body.is_empty() { 59 | true => request, 60 | false => request.header("content-type", "application/json"), 61 | } 62 | } 63 | 64 | /// perform the HTTP request 65 | pub async fn request(&self, req: R, logger: Option<&slog::Logger>) -> Result 66 | where 67 | R: Request, 68 | R::Response: DeserializeOwned, 69 | { 70 | let url = self.url(R::PATH); 71 | let mut url = Url::parse(&url).expect("failed to parse url"); 72 | 73 | if matches!(R::METHOD, Method::GET | Method::DELETE) && R::HAS_PAYLOAD { 74 | url.set_query(Some( 75 | &serde_urlencoded::to_string(&req).expect("failed to encode url payload"), 76 | )); 77 | } 78 | 79 | let body = match R::METHOD { 80 | Method::PUT | Method::POST => { 81 | serde_json::to_string(&req).expect("failed to json encode body payload") 82 | } 83 | _ => "".to_string(), 84 | }; 85 | 86 | let request = self.unsigned(&req, &url, &body); 87 | 88 | // do some logging if logger provided 89 | if let Some(log) = logger { 90 | slog::debug!(log, "HttpClient {} {}", R::METHOD, url); 91 | } 92 | 93 | let response = request 94 | .header("user-agent", "quantmind-trading") 95 | .body(body) 96 | .send() 97 | .await 98 | .unwrap(); 99 | 100 | self.handle_response::(&R::METHOD, &url, response) 101 | .await 102 | } 103 | 104 | async fn handle_response( 105 | &self, 106 | method: &Method, 107 | url: &Url, 108 | resp: Response, 109 | ) -> Result { 110 | let status = resp.status(); 111 | match resp.text().await { 112 | Ok(content) => { 113 | if status.is_success() { 114 | serde_json::from_str::(&content).map_err(|err| { 115 | Error::new( 116 | method.clone(), 117 | url.clone(), 118 | status, 119 | ErrorKind::InvalidContentMatch, 120 | err.to_string(), 121 | content, 122 | ) 123 | }) 124 | } else { 125 | Err(Error::new( 126 | method.clone(), 127 | url.clone(), 128 | status, 129 | ErrorKind::BadStatus, 130 | "".to_string(), 131 | content, 132 | )) 133 | } 134 | } 135 | Err(err) => Err(Error::new( 136 | method.clone(), 137 | url.clone(), 138 | status, 139 | ErrorKind::BadContent, 140 | err.to_string(), 141 | "".to_string(), 142 | )), 143 | } 144 | } 145 | } 146 | 147 | fn trim(text: String, len: usize) -> String { 148 | let text_len = text.len(); 149 | match text_len > len + 3 { 150 | true => { 151 | let mut msg = text; 152 | msg.truncate(len); 153 | msg.push_str(&format!("...({} more characters)", text_len - len)); 154 | msg 155 | } 156 | false => text, 157 | } 158 | } 159 | 160 | impl Error { 161 | pub fn new( 162 | method: Method, 163 | url: Url, 164 | status: StatusCode, 165 | error_kind: ErrorKind, 166 | err: String, 167 | content: String, 168 | ) -> Self { 169 | Error { 170 | method, 171 | url, 172 | status: status.as_str().to_string(), 173 | err, 174 | error_kind, 175 | content: trim(content, 1000), 176 | } 177 | } 178 | 179 | pub fn kind(&self) -> ErrorKind { 180 | self.error_kind 181 | } 182 | } 183 | 184 | impl fmt::Display for Error { 185 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 186 | write!( 187 | f, 188 | "fluid::http::Error {} {} - response status {}\n{:?} {}\n{}", 189 | self.method, self.url, self.status, self.error_kind, self.err, self.content 190 | ) 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /common/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! the common crate defines all the common structs, functions and enums 2 | //! 3 | //! It provides tooling such as logging, environment variable config and so on 4 | mod backoff; 5 | mod http; 6 | mod logging; 7 | mod orders; 8 | mod worker; 9 | mod ws; 10 | 11 | pub use self::http::*; 12 | pub use self::logging::*; 13 | pub use self::orders::*; 14 | pub use self::worker::*; 15 | pub use self::ws::*; 16 | -------------------------------------------------------------------------------- /common/src/logging.rs: -------------------------------------------------------------------------------- 1 | use config::Config; 2 | extern crate anyhow; 3 | use slog::{o, Drain, Logger}; 4 | 5 | /// Initialise slog 6 | /// 7 | /// If the `json_logs` environment variable is truthy it will enable a json logger 8 | pub fn init_logging(config: &Config) -> anyhow::Result { 9 | let use_json_logs = config.get_bool("json_logs").unwrap_or(false); 10 | if use_json_logs { 11 | let decorator = slog_json::Json::default(std::io::stdout()).fuse(); 12 | let drain = slog_async::Async::new(decorator).build().fuse(); 13 | Ok(slog::Logger::root(drain, o!())) 14 | } else { 15 | let decorator = slog_term::TermDecorator::new().build(); 16 | let drain = slog_term::FullFormat::new(decorator).build().fuse(); 17 | let drain = slog_async::Async::new(drain).chan_size(1024).build().fuse(); 18 | Ok(slog::Logger::root(drain, o!())) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /common/src/orders.rs: -------------------------------------------------------------------------------- 1 | //! Level 2 Orderbook implementation 2 | //! 3 | //! A level2 order book is a datastructure which maintains ordered price-volume pairs 4 | use rust_decimal::prelude::*; 5 | use std::collections::BTreeMap; 6 | use std::error::Error; 7 | use std::{cmp, fmt}; 8 | 9 | type L2Map = BTreeMap; 10 | 11 | /// Orderbook side 12 | pub enum Side { 13 | Bid, 14 | Ask, 15 | } 16 | 17 | /// One side of a Level 2 Order book 18 | #[derive(Clone, Debug)] 19 | pub struct L2 { 20 | /// ordered mapping of prices to volumes 21 | orders: L2Map, 22 | desc: bool, 23 | } 24 | 25 | #[derive(Debug)] 26 | pub struct InconsistentBook { 27 | pub details: String, 28 | pub asset: String, 29 | } 30 | 31 | /// A `Book` represents a level 2 order book data structure 32 | #[derive(Debug, Clone)] 33 | pub struct Book { 34 | /// asset name 35 | pub asset: String, 36 | /// level 2 bid prices & sizes 37 | pub bids: L2, 38 | /// level 2 ask prices & sizes 39 | pub asks: L2, 40 | } 41 | 42 | /// Calculate ask-bid spread 43 | pub fn bid_ask_spread(bid: Option, ask: Option) -> Option { 44 | match bid { 45 | Some(b) => ask.map(|a| a - b), 46 | None => None, 47 | } 48 | } 49 | 50 | pub struct L2Iterator<'a> { 51 | iter: std::collections::btree_map::Iter<'a, Decimal, Decimal>, 52 | desc: bool, 53 | } 54 | 55 | impl L2 { 56 | fn new(desc: bool) -> L2 { 57 | L2 { 58 | orders: BTreeMap::new(), 59 | desc, 60 | } 61 | } 62 | 63 | /// Returns the depth levels 64 | pub fn len(&self) -> usize { 65 | self.orders.len() 66 | } 67 | 68 | /// Returns true if this orderbook side is empty 69 | pub fn is_empty(&self) -> bool { 70 | self.orders.is_empty() 71 | } 72 | 73 | /// Set a new price/volume into the book side 74 | /// 75 | /// # Arguments 76 | /// 77 | /// * `price` - the price level 78 | /// * `volume` - volume fro the price, if 0 the price level will be removed 79 | pub fn set(&mut self, price: Decimal, volume: Decimal) { 80 | match volume.is_zero() { 81 | true => self.orders.remove(&price), 82 | false => self.orders.insert(price, volume), 83 | }; 84 | } 85 | 86 | /// Set a new price/volume into the book side 87 | /// 88 | /// # Arguments 89 | /// 90 | /// * `price` - the price level 91 | /// * `volume` - volume fro the price, if 0 the price level will be removed 92 | pub fn set_str(&mut self, price: &str, volume: &str) { 93 | self.set( 94 | Decimal::from_str(price).unwrap(), 95 | Decimal::from_str(volume).unwrap(), 96 | ) 97 | } 98 | 99 | /// Update the order side with a vector of price/volume tuples 100 | pub fn update(&mut self, price_volume: &[(String, String)]) { 101 | for (price, volume) in price_volume.iter() { 102 | self.set_str(price, volume); 103 | } 104 | } 105 | 106 | /// Returns the (price, volume) tuple at the best price if available 107 | pub fn best(&self) -> Option<(&Decimal, &Decimal)> { 108 | if self.desc { 109 | self.orders.iter().next_back() 110 | // use this once available in stable 111 | //self.orders.last_key_value() 112 | } else { 113 | self.orders.iter().next() 114 | // self.orders.first_key_value() 115 | } 116 | } 117 | 118 | /// Best price in the l2 side 119 | pub fn best_price(&self) -> Option { 120 | self.best().map(|(price, _)| *price) 121 | } 122 | 123 | /// Best of price 124 | /// 125 | /// This function returns the best price between the price provided and 126 | /// the current best price in the l2 side 127 | pub fn best_of(&self, price: Option) -> Option { 128 | match self.best_price() { 129 | Some(best) => match price { 130 | Some(other_price) => match self.desc { 131 | true => Some(cmp::max(best, other_price)), 132 | false => Some(cmp::min(best, other_price)), 133 | }, 134 | None => Some(best), 135 | }, 136 | None => price, 137 | } 138 | } 139 | 140 | /// (price, volume) tuple Iterator 141 | pub fn iter(&self) -> L2Iterator { 142 | L2Iterator { 143 | iter: self.orders.iter(), 144 | desc: self.desc, 145 | } 146 | } 147 | 148 | fn trim(&self, max_depth: usize) -> Self { 149 | let mut orders = L2Map::new(); 150 | for (i, (price, volume)) in self.iter().enumerate() { 151 | if i >= max_depth { 152 | break; 153 | } 154 | orders.insert(*price, *volume); 155 | } 156 | Self { 157 | orders, 158 | desc: self.desc, 159 | } 160 | } 161 | } 162 | 163 | impl<'a> Iterator for L2Iterator<'a> { 164 | type Item = (&'a Decimal, &'a Decimal); 165 | 166 | fn next(&mut self) -> Option { 167 | match self.desc { 168 | false => self.iter.next(), 169 | true => self.iter.next_back(), 170 | } 171 | } 172 | } 173 | 174 | impl InconsistentBook { 175 | pub fn new(msg: &str, asset: &str) -> Self { 176 | InconsistentBook { 177 | details: msg.to_owned(), 178 | asset: asset.to_owned(), 179 | } 180 | } 181 | } 182 | 183 | impl fmt::Display for InconsistentBook { 184 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 185 | write!(f, "{}", self.details) 186 | } 187 | } 188 | 189 | impl Error for InconsistentBook { 190 | fn description(&self) -> &str { 191 | &self.details 192 | } 193 | } 194 | 195 | impl Book { 196 | pub fn new(asset: &str) -> Self { 197 | Self { 198 | asset: asset.to_owned(), 199 | bids: L2::new(true), 200 | asks: L2::new(false), 201 | } 202 | } 203 | 204 | /// Check if the book is consistent 205 | /// 206 | /// A consistent book has the best bid price lower than the best ask price. 207 | /// In other words no crossing allowed 208 | pub fn is_consistent(&self) -> bool { 209 | let bid = match self.bids.best() { 210 | None => return true, 211 | Some((price, _)) => price, 212 | }; 213 | let ask = match self.asks.best() { 214 | None => return true, 215 | Some((price, _)) => price, 216 | }; 217 | bid < ask 218 | } 219 | 220 | pub fn spread(&self) -> Option { 221 | bid_ask_spread(self.bids.best_price(), self.asks.best_price()) 222 | } 223 | 224 | /// Return a new book trimmed a max depth 225 | pub fn trim(&self, max_depth: usize) -> Self { 226 | Self { 227 | asset: self.asset.to_owned(), 228 | asks: self.asks.trim(max_depth), 229 | bids: self.bids.trim(max_depth), 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /common/src/worker.rs: -------------------------------------------------------------------------------- 1 | use crate::logging::init_logging; 2 | use crate::orders::Book; 3 | use anyhow::Result; 4 | use async_channel::{unbounded, Receiver, Sender}; 5 | use config::builder::{ConfigBuilder, DefaultState}; 6 | use config::{Config, ConfigError, Environment}; 7 | use serde::Deserialize; 8 | use serde_json::Value; 9 | use slog::{error, info, Logger}; 10 | 11 | pub type CfgBuilder = ConfigBuilder; 12 | 13 | /// Worker Context 14 | /// 15 | /// A context is the basic configuration for a worker 16 | #[derive(Clone)] 17 | pub struct Context { 18 | /// name of the worker 19 | pub name: String, 20 | /// configuration 21 | pub cfg: Config, 22 | /// logging 23 | pub logger: Logger, 24 | /// Use this to send messages to another worker 25 | pub sender: Sender, 26 | /// Use this to receive messages from another worker 27 | pub receiver: Receiver, 28 | } 29 | 30 | #[derive(Debug, Clone)] 31 | pub struct WsInfo { 32 | pub name: String, 33 | pub url: String, 34 | } 35 | 36 | /// Websocket payload 37 | /// 38 | /// This struct is used by the websocket consumer to communicate back to the main application 39 | /// once a new websocket message arrives 40 | #[derive(Debug, Clone)] 41 | pub struct WsPayload { 42 | /// gateway name 43 | pub name: String, 44 | /// websocket url 45 | pub url: String, 46 | /// message payload 47 | pub value: Value, 48 | } 49 | 50 | /// Gateway command 51 | /// 52 | /// This struct is used by the websocket consumer to communicate back to the main application 53 | /// once a new websocket message arrives 54 | #[derive(Debug, Clone)] 55 | pub struct NewBookSnapshot { 56 | /// gateway name 57 | pub name: String, 58 | } 59 | 60 | /// Orderbook snapshot message 61 | /// 62 | /// An orderbook snapshot is a representation of the order book at a given time. 63 | /// It can be used as the starting point of an in-memory order book. 64 | #[derive(Debug, Clone)] 65 | pub struct BookSnapshot { 66 | /// gateway name 67 | pub name: String, 68 | /// sequence number (use this to discard deltas with sequence prior to this) 69 | pub sequence: usize, 70 | /// the order book snapshot 71 | pub book: Book, 72 | } 73 | 74 | /// Internal message enum 75 | /// 76 | /// This enum is used to send messages between different coroutines 77 | #[derive(Debug, Clone)] 78 | pub enum InnerMessage { 79 | /// heartbeat message 80 | Heartbeat, 81 | /// clean exit 82 | Exit, 83 | /// exit with failure 84 | Failure, 85 | /// websocket message 86 | WsConnected(WsInfo), 87 | /// websocket disconnect 88 | WsDisconnected(WsInfo), 89 | /// websocket payload 90 | WsPayload(WsPayload), 91 | /// request for a new BookSnapshot 92 | NewBookSnapshot(NewBookSnapshot), 93 | /// Orderbook snapshot 94 | BookSnapshot(BookSnapshot), 95 | } 96 | 97 | /// A context for a courotine worker 98 | pub type WorkerContext = Context; 99 | 100 | pub fn create_config() -> CfgBuilder { 101 | Config::builder().add_source(Environment::default()) 102 | } 103 | 104 | impl Context { 105 | pub fn new(name: &str, config: Option) -> Self { 106 | let (sender, receiver) = unbounded(); 107 | let cfg = match config { 108 | Some(cfg) => cfg, 109 | None => create_config().build().expect("config"), 110 | }; 111 | let logger = init_logging(&cfg).unwrap(); 112 | Self { 113 | name: name.to_owned(), 114 | cfg, 115 | logger, 116 | sender, 117 | receiver, 118 | } 119 | } 120 | 121 | /// Get a value from config or a default one 122 | pub fn get_or<'de, C: Deserialize<'de>>( 123 | &self, 124 | key: &str, 125 | default: C, 126 | ) -> Result { 127 | let v = self.cfg.get(key); 128 | 129 | if let Err(ConfigError::NotFound(_)) = v { 130 | Ok(default) 131 | } else { 132 | v 133 | } 134 | } 135 | 136 | pub fn try_send(&self, msg: T) { 137 | self.sender.try_send(msg).unwrap(); 138 | } 139 | 140 | pub async fn send(&self, msg: T) { 141 | self.sender.send(msg).await.unwrap(); 142 | } 143 | } 144 | 145 | pub async fn wrap_result(context: &WorkerContext, result: Result<()>) { 146 | match result { 147 | Ok(()) => { 148 | info!(context.logger, "{} - exited", context.name); 149 | context.send(InnerMessage::Exit).await; 150 | } 151 | Err(err) => { 152 | error!( 153 | context.logger, 154 | "{} - unexpected error - {}", context.name, err 155 | ); 156 | context.send(InnerMessage::Failure).await; 157 | } 158 | }; 159 | } 160 | -------------------------------------------------------------------------------- /common/src/ws.rs: -------------------------------------------------------------------------------- 1 | use crate::backoff::{millis_sleep, Backoff}; 2 | use crate::worker::{Context, InnerMessage, WsInfo, WsPayload}; 3 | use anyhow::{Error, Result}; 4 | use async_channel::{unbounded, Receiver, Sender}; 5 | use futures_util::{SinkExt, StreamExt}; 6 | use serde::Serialize; 7 | use serde_json::{to_value, Value}; 8 | use slog::{debug, error, info, warn}; 9 | use std::time::{Duration, Instant}; 10 | use tokio::io; 11 | use tokio_tungstenite::{connect_async, tungstenite::protocol::Message, WebSocketStream}; 12 | 13 | /// Websocket consumer 14 | /// 15 | /// Listen for new messages from websocket and write new commands into it 16 | #[derive(Clone)] 17 | pub struct WsConsumer { 18 | pub context: Context, 19 | pub sender: Sender, 20 | receiver: Receiver, 21 | heartbeat_sender: Sender<()>, 22 | heartbeat_receiver: Receiver<()>, 23 | heartbeat: Duration, 24 | ws_url: String, 25 | } 26 | 27 | impl WsConsumer { 28 | /// Create a new websocket consumer 29 | pub fn new(context: &Context, ws_url: &str) -> Self { 30 | let (sender, receiver) = unbounded(); 31 | let (heartbeat_sender, heartbeat_receiver) = unbounded(); 32 | let heartbeat_millis: u64 = context 33 | .get_or("websocket_heartbeat", 5000) 34 | .expect("websocket heartbeat"); 35 | WsConsumer { 36 | context: context.clone(), 37 | sender, 38 | receiver, 39 | heartbeat_sender, 40 | heartbeat_receiver, 41 | ws_url: ws_url.to_string(), 42 | heartbeat: Duration::from_millis(heartbeat_millis), 43 | } 44 | } 45 | 46 | pub fn info(&self) -> WsInfo { 47 | WsInfo { 48 | name: self.context.name.clone(), 49 | url: self.ws_url.clone(), 50 | } 51 | } 52 | 53 | /// schedule a write into the websocket 54 | pub fn write(&self, message: T) { 55 | self.sender.try_send(to_value(message).unwrap()).unwrap(); 56 | } 57 | 58 | fn payload(&self, value: Value) -> WsPayload { 59 | WsPayload { 60 | name: self.context.name.clone(), 61 | url: self.ws_url.clone(), 62 | value, 63 | } 64 | } 65 | 66 | pub fn get_url(&self) -> &str { 67 | &self.ws_url 68 | } 69 | 70 | fn create_ws_request(&self) -> &str { 71 | &self.ws_url 72 | } 73 | 74 | // coroutine for consuming and writing messages into a websocket 75 | pub async fn run(&self) -> Result<()> { 76 | let mut backoff = Backoff::new(10, 1, 20, 2); 77 | let logger = self.context.logger.clone(); 78 | 79 | // heartbeat to check health of connection 80 | let heartbeat_sender = self.heartbeat_sender.clone(); 81 | let heartbeat = self.heartbeat; 82 | tokio::spawn(async move { 83 | loop { 84 | tokio::time::sleep(heartbeat).await; 85 | heartbeat_sender.send(()).await.unwrap(); 86 | } 87 | }); 88 | 89 | // main loop with backoff-reconnection 90 | loop { 91 | match backoff.next() { 92 | Some(delay) => { 93 | if delay > 0 { 94 | info!( 95 | logger, 96 | "attempt {} to reconnect to websocket in {} seconds", 97 | backoff.count(), 98 | delay 99 | ); 100 | millis_sleep(1000 * delay as u64).await; 101 | } 102 | } 103 | None => { 104 | return Err(Error::msg(format!( 105 | "failed to reconnect to websocket after {} attempts", 106 | backoff.count() 107 | ))); 108 | } 109 | } 110 | 111 | info!(logger, "connecting with websocket {}", &self.ws_url); 112 | let ws_stream = match connect_async(self.create_ws_request()).await { 113 | Ok((ws_stream, _)) => { 114 | warn!(logger, "connected with websocket {}", &self.ws_url); 115 | backoff.reset(); 116 | ws_stream 117 | } 118 | Err(err) => { 119 | warn!( 120 | logger, 121 | "failed to connect with websocket {}: {}", &self.ws_url, err 122 | ); 123 | continue; 124 | } 125 | }; 126 | 127 | self.stream(ws_stream).await.unwrap_or_else(|err| { 128 | error!(logger, "{}", err); 129 | }); 130 | 131 | self.context 132 | .send(InnerMessage::WsDisconnected(self.info())) 133 | .await; 134 | } 135 | } 136 | 137 | async fn stream(&self, mut ws_stream: WebSocketStream) -> Result<()> 138 | where 139 | S: io::AsyncRead + io::AsyncWrite + Unpin, 140 | { 141 | let logger = &self.context.logger; 142 | let url = &self.ws_url; 143 | let mut last_frame_instant = Instant::now(); 144 | let mut num_messages_since_last_heartbeat = 0; 145 | // send connected message to the main application 146 | self.context 147 | .send(InnerMessage::WsConnected(self.info())) 148 | .await; 149 | loop { 150 | tokio::select! { 151 | // Handle stream of messages from exchange 152 | Some(frame) = ws_stream.next() => { 153 | last_frame_instant = Instant::now(); 154 | num_messages_since_last_heartbeat += 1; 155 | match frame { 156 | Ok(Message::Text(ref text)) => { 157 | let value: Value = match serde_json::from_str(text.as_ref()) { 158 | Ok(value) => value, 159 | Err(err) => { 160 | let mut context = String::from("malformed json message: "); 161 | context.push_str(text); 162 | return Err(Error::new(err).context(context)); 163 | } 164 | }; 165 | // send websocket message to the main task 166 | self.context.send(InnerMessage::WsPayload(self.payload(value))).await; 167 | } 168 | Ok(Message::Close(_)) =>{ 169 | warn!(logger, "received a close frame, stop streaming and reconnect"); 170 | return Ok(()); 171 | } 172 | Ok(Message::Ping(msg)) =>{ 173 | info!(logger, "send a pong message after receiving a server ping"); 174 | ws_stream.send(Message::Pong(msg)).await?; 175 | } 176 | Ok(Message::Pong(_)) =>{ 177 | info!(logger, "got a pong message from server"); 178 | } 179 | Ok(Message::Binary(_)) =>{ 180 | warn!(logger, "got a binary message from server - skip it"); 181 | } 182 | Ok(Message::Frame(_)) =>{ 183 | warn!(logger, "got a raw frame message from server - skip it"); 184 | } 185 | Err(err) => { 186 | return Err(Error::msg(format!("error while streaming websocket: {}", err))); 187 | } 188 | }; 189 | } 190 | // write new messages 191 | Ok(message) = self.receiver.recv() => { 192 | debug!(logger, "write new message into websocket: {}", message); 193 | ws_stream.send(Message::Text(message.to_string())).await?; 194 | } 195 | // heartbeat 196 | Ok(_) = self.heartbeat_receiver.recv() => { 197 | if Instant::now() - last_frame_instant > self.heartbeat && num_messages_since_last_heartbeat == 0 { 198 | warn!(logger, "{} no messages received since last heartbeat, exit receiving loop and reconnect", self.context.name); 199 | return Ok(()); 200 | } else { 201 | debug!(logger, "{} received {} messages since last heartbeat", url, num_messages_since_last_heartbeat); 202 | num_messages_since_last_heartbeat = 0; 203 | } 204 | } 205 | } 206 | } 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /common/tests/test_context.rs: -------------------------------------------------------------------------------- 1 | use common::Context; 2 | 3 | type SimpleContext = Context<()>; 4 | 5 | #[test] 6 | fn simple_context() { 7 | let context = SimpleContext::new("test", None); 8 | assert_eq!(context.name, "test"); 9 | } 10 | -------------------------------------------------------------------------------- /common/tests/test_orders.rs: -------------------------------------------------------------------------------- 1 | use common::Book; 2 | use rust_decimal::prelude::*; 3 | 4 | #[test] 5 | fn empty_book() { 6 | let book = Book::new("ethbtc"); 7 | assert_eq!(book.asset, "ethbtc"); 8 | assert_eq!(book.bids.len(), 0); 9 | assert_eq!(book.asks.len(), 0); 10 | assert_eq!(book.is_consistent(), true); 11 | } 12 | 13 | #[test] 14 | fn insert_asks() { 15 | let mut book = Book::new("ethbtc"); 16 | book.asks.set_str("50", "65.5"); 17 | book.asks.set_str("45.0", "35"); 18 | assert_eq!(book.asks.len(), 2); 19 | let (price, volume) = book.asks.best().unwrap(); 20 | assert_eq!(price.to_string(), "45.0"); 21 | assert_eq!(volume.to_string(), "35"); 22 | book.asks.set_str("45.0", "0.0"); 23 | assert_eq!(book.asks.len(), 1); 24 | book.asks.set_str("45.0", "0.0"); 25 | assert_eq!(book.asks.len(), 1); 26 | let (price, volume) = book.asks.best().unwrap(); 27 | assert_eq!(price.to_string(), "50"); 28 | assert_eq!(volume.to_string(), "65.5"); 29 | assert_eq!( 30 | book.asks.best_price().unwrap(), 31 | Decimal::from_str("50.0").unwrap() 32 | ); 33 | } 34 | 35 | #[test] 36 | fn insert_bids() { 37 | let mut book = Book::new("adabtc"); 38 | book.bids.set_str("45.0", "65.0"); 39 | book.bids.set_str("50", "35.0"); 40 | assert_eq!(book.bids.len(), 2); 41 | let (price, volume) = book.bids.best().unwrap(); 42 | assert_eq!(price.to_string(), "50"); 43 | assert_eq!(volume.to_string(), "35.0"); 44 | book.bids.set_str("45.0", "0.0"); 45 | assert_eq!(book.bids.len(), 1); 46 | book.bids.set_str("45.0", "0.0"); 47 | assert_eq!(book.bids.len(), 1); 48 | let (price, volume) = book.bids.best().unwrap(); 49 | assert_eq!(price.to_string(), "50"); 50 | assert_eq!(volume.to_string(), "35.0"); 51 | } 52 | 53 | #[test] 54 | fn inconsistent_book() { 55 | let mut book = Book::new("adabtc"); 56 | book.bids.set_str("51.2", "100.0"); 57 | book.asks.set_str("49.1", "25.0"); 58 | assert_eq!(book.is_consistent(), false); 59 | assert_eq!(book.spread(), Some(Decimal::from_str("-2.1").unwrap())); 60 | } 61 | 62 | #[test] 63 | fn insert_best_of() { 64 | let mut book = Book::new("ethbtc"); 65 | book.asks.set_str("50", "65.5"); 66 | book.asks.set_str("45.0", "35"); 67 | assert_eq!( 68 | book.asks.best_price().unwrap(), 69 | Decimal::from_str("45").unwrap() 70 | ); 71 | assert_eq!( 72 | book.asks 73 | .best_of(Some(Decimal::from_str("48").unwrap())) 74 | .unwrap(), 75 | Decimal::from_str("45").unwrap() 76 | ); 77 | assert_eq!( 78 | book.asks 79 | .best_of(Some(Decimal::from_str("43.2").unwrap())) 80 | .unwrap(), 81 | Decimal::from_str("43.2").unwrap() 82 | ); 83 | } 84 | -------------------------------------------------------------------------------- /devops/common: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | export DOCKER_URL=ghcr.io 3 | -------------------------------------------------------------------------------- /devops/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | gateway: 6 | image: kong 7 | restart: always 8 | ports: 9 | - 90:8000 10 | - 91:8001 11 | volumes: 12 | - ${PWD}/devops:/home/kong 13 | depends_on: 14 | - kollector 15 | links: 16 | - kollector:kollector 17 | environment: 18 | KONG_DATABASE: "off" 19 | KONG_DECLARATIVE_CONFIG: "/home/kong/kong.yml" 20 | KONG_ADMIN_LISTEN: 0.0.0.0:8001 21 | env_file: 22 | - ../.env 23 | 24 | kollector: 25 | image: kollector 26 | build: 27 | context: ../ 28 | dockerfile: ./devops/kollector.dockerfile 29 | command: 30 | - kollector 31 | - "-p" 32 | - btcusdt,ethbtc,ltcbtc,xrpbtc 33 | ports: 34 | - 8050:8050 35 | environment: 36 | APP_GRPC_HOST: "0.0.0.0" 37 | 38 | kollector-web: 39 | image: kollector-web 40 | build: 41 | context: ../ 42 | dockerfile: ./devops/web.dockerfile 43 | ports: 44 | - 4000:3000 45 | -------------------------------------------------------------------------------- /devops/docker-login: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./devops/common 3 | echo "Logging in to Github ${DOCKER_URL}" 4 | docker login ${DOCKER_URL} -u qmbot -p ${GITHUB_TOKEN} 5 | echo "-----------------------------------------" 6 | 7 | echo "Logging in to Dockerhub" 8 | docker login -u lsbardel -p ${DOCKER_HUB_TOKEN} 9 | echo "-----------------------------------------" 10 | -------------------------------------------------------------------------------- /devops/kollector.dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.60 AS builder 2 | 3 | ARG BUILD_FLAG= 4 | 5 | RUN apt update 6 | RUN apt install -y protobuf-compiler 7 | RUN protoc --version 8 | 9 | WORKDIR /kollector 10 | 11 | COPY . . 12 | 13 | RUN cargo build $BUILD_FLAG 14 | RUN cargo test $BUILD_FLAG 15 | 16 | FROM debian:bullseye-slim 17 | 18 | RUN apt update 19 | RUN apt install -y openssl ca-certificates 20 | 21 | ARG BIN=/usr/local/bin 22 | ARG RELEASE=debug 23 | 24 | COPY --from=builder /kollector/target/$RELEASE/service ${BIN}/kollector 25 | RUN kollector --help 26 | -------------------------------------------------------------------------------- /devops/kong.yml: -------------------------------------------------------------------------------- 1 | _format_version: "2.1" 2 | _transform: true 3 | 4 | services: 5 | - name: kollector-status 6 | url: http://kollector:8050 7 | routes: 8 | - name: kollector-http 9 | paths: 10 | - /rest 11 | 12 | - name: kollector 13 | url: http://kollector:50060 14 | routes: 15 | - name: kollector-grpc 16 | strip_path: false 17 | paths: 18 | - / 19 | plugins: 20 | - name: grpc-web 21 | - name: cors 22 | -------------------------------------------------------------------------------- /devops/lint-py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | isort . $1 4 | black . $1 5 | flake8 6 | mypy pyservice 7 | -------------------------------------------------------------------------------- /devops/pyservice.dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | ENV PYTHONPATH $PYTHONPATH:/app 4 | 5 | WORKDIR /app 6 | # 7 | # INSTALL python dependencies & test 8 | COPY ./pyproject.toml ./poetry.lock ./ 9 | RUN pip install -U poetry 10 | RUN poetry config virtualenvs.create false 11 | RUN poetry install --no-interaction --no-ansi 12 | RUN rm pyproject.toml poetry.lock 13 | RUN rm -rf /root/.cache 14 | 15 | COPY . . 16 | 17 | RUN pytest 18 | RUN ./devops/lint-py 19 | -------------------------------------------------------------------------------- /devops/web.dockerfile: -------------------------------------------------------------------------------- 1 | FROM node as builder 2 | 3 | WORKDIR /web 4 | COPY package.json yarn.lock ./ 5 | RUN yarn 6 | COPY webpack.config.js tsconfig.json web ./ 7 | COPY web ./web 8 | COPY @types ./@types 9 | RUN yarn build 10 | 11 | FROM node 12 | 13 | WORKDIR /web 14 | RUN npm install -g http-server 15 | COPY web/index.html ./sources/index.html 16 | COPY --from=builder /web/dist sources/dist 17 | RUN ls -la 18 | 19 | CMD ["http-server", "/web/sources", "-p", "3000", "-d"] 20 | -------------------------------------------------------------------------------- /gateways/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gateways" 3 | version = "0.1.0" 4 | edition = "2021" 5 | homepage = "https://github.com/quantmind/kollector" 6 | repository = "https://github.com/quantmind/kollector" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | anyhow = "1.0.57" 12 | async-channel = "1.6.1" 13 | common = { path = "../common" } 14 | futures-util = "0.3.21" 15 | reqwest = "0.11.10" 16 | serde = { version = "^1.0.137", features = ["derive"] } 17 | serde_json = "1.0.81" 18 | slog = "2.7.0" 19 | tokio = { version = "^1.18.2", features = ["full"] } 20 | -------------------------------------------------------------------------------- /gateways/src/binance/gateway.rs: -------------------------------------------------------------------------------- 1 | use super::models; 2 | use super::requests; 3 | use crate::{Gateway, WsUpdate}; 4 | use common::{ 5 | Book, BookSnapshot, HttpClient, InnerMessage, NewBookSnapshot, WorkerContext, WsConsumer, 6 | }; 7 | use serde_json::{from_value, Value}; 8 | use slog::{error, info, warn}; 9 | use std::collections::HashMap; 10 | use std::time::Duration; 11 | 12 | struct BookWithBuffer { 13 | updates: Vec, 14 | book: Option, 15 | } 16 | 17 | /// Binance Gateway 18 | /// 19 | /// The binance gateway requires to fetch order book snapshots via Rest 20 | /// and maintain the book updates in memory with messages coming from the websocket api. 21 | /// For more information check 22 | /// [binance documentation](https://github.com/binance/binance-spot-api-docs/blob/master/web-socket-streams.md) 23 | pub struct Binance { 24 | context: WorkerContext, 25 | ws: WsConsumer, 26 | max_depth: usize, 27 | msg_id: usize, 28 | books: HashMap, 29 | } 30 | 31 | impl Gateway for Binance { 32 | fn name(&self) -> &str { 33 | &self.context.name 34 | } 35 | 36 | fn ws_consumer(&self) -> WsConsumer { 37 | self.ws.clone() 38 | } 39 | 40 | // request book snapshot at regular intervals 41 | // this is so we have a consistent book 42 | fn setup(&self) { 43 | let context = self.context.clone(); 44 | let heartbeat_millis: u64 = context 45 | .get_or("binance_snapshot_heartbeat", 10000) 46 | .expect("websocket binance_snapshot_heartbeat"); 47 | let mut delay = 2; 48 | tokio::spawn(async move { 49 | loop { 50 | tokio::time::sleep(Duration::from_millis(delay)).await; 51 | context 52 | .send(InnerMessage::NewBookSnapshot(NewBookSnapshot { 53 | name: context.name.to_owned(), 54 | })) 55 | .await; 56 | delay = heartbeat_millis; 57 | } 58 | }); 59 | } 60 | 61 | fn subscribe(&mut self, symbols: &[String]) { 62 | self.msg_id += 1; 63 | self.ws.write(models::WsMessage::subscribe( 64 | self.msg_id, 65 | "depth@100ms", 66 | symbols, 67 | )); 68 | } 69 | 70 | fn unsubscribe(&mut self, symbols: &[String]) { 71 | self.msg_id += 1; 72 | self.ws.write(models::WsMessage::unsubscribe( 73 | self.msg_id, 74 | "depth@100ms", 75 | symbols, 76 | )); 77 | } 78 | 79 | fn request_snapshot(&mut self) { 80 | let mut assets = vec![]; 81 | for (asset, bf) in self.books.iter_mut() { 82 | assets.push(asset.to_owned()); 83 | bf.reset(); 84 | } 85 | fetch_snapshots(assets, self.http(), self.context.clone()); 86 | } 87 | 88 | fn on_websocket_message(&mut self, value: Value) -> Option { 89 | let result: Result = from_value(value.clone()); 90 | match result { 91 | Ok(o) => { 92 | if let Some(data) = o.data { 93 | match data { 94 | models::WsData::BookUpdate(book) => { 95 | return self.book_snapshot(book); 96 | } 97 | } 98 | }; 99 | } 100 | Err(err) => { 101 | warn!(self.context.logger, "{}. {}", err, value); 102 | } 103 | } 104 | None 105 | } 106 | 107 | fn on_book_snapshot(&mut self, snapshot: BookSnapshot) -> Option { 108 | match self.books.get_mut(&snapshot.book.asset) { 109 | Some(bf) => { 110 | info!( 111 | self.context.logger, 112 | "{} received an orderbook snapshot {}", snapshot.name, snapshot.book.asset 113 | ); 114 | return bf.on_book_snapshot(&snapshot, self.max_depth); 115 | } 116 | None => { 117 | warn!( 118 | self.context.logger, 119 | "received an unknown orderbook snapshot {:?}", snapshot 120 | ); 121 | } 122 | } 123 | None 124 | } 125 | } 126 | 127 | impl Binance { 128 | /// Create a new Binance gateway 129 | pub fn new(context: &WorkerContext, max_depth: usize, _pairs: &[String]) -> Self { 130 | let mut context = context.clone(); 131 | context.name = "binance".to_owned(); 132 | let ws_url: &str = context 133 | .get_or( 134 | "binance_spot_ws_url", 135 | "wss://stream.binance.com:9443/stream", 136 | ) 137 | .expect("Binance websocket url"); 138 | let ws = WsConsumer::new(&context, ws_url); 139 | Self { 140 | context, 141 | ws, 142 | max_depth, 143 | msg_id: 0, 144 | books: HashMap::new(), 145 | } 146 | } 147 | 148 | // Http client 149 | fn http(&self) -> HttpClient { 150 | let api_url: &str = self 151 | .context 152 | .get_or("binance_spot_url", "https://api.binance.com") 153 | .expect("Binance api url"); 154 | HttpClient::new(api_url) 155 | } 156 | 157 | fn book_snapshot(&mut self, book_update: models::BookUpdate) -> Option { 158 | let asset = book_update.s.to_lowercase(); 159 | let bf = self.books.entry(asset).or_insert_with(BookWithBuffer::new); 160 | match &mut bf.book { 161 | Some(book) => { 162 | book.asks.update(&book_update.a); 163 | book.bids.update(&book_update.b); 164 | Some(WsUpdate::Book(book.trim(self.max_depth))) 165 | } 166 | None => { 167 | // push the update in the buffer 168 | bf.updates.push(book_update); 169 | None 170 | } 171 | } 172 | } 173 | } 174 | 175 | fn fetch_snapshots(assets: Vec, http: HttpClient, context: WorkerContext) { 176 | tokio::spawn(async move { 177 | for asset in assets.iter() { 178 | let request = requests::GetDepth::new(asset, 1000); 179 | info!( 180 | context.logger, 181 | "{} fetching orderbook {} snapshot via rest", context.name, asset 182 | ); 183 | let result = http.request(request, Some(&context.logger)).await; 184 | match result { 185 | Ok(b) => { 186 | let mut book = Book::new(asset); 187 | book.asks.update(&b.asks); 188 | book.bids.update(&b.bids); 189 | context 190 | .send(InnerMessage::BookSnapshot(BookSnapshot { 191 | name: context.name.to_owned(), 192 | sequence: b.last_update_id, 193 | book, 194 | })) 195 | .await; 196 | } 197 | Err(err) => { 198 | error!( 199 | context.logger, 200 | "{} - unexpected error - {}", context.name, err 201 | ); 202 | context.send(InnerMessage::Failure).await; 203 | } 204 | } 205 | } 206 | }); 207 | } 208 | 209 | impl BookWithBuffer { 210 | fn new() -> Self { 211 | Self { 212 | updates: vec![], 213 | book: None, 214 | } 215 | } 216 | 217 | fn reset(&mut self) { 218 | self.book = None; 219 | } 220 | 221 | fn on_book_snapshot(&mut self, snapshot: &BookSnapshot, max_depth: usize) -> Option { 222 | let mut book = snapshot.book.clone(); 223 | for update in self.updates.iter() { 224 | if update.u > snapshot.sequence { 225 | book.asks.update(&update.a); 226 | book.bids.update(&update.b); 227 | } 228 | } 229 | self.updates = vec![]; 230 | let ob = book.trim(max_depth); 231 | self.book = Some(book); 232 | Some(ob) 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /gateways/src/binance/mod.rs: -------------------------------------------------------------------------------- 1 | mod gateway; 2 | mod models; 3 | mod requests; 4 | 5 | pub use self::gateway::Binance; 6 | -------------------------------------------------------------------------------- /gateways/src/binance/models.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Debug, Clone, Serialize)] 4 | pub struct WsMessage { 5 | pub id: usize, 6 | pub method: String, 7 | pub params: Vec, 8 | } 9 | 10 | #[derive(Debug, Clone, Deserialize)] 11 | #[serde(rename_all = "camelCase")] 12 | pub struct BookSnapshot { 13 | pub last_update_id: usize, 14 | pub asks: Vec<(String, String)>, 15 | pub bids: Vec<(String, String)>, 16 | } 17 | 18 | #[derive(Debug, Clone, Deserialize)] 19 | pub struct BookUpdate { 20 | pub s: String, 21 | pub u: usize, 22 | pub a: Vec<(String, String)>, 23 | pub b: Vec<(String, String)>, 24 | #[serde(rename = "E")] 25 | pub time: usize, 26 | } 27 | 28 | #[derive(Debug, Clone, Deserialize)] 29 | #[serde(tag = "e", rename_all = "snake_case")] 30 | pub enum WsData { 31 | #[serde(rename = "depthUpdate")] 32 | BookUpdate(BookUpdate), 33 | } 34 | 35 | #[derive(Debug, Clone, Deserialize)] 36 | pub struct WsResponse { 37 | pub data: Option, 38 | } 39 | 40 | impl WsMessage { 41 | pub fn subscribe(id: usize, channel: &str, symbols: &[String]) -> Self { 42 | Self::op("SUBSCRIBE", id, channel, symbols) 43 | } 44 | 45 | pub fn unsubscribe(id: usize, channel: &str, symbols: &[String]) -> Self { 46 | Self::op("UNSUBSCRIBE", id, channel, symbols) 47 | } 48 | 49 | fn op(method: &str, id: usize, channel: &str, symbols: &[String]) -> Self { 50 | Self { 51 | id, 52 | method: method.to_owned(), 53 | params: symbols 54 | .iter() 55 | .map(|symbol| format!("{}@{}", symbol, channel)) 56 | .collect(), 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /gateways/src/binance/requests.rs: -------------------------------------------------------------------------------- 1 | use super::models; 2 | use common::Request; 3 | use reqwest::Method; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, Deserialize, Serialize)] 7 | pub struct GetDepth { 8 | symbol: String, 9 | limit: usize, 10 | } 11 | 12 | impl Request for GetDepth { 13 | const METHOD: Method = Method::GET; 14 | const PATH: &'static str = "/api/v3/depth"; 15 | const HAS_PAYLOAD: bool = true; 16 | type Response = models::BookSnapshot; 17 | } 18 | 19 | impl GetDepth { 20 | pub fn new(symbol: &str, limit: usize) -> Self { 21 | Self { 22 | symbol: symbol.to_uppercase(), 23 | limit, 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /gateways/src/bitstamp/gateway.rs: -------------------------------------------------------------------------------- 1 | use super::models; 2 | use crate::{Gateway, WsUpdate}; 3 | use common::{Book, Context, InnerMessage, WsConsumer}; 4 | use serde_json::{from_value, Value}; 5 | use slog::{info, warn}; 6 | 7 | /// Bitstamp Gateway 8 | /// 9 | /// This gatewat stream order book snapshots from 10 | /// [bitstamp websocket api](https://www.bitstamp.net/websocket/v2/) 11 | pub struct Bitstamp { 12 | context: Context, 13 | ws: WsConsumer, 14 | max_depth: usize, 15 | } 16 | 17 | impl Gateway for Bitstamp { 18 | fn name(&self) -> &str { 19 | &self.context.name 20 | } 21 | 22 | fn ws_consumer(&self) -> WsConsumer { 23 | self.ws.clone() 24 | } 25 | 26 | fn subscribe(&mut self, symbols: &[String]) { 27 | for symbol in symbols.iter() { 28 | self.ws 29 | .write(models::Command::subscribe("order_book", symbol)); 30 | } 31 | } 32 | 33 | fn unsubscribe(&mut self, symbols: &[String]) { 34 | for symbol in symbols.iter() { 35 | self.ws 36 | .write(models::Command::unsubscribe("order_book", symbol)); 37 | } 38 | } 39 | 40 | fn on_websocket_message(&mut self, value: Value) -> Option { 41 | let result: Result = from_value(value.clone()); 42 | match result { 43 | Ok(o) => match o { 44 | models::WsResponse::Subscriptions(sub) => { 45 | info!(self.context.logger, "{:?}", sub); 46 | } 47 | models::WsResponse::Book(ref book) => { 48 | return self.book_snapshot(book); 49 | } 50 | }, 51 | Err(err) => { 52 | warn!(self.context.logger, "{}. {}", err, value); 53 | } 54 | }; 55 | None 56 | } 57 | } 58 | 59 | impl Bitstamp { 60 | /// Create a new Bitstamp gateway 61 | pub fn new(context: &Context, max_depth: usize, _pairs: &[String]) -> Self { 62 | let mut context = context.clone(); 63 | context.name = "bitstamp".to_owned(); 64 | let ws_url: &str = context 65 | .get_or("bitstamp_ws_url", "wss://ws.bitstamp.net") 66 | .expect("bitstamp_ws_url"); 67 | let ws = WsConsumer::new(&context, ws_url); 68 | Self { 69 | context, 70 | ws, 71 | max_depth, 72 | } 73 | } 74 | 75 | fn book_snapshot(&self, book: &models::Book) -> Option { 76 | let mut ob = Book::new(&book.channel.split('_').last().unwrap().to_lowercase()); 77 | ob.asks.update(&book.data.asks); 78 | ob.bids.update(&book.data.bids); 79 | Some(WsUpdate::Book(ob.trim(self.max_depth))) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /gateways/src/bitstamp/mod.rs: -------------------------------------------------------------------------------- 1 | mod gateway; 2 | mod models; 3 | 4 | pub use self::gateway::Bitstamp; 5 | -------------------------------------------------------------------------------- /gateways/src/bitstamp/models.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Debug, Clone, Serialize, Deserialize)] 4 | pub struct Channel { 5 | channel: String, 6 | } 7 | 8 | #[derive(Debug, Clone, Serialize)] 9 | pub struct Command { 10 | pub event: String, 11 | data: Channel, 12 | } 13 | 14 | #[derive(Debug, Clone, Deserialize)] 15 | pub struct BookData { 16 | pub asks: Vec<(String, String)>, 17 | pub bids: Vec<(String, String)>, 18 | pub microtimestamp: String, 19 | } 20 | 21 | #[derive(Debug, Clone, Deserialize)] 22 | pub struct Book { 23 | pub channel: String, 24 | pub data: BookData, 25 | } 26 | 27 | #[derive(Debug, Clone, Deserialize)] 28 | #[serde(tag = "event")] 29 | pub enum WsResponse { 30 | #[serde(rename = "bts:subscription_succeeded")] 31 | Subscriptions(Channel), 32 | #[serde(rename = "data")] 33 | Book(Book), 34 | } 35 | 36 | impl Command { 37 | pub fn subscribe(channel: &str, pair: &str) -> Self { 38 | Self::op("bts:subscribe", channel, pair) 39 | } 40 | 41 | pub fn unsubscribe(channel: &str, pair: &str) -> Self { 42 | Self::op("bts:unsubscribe", channel, pair) 43 | } 44 | 45 | fn op(event: &str, channel: &str, pair: &str) -> Self { 46 | Self { 47 | event: event.to_owned(), 48 | data: Channel { 49 | channel: format!("{}_{}", channel, pair), 50 | }, 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /gateways/src/gateway.rs: -------------------------------------------------------------------------------- 1 | use common::{Book, BookSnapshot, WsConsumer}; 2 | use serde_json::Value; 3 | 4 | /// A websocket update 5 | /// 6 | /// This can be extended to handle other type of updates 7 | pub enum WsUpdate { 8 | Book(Book), 9 | } 10 | 11 | /// A Gateway trait 12 | /// 13 | /// A gateway implements the middleware for connecting to remote crypto exchanges 14 | /// and parse orderbook (or other) messages in a streaming fashion. 15 | pub trait Gateway { 16 | fn setup(&self) {} 17 | 18 | /// Request book snapshots 19 | fn request_snapshot(&mut self) {} 20 | 21 | /// gateway name 22 | fn name(&self) -> &str; 23 | 24 | /// subscribe to a set of assets 25 | /// 26 | /// assets are lowercase, the gateway implementation is responsible 27 | /// for mapping names to the relevant exchange asset names 28 | fn subscribe(&mut self, symbols: &[String]); 29 | 30 | /// unsubscribe from a set of assets 31 | fn unsubscribe(&mut self, symbols: &[String]); 32 | 33 | /// handle a websocket message from exchange 34 | fn on_websocket_message(&mut self, value: Value) -> Option; 35 | 36 | /// handle a book snapshot 37 | fn on_book_snapshot(&mut self, _snapshot: BookSnapshot) -> Option { 38 | None 39 | } 40 | 41 | /// return the websocket consumer for this gateway 42 | fn ws_consumer(&self) -> WsConsumer; 43 | } 44 | -------------------------------------------------------------------------------- /gateways/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Connect and map remote exchanges messages 2 | mod binance; 3 | mod bitstamp; 4 | mod gateway; 5 | 6 | pub use self::binance::Binance; 7 | pub use self::bitstamp::Bitstamp; 8 | pub use self::gateway::*; 9 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from pyservice.app import start_app 2 | 3 | if __name__ == "__main__": 4 | start_app() 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kollector", 3 | "version": "0.1.0", 4 | "description": "Simple front-end for grpc order books", 5 | "main": "web/index.ts", 6 | "repository": "git@github.com:quantmind/kollector.git", 7 | "author": "luca@quantmind.com", 8 | "license": "MIT", 9 | "scripts": { 10 | "lint": "eslint 'app/**/*'", 11 | "fix": "eslint 'app/**/*' --fix", 12 | "precommit": "lint-staged", 13 | "build": "export NODE_ENV=production && webpack", 14 | "watch": "export NODE_ENV=development && webpack serve --progress", 15 | "dev": "export NODE_ENV=production && webpack serve", 16 | "postinstall": "husky install" 17 | }, 18 | "lint-staged": { 19 | "*.{ts,tsx,js,jsx}": [ 20 | "yarn fix" 21 | ], 22 | "*.{json,css,scss}": [ 23 | "prettier --write" 24 | ] 25 | }, 26 | "husky": { 27 | "hooks": { 28 | "pre-commit": "lint-staged", 29 | "post-commit": "git update-index --again" 30 | } 31 | }, 32 | "devDependencies": { 33 | "@types/d3": "^7.1.0", 34 | "@types/lodash.debounce": "^4.0.7", 35 | "@types/react-dom": "^18.0.4", 36 | "@types/react-router-dom": "^5.3.3", 37 | "dotenv": "^16.0.1", 38 | "husky": "^8.0.1", 39 | "lint-staged": "^12.4.1", 40 | "source-map-loader": "^3.0.1", 41 | "ts-loader": "^9.3.0", 42 | "typescript": "^4.6.4", 43 | "webpack": "^5.72.1", 44 | "webpack-cli": "^4.9.2", 45 | "webpack-dev-server": "^4.9.0", 46 | "webpack-require-from": "^1.8.6" 47 | }, 48 | "dependencies": { 49 | "@emotion/react": "^11.9.0", 50 | "@emotion/styled": "^11.8.1", 51 | "@mui/material": "^5.8.0", 52 | "@observablehq/plot": "^0.4.3", 53 | "google-protobuf": "^3.20.1", 54 | "grpc-web": "^1.3.1", 55 | "lodash.debounce": "^4.0.8", 56 | "react": "^18.1.0", 57 | "react-dom": "^18.1.0", 58 | "react-router-dom": "^6.3.0" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "kollector" 3 | version = "0.1.0" 4 | description = "Collect orderbook data from crypto exchanges and publish as GRPC" 5 | authors = ["Luca "] 6 | license = "MIT" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.9" 10 | aiohttp = "^3.8.1" 11 | click = "^8.1.3" 12 | sortedcontainers = "^2.4.0" 13 | rich = "^12.4.4" 14 | numpy = "^1.22.4" 15 | 16 | [tool.poetry.dev-dependencies] 17 | pytest = "^7.1.2" 18 | flake8 = "black" 19 | black = "^22.3.0" 20 | isort = "^5.10.1" 21 | mypy = "^0.961" 22 | pytest-asyncio = "^0.18.3" 23 | aioresponses = "^0.7.3" 24 | 25 | [build-system] 26 | requires = ["poetry-core>=1.0.0"] 27 | build-backend = "poetry.core.masonry.api" 28 | 29 | 30 | [tool.black] 31 | line-length = 88 32 | exclude = ''' 33 | 34 | ( 35 | /( 36 | \.eggs 37 | | \.git 38 | | \.github 39 | | \.husky 40 | | \.vscode 41 | | \.mypy_cache 42 | | \.pytest_cache 43 | | \.tox 44 | | \.venv 45 | | \.gitsubmodules 46 | | common 47 | | devops 48 | | dist 49 | | gateways 50 | | node_modules 51 | | service 52 | | target 53 | | web 54 | )/ 55 | ) 56 | ''' 57 | -------------------------------------------------------------------------------- /pyservice/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantmind/kollector/ca09445cfc9f69299c7b52919a25c9459fd42997/pyservice/__init__.py -------------------------------------------------------------------------------- /pyservice/app.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import click 4 | from aiohttp import web 5 | 6 | from . import config 7 | from .binance import Binance 8 | from .console import ConsoleUI 9 | 10 | routes = web.RouteTableDef() 11 | 12 | 13 | @routes.get("/status") 14 | async def status(request: web.Request) -> web.Response: 15 | """status probe""" 16 | status = {gateway.name: gateway.status() for gateway in request.app["gateways"]} 17 | return web.json_response(status) 18 | 19 | 20 | def create_app( 21 | console: bool = False, pairs: str = "", close_ws_every: int = 0 22 | ) -> web.Application: 23 | """Create aiohttp web application and register gateways""" 24 | app = web.Application() 25 | publisher = ConsoleUI.setup(app) if console else None 26 | pairs_ = [ 27 | pair.strip().lower() for pair in (pairs or config.CURRENCY_PAIRS).split(",") 28 | ] 29 | app["gateways"] = [ 30 | Binance.setup( 31 | app, publisher=publisher, pairs=pairs_, close_ws_every=close_ws_every 32 | ) 33 | ] 34 | app.router.add_routes(routes) 35 | return app 36 | 37 | 38 | @click.command() 39 | @click.option("--console", default=False, is_flag=True, help="add terminal console") 40 | @click.option( 41 | "--pairs", 42 | default=config.CURRENCY_PAIRS, 43 | help="comma separated list of currency pairs to subscribe", 44 | show_default=True, 45 | ) 46 | @click.option( 47 | "--drop", 48 | default=0, 49 | type=int, 50 | help="number of seconds after which the websocket connection is dropped", 51 | show_default=True, 52 | ) 53 | @click.option( 54 | "--port", default=3010, type=int, help="port to listen on", show_default=True 55 | ) 56 | def start_app(console: bool, pairs: str, drop: int, port: int) -> None: 57 | """Start the service""" 58 | if not console: 59 | logging.basicConfig(level=logging.INFO) 60 | app = create_app(console=console, pairs=pairs, close_ws_every=drop) 61 | web.run_app(app, port=port) 62 | -------------------------------------------------------------------------------- /pyservice/binance.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from collections import defaultdict 3 | from typing import Any 4 | 5 | from aiohttp import ClientSession, WSMessage 6 | 7 | from . import config 8 | from .gateway import WebsocketGateway, Worker 9 | 10 | 11 | class Binance(WebsocketGateway): 12 | """Connect to binance websocket and subscribe to depth updates""" 13 | 14 | def __init__(self, **kwargs: Any) -> None: 15 | super().__init__(**kwargs) 16 | self._inflight_snapshot: dict[str, bool] = defaultdict(bool) 17 | self._updates: dict[str, list[dict]] = defaultdict(list) 18 | self._id: int = 0 19 | self.add_worker(Worker(self._refresh_snapshots)) 20 | 21 | def ws_url(self) -> str: 22 | return config.BINANCE_SPOT_WS_URL 23 | 24 | def on_ws_connection(self) -> None: 25 | """New connection, subscribe to depth channels""" 26 | super().on_ws_connection() 27 | self.books.clear() 28 | self._updates.clear() 29 | self._inflight_snapshot.clear() 30 | self._rpc_write( 31 | "SUBSCRIBE", [f"{pair}@depth@100ms".lower() for pair in self.pairs] 32 | ) 33 | 34 | def on_text_message(self, msg: WSMessage) -> None: 35 | """Handle a text message from websocket""" 36 | data = msg.json() 37 | if data.get("e") == "depthUpdate": 38 | self._on_book_update(data["s"].lower(), data) 39 | 40 | # INTERNALS 41 | 42 | def _rpc_write(self, method: str, params: list) -> None: 43 | msg = dict(method=method, id=self._rpc_id(), params=params) 44 | self.logger.info("sending %s", msg) 45 | self.write_json(msg) 46 | 47 | def _rpc_id(self) -> int: 48 | self._id += 1 49 | return self._id 50 | 51 | def _on_book_update(self, pair: str, data: dict) -> None: 52 | """New depth update""" 53 | if pair in self.books: 54 | self._update_book(pair, data) 55 | else: 56 | # book snapshot is missing - request one if not done already 57 | self._request_snapshot(pair) 58 | self._updates[pair].append(data) 59 | 60 | async def _get_snapshot(self, pair: str) -> None: 61 | """Request a snapshot of the book for a pair""" 62 | async with ClientSession() as session: 63 | self.logger.info("fetch snapshot for %s", pair) 64 | response = await session.get( 65 | f"{config.BINANCE_SPOT_URL}/api/v3/depth", 66 | params=dict(symbol=pair.upper()), 67 | ) 68 | snapshot = await response.json() 69 | book = self.new_book() 70 | sequence = snapshot["lastUpdateId"] 71 | book.asks.update(snapshot.get("asks", ())) 72 | book.bids.update(snapshot.get("bids", ())) 73 | self.books[pair] = book 74 | self._inflight_snapshot[pair] = False 75 | for update in self._updates.pop(pair, ()): 76 | if update["u"] > sequence: 77 | # stop updating if book inconsistent 78 | if not self._update_book(pair, update): 79 | return 80 | self.logger.info("snapshot for %s populated - ready for updates", pair) 81 | 82 | def _update_book(self, pair: str, update: dict) -> bool: 83 | book = self.books[pair] 84 | book.asks.update(update.get("a", ())) 85 | book.bids.update(update.get("b", ())) 86 | if not book.is_consistent(): 87 | if not self._inflight_snapshot[pair]: 88 | self.logger.warning("book is inconsistent - request snapshot") 89 | self._request_snapshot(pair) 90 | return False 91 | return True 92 | 93 | def _request_snapshot(self, pair: str) -> None: 94 | """Request a snapshot of the book for a pair""" 95 | if not self._inflight_snapshot[pair]: 96 | self.books.pop(pair, None) 97 | self._inflight_snapshot[pair] = True 98 | self.execute(self._get_snapshot, pair) 99 | 100 | async def _refresh_snapshots(self) -> None: 101 | """Refresh snapshots for all pairs at regular intervals 102 | 103 | This is so we keep the book consistent for deep levels too 104 | """ 105 | while True: 106 | for pair in tuple(self.books): 107 | self._request_snapshot(pair) 108 | await asyncio.sleep(config.BINANCE_REFRESH_SNAPSHOT_INTERVAL) 109 | -------------------------------------------------------------------------------- /pyservice/book.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | from math import exp 3 | from operator import neg 4 | from typing import Any, Collection, Iterable, Iterator, NamedTuple 5 | 6 | from sortedcontainers import SortedDict 7 | 8 | BID = 0 9 | ASK = 1 10 | ZERO = Decimal(0) 11 | HALF = Decimal("0.5") 12 | NAN = Decimal("nan") 13 | NumberOrString = float | Decimal | str 14 | 15 | 16 | def as_decimal(value: NumberOrString) -> Decimal: 17 | if isinstance(value, float): 18 | return Decimal(str(value)) 19 | elif isinstance(value, Decimal): 20 | return value 21 | else: 22 | return Decimal(value) 23 | 24 | 25 | class SideHelper(NamedTuple): 26 | side: int 27 | sign: int 28 | name: str 29 | 30 | 31 | class L2(SortedDict): 32 | @property 33 | def best(self) -> tuple[Decimal, Decimal]: 34 | try: 35 | return self.peekitem(0) 36 | except IndexError: 37 | return (NAN, ZERO) 38 | 39 | @property 40 | def best_price(self) -> Decimal: 41 | return self.best[0] 42 | 43 | def update(self, data: Iterable[tuple[NumberOrString, NumberOrString]]) -> None: 44 | for price, volume in data: 45 | self.set(price, volume) 46 | 47 | def set( 48 | self, price: NumberOrString, volume: NumberOrString 49 | ) -> tuple[Decimal, Decimal]: 50 | price = as_decimal(price) 51 | volume = as_decimal(volume) 52 | if volume == ZERO: 53 | self.pop(price, None) 54 | else: 55 | self[price] = volume 56 | return (price, volume) 57 | 58 | def depth_decay(self, level: int = 3, decay: float = 0) -> float: 59 | return sum(self._first(level, decay=decay)) 60 | 61 | def _first(self, level: int, decay: float = 0) -> Iterator[float]: 62 | for idx, key in enumerate(self): 63 | if idx >= level: 64 | break 65 | d = exp(-decay * idx) 66 | yield d * float(self[key]) 67 | 68 | 69 | class Book(Collection[Decimal]): 70 | """Level 2 order book""" 71 | 72 | def __init__(self, timestamp: int = 0, max_depth: int = 0) -> None: 73 | self.timestamp: int = timestamp 74 | self.sides: dict[int, L2] = {BID: L2(neg), ASK: L2()} 75 | self.max_depth: int = max_depth 76 | 77 | def __repr__(self) -> str: 78 | return f"bids: {repr(self.bids)}, asks: {repr(self.asks)}" 79 | 80 | def __len__(self) -> int: 81 | return len(self.bids) + len(self.asks) 82 | 83 | def __iter__(self) -> Iterator[Decimal]: 84 | yield from self.asks 85 | yield from self.bids 86 | 87 | def __contains__(self, price: Any) -> bool: 88 | return price in self.asks or price in self.bids 89 | 90 | @property 91 | def asks(self) -> L2: 92 | return self.sides[ASK] 93 | 94 | @property 95 | def bids(self) -> L2: 96 | return self.sides[BID] 97 | 98 | def best_bid(self) -> Decimal: 99 | return self.bids.best_price 100 | 101 | def best_ask(self) -> Decimal: 102 | return self.asks.best_price 103 | 104 | def best_volume_bid(self) -> Decimal: 105 | return self.bids.best_volume 106 | 107 | def best_volume_ask(self) -> Decimal: 108 | return self.asks.best_volume 109 | 110 | def spread(self) -> Decimal: 111 | return self.best_ask() - self.best_bid() 112 | 113 | def mid(self, weighted: bool = False) -> Decimal: 114 | bid = self.best_bid() 115 | ask = self.best_ask() 116 | if bid == bid and ask == ask: 117 | if weighted: 118 | volume_bid = self.best_volume_bid() 119 | volume_ask = self.best_volume_ask() 120 | return (volume_bid * bid + volume_ask * ask) / (volume_bid + volume_ask) 121 | return HALF * (bid + ask) 122 | elif bid == bid: 123 | return bid 124 | else: 125 | return ask 126 | 127 | def imbalance(self, depth: int = 3, decay: float = 0.5) -> float: 128 | volume_bid = self.bids.depth_decay(depth, decay) 129 | volume_ask = self.asks.depth_decay(depth, decay) 130 | volume = volume_bid + volume_ask 131 | return 0 if volume == 0 else (volume_bid - volume_ask) / volume 132 | 133 | def is_consistent(self) -> bool: 134 | """Check if the book is consistent""" 135 | bid = self.best_bid() 136 | ask = self.best_ask() 137 | if bid == bid and ask == ask: 138 | return bid < ask 139 | else: 140 | return True 141 | -------------------------------------------------------------------------------- /pyservice/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | STALE_WEBSOCKET_TIMEOUT = int(os.getenv("STALE_WEBSOCKET_TIMEOUT", "10")) 4 | BOOK_PUBLISH_INTERVAL = float(os.getenv("BOOK_PUBLISH_INTERVAL", "0.5")) 5 | CURRENCY_PAIRS = os.getenv("CURRENCY_PAIRS", "btcusdt,ethusdt") 6 | 7 | BINANCE_SPOT_WS_URL = os.getenv( 8 | "BINANCE_SPOT_WS_URL", "wss://stream.binance.com:9443/ws" 9 | ) 10 | BINANCE_SPOT_URL = os.getenv("BINANCE_SPOT_URL", "https://api.binance.com") 11 | BINANCE_REFRESH_SNAPSHOT_INTERVAL = int( 12 | os.getenv("BINANCE_REFRESH_SNAPSHOT_INTERVAL", "10") 13 | ) 14 | -------------------------------------------------------------------------------- /pyservice/console.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from rich.align import Align 4 | from rich.console import Console, Group 5 | from rich.live import Live 6 | from rich.table import Table 7 | 8 | from .book import ZERO, Book 9 | from .gateway import Publisher 10 | from .workers import Worker, Workers 11 | 12 | 13 | class ConsoleUI(Workers, Publisher): 14 | def __init__(self, levels: int = 10) -> None: 15 | super().__init__() 16 | self.levels = levels 17 | self.console = Console() 18 | self._books: asyncio.Queue[list[Align]] = asyncio.Queue() 19 | self.add_worker(Worker(self._live)) 20 | 21 | async def publish_books(self, books: dict[str, Book]) -> None: 22 | """Publish data to console.""" 23 | if books: 24 | tables = [self.create_table(pair, books[pair]) for pair in sorted(books)] 25 | await self._books.put(tables) 26 | 27 | def create_table(self, symbol: str, book: Book) -> Align: 28 | """Create a table for a given orderbook 29 | 30 | Display the top self.levels only 31 | """ 32 | w = 15 33 | table = Table( 34 | title=( 35 | f"[b]{symbol.upper()}[/b] mid: " 36 | f"{book.mid()} imbalance: {round(book.imbalance(), 2)}" 37 | ) 38 | ) 39 | table.add_column("Bid CumVol", style="bright_green", justify="right", width=w) 40 | table.add_column("Bid Volume", style="bright_green", justify="right", width=w) 41 | table.add_column("Bid Price", style="bright_green", justify="right", width=w) 42 | table.add_column("Ask Price", style="bright_red", justify="left", width=w) 43 | table.add_column("Ask Volume", style="bright_red", justify="left", width=w) 44 | table.add_column("Ask CumVol", style="bright_red", justify="left", width=w) 45 | ask_cum = ZERO 46 | bid_cum = ZERO 47 | for (bid_price, bid_volume), (ask_price, ask_volume), _ in zip( 48 | book.bids.items(), book.asks.items(), range(self.levels) 49 | ): 50 | ask_cum += ask_volume 51 | bid_cum += bid_volume 52 | table.add_row( 53 | str(bid_cum), 54 | str(bid_volume), 55 | str(bid_price), 56 | str(ask_price), 57 | str(ask_volume), 58 | str(ask_cum), 59 | ) 60 | return Align.center(table) 61 | 62 | async def _live(self) -> None: 63 | """Coroutine for refreshing the console""" 64 | with Live(console=self.console, screen=True, auto_refresh=False) as live: 65 | while True: 66 | books = await self._books.get() 67 | live.update(Group(*books), refresh=True) 68 | -------------------------------------------------------------------------------- /pyservice/gateway.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import time 4 | from typing import Any, Sequence, cast 5 | 6 | from aiohttp import ( 7 | ClientError, 8 | ClientSession, 9 | ClientWebSocketResponse, 10 | WSMessage, 11 | WSMsgType, 12 | ) 13 | 14 | from . import config 15 | from .book import Book 16 | from .workers import Worker, Workers 17 | 18 | 19 | class BadState(Exception): 20 | pass 21 | 22 | 23 | class WebsocketReconnect(Exception): 24 | pass 25 | 26 | 27 | class Publisher: 28 | async def publish_books(self, books: dict[str, Book]) -> None: 29 | """Publish a book to the websocket""" 30 | pass 31 | 32 | 33 | class SimpleBackOff: 34 | def __init__(self, max_delay: float = 10.0, increase_by: float = 1.2) -> None: 35 | self.max_delay = max_delay 36 | self.increase_by = increase_by 37 | self.delay = 0.0 38 | 39 | def next(self) -> float: 40 | self.delay = min( 41 | self.increase_by * self.delay if self.delay > 0 else 1.0, self.max_delay 42 | ) 43 | return round(self.delay, 1) 44 | 45 | def reset(self) -> None: 46 | self.delay = 0.0 47 | 48 | 49 | class WebsocketGateway(Workers): 50 | """A Gateway consuming websocket messages.""" 51 | 52 | def __init__( 53 | self, 54 | publisher: Publisher = None, 55 | pairs: Sequence[str] = (), 56 | close_ws_every: int = 0, 57 | ) -> None: 58 | super().__init__() 59 | self._last_update: float = time.time() 60 | self._message_received: int = 0 61 | self._ws_connection: ClientWebSocketResponse | None = None 62 | self._close_ws_every = close_ws_every 63 | self._backoff = SimpleBackOff() 64 | self.books: dict[str, Book] = {} 65 | self.publisher = publisher or Publisher() 66 | self.pairs: tuple[str, ...] = tuple(pairs) 67 | self.add_worker(Worker(self._connect_and_listen)) 68 | self.add_worker(Worker(self._publish_books)) 69 | 70 | # WebsocketGateway interface 71 | 72 | def ws_url(self) -> str: 73 | """Return the websocket url""" 74 | raise NotImplementedError 75 | 76 | def on_ws_connection(self) -> None: 77 | """Callback when a new websocket is connected""" 78 | self.logger.warning("new websocket connection with %s", self.ws_url()) 79 | # this is to test a dropped connection 80 | if self._close_ws_every > 0: 81 | asyncio.get_event_loop().call_later( 82 | self._close_ws_every, self._drop_connection 83 | ) 84 | 85 | def on_text_message(self, msg: WSMessage) -> None: 86 | """Handle a text message from websocket 87 | 88 | Subclasses should implement this method innit!. 89 | """ 90 | self.logger.info("Websocket text message: %s", msg.data) 91 | 92 | def on_error_message(self, msg: WSMessage) -> None: 93 | """Handle an error message from websocket""" 94 | self.logger.warning("Websocket error: %s", msg.data) 95 | raise WebsocketReconnect 96 | 97 | def write_json(self, msg: Any) -> None: 98 | self.write(json.dumps(msg)) 99 | 100 | def write(self, msg: str) -> None: 101 | self.execute(self._send_str, msg) 102 | 103 | def new_book(self) -> Book: 104 | return Book() 105 | 106 | # Workers interface 107 | 108 | def status(self) -> dict: 109 | """Am I doing good? If not raise an error""" 110 | gap: float = time.time() - self._last_update 111 | if gap > config.STALE_WEBSOCKET_TIMEOUT: 112 | raise BadState("Websocket connection is stale for %s seconds" % gap) 113 | return dict( 114 | last_update=self._last_update, message_received=self._message_received 115 | ) 116 | 117 | # INTERNALS 118 | 119 | async def _send_str(self, msg: str) -> None: 120 | if self._ws_connection is not None: 121 | await self._ws_connection.send_str(msg) 122 | else: 123 | self.logger.warning("Websocket connection is closed") 124 | 125 | async def _connect_and_listen(self) -> None: 126 | """Coroutine for connecting and listening to websocket""" 127 | while True: 128 | async with ClientSession() as session: 129 | try: 130 | async with session.ws_connect(self.ws_url()) as ws_connection: 131 | self._ws_connection = ws_connection 132 | self._backoff.reset() 133 | try: 134 | self.on_ws_connection() 135 | await self._listen_and_consume_messages() 136 | except WebsocketReconnect: 137 | pass 138 | except ClientError as exc: 139 | self.logger.warning("Websocket connection error: %s", exc) 140 | delay = self._backoff.next() 141 | self.logger.warning("reconnect with websocket in %s seconds", delay) 142 | self._ws_connection = None 143 | await asyncio.sleep(delay) 144 | 145 | async def _listen_and_consume_messages(self) -> None: 146 | """Coroutine for consuming websocket messages""" 147 | async for msg in cast(ClientWebSocketResponse, self._ws_connection): 148 | self._last_update = time.time() 149 | self._message_received += 1 150 | match msg.type: 151 | case WSMsgType.TEXT: 152 | self.on_text_message(msg) 153 | case WSMsgType.ERROR: 154 | self.on_error_message(msg) 155 | case WSMsgType.CLOSE: 156 | self.logger.warning("Websocket connection closed") 157 | raise WebsocketReconnect 158 | case _: 159 | self.logger.warning("unhandled message type: %s", msg.type) 160 | # release loop to avoid starvation from greedy websocket connections 161 | await asyncio.sleep(0) 162 | 163 | async def _publish_books(self) -> None: 164 | while True: 165 | if len(self.books) == len(self.pairs): 166 | await self.publisher.publish_books(self.books) 167 | await asyncio.sleep(config.BOOK_PUBLISH_INTERVAL) 168 | 169 | def _drop_connection(self) -> None: 170 | if self._ws_connection is not None: 171 | self.logger.info("close websocket connection") 172 | self.execute(self._ws_connection.close) 173 | -------------------------------------------------------------------------------- /pyservice/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quantmind/kollector/ca09445cfc9f69299c7b52919a25c9459fd42997/pyservice/tests/__init__.py -------------------------------------------------------------------------------- /pyservice/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import random 2 | from typing import Any, AsyncGenerator 3 | from unittest import mock 4 | 5 | import numpy as np 6 | import pytest 7 | from aiohttp import ClientSession 8 | from aiohttp.test_utils import TestClient, TestServer 9 | from aiohttp.web import Application 10 | from aioresponses import aioresponses 11 | 12 | from pyservice import config, gateway, workers 13 | from pyservice.app import create_app 14 | from pyservice.book import Book 15 | 16 | from .ws import websocket 17 | 18 | 19 | @pytest.fixture 20 | def app() -> Application: 21 | return create_app(pairs="BTCUSDT") 22 | 23 | 24 | @pytest.fixture(autouse=True) 25 | def mocks(monkeypatch: Any) -> None: 26 | monkeypatch.setattr( 27 | gateway, 28 | "ClientSession", 29 | lambda: ClientSession(ws_response_class=mock.MagicMock), 30 | ) 31 | monkeypatch.setattr(workers, "bail_out", mock.MagicMock) 32 | 33 | 34 | def mock_binance_responses(r: aioresponses) -> None: 35 | r.get( 36 | f"{config.BINANCE_SPOT_URL}/api/v3/depth?symbol=BTCUSDT", 37 | payload=dict(lastUpdateId=19857594115, bids=[], asks=[]), 38 | repeat=True, 39 | ) 40 | r.get(config.BINANCE_SPOT_WS_URL, callback=websocket) 41 | 42 | 43 | @pytest.fixture 44 | async def app_cli(app: Application) -> AsyncGenerator: 45 | with aioresponses(passthrough=["http://127.0.0.1"]) as r: 46 | mock_binance_responses(r) 47 | client = TestClient(TestServer(app)) 48 | await client.start_server() 49 | try: 50 | yield client 51 | finally: 52 | await client.close() 53 | 54 | 55 | @pytest.fixture 56 | def book() -> Book: 57 | return Book() 58 | 59 | 60 | @pytest.fixture 61 | def rbook() -> Book: 62 | return random_book() 63 | 64 | 65 | def random_book(n_min: int = 30, n_max: int = 50) -> Book: 66 | book = Book() 67 | N = random.randint(n_min, n_max) 68 | prices = [(80 + 40 * random.random()) for _ in range(N)] 69 | volumes = [(1 + 10 * random.random()) for _ in range(N)] 70 | mid = np.mean(prices) 71 | for p, v in zip(prices, volumes): 72 | if p > mid: 73 | book.asks.set(p, v) 74 | else: 75 | book.bids.set(p, v) 76 | return book 77 | -------------------------------------------------------------------------------- /pyservice/tests/test_app.py: -------------------------------------------------------------------------------- 1 | from aiohttp.test_utils import TestClient 2 | 3 | 4 | async def test_status(app_cli: TestClient) -> None: 5 | response = await app_cli.get("/status") 6 | assert response.status == 200 7 | data = await response.json() 8 | assert data["binance"] 9 | -------------------------------------------------------------------------------- /pyservice/tests/test_book.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | from pyservice.book import HALF, Book 4 | 5 | 6 | def test_empty_order_book(book: Book) -> None: 7 | assert len(book.sides) == 2 8 | assert len(book.bids) == 0 9 | assert len(book.asks) == 0 10 | assert book.best_bid() != book.best_bid() 11 | assert book.best_ask() != book.best_ask() 12 | assert book.is_consistent() 13 | assert book.imbalance() == 0 14 | assert book.mid() != book.mid() 15 | p, v = book.bids.best 16 | assert p != p 17 | assert v == 0 18 | 19 | 20 | def test_only_bid(book: Book) -> None: 21 | assert len(book.sides) == 2 22 | assert len(book.bids) == 0 23 | assert len(book.asks) == 0 24 | book.bids.set(1.2, 10) 25 | assert len(book.bids) == 1 26 | assert len(book.asks) == 0 27 | assert book.is_consistent() 28 | assert book.imbalance() == 1 29 | 30 | 31 | def test_only_ask(book: Book) -> None: 32 | assert len(book.sides) == 2 33 | assert len(book.bids) == 0 34 | assert len(book.asks) == 0 35 | book.asks.set(1.2, 10) 36 | assert len(book.bids) == 0 37 | assert len(book.asks) == 1 38 | assert book.is_consistent() 39 | assert book.imbalance(1) == -1 40 | 41 | 42 | def test_order_book(book: Book) -> None: 43 | book.bids.set(1.1, 10) 44 | book.asks.set(1.2, 10) 45 | assert len(book.bids) == 1 46 | assert len(book.asks) == 1 47 | assert book.is_consistent() 48 | assert book.bids.best[1] == Decimal(10) 49 | assert book.asks.best[1] == Decimal(10) 50 | assert book.imbalance() == 0 51 | book.asks.set(1.2, 5) 52 | assert book.bids.best[1] == Decimal(10) 53 | assert book.asks.best[1] == Decimal(5) 54 | assert book.imbalance(1) == float( 55 | (Decimal(10) - Decimal(5)) / (Decimal(10) + Decimal(5)) 56 | ) 57 | 58 | 59 | def test_ordering(rbook: Book) -> None: 60 | assert rbook.is_consistent() 61 | b0 = float("infinity") 62 | for bid in rbook.bids: 63 | assert b0 > bid 64 | b0 = bid 65 | a0 = 0 66 | for ask in rbook.asks: 67 | assert a0 < ask 68 | a0 = ask 69 | 70 | 71 | def test_mid(rbook: Book) -> None: 72 | mid = rbook.mid() 73 | for bid in rbook.bids: 74 | assert bid < mid 75 | for ask in rbook.asks: 76 | assert ask > mid 77 | 78 | 79 | def test_container(rbook: Book) -> None: 80 | assert rbook 81 | assert len(rbook) == len(rbook.bids) + len(rbook.asks) 82 | prices = list(rbook) 83 | assert len(prices) == len(rbook) 84 | for p in prices: 85 | assert p in rbook 86 | 87 | 88 | def test_replace(rbook: Book) -> None: 89 | assert rbook.is_consistent() 90 | prices = rbook.bids if len(rbook.bids) > len(rbook.asks) else rbook.asks 91 | price_list = list(prices.items()) 92 | N = len(price_list) 93 | # 94 | h = int(N / 2) 95 | price = price_list[h][0] 96 | assert price_list[h][1] 97 | # 98 | # add new price 99 | price = HALF * (price_list[h - 1][0] + price_list[h][0]) 100 | prices[price] = 50 101 | assert len(prices) == N + 1 102 | 103 | 104 | def test_remove_level(book: Book) -> None: 105 | assert book.mid() != book.mid() 106 | assert book.spread() != book.spread() 107 | book.bids.set(45, 20) 108 | assert len(book.bids) == 1 109 | assert book.mid() == Decimal(45) 110 | assert book.spread() != book.spread() 111 | book.bids.set(45, 0) 112 | assert book.mid() != book.mid() 113 | assert book.spread() != book.spread() 114 | assert len(book.bids) == 0 115 | -------------------------------------------------------------------------------- /pyservice/tests/ws.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import hashlib 3 | from typing import Any 4 | from unittest.mock import MagicMock 5 | 6 | from aiohttp import ClientResponse, hdrs 7 | from aiohttp.http import WS_KEY 8 | from aioresponses import CallbackResult 9 | from yarl import URL 10 | 11 | 12 | def websocket(url: str, headers: dict, **kwargs: Any) -> CallbackResult: 13 | sec_key = headers[hdrs.SEC_WEBSOCKET_KEY].encode("utf-8") 14 | accept = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest()).decode() 15 | headers = { 16 | hdrs.SEC_WEBSOCKET_ACCEPT: accept, 17 | hdrs.CONNECTION: "upgrade", 18 | hdrs.UPGRADE: "websocket", 19 | } 20 | return CallbackResult( 21 | status=101, 22 | headers=headers, 23 | response_class=websocket_response_factory, # type: ignore 24 | ) 25 | 26 | 27 | def connection_mock() -> MagicMock: 28 | connection = MagicMock() 29 | transport = MagicMock() 30 | transport.is_closing.return_value = False 31 | connection.transport = transport 32 | return connection 33 | 34 | 35 | def websocket_response_factory(method: str, url: URL, **kwargs: Any) -> ClientResponse: 36 | resp = ClientResponse(method, url, **kwargs) 37 | resp._connection = connection_mock() 38 | return resp 39 | -------------------------------------------------------------------------------- /pyservice/workers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from functools import partial 4 | from typing import Any, Awaitable, Callable, Optional 5 | 6 | from aiohttp.web import Application, GracefulExit 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | WorkerType = Callable[[], Awaitable] 12 | # need better typing for this 13 | ExecutorType = Callable 14 | 15 | 16 | def bail_out() -> None: 17 | raise GracefulExit 18 | 19 | 20 | class Worker: 21 | def __init__(self, worker_func: WorkerType, logger: logging.Logger = logger): 22 | self.worker_func = worker_func 23 | self.logger = logger 24 | 25 | async def __call__(self) -> None: 26 | try: 27 | await self.worker_func() 28 | except Exception: 29 | self.logger.exception( 30 | "unhandled exception - bailing out", 31 | ) 32 | asyncio.get_event_loop().call_soon(bail_out) 33 | else: 34 | self.logger.exception("worker bailing out") 35 | asyncio.get_event_loop().call_soon(bail_out) 36 | 37 | 38 | class Workers: 39 | """Maintain a pool of asynchronous workers 40 | 41 | They maintain in memory normalized data to stream to other consumers 42 | """ 43 | 44 | def __init__(self) -> None: 45 | super().__init__() 46 | self.logger: logging.Logger = logger.getChild(self.name) 47 | self._output: asyncio.Queue = asyncio.Queue() 48 | self._workers: list[Worker] = [] 49 | self._tasks: Optional[tuple[asyncio.Task, ...]] = None 50 | self.add_worker(Worker(self._async_execution)) 51 | 52 | @property 53 | def name(self) -> str: 54 | """This is my name""" 55 | return self.__class__.__name__.lower() 56 | 57 | @classmethod 58 | def setup(cls, app: Application, **kwargs: Any) -> "Workers": 59 | """Create the workers and register startup and shutdown events with the app""" 60 | worker = cls(**kwargs) 61 | app.on_startup.append(worker.on_startup) 62 | app.on_shutdown.append(worker.on_shutdown) 63 | return worker 64 | 65 | def status(self) -> dict: 66 | """Return a status dict or raise an exception if in a bad state""" 67 | return {} 68 | 69 | def execute(self, executor: ExecutorType, *args: Any) -> None: 70 | self._output.put_nowait(partial(executor, *args)) 71 | 72 | def add_worker(self, worker: Worker) -> None: 73 | self._workers.append(worker) 74 | 75 | async def on_startup(self, app: Application) -> None: 76 | """register startup event with main app""" 77 | self._tasks = tuple(asyncio.create_task(worker()) for worker in self._workers) 78 | 79 | async def on_shutdown(self, app: Application) -> None: 80 | """register shutdown event with main app""" 81 | if self._tasks: 82 | self.logger.warning("closing %d background tasks", len(self._tasks)) 83 | for task in self._tasks: 84 | task.cancel() 85 | try: 86 | await asyncio.gather(*self._tasks) 87 | except asyncio.CancelledError: 88 | pass 89 | 90 | async def _async_execution(self) -> None: 91 | """Coroutine for executing async commands""" 92 | while True: 93 | executor = await self._output.get() 94 | await executor() 95 | await asyncio.sleep(0) 96 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Collect & stream orderbooks 2 | 3 | [![build](https://github.com/quantmind/kollector/actions/workflows/build.yml/badge.svg)](https://github.com/quantmind/kollector/actions/workflows/build.yml) 4 | 5 | A small service for collecting and streaming orderbook L2 data from crypto exchanges. 6 | 7 | ## Development 8 | 9 | * you need rust toolchain installed (this has been developed with rust 1.6) 10 | * run `make build` to build and test the rust application 11 | * [protobuf definitions](./service/proto/orderbook.proto) 12 | * code documentation https://quantmind.github.io/kollector/common/ 13 | * `yarn watch` will start the web server for development (serving on http://localhost:3000) 14 | 15 | 16 | ## Running the App 17 | 18 | You can run the e2e app using docker rather than building from source. 19 | To run server and a web client, make sure you have `docker-compose` installed and launch `make start`. 20 | 21 | The command will start: 22 | 23 | * a [kong](https://github.com/Kong/kong) gateway server configured for [grpc-web](https://docs.konghq.com/hub/kong-inc/grpc-web/) 24 | * the rust `kollector` service 25 | * the web server serving the front-end application on http://localhost:4000 26 | 27 | ![book](https://user-images.githubusercontent.com/144320/169648803-adf7fa98-2701-4695-b8c6-369a66883e1f.gif) 28 | 29 | ## Python application 30 | 31 | * A small python application to stream orderbooks from binance and display as table in the console 32 | * tested with python 3.10 only 33 | * Install the app via poetry `poetry install` (you need poetry first) 34 | * Run the application via `poetry run python main.py --console` or `make service-py` 35 | 36 | ![python-book](https://user-images.githubusercontent.com/144320/174490981-31093f56-6101-4492-8bc5-83033d5219c3.gif) 37 | -------------------------------------------------------------------------------- /service/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "service" 3 | version = "0.1.0" 4 | edition = "2021" 5 | homepage = "https://github.com/quantmind/kollector" 6 | repository = "https://github.com/quantmind/kollector" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | common = { path = "../common" } 12 | gateways = { path = "../gateways" } 13 | async-channel = "1.6.1" 14 | clap = { version = "3.1.18", features = ["derive"] } 15 | dotenv = "0.15.0" 16 | futures-util = "0.3.21" 17 | prost = "0.10.3" 18 | tokio = { version = "^1.18.2", features = ["full"] } 19 | tokio-stream = "0.1.8" 20 | tonic = { version = "^0.7.2", features = ["tls", "compression"] } 21 | slog = "2.7.0" 22 | ctrlc = "3.2.2" 23 | prometheus = "0.13.0" 24 | lazy_static = "1.4.0" 25 | rust_decimal = "1.23.1" 26 | hyper = "0.14.18" 27 | anyhow = "1.0.57" 28 | serde = "1.0.137" 29 | 30 | [build-dependencies] 31 | tonic-build = { version = "^0.7.2", features = ["prost", "compression"] } 32 | -------------------------------------------------------------------------------- /service/build.rs: -------------------------------------------------------------------------------- 1 | use std::{env, path::PathBuf}; 2 | 3 | fn main() { 4 | let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); 5 | tonic_build::configure() 6 | .file_descriptor_set_path(out_dir.join("orderbook.bin")) 7 | .compile(&["proto/orderbook.proto"], &["proto"]) 8 | .unwrap(); 9 | } 10 | -------------------------------------------------------------------------------- /service/proto/orderbook.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package orderbook; 4 | 5 | service OrderbookAggregator { 6 | rpc BookSummary(BookRequest) returns (stream Summary); 7 | rpc Info(Empty) returns (ServiceInfo); 8 | } 9 | 10 | message Empty {} 11 | 12 | message BookRequest { 13 | string pair = 1; 14 | } 15 | 16 | message ServiceInfo { 17 | repeated string pairs = 1; 18 | uint64 max_depth = 2; 19 | } 20 | 21 | 22 | message Summary { 23 | double spread = 1; 24 | repeated Level bids = 2; 25 | repeated Level asks = 3; 26 | } 27 | 28 | 29 | message Level { 30 | string exchange = 1; 31 | double price = 2; 32 | double amount = 3; 33 | } 34 | -------------------------------------------------------------------------------- /service/src/grpc.rs: -------------------------------------------------------------------------------- 1 | pub mod orderbook { 2 | tonic::include_proto!("orderbook"); 3 | } 4 | use common::{bid_ask_spread, wrap_result, Book, Context, WorkerContext, L2}; 5 | use futures_util::Stream; 6 | use orderbook::{ 7 | orderbook_aggregator_server as obs, BookRequest, Empty, Level, ServiceInfo, Summary, 8 | }; 9 | use rust_decimal::prelude::*; 10 | use slog::info; 11 | use std::collections::HashMap; 12 | use std::{net::ToSocketAddrs, pin::Pin}; 13 | use tokio::sync::mpsc; 14 | use tokio_stream::{wrappers::ReceiverStream, StreamExt}; 15 | use tonic::{transport::Server, Request, Response, Status}; 16 | 17 | /// An hashmap mapping an exchange name with the orderbook for a given asset 18 | pub type AssetBooks = HashMap; 19 | type GrpcResult = Result, Status>; 20 | type GrpcContext = Context<(String, Summary)>; 21 | 22 | /// Orderbook Aggregator GRPC server 23 | /// 24 | /// This struct implements the BookSummary streamying method for the GRPC server. 25 | /// The server receive messages from the main application and broadcast them 26 | /// to grpc upstream clients. 27 | #[derive(Clone)] 28 | pub struct OrderbookAggregator { 29 | /// use this to send messages to the Orderbook Aggregator service 30 | pub context: GrpcContext, 31 | pairs: Vec, 32 | max_depth: usize, 33 | } 34 | 35 | /// Start serving the GRPC 36 | /// 37 | /// The port is configured via the `app_grpc_port` environment variable and defaults to 50060. 38 | pub fn serve_grpc(server: OrderbookAggregator, context: &WorkerContext) { 39 | let ctx = context.clone(); 40 | tokio::spawn(async move { 41 | let host: String = server 42 | .context 43 | .get_or("app_grpc_host", "[::1]".to_owned()) 44 | .expect("GRPC host"); 45 | let port: u16 = server 46 | .context 47 | .get_or("app_grpc_port", 50060) 48 | .expect("GRPC port"); 49 | let addr = format!("{}:{}", host, port) 50 | .to_socket_addrs() 51 | .unwrap() 52 | .next() 53 | .unwrap(); 54 | info!(server.context.logger, "start the GRPC server {}", addr); 55 | let result = Server::builder() 56 | .accept_http1(true) 57 | .add_service(obs::OrderbookAggregatorServer::new(server)) 58 | .serve(addr) 59 | .await 60 | .map_err(anyhow::Error::new); 61 | wrap_result(&ctx, result).await; 62 | }); 63 | } 64 | 65 | /// Extract the book Summary protobuf message from asset order books from exchanges 66 | pub fn book_summary(asset_books: &AssetBooks) -> Summary { 67 | let mut summary = Summary::default(); 68 | let mut best_ask: Option = None; 69 | let mut best_bid: Option = None; 70 | for (exchange, book) in asset_books.iter() { 71 | best_ask = update_summary_side(&mut summary.asks, &book.asks, exchange, best_ask); 72 | best_bid = update_summary_side(&mut summary.bids, &book.bids, exchange, best_bid); 73 | } 74 | summary 75 | .asks 76 | .sort_by(|a, b| a.price.partial_cmp(&b.price).unwrap()); 77 | summary 78 | .bids 79 | .sort_by(|a, b| b.price.partial_cmp(&a.price).unwrap()); 80 | summary.spread = bid_ask_spread(best_bid, best_ask) 81 | .unwrap_or(Decimal::ZERO) 82 | .to_f64() 83 | .unwrap(); 84 | summary 85 | } 86 | 87 | fn update_summary_side( 88 | levels: &mut Vec, 89 | book_side: &L2, 90 | exchange: &str, 91 | best_price: Option, 92 | ) -> Option { 93 | for (price, volume) in book_side.iter() { 94 | levels.push(Level { 95 | exchange: exchange.to_owned(), 96 | price: price.to_f64().unwrap(), 97 | amount: volume.to_f64().unwrap(), 98 | }); 99 | } 100 | book_side.best_of(best_price) 101 | } 102 | 103 | impl OrderbookAggregator { 104 | /// create a new OrderbookAggregator server 105 | pub fn new(pairs: &[String], max_depth: usize) -> Self { 106 | Self { 107 | context: GrpcContext::new("grpc", None), 108 | pairs: pairs.to_owned(), 109 | max_depth, 110 | } 111 | } 112 | } 113 | 114 | #[tonic::async_trait] 115 | impl obs::OrderbookAggregator for OrderbookAggregator { 116 | type BookSummaryStream = Pin> + Send>>; 117 | 118 | async fn book_summary( 119 | &self, 120 | request: Request, 121 | ) -> GrpcResult { 122 | // get a new receiver for this connection 123 | let mut context = self.context.clone(); 124 | let pair = request.get_ref().pair.to_owned(); 125 | info!(context.logger, "new connection for pair {}", pair); 126 | 127 | let (tx, rx) = mpsc::channel(128); 128 | 129 | tokio::spawn(async move { 130 | while let Some((asset, message)) = context.receiver.next().await { 131 | if pair == asset { 132 | match tx.send(Result::<_, Status>::Ok(message)).await { 133 | Ok(_) => { 134 | // item (server response) was queued to be send to client 135 | } 136 | Err(_item) => { 137 | // output_stream was build from rx and both are dropped 138 | break; 139 | } 140 | } 141 | } 142 | } 143 | info!(context.logger, "client disconnected from pair {}", pair); 144 | }); 145 | 146 | let output_stream = ReceiverStream::new(rx); 147 | 148 | Ok(Response::new( 149 | Box::pin(output_stream) as Self::BookSummaryStream 150 | )) 151 | } 152 | 153 | async fn info(&self, _request: Request) -> GrpcResult { 154 | let reply = ServiceInfo { 155 | max_depth: self.max_depth.to_u64().unwrap(), 156 | pairs: self.pairs.to_owned(), 157 | }; 158 | Ok(Response::new(reply)) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /service/src/http.rs: -------------------------------------------------------------------------------- 1 | use common::{wrap_result, WorkerContext}; 2 | extern crate hyper; 3 | use hyper::service::{make_service_fn, service_fn}; 4 | use hyper::{header::CONTENT_TYPE, Body, Method, Request, Response, Server, StatusCode}; 5 | use prometheus::{Encoder, TextEncoder}; 6 | use std::net::SocketAddr; 7 | extern crate anyhow; 8 | use slog::info; 9 | 10 | async fn http_handler(req: Request) -> Result, hyper::Error> { 11 | match (req.method(), req.uri().path()) { 12 | (&Method::GET, "/status") => Ok(Response::new("OK".into())), 13 | (&Method::GET, "/metrics") => serve_prometheus_req(req).await, 14 | _ => not_found(req), 15 | } 16 | } 17 | 18 | fn not_found(_req: Request) -> Result, hyper::Error> { 19 | let mut not_found = Response::default(); 20 | *not_found.status_mut() = StatusCode::NOT_FOUND; 21 | Ok(not_found) 22 | } 23 | 24 | async fn serve_prometheus_req(_req: Request) -> Result, hyper::Error> { 25 | let encoder = TextEncoder::new(); 26 | 27 | let metric_families = prometheus::gather(); 28 | 29 | let mut buffer = vec![]; 30 | encoder.encode(&metric_families, &mut buffer).unwrap(); 31 | 32 | let response = Response::builder() 33 | .status(200) 34 | .header(CONTENT_TYPE, encoder.format_type()) 35 | .body(Body::from(buffer)) 36 | .unwrap(); 37 | 38 | Ok(response) 39 | } 40 | 41 | /// Starts an HTTP service 42 | /// 43 | /// This service expose a liveness probe and prometheus metrics. 44 | /// The port is configured via the `app_http_port` environment variable and defaults to 8050. 45 | pub async fn start_http_service(context: WorkerContext) { 46 | let port: u16 = context.get_or("app_http_port", 8050).expect("HTTP port"); 47 | // create the redis client 48 | // let _redis_cli = context.redis_cli("REDIS_URL").unwrap(); 49 | let addr = SocketAddr::from(([0, 0, 0, 0], port)); 50 | let make_service = make_service_fn(|_conn| async { 51 | // service_fn converts our function into a `Service` 52 | Ok::<_, hyper::Error>(service_fn(http_handler)) 53 | }); 54 | let server = Server::bind(&addr).serve(make_service); 55 | info!(context.logger, "start http server on {}", addr); 56 | let result = server.await.map_err(anyhow::Error::new); 57 | wrap_result(&context, result).await; 58 | } 59 | -------------------------------------------------------------------------------- /service/src/kollector.rs: -------------------------------------------------------------------------------- 1 | use crate::grpc::{book_summary, serve_grpc, AssetBooks, OrderbookAggregator}; 2 | use crate::http::start_http_service; 3 | use common::{wrap_result, Book, Context, InnerMessage}; 4 | use gateways::{Gateway, WsUpdate}; 5 | use slog::{error, info, warn}; 6 | use std::collections::{HashMap, HashSet}; 7 | 8 | /// The Kollector is the main the main service 9 | pub struct Kollector { 10 | pub context: Context, 11 | pub max_depth: usize, 12 | pub pairs: Vec, 13 | gateways: HashMap>, 14 | books: HashMap, 15 | grpc: Option, 16 | } 17 | 18 | impl Kollector { 19 | /// Create a new Kollector service 20 | /// 21 | /// # Arguments 22 | /// 23 | /// * pairs - comma separated list of crypto pairs 24 | /// * max_depth - maximum depth of order book to stream 25 | pub fn new(pairs: &str, max_depth: usize) -> Self { 26 | let pairs_: Vec = HashSet::::from_iter(pairs.split(',').map(String::from)) 27 | .into_iter() 28 | .collect(); 29 | Self { 30 | context: Context::::new("kollector", None), 31 | gateways: HashMap::new(), 32 | books: HashMap::new(), 33 | pairs: pairs_, 34 | grpc: None, 35 | max_depth, 36 | } 37 | } 38 | 39 | /// Spawn a gateway 40 | /// 41 | /// This method add a new gateway to the gateway's hashmap and 42 | /// start the websocket coroutine which connect to the exchange. 43 | /// 44 | /// This method should be called before running the service. 45 | pub fn spawn_gateway(&mut self, gateway: Box) { 46 | gateway.setup(); 47 | let ws = gateway.ws_consumer(); 48 | self.gateways.insert(gateway.name().to_owned(), gateway); 49 | tokio::spawn(async move { 50 | let result = ws.run().await; 51 | wrap_result(&ws.context, result).await; 52 | }); 53 | } 54 | 55 | /// Spawn the grpc server 56 | /// 57 | /// This method should be called before running the service 58 | pub fn spawn_grpc(&mut self) { 59 | let grpc = OrderbookAggregator::new(&self.pairs, self.max_depth); 60 | self.grpc = Some(grpc.clone()); 61 | serve_grpc(grpc, &self.context); 62 | } 63 | 64 | /// Add web service 65 | /// 66 | /// Add a web service for prometheus metrics and k8s liveness probe 67 | pub fn spawn_http(&self) { 68 | let service = self.context.clone(); 69 | tokio::spawn(async move { 70 | start_http_service(service).await; 71 | }); 72 | } 73 | 74 | /// Add Ctrl-C handler 75 | pub fn handle_ctrlc(&self) { 76 | // handle shutdown 77 | let signal = self.context.clone(); 78 | ctrlc::set_handler(move || { 79 | signal.sender.try_send(InnerMessage::Exit).unwrap(); 80 | }) 81 | .expect("Error setting Ctrl-C handler"); 82 | } 83 | 84 | /// Main coroutine 85 | /// 86 | /// This coroutine runs the main part of the kollector service 87 | pub async fn run(&mut self) { 88 | let context = self.context.clone(); 89 | 90 | for gateway in self.gateways.values_mut() { 91 | info!( 92 | context.logger, 93 | "subscribe to {} {:?}", 94 | gateway.name(), 95 | self.pairs 96 | ); 97 | gateway.subscribe(&self.pairs); 98 | } 99 | 100 | info!( 101 | context.logger, 102 | "start {} with depth {}", context.name, self.max_depth 103 | ); 104 | loop { 105 | match context.receiver.recv().await { 106 | Ok(InnerMessage::Failure) => { 107 | warn!(context.logger, "exit main worker after failure"); 108 | return; 109 | } 110 | Ok(InnerMessage::Exit) => { 111 | warn!(context.logger, "exit main worker"); 112 | return; 113 | } 114 | // 115 | Ok(InnerMessage::NewBookSnapshot(snapshot)) => { 116 | match self.gateways.get_mut(&snapshot.name) { 117 | Some(gw) => { 118 | gw.request_snapshot(); 119 | } 120 | None => { 121 | error!( 122 | context.logger, 123 | "snapshot from an unknown gateway {}", snapshot.name 124 | ); 125 | } 126 | } 127 | } 128 | // 129 | Ok(InnerMessage::BookSnapshot(snapshot)) => { 130 | let name = snapshot.name.to_owned(); 131 | match self.gateways.get_mut(&name) { 132 | Some(gw) => { 133 | if let Some(book) = gw.on_book_snapshot(snapshot) { 134 | self.update_book(&name, book); 135 | } 136 | } 137 | None => { 138 | error!(context.logger, "snapshot from an unknown gateway {}", name); 139 | } 140 | } 141 | } 142 | // websocket payload 143 | Ok(InnerMessage::WsPayload(ws_payload)) => { 144 | match self.gateways.get_mut(&ws_payload.name) { 145 | Some(gw) => { 146 | if let Some(update) = gw.on_websocket_message(ws_payload.value) { 147 | match update { 148 | WsUpdate::Book(book) => { 149 | self.update_book(&ws_payload.name, book); 150 | } 151 | } 152 | } 153 | } 154 | None => { 155 | error!( 156 | context.logger, 157 | "message from an unknown gateway {}", ws_payload.name 158 | ); 159 | } 160 | }; 161 | } 162 | Ok(_) => { 163 | // skip any other message 164 | } 165 | // 166 | Err(err) => { 167 | error!( 168 | context.logger, 169 | "Main loop could not receive message: {}", err 170 | ); 171 | return; 172 | } 173 | } 174 | } 175 | } 176 | 177 | fn update_book(&mut self, name: &str, book: Book) { 178 | let asset = book.asset.to_owned(); 179 | let asset_books = self 180 | .books 181 | .entry(asset.to_owned()) 182 | .or_insert_with(HashMap::new); 183 | asset_books.insert(name.to_owned(), book); 184 | let summary = book_summary(asset_books); 185 | // broadcast the summary to listeners (for now grpc only) 186 | if let Some(grpc) = &self.grpc { 187 | grpc.context.try_send((asset, summary)); 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /service/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! GRPC Service exposing streaming endpoint of order book updates 2 | mod grpc; 3 | mod http; 4 | mod kollector; 5 | 6 | pub use self::grpc::*; 7 | pub use self::http::*; 8 | pub use self::kollector::*; 9 | -------------------------------------------------------------------------------- /service/src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use gateways::{Binance, Bitstamp}; 3 | use service::Kollector; 4 | 5 | #[derive(Parser, Debug)] 6 | #[clap(author, version, about, long_about = None)] 7 | struct Args { 8 | /// Comma separated list of currency pairs 9 | #[clap(short, long)] 10 | pairs: String, 11 | 12 | /// Maximum number of order book levels to track 13 | #[clap(short, long, default_value_t = 10)] 14 | max_depth: usize, 15 | } 16 | 17 | #[macro_export] 18 | macro_rules! run_gateway { 19 | ($kollector: ident, $Gateway: ident) => { 20 | $kollector.spawn_gateway(Box::new($Gateway::new( 21 | &$kollector.context, 22 | $kollector.max_depth, 23 | &$kollector.pairs, 24 | ))) 25 | }; 26 | } 27 | 28 | #[tokio::main] 29 | async fn main() { 30 | dotenv::from_path(".env").ok(); 31 | let app = Args::parse(); 32 | 33 | // create the service 34 | let mut kollector = Kollector::new(&app.pairs, app.max_depth); 35 | // add Ctrl-c handler 36 | kollector.handle_ctrlc(); 37 | // spawn the GRPC server 38 | kollector.spawn_grpc(); 39 | // spawn gateways 40 | run_gateway!(kollector, Binance); 41 | run_gateway!(kollector, Bitstamp); 42 | // spawn the HTTP server 43 | kollector.spawn_http(); 44 | // run the main application 45 | kollector.run().await; 46 | } 47 | -------------------------------------------------------------------------------- /service/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use lazy_static::lazy_static; 2 | use prometheus as prom; 3 | 4 | lazy_static! { 5 | pub static ref INPUT_QUEUE_LENGTH: prom::GaugeVec = prom::register_gauge_vec!( 6 | "kollector_input_message_queue", 7 | "Kollector internal message queue length", 8 | &["gateway"] 9 | ) 10 | .unwrap(); 11 | pub static ref OUTPUT_QUEUE_LENGTH: prom::GaugeVec = prom::register_gauge_vec!( 12 | "kollector_output_message_queue", 13 | "Kollector output internal message queue length", 14 | &["gateway"] 15 | ) 16 | .unwrap(); 17 | } 18 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = __pycache__,.venv,dist,node_modules,target,common,devops,gateways,service,web 3 | max-line-length = 88 4 | ignore = A001,A002,A003,B902,C816,C815,C812,W503,E203 5 | per-file-ignores = 6 | research/*:E402,E501,I100,I202,F704 7 | 8 | 9 | [tool:pytest] 10 | asyncio_mode = auto 11 | testpaths = 12 | pyservice/tests/ 13 | filterwarnings = 14 | ignore::DeprecationWarning 15 | ignore::UserWarning 16 | 17 | [isort] 18 | line_length = 88 19 | skip=.venv,dist,node_modules,target,common,devops,gateways,service,web 20 | multi_line_output=3 21 | include_trailing_comma=True 22 | 23 | [mypy] 24 | ignore_missing_imports=True 25 | disallow_untyped_calls=False 26 | warn_return_any=False 27 | disallow_untyped_defs=True 28 | warn_no_return=True 29 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "jsx": "react", 4 | "module": "esnext", 5 | "moduleResolution": "node", 6 | "allowSyntheticDefaultImports": true, 7 | "esModuleInterop": true, 8 | "sourceMap": true, 9 | "strict": true, 10 | "target": "esnext", 11 | "allowJs": true, 12 | "experimentalDecorators": true, 13 | "resolveJsonModule": true, 14 | "isolatedModules": false, 15 | "baseUrl": ".", 16 | "paths": { 17 | "@fluidily/*": ["./app/*"], 18 | } 19 | }, 20 | "exclude": ["node_modules"], 21 | "include": ["web/**/*.ts", "web/**/*.js", "app/**/*.tsx", "@types/*.ts", "web/Index.tsx"] 22 | } 23 | -------------------------------------------------------------------------------- /web/Index.tsx: -------------------------------------------------------------------------------- 1 | import CssBaseline from "@mui/material/CssBaseline"; 2 | import { createTheme, ThemeProvider } from "@mui/material/styles"; 3 | import React from "react"; 4 | import { createRoot } from "react-dom/client"; 5 | import { BrowserRouter, Route, Routes } from "react-router-dom"; 6 | import Main from "./Main"; 7 | 8 | const root = document.getElementById("__kollector"); 9 | 10 | if (root) { 11 | const app = createRoot(root); 12 | 13 | const darkTheme = createTheme({ 14 | palette: { 15 | mode: "dark", 16 | }, 17 | }); 18 | 19 | app.render( 20 | 21 | 22 | 23 | 24 | } /> 25 | 26 | 27 | 28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /web/Main.tsx: -------------------------------------------------------------------------------- 1 | import Box from "@mui/material/Box"; 2 | import Container from "@mui/material/Container"; 3 | import FormControl from "@mui/material/FormControl"; 4 | import FormHelperText from "@mui/material/FormHelperText"; 5 | import MenuItem from "@mui/material/MenuItem"; 6 | import Select from "@mui/material/Select"; 7 | import { useTheme } from "@mui/material/styles"; 8 | import Typography from "@mui/material/Typography"; 9 | import * as Plot from "@observablehq/plot"; 10 | import { extent, format } from "d3"; 11 | import React from "react"; 12 | import { useSearchParams } from "react-router-dom"; 13 | import GrpcService from "./service"; 14 | import { PlotReact } from "./Viz"; 15 | 16 | const formatImbalance = format(".3f"); 17 | 18 | interface Info { 19 | pairs: string[]; 20 | } 21 | 22 | interface Level { 23 | exchange: string; 24 | price: number; 25 | amount: number; 26 | side: string; 27 | group: string; 28 | } 29 | 30 | interface Summary { 31 | spread: number; 32 | asks: Level[]; 33 | bids: Level[]; 34 | } 35 | 36 | const Main = () => { 37 | const [searchParams, setSearchParams] = useSearchParams(); 38 | const [info, setInfo] = React.useState(null); 39 | const grpc = new GrpcService(); 40 | const getInfo = info ? true : false; 41 | 42 | const handleChange = (e: any) => { 43 | setSearchParams({ pair: e.target.value }); 44 | }; 45 | 46 | React.useEffect(() => { 47 | grpc.info(setInfo); 48 | }, [getInfo]); 49 | 50 | if (!info) return null; 51 | 52 | const pair = searchParams.get("pair") || info.pairs[0]; 53 | 54 | return ( 55 | 56 | 57 | Orderbook stream 58 | 59 | 60 | 61 | 68 | Crypto pair 69 | 70 | 71 | 72 | 73 | ); 74 | }; 75 | 76 | const Inner = ({ pair, grpc }: { pair: string; grpc: GrpcService }) => { 77 | const theme = useTheme(); 78 | const plot = React.useRef(null); 79 | const [summary, setData] = React.useState(null); 80 | 81 | const storeChart = (chart: any) => { 82 | plot.current = chart; 83 | }; 84 | 85 | React.useEffect(() => { 86 | const stream = grpc.streamPair(pair, setData); 87 | return () => { 88 | stream.cancel(); 89 | }; 90 | }, [pair]); 91 | 92 | if (!summary) return null; 93 | const plotData = summary.bids.concat(summary.asks); 94 | const [askAmount, asks] = cumsum(summary.asks); 95 | const [bidAmount, bids] = cumsum(summary.bids); 96 | const imbalance = formatImbalance( 97 | (bidAmount - askAmount) / (bidAmount + askAmount) 98 | ); 99 | plotData.sort(by_price); 100 | const groups = Array.from(new Set(plotData.map((d: Level) => d.group))); 101 | groups.sort(); 102 | 103 | // plot options 104 | const options = { 105 | title: "Orderbooks", 106 | marginLeft: 100, 107 | marginTop: 50, 108 | marginBottom: 50, 109 | style: { 110 | background: theme.palette.background.default, 111 | color: theme.palette.text.primary, 112 | fontSize: 14, 113 | }, 114 | x: { 115 | grid: true, 116 | }, 117 | y: { 118 | grid: true, 119 | domain: price_domain(summary.asks, summary.bids), 120 | }, 121 | color: { 122 | type: "ordinal", 123 | scheme: "spectral", 124 | legend: true, 125 | }, 126 | marks: [ 127 | Plot.ruleY(plotData, { 128 | x: "amount", 129 | y: "price", 130 | stroke: "group", 131 | }), 132 | Plot.dot(plotData, { x: "amount", y: "price", fill: "group", r: 6 }), 133 | Plot.areaX(asks, { 134 | y: "price", 135 | x2: "amount", 136 | fill: "red", 137 | curve: "step", 138 | fillOpacity: 0.2, 139 | }), 140 | Plot.lineX(asks, { 141 | y: "price", 142 | x: "amount", 143 | stroke: "red", 144 | curve: "step", 145 | strokeOpacity: 0.5, 146 | }), 147 | Plot.areaX(bids, { 148 | y: "price", 149 | x2: "amount", 150 | fill: "green", 151 | curve: "step", 152 | fillOpacity: 0.2, 153 | }), 154 | Plot.lineX(bids, { 155 | y: "price", 156 | x: "amount", 157 | stroke: "green", 158 | curve: "step", 159 | strokeOpacity: 0.5, 160 | }), 161 | ], 162 | }; 163 | 164 | return ( 165 | <> 166 | 167 | Spread: {summary.spread} - Book imbalance: {imbalance} 168 | 169 | 170 | 176 | 177 | 178 | ); 179 | }; 180 | 181 | const price = (l: Level) => l.price; 182 | const by_price = (l1: Level, l2: Level) => (l1.price > l2.price ? 1 : -1); 183 | 184 | // keep the plt stable rather than bouncing around 185 | const price_domain = (asks: Level[], bids: Level[]) => { 186 | const [a1, a2] = extent(asks, price) as [number, number]; 187 | const [b1, b2] = extent(bids, price) as [number, number]; 188 | const mid = (a1 + b2) / 2; 189 | const d = 1.05 * Math.max(a2 - mid, mid - b1); 190 | return [mid - d, mid + d]; 191 | }; 192 | 193 | const cumsum = (levels: Level[]): [number, any] => { 194 | let amount = 0; 195 | let depth = levels.map((l: Level) => { 196 | amount += l.amount; 197 | return { amount, price: l.price }; 198 | }); 199 | return [amount, depth]; 200 | }; 201 | 202 | export default Main; 203 | -------------------------------------------------------------------------------- /web/Viz/Chart.tsx: -------------------------------------------------------------------------------- 1 | import Box from "@mui/material/Box"; 2 | import debounce from "lodash.debounce"; 3 | import React from "react"; 4 | 5 | export interface Size { 6 | width: number; 7 | height: number; 8 | } 9 | 10 | export interface PlotReactProps { 11 | options: O; 12 | height?: string | number; 13 | width?: number; 14 | maxHeight?: number; 15 | wait?: number; 16 | onDelete?: (chart: T) => void; 17 | onCreate?: (chart: T) => void; 18 | } 19 | 20 | export interface InnerPlotReactProps extends PlotReactProps { 21 | createChart: (el: HTMLElement, options: O, size: Size) => T; 22 | resizeChart: (el: HTMLElement, options: O, size: Size, chart: T) => void; 23 | destroy: (chart: T) => void; 24 | } 25 | 26 | // Generic Charting React Element 27 | // 28 | // Supports 29 | // - uPlot 30 | // - Observable Plot 31 | // 32 | // Allow to resize window and set maximum height and or aspect ratio 33 | export const Chart = (props: InnerPlotReactProps) => { 34 | const { 35 | createChart, 36 | resizeChart, 37 | destroy, 38 | height = "70%", 39 | width = 0, 40 | wait = 100, 41 | maxHeight = 10000, 42 | onCreate, 43 | onDelete, 44 | options, 45 | } = props; 46 | const resizable = height.constructor.name == "String"; 47 | const heightStr = height as string; 48 | const heightPct = resizable 49 | ? +heightStr.substring(0, heightStr.length - 1) 50 | : 0; 51 | const chartRef = React.useRef(null); 52 | const targetRef = React.useRef(null); 53 | const [size, setSize] = React.useState({ 54 | width, 55 | height: resizable ? 0 : (height as number), 56 | }); 57 | 58 | const plotSize = (): Size | undefined => { 59 | const s: Size = { height: 0, width: targetRef.current?.offsetWidth || 0 }; 60 | if (size.width !== s.width) { 61 | s.height = Math.min(Math.round(0.01 * heightPct * s.width), maxHeight); 62 | return s; 63 | } 64 | }; 65 | // Create the chart 66 | const create = () => { 67 | chartRef.current = createChart( 68 | targetRef.current as HTMLElement, 69 | options, 70 | resizable ? plotSize() || size : size 71 | ); 72 | 73 | if (onCreate) onCreate(chartRef.current); 74 | }; 75 | 76 | const doDestroy = (chart: T | null) => { 77 | if (chart) { 78 | if (onDelete) onDelete(chart as T); 79 | destroy(chart as T); 80 | chartRef.current = null; 81 | } 82 | }; 83 | 84 | // 85 | React.useEffect(() => { 86 | if (resizable) { 87 | const newSize = plotSize(); 88 | if (newSize) { 89 | setSize(newSize); 90 | return; 91 | } 92 | } 93 | create(); 94 | const current = chartRef.current; 95 | 96 | let handleResize: any = null; 97 | 98 | if (resizable) { 99 | handleResize = debounce(() => { 100 | const newSize = plotSize(); 101 | if (newSize) { 102 | setSize(newSize); 103 | } 104 | }, wait); 105 | window.addEventListener("resize", handleResize); 106 | } 107 | 108 | return () => { 109 | if (handleResize) { 110 | window.removeEventListener("resize", handleResize); 111 | } 112 | doDestroy(current); 113 | }; 114 | }, [size, options]); 115 | 116 | return ( 117 | 121 | ); 122 | }; 123 | -------------------------------------------------------------------------------- /web/Viz/PlotReact.tsx: -------------------------------------------------------------------------------- 1 | import { plot } from "@observablehq/plot"; 2 | import React from "react"; 3 | import { Chart, PlotReactProps, Size } from "./Chart"; 4 | 5 | type ObservablePlot = typeof plot; 6 | type ObservablePlotOptions = any; 7 | 8 | const PlotReact = ( 9 | props: PlotReactProps 10 | ) => { 11 | const createChart = ( 12 | el: HTMLElement, 13 | options: ObservablePlotOptions, 14 | size: Size 15 | ): ObservablePlot => { 16 | const chart = plot({ ...options, ...size }); 17 | el.append(chart); 18 | return chart; 19 | }; 20 | 21 | const destroy = (chart: ObservablePlot) => { 22 | chart.remove(); 23 | }; 24 | 25 | return ( 26 | 32 | ); 33 | }; 34 | 35 | export default PlotReact; 36 | -------------------------------------------------------------------------------- /web/Viz/index.ts: -------------------------------------------------------------------------------- 1 | export { default as PlotReact } from "./PlotReact"; 2 | -------------------------------------------------------------------------------- /web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | GRPC Orderbooks 6 | 7 | 8 |
9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /web/proto/orderbook_grpc_web_pb.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileoverview gRPC-Web generated client stub for orderbook 3 | * @enhanceable 4 | * @public 5 | */ 6 | 7 | // GENERATED CODE -- DO NOT EDIT! 8 | 9 | 10 | /* eslint-disable */ 11 | // @ts-nocheck 12 | 13 | 14 | 15 | const grpc = {}; 16 | grpc.web = require('grpc-web'); 17 | 18 | const proto = {}; 19 | proto.orderbook = require('./orderbook_pb.js'); 20 | 21 | /** 22 | * @param {string} hostname 23 | * @param {?Object} credentials 24 | * @param {?grpc.web.ClientOptions} options 25 | * @constructor 26 | * @struct 27 | * @final 28 | */ 29 | proto.orderbook.OrderbookAggregatorClient = 30 | function(hostname, credentials, options) { 31 | if (!options) options = {}; 32 | options.format = 'text'; 33 | 34 | /** 35 | * @private @const {!grpc.web.GrpcWebClientBase} The client 36 | */ 37 | this.client_ = new grpc.web.GrpcWebClientBase(options); 38 | 39 | /** 40 | * @private @const {string} The hostname 41 | */ 42 | this.hostname_ = hostname; 43 | 44 | }; 45 | 46 | 47 | /** 48 | * @param {string} hostname 49 | * @param {?Object} credentials 50 | * @param {?grpc.web.ClientOptions} options 51 | * @constructor 52 | * @struct 53 | * @final 54 | */ 55 | proto.orderbook.OrderbookAggregatorPromiseClient = 56 | function(hostname, credentials, options) { 57 | if (!options) options = {}; 58 | options.format = 'text'; 59 | 60 | /** 61 | * @private @const {!grpc.web.GrpcWebClientBase} The client 62 | */ 63 | this.client_ = new grpc.web.GrpcWebClientBase(options); 64 | 65 | /** 66 | * @private @const {string} The hostname 67 | */ 68 | this.hostname_ = hostname; 69 | 70 | }; 71 | 72 | 73 | /** 74 | * @const 75 | * @type {!grpc.web.MethodDescriptor< 76 | * !proto.orderbook.BookRequest, 77 | * !proto.orderbook.Summary>} 78 | */ 79 | const methodDescriptor_OrderbookAggregator_BookSummary = new grpc.web.MethodDescriptor( 80 | '/orderbook.OrderbookAggregator/BookSummary', 81 | grpc.web.MethodType.SERVER_STREAMING, 82 | proto.orderbook.BookRequest, 83 | proto.orderbook.Summary, 84 | /** 85 | * @param {!proto.orderbook.BookRequest} request 86 | * @return {!Uint8Array} 87 | */ 88 | function(request) { 89 | return request.serializeBinary(); 90 | }, 91 | proto.orderbook.Summary.deserializeBinary 92 | ); 93 | 94 | 95 | /** 96 | * @param {!proto.orderbook.BookRequest} request The request proto 97 | * @param {?Object=} metadata User defined 98 | * call metadata 99 | * @return {!grpc.web.ClientReadableStream} 100 | * The XHR Node Readable Stream 101 | */ 102 | proto.orderbook.OrderbookAggregatorClient.prototype.bookSummary = 103 | function(request, metadata) { 104 | return this.client_.serverStreaming(this.hostname_ + 105 | '/orderbook.OrderbookAggregator/BookSummary', 106 | request, 107 | metadata || {}, 108 | methodDescriptor_OrderbookAggregator_BookSummary); 109 | }; 110 | 111 | 112 | /** 113 | * @param {!proto.orderbook.BookRequest} request The request proto 114 | * @param {?Object=} metadata User defined 115 | * call metadata 116 | * @return {!grpc.web.ClientReadableStream} 117 | * The XHR Node Readable Stream 118 | */ 119 | proto.orderbook.OrderbookAggregatorPromiseClient.prototype.bookSummary = 120 | function(request, metadata) { 121 | return this.client_.serverStreaming(this.hostname_ + 122 | '/orderbook.OrderbookAggregator/BookSummary', 123 | request, 124 | metadata || {}, 125 | methodDescriptor_OrderbookAggregator_BookSummary); 126 | }; 127 | 128 | 129 | /** 130 | * @const 131 | * @type {!grpc.web.MethodDescriptor< 132 | * !proto.orderbook.Empty, 133 | * !proto.orderbook.ServiceInfo>} 134 | */ 135 | const methodDescriptor_OrderbookAggregator_Info = new grpc.web.MethodDescriptor( 136 | '/orderbook.OrderbookAggregator/Info', 137 | grpc.web.MethodType.UNARY, 138 | proto.orderbook.Empty, 139 | proto.orderbook.ServiceInfo, 140 | /** 141 | * @param {!proto.orderbook.Empty} request 142 | * @return {!Uint8Array} 143 | */ 144 | function(request) { 145 | return request.serializeBinary(); 146 | }, 147 | proto.orderbook.ServiceInfo.deserializeBinary 148 | ); 149 | 150 | 151 | /** 152 | * @param {!proto.orderbook.Empty} request The 153 | * request proto 154 | * @param {?Object} metadata User defined 155 | * call metadata 156 | * @param {function(?grpc.web.RpcError, ?proto.orderbook.ServiceInfo)} 157 | * callback The callback function(error, response) 158 | * @return {!grpc.web.ClientReadableStream|undefined} 159 | * The XHR Node Readable Stream 160 | */ 161 | proto.orderbook.OrderbookAggregatorClient.prototype.info = 162 | function(request, metadata, callback) { 163 | return this.client_.rpcCall(this.hostname_ + 164 | '/orderbook.OrderbookAggregator/Info', 165 | request, 166 | metadata || {}, 167 | methodDescriptor_OrderbookAggregator_Info, 168 | callback); 169 | }; 170 | 171 | 172 | /** 173 | * @param {!proto.orderbook.Empty} request The 174 | * request proto 175 | * @param {?Object=} metadata User defined 176 | * call metadata 177 | * @return {!Promise} 178 | * Promise that resolves to the response 179 | */ 180 | proto.orderbook.OrderbookAggregatorPromiseClient.prototype.info = 181 | function(request, metadata) { 182 | return this.client_.unaryCall(this.hostname_ + 183 | '/orderbook.OrderbookAggregator/Info', 184 | request, 185 | metadata || {}, 186 | methodDescriptor_OrderbookAggregator_Info); 187 | }; 188 | 189 | 190 | module.exports = proto.orderbook; 191 | 192 | -------------------------------------------------------------------------------- /web/proto/orderbook_pb.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileoverview 3 | * @enhanceable 4 | * @suppress {messageConventions} JS Compiler reports an error if a variable or 5 | * field starts with 'MSG_' and isn't a translatable message. 6 | * @public 7 | */ 8 | // GENERATED CODE -- DO NOT EDIT! 9 | 10 | var jspb = require('google-protobuf'); 11 | var goog = jspb; 12 | var global = Function('return this')(); 13 | 14 | goog.exportSymbol('proto.orderbook.BookRequest', null, global); 15 | goog.exportSymbol('proto.orderbook.Empty', null, global); 16 | goog.exportSymbol('proto.orderbook.Level', null, global); 17 | goog.exportSymbol('proto.orderbook.ServiceInfo', null, global); 18 | goog.exportSymbol('proto.orderbook.Summary', null, global); 19 | 20 | /** 21 | * Generated by JsPbCodeGenerator. 22 | * @param {Array=} opt_data Optional initial data array, typically from a 23 | * server response, or constructed directly in Javascript. The array is used 24 | * in place and becomes part of the constructed object. It is not cloned. 25 | * If no data is provided, the constructed object will be empty, but still 26 | * valid. 27 | * @extends {jspb.Message} 28 | * @constructor 29 | */ 30 | proto.orderbook.Empty = function(opt_data) { 31 | jspb.Message.initialize(this, opt_data, 0, -1, null, null); 32 | }; 33 | goog.inherits(proto.orderbook.Empty, jspb.Message); 34 | if (goog.DEBUG && !COMPILED) { 35 | proto.orderbook.Empty.displayName = 'proto.orderbook.Empty'; 36 | } 37 | 38 | 39 | if (jspb.Message.GENERATE_TO_OBJECT) { 40 | /** 41 | * Creates an object representation of this proto suitable for use in Soy templates. 42 | * Field names that are reserved in JavaScript and will be renamed to pb_name. 43 | * To access a reserved field use, foo.pb_, eg, foo.pb_default. 44 | * For the list of reserved names please see: 45 | * com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS. 46 | * @param {boolean=} opt_includeInstance Whether to include the JSPB instance 47 | * for transitional soy proto support: http://goto/soy-param-migration 48 | * @return {!Object} 49 | */ 50 | proto.orderbook.Empty.prototype.toObject = function(opt_includeInstance) { 51 | return proto.orderbook.Empty.toObject(opt_includeInstance, this); 52 | }; 53 | 54 | 55 | /** 56 | * Static version of the {@see toObject} method. 57 | * @param {boolean|undefined} includeInstance Whether to include the JSPB 58 | * instance for transitional soy proto support: 59 | * http://goto/soy-param-migration 60 | * @param {!proto.orderbook.Empty} msg The msg instance to transform. 61 | * @return {!Object} 62 | * @suppress {unusedLocalVariables} f is only used for nested messages 63 | */ 64 | proto.orderbook.Empty.toObject = function(includeInstance, msg) { 65 | var f, obj = { 66 | pair: jspb.Message.getFieldWithDefault(msg, 1, "") 67 | }; 68 | 69 | if (includeInstance) { 70 | obj.$jspbMessageInstance = msg; 71 | } 72 | return obj; 73 | }; 74 | } 75 | 76 | 77 | /** 78 | * Deserializes binary data (in protobuf wire format). 79 | * @param {jspb.ByteSource} bytes The bytes to deserialize. 80 | * @return {!proto.orderbook.Empty} 81 | */ 82 | proto.orderbook.Empty.deserializeBinary = function(bytes) { 83 | var reader = new jspb.BinaryReader(bytes); 84 | var msg = new proto.orderbook.Empty; 85 | return proto.orderbook.Empty.deserializeBinaryFromReader(msg, reader); 86 | }; 87 | 88 | 89 | /** 90 | * Deserializes binary data (in protobuf wire format) from the 91 | * given reader into the given message object. 92 | * @param {!proto.orderbook.Empty} msg The message object to deserialize into. 93 | * @param {!jspb.BinaryReader} reader The BinaryReader to use. 94 | * @return {!proto.orderbook.Empty} 95 | */ 96 | proto.orderbook.Empty.deserializeBinaryFromReader = function(msg, reader) { 97 | while (reader.nextField()) { 98 | if (reader.isEndGroup()) { 99 | break; 100 | } 101 | var field = reader.getFieldNumber(); 102 | switch (field) { 103 | case 1: 104 | var value = /** @type {string} */ (reader.readString()); 105 | msg.setPair(value); 106 | break; 107 | default: 108 | reader.skipField(); 109 | break; 110 | } 111 | } 112 | return msg; 113 | }; 114 | 115 | 116 | /** 117 | * Serializes the message to binary data (in protobuf wire format). 118 | * @return {!Uint8Array} 119 | */ 120 | proto.orderbook.Empty.prototype.serializeBinary = function() { 121 | var writer = new jspb.BinaryWriter(); 122 | proto.orderbook.Empty.serializeBinaryToWriter(this, writer); 123 | return writer.getResultBuffer(); 124 | }; 125 | 126 | 127 | /** 128 | * Serializes the given message to binary data (in protobuf wire 129 | * format), writing to the given BinaryWriter. 130 | * @param {!proto.orderbook.Empty} message 131 | * @param {!jspb.BinaryWriter} writer 132 | * @suppress {unusedLocalVariables} f is only used for nested messages 133 | */ 134 | proto.orderbook.Empty.serializeBinaryToWriter = function(message, writer) { 135 | var f = undefined; 136 | f = message.getPair(); 137 | if (f.length > 0) { 138 | writer.writeString( 139 | 1, 140 | f 141 | ); 142 | } 143 | }; 144 | 145 | 146 | /** 147 | * optional string pair = 1; 148 | * @return {string} 149 | */ 150 | proto.orderbook.Empty.prototype.getPair = function() { 151 | return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); 152 | }; 153 | 154 | 155 | /** @param {string} value */ 156 | proto.orderbook.Empty.prototype.setPair = function(value) { 157 | jspb.Message.setProto3StringField(this, 1, value); 158 | }; 159 | 160 | 161 | 162 | /** 163 | * Generated by JsPbCodeGenerator. 164 | * @param {Array=} opt_data Optional initial data array, typically from a 165 | * server response, or constructed directly in Javascript. The array is used 166 | * in place and becomes part of the constructed object. It is not cloned. 167 | * If no data is provided, the constructed object will be empty, but still 168 | * valid. 169 | * @extends {jspb.Message} 170 | * @constructor 171 | */ 172 | proto.orderbook.BookRequest = function(opt_data) { 173 | jspb.Message.initialize(this, opt_data, 0, -1, null, null); 174 | }; 175 | goog.inherits(proto.orderbook.BookRequest, jspb.Message); 176 | if (goog.DEBUG && !COMPILED) { 177 | proto.orderbook.BookRequest.displayName = 'proto.orderbook.BookRequest'; 178 | } 179 | 180 | 181 | if (jspb.Message.GENERATE_TO_OBJECT) { 182 | /** 183 | * Creates an object representation of this proto suitable for use in Soy templates. 184 | * Field names that are reserved in JavaScript and will be renamed to pb_name. 185 | * To access a reserved field use, foo.pb_, eg, foo.pb_default. 186 | * For the list of reserved names please see: 187 | * com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS. 188 | * @param {boolean=} opt_includeInstance Whether to include the JSPB instance 189 | * for transitional soy proto support: http://goto/soy-param-migration 190 | * @return {!Object} 191 | */ 192 | proto.orderbook.BookRequest.prototype.toObject = function(opt_includeInstance) { 193 | return proto.orderbook.BookRequest.toObject(opt_includeInstance, this); 194 | }; 195 | 196 | 197 | /** 198 | * Static version of the {@see toObject} method. 199 | * @param {boolean|undefined} includeInstance Whether to include the JSPB 200 | * instance for transitional soy proto support: 201 | * http://goto/soy-param-migration 202 | * @param {!proto.orderbook.BookRequest} msg The msg instance to transform. 203 | * @return {!Object} 204 | * @suppress {unusedLocalVariables} f is only used for nested messages 205 | */ 206 | proto.orderbook.BookRequest.toObject = function(includeInstance, msg) { 207 | var f, obj = { 208 | pair: jspb.Message.getFieldWithDefault(msg, 1, "") 209 | }; 210 | 211 | if (includeInstance) { 212 | obj.$jspbMessageInstance = msg; 213 | } 214 | return obj; 215 | }; 216 | } 217 | 218 | 219 | /** 220 | * Deserializes binary data (in protobuf wire format). 221 | * @param {jspb.ByteSource} bytes The bytes to deserialize. 222 | * @return {!proto.orderbook.BookRequest} 223 | */ 224 | proto.orderbook.BookRequest.deserializeBinary = function(bytes) { 225 | var reader = new jspb.BinaryReader(bytes); 226 | var msg = new proto.orderbook.BookRequest; 227 | return proto.orderbook.BookRequest.deserializeBinaryFromReader(msg, reader); 228 | }; 229 | 230 | 231 | /** 232 | * Deserializes binary data (in protobuf wire format) from the 233 | * given reader into the given message object. 234 | * @param {!proto.orderbook.BookRequest} msg The message object to deserialize into. 235 | * @param {!jspb.BinaryReader} reader The BinaryReader to use. 236 | * @return {!proto.orderbook.BookRequest} 237 | */ 238 | proto.orderbook.BookRequest.deserializeBinaryFromReader = function(msg, reader) { 239 | while (reader.nextField()) { 240 | if (reader.isEndGroup()) { 241 | break; 242 | } 243 | var field = reader.getFieldNumber(); 244 | switch (field) { 245 | case 1: 246 | var value = /** @type {string} */ (reader.readString()); 247 | msg.setPair(value); 248 | break; 249 | default: 250 | reader.skipField(); 251 | break; 252 | } 253 | } 254 | return msg; 255 | }; 256 | 257 | 258 | /** 259 | * Serializes the message to binary data (in protobuf wire format). 260 | * @return {!Uint8Array} 261 | */ 262 | proto.orderbook.BookRequest.prototype.serializeBinary = function() { 263 | var writer = new jspb.BinaryWriter(); 264 | proto.orderbook.BookRequest.serializeBinaryToWriter(this, writer); 265 | return writer.getResultBuffer(); 266 | }; 267 | 268 | 269 | /** 270 | * Serializes the given message to binary data (in protobuf wire 271 | * format), writing to the given BinaryWriter. 272 | * @param {!proto.orderbook.BookRequest} message 273 | * @param {!jspb.BinaryWriter} writer 274 | * @suppress {unusedLocalVariables} f is only used for nested messages 275 | */ 276 | proto.orderbook.BookRequest.serializeBinaryToWriter = function(message, writer) { 277 | var f = undefined; 278 | f = message.getPair(); 279 | if (f.length > 0) { 280 | writer.writeString( 281 | 1, 282 | f 283 | ); 284 | } 285 | }; 286 | 287 | 288 | /** 289 | * optional string pair = 1; 290 | * @return {string} 291 | */ 292 | proto.orderbook.BookRequest.prototype.getPair = function() { 293 | return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); 294 | }; 295 | 296 | 297 | /** @param {string} value */ 298 | proto.orderbook.BookRequest.prototype.setPair = function(value) { 299 | jspb.Message.setProto3StringField(this, 1, value); 300 | }; 301 | 302 | 303 | 304 | /** 305 | * Generated by JsPbCodeGenerator. 306 | * @param {Array=} opt_data Optional initial data array, typically from a 307 | * server response, or constructed directly in Javascript. The array is used 308 | * in place and becomes part of the constructed object. It is not cloned. 309 | * If no data is provided, the constructed object will be empty, but still 310 | * valid. 311 | * @extends {jspb.Message} 312 | * @constructor 313 | */ 314 | proto.orderbook.ServiceInfo = function(opt_data) { 315 | jspb.Message.initialize(this, opt_data, 0, -1, proto.orderbook.ServiceInfo.repeatedFields_, null); 316 | }; 317 | goog.inherits(proto.orderbook.ServiceInfo, jspb.Message); 318 | if (goog.DEBUG && !COMPILED) { 319 | proto.orderbook.ServiceInfo.displayName = 'proto.orderbook.ServiceInfo'; 320 | } 321 | /** 322 | * List of repeated fields within this message type. 323 | * @private {!Array} 324 | * @const 325 | */ 326 | proto.orderbook.ServiceInfo.repeatedFields_ = [1]; 327 | 328 | 329 | 330 | if (jspb.Message.GENERATE_TO_OBJECT) { 331 | /** 332 | * Creates an object representation of this proto suitable for use in Soy templates. 333 | * Field names that are reserved in JavaScript and will be renamed to pb_name. 334 | * To access a reserved field use, foo.pb_, eg, foo.pb_default. 335 | * For the list of reserved names please see: 336 | * com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS. 337 | * @param {boolean=} opt_includeInstance Whether to include the JSPB instance 338 | * for transitional soy proto support: http://goto/soy-param-migration 339 | * @return {!Object} 340 | */ 341 | proto.orderbook.ServiceInfo.prototype.toObject = function(opt_includeInstance) { 342 | return proto.orderbook.ServiceInfo.toObject(opt_includeInstance, this); 343 | }; 344 | 345 | 346 | /** 347 | * Static version of the {@see toObject} method. 348 | * @param {boolean|undefined} includeInstance Whether to include the JSPB 349 | * instance for transitional soy proto support: 350 | * http://goto/soy-param-migration 351 | * @param {!proto.orderbook.ServiceInfo} msg The msg instance to transform. 352 | * @return {!Object} 353 | * @suppress {unusedLocalVariables} f is only used for nested messages 354 | */ 355 | proto.orderbook.ServiceInfo.toObject = function(includeInstance, msg) { 356 | var f, obj = { 357 | pairsList: jspb.Message.getRepeatedField(msg, 1), 358 | maxDepth: jspb.Message.getFieldWithDefault(msg, 2, 0) 359 | }; 360 | 361 | if (includeInstance) { 362 | obj.$jspbMessageInstance = msg; 363 | } 364 | return obj; 365 | }; 366 | } 367 | 368 | 369 | /** 370 | * Deserializes binary data (in protobuf wire format). 371 | * @param {jspb.ByteSource} bytes The bytes to deserialize. 372 | * @return {!proto.orderbook.ServiceInfo} 373 | */ 374 | proto.orderbook.ServiceInfo.deserializeBinary = function(bytes) { 375 | var reader = new jspb.BinaryReader(bytes); 376 | var msg = new proto.orderbook.ServiceInfo; 377 | return proto.orderbook.ServiceInfo.deserializeBinaryFromReader(msg, reader); 378 | }; 379 | 380 | 381 | /** 382 | * Deserializes binary data (in protobuf wire format) from the 383 | * given reader into the given message object. 384 | * @param {!proto.orderbook.ServiceInfo} msg The message object to deserialize into. 385 | * @param {!jspb.BinaryReader} reader The BinaryReader to use. 386 | * @return {!proto.orderbook.ServiceInfo} 387 | */ 388 | proto.orderbook.ServiceInfo.deserializeBinaryFromReader = function(msg, reader) { 389 | while (reader.nextField()) { 390 | if (reader.isEndGroup()) { 391 | break; 392 | } 393 | var field = reader.getFieldNumber(); 394 | switch (field) { 395 | case 1: 396 | var value = /** @type {string} */ (reader.readString()); 397 | msg.addPairs(value); 398 | break; 399 | case 2: 400 | var value = /** @type {number} */ (reader.readUint64()); 401 | msg.setMaxDepth(value); 402 | break; 403 | default: 404 | reader.skipField(); 405 | break; 406 | } 407 | } 408 | return msg; 409 | }; 410 | 411 | 412 | /** 413 | * Serializes the message to binary data (in protobuf wire format). 414 | * @return {!Uint8Array} 415 | */ 416 | proto.orderbook.ServiceInfo.prototype.serializeBinary = function() { 417 | var writer = new jspb.BinaryWriter(); 418 | proto.orderbook.ServiceInfo.serializeBinaryToWriter(this, writer); 419 | return writer.getResultBuffer(); 420 | }; 421 | 422 | 423 | /** 424 | * Serializes the given message to binary data (in protobuf wire 425 | * format), writing to the given BinaryWriter. 426 | * @param {!proto.orderbook.ServiceInfo} message 427 | * @param {!jspb.BinaryWriter} writer 428 | * @suppress {unusedLocalVariables} f is only used for nested messages 429 | */ 430 | proto.orderbook.ServiceInfo.serializeBinaryToWriter = function(message, writer) { 431 | var f = undefined; 432 | f = message.getPairsList(); 433 | if (f.length > 0) { 434 | writer.writeRepeatedString( 435 | 1, 436 | f 437 | ); 438 | } 439 | f = message.getMaxDepth(); 440 | if (f !== 0) { 441 | writer.writeUint64( 442 | 2, 443 | f 444 | ); 445 | } 446 | }; 447 | 448 | 449 | /** 450 | * repeated string pairs = 1; 451 | * @return {!Array} 452 | */ 453 | proto.orderbook.ServiceInfo.prototype.getPairsList = function() { 454 | return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); 455 | }; 456 | 457 | 458 | /** @param {!Array} value */ 459 | proto.orderbook.ServiceInfo.prototype.setPairsList = function(value) { 460 | jspb.Message.setField(this, 1, value || []); 461 | }; 462 | 463 | 464 | /** 465 | * @param {!string} value 466 | * @param {number=} opt_index 467 | */ 468 | proto.orderbook.ServiceInfo.prototype.addPairs = function(value, opt_index) { 469 | jspb.Message.addToRepeatedField(this, 1, value, opt_index); 470 | }; 471 | 472 | 473 | proto.orderbook.ServiceInfo.prototype.clearPairsList = function() { 474 | this.setPairsList([]); 475 | }; 476 | 477 | 478 | /** 479 | * optional uint64 max_depth = 2; 480 | * @return {number} 481 | */ 482 | proto.orderbook.ServiceInfo.prototype.getMaxDepth = function() { 483 | return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); 484 | }; 485 | 486 | 487 | /** @param {number} value */ 488 | proto.orderbook.ServiceInfo.prototype.setMaxDepth = function(value) { 489 | jspb.Message.setProto3IntField(this, 2, value); 490 | }; 491 | 492 | 493 | 494 | /** 495 | * Generated by JsPbCodeGenerator. 496 | * @param {Array=} opt_data Optional initial data array, typically from a 497 | * server response, or constructed directly in Javascript. The array is used 498 | * in place and becomes part of the constructed object. It is not cloned. 499 | * If no data is provided, the constructed object will be empty, but still 500 | * valid. 501 | * @extends {jspb.Message} 502 | * @constructor 503 | */ 504 | proto.orderbook.Summary = function(opt_data) { 505 | jspb.Message.initialize(this, opt_data, 0, -1, proto.orderbook.Summary.repeatedFields_, null); 506 | }; 507 | goog.inherits(proto.orderbook.Summary, jspb.Message); 508 | if (goog.DEBUG && !COMPILED) { 509 | proto.orderbook.Summary.displayName = 'proto.orderbook.Summary'; 510 | } 511 | /** 512 | * List of repeated fields within this message type. 513 | * @private {!Array} 514 | * @const 515 | */ 516 | proto.orderbook.Summary.repeatedFields_ = [2,3]; 517 | 518 | 519 | 520 | if (jspb.Message.GENERATE_TO_OBJECT) { 521 | /** 522 | * Creates an object representation of this proto suitable for use in Soy templates. 523 | * Field names that are reserved in JavaScript and will be renamed to pb_name. 524 | * To access a reserved field use, foo.pb_, eg, foo.pb_default. 525 | * For the list of reserved names please see: 526 | * com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS. 527 | * @param {boolean=} opt_includeInstance Whether to include the JSPB instance 528 | * for transitional soy proto support: http://goto/soy-param-migration 529 | * @return {!Object} 530 | */ 531 | proto.orderbook.Summary.prototype.toObject = function(opt_includeInstance) { 532 | return proto.orderbook.Summary.toObject(opt_includeInstance, this); 533 | }; 534 | 535 | 536 | /** 537 | * Static version of the {@see toObject} method. 538 | * @param {boolean|undefined} includeInstance Whether to include the JSPB 539 | * instance for transitional soy proto support: 540 | * http://goto/soy-param-migration 541 | * @param {!proto.orderbook.Summary} msg The msg instance to transform. 542 | * @return {!Object} 543 | * @suppress {unusedLocalVariables} f is only used for nested messages 544 | */ 545 | proto.orderbook.Summary.toObject = function(includeInstance, msg) { 546 | var f, obj = { 547 | spread: +jspb.Message.getFieldWithDefault(msg, 1, 0.0), 548 | bidsList: jspb.Message.toObjectList(msg.getBidsList(), 549 | proto.orderbook.Level.toObject, includeInstance), 550 | asksList: jspb.Message.toObjectList(msg.getAsksList(), 551 | proto.orderbook.Level.toObject, includeInstance) 552 | }; 553 | 554 | if (includeInstance) { 555 | obj.$jspbMessageInstance = msg; 556 | } 557 | return obj; 558 | }; 559 | } 560 | 561 | 562 | /** 563 | * Deserializes binary data (in protobuf wire format). 564 | * @param {jspb.ByteSource} bytes The bytes to deserialize. 565 | * @return {!proto.orderbook.Summary} 566 | */ 567 | proto.orderbook.Summary.deserializeBinary = function(bytes) { 568 | var reader = new jspb.BinaryReader(bytes); 569 | var msg = new proto.orderbook.Summary; 570 | return proto.orderbook.Summary.deserializeBinaryFromReader(msg, reader); 571 | }; 572 | 573 | 574 | /** 575 | * Deserializes binary data (in protobuf wire format) from the 576 | * given reader into the given message object. 577 | * @param {!proto.orderbook.Summary} msg The message object to deserialize into. 578 | * @param {!jspb.BinaryReader} reader The BinaryReader to use. 579 | * @return {!proto.orderbook.Summary} 580 | */ 581 | proto.orderbook.Summary.deserializeBinaryFromReader = function(msg, reader) { 582 | while (reader.nextField()) { 583 | if (reader.isEndGroup()) { 584 | break; 585 | } 586 | var field = reader.getFieldNumber(); 587 | switch (field) { 588 | case 1: 589 | var value = /** @type {number} */ (reader.readDouble()); 590 | msg.setSpread(value); 591 | break; 592 | case 2: 593 | var value = new proto.orderbook.Level; 594 | reader.readMessage(value,proto.orderbook.Level.deserializeBinaryFromReader); 595 | msg.addBids(value); 596 | break; 597 | case 3: 598 | var value = new proto.orderbook.Level; 599 | reader.readMessage(value,proto.orderbook.Level.deserializeBinaryFromReader); 600 | msg.addAsks(value); 601 | break; 602 | default: 603 | reader.skipField(); 604 | break; 605 | } 606 | } 607 | return msg; 608 | }; 609 | 610 | 611 | /** 612 | * Serializes the message to binary data (in protobuf wire format). 613 | * @return {!Uint8Array} 614 | */ 615 | proto.orderbook.Summary.prototype.serializeBinary = function() { 616 | var writer = new jspb.BinaryWriter(); 617 | proto.orderbook.Summary.serializeBinaryToWriter(this, writer); 618 | return writer.getResultBuffer(); 619 | }; 620 | 621 | 622 | /** 623 | * Serializes the given message to binary data (in protobuf wire 624 | * format), writing to the given BinaryWriter. 625 | * @param {!proto.orderbook.Summary} message 626 | * @param {!jspb.BinaryWriter} writer 627 | * @suppress {unusedLocalVariables} f is only used for nested messages 628 | */ 629 | proto.orderbook.Summary.serializeBinaryToWriter = function(message, writer) { 630 | var f = undefined; 631 | f = message.getSpread(); 632 | if (f !== 0.0) { 633 | writer.writeDouble( 634 | 1, 635 | f 636 | ); 637 | } 638 | f = message.getBidsList(); 639 | if (f.length > 0) { 640 | writer.writeRepeatedMessage( 641 | 2, 642 | f, 643 | proto.orderbook.Level.serializeBinaryToWriter 644 | ); 645 | } 646 | f = message.getAsksList(); 647 | if (f.length > 0) { 648 | writer.writeRepeatedMessage( 649 | 3, 650 | f, 651 | proto.orderbook.Level.serializeBinaryToWriter 652 | ); 653 | } 654 | }; 655 | 656 | 657 | /** 658 | * optional double spread = 1; 659 | * @return {number} 660 | */ 661 | proto.orderbook.Summary.prototype.getSpread = function() { 662 | return /** @type {number} */ (+jspb.Message.getFieldWithDefault(this, 1, 0.0)); 663 | }; 664 | 665 | 666 | /** @param {number} value */ 667 | proto.orderbook.Summary.prototype.setSpread = function(value) { 668 | jspb.Message.setProto3FloatField(this, 1, value); 669 | }; 670 | 671 | 672 | /** 673 | * repeated Level bids = 2; 674 | * @return {!Array} 675 | */ 676 | proto.orderbook.Summary.prototype.getBidsList = function() { 677 | return /** @type{!Array} */ ( 678 | jspb.Message.getRepeatedWrapperField(this, proto.orderbook.Level, 2)); 679 | }; 680 | 681 | 682 | /** @param {!Array} value */ 683 | proto.orderbook.Summary.prototype.setBidsList = function(value) { 684 | jspb.Message.setRepeatedWrapperField(this, 2, value); 685 | }; 686 | 687 | 688 | /** 689 | * @param {!proto.orderbook.Level=} opt_value 690 | * @param {number=} opt_index 691 | * @return {!proto.orderbook.Level} 692 | */ 693 | proto.orderbook.Summary.prototype.addBids = function(opt_value, opt_index) { 694 | return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.orderbook.Level, opt_index); 695 | }; 696 | 697 | 698 | proto.orderbook.Summary.prototype.clearBidsList = function() { 699 | this.setBidsList([]); 700 | }; 701 | 702 | 703 | /** 704 | * repeated Level asks = 3; 705 | * @return {!Array} 706 | */ 707 | proto.orderbook.Summary.prototype.getAsksList = function() { 708 | return /** @type{!Array} */ ( 709 | jspb.Message.getRepeatedWrapperField(this, proto.orderbook.Level, 3)); 710 | }; 711 | 712 | 713 | /** @param {!Array} value */ 714 | proto.orderbook.Summary.prototype.setAsksList = function(value) { 715 | jspb.Message.setRepeatedWrapperField(this, 3, value); 716 | }; 717 | 718 | 719 | /** 720 | * @param {!proto.orderbook.Level=} opt_value 721 | * @param {number=} opt_index 722 | * @return {!proto.orderbook.Level} 723 | */ 724 | proto.orderbook.Summary.prototype.addAsks = function(opt_value, opt_index) { 725 | return jspb.Message.addToRepeatedWrapperField(this, 3, opt_value, proto.orderbook.Level, opt_index); 726 | }; 727 | 728 | 729 | proto.orderbook.Summary.prototype.clearAsksList = function() { 730 | this.setAsksList([]); 731 | }; 732 | 733 | 734 | 735 | /** 736 | * Generated by JsPbCodeGenerator. 737 | * @param {Array=} opt_data Optional initial data array, typically from a 738 | * server response, or constructed directly in Javascript. The array is used 739 | * in place and becomes part of the constructed object. It is not cloned. 740 | * If no data is provided, the constructed object will be empty, but still 741 | * valid. 742 | * @extends {jspb.Message} 743 | * @constructor 744 | */ 745 | proto.orderbook.Level = function(opt_data) { 746 | jspb.Message.initialize(this, opt_data, 0, -1, null, null); 747 | }; 748 | goog.inherits(proto.orderbook.Level, jspb.Message); 749 | if (goog.DEBUG && !COMPILED) { 750 | proto.orderbook.Level.displayName = 'proto.orderbook.Level'; 751 | } 752 | 753 | 754 | if (jspb.Message.GENERATE_TO_OBJECT) { 755 | /** 756 | * Creates an object representation of this proto suitable for use in Soy templates. 757 | * Field names that are reserved in JavaScript and will be renamed to pb_name. 758 | * To access a reserved field use, foo.pb_, eg, foo.pb_default. 759 | * For the list of reserved names please see: 760 | * com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS. 761 | * @param {boolean=} opt_includeInstance Whether to include the JSPB instance 762 | * for transitional soy proto support: http://goto/soy-param-migration 763 | * @return {!Object} 764 | */ 765 | proto.orderbook.Level.prototype.toObject = function(opt_includeInstance) { 766 | return proto.orderbook.Level.toObject(opt_includeInstance, this); 767 | }; 768 | 769 | 770 | /** 771 | * Static version of the {@see toObject} method. 772 | * @param {boolean|undefined} includeInstance Whether to include the JSPB 773 | * instance for transitional soy proto support: 774 | * http://goto/soy-param-migration 775 | * @param {!proto.orderbook.Level} msg The msg instance to transform. 776 | * @return {!Object} 777 | * @suppress {unusedLocalVariables} f is only used for nested messages 778 | */ 779 | proto.orderbook.Level.toObject = function(includeInstance, msg) { 780 | var f, obj = { 781 | exchange: jspb.Message.getFieldWithDefault(msg, 1, ""), 782 | price: +jspb.Message.getFieldWithDefault(msg, 2, 0.0), 783 | amount: +jspb.Message.getFieldWithDefault(msg, 3, 0.0) 784 | }; 785 | 786 | if (includeInstance) { 787 | obj.$jspbMessageInstance = msg; 788 | } 789 | return obj; 790 | }; 791 | } 792 | 793 | 794 | /** 795 | * Deserializes binary data (in protobuf wire format). 796 | * @param {jspb.ByteSource} bytes The bytes to deserialize. 797 | * @return {!proto.orderbook.Level} 798 | */ 799 | proto.orderbook.Level.deserializeBinary = function(bytes) { 800 | var reader = new jspb.BinaryReader(bytes); 801 | var msg = new proto.orderbook.Level; 802 | return proto.orderbook.Level.deserializeBinaryFromReader(msg, reader); 803 | }; 804 | 805 | 806 | /** 807 | * Deserializes binary data (in protobuf wire format) from the 808 | * given reader into the given message object. 809 | * @param {!proto.orderbook.Level} msg The message object to deserialize into. 810 | * @param {!jspb.BinaryReader} reader The BinaryReader to use. 811 | * @return {!proto.orderbook.Level} 812 | */ 813 | proto.orderbook.Level.deserializeBinaryFromReader = function(msg, reader) { 814 | while (reader.nextField()) { 815 | if (reader.isEndGroup()) { 816 | break; 817 | } 818 | var field = reader.getFieldNumber(); 819 | switch (field) { 820 | case 1: 821 | var value = /** @type {string} */ (reader.readString()); 822 | msg.setExchange(value); 823 | break; 824 | case 2: 825 | var value = /** @type {number} */ (reader.readDouble()); 826 | msg.setPrice(value); 827 | break; 828 | case 3: 829 | var value = /** @type {number} */ (reader.readDouble()); 830 | msg.setAmount(value); 831 | break; 832 | default: 833 | reader.skipField(); 834 | break; 835 | } 836 | } 837 | return msg; 838 | }; 839 | 840 | 841 | /** 842 | * Serializes the message to binary data (in protobuf wire format). 843 | * @return {!Uint8Array} 844 | */ 845 | proto.orderbook.Level.prototype.serializeBinary = function() { 846 | var writer = new jspb.BinaryWriter(); 847 | proto.orderbook.Level.serializeBinaryToWriter(this, writer); 848 | return writer.getResultBuffer(); 849 | }; 850 | 851 | 852 | /** 853 | * Serializes the given message to binary data (in protobuf wire 854 | * format), writing to the given BinaryWriter. 855 | * @param {!proto.orderbook.Level} message 856 | * @param {!jspb.BinaryWriter} writer 857 | * @suppress {unusedLocalVariables} f is only used for nested messages 858 | */ 859 | proto.orderbook.Level.serializeBinaryToWriter = function(message, writer) { 860 | var f = undefined; 861 | f = message.getExchange(); 862 | if (f.length > 0) { 863 | writer.writeString( 864 | 1, 865 | f 866 | ); 867 | } 868 | f = message.getPrice(); 869 | if (f !== 0.0) { 870 | writer.writeDouble( 871 | 2, 872 | f 873 | ); 874 | } 875 | f = message.getAmount(); 876 | if (f !== 0.0) { 877 | writer.writeDouble( 878 | 3, 879 | f 880 | ); 881 | } 882 | }; 883 | 884 | 885 | /** 886 | * optional string exchange = 1; 887 | * @return {string} 888 | */ 889 | proto.orderbook.Level.prototype.getExchange = function() { 890 | return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); 891 | }; 892 | 893 | 894 | /** @param {string} value */ 895 | proto.orderbook.Level.prototype.setExchange = function(value) { 896 | jspb.Message.setProto3StringField(this, 1, value); 897 | }; 898 | 899 | 900 | /** 901 | * optional double price = 2; 902 | * @return {number} 903 | */ 904 | proto.orderbook.Level.prototype.getPrice = function() { 905 | return /** @type {number} */ (+jspb.Message.getFieldWithDefault(this, 2, 0.0)); 906 | }; 907 | 908 | 909 | /** @param {number} value */ 910 | proto.orderbook.Level.prototype.setPrice = function(value) { 911 | jspb.Message.setProto3FloatField(this, 2, value); 912 | }; 913 | 914 | 915 | /** 916 | * optional double amount = 3; 917 | * @return {number} 918 | */ 919 | proto.orderbook.Level.prototype.getAmount = function() { 920 | return /** @type {number} */ (+jspb.Message.getFieldWithDefault(this, 3, 0.0)); 921 | }; 922 | 923 | 924 | /** @param {number} value */ 925 | proto.orderbook.Level.prototype.setAmount = function(value) { 926 | jspb.Message.setProto3FloatField(this, 3, value); 927 | }; 928 | 929 | 930 | goog.object.extend(exports, proto.orderbook); 931 | -------------------------------------------------------------------------------- /web/service.ts: -------------------------------------------------------------------------------- 1 | import { OrderbookAggregatorClient } from "./proto/orderbook_grpc_web_pb"; 2 | 3 | const { 4 | BookRequest, 5 | Summary, 6 | Level, 7 | Empty, 8 | } = require("./proto/orderbook_pb.js"); 9 | 10 | class GrpcService { 11 | cli: OrderbookAggregatorClient; 12 | 13 | constructor() { 14 | // @ts-ignore 15 | this.cli = new OrderbookAggregatorClient(STREAMING_URL, null, {}); 16 | } 17 | 18 | info(onData: any) { 19 | const request = new Empty(); 20 | this.cli.info(request, {}, (_: any, response: any) => { 21 | onData({ 22 | pairs: response.getPairsList(), 23 | }); 24 | }); 25 | } 26 | 27 | streamPair(pair: string, onData: any) { 28 | const request = new BookRequest(); 29 | request.setPair(pair); 30 | const stream = this.cli.bookSummary(request, {}); 31 | stream.on("data", (summary: typeof Summary) => { 32 | onData({ 33 | spread: summary.getSpread(), 34 | asks: summary.getAsksList().map(level_record("asks")), 35 | bids: summary.getBidsList().map(level_record("bids")), 36 | }); 37 | }); 38 | return stream; 39 | } 40 | } 41 | 42 | const level_record = 43 | (side: string) => 44 | (level: typeof Level): Record => ({ 45 | exchange: level.getExchange(), 46 | price: level.getPrice(), 47 | amount: level.getAmount(), 48 | group: `${level.getExchange()} - ${side}`, 49 | side, 50 | }); 51 | 52 | export default GrpcService; 53 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | const logger = require("console"); 3 | const path = require("path"); 4 | const webpack = require("webpack"); 5 | 6 | 7 | const STATIC_PATH = "/dist/"; 8 | const mode = 9 | process.env.NODE_ENV === "production" ? "production" : "development"; 10 | const PWD = process.cwd(); 11 | const resolvePath = (relativePath) => path.resolve(PWD, relativePath); 12 | const env = { 13 | process: JSON.stringify({}), 14 | STREAMING_URL: JSON.stringify( 15 | process.env.STREAMING_URL || "http://localhost:90" 16 | ), 17 | }; 18 | 19 | console.info(env); 20 | 21 | const config = { 22 | mode, 23 | entry: { 24 | block: "./web/Index.tsx", 25 | }, 26 | plugins: [ 27 | new webpack.DefinePlugin(env), 28 | ], 29 | output: { 30 | publicPath: STATIC_PATH, 31 | path: resolvePath(`.${STATIC_PATH}`), 32 | filename: "[name].js", 33 | chunkFilename: "[name].bundle.js", 34 | libraryTarget: "umd", 35 | }, 36 | devtool: "source-map", 37 | optimization: { 38 | minimize: mode === "production", 39 | }, 40 | resolve: { 41 | extensions: [".js", ".ts", ".jsx", ".tsx"], 42 | }, 43 | module: { 44 | rules: [ 45 | { 46 | test: /\.ts(x?)$/, 47 | exclude: [/node_modules/, /third_party/, /server/], 48 | use: { 49 | loader: "ts-loader", 50 | }, 51 | }, 52 | { enforce: "pre", test: /\.js$/, loader: "source-map-loader" }, 53 | { 54 | test: /\.(s?)css$/, 55 | use: ["style-loader", "css-loader"], 56 | //use: ["style-loader", "css-loader", "sass-loader"], 57 | }, 58 | { 59 | test: /\.(png|jpe?g|gif|woff|woff2|eot|ttf|svg)$/i, 60 | use: [ 61 | { 62 | loader: "file-loader", 63 | }, 64 | ], 65 | }, 66 | ], 67 | }, 68 | }; 69 | 70 | if (mode === "development") { 71 | logger.log("Looks like we are in development mode"); 72 | 73 | config.devServer = { 74 | port: 3000, 75 | hot: true, 76 | static: { 77 | directory: path.join(__dirname, 'web'), 78 | }, 79 | } 80 | } 81 | 82 | module.exports = config; 83 | --------------------------------------------------------------------------------