├── .dockerignore ├── .envrc.dev ├── .gitattributes ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── README.md ├── conf ├── clear_pm2_logs.sh ├── compress.sh ├── logrotate.sh ├── logrotate │ ├── logrotate.bbo.conf │ ├── logrotate.candlestick.conf │ ├── logrotate.funding_rate.conf │ ├── logrotate.l2_event.conf │ ├── logrotate.l2_snapshot.conf │ ├── logrotate.l2_topk.conf │ ├── logrotate.l3_event.conf │ ├── logrotate.open_interest.conf │ ├── logrotate.other.conf │ ├── logrotate.ticker.conf │ └── logrotate.trade.conf ├── pm2 │ ├── pm2.bbo.config.js │ ├── pm2.candlestick.config.js │ ├── pm2.funding_rate.config.js │ ├── pm2.l2_event.config.js │ ├── pm2.l2_snapshot.config.js │ ├── pm2.l2_topk.config.js │ ├── pm2.l3_event.config.js │ ├── pm2.open_interest.config.js │ ├── pm2.other.config.js │ ├── pm2.ticker.config.js │ └── pm2.trade.config.js ├── rclone.conf ├── run_crawlers.sh └── upload.sh └── src ├── lib.rs ├── main.rs ├── misc_crawlers ├── binance.rs ├── bitmex.rs ├── bybit.rs ├── coinbase_pro.rs ├── huobi.rs ├── mod.rs ├── okx.rs └── utils.rs └── writers ├── file_writer.rs └── mod.rs /.dockerignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.envrc.dev: -------------------------------------------------------------------------------- 1 | export https_proxy="socks5://127.0.0.1:9050" 2 | export DATA_DIR="/tmp/carbonbot" 3 | export RUST_BACKTRACE=1 4 | export REST_RETRY_COUNT=5 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-detectable=false 2 | *.sh linguist-detectable=false 3 | Dockerfile linguist-detectable=false 4 | 5 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Cargo build 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions-rs/toolchain@v1 12 | with: 13 | toolchain: nightly 14 | override: true 15 | - uses: actions-rs/cargo@v1 16 | with: 17 | command: build 18 | args: --release --all-features 19 | test: 20 | name: Cargo test 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v2 24 | - uses: actions-rs/toolchain@v1 25 | with: 26 | toolchain: nightly 27 | override: true 28 | - uses: actions-rs/cargo@v1 29 | with: 30 | command: test 31 | fmt: 32 | name: Cargo fmt 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/checkout@v2 36 | - uses: actions-rs/toolchain@v1 37 | with: 38 | toolchain: nightly 39 | override: true 40 | components: rustfmt 41 | - uses: actions-rs/cargo@v1 42 | with: 43 | command: fmt 44 | args: -- --check 45 | 46 | check: 47 | name: Cargo check 48 | runs-on: ubuntu-latest 49 | steps: 50 | - uses: actions/checkout@v2 51 | - uses: actions-rs/toolchain@v1 52 | with: 53 | toolchain: nightly 54 | override: true 55 | - uses: actions-rs/cargo@v1 56 | with: 57 | command: check 58 | 59 | clippy: 60 | name: Cargo clippy 61 | runs-on: ubuntu-latest 62 | steps: 63 | - uses: actions/checkout@v2 64 | - uses: actions-rs/toolchain@v1 65 | with: 66 | toolchain: nightly 67 | override: true 68 | components: clippy 69 | - uses: actions-rs/cargo@v1 70 | with: 71 | command: clippy 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "carbonbot" 3 | version = "2.12.3" 4 | authors = ["soulmachine "] 5 | edition = "2021" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | crypto-crawler = "4.7.8" 11 | crypto-market-type = "1.1.5" 12 | crypto-msg-type = "1.0.11" 13 | crypto-ws-client = "4.12.11" 14 | once_cell = "1.17.1" 15 | redis = "0.22.3" 16 | reopen = { version = "1.0.3", features = ["signals"] } 17 | serde_json = "1.0.93" 18 | signal-hook = { version = "0.3.15", features = ["extended-siginfo"] } 19 | log = "0.4.17" 20 | env_logger = "0.10.0" 21 | tokio = { version = "1.25.0", features = ["rt-multi-thread", "macros"] } 22 | 23 | [profile.release] 24 | lto = true 25 | strip = "debuginfo" 26 | codegen-units = 1 27 | opt-level="z" 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:latest AS builder 2 | 3 | RUN mkdir /project 4 | WORKDIR /project 5 | 6 | COPY ./Cargo.toml ./Cargo.toml 7 | COPY ./src/ ./src/ 8 | 9 | RUN apt -qy update && apt -qy install pkg-config libssl-dev \ 10 | && RUSTFLAGS="-C target-cpu=x86-64-v3" cargo build --release 11 | 12 | 13 | FROM node:bullseye-slim 14 | 15 | COPY --from=builder /project/target/release/carbonbot /usr/local/bin/carbonbot 16 | 17 | # procps provides the ps command, which is needed by pm2 18 | RUN apt-get -qy update && apt-get -qy --no-install-recommends install \ 19 | ca-certificates curl htop iputils-ping logrotate procps pigz sudo tree xz-utils \ 20 | && chown -R node:node /var/lib/logrotate/ \ 21 | && npm install pm2 -g --production \ 22 | && apt-get -qy install gzip unzip && curl https://rclone.org/install.sh | bash \ 23 | && echo "node ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers \ 24 | && apt-get -qy autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* 25 | 26 | # Install fixuid 27 | RUN ARCH="$(dpkg --print-architecture)" && \ 28 | curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.5.1/fixuid-0.5.1-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \ 29 | chown root:root /usr/local/bin/fixuid && \ 30 | chmod 4755 /usr/local/bin/fixuid && \ 31 | mkdir -p /etc/fixuid && \ 32 | printf "user: node\ngroup: node\npaths:\n - /home/node\n - /var/lib/logrotate/\n" > /etc/fixuid/config.yml 33 | 34 | COPY --chown=node:node ./conf/pm2/pm2.*.config.js /home/node/ 35 | COPY ./conf/logrotate/logrotate.*.conf /usr/local/etc/ 36 | COPY --chown=node:node ./conf/rclone.conf /home/node/.config/rclone/rclone.conf 37 | COPY ./conf/logrotate.sh /usr/local/bin/logrotate.sh 38 | COPY ./conf/compress.sh /usr/local/bin/compress.sh 39 | COPY ./conf/upload.sh /usr/local/bin/upload.sh 40 | 41 | ENV RUST_LOG "warn" 42 | ENV RUST_BACKTRACE 1 43 | 44 | VOLUME [ "/carbonbot_data" ] 45 | ENV DATA_DIR /carbonbot_data 46 | 47 | USER node:node 48 | ENV USER node 49 | WORKDIR /home/node 50 | 51 | ENTRYPOINT ["fixuid", "-q"] 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # carbonbot 2 | 3 | [![Discord](https://img.shields.io/discord/1043987684164649020?logo=discord)](https://discord.gg/Vych8DNZU2) 4 | ========== 5 | 6 | A CLI tool based on the crypto-crawler-rs library to crawl trade, level2, level3, ticker, funding rate, etc. 7 | 8 | ## 1. Sample Data Download 9 | 10 | ```bash 11 | aws s3 ls --request-payer requester s3://carbonbot/ 12 | aws s3 sync --request-payer requester s3://carbonbot/monthly/parsed . 13 | ``` 14 | 15 | The S3 bucket `s3://carbonbot` has **Requester Pays** enabled, so users only pay for the data transfer fee, while I pay for the data storage fee. 16 | 17 | ## 2. Run Crawlers 18 | 19 | Copy `conf/run_crawlers.sh` to somewhere, change `LOCAL_TMP_DIR` to a local SSD directory and `DEST_DIR` to a directory on a large disk, and run this shell script. Run `docker ps` and you'll see all crawlers are running! 20 | 21 | Use `tail -f file` to check files under `LOCAL_TMP_DIR`, you'll see data in realtime; watch the `DEST_DIR` dirctory, you'll see new files are moved from `LOCAL_TMP_DIR` to `DEST_DIR` every 15 minutes. 22 | 23 | ## 3. Output Destinations 24 | 25 | Crawlers running in the `ghcr.io/crypto-crawler/carbonbot:latest` container write data to the local temporary path `/carbonbot_data` first, then move data to multiple destinations every 15 minutes. 26 | 27 | Four kinds of destinations are supported: directory, AWS S3, MinIO and Redis. 28 | 29 | ### Directory 30 | 31 | To save data to a local directory or a NFS directory, users need to mount this directory into the docker container, and specify a `DEST_DIR` environment variable pointing to this directory. For example: 32 | 33 | ```bash 34 | docker run -d --name carbonbot-trade --restart always -v $YOUR_LOCAL_PATH:/carbonbot_data -v $DEST_DIR:/dest_dir -e DEST_DIR=/dest_dir -u "$(id -u):$(id -g)" ghcr.io/crypto-crawler/carbonbot:latest pm2-runtime start pm2.trade.config.js 35 | ``` 36 | 37 | ### AWS S3 38 | 39 | To upload data to AWS S3 automatically, uses need to specify three environment variables, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_S3_DIR`. For example: 40 | 41 | ```bash 42 | docker run -d --name carbonbot-trade --restart always -v $YOUR_LOCAL_PATH:/carbonbot_data -e AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY" -e AWS_SECRET_ACCESS_KEY="YOUR_SECRET_KEY" -e AWS_S3_DIR="s3://YOUR_BUCKET/path" -u "$(id -u):$(id -g)" ghcr.io/crypto-crawler/carbonbot:latest pm2-runtime start pm2.trade.config.js 43 | ``` 44 | 45 | Optionally, users can specify the `AWS_REGION` environment variable, see [Configuring the AWS SDK for Go 46 | ](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). 47 | 48 | ### MinIO 49 | 50 | To upload data to AWS S3 automatically, users need to specify three environment variables, `MINIO_ACCESS_KEY_ID`, `MINIO_SECRET_ACCESS_KEY`, `MINIO_ENDPOINT_URL` and `MINIO_DIR`. For example: 51 | 52 | ```bash 53 | docker run -d --name carbonbot-trade --restart always -v $YOUR_LOCAL_PATH:/carbonbot_data -e MINIO_ACCESS_KEY_ID="YOUR_ACCESS_KEY" -e MINIO_SECRET_ACCESS_KEY="YOUR_SECRET_KEY" -e MINIO_ENDPOINT_URL="http://ip:9000" -e MINIO_DIR="minio://YOUR_BUCKET/path" -u "$(id -u):$(id -g)" ghcr.io/crypto-crawler/carbonbot:latest pm2-runtime start pm2.trade.config.js 54 | ``` 55 | 56 | ### Redis 57 | 58 | To output data to Redis, users needs to specify a `REDIS_URL` environment variable. For example: 59 | 60 | ```bash 61 | docker run -d --name carbonbot-trade --restart always -v $YOUR_LOCAL_PATH:/carbonbot_data -e REDIS_URL=redis://172.17.0.1:6379 -u "$(id -u):$(id -g)" ghcr.io/crypto-crawler/carbonbot:latest pm2-runtime start pm2.trade.config.js 62 | ``` 63 | 64 | 65 | ## 4. Build 66 | 67 | ```bash 68 | docker build -t ghcr.io/crypto-crawler/carbonbot:latest . 69 | docker push ghcr.io/crypto-crawler/carbonbot:latest 70 | ``` 71 | -------------------------------------------------------------------------------- /conf/clear_pm2_logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | containers=$(docker ps --format '{{.Names}}' -f ancestor=ghcr.io/crypto-crawler/carbonbot:latest) 4 | 5 | echo "Cleaning up pm2 logs..." 6 | 7 | for container in ${containers[@]}; do 8 | echo "$container" 9 | docker exec -it "$container" bash -c "truncate -s 0 /home/node/.pm2/logs/*.log; pm2 reset all -s" 10 | done 11 | -------------------------------------------------------------------------------- /conf/compress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Linted by https://www.shellcheck.net/ 3 | 4 | msg_type=$1 # trade, l2_event, etc. 5 | 6 | if [[ -z "${DATA_DIR}" ]]; then 7 | echo "DATA_DIR must be set" >&2 8 | exit 1 9 | fi 10 | 11 | # Infinite while loop 12 | while : 13 | do 14 | # Find .json files older than 1 minute and compress them 15 | find "$DATA_DIR/$msg_type" -name "*.json" -type f -mmin +1 | xargs -r -n 1 pigz --best -f 16 | sleep 3 17 | done 18 | -------------------------------------------------------------------------------- /conf/logrotate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Linted by https://www.shellcheck.net/ 3 | 4 | # Inside Docker logrotate always run at first time, which is not expected. 5 | # This script is a thin wrapper around logrotate, to make it skip the first 6 | # run if timestamp is not at "*/15 * * * *" 7 | 8 | # https://unix.stackexchange.com/a/79372/40515 9 | minute=$(date +%-M) 10 | 11 | if [[ -z "${DATA_DIR}" ]]; then 12 | # do nothing if DATA_DIR is not set 13 | exit 0 14 | fi 15 | 16 | if [ ! -f /tmp/logrotate.first.done ] ; then 17 | if [ $(( minute % 15 )) != 0 ]; then 18 | echo "Fist time run and timestamp is not 15 minutes, skipped" 19 | else 20 | echo "$(date --rfc-3339=seconds) rotating files" 21 | logrotate "$@" 22 | fi 23 | touch /tmp/logrotate.first.done 24 | else 25 | echo "$(date --rfc-3339=seconds) rotating files" 26 | logrotate "$@" 27 | fi 28 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.bbo.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/bbo/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/bbo/*/*/*.bbo {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.candlestick.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/candlestick/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/candlestick/*/*/*.candlestick {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.funding_rate.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/funding_rate/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/funding_rate/*/*/*.funding_rate {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.l2_event.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/l2_event/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/l2_event/*/*/*.l2_event {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.l2_snapshot.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/l2_snapshot/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/l2_snapshot/*/*/*.l2_snapshot {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.l2_topk.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/l2_topk/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/l2_topk/*/*/*.l2_topk {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.l3_event.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/l3_event/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/l3_event/*/*/*.l3_event {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.open_interest.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/open_interest/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/open_interest/*/*/*.open_interest {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.other.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/other/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/other/*/*/*.other {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.ticker.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/ticker/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/ticker/*/*/*.ticker {} 16 | -------------------------------------------------------------------------------- /conf/logrotate/logrotate.trade.conf: -------------------------------------------------------------------------------- 1 | missingok 2 | notifempty 3 | rotate 99999 4 | size 64 5 | 6 | dateext 7 | dateformat .%Y-%m-%d-%H-%M.json 8 | 9 | postrotate 10 | pid_file="/tmp/carbonbot-pids/trade/$(basename $1)" 11 | pid=$(cat "$pid_file") 12 | kill -HUP "$pid" 2> /dev/null || echo "$(basename $1) not running" 13 | endscript 14 | 15 | /carbonbot_data/trade/*/*/*.trade {} 16 | -------------------------------------------------------------------------------- /conf/pm2/pm2.bbo.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_swap", // linear_future + linear_swap 6 | "inverse_swap", // inverse_future + inverse_swap 7 | ], 8 | bitmex: ["unknown"], // unknown means all 9 | deribit: ["inverse_future", "inverse_swap", "european_option"], 10 | gate: ["spot", "linear_swap", "inverse_swap"], 11 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 12 | kraken: ["spot"], 13 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 14 | okx: [ 15 | "spot", 16 | "linear_future", 17 | "inverse_future", 18 | "linear_swap", 19 | "inverse_swap", 20 | "european_option", 21 | ], 22 | }; 23 | 24 | const apps = []; 25 | 26 | Object.keys(market_types).forEach((exchange) => { 27 | market_types[exchange].forEach((market_ype) => { 28 | const app = { 29 | name: `crawler-bbo-${exchange}-${market_ype}`, 30 | script: "carbonbot", 31 | args: `${exchange} ${market_ype} bbo`, 32 | exec_interpreter: "none", 33 | exec_mode: "fork_mode", 34 | instances: 1, 35 | exp_backoff_restart_delay: 5000, 36 | }; 37 | 38 | apps.push(app); 39 | }); 40 | }); 41 | 42 | apps.push({ 43 | name: "logrotate", 44 | script: "/usr/local/bin/logrotate.sh", 45 | args: "/usr/local/etc/logrotate.bbo.conf", 46 | exec_interpreter: "none", 47 | exec_mode: "fork_mode", 48 | cron_restart: "*/15 * * * *", 49 | autorestart: false, 50 | }); 51 | 52 | apps.push({ 53 | name: "compress", 54 | script: "/usr/local/bin/compress.sh", 55 | args: "bbo", 56 | exec_interpreter: "bash", 57 | exec_mode: "fork_mode", 58 | instances: 1, 59 | restart_delay: 5000, // 5 seconds 60 | }); 61 | 62 | apps.push({ 63 | name: "upload", 64 | script: "/usr/local/bin/upload.sh", 65 | args: "bbo", 66 | exec_interpreter: "bash", 67 | exec_mode: "fork_mode", 68 | instances: 1, 69 | restart_delay: 5000, // 5 seconds 70 | }); 71 | 72 | module.exports = { 73 | apps, 74 | }; 75 | -------------------------------------------------------------------------------- /conf/pm2/pm2.candlestick.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_future", 6 | "inverse_future", 7 | "linear_swap", 8 | "inverse_swap", 9 | ], 10 | bitfinex: ["spot", "linear_swap"], 11 | bitget: ["spot", "inverse_swap", "linear_swap", "inverse_future"], 12 | bitmex: ["unknown"], // unknown means all 13 | bybit: ["inverse_future", "inverse_swap", "linear_swap"], 14 | deribit: ["inverse_future", "inverse_swap", "european_option"], 15 | gate: ["spot", "inverse_future", "linear_future", "inverse_swap", "linear_swap"], 16 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 17 | kraken: ["spot"], 18 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 19 | mexc: ["spot", "linear_swap", "inverse_swap"], 20 | okx: [ 21 | "spot", 22 | "linear_future", 23 | "inverse_future", 24 | "linear_swap", 25 | "inverse_swap", 26 | "european_option", 27 | ], 28 | zb: ["spot", "linear_swap"], 29 | zbg: ["spot", "inverse_swap", "linear_swap"], 30 | }; 31 | 32 | const apps = []; 33 | 34 | Object.keys(market_types).forEach((exchange) => { 35 | market_types[exchange].forEach((market_ype) => { 36 | const app = { 37 | name: `crawler-candlestick-${exchange}-${market_ype}`, 38 | script: "carbonbot", 39 | args: `${exchange} ${market_ype} candlestick`, 40 | exec_interpreter: "none", 41 | exec_mode: "fork_mode", 42 | instances: 1, 43 | exp_backoff_restart_delay: 5000, 44 | }; 45 | 46 | apps.push(app); 47 | }); 48 | }); 49 | 50 | apps.push({ 51 | name: "logrotate", 52 | script: "/usr/local/bin/logrotate.sh", 53 | args: "/usr/local/etc/logrotate.candlestick.conf", 54 | exec_interpreter: "none", 55 | exec_mode: "fork_mode", 56 | cron_restart: "*/15 * * * *", 57 | autorestart: false, 58 | }); 59 | 60 | apps.push({ 61 | name: "compress", 62 | script: "/usr/local/bin/compress.sh", 63 | args: "candlestick", 64 | exec_interpreter: "bash", 65 | exec_mode: "fork_mode", 66 | instances: 1, 67 | restart_delay: 5000, // 5 seconds 68 | }); 69 | 70 | apps.push({ 71 | name: "upload", 72 | script: "/usr/local/bin/upload.sh", 73 | args: "candlestick", 74 | exec_interpreter: "bash", 75 | exec_mode: "fork_mode", 76 | instances: 1, 77 | restart_delay: 5000, // 5 seconds 78 | }); 79 | 80 | module.exports = { 81 | apps, 82 | }; 83 | -------------------------------------------------------------------------------- /conf/pm2/pm2.funding_rate.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: ["linear_swap", "inverse_swap"], 4 | bitmex: ["unknown"], // unknown means all 5 | huobi: ["linear_swap", "inverse_swap"], 6 | okx: ["linear_swap", "inverse_swap"], 7 | }; 8 | 9 | const apps = []; 10 | 11 | Object.keys(market_types).forEach((exchange) => { 12 | market_types[exchange].forEach((market_ype) => { 13 | const app = { 14 | name: `crawler-funding-rate-${exchange}-${market_ype}`, 15 | script: "carbonbot", 16 | args: `${exchange} ${market_ype} funding_rate`, 17 | exec_interpreter: "none", 18 | exec_mode: "fork_mode", 19 | instances: 1, 20 | exp_backoff_restart_delay: 5000, 21 | }; 22 | 23 | apps.push(app); 24 | }); 25 | }); 26 | 27 | apps.push({ 28 | name: "logrotate", 29 | script: "/usr/local/bin/logrotate.sh", 30 | args: "/usr/local/etc/logrotate.funding_rate.conf", 31 | exec_interpreter: "none", 32 | exec_mode: "fork_mode", 33 | cron_restart: "*/15 * * * *", 34 | autorestart: false, 35 | }); 36 | 37 | apps.push({ 38 | name: "compress", 39 | script: "/usr/local/bin/compress.sh", 40 | args: "funding_rate", 41 | exec_interpreter: "bash", 42 | exec_mode: "fork_mode", 43 | instances: 1, 44 | restart_delay: 5000, // 5 seconds 45 | }); 46 | 47 | apps.push({ 48 | name: "upload", 49 | script: "/usr/local/bin/upload.sh", 50 | args: "funding_rate", 51 | exec_interpreter: "bash", 52 | exec_mode: "fork_mode", 53 | instances: 1, 54 | restart_delay: 5000, // 5 seconds 55 | }); 56 | 57 | module.exports = { 58 | apps, 59 | }; 60 | -------------------------------------------------------------------------------- /conf/pm2/pm2.l2_event.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_future", 6 | "inverse_future", 7 | "linear_swap", 8 | "inverse_swap", 9 | ], 10 | bitfinex: ["spot", "linear_swap"], 11 | bitget: ["spot", "inverse_swap", "linear_swap", "inverse_future"], 12 | bithumb: ["spot"], 13 | bitmex: ["unknown"], // unknown means all 14 | bitstamp: ["spot"], 15 | bybit: ["inverse_future", "inverse_swap", "linear_swap"], 16 | coinbase_pro: ["spot"], 17 | deribit: ["inverse_future", "inverse_swap", "european_option"], 18 | dydx: ["linear_swap"], 19 | gate: ["spot", "inverse_future", "linear_future", "inverse_swap", "linear_swap"], 20 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 21 | kraken: ["spot", "inverse_future", "inverse_swap"], 22 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 23 | mexc: ["spot", "linear_swap", "inverse_swap"], 24 | okx: [ 25 | "spot", 26 | "linear_future", 27 | "inverse_future", 28 | "linear_swap", 29 | "inverse_swap", 30 | "european_option", 31 | ], 32 | zb: ["linear_swap"], 33 | zbg: ["spot", "inverse_swap", "linear_swap"], 34 | }; 35 | 36 | const apps = []; 37 | 38 | Object.keys(market_types).forEach((exchange) => { 39 | market_types[exchange].forEach((market_ype) => { 40 | const app = { 41 | name: `crawler-l2_event-${exchange}-${market_ype}`, 42 | script: "carbonbot", 43 | args: `${exchange} ${market_ype} l2_event`, 44 | exec_interpreter: "none", 45 | exec_mode: "fork_mode", 46 | instances: 1, 47 | exp_backoff_restart_delay: 5000, 48 | }; 49 | 50 | apps.push(app); 51 | }); 52 | }); 53 | 54 | apps.push({ 55 | name: "logrotate", 56 | script: "/usr/local/bin/logrotate.sh", 57 | args: "/usr/local/etc/logrotate.l2_event.conf", 58 | exec_interpreter: "none", 59 | exec_mode: "fork_mode", 60 | cron_restart: "*/15 * * * *", 61 | autorestart: false, 62 | }); 63 | 64 | apps.push({ 65 | name: "compress", 66 | script: "/usr/local/bin/compress.sh", 67 | args: "l2_event", 68 | exec_interpreter: "bash", 69 | exec_mode: "fork_mode", 70 | instances: 1, 71 | restart_delay: 5000, // 5 seconds 72 | }); 73 | 74 | apps.push({ 75 | name: "upload", 76 | script: "/usr/local/bin/upload.sh", 77 | args: "l2_event", 78 | exec_interpreter: "bash", 79 | exec_mode: "fork_mode", 80 | instances: 1, 81 | restart_delay: 5000, // 5 seconds 82 | }); 83 | 84 | module.exports = { 85 | apps, 86 | }; 87 | -------------------------------------------------------------------------------- /conf/pm2/pm2.l2_snapshot.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_future", 6 | "inverse_future", 7 | "linear_swap", 8 | "inverse_swap", 9 | ], 10 | bitfinex: ["spot", "linear_swap"], 11 | bitget: ["spot", "inverse_swap", "linear_swap", "inverse_future"], 12 | bithumb: ["spot"], 13 | bitmex: ["unknown"], // unknown means all 14 | bitstamp: ["spot"], 15 | bybit: ["inverse_future", "inverse_swap", "linear_swap"], 16 | coinbase_pro: ["spot"], 17 | deribit: ["inverse_future", "inverse_swap", "european_option"], 18 | dydx: ["linear_swap"], 19 | gate: ["spot", "inverse_future", "linear_future", "linear_swap", "inverse_swap"], 20 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 21 | kraken: ["spot", "inverse_future", "inverse_swap"], 22 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 23 | mexc: ["spot", "linear_swap", "inverse_swap"], 24 | okx: [ 25 | "spot", 26 | "linear_future", 27 | "inverse_future", 28 | "linear_swap", 29 | "inverse_swap", 30 | "european_option", 31 | ], 32 | zb: ["spot", "linear_swap"], 33 | zbg: ["spot", "inverse_swap", "linear_swap"], 34 | }; 35 | 36 | const apps = []; 37 | 38 | Object.keys(market_types).forEach((exchange) => { 39 | market_types[exchange].forEach((market_ype) => { 40 | const app = { 41 | name: `crawler-l2_snapshot-${exchange}-${market_ype}`, 42 | script: "carbonbot", 43 | args: `${exchange} ${market_ype} l2_snapshot`, 44 | exec_interpreter: "none", 45 | exec_mode: "fork_mode", 46 | instances: 1, 47 | exp_backoff_restart_delay: 5000, 48 | }; 49 | 50 | apps.push(app); 51 | }); 52 | }); 53 | 54 | apps.push({ 55 | name: "logrotate", 56 | script: "/usr/local/bin/logrotate.sh", 57 | args: "/usr/local/etc/logrotate.l2_snapshot.conf", 58 | exec_interpreter: "none", 59 | exec_mode: "fork_mode", 60 | cron_restart: "*/15 * * * *", 61 | autorestart: false, 62 | }); 63 | 64 | apps.push({ 65 | name: "compress", 66 | script: "/usr/local/bin/compress.sh", 67 | args: "l2_snapshot", 68 | exec_interpreter: "bash", 69 | exec_mode: "fork_mode", 70 | instances: 1, 71 | restart_delay: 5000, // 5 seconds 72 | }); 73 | 74 | apps.push({ 75 | name: "upload", 76 | script: "/usr/local/bin/upload.sh", 77 | args: "l2_snapshot", 78 | exec_interpreter: "bash", 79 | exec_mode: "fork_mode", 80 | instances: 1, 81 | restart_delay: 5000, // 5 seconds 82 | }); 83 | 84 | module.exports = { 85 | apps, 86 | }; 87 | -------------------------------------------------------------------------------- /conf/pm2/pm2.l2_topk.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_future", 6 | "inverse_future", 7 | "linear_swap", 8 | "inverse_swap", 9 | ], 10 | bitget: ["spot", "inverse_swap", "linear_swap", "inverse_future"], 11 | bitmex: ["unknown"], // unknown means all 12 | bitstamp: ["spot"], 13 | deribit: ["inverse_future", "inverse_swap", "european_option"], 14 | gate: ["spot", "inverse_swap", "linear_swap"], 15 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 16 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 17 | mexc: ["spot", "linear_swap", "inverse_swap"], 18 | okx: [ 19 | "spot", 20 | "linear_future", 21 | "inverse_future", 22 | "linear_swap", 23 | "inverse_swap", 24 | "european_option", 25 | ], 26 | zb: ["spot", "linear_swap"], 27 | }; 28 | 29 | const apps = []; 30 | 31 | Object.keys(market_types).forEach((exchange) => { 32 | market_types[exchange].forEach((market_ype) => { 33 | const app = { 34 | name: `crawler-l2_topk-${exchange}-${market_ype}`, 35 | script: "carbonbot", 36 | args: `${exchange} ${market_ype} l2_topk`, 37 | exec_interpreter: "none", 38 | exec_mode: "fork_mode", 39 | instances: 1, 40 | exp_backoff_restart_delay: 5000, 41 | }; 42 | 43 | apps.push(app); 44 | }); 45 | }); 46 | 47 | apps.push({ 48 | name: "logrotate", 49 | script: "/usr/local/bin/logrotate.sh", 50 | args: "/usr/local/etc/logrotate.l2_topk.conf", 51 | exec_interpreter: "none", 52 | exec_mode: "fork_mode", 53 | cron_restart: "*/15 * * * *", 54 | autorestart: false, 55 | }); 56 | 57 | apps.push({ 58 | name: "compress", 59 | script: "/usr/local/bin/compress.sh", 60 | args: "l2_topk", 61 | exec_interpreter: "bash", 62 | exec_mode: "fork_mode", 63 | instances: 1, 64 | restart_delay: 5000, // 5 seconds 65 | }); 66 | 67 | apps.push({ 68 | name: "upload", 69 | script: "/usr/local/bin/upload.sh", 70 | args: "l2_topk", 71 | exec_interpreter: "bash", 72 | exec_mode: "fork_mode", 73 | instances: 1, 74 | restart_delay: 5000, // 5 seconds 75 | }); 76 | 77 | module.exports = { 78 | apps, 79 | }; 80 | -------------------------------------------------------------------------------- /conf/pm2/pm2.l3_event.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | bitfinex: ["spot", "linear_swap"], 4 | bitstamp: ["spot"], 5 | coinbase_pro: ["spot"], 6 | }; 7 | 8 | const apps = []; 9 | 10 | Object.keys(market_types).forEach((exchange) => { 11 | market_types[exchange].forEach((market_ype) => { 12 | const app = { 13 | name: `crawler-l3_event-${exchange}-${market_ype}`, 14 | script: "carbonbot", 15 | args: `${exchange} ${market_ype} l3_event`, 16 | exec_interpreter: "none", 17 | exec_mode: "fork_mode", 18 | instances: 1, 19 | exp_backoff_restart_delay: 5000, 20 | }; 21 | 22 | apps.push(app); 23 | }); 24 | }); 25 | 26 | apps.push({ 27 | name: "logrotate", 28 | script: "/usr/local/bin/logrotate.sh", 29 | args: "/usr/local/etc/logrotate.l3_event.conf", 30 | exec_interpreter: "none", 31 | exec_mode: "fork_mode", 32 | cron_restart: "*/15 * * * *", 33 | autorestart: false, 34 | }); 35 | 36 | apps.push({ 37 | name: "compress", 38 | script: "/usr/local/bin/compress.sh", 39 | args: "l3_event", 40 | exec_interpreter: "bash", 41 | exec_mode: "fork_mode", 42 | instances: 1, 43 | restart_delay: 5000, // 5 seconds 44 | }); 45 | 46 | apps.push({ 47 | name: "upload", 48 | script: "/usr/local/bin/upload.sh", 49 | args: "l3_event", 50 | exec_interpreter: "bash", 51 | exec_mode: "fork_mode", 52 | instances: 1, 53 | restart_delay: 5000, // 5 seconds 54 | }); 55 | 56 | module.exports = { 57 | apps, 58 | }; 59 | -------------------------------------------------------------------------------- /conf/pm2/pm2.open_interest.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: ["linear_future", "inverse_future", "linear_swap", "inverse_swap"], 4 | bitget: ["inverse_swap", "linear_swap"], 5 | bybit: ["inverse_future", "inverse_swap", "linear_swap"], 6 | deribit: ["unknown"], // https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency={BTC,ETH,SOL,USDC} contains all markets 7 | dydx: ["linear_swap"], 8 | gate: ["linear_swap", "inverse_swap"], 9 | huobi: ["inverse_future", "linear_swap", "inverse_swap"], 10 | kucoin: ["unknown"], // https://api-futures.kucoin.com/api/v1/contracts/active contains all open interests 11 | okx: [ 12 | "linear_future", 13 | "inverse_future", 14 | "linear_swap", 15 | "inverse_swap", 16 | "european_option", 17 | ], 18 | zbg: ["inverse_swap", "linear_swap"], 19 | }; 20 | 21 | const apps = []; 22 | 23 | Object.keys(market_types).forEach((exchange) => { 24 | market_types[exchange].forEach((market_ype) => { 25 | const app = { 26 | name: `crawler-open_interest-${exchange}-${market_ype}`, 27 | script: "carbonbot", 28 | args: `${exchange} ${market_ype} open_interest`, 29 | exec_interpreter: "none", 30 | exec_mode: "fork_mode", 31 | instances: 1, 32 | exp_backoff_restart_delay: 5000, 33 | }; 34 | 35 | apps.push(app); 36 | }); 37 | }); 38 | 39 | apps.push({ 40 | name: "logrotate", 41 | script: "/usr/local/bin/logrotate.sh", 42 | args: "/usr/local/etc/logrotate.open_interest.conf", 43 | exec_interpreter: "none", 44 | exec_mode: "fork_mode", 45 | cron_restart: "*/15 * * * *", 46 | autorestart: false, 47 | }); 48 | 49 | apps.push({ 50 | name: "compress", 51 | script: "/usr/local/bin/compress.sh", 52 | args: "open_interest", 53 | exec_interpreter: "bash", 54 | exec_mode: "fork_mode", 55 | instances: 1, 56 | restart_delay: 5000, // 5 seconds 57 | }); 58 | 59 | apps.push({ 60 | name: "upload", 61 | script: "/usr/local/bin/upload.sh", 62 | args: "open_interest", 63 | exec_interpreter: "bash", 64 | exec_mode: "fork_mode", 65 | instances: 1, 66 | restart_delay: 5000, // 5 seconds 67 | }); 68 | 69 | module.exports = { 70 | apps, 71 | }; 72 | -------------------------------------------------------------------------------- /conf/pm2/pm2.other.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: ["inverse_swap", "linear_swap"], 4 | bitmex: ["unknown"], 5 | bybit: ["inverse_future", "inverse_swap"], 6 | coinbase_pro: ["spot"], 7 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 8 | okx: ["unknown"], 9 | }; 10 | 11 | const apps = []; 12 | 13 | Object.keys(market_types).forEach((exchange) => { 14 | market_types[exchange].forEach((market_ype) => { 15 | const app = { 16 | name: `crawler-other-${exchange}-${market_ype}`, 17 | script: "carbonbot", 18 | args: `${exchange} ${market_ype} other`, 19 | exec_interpreter: "none", 20 | exec_mode: "fork_mode", 21 | instances: 1, 22 | exp_backoff_restart_delay: 5000, 23 | }; 24 | 25 | apps.push(app); 26 | }); 27 | }); 28 | 29 | apps.push({ 30 | name: "logrotate", 31 | script: "/usr/local/bin/logrotate.sh", 32 | args: "/usr/local/etc/logrotate.other.conf", 33 | exec_interpreter: "none", 34 | exec_mode: "fork_mode", 35 | cron_restart: "*/15 * * * *", 36 | autorestart: false, 37 | }); 38 | 39 | apps.push({ 40 | name: "compress", 41 | script: "/usr/local/bin/compress.sh", 42 | args: "other", 43 | exec_interpreter: "bash", 44 | exec_mode: "fork_mode", 45 | instances: 1, 46 | restart_delay: 5000, // 5 seconds 47 | }); 48 | 49 | apps.push({ 50 | name: "upload", 51 | script: "/usr/local/bin/upload.sh", 52 | args: "other", 53 | exec_interpreter: "bash", 54 | exec_mode: "fork_mode", 55 | instances: 1, 56 | restart_delay: 5000, // 5 seconds 57 | }); 58 | 59 | module.exports = { 60 | apps, 61 | }; 62 | -------------------------------------------------------------------------------- /conf/pm2/pm2.ticker.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_swap", // linear_future + linear_swap 6 | "inverse_swap", // inverse_future + inverse_swap 7 | // "european_option", 8 | ], 9 | bitfinex: ["spot", "linear_swap"], 10 | bitget: ["spot", "inverse_swap", "linear_swap", "inverse_future"], 11 | bithumb: ["spot"], 12 | bybit: ["inverse_future", "inverse_swap", "linear_swap"], 13 | coinbase_pro: ["spot"], 14 | deribit: ["inverse_future", "inverse_swap", "european_option"], 15 | gate: ["spot", "inverse_future", "linear_future", "inverse_swap", "linear_swap"], 16 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 17 | kraken: ["spot", "inverse_future", "inverse_swap"], 18 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 19 | mexc: ["linear_swap", "inverse_swap"], 20 | okx: [ 21 | "spot", 22 | "linear_future", 23 | "inverse_future", 24 | "linear_swap", 25 | "inverse_swap", 26 | "european_option", 27 | ], 28 | zb: ["spot", "linear_swap"], 29 | zbg: ["spot", "inverse_swap", "linear_swap"], 30 | }; 31 | 32 | const apps = []; 33 | 34 | Object.keys(market_types).forEach((exchange) => { 35 | market_types[exchange].forEach((market_ype) => { 36 | const app = { 37 | name: `crawler-ticker-${exchange}-${market_ype}`, 38 | script: "carbonbot", 39 | args: `${exchange} ${market_ype} ticker`, 40 | exec_interpreter: "none", 41 | exec_mode: "fork_mode", 42 | instances: 1, 43 | exp_backoff_restart_delay: 5000, 44 | }; 45 | 46 | apps.push(app); 47 | }); 48 | }); 49 | 50 | apps.push({ 51 | name: "logrotate", 52 | script: "/usr/local/bin/logrotate.sh", 53 | args: "/usr/local/etc/logrotate.ticker.conf", 54 | exec_interpreter: "none", 55 | exec_mode: "fork_mode", 56 | cron_restart: "*/15 * * * *", 57 | autorestart: false, 58 | }); 59 | 60 | apps.push({ 61 | name: "compress", 62 | script: "/usr/local/bin/compress.sh", 63 | args: "ticker", 64 | exec_interpreter: "bash", 65 | exec_mode: "fork_mode", 66 | instances: 1, 67 | restart_delay: 5000, // 5 seconds 68 | }); 69 | 70 | apps.push({ 71 | name: "upload", 72 | script: "/usr/local/bin/upload.sh", 73 | args: "ticker", 74 | exec_interpreter: "bash", 75 | exec_mode: "fork_mode", 76 | instances: 1, 77 | restart_delay: 5000, // 5 seconds 78 | }); 79 | 80 | module.exports = { 81 | apps, 82 | }; 83 | -------------------------------------------------------------------------------- /conf/pm2/pm2.trade.config.js: -------------------------------------------------------------------------------- 1 | // see src/market_type.rs in crypto-markets 2 | const market_types = { 3 | binance: [ 4 | "spot", 5 | "linear_future", 6 | "inverse_future", 7 | "linear_swap", 8 | "inverse_swap", 9 | // "european_option", 10 | ], 11 | bitfinex: ["spot", "linear_swap"], 12 | bitget: ["spot", "inverse_swap", "linear_swap", "inverse_future"], 13 | bithumb: ["spot"], 14 | bitmex: ["unknown"], // unknown means all 15 | bitstamp: ["spot"], 16 | bybit: ["inverse_future", "inverse_swap", "linear_swap"], 17 | coinbase_pro: ["spot"], 18 | deribit: ["inverse_future", "european_option"], // inverse_swap is included in inverse_future 19 | dydx: ["linear_swap"], 20 | gate: ["spot", "inverse_future", "linear_future", "inverse_swap", "linear_swap"], 21 | huobi: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 22 | kraken: ["spot", "inverse_future", "inverse_swap"], 23 | kucoin: ["spot", "inverse_future", "linear_swap", "inverse_swap"], 24 | mexc: ["spot", "linear_swap", "inverse_swap"], 25 | okx: [ 26 | "spot", 27 | "linear_future", 28 | "inverse_future", 29 | "linear_swap", 30 | "inverse_swap", 31 | "european_option", 32 | ], 33 | zb: ["spot", "linear_swap"], 34 | zbg: ["spot", "inverse_swap", "linear_swap"], 35 | }; 36 | 37 | const apps = []; 38 | 39 | Object.keys(market_types).forEach((exchange) => { 40 | market_types[exchange].forEach((market_ype) => { 41 | const app = { 42 | name: `crawler-trade-${exchange}-${market_ype}`, 43 | script: "carbonbot", 44 | args: `${exchange} ${market_ype} trade`, 45 | exec_interpreter: "none", 46 | exec_mode: "fork_mode", 47 | instances: 1, 48 | exp_backoff_restart_delay: 5000, 49 | }; 50 | 51 | apps.push(app); 52 | }); 53 | }); 54 | 55 | apps.push({ 56 | name: "logrotate", 57 | script: "/usr/local/bin/logrotate.sh", 58 | args: "/usr/local/etc/logrotate.trade.conf", 59 | exec_interpreter: "none", 60 | exec_mode: "fork_mode", 61 | cron_restart: "*/15 * * * *", 62 | autorestart: false, 63 | }); 64 | 65 | apps.push({ 66 | name: "compress", 67 | script: "/usr/local/bin/compress.sh", 68 | args: "trade", 69 | exec_interpreter: "bash", 70 | exec_mode: "fork_mode", 71 | instances: 1, 72 | restart_delay: 5000, // 5 seconds 73 | }); 74 | 75 | apps.push({ 76 | name: "upload", 77 | script: "/usr/local/bin/upload.sh", 78 | args: "trade", 79 | exec_interpreter: "bash", 80 | exec_mode: "fork_mode", 81 | instances: 1, 82 | restart_delay: 5000, // 5 seconds 83 | }); 84 | 85 | module.exports = { 86 | apps, 87 | }; 88 | -------------------------------------------------------------------------------- /conf/rclone.conf: -------------------------------------------------------------------------------- 1 | [s3] 2 | type = s3 3 | provider = AWS 4 | env_auth = true 5 | no_check_bucket=true 6 | storage_class = STANDARD 7 | 8 | [minio] 9 | type = s3 10 | provider = Minio 11 | env_auth = true 12 | no_check_bucket=true 13 | -------------------------------------------------------------------------------- /conf/run_crawlers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | LOCAL_TMP_DIR="/your/local/path" # required, better be a local SSD 3 | 4 | # Four kinds of destinations are supported: directory, AWS S3, MinIO and Redis 5 | 6 | # optional 7 | DEST_DIR="/your/dest/directory" # can be either a local directory or a NFS directory 8 | 9 | # optional 10 | AWS_ACCESS_KEY_ID="you access key" 11 | AWS_SECRET_ACCESS_KEY="your secret key" 12 | AWS_S3_DIR="s3://bucket/path" 13 | 14 | # optional 15 | MINIO_ACCESS_KEY_ID="your access key" 16 | MINIO_SECRET_ACCESS_KEY="your secret key" 17 | MINIO_DIR="minio://bucket/path" 18 | MINIO_ENDPOINT_URL="http://ip-address:9000" 19 | 20 | # optional 21 | REDIS_URL="redis://@ip:6379" 22 | 23 | 24 | docker pull ghcr.io/crypto-crawler/carbonbot:latest 25 | 26 | mkdir -p $LOCAL_TMP_DIR 27 | 28 | # l2_snapshot and open_interest are not included, better deploy them in a different network 29 | msg_types=("trade" "l2_event" "l2_topk" "l3_event" "bbo" "ticker" "candlestick" "funding_rate" "other") 30 | 31 | for msg_type in ${msg_types[@]}; do 32 | docker stop carbonbot-$msg_type && docker rm carbonbot-$msg_type 33 | # Store data to dest directory only 34 | docker run -d --name carbonbot-$msg_type --restart unless-stopped -v $LOCAL_TMP_DIR:/carbonbot_data -v $DEST_DIR:/dest_dir -u "$(id -u):$(id -g)" ghcr.io/crypto-crawler/carbonbot:latest pm2-runtime start pm2.$msg_type.config.js 35 | # Store data to dest directory, AWS S3 and MinIO 36 | # docker run -d --name carbonbot-$msg_type --restart unless-stopped -v $LOCAL_TMP_DIR:/carbonbot_data -v $DEST_DIR:/dest_dir -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_S3_DIR=$AWS_S3_DIR -e MINIO_ACCESS_KEY_ID=$MINIO_ACCESS_KEY_ID -e MINIO_SECRET_ACCESS_KEY=$MINIO_SECRET_ACCESS_KEY -e MINIO_ENDPOINT_URL=$MINIO_ENDPOINT_URL -e MINIO_DIR=$MINIO_DIR -u "$(id -u):$(id -g)" ghcr.io/crypto-crawler/carbonbot:latest pm2-runtime start pm2.$msg_type.config.js 37 | done 38 | 39 | docker system prune -af 40 | -------------------------------------------------------------------------------- /conf/upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Linted by https://www.shellcheck.net/ 3 | 4 | # This script aims to harvest .json files generated by logrotate, 5 | # compress and upload them to S3, MinIO or NAS. 6 | 7 | msg_type=$1 # trade, l2_event, etc. 8 | 9 | if [[ -z "${DATA_DIR}" ]]; then 10 | echo "DATA_DIR must be set" >&2 11 | exit 1 12 | fi 13 | 14 | if [[ -z "${DEST_DIR}" && -z "${AWS_S3_DIR}" && -z "${MINIO_DIR}" && -z "${REDIS_URL}" ]]; then 15 | echo "At least one of DEST_DIR, AWS_S3_DIR, MINIO_DIR or REDIS_URL must be set" >&2 16 | exit 1 17 | fi 18 | 19 | num_destinations=0 20 | 21 | if [[ -n "${AWS_S3_DIR}" ]]; then 22 | if [[ -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" ]]; then 23 | echo "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set" >&2 24 | exit 1 25 | fi 26 | num_destinations=$((num_destinations+1)) 27 | fi 28 | 29 | if [[ -n "${MINIO_DIR}" ]]; then 30 | if [[ -z "${MINIO_ACCESS_KEY_ID}" || -z "${MINIO_SECRET_ACCESS_KEY}" || -z "${MINIO_ENDPOINT_URL}" ]]; then 31 | echo "MINIO_ACCESS_KEY_ID, MINIO_SECRET_ACCESS_KEY and MINIO_ENDPOINT_URL must be set" >&2 32 | exit 1 33 | fi 34 | num_destinations=$((num_destinations+1)) 35 | fi 36 | 37 | if [[ -n "${DEST_DIR}" ]]; then 38 | mkdir -p "$DEST_DIR/$msg_type" 39 | num_destinations=$((num_destinations+1)) 40 | fi 41 | 42 | # Infinite while loop 43 | while : 44 | do 45 | # Upload .json.gz and .json.xz files older than 1 minute 46 | if [[ -n "${AWS_S3_DIR}" ]]; then 47 | if [ $num_destinations -gt 1 ]; then sub_command="copy"; else sub_command="move"; fi 48 | if rclone --s3-region "${AWS_REGION:-us-east-1}" --immutable --contimeout=1s --retries 1 --low-level-retries 1 $sub_command "$DATA_DIR/$msg_type" "$AWS_S3_DIR/$msg_type" --include '*.json.gz' --include '*.json.xz' --min-age 1m --no-traverse --transfers=8; then 49 | num_destinations=$((num_destinations-1)) 50 | fi 51 | fi 52 | if [[ -n "${MINIO_DIR}" ]]; then 53 | if [ $num_destinations -gt 1 ]; then sub_command="copy"; else sub_command="move"; fi 54 | if rclone --s3-access-key-id "$MINIO_ACCESS_KEY_ID" --s3-secret-access-key "$MINIO_SECRET_ACCESS_KEY" --s3-endpoint "$MINIO_ENDPOINT_URL" --immutable --contimeout=1s --retries 1 --low-level-retries 1 $sub_command "$DATA_DIR/$msg_type" "$MINIO_DIR/$msg_type" --include '*.json.gz' --include '*.json.xz' --min-age 1m --no-traverse --transfers=8; then 55 | num_destinations=$((num_destinations-1)) 56 | fi 57 | fi 58 | if [[ -n "${DEST_DIR}" ]]; then 59 | if [ $num_destinations -gt 1 ]; then sub_command="copy"; else sub_command="move"; fi 60 | if rclone $sub_command "$DATA_DIR/$msg_type" "$DEST_DIR/$msg_type" --include '*.json.gz' --include '*.json.xz' --min-age 1m --no-traverse --transfers=8; then 61 | num_destinations=$((num_destinations-1)) 62 | fi 63 | fi 64 | 65 | sleep 3 66 | done 67 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod misc_crawlers; 2 | pub(crate) mod writers; 3 | 4 | pub use misc_crawlers::crawl_other; 5 | pub use writers::create_writer_threads; 6 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use carbonbot::{crawl_other, create_writer_threads}; 2 | use crypto_crawler::*; 3 | use crypto_market_type::MarketType; 4 | use crypto_msg_type::MessageType; 5 | use log::*; 6 | use once_cell::sync::Lazy; 7 | use std::{collections::HashMap, env, str::FromStr}; 8 | 9 | // Exchanges with poor liquidity 10 | #[allow(clippy::type_complexity)] 11 | static HEARTBEAT_CONFIGS: Lazy>>> = 12 | Lazy::new(|| { 13 | let mut trade_configs: HashMap> = HashMap::new(); 14 | trade_configs.insert( 15 | "bitfinex".to_string(), 16 | HashMap::from([(MarketType::Spot, 180)]), 17 | ); 18 | trade_configs.insert( 19 | "gate".to_string(), 20 | HashMap::from([ 21 | (MarketType::InverseFuture, 600), 22 | (MarketType::LinearFuture, 600), 23 | ]), 24 | ); 25 | trade_configs.insert( 26 | "kraken".to_string(), 27 | HashMap::from([ 28 | (MarketType::InverseFuture, 420), 29 | (MarketType::InverseSwap, 180), 30 | ]), 31 | ); 32 | trade_configs.insert( 33 | "zb".to_string(), 34 | HashMap::from([(MarketType::LinearSwap, 360)]), 35 | ); 36 | trade_configs.insert( 37 | "zbg".to_string(), 38 | HashMap::from([ 39 | (MarketType::InverseSwap, 540), 40 | (MarketType::LinearSwap, 540), 41 | ]), 42 | ); 43 | 44 | let mut l2_event_configs: HashMap> = HashMap::new(); 45 | l2_event_configs.insert( 46 | "bitfinex".to_string(), 47 | HashMap::from([(MarketType::Spot, 120), (MarketType::LinearSwap, 150)]), 48 | ); 49 | l2_event_configs.insert("gate".to_string(), HashMap::from([(MarketType::Spot, 120)])); 50 | l2_event_configs.insert("mexc".to_string(), HashMap::from([(MarketType::Spot, 120)])); 51 | l2_event_configs.insert( 52 | "zb".to_string(), 53 | HashMap::from([(MarketType::LinearSwap, 120)]), 54 | ); 55 | l2_event_configs.insert( 56 | "zbg".to_string(), 57 | HashMap::from([ 58 | (MarketType::InverseSwap, 120), 59 | (MarketType::LinearSwap, 120), 60 | ]), 61 | ); 62 | 63 | HashMap::from([ 64 | (MessageType::Trade, trade_configs.clone()), 65 | (MessageType::Ticker, trade_configs), 66 | (MessageType::L2Event, l2_event_configs.clone()), 67 | (MessageType::L3Event, l2_event_configs.clone()), 68 | (MessageType::L2TopK, l2_event_configs.clone()), 69 | (MessageType::BBO, l2_event_configs), 70 | ]) 71 | }); 72 | 73 | fn get_message_gap(exchange: &'static str, market_type: MarketType, msg_type: MessageType) -> u64 { 74 | if let Some(x) = HEARTBEAT_CONFIGS.get(&msg_type) { 75 | if let Some(y) = x.get(exchange) { 76 | if let Some(z) = y.get(&market_type) { 77 | return *z; 78 | } 79 | } 80 | } 81 | 82 | match msg_type { 83 | MessageType::Trade | MessageType::Ticker => match market_type { 84 | MarketType::Spot | MarketType::LinearSwap => 60, 85 | MarketType::InverseFuture | MarketType::InverseSwap | MarketType::LinearFuture => 120, 86 | _ => 360, // 6 minutes 87 | }, 88 | MessageType::L2Event | MessageType::L3Event | MessageType::L2TopK | MessageType::BBO => { 89 | match market_type { 90 | MarketType::Spot | MarketType::LinearSwap => 15, 91 | MarketType::InverseFuture | MarketType::InverseSwap | MarketType::LinearFuture => { 92 | 25 93 | } 94 | _ => 120, // 2 minutes 95 | } 96 | } 97 | MessageType::FundingRate => { 98 | match exchange { 99 | "bitmex" => 3600 * 8 + 5, // Sent every funding interval (usually 8hrs), see https://www.bitmex.com/app/wsAPI 100 | "huobi" => 65, // Funding rate will be pushed every 60 seconds by default, see https://huobiapi.github.io/docs/coin_margined_swap/v1/en/#unsubscribe-funding-rate-data-no-authentication-unsub 101 | "okx" => 95, // Data will be pushed in 30s to 90s, see https://www.okx.com/docs-v5/en/#websocket-api-public-channel-funding-rate-channel 102 | _ => 3600, 103 | } 104 | } 105 | MessageType::Candlestick => 180, 106 | _ => 300, // 5 minutes 107 | } 108 | } 109 | 110 | pub async fn crawl( 111 | exchange: &'static str, 112 | market_type: MarketType, 113 | msg_type: MessageType, 114 | data_dir: Option, 115 | redis_url: Option, 116 | symbols: Option<&[String]>, 117 | ) { 118 | info!("Started to crawl {} {} {}", exchange, market_type, msg_type); 119 | if data_dir.is_none() && redis_url.is_none() { 120 | error!("Both DATA_DIR and REDIS_URL are not set"); 121 | return; 122 | } 123 | let (tx, rx) = std::sync::mpsc::channel::(); 124 | let timeout_secs = get_message_gap(exchange, market_type, msg_type); 125 | let writer_threads = create_writer_threads(rx, data_dir, redis_url, timeout_secs); 126 | 127 | if msg_type == MessageType::Candlestick { 128 | crawl_candlestick(exchange, market_type, None, tx).await; 129 | } else if msg_type == MessageType::OpenInterest { 130 | tokio::task::spawn_blocking(move || crawl_open_interest(exchange, market_type, tx)); 131 | } else if msg_type == MessageType::Other { 132 | crawl_other(exchange, market_type, tx).await; 133 | } else { 134 | match msg_type { 135 | MessageType::BBO => { 136 | crawl_bbo(exchange, market_type, symbols, tx).await; 137 | } 138 | MessageType::Trade => { 139 | crawl_trade(exchange, market_type, symbols, tx).await; 140 | } 141 | MessageType::L2Event => { 142 | crawl_l2_event(exchange, market_type, symbols, tx).await; 143 | } 144 | MessageType::L3Event => { 145 | crawl_l3_event(exchange, market_type, symbols, tx).await; 146 | } 147 | MessageType::L2Snapshot => { 148 | let symbols = if let Some(symbols) = symbols { 149 | symbols.to_vec() 150 | } else { 151 | vec![] 152 | }; 153 | tokio::task::spawn_blocking(move || { 154 | let symbols_local = symbols; 155 | crawl_l2_snapshot(exchange, market_type, Some(&symbols_local), tx) 156 | }); 157 | } 158 | MessageType::L2TopK => { 159 | crawl_l2_topk(exchange, market_type, symbols, tx).await; 160 | } 161 | MessageType::L3Snapshot => { 162 | let symbols = if let Some(symbols) = symbols { 163 | symbols.to_vec() 164 | } else { 165 | vec![] 166 | }; 167 | tokio::task::spawn_blocking(move || { 168 | let symbols_local = symbols; 169 | crawl_l3_snapshot(exchange, market_type, Some(&symbols_local), tx) 170 | }); 171 | } 172 | MessageType::Ticker => { 173 | crawl_ticker(exchange, market_type, symbols, tx).await; 174 | } 175 | MessageType::FundingRate => { 176 | crawl_funding_rate(exchange, market_type, symbols, tx).await 177 | } 178 | _ => panic!("Not implemented"), 179 | }; 180 | } 181 | for thread in writer_threads { 182 | thread.join().unwrap(); 183 | } 184 | } 185 | 186 | #[tokio::main(flavor = "multi_thread")] 187 | async fn main() { 188 | env_logger::init(); 189 | 190 | let args: Vec = env::args().collect(); 191 | if args.len() != 4 && args.len() != 5 { 192 | println!("Usage: carbonbot [comma_seperated_symbols]"); 193 | return; 194 | } 195 | 196 | let exchange: &'static str = Box::leak(args[1].clone().into_boxed_str()); 197 | 198 | let market_type_str = args[2].as_str(); 199 | let market_type = MarketType::from_str(market_type_str); 200 | if market_type.is_err() { 201 | println!("Unknown market type: {}", market_type_str); 202 | return; 203 | } 204 | let market_type = market_type.unwrap(); 205 | 206 | let msg_type_str = args[3].as_str(); 207 | let msg_type = MessageType::from_str(msg_type_str); 208 | if msg_type.is_err() { 209 | println!("Unknown msg type: {}", msg_type_str); 210 | return; 211 | } 212 | let msg_type = msg_type.unwrap(); 213 | 214 | let data_dir = if std::env::var("DATA_DIR").is_err() { 215 | info!("The DATA_DIR environment variable does not exist"); 216 | None 217 | } else { 218 | let url = std::env::var("DATA_DIR").unwrap(); 219 | Some(url) 220 | }; 221 | 222 | let redis_url = if std::env::var("REDIS_URL").is_err() { 223 | info!("The REDIS_URL environment variable does not exist"); 224 | None 225 | } else { 226 | let url = std::env::var("REDIS_URL").unwrap(); 227 | Some(url) 228 | }; 229 | 230 | let specified_symbols = if args.len() == 4 { 231 | Vec::new() 232 | } else { 233 | let mut symbols = fetch_symbols_retry(exchange, market_type); 234 | symbols.retain(|symbol| args[4].split(',').any(|part| symbol.contains(part))); 235 | info!("target symbols: {:?}", symbols); 236 | symbols 237 | }; 238 | 239 | if data_dir.is_none() && redis_url.is_none() { 240 | panic!("The environment variable DATA_DIR and REDIS_URL are not set, at least one of them should be set"); 241 | } 242 | 243 | let pid = std::process::id(); 244 | // write pid to file 245 | { 246 | let mut dir = std::env::temp_dir() 247 | .join("carbonbot-pids") 248 | .join(msg_type_str); 249 | let _ = std::fs::create_dir_all(&dir); 250 | dir.push(format!("{}.{}.{}", exchange, market_type_str, msg_type_str)); 251 | std::fs::write(dir.as_path(), pid.to_string()).expect("Unable to write pid to file"); 252 | } 253 | crawl( 254 | exchange, 255 | market_type, 256 | msg_type, 257 | data_dir, 258 | redis_url, 259 | if specified_symbols.is_empty() { 260 | None 261 | } else { 262 | Some(&specified_symbols) 263 | }, 264 | ) 265 | .await; 266 | } 267 | -------------------------------------------------------------------------------- /src/misc_crawlers/binance.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use super::utils::create_conversion_thread; 4 | use crypto_crawler::Message; 5 | use crypto_market_type::MarketType; 6 | use crypto_msg_type::MessageType; 7 | use crypto_ws_client::*; 8 | 9 | pub(super) async fn crawl_other(market_type: MarketType, tx: Sender) { 10 | let tx = create_conversion_thread("binance".to_string(), MessageType::Other, market_type, tx); 11 | let commands = 12 | vec![r#"{"id":9527,"method":"SUBSCRIBE","params":["!forceOrder@arr"]}"#.to_string()]; 13 | 14 | match market_type { 15 | MarketType::InverseSwap | MarketType::InverseFuture => { 16 | let ws_client = BinanceInverseWSClient::new(tx, None).await; 17 | ws_client.send(&commands).await; 18 | ws_client.run().await; 19 | ws_client.close(); 20 | } 21 | MarketType::LinearSwap | MarketType::LinearFuture => { 22 | let ws_client = BinanceLinearWSClient::new(tx, None).await; 23 | ws_client.send(&commands).await; 24 | ws_client.run().await; 25 | ws_client.close(); 26 | } 27 | _ => panic!("Unknown market_type {}", market_type), 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/misc_crawlers/bitmex.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use super::utils::create_conversion_thread; 4 | use crypto_crawler::Message; 5 | use crypto_market_type::MarketType; 6 | use crypto_msg_type::MessageType; 7 | use crypto_ws_client::*; 8 | 9 | pub(super) async fn crawl_other(market_type: MarketType, tx: Sender) { 10 | assert_eq!(market_type, MarketType::Unknown); 11 | let tx = create_conversion_thread("bitmex".to_string(), MessageType::Other, market_type, tx); 12 | let commands: Vec = vec![ 13 | "announcement", 14 | "connected", 15 | "instrument", 16 | "insurance", 17 | "liquidation", 18 | "publicNotifications", 19 | "settlement", 20 | ] 21 | .into_iter() 22 | .map(|x| format!(r#"{{"op":"subscribe","args":["{}"]}}"#, x)) 23 | .collect(); 24 | 25 | let ws_client = BitmexWSClient::new(tx, None).await; 26 | ws_client.send(&commands).await; 27 | ws_client.run().await; 28 | ws_client.close(); 29 | } 30 | -------------------------------------------------------------------------------- /src/misc_crawlers/bybit.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use super::utils::create_conversion_thread; 4 | use crypto_crawler::Message; 5 | use crypto_market_type::MarketType; 6 | use crypto_msg_type::MessageType; 7 | use crypto_ws_client::*; 8 | 9 | pub(super) async fn crawl_other(market_type: MarketType, tx: Sender) { 10 | let tx = create_conversion_thread("bybit".to_string(), MessageType::Other, market_type, tx); 11 | let commands = vec![r#"{"op":"subscribe","args":["insurance","liquidation"]}"#.to_string()]; 12 | 13 | match market_type { 14 | MarketType::InverseFuture | MarketType::InverseSwap => { 15 | let ws_client = BybitInverseWSClient::new(tx, None).await; 16 | ws_client.send(&commands).await; 17 | ws_client.run().await; 18 | ws_client.close(); 19 | } 20 | _ => panic!("Unknown market_type {}", market_type), 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/misc_crawlers/coinbase_pro.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use super::utils::create_conversion_thread; 4 | use crypto_crawler::Message; 5 | use crypto_market_type::MarketType; 6 | use crypto_msg_type::MessageType; 7 | use crypto_ws_client::*; 8 | 9 | pub(super) async fn crawl_other(market_type: MarketType, tx: Sender) { 10 | let tx = create_conversion_thread( 11 | "coinbase_pro".to_string(), 12 | MessageType::Other, 13 | market_type, 14 | tx, 15 | ); 16 | let commands: Vec = 17 | vec![r#"{"type": "subscribe","channels":[{ "name": "status"}]}"#.to_string()]; 18 | 19 | let ws_client = CoinbaseProWSClient::new(tx, None).await; 20 | ws_client.send(&commands).await; 21 | ws_client.run().await; 22 | ws_client.close(); 23 | } 24 | -------------------------------------------------------------------------------- /src/misc_crawlers/huobi.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use super::utils::create_conversion_thread; 4 | use crypto_crawler::Message; 5 | use crypto_market_type::MarketType; 6 | use crypto_msg_type::MessageType; 7 | use crypto_ws_client::*; 8 | 9 | pub(super) async fn crawl_other(market_type: MarketType, tx: Sender) { 10 | let tx = create_conversion_thread("huobi".to_string(), MessageType::Other, market_type, tx); 11 | let commands = vec![r#"{"sub":"market.overview","id":"crypto-ws-client"}"#.to_string()]; 12 | 13 | match market_type { 14 | MarketType::Spot => { 15 | let ws_client = HuobiSpotWSClient::new(tx, None).await; 16 | ws_client.send(&commands).await; 17 | ws_client.run().await; 18 | ws_client.close(); 19 | } 20 | MarketType::InverseFuture => { 21 | let ws_client = HuobiSpotWSClient::new(tx, None).await; 22 | ws_client.send(&commands).await; 23 | ws_client.run().await; 24 | ws_client.close(); 25 | } 26 | MarketType::LinearSwap => { 27 | let ws_client = HuobiSpotWSClient::new(tx, None).await; 28 | ws_client.send(&commands).await; 29 | ws_client.run().await; 30 | ws_client.close(); 31 | } 32 | MarketType::InverseSwap => { 33 | let ws_client = HuobiSpotWSClient::new(tx, None).await; 34 | ws_client.send(&commands).await; 35 | ws_client.run().await; 36 | ws_client.close(); 37 | } 38 | MarketType::EuropeanOption => { 39 | let ws_client = HuobiSpotWSClient::new(tx, None).await; 40 | ws_client.send(&commands).await; 41 | ws_client.run().await; 42 | ws_client.close(); 43 | } 44 | _ => panic!("Unknown market_type {}", market_type), 45 | }; 46 | } 47 | -------------------------------------------------------------------------------- /src/misc_crawlers/mod.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use crypto_crawler::Message; 4 | use crypto_market_type::MarketType; 5 | 6 | mod binance; 7 | mod bitmex; 8 | mod bybit; 9 | mod coinbase_pro; 10 | mod huobi; 11 | mod okx; 12 | 13 | mod utils; 14 | 15 | pub async fn crawl_other(exchange: &str, market_type: MarketType, tx: Sender) { 16 | match exchange { 17 | "binance" => binance::crawl_other(market_type, tx).await, 18 | "bitmex" => bitmex::crawl_other(market_type, tx).await, 19 | "bybit" => bybit::crawl_other(market_type, tx).await, 20 | "coinbase_pro" => coinbase_pro::crawl_other(market_type, tx).await, 21 | "huobi" => huobi::crawl_other(market_type, tx).await, 22 | "okx" => { 23 | assert_eq!(market_type, MarketType::Unknown); 24 | okx::crawl_other(tx).await 25 | } 26 | _ => panic!("Unknown exchange {}", exchange), 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/misc_crawlers/okx.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use super::utils::create_conversion_thread; 4 | use crypto_crawler::Message; 5 | use crypto_market_type::MarketType; 6 | use crypto_msg_type::MessageType; 7 | use crypto_ws_client::*; 8 | 9 | pub(super) async fn crawl_other(tx: Sender) { 10 | let tx = create_conversion_thread( 11 | "okx".to_string(), 12 | MessageType::Other, 13 | MarketType::Unknown, 14 | tx, 15 | ); 16 | let commands = 17 | vec![r#"{"op":"subscribe","args":[{"channel":"instruments","instType":"SPOT"},{"channel":"instruments","instType":"MARGIN"},{"channel":"instruments","instType":"SWAP"},{"channel":"instruments","instType":"FUTURES"},{"channel":"instruments","instType":"OPTION"},{"channel":"public-struc-block-trades"},{"channel":"status"},{"channel":"opt-summary","uly":"BTC-USD"},{"channel":"opt-summary","uly":"ETH-USD"},{"channel":"opt-summary","uly":"SOL-USD"}]}"#.to_string()]; 18 | 19 | let ws_client = OkxWSClient::new(tx, None).await; 20 | ws_client.send(&commands).await; 21 | ws_client.run().await; 22 | ws_client.close(); 23 | } 24 | -------------------------------------------------------------------------------- /src/misc_crawlers/utils.rs: -------------------------------------------------------------------------------- 1 | use std::sync::mpsc::Sender; 2 | 3 | use crypto_crawler::Message; 4 | use crypto_market_type::MarketType; 5 | use crypto_msg_type::MessageType; 6 | 7 | // create a thread to convert Sender Sender 8 | pub(super) fn create_conversion_thread( 9 | exchange: String, 10 | msg_type: MessageType, 11 | market_type: MarketType, 12 | tx: Sender, 13 | ) -> Sender { 14 | let (tx_raw, rx_raw) = std::sync::mpsc::channel(); 15 | std::thread::spawn(move || { 16 | for json in rx_raw { 17 | let msg = Message::new(exchange.clone(), market_type, msg_type, json); 18 | tx.send(msg).unwrap(); 19 | } 20 | }); 21 | tx_raw 22 | } 23 | -------------------------------------------------------------------------------- /src/writers/file_writer.rs: -------------------------------------------------------------------------------- 1 | use super::Writer; 2 | 3 | use log::*; 4 | use reopen::Reopen; 5 | use std::{ 6 | fs, 7 | io::{BufWriter, Error, Write}, 8 | path::Path, 9 | }; 10 | 11 | #[cfg(not(windows))] 12 | use signal_hook::consts::signal::SIGHUP; 13 | #[cfg(windows)] // Windows has a very limited set of signals, but make it compile at least :-( 14 | use signal_hook::consts::signal::SIGINT as SIGHUP; 15 | 16 | fn open>(p: P) -> Result, Error> { 17 | info!("reopen {}", p.as_ref().display()); 18 | let file = fs::OpenOptions::new() 19 | .create(true) 20 | .write(true) 21 | .append(true) 22 | .open(p)?; 23 | Ok(BufWriter::new(file)) 24 | } 25 | 26 | pub struct FileWriter { 27 | file: Reopen>, 28 | path: String, 29 | } 30 | 31 | impl FileWriter { 32 | pub fn new(path: &str) -> Self { 33 | let path_clone = path.to_string(); 34 | let file = Reopen::new(Box::new(move || open(&path_clone))).unwrap(); 35 | // Make sure it gets reopened on SIGHUP 36 | file.handle().register_signal(SIGHUP).unwrap(); 37 | 38 | FileWriter { 39 | file, 40 | path: path.to_string(), 41 | } 42 | } 43 | } 44 | 45 | impl Writer for FileWriter { 46 | fn write(&mut self, s: &str) { 47 | if let Err(e) = writeln!(self.file, "{}", s.trim()) { 48 | error!("{}, {}", self.path, e); 49 | } 50 | } 51 | 52 | fn close(&mut self) { 53 | if let Err(e) = self.file.flush() { 54 | error!("{}, {}", self.path, e); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/writers/mod.rs: -------------------------------------------------------------------------------- 1 | pub(super) mod file_writer; 2 | 3 | use crypto_crawler::*; 4 | use log::*; 5 | use redis::{self, Commands}; 6 | use std::{ 7 | collections::HashMap, 8 | path::Path, 9 | sync::mpsc::{Receiver, Sender}, 10 | thread::JoinHandle, 11 | time::{Duration, Instant}, 12 | }; 13 | 14 | pub trait Writer { 15 | fn write(&mut self, s: &str); 16 | fn close(&mut self); 17 | } 18 | 19 | pub use file_writer::FileWriter; 20 | 21 | // If there are no messages coming in for `timeout` seconds, exit the process. 22 | fn create_file_writer_thread( 23 | rx: Receiver, 24 | data_dir: String, 25 | tx_redis: Option>, 26 | timeout_secs: u64, 27 | ) -> JoinHandle<()> { 28 | std::thread::spawn(move || { 29 | let timeout = Duration::from_secs(timeout_secs); 30 | let mut writers: HashMap = HashMap::new(); 31 | 32 | let start_time = Instant::now(); 33 | while let Ok(msg) = rx.recv_timeout(timeout) { 34 | let file_name = format!("{}.{}.{}", msg.exchange, msg.market_type, msg.msg_type); 35 | if !writers.contains_key(&file_name) { 36 | let data_dir = Path::new(&data_dir) 37 | .join(msg.msg_type.to_string()) 38 | .join(&msg.exchange) 39 | .join(msg.market_type.to_string()) 40 | .into_os_string(); 41 | std::fs::create_dir_all(data_dir.as_os_str()).unwrap(); 42 | let file_path = Path::new(data_dir.as_os_str()) 43 | .join(file_name.clone()) 44 | .into_os_string(); 45 | writers.insert( 46 | file_name.clone(), 47 | FileWriter::new(file_path.as_os_str().to_str().unwrap()), 48 | ); 49 | } 50 | 51 | if let Some(writer) = writers.get_mut(&file_name) { 52 | // JSON, serde_json::to_string(&msg); CSV, msg.to_tsv_string() 53 | let s = serde_json::to_string(&msg).unwrap(); 54 | writer.write(&s); 55 | } 56 | // copy to redis 57 | if let Some(ref tx_redis) = tx_redis { 58 | tx_redis.send(msg).unwrap(); 59 | } 60 | } 61 | 62 | for mut writer in writers { 63 | writer.1.close(); 64 | } 65 | 66 | let elapsed = start_time.elapsed().as_secs(); 67 | error!( 68 | "This crawler has been running stably for {} seconds, until there are no messages for {} seconds, exiting now", 69 | elapsed, timeout_secs 70 | ); 71 | std::process::exit(1); // pm2 will restart this process 72 | }) 73 | } 74 | 75 | fn connect_redis(redis_url: &str) -> Result { 76 | assert!(!redis_url.is_empty(), "redis_url is empty"); 77 | 78 | let mut redis_error: Option = None; 79 | let mut conn: Option = None; 80 | for _ in 0..3 { 81 | match redis::Client::open(redis_url) { 82 | Ok(client) => match client.get_connection() { 83 | Ok(connection) => { 84 | conn = Some(connection); 85 | break; 86 | } 87 | Err(err) => redis_error = Some(err), 88 | }, 89 | Err(err) => redis_error = Some(err), 90 | } 91 | } 92 | 93 | if let Some(connection) = conn { 94 | Ok(connection) 95 | } else { 96 | Err(redis_error.unwrap()) 97 | } 98 | } 99 | 100 | fn create_redis_writer_thread(rx: Receiver, redis_url: String) -> JoinHandle<()> { 101 | std::thread::spawn(move || { 102 | let mut redis_conn = connect_redis(&redis_url).unwrap(); 103 | for msg in rx { 104 | let msg_type = msg.msg_type; 105 | let s = serde_json::to_string(&msg).unwrap(); 106 | let topic = format!("carbonbot:{}", msg_type); 107 | if let Err(err) = redis_conn.publish::<&str, String, i64>(&topic, s) { 108 | error!("{}", err); 109 | return; 110 | } 111 | } 112 | }) 113 | } 114 | 115 | #[allow(clippy::unnecessary_unwrap)] 116 | pub fn create_writer_threads( 117 | rx: Receiver, 118 | data_dir: Option, 119 | redis_url: Option, 120 | timeout_secs: u64, 121 | ) -> Vec> { 122 | let mut threads = Vec::new(); 123 | if data_dir.is_none() && redis_url.is_none() { 124 | error!("Both DATA_DIR and REDIS_URL are not set"); 125 | return threads; 126 | } 127 | 128 | if data_dir.is_some() && redis_url.is_some() { 129 | // channel for Redis 130 | let (tx_redis, rx_redis) = std::sync::mpsc::channel::(); 131 | threads.push(create_file_writer_thread( 132 | rx, 133 | data_dir.unwrap(), 134 | Some(tx_redis), 135 | timeout_secs, 136 | )); 137 | threads.push(create_redis_writer_thread(rx_redis, redis_url.unwrap())); 138 | } else if data_dir.is_some() { 139 | threads.push(create_file_writer_thread( 140 | rx, 141 | data_dir.unwrap(), 142 | None, 143 | timeout_secs, 144 | )) 145 | } else { 146 | threads.push(create_redis_writer_thread(rx, redis_url.unwrap())); 147 | } 148 | threads 149 | } 150 | --------------------------------------------------------------------------------