├── VERSION ├── .flake8 ├── .github ├── PULL_REQUEST_TEMPLATE ├── ISSUE_TEMPLATE │ ├── question.md │ ├── feature_request.md │ └── bug_report.md └── FUNDING.yml ├── Makefile ├── examples ├── compose │ └── cryptostore2tty │ │ ├── docker-compose.yml │ │ └── README.md └── tcp.py ├── tools ├── config.yaml └── generate.py ├── Dockerfile ├── CHANGES.md ├── .gitignore ├── README.md ├── LICENSE ├── docs └── overview.md └── cryptostore.py /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.4 2 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E501,F405,F403 3 | exclude = tests/*,docs/*,*.md,__init__.py,cryptofeed/exchanges.py,cryptofeed/providers.py -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE: -------------------------------------------------------------------------------- 1 | ### Description of code - what bug does this fix / what feature does this add? 2 | 3 | - [ ] - Tested 4 | - [ ] - Changelog updated 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Questions about the library/behavior/etc 4 | title: '' 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VER=`cat VERSION` 2 | 3 | build: Dockerfile cryptostore.py 4 | docker build . -t ghcr.io/bmoscon/cryptostore:latest 5 | 6 | release: build 7 | docker build . -t ghcr.io/bmoscon/cryptostore:${VER} 8 | -------------------------------------------------------------------------------- /examples/compose/cryptostore2tty/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | services: 3 | cryptostore2tty: 4 | image: ghcr.io/bmoscon/cryptostore:latest 5 | environment: 6 | - EXCHANGE=BINANCE 7 | - CHANNELS=trades 8 | - SYMBOLS=BTC-USDT 9 | - BACKEND=TTY 10 | -------------------------------------------------------------------------------- /tools/config.yaml: -------------------------------------------------------------------------------- 1 | exchanges: 2 | - COINBASE 3 | - BINANCE 4 | 5 | symbols: 6 | COINBASE: ALL 7 | BINANCE: ALL 8 | 9 | channels: 10 | - l2book 11 | - trades 12 | 13 | symbols_per_channel: 10 14 | 15 | backend: INFLUX 16 | org: crypto 17 | bucket: crypto 18 | token: TOKEN 19 | host: http://127.0.0.1:8086 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim-bullseye 2 | 3 | RUN apt update 4 | RUN apt install gcc git -y 5 | 6 | RUN pip install --no-cache-dir cython 7 | RUN pip install --no-cache-dir cryptofeed 8 | RUN pip install --no-cache-dir redis 9 | RUN pip install --no-cache-dir pymongo[srv] 10 | RUN pip install --no-cache-dir motor 11 | RUN pip install --no-cache-dir asyncpg 12 | 13 | COPY cryptostore.py /cryptostore.py 14 | 15 | CMD ["/cryptostore.py"] 16 | ENTRYPOINT ["python"] 17 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: bmoscon 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: Feature Request 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | * Provide config (at least as much as is relevant) 16 | * redis or kafka? 17 | * What behavior did you see? 18 | * Provide any tracebacks if applicable 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Operating System:** 27 | - macOS, linux, etc 28 | 29 | **Cryptofeed Version** 30 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ## Changelog 2 | 3 | ### 1.0.3 (2023-01-28) 4 | * Feature: Add tool to generate docker run commands 5 | * Bugfix: Fixed a kwarg typo 6 | 7 | ### 1.0.2 (2022-03-01) 8 | * Feature: QuestDB support 9 | 10 | ### 1.0.1 (2022-02-26) 11 | * Feature: Add support for socket backends (TCP, UDS, UDP) 12 | * Example: Add example for receiving TCP messages from cryptofeed via container 13 | * Feature: Support for InfluxDB 14 | 15 | ### 1.0.0 (2022-02-18) 16 | * Update: Old repo moved to legacy branch, new version (1.X) created. 17 | * Feature: Support for public channels on Redis and Mongo 18 | * Bugfix: Input validation, remove unused callbacks from Feed. 19 | * Docs: Start documentation. 20 | * Feature: Add support for Redis Streams 21 | * Feature: Add support for Postgres 22 | -------------------------------------------------------------------------------- /examples/tcp.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (C) 2018-2024 Bryant Moscon - bmoscon@gmail.com 3 | 4 | Please see the LICENSE file for the terms and conditions 5 | associated with this software. 6 | 7 | 8 | Run this file, then start the docker container with 9 | 10 | 11 | docker run -e EXCHANGE='COINBASE' \ 12 | -e CHANNELS='trades' \ 13 | -e SYMBOLS='BTC-USD' \ 14 | -e BACKEND='TCP' \ 15 | -e HOST='tcp://127.0.0.1' \ 16 | -e PORT=8080 \ 17 | cryptostore:latest 18 | ''' 19 | import asyncio 20 | from decimal import Decimal 21 | import json 22 | 23 | 24 | async def reader(reader, writer): 25 | while True: 26 | data = await reader.read(1024 * 640) 27 | message = data.decode() 28 | # if multiple messages are received back to back, 29 | # need to make sure they are formatted as if in an array 30 | message = message.replace("}{", "},{") 31 | message = f"[{message}]" 32 | message = json.loads(message, parse_float=Decimal) 33 | 34 | addr = writer.get_extra_info('peername') 35 | 36 | print(f"Received {message!r} from {addr!r}") 37 | 38 | 39 | async def main(): 40 | server = await asyncio.start_server(reader, '127.0.0.1', 8080) 41 | await server.serve_forever() 42 | 43 | 44 | if __name__ == '__main__': 45 | asyncio.run(main()) 46 | -------------------------------------------------------------------------------- /tools/generate.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (C) 2018-2024 Bryant Moscon - bmoscon@gmail.com 3 | Please see the LICENSE file for the terms and conditions 4 | associated with this software. 5 | 6 | 7 | Requires a config file named "config.yaml" in the same directory, 8 | see provided example. 9 | ''' 10 | import yaml 11 | 12 | from cryptofeed.exchanges import EXCHANGE_MAP 13 | 14 | 15 | def main(): 16 | with open("config.yaml", 'r') as fp: 17 | config = yaml.safe_load(fp) 18 | 19 | kwargs = config.copy() 20 | [kwargs.pop(key) for key in ('backend', 'channels', 'exchanges', 'symbols', 'symbols_per_channel')] 21 | kwargs_str = '-e ' + ' -e '.join([f"{key.upper()}='{value}'" for key, value in kwargs.items()]) 22 | 23 | for exchange in config['exchanges']: 24 | eobj = EXCHANGE_MAP[exchange.upper()]() 25 | symbols = eobj.symbols() 26 | 27 | if config['symbols'][exchange] != 'ALL': 28 | if any([s not in symbols for s in config['symbols'][exchange]]): 29 | print("Invalid symbol specified") 30 | return 31 | symbols = config['symbols'][exchange] 32 | 33 | for i in range(0, len(symbols), config['symbols_per_channel']): 34 | for chan in config['channels']: 35 | print(f"docker run -e EXCHANGE={exchange.upper()} -e CHANNELS={chan} -e SYMBOLS='{','.join(symbols[i:i+config['symbols_per_channel']])}' -e BACKEND={config['backend']} {kwargs_str} ghcr.io/bmoscon/cryptostore:latest") 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .vscode/ 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | # PyCharm / jetbrains 109 | .idea 110 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cryptostore 2 | 3 | [![License](https://img.shields.io/badge/license-XFree86-blue.svg)](LICENSE) 4 | ![Python](https://img.shields.io/badge/Python-3.8+-green.svg) 5 | [![Docker Image](https://img.shields.io/badge/Docker-cryptostore-brightgreen.svg)](https://github.com/bmoscon/cryptostore/pkgs/container/cryptostore) 6 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/da2a982c976649e193c807895ee7a33c)](https://www.codacy.com/manual/bmoscon/cryptostore?utm_source=github.com&utm_medium=referral&utm_content=bmoscon/cryptostore&utm_campaign=Badge_Grade) 7 | 8 | Cryptostore is an application wrapper around [Cryptofeed](https://github.com/bmoscon/cryptofeed) that runs in containers, for the purpose of storing cryptocurrency data directly to various databases and message protocols. 9 | 10 | This project assumes familiarity with Docker, but some basic commands are available to get you started below. 11 | 12 | ### Using a Prebuilt Container Image 13 | 14 | Docker images are hosted in GitHub and can be pulled using the following command: 15 | 16 | ``` 17 | docker pull ghcr.io/bmoscon/cryptostore:latest 18 | ``` 19 | 20 | ### To Build a Container From Source 21 | 22 | ``` 23 | docker build . -t "cryptostore:latest" 24 | ``` 25 | 26 | 27 | ### To Run a Container 28 | 29 | ``` 30 | docker run -e EXCHANGE='COINBASE' -e CHANNELS='trades' -e SYMBOLS='BTC-USD' -e BACKEND='REDIS' cryptostore:latest 31 | ``` 32 | 33 | **Note**: if you've pulled the image from GitHub, the container name will be `ghcr.io/bmoscon/cryptostore:latest` as opposed to `cryptostore:latest`. 34 | 35 | 36 | Depending on your operating system and how your backends are set up, networking configuration may need to be supplied to docker, or other backend specific environment variables might need to be supplied. 37 | 38 | Configuration is passed to the container via environment variables. `CHANNELS` and `SYMBOLS` can be single values, or list of values. Only one exchange per container is supported. 39 | 40 | 41 | ### Documentation 42 | 43 | For more information about usage, see the [documentation](docs/). 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2018-2024 Bryant Moscon - bmoscon@gmail.com 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to 5 | deal in the Software without restriction, including without limitation the 6 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | sell copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | 1. Redistributions of source code must retain the above copyright notice, 11 | this list of conditions, and the following disclaimer. 12 | 13 | 2. Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution, and in the same 16 | place and form as other copyright, license and disclaimer information. 17 | 18 | 3. The end-user documentation included with the redistribution, if any, must 19 | include the following acknowledgment: "This product includes software 20 | developed by Bryant Moscon (http://www.bryantmoscon.com/)", in the same 21 | place and form as other third-party acknowledgments. Alternately, this 22 | acknowledgment may appear in the software itself, in the same form and 23 | location as other such third-party acknowledgments. 24 | 25 | 4. Except as contained in this notice, the name of the author, Bryant Moscon, 26 | shall not be used in advertising or otherwise to promote the sale, use or 27 | other dealings in this Software without prior written authorization from 28 | the author. 29 | 30 | 31 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 32 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 33 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 34 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 35 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 36 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 37 | THE SOFTWARE. 38 | -------------------------------------------------------------------------------- /docs/overview.md: -------------------------------------------------------------------------------- 1 | ## High Level Overview 2 | 3 | 4 | Cryptostore utilizes the supported backends of cryptofeed to store data from exchanges in various databases and other services. The following are the supported backends along with the flag that should be used in the BACKEND environment variable: 5 | 6 | * Redis Sorted Sets (ZSET) - `REDIS` 7 | * Redis Streams - `REDISSTREAM` 8 | * MongoDB - `MONGO` 9 | * Postgres - `POSTGRES` 10 | * TCP - `TCP` 11 | * UDP - `UDP` 12 | * UDS - `UDS` 13 | * InfluxDB - `INFLUX` 14 | * QuestDB - `QUEST` 15 | 16 | Cryptostore runs in a docker container, and expects configuration to be provided to it via environment variables. The env vars it utilizes (not all are required for all configurations) are: 17 | 18 | * `EXCHANGE` - the exchange. Only one can be specified. Run multiple containers to collect data from more than 1 exchange. Should be in all caps. 19 | * `SYMBOLS` - a single symbol, or a list of symbols. Must follow cryptofeed naming conventions for symbols. 20 | * `CHANNELS` - cryptofeed data channels (`trades`, `l2_book`, `ticker`, etc.) Can be a single channel or a list of channels. 21 | * `CONFIG` - path to cryptofeed config file (must be built into the container or otherwise accessible in the container). Not required. 22 | * `BACKEND` - Backend to be used, see list of supported backends above. 23 | * `SAVE_RAW` - Keep raw data and save it to file. Default is False, and even when True you still have to specify a backend for the processed data. To persist the raw data files you should bind the /raw_data folder in the container to a local volume. E.g. add `-v /your/local/path:/raw_data` to your `docker run` command. 24 | * `SNAPSHOT_ONLY` - Valid for orderbook channel only, specifies that only complete books should be stored. Default is False. 25 | * `SNAPSHOT_INTERVAL` - Specifies how often snapshot is stored in terms of number of delta updates between snapshots. Default is 1000. 26 | * `HOST` - Host for backend. Defaults to `localhost`. TCP, UDP and UDS require the protocol to be prepended to the host/url. E.g. `tcp://127.0.0.1`, `uds://udsfile.tmp`, etc. 27 | * `PORT` - Port for service. Defaults to backend default. 28 | * `CANDLE_INTERVAL` - Used for candles. Default is 1m. 29 | * `DATABASE` - Specify the database for MongoDB or Postgres 30 | * `USER` - the username for Postgres 31 | * `PASSWORD` - the password for the specified Postgres user 32 | * `ORG` - the InfluxDB organization 33 | * `BUCKET` - the InfluxDB bucket 34 | * `TOKEN` - the InfluxDB token 35 | 36 | 37 | ### Example 38 | 39 | An example of how the configuration would look when subscribing to BTC-USD and ETH-USD on Coinbase for Trades, Ticker and L2 Book channels, using Redis Streams: 40 | 41 | ``` 42 | docker run -e EXCHANGE='COINBASE' \ 43 | -e CHANNELS='trades,ticker,l2_book' \ 44 | -e SYMBOLS='BTC-USD,ETH-USD' \ 45 | -e BACKEND='REDISSTREAM' \ 46 | ... networking and other params ... \ 47 | cryptostore:latest 48 | ``` 49 | -------------------------------------------------------------------------------- /examples/compose/cryptostore2tty/README.md: -------------------------------------------------------------------------------- 1 | Usage: `docker compose up` 2 | 3 | Console displays 4 | ``` 5 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00221000 price: 31151.25000000 id: 2673492213 type: None timestamp: 1689354730.092 6 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00219000 price: 31151.32000000 id: 2673492214 type: None timestamp: 1689354730.092 7 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00107000 price: 31151.33000000 id: 2673492215 type: None timestamp: 1689354730.092 8 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00033000 price: 31151.59000000 id: 2673492216 type: None timestamp: 1689354730.092 9 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00087000 price: 31151.72000000 id: 2673492217 type: None timestamp: 1689354730.092 10 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00081000 price: 31151.99000000 id: 2673492218 type: None timestamp: 1689354730.096 11 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00081000 price: 31151.99000000 id: 2673492219 type: None timestamp: 1689354730.096 12 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00347000 price: 31151.99000000 id: 2673492220 type: None timestamp: 1689354730.117 13 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00238000 price: 31151.99000000 id: 2673492221 type: None timestamp: 1689354730.122 14 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00174000 price: 31151.99000000 id: 2673492222 type: None timestamp: 1689354730.134 15 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00123000 price: 31151.99000000 id: 2673492223 type: None timestamp: 1689354730.145 16 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00123000 price: 31151.99000000 id: 2673492224 type: None timestamp: 1689354730.156 17 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00123000 price: 31151.99000000 id: 2673492225 type: None timestamp: 1689354730.182 18 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: sell amount: 0.00162000 price: 31151.98000000 id: 2673492226 type: None timestamp: 1689354730.213 19 | cryptostore-cryptostore-1 | 2023-07-14 17:12:10 - exchange: BINANCE symbol: BTC-USDT side: buy amount: 0.00369000 price: 31151.99000000 id: 2673492227 type: None timestamp: 1689354730.385 20 | ``` 21 | 22 | but data are not stored 23 | -------------------------------------------------------------------------------- /cryptostore.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (C) 2018-2024 Bryant Moscon - bmoscon@gmail.com 3 | Please see the LICENSE file for the terms and conditions 4 | associated with this software. 5 | ''' 6 | from datetime import datetime 7 | import os 8 | 9 | from cryptofeed import FeedHandler 10 | from cryptofeed.raw_data_collection import AsyncFileCallback 11 | from cryptofeed.exchanges import EXCHANGE_MAP 12 | from cryptofeed.feed import Feed 13 | from cryptofeed.defines import L2_BOOK, TICKER, TRADES, FUNDING, CANDLES, OPEN_INTEREST, LIQUIDATIONS 14 | from cryptofeed.backends.redis import BookRedis, TradeRedis, TickerRedis, FundingRedis, CandlesRedis, OpenInterestRedis, LiquidationsRedis 15 | from cryptofeed.backends.redis import BookStream, TradeStream, TickerStream, FundingStream, CandlesStream, OpenInterestStream, LiquidationsStream 16 | from cryptofeed.backends.mongo import BookMongo, TradeMongo, TickerMongo, FundingMongo, CandlesMongo, OpenInterestMongo, LiquidationsMongo 17 | from cryptofeed.backends.postgres import BookPostgres, TradePostgres, TickerPostgres, FundingPostgres, CandlesPostgres, OpenInterestPostgres, LiquidationsPostgres 18 | from cryptofeed.backends.socket import BookSocket, TradeSocket, TickerSocket, FundingSocket, CandlesSocket, OpenInterestSocket, LiquidationsSocket 19 | from cryptofeed.backends.influxdb import BookInflux, TradeInflux, TickerInflux, FundingInflux, CandlesInflux, OpenInterestInflux, LiquidationsInflux 20 | from cryptofeed.backends.quest import BookQuest, TradeQuest, TickerQuest, FundingQuest, CandlesQuest, OpenInterestQuest, LiquidationsQuest 21 | 22 | 23 | async def tty(obj, receipt_ts): 24 | # For debugging purposes 25 | rts = datetime.utcfromtimestamp(receipt_ts).strftime('%Y-%m-%d %H:%M:%S') 26 | print(f"{rts} - {obj}") 27 | 28 | 29 | def load_config() -> Feed: 30 | exchange = os.environ.get('EXCHANGE') 31 | symbols = os.environ.get('SYMBOLS') 32 | 33 | if symbols is None: 34 | raise ValueError("Symbols must be specified") 35 | symbols = symbols.split(",") 36 | 37 | channels = os.environ.get('CHANNELS') 38 | if channels is None: 39 | raise ValueError("Channels must be specified") 40 | channels = channels.split(",") 41 | 42 | config = os.environ.get('CONFIG') 43 | backend = os.environ.get('BACKEND') 44 | snap_only = os.environ.get('SNAPSHOT_ONLY', False) 45 | if snap_only: 46 | if snap_only.lower().startswith('f'): 47 | snap_only = False 48 | elif snap_only.lower().startswith('t'): 49 | snap_only = True 50 | else: 51 | raise ValueError('Invalid value specified for SNAPSHOT_ONLY') 52 | snap_interval = os.environ.get('SNAPSHOT_INTERVAL', 1000) 53 | snap_interval = int(snap_interval) 54 | host = os.environ.get('HOST', '127.0.0.1') 55 | port = os.environ.get('PORT') 56 | if port: 57 | port = int(port) 58 | candle_interval = os.environ.get('CANDLE_INTERVAL', '1m') 59 | database = os.environ.get('DATABASE') 60 | user = os.environ.get('USER') 61 | password = os.environ.get('PASSWORD') 62 | org = os.environ.get('ORG') 63 | bucket = os.environ.get('BUCKET') 64 | token = os.environ.get('TOKEN') 65 | 66 | cbs = None 67 | allowed_backends = ['REDIS', 'REDISSTREAM', 'MONGO', 'POSTGRES', 'TCP', 'UDP', 'UDS', 'INFLUX', 'QUEST', 'TTY'] 68 | if backend in allowed_backends: 69 | if backend == 'REDIS' or backend == 'REDISSTREAM': 70 | kwargs = {'host': host, 'port': port if port else 6379} 71 | cbs = { 72 | L2_BOOK: BookRedis(snapshot_interval=snap_interval, snapshots_only=snap_only, **kwargs) if backend == 'REDIS' else BookStream(snapshot_interval=snap_interval, snapshots_only=snap_only, **kwargs), 73 | TRADES: TradeRedis(**kwargs) if backend == 'REDIS' else TradeStream(**kwargs), 74 | TICKER: TickerRedis(**kwargs) if backend == 'REDIS' else TickerStream(**kwargs), 75 | FUNDING: FundingRedis(**kwargs) if backend == 'REDIS' else FundingStream(**kwargs), 76 | CANDLES: CandlesRedis(**kwargs) if backend == 'REDIS' else CandlesStream(**kwargs), 77 | OPEN_INTEREST: OpenInterestRedis(**kwargs) if backend == 'REDIS' else OpenInterestStream(**kwargs), 78 | LIQUIDATIONS: LiquidationsRedis(**kwargs) if backend == 'REDIS' else LiquidationsStream(**kwargs) 79 | } 80 | elif backend == 'MONGO': 81 | kwargs = {'host': host, 'port': port if port else 27101} 82 | cbs = { 83 | L2_BOOK: BookMongo(database, snapshot_interval=snap_interval, snapshots_only=snap_only, **kwargs), 84 | TRADES: TradeMongo(database, **kwargs), 85 | TICKER: TickerMongo(database, **kwargs), 86 | FUNDING: FundingMongo(database, **kwargs), 87 | CANDLES: CandlesMongo(database, **kwargs), 88 | OPEN_INTEREST: OpenInterestMongo(database, **kwargs), 89 | LIQUIDATIONS: LiquidationsMongo(database, **kwargs) 90 | } 91 | elif backend == 'POSTGRES': 92 | kwargs = {'db': database, 'host': host, 'port': port if port else 5432, 'user': user, 'pw': password} 93 | cbs = { 94 | L2_BOOK: BookPostgres(snapshot_interval=snap_interval, snapshots_only=snap_only, **kwargs), 95 | TRADES: TradePostgres(**kwargs), 96 | TICKER: TickerPostgres(**kwargs), 97 | FUNDING: FundingPostgres(**kwargs), 98 | CANDLES: CandlesPostgres(**kwargs), 99 | OPEN_INTEREST: OpenInterestPostgres(**kwargs), 100 | LIQUIDATIONS: LiquidationsPostgres(**kwargs) 101 | } 102 | elif backend in ('TCP', 'UDP', 'UDS'): 103 | kwargs = {'port': port} 104 | cbs = { 105 | L2_BOOK: BookSocket(host, snapshot_interval=snap_interval, snapshots_only=snap_only, **kwargs), 106 | TRADES: TradeSocket(host, **kwargs), 107 | TICKER: TickerSocket(host, **kwargs), 108 | FUNDING: FundingSocket(host, **kwargs), 109 | CANDLES: CandlesSocket(host, **kwargs), 110 | OPEN_INTEREST: OpenInterestSocket(host, **kwargs), 111 | LIQUIDATIONS: LiquidationsSocket(host, **kwargs) 112 | } 113 | elif backend == 'INFLUX': 114 | args = (host, org, bucket, token) 115 | cbs = { 116 | L2_BOOK: BookInflux(*args, snapshot_interval=snap_interval, snapshots_only=snap_only), 117 | TRADES: TradeInflux(*args), 118 | TICKER: TickerInflux(*args), 119 | FUNDING: FundingInflux(*args), 120 | CANDLES: CandlesInflux(*args), 121 | OPEN_INTEREST: OpenInterestInflux(*args), 122 | LIQUIDATIONS: LiquidationsInflux(*args) 123 | } 124 | elif backend == 'QUEST': 125 | kwargs = {'host': host, 'port': port if port else 9009} 126 | cbs = { 127 | L2_BOOK: BookQuest(**kwargs), 128 | TRADES: TradeQuest(**kwargs), 129 | TICKER: TickerQuest(**kwargs), 130 | FUNDING: FundingQuest(**kwargs), 131 | CANDLES: CandlesQuest(**kwargs), 132 | OPEN_INTEREST: OpenInterestQuest(**kwargs), 133 | LIQUIDATIONS: LiquidationsQuest(**kwargs) 134 | } 135 | elif backend == 'TTY': 136 | cbs = { 137 | L2_BOOK: tty, 138 | TRADES: tty, 139 | TICKER: tty, 140 | FUNDING: tty, 141 | CANDLES: tty, 142 | OPEN_INTEREST: tty, 143 | LIQUIDATIONS: tty 144 | } 145 | else: 146 | raise ValueError(f"Invalid backend '{backend}' specified - must be in {allowed_backends}") 147 | 148 | # Prune unused callbacks 149 | remove = [chan for chan in cbs if chan not in channels] 150 | for r in remove: 151 | del cbs[r] 152 | 153 | return EXCHANGE_MAP[exchange](candle_interval=candle_interval, symbols=symbols, channels=channels, config=config, callbacks=cbs) 154 | 155 | 156 | def main(): 157 | save_raw = os.environ.get('SAVE_RAW', False) 158 | if save_raw: 159 | if save_raw.lower().startswith('f'): 160 | save_raw = False 161 | elif save_raw.lower().startswith('t'): 162 | save_raw = True 163 | else: 164 | raise ValueError('Invalid value specified for SAVE_RAW') 165 | 166 | fh = FeedHandler(raw_data_collection=AsyncFileCallback("./raw_data") if save_raw else None) 167 | cfg = load_config() 168 | fh.add_feed(cfg) 169 | fh.run() 170 | 171 | 172 | if __name__ == '__main__': 173 | main() 174 | --------------------------------------------------------------------------------