├── .gitignore ├── .gitmodules ├── CHANGELOG.md ├── README.md ├── clients ├── docker-compose.yml └── env.template ├── dashboard ├── docker-compose.yml └── env.template ├── ingest ├── docker-compose.yml └── env.template ├── rest ├── docker-compose.yml └── env.template └── transformation ├── docker-compose.yml └── env.template /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "clients/zec-client"] 2 | path = clients/zec-client 3 | url = https://github.com/graphsense/zec-client 4 | branch = master 5 | [submodule "clients/ltc-client"] 6 | path = clients/ltc-client 7 | url = https://github.com/graphsense/ltc-client 8 | branch = master 9 | [submodule "clients/bch-client"] 10 | path = clients/bch-client 11 | url = https://github.com/graphsense/bch-client 12 | branch = master 13 | [submodule "clients/btc-client"] 14 | path = clients/btc-client 15 | url = https://github.com/graphsense/btc-client 16 | branch = master 17 | [submodule "clients/eth-client"] 18 | path = clients/eth-client 19 | url = https://github.com/graphsense/eth-client 20 | branch = master 21 | [submodule "ingest/graphsense-blocksci"] 22 | path = ingest/graphsense-blocksci 23 | url = https://github.com/graphsense/graphsense-blocksci 24 | branch = master 25 | [submodule "ingest/graphsense-ethereum-etl"] 26 | path = ingest/graphsense-ethereum-etl 27 | url = https://github.com/graphsense/graphsense-ethereum-etl 28 | branch = master 29 | [submodule "transformation/graphsense-transformation"] 30 | path = transformation/graphsense-transformation 31 | url = https://github.com/graphsense/graphsense-transformation 32 | branch = master 33 | [submodule "transformation/graphsense-ethereum-transformation"] 34 | path = transformation/graphsense-ethereum-transformation 35 | url = https://github.com/graphsense/graphsense-ethereum-transformation 36 | branch = master 37 | [submodule "rest/graphsense-rest"] 38 | path = rest/graphsense-rest 39 | url = https://github.com/graphsense/graphsense-rest 40 | branch = master 41 | [submodule "dashboard/graphsense-dashboard"] 42 | path = dashboard/graphsense-dashboard 43 | url = https://github.com/graphsense/graphsense-dashboard 44 | branch = master 45 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). 5 | 6 | ## [0.5.2] - 2022-03-25 7 | 8 | ## [0.5.1] - 2021-11-30 9 | ### Added 10 | - bitcoin-etl based ingest to support BTC Taproot upgrade (graphsense/graphsense-bitcoin-etl) 11 | 12 | ## [0.5.0] - 2021-06-02 13 | ### Added 14 | - Ethereum support (blocks and external transactions) 15 | 16 | ## [0.4.5] - 2020-11-19 17 | - Initial release 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GraphSense Setup 2 | 3 | Docker compose setup for GraphSense components. 4 | 5 | 6 | ## About 7 | 8 | The goal of this project is to allow an out-of-the-box yet customizable setup 9 | of all required GraphSense components. Graphsense consists of several services. 10 | 11 | List of components: 12 | - `clients`: dockerized cryptocurrency clients for Bitcoin, Bitcoin Cash, 13 | Litecoin, Zcash and Ethereum (`geth`). 14 | - `ingest`: these services are part of the 15 | [graphsense-blocksci][gs-blocksci] project to parse and ingest 16 | Blockchain data and exchange rates for all supported UTXO currencies. 17 | Ethereum support is provided by the [graphsense-ethereum-etl][gs-ethereum-etl] 18 | component. 19 | - `transformation`: a Spark job that reads raw data from Cassandra and writes 20 | the computed the address and entity graphs to a transformed keyspace. 21 | - `rest`: a Flask REST service exposing denormalized views computed by the 22 | transformation pipeline. 23 | - `dashboard`: a front-end dashboard for interactive cryptocurrency analysis. 24 | 25 | Although each one is required for the platform to work, some of them must be run 26 | in a sequential order. 27 | 28 | 29 | ## Prerequisites 30 | 31 | - Git version control system 32 | - [Docker][docker], see e.g. https://docs.docker.com/engine/install/ 33 | - Docker Compose: https://docs.docker.com/compose/install/ 34 | - a machine with at least 60GB RAM to run the `graphsense-blocksci` component 35 | - standalone [Apache Spark][apache-spark] cluster (version 3.1.2/Scala 2.12) 36 | - [Apache Cassandra][apache-cassandra] cluster 37 | 38 | All containers run with UID 10000 (user `dockeruser`). Ensure that a user 39 | UID `10000` exists on your local system(s) and that mapped local volumes are 40 | owned by this user. 41 | 42 | 43 | ## Setup 44 | 45 | Each GraphSense component has its own `Dockerfile` and can be built and run 46 | individually (please consult the `README` files). Most of the times however, 47 | one would want to quickly set up a set of services and be sure that they are 48 | going to work well together. 49 | 50 | For that, we provide a set of `docker-compose.yml` files in the corresponding 51 | subdirectories. The source code of all required GraphSense components is 52 | included through Git submodules. To fetch the source code for all components use 53 | 54 | ``` 55 | git clone https://github.com/graphsense/graphsense-setup.git . 56 | git submodule init 57 | git submodule update 58 | ``` 59 | 60 | The components must be set up/run in the following order: 61 | 62 | - clients 63 | - ingest 64 | - transformation 65 | - rest 66 | - dashboard 67 | 68 | In each subdirectory copy the `env.template` file to `.env` and fill in all 69 | parameters in the `.env` files. Verify that all parameters have a value set: 70 | 71 | ``` 72 | docker-compose config 73 | ``` 74 | 75 | 76 | ### Clients 77 | 78 | The `client` subdirectory contains the source code and Docker setup for 79 | all supported cryptocurrency clients. To build the Docker images, change 80 | to the `client` directory and execute 81 | 82 | ``` 83 | docker-compose build 84 | ``` 85 | 86 | In the Docker Compose file, we define the following services: 87 | 88 | - `bitcoin-client` 89 | - `bitcoin-cash-client` 90 | - `litecoin-client` 91 | - `zcash-client` 92 | - `ethereum-client` 93 | 94 | Before starting the containers, create all directories as defined in the `.env` 95 | file (must be writable by UID 10000). To start all services in detached mode 96 | (i.e., to run the containers in the 97 | background), execute 98 | 99 | ``` 100 | docker-compose up -d 101 | ``` 102 | 103 | List all running containers: 104 | 105 | ``` 106 | > docker-compose ps 107 | Name Command State Ports 108 | ------------------------------------------------------------------------------ 109 | bitcoin bitcoind -conf=/opt/graphs ... Up 0.0.0.0:8332->8332/tcp 110 | bitcoin-cash bitcoind -conf=/opt/graphs ... Up 0.0.0.0:8432->8432/tcp 111 | geth geth --syncmode full --rpc ... Up 30303/tcp, 30303/udp, 112 | 0.0.0.0:8545->8545/tcp, 113 | 8546/tcp, 8547/tcp 114 | litecoin litecoind -conf=/opt/graph ... Up 0.0.0.0:8532->8532/tcp 115 | zcash zcashd -conf=/opt/graphsen ... Up 0.0.0.0:8632->8632/tcp 116 | ``` 117 | 118 | To keep track of the client synchronization process, use 119 | 120 | ``` 121 | # all clients 122 | docker-compose logs 123 | # only BTC 124 | docker-compose logs -f bitcoin-client 125 | ``` 126 | 127 | 128 | ### Ingest 129 | 130 | The `ingest` directory includes the `graphsense-blocksci` component, which 131 | employs [BlockSci][blocksci] for parsing blockchains and filtering CoinJoins. 132 | 133 | The Docker Compose file defines services 134 | - to parse the blockchain data gathered by the cryptocurrency clients and 135 | ingest it to an existing, external Apache Cassandra cluster, and 136 | - to retrieve and ingest exchange rates from [CoinDesk][coindesk] or 137 | [CoinMarketCap][coinmarketcap]. 138 | 139 | As above for the clients, build the Docker image using 140 | 141 | ``` 142 | docker-compose build 143 | ``` 144 | 145 | To parse and ingest the data, the Docker Compose setup defines three services 146 | for each cryptocurrency, which should run sequentially. E.g., for Bitcoin 147 | 148 | ``` 149 | # blockchain parsing 150 | docker-compose run parser-btc 151 | # ingest parsed data into raw Cassandra keyspace 152 | # (by default, up to the last block of previous day, since exchange rates 153 | # are not available for the current day) 154 | docker-compose run ingest-btc 155 | # ingest exchange rates to Cassandra 156 | docker-compose run ingest-exchange-rates-btc 157 | ``` 158 | 159 | To parse the Bitcoin blockchain, BlockSci requires at least 60GB of RAM. 160 | As of November 2020, BlockSci is no longer actively being under development, 161 | which means that the BTC Taproot upgrade (BTC block 709,632, 2021-11-14) 162 | is not supported. To ingest recent BTC transactions, please use the 163 | [bitcoin-etl][bitcoin-etl] based ingest script 164 | (see [graphsense-bitcoin-etl][graphsense-bitcoin-etl]). 165 | 166 | After successful parsing and ingest, execute 167 | 168 | ``` 169 | docker-compose down 170 | ``` 171 | 172 | to remove all containers and volumes. 173 | 174 | Ethereum support is provided by the `graphsense-ethereum-etl` component. 175 | Ingest of blocks and transactions 176 | 177 | ``` 178 | docker-compose run ingest-eth 179 | ``` 180 | 181 | Ingest of ETH exchange rates 182 | 183 | ``` 184 | docker-compose run ingest-exchange-rates-eth 185 | ``` 186 | 187 | #### GraphSense tagpacks (optional) 188 | 189 | There is currently no Docker setup for the ingest of attribution tags. 190 | The ingest of TagPacks to the raw Cassandra keyspace needs to be 191 | performed manually, see [graphsense-tagpack-tool][gs-tagpack-tool] 192 | for further information. 193 | 194 | 195 | ### Transformation 196 | 197 | The `transformation` directory contains dockerized versions of the Spark 198 | transformation pipelines ([graphsense-transformation][gs-transformation] 199 | and [graphsense-ethereum-transformation][gs-eth-transformation]). 200 | 201 | The current Docker setup requires an existing, external Spark standalone cluster 202 | (Spark version 3.1.2), which could also be deployed using Docker 203 | (as [Docker Swarm](https://docs.docker.com/get-started/swarm-deploy/)). 204 | The environment variable `SPARK_DRIVER_HOST` specifies the network address of 205 | the host machine where the container will be running. Spark cluster nodes should 206 | be able to resolve this address. This is necessary for communication between 207 | executors and the driver program. For detailed technical information see 208 | [SPARK-4563](https://issues.apache.org/jira/browse/SPARK-4563). 209 | The option `spark.driver.bindAddress` is set to `0.0.0.0` inside the container. 210 | It allows a different address from the local one to be advertised to 211 | executors or external systems, which is necessary when running containers 212 | with bridged networking. For this to properly work, the different 213 | ports used by the driver need to be forwarded from the container's host 214 | (`SPARK_DRIVER_PORT`,`SPARK_UI_PORT`, `SPARK_BLOCKMGR_PORT`). 215 | 216 | After finishing the Spark configuration edit the arguments section in the 217 | `.env` file, and build and run the containers. For the transformation 218 | pipeline for TUXO currencies, use 219 | 220 | ``` 221 | docker-compose build 222 | docker-compose run --service-ports transformation 223 | ``` 224 | 225 | and for the Ethereum transformation, use 226 | 227 | ``` 228 | docker-compose build 229 | docker-compose run --service-ports ethereum-transformation 230 | ``` 231 | 232 | Note that the `--service-ports` option is required, to enable the port mapping 233 | to the host. 234 | 235 | 236 | ### Rest 237 | 238 | The `rest` subdirectory contains the Docker Compose setup for the 239 | `graphsense-rest` component. 240 | Change to the `rest` directory and copy the template config file 241 | 242 | ``` 243 | mkdir -p graphsense-rest/instance 244 | cp graphsense-rest/conf/config.yaml.template graphsense-rest/instance/config.yaml 245 | ``` 246 | 247 | and edit all required parameters. 248 | 249 | Furthermore, edit the `.env` file and build and start the service 250 | 251 | ``` 252 | docker-compose build 253 | docker-compose up -d 254 | ``` 255 | 256 | The REST service will be accessible at `0.0.0.0:REST_PORT`. `REST_PORT` is 257 | configured through the `.env` file. 258 | 259 | The view the generated API documentation, open `http://127.0.0.1:REST_PORT/ui` 260 | in a web browser. 261 | 262 | ### Dashboard 263 | 264 | Edit the `.env` file and build and start the service: 265 | 266 | ``` 267 | docker-compose build 268 | docker-compose up -d 269 | ``` 270 | 271 | The dashboard is going to be accessible at `0.0.0.0:DASHBOARD_PORT`. 272 | `DASHBOARD_PORT` is configured through the `.env` file. 273 | 274 | 275 | [apache-spark]: https://spark.apache.org/downloads.html 276 | [apache-cassandra]: http://cassandra.apache.org/download 277 | [gs-blocksci]: https://github.com/graphsense/graphsense-blocksci 278 | [gs-ethereum-etl]: https://github.com/graphsense/graphsense-ethereum-etl 279 | [docker]: https://www.docker.com 280 | [blocksci]: https://github.com/citp/BlockSci 281 | [bitcoin-etl]: https://github.com/blockchain-etl/bitcoin-etl 282 | [graphsense-bitcoin-etl]: https://github.com/graphsense/graphsense-bitcoin-etl 283 | [coindesk]: https://www.coindesk.com/api 284 | [coinmarketcap]: https://coinmarketcap.com 285 | [gs-tagpack-tool]: https://github.com/graphsense/graphsense-tagpack-tool 286 | [gs-transformation]: https://github.com/graphsense/graphsense-transformation 287 | [gs-eth-transformation]: https://github.com/graphsense/graphsense-ethereum-transformation 288 | -------------------------------------------------------------------------------- /clients/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.5" 2 | 3 | services: 4 | 5 | bitcoin-client: 6 | image: bitcoin 7 | container_name: bitcoin 8 | hostname: bitcoin 9 | build: 10 | context: btc-client 11 | volumes: 12 | - ${BTC_BLOCK_DATADIR}:/opt/graphsense/data 13 | - ${BTC_CLIENT_CONF}:/opt/graphsense/client.conf 14 | networks: 15 | - graphsense-net 16 | ports: 17 | - 8332:8332 18 | restart: always 19 | 20 | bitcoin-cash-client: 21 | image: bitcoin-cash 22 | container_name: bitcoin-cash 23 | hostname: bitcoin-cash 24 | build: 25 | context: bch-client 26 | volumes: 27 | - ${BCH_BLOCK_DATADIR}:/opt/graphsense/data 28 | - ${BCH_CLIENT_CONF}:/opt/graphsense/client.conf 29 | networks: 30 | - graphsense-net 31 | ports: 32 | - 8432:8432 33 | restart: always 34 | 35 | litecoin-client: 36 | image: litecoin 37 | container_name: litecoin 38 | hostname: litecoin 39 | build: 40 | context: ltc-client 41 | volumes: 42 | - ${LTC_BLOCK_DATADIR}:/opt/graphsense/data 43 | - ${LTC_CLIENT_CONF}:/opt/graphsense/client.conf 44 | networks: 45 | - graphsense-net 46 | ports: 47 | - 8532:8532 48 | restart: always 49 | 50 | zcash-client: 51 | image: zcash 52 | container_name: zcash 53 | hostname: zcash 54 | build: 55 | context: zec-client 56 | volumes: 57 | - ${ZEC_BLOCK_DATADIR}:/opt/graphsense/data 58 | - ${ZEC_CLIENT_CONF}:/opt/graphsense/client.conf 59 | networks: 60 | - graphsense-net 61 | ports: 62 | - 8632:8632 63 | restart: always 64 | 65 | ethereum-client: 66 | image: openethereum 67 | container_name: openethereum 68 | hostname: openethereum 69 | build: 70 | context: eth-client 71 | volumes: 72 | - ${ETH_BLOCK_DATADIR}:/home/dockeruser/openethereum 73 | - ./eth-client/config.toml:/home/dockeruser/.local/share/io.parity.ethereum/config.toml 74 | networks: 75 | - graphsense-net 76 | ports: 77 | - 8545:8545 78 | restart: always 79 | 80 | 81 | networks: 82 | graphsense-net: 83 | name: graphsense-global-net 84 | -------------------------------------------------------------------------------- /clients/env.template: -------------------------------------------------------------------------------- 1 | # directories to store synced data from cryptocurrency clients 2 | BTC_BLOCK_DATADIR= 3 | BCH_BLOCK_DATADIR= 4 | LTC_BLOCK_DATADIR= 5 | ZEC_BLOCK_DATADIR= 6 | ETH_BLOCK_DATADIR= 7 | # config files for UTXO clients 8 | BTC_CLIENT_CONF=./btc-client/docker/client.conf 9 | BCH_CLIENT_CONF=./bch-client/docker/client.conf 10 | LTC_CLIENT_CONF=./ltc-client/docker/client.conf 11 | ZEC_CLIENT_CONF=./zec-client/docker/client.conf 12 | -------------------------------------------------------------------------------- /dashboard/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.1" 2 | 3 | services: 4 | 5 | graphsense-dashboard: 6 | image: graphsense-dashboard 7 | build: 8 | context: graphsense-dashboard 9 | ports: 10 | - "$DASHBOARD_PORT:8000" 11 | environment: 12 | REST_ENDPOINT: "${REST_ENDPOINT}" 13 | TITANIUM_REPORT_GENERATION_URL: "${TITANIUM_REPORT_GENERATION_URL}" 14 | command: ["nginx", "-g", "daemon off;"] 15 | restart: always 16 | -------------------------------------------------------------------------------- /dashboard/env.template: -------------------------------------------------------------------------------- 1 | # endpoint GraphSense REST API 2 | REST_ENDPOINT="https://api.graphsense.info" 3 | # exposed port of dashboard 4 | DASHBOARD_PORT=8000 5 | # webservice for report generation (optional) 6 | TITANIUM_REPORT_GENERATION_URL="https://titanium-report.graphsense.info" 7 | -------------------------------------------------------------------------------- /ingest/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.1" 2 | 3 | services: 4 | 5 | # BTC 6 | parser-btc: 7 | image: graphsense-blocksci 8 | build: 9 | context: graphsense-blocksci 10 | volumes: 11 | - "$BTC_BLOCKSCI_DATADIR:/var/data/blocksci_data/:rw" 12 | - "$BTC_BLOCK_DATADIR:/var/data/block_data/:ro" 13 | - "$BTC_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 14 | command: ["blocksci_parser", "/opt/graphsense/blocksci.cfg", "update"] 15 | 16 | ingest-btc: 17 | image: graphsense-blocksci 18 | build: 19 | context: graphsense-blocksci 20 | environment: 21 | - CASSANDRA_HOST=${CASSANDRA_HOST} 22 | - RAW_KEYSPACE=${BTC_RAW_KEYSPACE} 23 | - PROCESSES=${PROCESSES} 24 | volumes: 25 | - "$BTC_BLOCKSCI_DATADIR:/var/data/blocksci_data/:ro" 26 | - "$BTC_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 27 | entrypoint: ["/docker-entrypoint.sh"] 28 | command: ["/usr/local/bin/ingest.sh"] 29 | 30 | ingest-exchange-rates-btc: 31 | image: graphsense-blocksci 32 | build: 33 | context: graphsense-blocksci 34 | environment: 35 | - CASSANDRA_HOST=${CASSANDRA_HOST} 36 | - RAW_KEYSPACE=${BTC_RAW_KEYSPACE} 37 | entrypoint: ["/docker-entrypoint.sh"] 38 | command: ["python3", "/usr/local/bin/ingest_rates_coindesk.py", 39 | "-d", "$CASSANDRA_HOST", 40 | "-k", "$BTC_RAW_KEYSPACE"] 41 | 42 | # BCH 43 | parser-bch: 44 | image: graphsense-blocksci 45 | build: 46 | context: graphsense-blocksci 47 | volumes: 48 | - "$BCH_BLOCKSCI_DATADIR:/var/data/blocksci_data/:rw" 49 | - "$BCH_BLOCK_DATADIR:/var/data/block_data/:ro" 50 | - "$BCH_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 51 | command: ["blocksci_parser", "/opt/graphsense/blocksci.cfg", "update"] 52 | 53 | ingest-bch: 54 | image: graphsense-blocksci 55 | build: 56 | context: graphsense-blocksci 57 | environment: 58 | - CASSANDRA_HOST=${CASSANDRA_HOST} 59 | - RAW_KEYSPACE=${BCH_RAW_KEYSPACE} 60 | - PROCESSES=${PROCESSES} 61 | volumes: 62 | - "$BCH_BLOCKSCI_DATADIR:/var/data/blocksci_data/:ro" 63 | - "$BCH_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 64 | entrypoint: ["/docker-entrypoint.sh"] 65 | command: ["/usr/local/bin/ingest.sh"] 66 | 67 | ingest-exchange-rates-bch: 68 | image: graphsense-blocksci 69 | build: 70 | context: graphsense-blocksci 71 | environment: 72 | - CASSANDRA_HOST=${CASSANDRA_HOST} 73 | - RAW_KEYSPACE=${BCH_RAW_KEYSPACE} 74 | entrypoint: ["/docker-entrypoint.sh"] 75 | command: ["python3", "/usr/local/bin/ingest_rates_coinmarketcap.py", 76 | "--cryptocurrency", "BCH", 77 | "-d", "$CASSANDRA_HOST", 78 | "-k", "$BCH_RAW_KEYSPACE"] 79 | 80 | # LTC 81 | parser-ltc: 82 | image: graphsense-blocksci 83 | build: 84 | context: graphsense-blocksci 85 | volumes: 86 | - "$LTC_BLOCKSCI_DATADIR:/var/data/blocksci_data/:rw" 87 | - "$LTC_BLOCK_DATADIR:/var/data/block_data/:ro" 88 | - "$LTC_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 89 | command: ["blocksci_parser", "/opt/graphsense/blocksci.cfg", "update"] 90 | 91 | ingest-ltc: 92 | image: graphsense-blocksci 93 | build: 94 | context: graphsense-blocksci 95 | environment: 96 | - CASSANDRA_HOST=${CASSANDRA_HOST} 97 | - RAW_KEYSPACE=${LTC_RAW_KEYSPACE} 98 | - PROCESSES=${PROCESSES} 99 | volumes: 100 | - "$LTC_BLOCKSCI_DATADIR:/var/data/blocksci_data/:ro" 101 | - "$LTC_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 102 | entrypoint: ["/docker-entrypoint.sh"] 103 | command: ["/usr/local/bin/ingest.sh"] 104 | 105 | ingest-exchange-rates-ltc: 106 | image: graphsense-blocksci 107 | build: 108 | context: graphsense-blocksci 109 | environment: 110 | - CASSANDRA_HOST=${CASSANDRA_HOST} 111 | - RAW_KEYSPACE=${LTC_RAW_KEYSPACE} 112 | entrypoint: ["/docker-entrypoint.sh"] 113 | command: ["python3", "/usr/local/bin/ingest_rates_coinmarketcap.py", 114 | "--cryptocurrency", "LTC", 115 | "-d", "$CASSANDRA_HOST", 116 | "-k", "$LTC_RAW_KEYSPACE"] 117 | 118 | # ZEC 119 | parser-zec: 120 | image: graphsense-blocksci 121 | build: 122 | context: graphsense-blocksci 123 | volumes: 124 | - "$ZEC_BLOCKSCI_DATADIR:/var/data/blocksci_data/:rw" 125 | - "$ZEC_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 126 | command: ["blocksci_parser", "/opt/graphsense/blocksci.cfg", "update"] 127 | 128 | ingest-zec: 129 | image: graphsense-blocksci 130 | build: 131 | context: graphsense-blocksci 132 | environment: 133 | - CASSANDRA_HOST=${CASSANDRA_HOST} 134 | - RAW_KEYSPACE=${ZEC_RAW_KEYSPACE} 135 | - PROCESSES=${PROCESSES} 136 | volumes: 137 | - "$ZEC_BLOCKSCI_DATADIR:/var/data/blocksci_data/:ro" 138 | - "$ZEC_BLOCKSCI_CONFIG:/opt/graphsense/blocksci.cfg" 139 | entrypoint: ["/docker-entrypoint.sh"] 140 | command: ["/usr/local/bin/ingest.sh"] 141 | 142 | ingest-exchange-rates-zec: 143 | image: graphsense-blocksci 144 | build: 145 | context: graphsense-blocksci 146 | environment: 147 | - CASSANDRA_HOST=${CASSANDRA_HOST} 148 | - RAW_KEYSPACE=${ZEC_RAW_KEYSPACE} 149 | entrypoint: ["/docker-entrypoint.sh"] 150 | command: ["python3", "/usr/local/bin/ingest_rates_coinmarketcap.py", 151 | "--cryptocurrency", "ZEC", 152 | "-d", "$CASSANDRA_HOST", 153 | "-k", "$ZEC_RAW_KEYSPACE"] 154 | 155 | ingest-eth: 156 | image: graphsense-ethereum-etl 157 | build: 158 | context: graphsense-ethereum-etl 159 | environment: 160 | - CASSANDRA_HOST=${CASSANDRA_HOST} 161 | - RAW_KEYSPACE=${ETH_RAW_KEYSPACE} 162 | - PROVIDER_URI=${GETH_IPC} 163 | volumes: 164 | - "${ETH_DSBULK_LOG_DIR}:/home/dockeruser/logs/" 165 | - "${ETH_IPC}:/var/run/jsonrpc.ipc:ro" 166 | entrypoint: ["/entrypoint.sh"] 167 | command: ["/usr/local/bin/ingest.sh"] 168 | 169 | ingest-exchange-rates-eth: 170 | image: graphsense-ethereum-etl 171 | build: 172 | context: graphsense-ethereum-etl 173 | environment: 174 | - CASSANDRA_HOST=${CASSANDRA_HOST} 175 | - RAW_KEYSPACE=${ETH_RAW_KEYSPACE} 176 | entrypoint: ["/entrypoint.sh"] 177 | command: ["/home/dockeruser/pandas-venv/bin/python3", 178 | "/usr/local/bin/ingest_rates_coinmarketcap.py", 179 | "--cryptocurrency", "ETH", 180 | "-d", "$CASSANDRA_HOST", 181 | "-k", "$ETH_RAW_KEYSPACE"] 182 | 183 | networks: 184 | default: 185 | external: 186 | name: graphsense-global-net 187 | -------------------------------------------------------------------------------- /ingest/env.template: -------------------------------------------------------------------------------- 1 | # number of parallel processes (Python multiprocessing) 2 | PROCESSES=24 3 | 4 | # Cassandra seed host; IP address (or resolvable hostname) 5 | CASSANDRA_HOST=192.168.243.101 6 | 7 | ### BTC (parsing in disk mode) 8 | # location of BlockSci config file on local machine 9 | BTC_BLOCKSCI_CONFIG=./graphsense-blocksci/conf/btc.cfg 10 | # local directory for BlockSci data 11 | BTC_BLOCKSCI_DATADIR= 12 | # local directory for BTC client data (for disk mode parsing) 13 | BTC_BLOCK_DATADIR= 14 | # name of raw Cassandra keyspace 15 | BTC_RAW_KEYSPACE=btc_raw 16 | 17 | ### BCH (parsing in disk mode) 18 | # location of BlockSci config file on local machine 19 | BCH_BLOCKSCI_CONFIG=./graphsense-blocksci/conf/bch.cfg 20 | # local directory for BlockSci data 21 | BCH_BLOCKSCI_DATADIR= 22 | # local directory for BCH client data (for disk mode parsing) 23 | BCH_BLOCK_DATADIR= 24 | # name of raw Cassandra keyspace 25 | BCH_RAW_KEYSPACE=bch_raw 26 | 27 | ### LTC (parsing in disk mode) 28 | # location of BlockSci config file on local machine 29 | LTC_BLOCKSCI_CONFIG=./graphsense-blocksci/conf/ltc.cfg 30 | # local directory for BlockSci data 31 | LTC_BLOCKSCI_DATADIR= 32 | # local directory for LTC client data (for disk mode parsing) 33 | LTC_BLOCK_DATADIR= 34 | # name of raw Cassandra keyspace 35 | LTC_RAW_KEYSPACE=ltc_raw 36 | 37 | ### ZEC (parsing in RPC mode) 38 | # location of BlockSci config file on local machine 39 | ZEC_BLOCKSCI_CONFIG=./graphsense-blocksci/conf/zec.cfg 40 | # local directory for BlockSci data 41 | ZEC_BLOCKSCI_DATADIR= 42 | # name of raw Cassandra keyspace 43 | ZEC_RAW_KEYSPACE=zec_raw 44 | 45 | ### ETH 46 | # URI of the web3 IPC provider 47 | ETH_IPC= 48 | # local directory for dsbulk log files 49 | ETH_DSBULK_LOG_DIR= 50 | # name of raw Cassandra keyspace 51 | ETH_RAW_KEYSPACE=eth_raw 52 | -------------------------------------------------------------------------------- /rest/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.1" 2 | 3 | services: 4 | 5 | graphsense-rest: 6 | image: graphsense-rest 7 | build: 8 | context: graphsense-rest 9 | args: 10 | NUM_WORKERS: "${NUM_WORKERS}" 11 | ports: 12 | - "${REST_PORT}:9000" 13 | restart: always 14 | -------------------------------------------------------------------------------- /rest/env.template: -------------------------------------------------------------------------------- 1 | # number of gunicorn worker processes 2 | NUM_WORKERS=3 3 | 4 | # TCP port where the rest service is going to be listening 5 | REST_PORT=9000 6 | -------------------------------------------------------------------------------- /transformation/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.1" 2 | 3 | services: 4 | 5 | transformation: 6 | build: 7 | context: graphsense-transformation 8 | ports: 9 | - "$SPARK_DRIVER_HOST:$SPARK_UI_PORT:$SPARK_UI_PORT" 10 | - "$SPARK_DRIVER_HOST:$SPARK_DRIVER_PORT:$SPARK_DRIVER_PORT" 11 | - "$SPARK_DRIVER_HOST:$SPARK_BLOCKMGR_PORT:$SPARK_BLOCKMGR_PORT" 12 | environment: 13 | - SPARK_UI_PORT=${SPARK_UI_PORT} 14 | - SPARK_DRIVER_PORT=${SPARK_DRIVER_PORT} 15 | - SPARK_BLOCKMGR_PORT=${SPARK_BLOCKMGR_PORT} 16 | - SPARK_DRIVER_HOST=${SPARK_DRIVER_HOST} 17 | - SPARK_MASTER=${SPARK_MASTER} 18 | - SPARK_EXECUTOR_MEMORY=${SPARK_EXECUTOR_MEMORY} 19 | - SPARK_LOCAL_DIR=${SPARK_LOCAL_DIR} 20 | - CASSANDRA_HOST=${CASSANDRA_HOST} 21 | - CURRENCY=${CURRENCY} 22 | - RAW_KEYSPACE=${RAW_KEYSPACE} 23 | - TGT_KEYSPACE=${TGT_KEYSPACE} 24 | - CHECKPOINT_DIR=${CHECKPOINT_DIR} 25 | volumes: 26 | - "$SPARK_LOCAL_DIR:$SPARK_LOCAL_DIR" 27 | entrypoint: ["/opt/graphsense/docker-entrypoint.sh"] 28 | command: ["/opt/graphsense/submit.sh"] 29 | 30 | ethereum-transformation: 31 | build: 32 | context: graphsense-ethereum-transformation 33 | ports: 34 | - "$SPARK_DRIVER_HOST:$SPARK_UI_PORT:$SPARK_UI_PORT" 35 | - "$SPARK_DRIVER_HOST:$SPARK_DRIVER_PORT:$SPARK_DRIVER_PORT" 36 | - "$SPARK_DRIVER_HOST:$SPARK_BLOCKMGR_PORT:$SPARK_BLOCKMGR_PORT" 37 | environment: 38 | - SPARK_UI_PORT=${SPARK_UI_PORT} 39 | - SPARK_DRIVER_PORT=${SPARK_DRIVER_PORT} 40 | - SPARK_BLOCKMGR_PORT=${SPARK_BLOCKMGR_PORT} 41 | - SPARK_DRIVER_HOST=${SPARK_DRIVER_HOST} 42 | - SPARK_MASTER=${SPARK_MASTER} 43 | - SPARK_EXECUTOR_MEMORY=${SPARK_EXECUTOR_MEMORY} 44 | - SPARK_LOCAL_DIR=${SPARK_LOCAL_DIR} 45 | - CASSANDRA_HOST=${CASSANDRA_HOST} 46 | - RAW_KEYSPACE=${RAW_KEYSPACE} 47 | - TGT_KEYSPACE=${TGT_KEYSPACE} 48 | volumes: 49 | - "$SPARK_LOCAL_DIR:$SPARK_LOCAL_DIR" 50 | entrypoint: ["/opt/graphsense/docker-entrypoint.sh"] 51 | command: ["/opt/graphsense/submit.sh"] 52 | 53 | networks: 54 | default: 55 | external: 56 | name: graphsense-global-net 57 | -------------------------------------------------------------------------------- /transformation/env.template: -------------------------------------------------------------------------------- 1 | ### Spark configuration 2 | ### see https://spark.apache.org/docs/latest/configuration.html 3 | # application web UI port 4 | SPARK_UI_PORT=4040 5 | # driver port 6 | SPARK_DRIVER_PORT=4041 7 | # block manager port 8 | SPARK_BLOCKMGR_PORT=4042 9 | # IP address of the host machine where the container will be running 10 | SPARK_DRIVER_HOST= 11 | # address of spark-master in external standalone cluster 12 | # spark://SPARK_MASTER_IP:7077 13 | SPARK_MASTER= 14 | # amount of memory to use per executor process, in the same format as JVM 15 | # memory strings with a size unit suffix ("k", "m", "g" or "t") (e.g. 192g). 16 | SPARK_EXECUTOR_MEMORY= 17 | # local directory for shuffle data and block manager disk data 18 | SPARK_LOCAL_DIR= 19 | 20 | ### arguments for Spark submit job 21 | # Cassandra seed host; IP address (or resolvable hostname) 22 | CASSANDRA_HOST= 23 | # cryptocurrency coin name (BTC, BCH, LTC or ZEC) 24 | CURRENCY= 25 | # name of raw Cassandra keyspace 26 | RAW_KEYSPACE= 27 | # name of transformed Cassandra keyspace (target keyspace of Spark application) 28 | TGT_KEYSPACE= 29 | # Spark checkpoint directory (HDFS directory, writable for used Docker UID) 30 | CHECKPOINT_DIR= 31 | --------------------------------------------------------------------------------