├── .gitignore ├── .gitlab-ci.yml ├── Dockerfile ├── LICENSE ├── README.md ├── coinmetrics-all-blockchains ├── CoinMetrics │ └── BlockChain │ │ └── All.hs └── coinmetrics-all-blockchains.cabal ├── coinmetrics-binance ├── CoinMetrics │ └── Binance.hs └── coinmetrics-binance.cabal ├── coinmetrics-bitcoin ├── CoinMetrics │ └── Bitcoin.hs └── coinmetrics-bitcoin.cabal ├── coinmetrics-cardano ├── CoinMetrics │ └── Cardano.hs └── coinmetrics-cardano.cabal ├── coinmetrics-cosmos ├── CoinMetrics │ └── Cosmos.hs └── coinmetrics-cosmos.cabal ├── coinmetrics-eos ├── CoinMetrics │ ├── EOS.hs │ └── EOSArchive.hs └── coinmetrics-eos.cabal ├── coinmetrics-ethereum ├── CoinMetrics │ └── Ethereum.hs └── coinmetrics-ethereum.cabal ├── coinmetrics-export ├── bintray-deb.json.in ├── bintray-rpm.json.in ├── bintray.json.in ├── coinmetrics-export.cabal └── coinmetrics-export.hs ├── coinmetrics-grin ├── CoinMetrics │ └── Grin.hs └── coinmetrics-grin.cabal ├── coinmetrics-iota ├── CoinMetrics │ └── Iota.hs └── coinmetrics-iota.cabal ├── coinmetrics-monero ├── CoinMetrics │ └── Monero.hs └── coinmetrics-monero.cabal ├── coinmetrics-monitor ├── bintray-deb.json.in ├── bintray-rpm.json.in ├── bintray.json.in ├── coinmetrics-monitor.cabal └── coinmetrics-monitor.hs ├── coinmetrics-nem ├── CoinMetrics │ └── Nem.hs └── coinmetrics-nem.cabal ├── coinmetrics-neo ├── CoinMetrics │ └── Neo.hs └── coinmetrics-neo.cabal ├── coinmetrics-ripple ├── CoinMetrics │ └── Ripple.hs └── coinmetrics-ripple.cabal ├── coinmetrics-rosetta ├── CoinMetrics │ └── Rosetta.hs └── coinmetrics-rosetta.cabal ├── coinmetrics-stellar ├── CoinMetrics │ └── Stellar.hs ├── coinmetrics-stellar.cabal ├── coinmetrics-stellar.cpp ├── default.nix └── shell.nix ├── coinmetrics-storage ├── CoinMetrics │ └── Export │ │ ├── Storage.hs │ │ └── Storage │ │ ├── AvroFile.hs │ │ ├── Elastic.hs │ │ ├── Postgres.hs │ │ ├── PostgresFile.hs │ │ └── RabbitMQ.hs └── coinmetrics-storage.cabal ├── coinmetrics-tendermint ├── CoinMetrics │ ├── Tendermint.hs │ └── Tendermint │ │ └── Amino.hs └── coinmetrics-tendermint.cabal ├── coinmetrics-tezos ├── CoinMetrics │ └── Tezos.hs └── coinmetrics-tezos.cabal ├── coinmetrics-tron ├── CoinMetrics │ └── Tron.hs └── coinmetrics-tron.cabal ├── coinmetrics-waves ├── CoinMetrics │ └── Waves.hs └── coinmetrics-waves.cabal ├── coinmetrics ├── CoinMetrics │ ├── BlockChain.hs │ ├── JsonRpc.hs │ ├── Prometheus.hs │ ├── Schema │ │ ├── Flatten.hs │ │ └── Util.hs │ ├── Util.hs │ └── WebCache.hs ├── coinmetrics.cabal └── schema │ └── elastic │ └── unified.json ├── default.nix ├── docs ├── blockchains.md ├── coinmetrics-export.md ├── coinmetrics-monitor.md └── docker_image.md ├── release.nix ├── stack-shell.nix ├── stack.yaml └── stack.yaml.lock /.gitignore: -------------------------------------------------------------------------------- 1 | .stack-work 2 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - build 3 | 4 | nix-image: 5 | stage: build 6 | script: 7 | - $(nix-build --no-out-link -QA pushImagesScript ./release.nix) 8 | tags: 9 | - coinmetrics-nix-build-runner 10 | only: 11 | - master 12 | 13 | nix-image-mr: 14 | stage: build 15 | script: 16 | - nix-build --no-out-link -QA pushImagesScript ./release.nix 17 | tags: 18 | - coinmetrics-nix-build-runner 19 | only: 20 | - merge_requests 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster 2 | 3 | RUN set -ex; \ 4 | if ! command -v gpg > /dev/null; then \ 5 | apt-get update; \ 6 | apt-get install -y --no-install-recommends \ 7 | netbase \ 8 | libnuma1 \ 9 | ca-certificates \ 10 | libpq5 \ 11 | ; \ 12 | rm -rf /var/lib/apt/lists/*; \ 13 | fi 14 | 15 | COPY coinmetrics-export coinmetrics-monitor /usr/bin/ 16 | 17 | RUN useradd -m -u 1000 -s /bin/bash coinmetrics 18 | USER coinmetrics 19 | WORKDIR /home/coinmetrics 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017-2018 Alexander Bich (aka Quyse Lert) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Haskell-based CoinMetrics.io tools 2 | 3 | These tools are used by CoinMetrics.io team for exporting data from blockchains into analytical databases and monitoring full nodes synchronization state. 4 | 5 | ## Utilities 6 | 7 | * [coinmetrics-export](docs/coinmetrics-export.md) - utility for exporting data from blockchains in formats suitable for inserting into analytics databases (SQL, Avro). 8 | * [coinmetrics-monitor](docs/coinmetrics-monitor.md) - utility for monitoring blockchain nodes and providing Prometheus-compatible metrics. 9 | 10 | ## Status 11 | 12 | The project is being used in production at Coin Metrics, but many things are in flux or fragile, and docs may be outdated. Command line interface is more or less stable but may change. Please use with caution. 13 | 14 | Supported blockchains: 15 | 16 | * Binance Chain 17 | * [Bitcoin](https://bitcoin.org/) 18 | * [Cardano](https://www.cardanohub.org/) 19 | * [Cosmos](https://cosmos.network/) 20 | * [EOS](https://eos.io/) 21 | * [Ethereum](https://www.ethereum.org/) 22 | * [Grin](https://grin-tech.org/) 23 | * [IOTA](https://iota.org/) 24 | * [Monero](https://getmonero.org/) 25 | * [NEM](https://nem.io/) 26 | * [NEO](https://neo.org/) 27 | * [Ripple](https://ripple.com/) 28 | * Generic [Rosetta API](https://www.rosetta-api.org/) (WIP) 29 | * [Stellar](https://www.stellar.org/) 30 | * Generic [Tendermint](https://tendermint.com/) 31 | * [Tezos](https://tezos.com/) 32 | * [Tron](https://tron.network/) 33 | * [Waves](https://wavesplatform.com/) 34 | 35 | ## Binaries 36 | 37 | There're no stable releases yet. All binaries are "bleeding edge". 38 | 39 | One easy way to run the tools is to use docker. 40 | 41 | Pull the latest version: 42 | ```bash 43 | docker pull coinmetrics/haskell-tools 44 | ``` 45 | 46 | Run e.g. `coinmetrics-export` tool: 47 | ```bash 48 | docker run -it --rm --net host coinmetrics/haskell-tools coinmetrics-export 49 | ``` 50 | 51 | ## Building from source 52 | 53 | Building with Nix is recommended, because dependencies are fetched and built automatically. 54 | 55 | ### With Nix 56 | 57 | ```bash 58 | nix build -Lf ./release.nix bins 59 | ``` 60 | 61 | ### With Stack + Nix 62 | 63 | ```bash 64 | stack build --nix 65 | ``` 66 | 67 | ### With Stack 68 | 69 | ```bash 70 | stack build 71 | ``` 72 | 73 | Required dependencies: `zlib`, `libpq`. 74 | 75 | Stellar export also requires [xdrpp](https://github.com/xdrpp/xdrpp) and Stellar XDR headers built with XDR compiler, which may be hard to build manually. By default Stellar support is enabled, so build will fail without them. You can disable Stellar support if you don't need it: `stack build --flag coinmetrics-all-blockchains:-stellar`. If you do need it, please use one of the Nix build methods above (with Stack or without). 76 | 77 | The code is only tested on Linux (but maybe works on other OSes too). 78 | -------------------------------------------------------------------------------- /coinmetrics-all-blockchains/CoinMetrics/BlockChain/All.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE CPP, OverloadedStrings #-} 2 | 3 | module CoinMetrics.BlockChain.All 4 | ( getSomeBlockChainInfo 5 | , allBlockChainTypes 6 | , blockchainTypesStr 7 | ) where 8 | 9 | import qualified Data.HashMap.Strict as HM 10 | import Data.Proxy 11 | import qualified Data.Text as T 12 | 13 | import CoinMetrics.BlockChain 14 | 15 | #if defined(CM_SUPPORT_BINANCE) 16 | import CoinMetrics.Binance 17 | #endif 18 | #if defined(CM_SUPPORT_BITCOIN) 19 | import CoinMetrics.Bitcoin 20 | #endif 21 | #if defined(CM_SUPPORT_CARDANO) 22 | import CoinMetrics.Cardano 23 | #endif 24 | #if defined(CM_SUPPORT_COSMOS) 25 | import CoinMetrics.Cosmos 26 | #endif 27 | #if defined(CM_SUPPORT_EOS) 28 | import CoinMetrics.EOS 29 | import CoinMetrics.EOSArchive 30 | #endif 31 | #if defined(CM_SUPPORT_ETHEREUM) 32 | import CoinMetrics.Ethereum 33 | #endif 34 | #if defined(CM_SUPPORT_GRIN) 35 | import CoinMetrics.Grin 36 | #endif 37 | #if defined(CM_SUPPORT_MONERO) 38 | import CoinMetrics.Monero 39 | #endif 40 | #if defined(CM_SUPPORT_NEM) 41 | import CoinMetrics.Nem 42 | #endif 43 | #if defined(CM_SUPPORT_NEO) 44 | import CoinMetrics.Neo 45 | #endif 46 | #if defined(CM_SUPPORT_RIPPLE) 47 | import CoinMetrics.Ripple 48 | #endif 49 | #if defined(CM_SUPPORT_ROSETTA) 50 | import CoinMetrics.Rosetta 51 | #endif 52 | #if defined(CM_SUPPORT_STELLAR) 53 | import CoinMetrics.Stellar 54 | #endif 55 | #if defined(CM_SUPPORT_TENDERMINT) 56 | import CoinMetrics.Tendermint 57 | #endif 58 | #if defined(CM_SUPPORT_TEZOS) 59 | import CoinMetrics.Tezos 60 | #endif 61 | #if defined(CM_SUPPORT_TRON) 62 | import CoinMetrics.Tron 63 | #endif 64 | #if defined(CM_SUPPORT_WAVES) 65 | import CoinMetrics.Waves 66 | #endif 67 | 68 | allBlockChainInfos :: HM.HashMap T.Text SomeBlockChainInfo 69 | allBlockChainInfos = HM.fromList infos 70 | where 71 | infos = 72 | #if defined(CM_SUPPORT_BINANCE) 73 | ("binance", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Binance)) : 74 | #endif 75 | 76 | #if defined(CM_SUPPORT_BITCOIN) 77 | ("bitcoin", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Bitcoin)) : 78 | #endif 79 | 80 | #if defined(CM_SUPPORT_CARDANO) 81 | ("cardano", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Cardano)) : 82 | #endif 83 | 84 | #if defined(CM_SUPPORT_COSMOS) 85 | ("cosmos", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Cosmos)) : 86 | #endif 87 | 88 | #if defined(CM_SUPPORT_EOS) 89 | ("eos", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Eos)) : 90 | ("eos_archive", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy EosArchive)) : 91 | #endif 92 | 93 | #if defined(CM_SUPPORT_ETHEREUM) 94 | ("ethereum", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Ethereum)) : 95 | #endif 96 | 97 | #if defined(CM_SUPPORT_GRIN) 98 | ("grin", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Grin)) : 99 | #endif 100 | 101 | #if defined(CM_SUPPORT_MONERO) 102 | ("monero", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Monero)) : 103 | #endif 104 | 105 | #if defined(CM_SUPPORT_NEM) 106 | ("nem", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Nem)) : 107 | #endif 108 | 109 | #if defined(CM_SUPPORT_NEO) 110 | ("neo", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Neo)) : 111 | #endif 112 | 113 | #if defined(CM_SUPPORT_RIPPLE) 114 | ("ripple", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Ripple)) : 115 | #endif 116 | 117 | #if defined(CM_SUPPORT_ROSETTA) 118 | ("rosetta", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Rosetta)) : 119 | #endif 120 | 121 | #if defined(CM_SUPPORT_STELLAR) 122 | ("stellar", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Stellar)) : 123 | #endif 124 | 125 | #if defined(CM_SUPPORT_TENDERMINT) 126 | ("tendermint", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy (Tendermint T.Text))) : 127 | #endif 128 | 129 | #if defined(CM_SUPPORT_TEZOS) 130 | ("tezos", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Tezos)) : 131 | #endif 132 | 133 | #if defined(CM_SUPPORT_TRON) 134 | ("tron", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Tron)) : 135 | #endif 136 | 137 | #if defined(CM_SUPPORT_WAVES) 138 | ("waves", SomeBlockChainInfo $ getBlockChainInfo (Proxy :: Proxy Waves)) : 139 | #endif 140 | 141 | [] 142 | 143 | -- | Get blockchain info by name. 144 | getSomeBlockChainInfo :: T.Text -> Maybe SomeBlockChainInfo 145 | getSomeBlockChainInfo = flip HM.lookup allBlockChainInfos 146 | 147 | -- | Get supported blockchain types. 148 | allBlockChainTypes :: [T.Text] 149 | allBlockChainTypes = HM.keys allBlockChainInfos 150 | 151 | blockchainTypesStr :: String 152 | blockchainTypesStr = T.unpack $ T.intercalate ", " allBlockChainTypes 153 | -------------------------------------------------------------------------------- /coinmetrics-all-blockchains/coinmetrics-all-blockchains.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-all-blockchains 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | flag binance 15 | description: Support Binance 16 | default: True 17 | manual: True 18 | 19 | flag bitcoin 20 | description: Support Bitcoin 21 | default: True 22 | manual: True 23 | 24 | flag cardano 25 | description: Support Cardano 26 | default: True 27 | manual: True 28 | 29 | flag cosmos 30 | description: Support Cosmos 31 | default: True 32 | manual: True 33 | 34 | flag eos 35 | description: Support Eos 36 | default: True 37 | manual: True 38 | 39 | flag ethereum 40 | description: Support Ethereum 41 | default: True 42 | manual: True 43 | 44 | flag grin 45 | description: Support Grin 46 | default: True 47 | manual: True 48 | 49 | flag monero 50 | description: Support Monero 51 | default: True 52 | manual: True 53 | 54 | flag nem 55 | description: Support Nem 56 | default: True 57 | manual: True 58 | 59 | flag neo 60 | description: Support Neo 61 | default: True 62 | manual: True 63 | 64 | flag ripple 65 | description: Support Ripple 66 | default: True 67 | manual: True 68 | 69 | flag rosetta 70 | description: Support Rosetta 71 | default: True 72 | manual: True 73 | 74 | flag stellar 75 | description: Support Stellar 76 | default: True 77 | manual: True 78 | 79 | flag tendermint 80 | description: Support Tendermint 81 | default: True 82 | manual: True 83 | 84 | flag tezos 85 | description: Support Tezos 86 | default: True 87 | manual: True 88 | 89 | flag tron 90 | description: Support Tron 91 | default: True 92 | manual: True 93 | 94 | flag waves 95 | description: Support Waves 96 | default: True 97 | manual: True 98 | 99 | library 100 | exposed-modules: 101 | CoinMetrics.BlockChain.All 102 | build-depends: 103 | base 104 | , coinmetrics 105 | , text 106 | , unordered-containers 107 | 108 | if flag(binance) 109 | build-depends: coinmetrics-binance 110 | cpp-options: -DCM_SUPPORT_BINANCE 111 | 112 | if flag(bitcoin) 113 | build-depends: coinmetrics-bitcoin 114 | cpp-options: -DCM_SUPPORT_BITCOIN 115 | 116 | if flag(cardano) 117 | build-depends: coinmetrics-cardano 118 | cpp-options: -DCM_SUPPORT_CARDANO 119 | 120 | if flag(cosmos) 121 | build-depends: coinmetrics-cosmos 122 | cpp-options: -DCM_SUPPORT_COSMOS 123 | 124 | if flag(eos) 125 | build-depends: coinmetrics-eos 126 | cpp-options: -DCM_SUPPORT_EOS 127 | 128 | if flag(ethereum) 129 | build-depends: coinmetrics-ethereum 130 | cpp-options: -DCM_SUPPORT_ETHEREUM 131 | 132 | if flag(grin) 133 | build-depends: coinmetrics-grin 134 | cpp-options: -DCM_SUPPORT_GRIN 135 | 136 | if flag(monero) 137 | build-depends: coinmetrics-monero 138 | cpp-options: -DCM_SUPPORT_MONERO 139 | 140 | if flag(nem) 141 | build-depends: coinmetrics-nem 142 | cpp-options: -DCM_SUPPORT_NEM 143 | 144 | if flag(neo) 145 | build-depends: coinmetrics-neo 146 | cpp-options: -DCM_SUPPORT_NEO 147 | 148 | if flag(ripple) 149 | build-depends: coinmetrics-ripple 150 | cpp-options: -DCM_SUPPORT_RIPPLE 151 | 152 | if flag(rosetta) 153 | build-depends: coinmetrics-rosetta 154 | cpp-options: -DCM_SUPPORT_ROSETTA 155 | 156 | if flag(stellar) 157 | build-depends: coinmetrics-stellar 158 | cpp-options: -DCM_SUPPORT_STELLAR 159 | 160 | if flag(tendermint) 161 | build-depends: coinmetrics-tendermint 162 | cpp-options: -DCM_SUPPORT_TENDERMINT 163 | 164 | if flag(tezos) 165 | build-depends: coinmetrics-tezos 166 | cpp-options: -DCM_SUPPORT_TEZOS 167 | 168 | if flag(tron) 169 | build-depends: coinmetrics-tron 170 | cpp-options: -DCM_SUPPORT_TRON 171 | 172 | if flag(waves) 173 | build-depends: coinmetrics-waves 174 | cpp-options: -DCM_SUPPORT_WAVES 175 | 176 | ghc-options: -Wall 177 | default-language: Haskell2010 178 | -------------------------------------------------------------------------------- /coinmetrics-binance/coinmetrics-binance.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-binance 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Binance 17 | build-depends: 18 | aeson 19 | , avro 20 | , base 21 | , bytestring 22 | , cereal 23 | , coinmetrics 24 | , coinmetrics-tendermint 25 | , hanalytics-base 26 | , hanalytics-postgres 27 | , http-client 28 | , memory 29 | , protobuf 30 | , text 31 | , vector 32 | ghc-options: -Wall 33 | default-language: Haskell2010 34 | -------------------------------------------------------------------------------- /coinmetrics-bitcoin/CoinMetrics/Bitcoin.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, LambdaCase, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Bitcoin 4 | ( Bitcoin(..) 5 | , BitcoinBlock(..) 6 | , BitcoinTransaction(..) 7 | , BitcoinVin(..) 8 | , BitcoinVout(..) 9 | ) where 10 | 11 | import Control.Exception 12 | import Control.Monad 13 | import qualified Data.Aeson as J 14 | import qualified Data.Aeson.Types as J 15 | import qualified Data.HashMap.Strict as HM 16 | import Data.Int 17 | import Data.Maybe 18 | import Data.Proxy 19 | import Data.Scientific 20 | import qualified Data.Text as T 21 | import Data.Time.Clock.POSIX 22 | import qualified Data.Vector as V 23 | import Numeric 24 | 25 | import CoinMetrics.BlockChain 26 | import CoinMetrics.JsonRpc 27 | import CoinMetrics.Schema.Flatten 28 | import CoinMetrics.Schema.Util 29 | import CoinMetrics.Util 30 | import Hanalytics.Schema 31 | 32 | newtype Bitcoin = Bitcoin JsonRpc 33 | 34 | data BitcoinBlock = BitcoinBlock 35 | { bb_hash :: {-# UNPACK #-} !HexString 36 | , bb_size :: {-# UNPACK #-} !Int64 37 | , bb_strippedsize :: !(Maybe Int64) 38 | , bb_weight :: !(Maybe Int64) 39 | , bb_height :: {-# UNPACK #-} !Int64 40 | , bb_version :: {-# UNPACK #-} !Int64 41 | , bb_tx :: !(V.Vector BitcoinTransaction) 42 | , bb_time :: {-# UNPACK #-} !Int64 43 | , bb_nonce :: {-# UNPACK #-} !Int64 44 | , bb_difficulty :: {-# UNPACK #-} !Double 45 | , bb_prevHash :: !(Maybe HexString) 46 | } 47 | 48 | instance HasBlockHeader BitcoinBlock where 49 | getBlockHeader BitcoinBlock 50 | { bb_hash = hash 51 | , bb_height = height 52 | , bb_time = time 53 | , bb_prevHash = prevHash 54 | } = BlockHeader 55 | { bh_height = height 56 | , bh_hash = hash 57 | , bh_prevHash = prevHash 58 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral time 59 | } 60 | 61 | newtype BitcoinBlockWrapper = BitcoinBlockWrapper 62 | { unwrapBitcoinBlock :: BitcoinBlock 63 | } 64 | 65 | instance J.FromJSON BitcoinBlockWrapper where 66 | parseJSON = J.withObject "bitcoin block" $ \fields -> fmap BitcoinBlockWrapper $ BitcoinBlock 67 | <$> (fields J..: "hash") 68 | <*> (fields J..: "size") 69 | <*> (fields J..:? "strippedsize") 70 | <*> (fields J..:? "weight") 71 | <*> (fields J..: "height") 72 | <*> (fields J..: "version") 73 | <*> (V.map unwrapBitcoinTransaction <$> fields J..: "tx") 74 | <*> (fields J..: "time") 75 | <*> (parseNonce =<< fields J..: "nonce") 76 | <*> (fields J..: "difficulty") 77 | <*> (fields J..:? "previousblockhash") 78 | 79 | data BitcoinBlockHeader = BitcoinBlockHeader 80 | { bbh_hash :: {-# UNPACK #-} !HexString 81 | , bbh_height :: {-# UNPACK #-} !Int64 82 | , bbh_time :: {-# UNPACK #-} !Int64 83 | , bbh_prevHash :: {-# UNPACK #-} !HexString 84 | } 85 | 86 | instance HasBlockHeader BitcoinBlockHeader where 87 | getBlockHeader BitcoinBlockHeader 88 | { bbh_hash = hash 89 | , bbh_height = height 90 | , bbh_time = time 91 | , bbh_prevHash = prevHash 92 | } = BlockHeader 93 | { bh_height = height 94 | , bh_hash = hash 95 | , bh_prevHash = Just prevHash 96 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral time 97 | } 98 | 99 | newtype BitcoinBlockHeaderWrapper = BitcoinBlockHeaderWrapper 100 | { unwrapBitcoinBlockHeader :: BitcoinBlockHeader 101 | } 102 | 103 | instance J.FromJSON BitcoinBlockHeaderWrapper where 104 | parseJSON = J.withObject "bitcoin block header" $ \fields -> fmap BitcoinBlockHeaderWrapper $ BitcoinBlockHeader 105 | <$> (fields J..: "hash") 106 | <*> (fields J..: "height") 107 | <*> (fields J..: "time") 108 | <*> (fields J..: "previousblockhash") 109 | 110 | data BitcoinTransaction = BitcoinTransaction 111 | { bt_txid :: {-# UNPACK #-} !HexString 112 | , bt_hash :: !(Maybe HexString) 113 | , bt_size :: !(Maybe Int64) 114 | , bt_vsize :: !(Maybe Int64) 115 | , bt_version :: {-# UNPACK #-} !Int64 116 | , bt_locktime :: {-# UNPACK #-} !Int64 117 | , bt_vin :: !(V.Vector BitcoinVin) 118 | , bt_vout :: !(V.Vector BitcoinVout) 119 | } 120 | 121 | newtype BitcoinTransactionWrapper = BitcoinTransactionWrapper 122 | { unwrapBitcoinTransaction :: BitcoinTransaction 123 | } 124 | 125 | instance J.FromJSON BitcoinTransactionWrapper where 126 | parseJSON = J.withObject "bitcoin transaction" $ \fields -> fmap BitcoinTransactionWrapper $ BitcoinTransaction 127 | <$> (fields J..: "txid") 128 | <*> (fields J..:? "hash") 129 | <*> (fields J..:? "size") 130 | <*> (fields J..:? "vsize") 131 | <*> (fields J..: "version") 132 | <*> (fields J..: "locktime") 133 | <*> (fields J..: "vin") 134 | <*> (V.map unwrapBitcoinVout <$> fields J..: "vout") 135 | 136 | data BitcoinVin = BitcoinVin 137 | { bvi_txid :: !(Maybe HexString) 138 | , bvi_vout :: !(Maybe Int64) 139 | , bvi_coinbase :: !(Maybe HexString) 140 | } 141 | 142 | data BitcoinVout = BitcoinVout 143 | { bvo_type :: !T.Text 144 | , bvo_value :: {-# UNPACK #-} !Scientific 145 | , bvo_addresses :: !(V.Vector T.Text) 146 | , bvo_asm :: !T.Text 147 | } 148 | 149 | newtype BitcoinVoutWrapper = BitcoinVoutWrapper 150 | { unwrapBitcoinVout :: BitcoinVout 151 | } 152 | 153 | instance J.FromJSON BitcoinVoutWrapper where 154 | parseJSON = J.withObject "bitcoin vout" $ \fields -> do 155 | scriptPubKey <- fields J..: "scriptPubKey" 156 | fmap BitcoinVoutWrapper $ BitcoinVout 157 | <$> (scriptPubKey J..: "type") 158 | <*> (fields J..: "value") 159 | <*> fmap (fromMaybe V.empty) (scriptPubKey J..:? "addresses") 160 | <*> (scriptPubKey J..: "asm") 161 | 162 | parseNonce :: J.Value -> J.Parser Int64 163 | parseNonce = \case 164 | -- Bitcoin Gold returns nonce in form of hex string 165 | J.String (readHex . T.unpack -> [(n, "")]) -> return n 166 | n -> J.parseJSON n 167 | 168 | 169 | genSchemaInstances [''BitcoinBlock, ''BitcoinTransaction, ''BitcoinVin, ''BitcoinVout] 170 | genFlattenedTypes "height" [| bb_height |] [("block", ''BitcoinBlock), ("transaction", ''BitcoinTransaction), ("vin", ''BitcoinVin), ("vout", ''BitcoinVout)] 171 | 172 | instance BlockChain Bitcoin where 173 | type Block Bitcoin = BitcoinBlock 174 | 175 | getBlockChainInfo _ = BlockChainInfo 176 | { bci_init = \BlockChainParams 177 | { bcp_httpManager = httpManager 178 | , bcp_httpRequest = httpRequest 179 | } -> return $ Bitcoin $ newJsonRpc httpManager httpRequest Nothing 180 | , bci_defaultApiUrls = ["http://127.0.0.1:8332/"] 181 | , bci_defaultBeginBlock = 0 182 | , bci_defaultEndBlock = -100 -- conservative rewrite limit 183 | , bci_heightFieldName = "height" 184 | , bci_schemas = standardBlockChainSchemas 185 | (schemaOf (Proxy :: Proxy BitcoinBlock)) 186 | [ schemaOf (Proxy :: Proxy BitcoinVin) 187 | , schemaOf (Proxy :: Proxy BitcoinVout) 188 | , schemaOf (Proxy :: Proxy BitcoinTransaction) 189 | ] 190 | "CREATE TABLE \"bitcoin\" OF \"BitcoinBlock\" (PRIMARY KEY (\"height\"));" 191 | , bci_flattenSuffixes = ["blocks", "transactions", "vins", "vouts"] 192 | , bci_flattenPack = let 193 | f (blocks, (transactions, vins, vouts)) = 194 | [ SomeBlocks (blocks :: [BitcoinBlock_flattened]) 195 | , SomeBlocks (transactions :: [BitcoinTransaction_flattened]) 196 | , SomeBlocks (vins :: [BitcoinVin_flattened]) 197 | , SomeBlocks (vouts :: [BitcoinVout_flattened]) 198 | ] 199 | in f . mconcat . map flatten 200 | } 201 | 202 | getBlockChainNodeInfo (Bitcoin jsonRpc) = do 203 | networkInfoJson <- jsonRpcRequest jsonRpc "getnetworkinfo" ([] :: V.Vector J.Value) 204 | codedVersion <- either fail return $ J.parseEither (J..: "version") networkInfoJson 205 | return BlockChainNodeInfo 206 | { bcni_version = getVersionString codedVersion 207 | } 208 | 209 | getCurrentBlockHeight (Bitcoin jsonRpc) = jsonRpcRequest jsonRpc "getblockcount" ([] :: V.Vector J.Value) 210 | 211 | getBlockHeaderByHeight (Bitcoin jsonRpc) blockHeight = do 212 | blockHash <- jsonRpcRequest jsonRpc "getblockhash" ([J.Number $ fromIntegral blockHeight] :: V.Vector J.Value) 213 | getBlockHeader . unwrapBitcoinBlockHeader <$> jsonRpcRequest jsonRpc "getblock" ([blockHash] :: V.Vector J.Value) 214 | 215 | getBlockByHeight (Bitcoin jsonRpc) blockHeight = do 216 | blockHash <- jsonRpcRequest jsonRpc "getblockhash" ([J.Number $ fromIntegral blockHeight] :: V.Vector J.Value) 217 | -- try get everything in one RPC call 218 | eitherBlock <- try $ unwrapBitcoinBlock <$> jsonRpcRequest jsonRpc "getblock" ([blockHash, J.Number 2] :: V.Vector J.Value) 219 | case eitherBlock of 220 | Right block -> return block 221 | Left SomeException {} -> do 222 | -- request block with transactions' hashes 223 | blockJson <- jsonRpcRequest jsonRpc "getblock" ([blockHash, J.Bool True] :: V.Vector J.Value) 224 | transactionsHashes <- either fail return $ J.parseEither (J..: "tx") blockJson 225 | transactions <- forM transactionsHashes $ \case 226 | -- some bitcoin clones return full transaction here anyhow, because fuck you 227 | -- well, less work needed in that case 228 | transaction@(J.Object {}) -> return transaction 229 | transactionHash@(J.String {}) -> jsonRpcRequest jsonRpc "getrawtransaction" ([transactionHash, J.Number 1] :: V.Vector J.Value) 230 | _ -> fail "wrong tx hash" 231 | fmap unwrapBitcoinBlock $ either fail return $ J.parseEither J.parseJSON $ J.Object $ HM.insert "tx" (J.Array transactions) blockJson 232 | 233 | getVersionString :: Int -> T.Text 234 | getVersionString n = T.pack $ showsPrec 0 v4 $ '.' : (showsPrec 0 v3 $ '.' : (showsPrec 0 v2 $ '.' : show v1)) where 235 | (p1, v1) = n `quotRem` 100 236 | (p2, v2) = p1 `quotRem` 100 237 | (p3, v3) = p2 `quotRem` 100 238 | v4 = p3 `rem` 100 239 | -------------------------------------------------------------------------------- /coinmetrics-bitcoin/coinmetrics-bitcoin.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-bitcoin 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Bitcoin 17 | build-depends: 18 | aeson 19 | , base 20 | , coinmetrics 21 | , hanalytics-base 22 | , scientific 23 | , text 24 | , time 25 | , unordered-containers 26 | , vector 27 | ghc-options: -Wall -Wno-tabs 28 | default-language: Haskell2010 29 | -------------------------------------------------------------------------------- /coinmetrics-cardano/CoinMetrics/Cardano.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Cardano 4 | ( Cardano(..) 5 | , CardanoBlock(..) 6 | , CardanoTransaction(..) 7 | , CardanoInput(..) 8 | , CardanoOutput(..) 9 | ) where 10 | 11 | import Control.Monad 12 | import qualified Data.Aeson as J 13 | import qualified Data.Aeson.Types as J 14 | import qualified Data.ByteString as B 15 | import qualified Data.HashMap.Strict as HM 16 | import Data.Int 17 | import Data.Proxy 18 | import Data.String 19 | import qualified Data.Text as T 20 | import qualified Data.Text.Encoding as T 21 | import Data.Time.Clock.POSIX 22 | import qualified Data.Vector as V 23 | import qualified Network.HTTP.Client as H 24 | 25 | import CoinMetrics.BlockChain 26 | import CoinMetrics.Schema.Flatten 27 | import CoinMetrics.Schema.Util 28 | import CoinMetrics.Util 29 | import Hanalytics.Schema 30 | 31 | -- | Cardano connector. 32 | data Cardano = Cardano 33 | { cardano_httpManager :: !H.Manager 34 | , cardano_httpRequest :: !H.Request 35 | } 36 | 37 | cardanoRequest :: J.FromJSON r => Cardano -> T.Text -> [(B.ByteString, Maybe B.ByteString)] -> IO r 38 | cardanoRequest Cardano 39 | { cardano_httpManager = httpManager 40 | , cardano_httpRequest = httpRequest 41 | } path params = do 42 | body <- H.responseBody <$> tryWithRepeat (H.httpLbs (H.setQueryString params httpRequest 43 | { H.path = T.encodeUtf8 path 44 | }) httpManager) 45 | either fail return $ J.eitherDecode body 46 | 47 | -- API: https://cardanodocs.com/technical/explorer/api 48 | 49 | data CardanoBlock = CardanoBlock 50 | { cb_height :: {-# UNPACK #-} !Int64 51 | , cb_epoch :: {-# UNPACK #-} !Int64 52 | , cb_slot :: {-# UNPACK #-} !Int64 53 | , cb_hash :: {-# UNPACK #-} !HexString 54 | , cb_timeIssued :: {-# UNPACK #-} !Int64 55 | , cb_totalSent :: !Integer 56 | , cb_size :: {-# UNPACK #-} !Int64 57 | , cb_blockLead :: !(Maybe HexString) 58 | , cb_fees :: !Integer 59 | , cb_transactions :: !(V.Vector CardanoTransaction) 60 | } 61 | 62 | instance HasBlockHeader CardanoBlock where 63 | getBlockHeader CardanoBlock 64 | { cb_height = height 65 | , cb_hash = hash 66 | , cb_timeIssued = timeIssued 67 | } = BlockHeader 68 | { bh_height = height 69 | , bh_hash = hash 70 | , bh_prevHash = Nothing 71 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timeIssued 72 | } 73 | 74 | newtype CardanoBlockWrapper = CardanoBlockWrapper 75 | { unwrapCardanoBlock :: CardanoBlock 76 | } 77 | 78 | instance J.FromJSON CardanoBlockWrapper where 79 | parseJSON = J.withObject "cardano block" $ \fields -> fmap CardanoBlockWrapper $ CardanoBlock 80 | <$> (fields J..: "height") 81 | <*> (fields J..: "cbeEpoch") 82 | <*> (fields J..: "cbeSlot") 83 | <*> (fields J..: "cbeBlkHash") 84 | <*> (fields J..: "cbeTimeIssued") 85 | <*> (decodeValue =<< fields J..: "cbeTotalSent") 86 | <*> (fields J..: "cbeSize") 87 | <*> (fields J..:? "cbeBlockLead") 88 | <*> (decodeValue =<< fields J..: "cbeFees") 89 | <*> (V.map unwrapCardanoTransaction <$> fields J..: "transactions") 90 | 91 | data CardanoTransaction = CardanoTransaction 92 | { ct_id :: {-# UNPACK #-} !HexString 93 | , ct_timeIssued :: {-# UNPACK #-} !Int64 94 | , ct_fees :: !Integer 95 | , ct_inputs :: !(V.Vector CardanoInput) 96 | , ct_outputs :: !(V.Vector CardanoOutput) 97 | } 98 | 99 | newtype CardanoTransactionWrapper = CardanoTransactionWrapper 100 | { unwrapCardanoTransaction :: CardanoTransaction 101 | } 102 | 103 | instance J.FromJSON CardanoTransactionWrapper where 104 | parseJSON = J.withObject "cardano transaction" $ \fields -> fmap CardanoTransactionWrapper $ CardanoTransaction 105 | <$> (fields J..: "ctsId") 106 | <*> (fields J..: "ctsTxTimeIssued") 107 | <*> (decodeValue =<< fields J..: "ctsFees") 108 | <*> (V.map unwrapCardanoInput <$> fields J..: "ctsInputs") 109 | <*> (V.map unwrapCardanoOutput <$> fields J..: "ctsOutputs") 110 | 111 | data CardanoInput = CardanoInput 112 | { ci_address :: !T.Text 113 | , ci_value :: !Integer 114 | , ci_txid :: !HexString 115 | , ci_output :: !Int64 116 | } 117 | 118 | newtype CardanoInputWrapper = CardanoInputWrapper 119 | { unwrapCardanoInput :: CardanoInput 120 | } 121 | 122 | instance J.FromJSON CardanoInputWrapper where 123 | parseJSON = J.withObject "cardano input" $ \fields -> fmap CardanoInputWrapper $ CardanoInput 124 | <$> (fields J..: "ctaAddress") 125 | <*> (decodeValue =<< fields J..: "ctaAmount") 126 | <*> (fields J..: "ctaTxHash") 127 | <*> (fields J..: "ctaTxIndex") 128 | 129 | data CardanoOutput = CardanoOutput 130 | { co_address :: !T.Text 131 | , co_value :: !Integer 132 | } 133 | 134 | newtype CardanoOutputWrapper = CardanoOutputWrapper 135 | { unwrapCardanoOutput :: CardanoOutput 136 | } 137 | 138 | instance J.FromJSON CardanoOutputWrapper where 139 | parseJSON = J.withObject "cardano output" $ \fields -> fmap CardanoOutputWrapper $ CardanoOutput 140 | <$> (fields J..: "ctaAddress") 141 | <*> (decodeValue =<< fields J..: "ctaAmount") 142 | 143 | decodeValue :: J.Value -> J.Parser Integer 144 | decodeValue = J.withObject "cardano value" $ \fields -> read . T.unpack <$> fields J..: "getCoin" 145 | 146 | genSchemaInstances [''CardanoBlock, ''CardanoTransaction, ''CardanoInput, ''CardanoOutput] 147 | genFlattenedTypes "height" [| cb_height |] [("block", ''CardanoBlock), ("transaction", ''CardanoTransaction), ("input", ''CardanoInput), ("output", ''CardanoOutput)] 148 | 149 | instance BlockChain Cardano where 150 | type Block Cardano = CardanoBlock 151 | 152 | getBlockChainInfo _ = BlockChainInfo 153 | { bci_init = \BlockChainParams 154 | { bcp_httpManager = httpManager 155 | , bcp_httpRequest = httpRequest 156 | } -> return Cardano 157 | { cardano_httpManager = httpManager 158 | , cardano_httpRequest = httpRequest 159 | } 160 | , bci_defaultApiUrls = ["http://127.0.0.1:8100/"] 161 | , bci_defaultBeginBlock = 2 162 | , bci_defaultEndBlock = 0 163 | , bci_heightFieldName = "height" 164 | , bci_schemas = standardBlockChainSchemas 165 | (schemaOf (Proxy :: Proxy CardanoBlock)) 166 | [ schemaOf (Proxy :: Proxy CardanoInput) 167 | , schemaOf (Proxy :: Proxy CardanoOutput) 168 | , schemaOf (Proxy :: Proxy CardanoTransaction) 169 | ] 170 | "CREATE TABLE \"cardano\" OF \"CardanoBlock\" (PRIMARY KEY (\"height\"));" 171 | , bci_flattenSuffixes = ["blocks", "transactions", "logs", "actions", "uncles"] 172 | , bci_flattenPack = let 173 | f (blocks, (transactions, inputs, outputs)) = 174 | [ SomeBlocks (blocks :: [CardanoBlock_flattened]) 175 | , SomeBlocks (transactions :: [CardanoTransaction_flattened]) 176 | , SomeBlocks (inputs :: [CardanoInput_flattened]) 177 | , SomeBlocks (outputs :: [CardanoOutput_flattened]) 178 | ] 179 | in f . mconcat . map flatten 180 | } 181 | 182 | -- pageSize param doesn't work anymore 183 | -- getCurrentBlockHeight cardano = either fail return =<< cardanoRequest cardano "/api/blocks/pages/total" [("pageSize", Just "1")] 184 | getCurrentBlockHeight cardano = either fail (return . (+ (-8)) . (* 10)) =<< cardanoRequest cardano "/api/blocks/pages/total" [] 185 | 186 | getBlockByHeight cardano blockHeight = do 187 | -- calculate page's index and block's index on page 188 | let 189 | pageIndex = (blockHeight + 8) `quot` 10 190 | blockIndexOnPage = blockHeight - (pageIndex * 10 - 8) 191 | reverseGet i v = v V.! (V.length v - 1 - i) 192 | -- get page with blocks 193 | pageObject <- either fail return =<< cardanoRequest cardano "/api/blocks/pages" 194 | [ ("page", Just $ fromString $ show pageIndex) 195 | , ("pageSize", Just "10") 196 | ] 197 | blockObject <- either fail return $ flip J.parseEither pageObject $ J.withArray "page" $ 198 | \((V.! 1) -> blocksObjectsObject) -> 199 | J.withArray "blocks" (J.parseJSON . reverseGet (fromIntegral blockIndexOnPage)) blocksObjectsObject 200 | blockHashText <- either fail return $ J.parseEither (J..: "cbeBlkHash") blockObject 201 | blockTxsBriefObjects <- either (\err -> 202 | if err == "No block found" 203 | then return V.empty -- working around https://github.com/input-output-hk/cardano-rest/issues/41 204 | else fail err 205 | ) return 206 | =<< cardanoRequest cardano ("/api/blocks/txs/" <> blockHashText) [("limit", Just "1000000000000000000")] 207 | blockTxs <- forM blockTxsBriefObjects $ \txBriefObject -> do 208 | txIdText <- either fail return $ J.parseEither (J..: "ctbId") txBriefObject 209 | txInputs <- either fail return $ J.parseEither (J..: "ctbInputs") txBriefObject 210 | -- forked node has more information about inputs, so replace them 211 | either fail return . J.parseEither J.parseJSON . J.Object 212 | =<< either fail (return . HM.insert "ctsInputs" txInputs) 213 | =<< cardanoRequest cardano ("/api/txs/summary/" <> txIdText) [] 214 | either fail (return . unwrapCardanoBlock) $ J.parseEither (J.parseJSON . J.Object) 215 | $ HM.insert "height" (J.Number $ fromIntegral blockHeight) 216 | $ HM.insert "transactions" (J.Array blockTxs) 217 | blockObject 218 | -------------------------------------------------------------------------------- /coinmetrics-cardano/coinmetrics-cardano.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-cardano 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Cardano 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , http-client 24 | , memory 25 | , text 26 | , time 27 | , unordered-containers 28 | , vector 29 | ghc-options: -Wall 30 | default-language: Haskell2010 31 | -------------------------------------------------------------------------------- /coinmetrics-cosmos/CoinMetrics/Cosmos.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GeneralizedNewtypeDeriving, OverloadedStrings, TypeFamilies #-} 2 | 3 | module CoinMetrics.Cosmos 4 | ( Cosmos(..) 5 | , CosmosBlock 6 | , CosmosTransaction(..) 7 | ) where 8 | 9 | import Control.Monad 10 | import qualified Crypto.Hash as C 11 | import qualified Data.Aeson as J 12 | import qualified Data.Aeson.Types as J 13 | import qualified Data.Avro as A 14 | import qualified Data.ByteArray.Encoding as BA 15 | import qualified Data.ByteString as B 16 | import Data.Proxy 17 | import Data.String 18 | import qualified Data.Text.Encoding as T 19 | import qualified Data.Vector as V 20 | import qualified Network.HTTP.Client as H 21 | 22 | import CoinMetrics.BlockChain 23 | import CoinMetrics.Tendermint 24 | import CoinMetrics.Util 25 | import Hanalytics.Schema 26 | import Hanalytics.Schema.Postgres 27 | 28 | newtype Cosmos = Cosmos (Tendermint CosmosTransaction) 29 | 30 | type CosmosBlock = TendermintBlock CosmosTransaction 31 | 32 | newtype CosmosTransaction = CosmosTransaction J.Value deriving (SchemableField, J.FromJSON, J.ToJSON, A.ToAvro, A.HasAvroSchema, ToPostgresText) 33 | 34 | instance TendermintTx CosmosTransaction where 35 | -- only calculate hex-encoded hash here; transaction is retrieved in JSON format later 36 | decodeTendermintTx t = do 37 | bytes <- either fail return $ BA.convertFromBase BA.Base64 $ T.encodeUtf8 t 38 | let 39 | hash = T.decodeUtf8 $ BA.convertToBase BA.Base16 (C.hash (bytes :: B.ByteString) :: C.Digest C.SHA256) 40 | return $ CosmosTransaction $ J.toJSON hash 41 | 42 | 43 | instance BlockChain Cosmos where 44 | type Block Cosmos = CosmosBlock 45 | 46 | getBlockChainInfo _ = BlockChainInfo 47 | { bci_init = fmap Cosmos . bci_init (getBlockChainInfo undefined) 48 | , bci_defaultApiUrls = ["http://127.0.0.1:1317/"] 49 | , bci_defaultBeginBlock = 1 50 | , bci_defaultEndBlock = 0 51 | , bci_heightFieldName = "height" 52 | , bci_schemas = standardBlockChainSchemas 53 | (schemaOf (Proxy :: Proxy CosmosBlock)) 54 | [ 55 | ] 56 | "CREATE TABLE \"cosmos\" OF \"CosmosBlock\" (PRIMARY KEY (\"height\"));" 57 | } 58 | 59 | getBlockChainNodeInfo (Cosmos Tendermint 60 | { tendermint_httpManager = httpManager 61 | , tendermint_httpRequest = httpRequest 62 | }) = do 63 | response <- tryWithRepeat $ H.httpLbs httpRequest 64 | { H.path = "/node_info" 65 | } httpManager 66 | result <- either fail return $ J.eitherDecode' $ H.responseBody response 67 | version <- either fail return $ J.parseEither ((J..: "application_version") >=> (J..: "version")) result 68 | return BlockChainNodeInfo 69 | { bcni_version = version 70 | } 71 | 72 | getCurrentBlockHeight (Cosmos Tendermint 73 | { tendermint_httpManager = httpManager 74 | , tendermint_httpRequest = httpRequest 75 | }) = do 76 | blockResponse <- tryWithRepeat $ H.httpLbs httpRequest 77 | { H.path = "/blocks/latest" 78 | } httpManager 79 | either fail (return . (tb_height :: TendermintBlock CosmosTransaction -> BlockHeight) . unwrapTendermintBlock) $ J.eitherDecode' $ H.responseBody blockResponse 80 | 81 | getBlockByHeight (Cosmos Tendermint 82 | { tendermint_httpManager = httpManager 83 | , tendermint_httpRequest = httpRequest 84 | }) height = do 85 | blockResponse <- tryWithRepeat $ H.httpLbs httpRequest 86 | { H.path = "/blocks/" <> fromString (show height) 87 | } httpManager 88 | block <- either fail (return . unwrapTendermintBlock) $ J.eitherDecode' $ H.responseBody blockResponse 89 | -- retrieve transactions 90 | decodedTransactions <- V.forM (tb_transactions block) $ \(CosmosTransaction (J.String hash)) -> do 91 | txResponse <- tryWithRepeat $ H.httpLbs httpRequest 92 | { H.path = "/txs/" <> T.encodeUtf8 hash 93 | } httpManager 94 | either fail return $ J.eitherDecode' $ H.responseBody txResponse 95 | return block 96 | { tb_transactions = decodedTransactions 97 | } 98 | -------------------------------------------------------------------------------- /coinmetrics-cosmos/coinmetrics-cosmos.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-cosmos 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Cosmos 17 | build-depends: 18 | aeson 19 | , avro 20 | , base 21 | , bytestring 22 | , coinmetrics 23 | , coinmetrics-tendermint 24 | , cryptonite 25 | , hanalytics-base 26 | , hanalytics-postgres 27 | , http-client 28 | , memory 29 | , text 30 | , vector 31 | ghc-options: -Wall 32 | default-language: Haskell2010 33 | -------------------------------------------------------------------------------- /coinmetrics-eos/CoinMetrics/EOS.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, LambdaCase, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.EOS 4 | ( Eos(..) 5 | , EosBlock(..) 6 | , EosTransaction(..) 7 | , EosAction(..) 8 | , EosAuthorization(..) 9 | ) where 10 | 11 | import qualified Data.Aeson as J 12 | import qualified Data.Aeson.Types as J 13 | import qualified Data.ByteString.Lazy as BL 14 | import Data.Int 15 | import Data.Maybe 16 | import Data.Proxy 17 | import qualified Data.Text as T 18 | import qualified Data.Text.Encoding.Error as T 19 | import qualified Data.Text.Lazy.Encoding as TL 20 | import Data.Time.Clock.POSIX 21 | import qualified Data.Vector as V 22 | import qualified Network.HTTP.Client as H 23 | 24 | import CoinMetrics.BlockChain 25 | import CoinMetrics.Schema.Util 26 | import CoinMetrics.Util 27 | import Hanalytics.Schema 28 | 29 | data Eos = Eos 30 | { eos_httpManager :: !H.Manager 31 | , eos_httpRequest :: !H.Request 32 | } 33 | 34 | data EosBlock = EosBlock 35 | { eb_id :: {-# UNPACK #-} !HexString 36 | , eb_number :: {-# UNPACK #-} !Int64 37 | , eb_timestamp :: {-# UNPACK #-} !Int64 38 | , eb_producer :: !T.Text 39 | , eb_ref_block_prefix :: {-# UNPACK #-} !Int64 40 | , eb_transactions :: !(V.Vector EosTransaction) 41 | } 42 | 43 | instance HasBlockHeader EosBlock where 44 | getBlockHeader EosBlock 45 | { eb_number = number 46 | , eb_timestamp = timestamp 47 | } = BlockHeader 48 | { bh_height = number 49 | , bh_hash = mempty 50 | , bh_prevHash = Nothing 51 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timestamp 52 | } 53 | 54 | newtype EosBlockWrapper = EosBlockWrapper 55 | { unwrapEosBlock :: EosBlock 56 | } 57 | 58 | instance J.FromJSON EosBlockWrapper where 59 | parseJSON = J.withObject "eos block" $ \fields -> fmap EosBlockWrapper $ EosBlock 60 | <$> (fields J..: "id") 61 | <*> (fields J..: "block_num") 62 | <*> (round . utcTimeToPOSIXSeconds . currentLocalTimeToUTC <$> fields J..: "timestamp") 63 | <*> (fields J..: "producer") 64 | <*> (fields J..: "ref_block_prefix") 65 | <*> (V.map unwrapEosTransaction <$> fields J..: "transactions") 66 | 67 | data EosTransaction = EosTransaction 68 | { et_id :: {-# UNPACK #-} !HexString 69 | , et_status :: !T.Text 70 | , et_cpu_usage_us :: {-# UNPACK #-} !Int64 71 | , et_net_usage_words :: {-# UNPACK #-} !Int64 72 | , et_expiration :: {-# UNPACK #-} !Int64 73 | , et_ref_block_num :: {-# UNPACK #-} !Int64 74 | , et_ref_block_prefix :: {-# UNPACK #-} !Int64 75 | , et_max_net_usage_words :: {-# UNPACK #-} !Int64 76 | , et_max_cpu_usage_ms :: {-# UNPACK #-} !Int64 77 | , et_delay_sec :: {-# UNPACK #-} !Int64 78 | , et_context_free_actions :: !(V.Vector EosAction) 79 | , et_actions :: !(V.Vector EosAction) 80 | } 81 | 82 | newtype EosTransactionWrapper = EosTransactionWrapper 83 | { unwrapEosTransaction :: EosTransaction 84 | } 85 | 86 | instance J.FromJSON EosTransactionWrapper where 87 | parseJSON = J.withObject "eos transaction" $ \fields -> do 88 | trxVal <- fields J..: "trx" 89 | fmap EosTransactionWrapper $ case trxVal of 90 | J.Object trx -> do 91 | trxTrans <- trx J..: "transaction" 92 | EosTransaction 93 | <$> (trx J..: "id") 94 | <*> (fields J..: "status") 95 | <*> (fields J..: "cpu_usage_us") 96 | <*> (fields J..: "net_usage_words") 97 | <*> (round . utcTimeToPOSIXSeconds . currentLocalTimeToUTC <$> trxTrans J..: "expiration") 98 | <*> (trxTrans J..: "ref_block_num") 99 | <*> (trxTrans J..: "ref_block_prefix") 100 | <*> (trxTrans J..: "max_net_usage_words") 101 | <*> (trxTrans J..: "max_cpu_usage_ms") 102 | <*> (trxTrans J..: "delay_sec") 103 | <*> (V.map unwrapEosAction <$> trxTrans J..: "context_free_actions") 104 | <*> (V.map unwrapEosAction <$> trxTrans J..: "actions") 105 | _ -> EosTransaction 106 | mempty -- id 107 | <$> (fields J..: "status") 108 | <*> (fields J..: "cpu_usage_us") 109 | <*> (fields J..: "net_usage_words") 110 | <*> return 0 111 | <*> return 0 112 | <*> return 0 113 | <*> return 0 114 | <*> return 0 115 | <*> return 0 116 | <*> return [] 117 | <*> return [] 118 | 119 | data EosAction = EosAction 120 | { ea_account :: !T.Text 121 | , ea_name :: !T.Text 122 | , ea_authorization :: !(V.Vector EosAuthorization) 123 | , ea_data :: {-# UNPACK #-} !HexString 124 | } 125 | 126 | newtype EosActionWrapper = EosActionWrapper 127 | { unwrapEosAction :: EosAction 128 | } 129 | 130 | instance J.FromJSON EosActionWrapper where 131 | parseJSON = J.withObject "eos action" $ \fields -> fmap EosActionWrapper $ EosAction 132 | <$> (fields J..: "account") 133 | <*> (fields J..: "name") 134 | <*> (fields J..: "authorization") 135 | <*> (maybe (fields J..: "data") return =<< fields J..:? "hex_data") 136 | 137 | data EosAuthorization = EosAuthorization 138 | { eau_actor :: !T.Text 139 | , eau_permission :: !T.Text 140 | } 141 | 142 | genSchemaInstances [''EosBlock, ''EosTransaction, ''EosAction, ''EosAuthorization] 143 | -- genFlattenedTypes "number" [| eb_number |] [("block", ''EthereumBlock), ("transaction", ''EthereumTransaction), ("log", ''EthereumLog), ("action", ''EthereumAction), ("uncle", ''EthereumUncleBlock)] 144 | 145 | instance BlockChain Eos where 146 | type Block Eos = EosBlock 147 | 148 | getBlockChainInfo _ = BlockChainInfo 149 | { bci_init = \BlockChainParams 150 | { bcp_httpManager = httpManager 151 | , bcp_httpRequest = httpRequest 152 | } -> return Eos 153 | { eos_httpManager = httpManager 154 | , eos_httpRequest = httpRequest 155 | } 156 | , bci_defaultApiUrls = ["http://127.0.0.1:8888/"] 157 | , bci_defaultBeginBlock = 1 158 | , bci_defaultEndBlock = 0 -- no need in a gap, as it uses irreversible block number 159 | , bci_heightFieldName = "number" 160 | , bci_schemas = standardBlockChainSchemas 161 | (schemaOf (Proxy :: Proxy EosBlock)) 162 | [ schemaOf (Proxy :: Proxy EosAuthorization) 163 | , schemaOf (Proxy :: Proxy EosAction) 164 | , schemaOf (Proxy :: Proxy EosTransaction) 165 | ] 166 | "CREATE TABLE \"eos\" OF \"EosBlock\" (PRIMARY KEY (\"number\"));" 167 | } 168 | 169 | getBlockChainNodeInfo Eos 170 | { eos_httpManager = httpManager 171 | , eos_httpRequest = httpRequest 172 | } = do 173 | response <- tryWithRepeat $ H.httpLbs httpRequest 174 | { H.path = "/v1/chain/get_info" 175 | } httpManager 176 | version <- either fail return $ J.parseEither (J..: "server_version_string") =<< J.eitherDecode' (fixUtf8 $ H.responseBody response) 177 | return BlockChainNodeInfo 178 | { bcni_version = fromMaybe version $ T.stripPrefix "v" version 179 | } 180 | 181 | getCurrentBlockHeight Eos 182 | { eos_httpManager = httpManager 183 | , eos_httpRequest = httpRequest 184 | } = do 185 | response <- tryWithRepeat $ H.httpLbs httpRequest 186 | { H.path = "/v1/chain/get_info" 187 | } httpManager 188 | either fail return $ J.parseEither (J..: "last_irreversible_block_num") =<< J.eitherDecode' (fixUtf8 $ H.responseBody response) 189 | 190 | getBlockByHeight Eos 191 | { eos_httpManager = httpManager 192 | , eos_httpRequest = httpRequest 193 | } blockHeight = do 194 | response <- tryWithRepeat $ H.httpLbs httpRequest 195 | { H.path = "/v1/chain/get_block" 196 | , H.requestBody = H.RequestBodyLBS $ J.encode $ J.Object 197 | [ ("block_num_or_id", J.Number $ fromIntegral blockHeight) 198 | ] 199 | } httpManager 200 | either fail (return . unwrapEosBlock) $ J.eitherDecode' $ fixUtf8 $ H.responseBody response 201 | 202 | fixUtf8 :: BL.ByteString -> BL.ByteString 203 | fixUtf8 = TL.encodeUtf8 . TL.decodeUtf8With T.lenientDecode 204 | -------------------------------------------------------------------------------- /coinmetrics-eos/coinmetrics-eos.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-eos 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.EOS 17 | CoinMetrics.EOSArchive 18 | build-depends: 19 | aeson 20 | , base 21 | , bytestring 22 | , cereal 23 | , coinmetrics 24 | , hanalytics-base 25 | , http-client 26 | , stm 27 | , text 28 | , time 29 | , unordered-containers 30 | , vector 31 | , websockets 32 | ghc-options: -Wall 33 | default-language: Haskell2010 34 | -------------------------------------------------------------------------------- /coinmetrics-ethereum/coinmetrics-ethereum.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-ethereum 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Ethereum 17 | build-depends: 18 | aeson 19 | , base 20 | , coinmetrics 21 | , hanalytics-base 22 | , text 23 | , time 24 | , unordered-containers 25 | , vector 26 | , vector-instances 27 | ghc-options: -Wall 28 | default-language: Haskell2010 29 | -------------------------------------------------------------------------------- /coinmetrics-export/bintray-deb.json.in: -------------------------------------------------------------------------------- 1 | { "package": 2 | { "name": "coinmetrics-export" 3 | , "repo": "haskell-tools-deb" 4 | , "subject": "coinmetrics" 5 | , "desc": "Utility exporting blockchain data into formats suitable for analysis by other tools" 6 | , "website_url": "https://coinmetrics.io/" 7 | , "issue_tracker_url": "https://github.com/coinmetrics/haskell-tools/issues" 8 | , "vcs_url": "https://github.com/coinmetrics/haskell-tools.git" 9 | , "licenses": ["MIT"] 10 | , "public_download_numbers": true 11 | , "public_stats": true 12 | , "attributes": [] 13 | } 14 | , "version": 15 | { "name": "0.0.0.${TRAVIS_BUILD_NUMBER}" 16 | , "desc": "Bleeding edge version built from master branch" 17 | , "vcs_tag": "${TRAVIS_TAG}" 18 | , "gpgSign": false 19 | } 20 | , "files": 21 | [ { "includePattern": "pkg/(coinmetrics-export_.*\\.deb)" 22 | , "uploadPattern": "$1" 23 | , "matrixParams": 24 | { "deb_distribution": "unstable" 25 | , "deb_component": "main" 26 | , "deb_architecture": "amd64" 27 | } 28 | } 29 | ] 30 | , "publish": true 31 | } 32 | -------------------------------------------------------------------------------- /coinmetrics-export/bintray-rpm.json.in: -------------------------------------------------------------------------------- 1 | { "package": 2 | { "name": "coinmetrics-export" 3 | , "repo": "haskell-tools-rpm" 4 | , "subject": "coinmetrics" 5 | , "desc": "Utility exporting blockchain data into formats suitable for analysis by other tools" 6 | , "website_url": "https://coinmetrics.io/" 7 | , "issue_tracker_url": "https://github.com/coinmetrics/haskell-tools/issues" 8 | , "vcs_url": "https://github.com/coinmetrics/haskell-tools.git" 9 | , "licenses": ["MIT"] 10 | , "public_download_numbers": true 11 | , "public_stats": true 12 | , "attributes": [] 13 | } 14 | , "version": 15 | { "name": "0.0.0.${TRAVIS_BUILD_NUMBER}" 16 | , "desc": "Bleeding edge version built from master branch" 17 | , "vcs_tag": "${TRAVIS_TAG}" 18 | , "gpgSign": false 19 | } 20 | , "files": 21 | [ { "includePattern": "pkg/(coinmetrics-export-.*\\.rpm)" 22 | , "uploadPattern": "$1" 23 | } 24 | ] 25 | , "publish": true 26 | } 27 | -------------------------------------------------------------------------------- /coinmetrics-export/bintray.json.in: -------------------------------------------------------------------------------- 1 | { "package": 2 | { "name": "coinmetrics-export" 3 | , "repo": "haskell-tools" 4 | , "subject": "coinmetrics" 5 | , "desc": "Utility exporting blockchain data into formats suitable for analysis by other tools" 6 | , "website_url": "https://coinmetrics.io/" 7 | , "issue_tracker_url": "https://github.com/coinmetrics/haskell-tools/issues" 8 | , "vcs_url": "https://github.com/coinmetrics/haskell-tools.git" 9 | , "licenses": ["MIT"] 10 | , "public_download_numbers": true 11 | , "public_stats": true 12 | , "attributes": [] 13 | } 14 | , "version": 15 | { "name": "0.0.0.${TRAVIS_BUILD_NUMBER}" 16 | , "desc": "Bleeding edge version built from master branch" 17 | , "vcs_tag": "${TRAVIS_TAG}" 18 | , "gpgSign": false 19 | } 20 | , "files": 21 | [ { "includePattern": "bin/(coinmetrics-export)" 22 | , "uploadPattern": "$1" 23 | } 24 | ] 25 | , "publish": true 26 | } 27 | -------------------------------------------------------------------------------- /coinmetrics-export/coinmetrics-export.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-export 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | executable coinmetrics-export 15 | main-is: coinmetrics-export.hs 16 | other-modules: 17 | build-depends: 18 | aeson 19 | , avro 20 | , base 21 | , bytestring 22 | , cereal 23 | , coinmetrics 24 | , coinmetrics-all-blockchains 25 | , coinmetrics-iota 26 | , coinmetrics-storage 27 | , connection 28 | , data-default 29 | , hanalytics-base 30 | , hanalytics-bigquery 31 | , hanalytics-postgres 32 | , directory 33 | , diskhash 34 | , http-client 35 | , http-client-tls 36 | , memory 37 | , optparse-applicative 38 | , postgresql-libpq 39 | , process 40 | , prometheus-client 41 | , stm 42 | , text 43 | , unordered-containers 44 | , vector 45 | -- -O0 added temporarily to avoid space leak 46 | ghc-options: -threaded -Wall -O0 47 | default-language: Haskell2010 48 | -------------------------------------------------------------------------------- /coinmetrics-grin/CoinMetrics/Grin.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Grin 4 | ( Grin(..) 5 | , GrinBlock(..) 6 | ) where 7 | 8 | import qualified Data.Aeson as J 9 | import qualified Data.Aeson.Types as J 10 | import Data.Int 11 | import Data.Proxy 12 | import qualified Data.Text as T 13 | import Data.Time.Clock.POSIX 14 | import Data.Time.ISO8601 15 | import qualified Data.Vector as V 16 | 17 | import CoinMetrics.BlockChain 18 | import CoinMetrics.JsonRpc 19 | import CoinMetrics.Schema.Flatten 20 | import CoinMetrics.Schema.Util 21 | import CoinMetrics.Util 22 | import Hanalytics.Schema 23 | 24 | newtype Grin = Grin JsonRpc 25 | 26 | -- API: https://docs.grin.mw/grin-rfcs/text/0007-node-api-v2/ 27 | 28 | data GrinBlock = GrinBlock 29 | { gb_hash :: {-# UNPACK #-} !HexString 30 | , gb_version :: {-# UNPACK #-} !Int64 31 | , gb_height :: {-# UNPACK #-} !Int64 32 | , gb_timestamp :: {-# UNPACK #-} !Int64 33 | , gb_nonce :: !Integer 34 | , gb_total_difficulty :: !Integer 35 | , gb_secondary_scaling :: {-# UNPACK #-} !Int64 36 | , gb_total_kernel_offset :: {-# UNPACK #-} !HexString 37 | , gb_inputs :: !(V.Vector HexString) 38 | , gb_outputs :: !(V.Vector GrinOutput) 39 | , gb_kernels :: !(V.Vector GrinKernel) 40 | } 41 | 42 | instance HasBlockHeader GrinBlock where 43 | getBlockHeader GrinBlock 44 | { gb_hash = hash 45 | , gb_height = height 46 | , gb_timestamp = timestamp 47 | } = BlockHeader 48 | { bh_height = height 49 | , bh_hash = hash 50 | , bh_prevHash = Nothing 51 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timestamp 52 | } 53 | 54 | newtype GrinBlockWrapper = GrinBlockWrapper 55 | { unwrapGrinBlock :: GrinBlock 56 | } 57 | 58 | instance J.FromJSON GrinBlockWrapper where 59 | parseJSON = J.withObject "grin block" $ \fields -> do 60 | header <- fields J..: "header" 61 | fmap GrinBlockWrapper $ GrinBlock 62 | <$> (header J..: "hash") 63 | <*> (header J..: "version") 64 | <*> (header J..: "height") 65 | <*> (decodeDate =<< header J..: "timestamp") 66 | <*> (header J..: "nonce") 67 | <*> (header J..: "total_difficulty") 68 | <*> (header J..: "secondary_scaling") 69 | <*> (header J..: "total_kernel_offset") 70 | <*> (fields J..: "inputs") 71 | <*> (V.map unwrapGrinOutput <$> fields J..: "outputs") 72 | <*> (V.map unwrapGrinKernel <$> fields J..: "kernels") 73 | 74 | data GrinOutput = GrinOutput 75 | { go_output_type :: !T.Text 76 | , go_commit :: {-# UNPACK #-} !HexString 77 | } 78 | 79 | data GrinVersion = GrinVersion 80 | { gvBlockHeaderVersion :: {-# UNPACK #-} !Int 81 | , gvNodeVersion :: {-# UNPACK #-} !T.Text 82 | } 83 | 84 | instance J.FromJSON GrinVersion where 85 | parseJSON = J.withObject "grin version response" $ \response -> 86 | GrinVersion <$> response J..: "block_header_version" 87 | <*> response J..: "node_version" 88 | 89 | data GrinTip = GrinTip 90 | { gtrHeight :: {-# UNPACK #-} !BlockHeight 91 | , gtrLastBlockPushed :: {-# UNPACK #-} !HexString 92 | , gtrPrevBlockToLast :: {-# UNPACK #-} !HexString 93 | , gtrTotalDifficulty :: {-# UNPACK #-} !Double 94 | } 95 | 96 | instance J.FromJSON GrinTip where 97 | parseJSON = J.withObject "grin tip response" $ \response -> 98 | GrinTip <$> response J..: "height" 99 | <*> response J..: "last_block_pushed" 100 | <*> response J..: "prev_block_to_last" 101 | <*> response J..: "total_difficulty" 102 | 103 | newtype GrinOutputWrapper = GrinOutputWrapper 104 | { unwrapGrinOutput :: GrinOutput 105 | } 106 | 107 | instance J.FromJSON GrinOutputWrapper where 108 | parseJSON = J.withObject "grin output" $ \fields -> fmap GrinOutputWrapper $ GrinOutput 109 | <$> (fields J..: "output_type") 110 | <*> (fields J..: "commit") 111 | 112 | data GrinKernel = GrinKernel 113 | { gk_features :: !T.Text 114 | , gk_fee :: !Integer 115 | , gk_lock_height :: {-# UNPACK #-} !Int64 116 | , gk_excess :: {-# UNPACK #-} !HexString 117 | } 118 | 119 | newtype GrinKernelWrapper = GrinKernelWrapper 120 | { unwrapGrinKernel :: GrinKernel 121 | } 122 | 123 | newtype GrinRPCResponse a = GrinRPCResponse { unwrapGrinRPCResponse :: a } 124 | 125 | -- TODO Implement more graceful error handling 126 | instance J.FromJSON a => J.FromJSON (GrinRPCResponse a) where 127 | parseJSON = J.withObject "grin RPC response" $ \response -> GrinRPCResponse <$> (J.parseJSON =<< response J..: "Ok") 128 | 129 | instance J.FromJSON GrinKernelWrapper where 130 | parseJSON = J.withObject "grin kernel" $ \fields -> fmap GrinKernelWrapper $ GrinKernel 131 | <$> (fields J..: "features") 132 | <*> (fields J..: "fee") 133 | <*> (fields J..: "lock_height") 134 | <*> (fields J..: "excess") 135 | 136 | decodeDate :: T.Text -> J.Parser Int64 137 | decodeDate (T.unpack -> t) = case parseISO8601 t of 138 | Just date -> return $ floor $ utcTimeToPOSIXSeconds date 139 | Nothing -> fail $ "wrong date: " <> t 140 | 141 | genSchemaInstances [''GrinBlock, ''GrinOutput, ''GrinKernel] 142 | genFlattenedTypes "height" [| gb_height |] [("block", ''GrinBlock), ("output", ''GrinOutput), ("kernel", ''GrinKernel)] 143 | 144 | instance BlockChain Grin where 145 | type Block Grin = GrinBlock 146 | 147 | getBlockChainInfo _ = BlockChainInfo 148 | { bci_init = \BlockChainParams 149 | { bcp_httpManager = httpManager 150 | , bcp_httpRequest = httpRequest 151 | } -> return $ Grin $ newJsonRpc httpManager httpRequest Nothing 152 | , bci_defaultApiUrls = ["http://127.0.0.1:3413/v2/foreign"] 153 | , bci_defaultBeginBlock = 0 154 | , bci_defaultEndBlock = -60 155 | , bci_heightFieldName = "height" 156 | , bci_schemas = standardBlockChainSchemas 157 | (schemaOf (Proxy :: Proxy GrinBlock)) 158 | [ schemaOf (Proxy :: Proxy GrinOutput) 159 | , schemaOf (Proxy :: Proxy GrinKernel) 160 | ] 161 | "CREATE TABLE \"grin\" OF \"GrinBlock\" (PRIMARY KEY (\"height\"));" 162 | , bci_flattenSuffixes = ["blocks", "outputs", "kernels"] 163 | , bci_flattenPack = let 164 | f (blocks, outputs, kernels) = 165 | [ SomeBlocks (blocks :: [GrinBlock_flattened]) 166 | , SomeBlocks (outputs :: [GrinOutput_flattened]) 167 | , SomeBlocks (kernels :: [GrinKernel_flattened]) 168 | ] 169 | in f . mconcat . map flatten 170 | } 171 | 172 | getBlockChainNodeInfo (Grin jsonRpc) = 173 | BlockChainNodeInfo . gvNodeVersion . unwrapGrinRPCResponse <$> jsonRpcRequest jsonRpc "get_version" ([] :: V.Vector J.Value) 174 | 175 | getCurrentBlockHeight (Grin jsonRpc) = gtrHeight . unwrapGrinRPCResponse <$> jsonRpcRequest jsonRpc "get_tip" ([] :: V.Vector J.Value) 176 | 177 | getBlockByHeight (Grin jsonRpc) blockHeight = 178 | unwrapGrinBlock . unwrapGrinRPCResponse <$> jsonRpcRequest jsonRpc "get_block" ([J.Number $ fromIntegral blockHeight, J.Null, J.Null] :: V.Vector J.Value) -------------------------------------------------------------------------------- /coinmetrics-grin/coinmetrics-grin.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-grin 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Grin 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , http-client 24 | , iso8601-time 25 | , text 26 | , time 27 | , vector 28 | ghc-options: -Wall 29 | default-language: Haskell2010 30 | -------------------------------------------------------------------------------- /coinmetrics-iota/CoinMetrics/Iota.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, LambdaCase, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell #-} 2 | 3 | module CoinMetrics.Iota 4 | ( IotaTransaction(..) 5 | , Iota(..) 6 | , newIota 7 | , iotaGetTips 8 | , iotaGetMilestones 9 | , iotaGetTransactions 10 | , deserIotaTransaction 11 | , iotaEmptyHash 12 | ) where 13 | 14 | import Control.Monad 15 | import qualified Data.Aeson as J 16 | import qualified Data.Aeson.Types as J 17 | import qualified Data.ByteString as B 18 | import Data.Char 19 | import Data.Either 20 | import Data.Int 21 | import qualified Data.Serialize as S 22 | import qualified Data.Text as T 23 | import qualified Data.Text.Encoding as T 24 | import qualified Data.Vector as V 25 | import qualified Network.HTTP.Client as H 26 | 27 | import CoinMetrics.Schema.Util 28 | import CoinMetrics.Util 29 | 30 | data IotaTransaction = IotaTransaction 31 | { it_hash :: !T.Text 32 | , it_signature :: !T.Text 33 | , it_address :: !T.Text 34 | , it_value :: !Int64 35 | , it_obsoleteTag :: !T.Text 36 | , it_timestamp :: !Int64 37 | , it_currentIndex :: !Int64 38 | , it_lastIndex :: !Int64 39 | , it_bundle :: !T.Text 40 | , it_trunkTransaction :: !T.Text 41 | , it_branchTransaction :: !T.Text 42 | , it_tag :: !T.Text 43 | , it_attachmentTimestamp :: !Int64 44 | , it_attachmentTimestampLowerBound :: !Int64 45 | , it_attachmentTimestampUpperBound :: !Int64 46 | , it_nonce :: !T.Text 47 | } 48 | 49 | genSchemaInstances [''IotaTransaction] 50 | 51 | -- | Deserialize transaction. 52 | deserIotaTransaction :: T.Text -> S.Get IotaTransaction 53 | deserIotaTransaction hash = IotaTransaction hash 54 | <$> (T.decodeUtf8 <$> S.getByteString 2187) 55 | <*> (T.decodeUtf8 <$> S.getByteString 81) 56 | <*> deserInt 11 57 | <*> (T.decodeUtf8 <$> (check16ZeroTrytes >> S.getByteString 27)) 58 | <*> deserInt 9 59 | <*> deserInt 9 60 | <*> deserInt 9 61 | <*> (T.decodeUtf8 <$> S.getByteString 81) 62 | <*> (T.decodeUtf8 <$> S.getByteString 81) 63 | <*> (T.decodeUtf8 <$> S.getByteString 81) 64 | <*> (T.decodeUtf8 <$> S.getByteString 27) 65 | <*> deserInt 9 66 | <*> deserInt 9 67 | <*> deserInt 9 68 | <*> (T.decodeUtf8 <$> S.getByteString 27) 69 | 70 | -- | Deserialize 27-trit (9-tryte) integer. 71 | deserInt :: Int -> S.Get Int64 72 | deserInt n = f 0 0 1 where 73 | f i s p = if i >= n then return s else do 74 | t <- deserTryte 75 | f (i + 1) (s + t * p) (p * 27) 76 | 77 | -- | Check that there's 16 zero trytes. 78 | check16ZeroTrytes :: S.Get () 79 | check16ZeroTrytes = do 80 | trytes <- S.getByteString 16 81 | unless (trytes == "9999999999999999") $ fail "wrong zero trytes" 82 | 83 | -- | Deserialize a tryte. 84 | deserTryte :: S.Get Int64 85 | deserTryte = f <$> S.getWord8 where 86 | f = \case 87 | 57 {- '9' -} -> 0 88 | 65 {- 'A' -} -> 1 89 | 66 {- 'B' -} -> 2 90 | 67 {- 'C' -} -> 3 91 | 68 {- 'D' -} -> 4 92 | 69 {- 'E' -} -> 5 93 | 70 {- 'F' -} -> 6 94 | 71 {- 'G' -} -> 7 95 | 72 {- 'H' -} -> 8 96 | 73 {- 'I' -} -> 9 97 | 74 {- 'J' -} -> 10 98 | 75 {- 'K' -} -> 11 99 | 76 {- 'L' -} -> 12 100 | 77 {- 'M' -} -> 13 101 | 78 {- 'N' -} -> -13 102 | 79 {- 'O' -} -> -12 103 | 80 {- 'P' -} -> -11 104 | 81 {- 'Q' -} -> -10 105 | 82 {- 'R' -} -> -9 106 | 83 {- 'S' -} -> -8 107 | 84 {- 'T' -} -> -7 108 | 85 {- 'U' -} -> -6 109 | 86 {- 'V' -} -> -5 110 | 87 {- 'W' -} -> -4 111 | 88 {- 'X' -} -> -3 112 | 89 {- 'Y' -} -> -2 113 | 90 {- 'Z' -} -> -1 114 | _ -> error "wrong tryte" 115 | 116 | -- | IOTA connector. 117 | data Iota = Iota 118 | { iota_httpManager :: !H.Manager 119 | , iota_httpRequest :: !H.Request 120 | } 121 | 122 | newIota :: H.Manager -> H.Request -> Iota 123 | newIota httpManager httpRequest = Iota 124 | { iota_httpManager = httpManager 125 | , iota_httpRequest = httpRequest 126 | { H.method = "POST" 127 | , H.requestHeaders = 128 | [ ("Content-Type", "application/json") 129 | , ("X-IOTA-API-Version", "1") 130 | ] 131 | } 132 | } 133 | 134 | iotaRequest :: J.FromJSON a => Iota -> (a -> J.Parser r) -> J.Value -> IO r 135 | iotaRequest Iota 136 | { iota_httpManager = httpManager 137 | , iota_httpRequest = httpRequest 138 | } parser value = do 139 | body <- H.responseBody <$> tryWithRepeat (H.httpLbs httpRequest 140 | { H.requestBody = H.RequestBodyLBS $ J.encode value 141 | } httpManager) 142 | result <- either fail return $ J.eitherDecode body 143 | either fail return $ J.parseEither parser result 144 | 145 | iotaGetTips :: Iota -> IO (V.Vector T.Text) 146 | iotaGetTips iota = iotaRequest iota (J..: "hashes") $ J.Object 147 | [ ("command", "getTips") 148 | ] 149 | 150 | iotaGetMilestones :: Iota -> IO (V.Vector T.Text) 151 | iotaGetMilestones iota = iotaRequest iota parseMilestones $ J.Object 152 | [ ("command", "getNodeInfo") 153 | ] where 154 | parseMilestones o = do 155 | milestone <- o J..: "latestMilestone" 156 | solidMilestone <- o J..: "latestSolidSubtangleMilestone" 157 | return $ V.fromList [milestone, solidMilestone] 158 | 159 | iotaGetTransactions :: Iota -> V.Vector T.Text -> IO (V.Vector (Maybe IotaTransaction)) 160 | iotaGetTransactions iota hashes = f <$> iotaRequest iota (J..: "trytes") (J.Object 161 | [ ("command", "getTrytes") 162 | , ("hashes", J.toJSON hashes) 163 | ]) 164 | where f = V.zipWith (\hash trytes -> (fromRight (error "wrong transaction") . S.runGet (deserIotaTransaction hash)) <$> filterOutMissingTransaction (T.encodeUtf8 trytes)) hashes 165 | 166 | filterOutMissingTransaction :: B.ByteString -> Maybe B.ByteString 167 | filterOutMissingTransaction trytes = if B.any (/= fromIntegral (ord '9')) trytes then Just trytes else Nothing 168 | 169 | iotaEmptyHash :: T.Text 170 | iotaEmptyHash = "999999999999999999999999999999999999999999999999999999999999999999999999999999999" 171 | -------------------------------------------------------------------------------- /coinmetrics-iota/coinmetrics-iota.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-iota 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Iota 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , cereal 22 | , coinmetrics 23 | , http-client 24 | , memory 25 | , text 26 | , vector 27 | ghc-options: -Wall 28 | default-language: Haskell2010 29 | -------------------------------------------------------------------------------- /coinmetrics-monero/CoinMetrics/Monero.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Monero 4 | ( Monero(..) 5 | , MoneroBlock(..) 6 | , MoneroTransaction(..) 7 | , MoneroTransactionInput(..) 8 | , MoneroTransactionOutput(..) 9 | ) where 10 | 11 | import qualified Data.Aeson as J 12 | import qualified Data.Aeson.Types as J 13 | import qualified Data.ByteString as B 14 | import qualified Data.ByteString.Lazy as BL 15 | import qualified Data.ByteString.Short as BS 16 | import qualified Data.HashMap.Lazy as HML 17 | import Data.Maybe 18 | import Data.Int 19 | import Data.Proxy 20 | import Data.Scientific 21 | import qualified Data.Text.Encoding as T 22 | import Data.Time.Clock.POSIX 23 | import qualified Data.Vector as V 24 | 25 | import CoinMetrics.BlockChain 26 | import CoinMetrics.JsonRpc 27 | import CoinMetrics.Schema.Flatten 28 | import CoinMetrics.Schema.Util 29 | import CoinMetrics.Util 30 | import Hanalytics.Schema 31 | 32 | newtype Monero = Monero JsonRpc 33 | 34 | data MoneroBlock = MoneroBlock 35 | { mb_height :: {-# UNPACK #-} !Int64 36 | , mb_hash :: {-# UNPACK #-} !HexString 37 | , mb_major_version :: {-# UNPACK #-} !Int64 38 | , mb_minor_version :: {-# UNPACK #-} !Int64 39 | , mb_difficulty :: {-# UNPACK #-} !Int64 40 | , mb_reward :: {-# UNPACK #-} !Int64 41 | , mb_timestamp :: {-# UNPACK #-} !Int64 42 | , mb_nonce :: {-# UNPACK #-} !Int64 43 | , mb_size :: {-# UNPACK #-} !Int64 44 | , mb_miner_tx :: !MoneroTransaction 45 | , mb_transactions :: !(V.Vector MoneroTransaction) 46 | } 47 | 48 | instance HasBlockHeader MoneroBlock where 49 | getBlockHeader MoneroBlock 50 | { mb_height = height 51 | , mb_hash = hash 52 | , mb_timestamp = timestamp 53 | } = BlockHeader 54 | { bh_height = height 55 | , bh_hash = hash 56 | , bh_prevHash = Nothing 57 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timestamp 58 | } 59 | 60 | newtype MoneroBlockWrapper = MoneroBlockWrapper 61 | { unwrapMoneroBlock :: MoneroBlock 62 | } 63 | 64 | instance J.FromJSON MoneroBlockWrapper where 65 | parseJSON = J.withObject "monero block" $ \fields -> fmap MoneroBlockWrapper $ MoneroBlock 66 | <$> (fields J..: "height") 67 | <*> (fields J..: "hash") 68 | <*> (fields J..: "major_version") 69 | <*> (fields J..: "minor_version") 70 | <*> (fields J..: "difficulty") 71 | <*> (fields J..: "reward") 72 | <*> (fields J..: "timestamp") 73 | <*> (fields J..: "nonce") 74 | <*> (fields J..: "size") 75 | <*> (unwrapMoneroTransaction <$> fields J..: "miner_tx") 76 | <*> (V.map unwrapMoneroTransaction <$> fields J..: "transactions") 77 | 78 | data MoneroTransaction = MoneroTransaction 79 | { mt_hash :: !(Maybe HexString) 80 | , mt_version :: {-# UNPACK #-} !Int64 81 | , mt_unlock_time :: !Scientific 82 | , mt_vin :: !(V.Vector MoneroTransactionInput) 83 | , mt_vout :: !(V.Vector MoneroTransactionOutput) 84 | , mt_extra :: {-# UNPACK #-} !HexString 85 | , mt_fee :: !(Maybe Int64) 86 | } 87 | 88 | newtype MoneroTransactionWrapper = MoneroTransactionWrapper 89 | { unwrapMoneroTransaction :: MoneroTransaction 90 | } 91 | 92 | instance J.FromJSON MoneroTransactionWrapper where 93 | parseJSON = J.withObject "monero transaction" $ \fields -> fmap MoneroTransactionWrapper $ MoneroTransaction 94 | <$> (fields J..:? "hash") 95 | <*> (fields J..: "version") 96 | <*> (fields J..: "unlock_time") 97 | <*> (V.map unwrapMoneroTransactionInput <$> fields J..: "vin") 98 | <*> (V.map unwrapMoneroTransactionOutput <$> fields J..: "vout") 99 | <*> (HexString . BS.toShort . B.pack <$> fields J..: "extra") 100 | <*> parseFee fields 101 | where 102 | parseFee fields = do 103 | maybeRctSignatures <- fields J..:? "rct_signatures" 104 | case maybeRctSignatures of 105 | Just rctSignatures -> rctSignatures J..:? "txnFee" 106 | Nothing -> return Nothing 107 | 108 | data MoneroTransactionInput = MoneroTransactionInput 109 | { mti_amount :: !(Maybe Int64) 110 | , mti_k_image :: !(Maybe HexString) 111 | , mti_key_offsets :: !(V.Vector Int64) 112 | , mti_height :: !(Maybe Int64) 113 | } 114 | 115 | newtype MoneroTransactionInputWrapper = MoneroTransactionInputWrapper 116 | { unwrapMoneroTransactionInput :: MoneroTransactionInput 117 | } 118 | 119 | instance J.FromJSON MoneroTransactionInputWrapper where 120 | parseJSON = J.withObject "monero transaction input" $ \fields -> do 121 | maybeKey <- fields J..:? "key" 122 | maybeGen <- fields J..:? "gen" 123 | fmap MoneroTransactionInputWrapper $ MoneroTransactionInput 124 | <$> traverse (J..: "amount") maybeKey 125 | <*> traverse (J..: "k_image") maybeKey 126 | <*> (fromMaybe V.empty <$> traverse (J..: "key_offsets") maybeKey) 127 | <*> traverse (J..: "height") maybeGen 128 | 129 | data MoneroTransactionOutput = MoneroTransactionOutput 130 | { mto_amount :: !Int64 131 | , mto_key :: {-# UNPACK #-} !HexString 132 | } 133 | 134 | newtype MoneroTransactionOutputWrapper = MoneroTransactionOutputWrapper 135 | { unwrapMoneroTransactionOutput :: MoneroTransactionOutput 136 | } 137 | 138 | instance J.FromJSON MoneroTransactionOutputWrapper where 139 | parseJSON = J.withObject "monero transaction output" $ \fields -> fmap MoneroTransactionOutputWrapper $ MoneroTransactionOutput 140 | <$> (fields J..: "amount") 141 | <*> ((J..: "key") =<< fields J..: "target") 142 | 143 | genSchemaInstances [''MoneroBlock, ''MoneroTransaction, ''MoneroTransactionInput, ''MoneroTransactionOutput] 144 | genFlattenedTypes "height" [| mb_height |] [("block", ''MoneroBlock), ("transaction", ''MoneroTransaction), ("input", ''MoneroTransactionInput), ("output", ''MoneroTransactionOutput)] 145 | 146 | instance BlockChain Monero where 147 | type Block Monero = MoneroBlock 148 | 149 | getBlockChainInfo _ = BlockChainInfo 150 | { bci_init = \BlockChainParams 151 | { bcp_httpManager = httpManager 152 | , bcp_httpRequest = httpRequest 153 | } -> return $ Monero $ newJsonRpc httpManager httpRequest Nothing 154 | , bci_defaultApiUrls = ["http://127.0.0.1:18081/json_rpc"] 155 | , bci_defaultBeginBlock = 0 156 | , bci_defaultEndBlock = -60 -- conservative rewrite limit 157 | , bci_heightFieldName = "height" 158 | , bci_schemas = standardBlockChainSchemas 159 | (schemaOf (Proxy :: Proxy MoneroBlock)) 160 | [ schemaOf (Proxy :: Proxy MoneroTransactionInput) 161 | , schemaOf (Proxy :: Proxy MoneroTransactionOutput) 162 | , schemaOf (Proxy :: Proxy MoneroTransaction) 163 | ] 164 | "CREATE TABLE \"monero\" OF \"MoneroBlock\" (PRIMARY KEY (\"height\"));" 165 | , bci_flattenSuffixes = ["blocks", "transactions", "inputs", "outputs"] 166 | , bci_flattenPack = let 167 | f (blocks, (transactions, inputs, outputs)) = 168 | [ SomeBlocks (blocks :: [MoneroBlock_flattened]) 169 | , SomeBlocks (transactions :: [MoneroTransaction_flattened]) 170 | , SomeBlocks (inputs :: [MoneroTransactionInput_flattened]) 171 | , SomeBlocks (outputs :: [MoneroTransactionOutput_flattened]) 172 | ] 173 | in f . mconcat . map flatten 174 | } 175 | 176 | getBlockChainNodeInfo (Monero jsonRpc) = do 177 | version <- either fail return . J.parseEither (J..: "version") =<< jsonRpcRequest jsonRpc "get_info" J.Null 178 | return BlockChainNodeInfo 179 | { bcni_version = version 180 | } 181 | 182 | getCurrentBlockHeight (Monero jsonRpc) = 183 | either fail (return . (+ (-1))) . J.parseEither (J..: "count") =<< jsonRpcRequest jsonRpc "get_block_count" J.Null 184 | 185 | getBlockByHeight (Monero jsonRpc) blockHeight = do 186 | blockInfo <- jsonRpcRequest jsonRpc "get_block" (J.Object [("height", J.toJSON blockHeight)]) 187 | J.Object blockHeaderFields <- either fail return $ J.parseEither (J..: "block_header") blockInfo 188 | Just (J.Bool False) <- return $ HML.lookup "orphan_status" blockHeaderFields 189 | Just blockHash <- return $ HML.lookup "hash" blockHeaderFields 190 | Just blockDifficulty <- return $ HML.lookup "difficulty" blockHeaderFields 191 | Just blockReward <- return $ HML.lookup "reward" blockHeaderFields 192 | Just blockSize <- return $ HML.lookup "block_size" blockHeaderFields 193 | blockJsonFields <- either fail return $ ((J.eitherDecode' . BL.fromStrict . T.encodeUtf8) =<<) $ J.parseEither (J..: "json") blockInfo 194 | txHashes <- either fail return $ J.parseEither (J..: "tx_hashes") blockJsonFields 195 | transactions <- if V.null txHashes 196 | then return V.empty 197 | else do 198 | transactionsJsons <- either fail return . J.parseEither (J..: "txs_as_json") =<< nonJsonRpcRequest jsonRpc "/get_transactions" (J.Object [("txs_hashes", J.Array txHashes), ("decode_as_json", J.Bool True)]) 199 | V.forM (V.zip txHashes transactionsJsons) $ \(txHash, transactionJson) -> do 200 | transactionFields <- either fail return $ J.eitherDecode' $ BL.fromStrict $ T.encodeUtf8 transactionJson 201 | return $ J.Object $ HML.insert "hash" txHash transactionFields 202 | let 203 | jsonBlock = J.Object 204 | $ HML.insert "height" (J.toJSON blockHeight) 205 | $ HML.insert "hash" blockHash 206 | $ HML.insert "difficulty" blockDifficulty 207 | $ HML.insert "reward" blockReward 208 | $ HML.insert "size" blockSize 209 | $ HML.insert "transactions" (J.Array transactions) 210 | blockJsonFields 211 | case J.fromJSON jsonBlock of 212 | J.Success block -> return $ unwrapMoneroBlock block 213 | J.Error err -> fail err 214 | -------------------------------------------------------------------------------- /coinmetrics-monero/coinmetrics-monero.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-monero 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Monero 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , scientific 24 | , text 25 | , time 26 | , unordered-containers 27 | , vector 28 | ghc-options: -Wall 29 | default-language: Haskell2010 30 | -------------------------------------------------------------------------------- /coinmetrics-monitor/bintray-deb.json.in: -------------------------------------------------------------------------------- 1 | { "package": 2 | { "name": "coinmetrics-monitor" 3 | , "repo": "haskell-tools-deb" 4 | , "subject": "coinmetrics" 5 | , "desc": "Utility monitoring blockchain nodes" 6 | , "website_url": "https://coinmetrics.io/" 7 | , "issue_tracker_url": "https://github.com/coinmetrics/haskell-tools/issues" 8 | , "vcs_url": "https://github.com/coinmetrics/haskell-tools.git" 9 | , "licenses": ["MIT"] 10 | , "public_download_numbers": true 11 | , "public_stats": true 12 | , "attributes": [] 13 | } 14 | , "version": 15 | { "name": "0.0.0.${TRAVIS_BUILD_NUMBER}" 16 | , "desc": "Bleeding edge version built from master branch" 17 | , "vcs_tag": "${TRAVIS_TAG}" 18 | , "gpgSign": false 19 | } 20 | , "files": 21 | [ { "includePattern": "pkg/(coinmetrics-monitor_.*\\.deb)" 22 | , "uploadPattern": "$1" 23 | , "matrixParams": 24 | { "deb_distribution": "unstable" 25 | , "deb_component": "main" 26 | , "deb_architecture": "amd64" 27 | } 28 | } 29 | ] 30 | , "publish": true 31 | } 32 | -------------------------------------------------------------------------------- /coinmetrics-monitor/bintray-rpm.json.in: -------------------------------------------------------------------------------- 1 | { "package": 2 | { "name": "coinmetrics-monitor" 3 | , "repo": "haskell-tools-rpm" 4 | , "subject": "coinmetrics" 5 | , "desc": "Utility monitoring blockchain nodes" 6 | , "website_url": "https://coinmetrics.io/" 7 | , "issue_tracker_url": "https://github.com/coinmetrics/haskell-tools/issues" 8 | , "vcs_url": "https://github.com/coinmetrics/haskell-tools.git" 9 | , "licenses": ["MIT"] 10 | , "public_download_numbers": true 11 | , "public_stats": true 12 | , "attributes": [] 13 | } 14 | , "version": 15 | { "name": "0.0.0.${TRAVIS_BUILD_NUMBER}" 16 | , "desc": "Bleeding edge version built from master branch" 17 | , "vcs_tag": "${TRAVIS_TAG}" 18 | , "gpgSign": false 19 | } 20 | , "files": 21 | [ { "includePattern": "pkg/(coinmetrics-monitor-.*\\.rpm)" 22 | , "uploadPattern": "$1" 23 | } 24 | ] 25 | , "publish": true 26 | } 27 | -------------------------------------------------------------------------------- /coinmetrics-monitor/bintray.json.in: -------------------------------------------------------------------------------- 1 | { "package": 2 | { "name": "coinmetrics-monitor" 3 | , "repo": "haskell-tools" 4 | , "subject": "coinmetrics" 5 | , "desc": "Utility monitoring blockchain nodes" 6 | , "website_url": "https://coinmetrics.io/" 7 | , "issue_tracker_url": "https://github.com/coinmetrics/haskell-tools/issues" 8 | , "vcs_url": "https://github.com/coinmetrics/haskell-tools.git" 9 | , "licenses": ["MIT"] 10 | , "public_download_numbers": true 11 | , "public_stats": true 12 | , "attributes": [] 13 | } 14 | , "version": 15 | { "name": "0.0.0.${TRAVIS_BUILD_NUMBER}" 16 | , "desc": "Bleeding edge version built from master branch" 17 | , "vcs_tag": "${TRAVIS_TAG}" 18 | , "gpgSign": false 19 | } 20 | , "files": 21 | [ { "includePattern": "bin/(coinmetrics-monitor)" 22 | , "uploadPattern": "$1" 23 | } 24 | ] 25 | , "publish": true 26 | } 27 | -------------------------------------------------------------------------------- /coinmetrics-monitor/coinmetrics-monitor.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-monitor 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | executable coinmetrics-monitor 15 | main-is: coinmetrics-monitor.hs 16 | other-modules: 17 | build-depends: 18 | base 19 | , coinmetrics 20 | , coinmetrics-all-blockchains 21 | , connection 22 | , data-default 23 | , http-client 24 | , http-client-tls 25 | , http-types 26 | , optparse-applicative 27 | , prometheus-client 28 | , stm 29 | , text 30 | , time 31 | , wai 32 | , warp 33 | ghc-options: -threaded -Wall 34 | default-language: Haskell2010 35 | -------------------------------------------------------------------------------- /coinmetrics-nem/CoinMetrics/Nem.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies #-} 2 | 3 | module CoinMetrics.Nem 4 | ( Nem(..) 5 | , NemBlock(..) 6 | , NemTransaction(..) 7 | , NemNestedTransaction(..) 8 | ) where 9 | 10 | import Control.Monad 11 | import qualified Data.Aeson as J 12 | import qualified Data.Aeson.Types as J 13 | import qualified Data.ByteArray.Encoding as BA 14 | import qualified Data.ByteString as B 15 | import qualified Data.ByteString.Short as BS 16 | import Data.Int 17 | import Data.Maybe 18 | import Data.Proxy 19 | import qualified Data.Text as T 20 | import qualified Data.Text.Encoding as T 21 | import Data.Time.Clock 22 | import Data.Time.Format 23 | import qualified Data.Vector as V 24 | import qualified Network.HTTP.Client as H 25 | 26 | import CoinMetrics.BlockChain 27 | import CoinMetrics.Schema.Util 28 | import CoinMetrics.Util 29 | import Hanalytics.Schema 30 | 31 | -- | Nem connector. 32 | data Nem = Nem 33 | { nem_httpManager :: !H.Manager 34 | , nem_httpRequest :: !H.Request 35 | } 36 | 37 | nemRequest :: J.FromJSON r => Nem -> T.Text -> Maybe J.Value -> [(B.ByteString, Maybe B.ByteString)] -> IO r 38 | nemRequest Nem 39 | { nem_httpManager = httpManager 40 | , nem_httpRequest = httpRequest 41 | } path maybeBody params = either fail return . J.eitherDecode' . H.responseBody =<< H.httpLbs (H.setQueryString params httpRequest 42 | { H.path = H.path httpRequest <> T.encodeUtf8 path 43 | , H.method = if isJust maybeBody then "POST" else "GET" 44 | , H.requestBody = maybe (H.requestBody httpRequest) (H.RequestBodyLBS . J.encode) maybeBody 45 | }) httpManager 46 | 47 | data NemBlock = NemBlock 48 | { nb_timeStamp :: {-# UNPACK #-} !Int64 49 | , nb_height :: {-# UNPACK #-} !Int64 50 | , nb_signer :: {-# UNPACK #-} !HexString 51 | , nb_transactions :: !(V.Vector NemTransaction) 52 | } 53 | 54 | instance HasBlockHeader NemBlock where 55 | getBlockHeader NemBlock 56 | { nb_timeStamp = timeStamp 57 | , nb_height = height 58 | } = BlockHeader 59 | { bh_height = height 60 | , bh_hash = mempty 61 | , bh_prevHash = Nothing 62 | , bh_timestamp = fromIntegral timeStamp `addUTCTime` nemBlockGenesisTimestamp 63 | } 64 | 65 | nemBlockGenesisTimestamp :: UTCTime 66 | nemBlockGenesisTimestamp = parseTimeOrError False defaultTimeLocale "%Y-%m-%dT%H:%M:%SZ" "2015-03-29T00:06:25Z" 67 | 68 | newtype NemBlockWrapper = NemBlockWrapper 69 | { unwrapNemBlock :: NemBlock 70 | } 71 | 72 | instance J.FromJSON NemBlockWrapper where 73 | parseJSON = J.withObject "nem block" $ \fields -> fmap NemBlockWrapper $ NemBlock 74 | <$> (fields J..: "timeStamp") 75 | <*> (fields J..: "height") 76 | <*> (fields J..: "signer") 77 | <*> (V.map unwrapNemTransaction <$> fields J..: "transactions") 78 | 79 | data NemTransaction = NemTransaction 80 | { nt_timeStamp :: {-# UNPACK #-} !Int64 81 | , nt_fee :: {-# UNPACK #-} !Int64 82 | , nt_type :: {-# UNPACK #-} !Int64 83 | , nt_deadline :: {-# UNPACK #-} !Int64 84 | , nt_signer :: {-# UNPACK #-} !HexString 85 | , nt_signerAddress :: !T.Text 86 | , nt_recipient :: !(Maybe T.Text) 87 | , nt_mode :: !(Maybe Int64) 88 | , nt_remoteAccount :: !(Maybe HexString) 89 | , nt_creationFee :: !(Maybe Int64) 90 | , nt_creationFeeSink :: !(Maybe T.Text) 91 | , nt_delta :: !(Maybe Int64) 92 | , nt_otherAccount :: !(Maybe T.Text) 93 | , nt_rentalFee :: !(Maybe Int64) 94 | , nt_rentalFeeSink :: !(Maybe T.Text) 95 | , nt_newPart :: !(Maybe T.Text) 96 | , nt_parent :: !(Maybe T.Text) 97 | , nt_amount :: !(Maybe Int64) 98 | , nt_otherTrans :: !(Maybe NemNestedTransaction) 99 | , nt_signatures :: !(V.Vector NemNestedTransaction) 100 | } 101 | 102 | newtype NemTransactionWrapper = NemTransactionWrapper 103 | { unwrapNemTransaction :: NemTransaction 104 | } 105 | 106 | instance J.FromJSON NemTransactionWrapper where 107 | parseJSON = J.withObject "nem transaction" $ \fields -> fmap NemTransactionWrapper $ NemTransaction 108 | <$> (fields J..: "timeStamp") 109 | <*> (fields J..: "fee") 110 | <*> (fields J..: "type") 111 | <*> (fields J..: "deadline") 112 | <*> (fields J..: "signer") 113 | <*> return "" 114 | <*> (fields J..:? "recipient") 115 | <*> (fields J..:? "mode") 116 | <*> (fields J..:? "remoteAccount") 117 | <*> (fields J..:? "creationFee") 118 | <*> (fields J..:? "creationFeeSink") 119 | <*> (fields J..:? "delta") 120 | <*> (fields J..:? "otherAccount") 121 | <*> (fields J..:? "rentalFee") 122 | <*> (fields J..:? "rentalFeeSink") 123 | <*> (fields J..:? "newPart") 124 | <*> (fields J..:? "parent") 125 | <*> (fields J..:? "amount") 126 | <*> (fields J..:? "otherTrans") 127 | <*> (fromMaybe V.empty <$> fields J..:? "signatures") 128 | 129 | data NemNestedTransaction = NemNestedTransaction 130 | { nnt_timeStamp :: {-# UNPACK #-} !Int64 131 | , nnt_fee :: {-# UNPACK #-} !Int64 132 | , nnt_type :: {-# UNPACK #-} !Int64 133 | , nnt_deadline :: {-# UNPACK #-} !Int64 134 | , nnt_signer :: {-# UNPACK #-} !HexString 135 | , nnt_recipient :: !(Maybe T.Text) 136 | , nnt_mode :: !(Maybe Int64) 137 | , nnt_remoteAccount :: !(Maybe HexString) 138 | , nnt_creationFee :: !(Maybe Int64) 139 | , nnt_creationFeeSink :: !(Maybe T.Text) 140 | , nnt_delta :: !(Maybe Int64) 141 | , nnt_otherAccount :: !(Maybe T.Text) 142 | , nnt_rentalFee :: !(Maybe Int64) 143 | , nnt_rentalFeeSink :: !(Maybe T.Text) 144 | , nnt_newPart :: !(Maybe T.Text) 145 | , nnt_parent :: !(Maybe T.Text) 146 | , nnt_amount :: !(Maybe Int64) 147 | } 148 | 149 | genSchemaInstances [''NemBlock, ''NemTransaction, ''NemNestedTransaction] 150 | 151 | instance BlockChain Nem where 152 | type Block Nem = NemBlock 153 | 154 | getBlockChainInfo _ = BlockChainInfo 155 | { bci_init = \BlockChainParams 156 | { bcp_httpManager = httpManager 157 | , bcp_httpRequest = httpRequest 158 | } -> return Nem 159 | { nem_httpManager = httpManager 160 | , nem_httpRequest = httpRequest 161 | { H.requestHeaders = [("Content-Type", "application/json")] 162 | } 163 | } 164 | , bci_defaultApiUrls = ["http://127.0.0.1:7890/"] 165 | , bci_defaultBeginBlock = 1 166 | , bci_defaultEndBlock = -360 -- actual rewrite limit 167 | , bci_heightFieldName = "height" 168 | , bci_schemas = standardBlockChainSchemas 169 | (schemaOf (Proxy :: Proxy NemBlock)) 170 | [ schemaOf (Proxy :: Proxy NemNestedTransaction) 171 | , schemaOf (Proxy :: Proxy NemTransaction) 172 | ] 173 | "CREATE TABLE \"nem\" OF \"NemBlock\" (PRIMARY KEY (\"height\"));" 174 | } 175 | 176 | getBlockChainNodeInfo nem = do 177 | version <- either fail return . J.parseEither ((J..: "version") <=< (J..: "metaData")) =<< nemRequest nem "/node/info" Nothing [] 178 | return BlockChainNodeInfo 179 | { bcni_version = version 180 | } 181 | 182 | getCurrentBlockHeight nem = tryWithRepeat $ either fail return 183 | . J.parseEither (J..: "height") 184 | =<< nemRequest nem "/chain/height" Nothing [] 185 | 186 | getBlockByHeight nem blockHeight = do 187 | block@NemBlock 188 | { nb_transactions = transactions 189 | } <- tryWithRepeat $ either fail (return . unwrapNemBlock) . J.parseEither J.parseJSON =<< nemRequest nem "/block/at/public" (Just $ J.Object [("height", J.Number $ fromIntegral blockHeight)]) [] 190 | signersAddresses <- V.forM transactions $ \NemTransaction 191 | { nt_signer = signer 192 | } -> tryWithRepeat $ either fail return 193 | . J.parseEither ((J..: "address") <=< (J..: "account")) 194 | =<< nemRequest nem "/account/get/from-public-key" Nothing [("publicKey", Just $ BA.convertToBase BA.Base16 $ BS.fromShort $ unHexString signer)] 195 | return block 196 | { nb_transactions = V.zipWith (\transaction signerAddress -> transaction 197 | { nt_signerAddress = signerAddress 198 | }) transactions signersAddresses 199 | } 200 | -------------------------------------------------------------------------------- /coinmetrics-nem/coinmetrics-nem.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-nem 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Nem 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , http-client 24 | , memory 25 | , text 26 | , time 27 | , vector 28 | ghc-options: -Wall 29 | default-language: Haskell2010 30 | -------------------------------------------------------------------------------- /coinmetrics-neo/CoinMetrics/Neo.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Neo 4 | ( Neo(..) 5 | , NeoBlock(..) 6 | , NeoTransaction(..) 7 | , NeoTransactionInput(..) 8 | , NeoTransactionOutput(..) 9 | ) where 10 | 11 | import qualified Data.Aeson as J 12 | import qualified Data.Aeson.Types as J 13 | import Data.Int 14 | import Data.Proxy 15 | import Data.Scientific 16 | import qualified Data.Text as T 17 | import Data.Time.Clock.POSIX 18 | import qualified Data.Vector as V 19 | 20 | import CoinMetrics.BlockChain 21 | import CoinMetrics.JsonRpc 22 | import CoinMetrics.Schema.Flatten 23 | import CoinMetrics.Schema.Util 24 | import CoinMetrics.Util 25 | import Hanalytics.Schema 26 | 27 | newtype Neo = Neo JsonRpc 28 | 29 | data NeoBlock = NeoBlock 30 | { nb_hash :: {-# UNPACK #-} !HexString 31 | , nb_size :: {-# UNPACK #-} !Int64 32 | , nb_time :: {-# UNPACK #-} !Int64 33 | , nb_index :: {-# UNPACK #-} !Int64 34 | , nb_tx :: !(V.Vector NeoTransaction) 35 | } 36 | 37 | instance HasBlockHeader NeoBlock where 38 | getBlockHeader NeoBlock 39 | { nb_hash = hash 40 | , nb_index = index 41 | , nb_time = time 42 | } = BlockHeader 43 | { bh_height = index 44 | , bh_hash = hash 45 | , bh_prevHash = Nothing 46 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral time 47 | } 48 | 49 | newtype NeoBlockWrapper = NeoBlockWrapper 50 | { unwrapNeoBlock :: NeoBlock 51 | } 52 | 53 | instance J.FromJSON NeoBlockWrapper where 54 | parseJSON = J.withObject "neo block" $ \fields -> fmap NeoBlockWrapper $ NeoBlock 55 | <$> (decode0xHexBytes =<< fields J..: "hash") 56 | <*> (fields J..: "size") 57 | <*> (fields J..: "time") 58 | <*> (fields J..: "index") 59 | <*> (V.map unwrapNeoTransaction <$> fields J..: "tx") 60 | 61 | data NeoTransaction = NeoTransaction 62 | { et_txid :: {-# UNPACK #-} !HexString 63 | , et_size :: {-# UNPACK #-} !Int64 64 | , et_type :: !T.Text 65 | , et_vin :: !(V.Vector NeoTransactionInput) 66 | , et_vout :: !(V.Vector NeoTransactionOutput) 67 | , et_sys_fee :: !Scientific 68 | , et_net_fee :: !Scientific 69 | } 70 | 71 | newtype NeoTransactionWrapper = NeoTransactionWrapper 72 | { unwrapNeoTransaction :: NeoTransaction 73 | } 74 | 75 | instance J.FromJSON NeoTransactionWrapper where 76 | parseJSON = J.withObject "neo transaction" $ \fields -> fmap NeoTransactionWrapper $ NeoTransaction 77 | <$> (decode0xHexBytes =<< fields J..: "txid") 78 | <*> (fields J..: "size") 79 | <*> (fields J..: "type") 80 | <*> (V.map unwrapNeoTransactionInput <$> fields J..: "vin") 81 | <*> (V.map unwrapNeoTransactionOutput <$> fields J..: "vout") 82 | <*> (decodeReadStr =<< fields J..: "sys_fee") 83 | <*> (decodeReadStr =<< fields J..: "net_fee") 84 | 85 | data NeoTransactionInput = NeoTransactionInput 86 | { nti_txid :: {-# UNPACK #-} !HexString 87 | , nti_vout :: {-# UNPACK #-} !Int64 88 | } 89 | 90 | newtype NeoTransactionInputWrapper = NeoTransactionInputWrapper 91 | { unwrapNeoTransactionInput :: NeoTransactionInput 92 | } 93 | 94 | instance J.FromJSON NeoTransactionInputWrapper where 95 | parseJSON = J.withObject "neo transaction input" $ \fields -> fmap NeoTransactionInputWrapper $ NeoTransactionInput 96 | <$> (decode0xHexBytes =<< fields J..: "txid") 97 | <*> (fields J..: "vout") 98 | 99 | data NeoTransactionOutput = NeoTransactionOutput 100 | { nto_asset :: {-# UNPACK #-} !HexString 101 | , nto_value :: !Scientific 102 | , nto_address :: !T.Text 103 | } 104 | 105 | newtype NeoTransactionOutputWrapper = NeoTransactionOutputWrapper 106 | { unwrapNeoTransactionOutput :: NeoTransactionOutput 107 | } 108 | 109 | instance J.FromJSON NeoTransactionOutputWrapper where 110 | parseJSON = J.withObject "neo transaction output" $ \fields -> fmap NeoTransactionOutputWrapper $ NeoTransactionOutput 111 | <$> (decode0xHexBytes =<< fields J..: "asset") 112 | <*> (decodeReadStr =<< fields J..: "value") 113 | <*> (fields J..: "address") 114 | 115 | 116 | genSchemaInstances [''NeoBlock, ''NeoTransaction, ''NeoTransactionInput, ''NeoTransactionOutput] 117 | genFlattenedTypes "index" [| nb_index |] [("block", ''NeoBlock), ("transaction", ''NeoTransaction), ("input", ''NeoTransactionInput), ("output", ''NeoTransactionOutput)] 118 | 119 | instance BlockChain Neo where 120 | type Block Neo = NeoBlock 121 | 122 | getBlockChainInfo _ = BlockChainInfo 123 | { bci_init = \BlockChainParams 124 | { bcp_httpManager = httpManager 125 | , bcp_httpRequest = httpRequest 126 | } -> return $ Neo $ newJsonRpc httpManager httpRequest Nothing 127 | , bci_defaultApiUrls = ["http://127.0.0.1:10332/"] 128 | , bci_defaultBeginBlock = 0 129 | , bci_defaultEndBlock = -1000 -- very conservative rewrite limit 130 | , bci_heightFieldName = "index" 131 | , bci_schemas = standardBlockChainSchemas 132 | (schemaOf (Proxy :: Proxy NeoBlock)) 133 | [ schemaOf (Proxy :: Proxy NeoTransactionInput) 134 | , schemaOf (Proxy :: Proxy NeoTransactionOutput) 135 | , schemaOf (Proxy :: Proxy NeoTransaction) 136 | ] 137 | "CREATE TABLE \"neo\" OF \"NeoBlock\" (PRIMARY KEY (\"index\"));" 138 | , bci_flattenSuffixes = ["blocks", "transactions", "inputs", "outputs"] 139 | , bci_flattenPack = let 140 | f (blocks, (transactions, inputs, outputs)) = 141 | [ SomeBlocks (blocks :: [NeoBlock_flattened]) 142 | , SomeBlocks (transactions :: [NeoTransaction_flattened]) 143 | , SomeBlocks (inputs :: [NeoTransactionInput_flattened]) 144 | , SomeBlocks (outputs :: [NeoTransactionOutput_flattened]) 145 | ] 146 | in f . mconcat . map flatten 147 | } 148 | 149 | getBlockChainNodeInfo (Neo jsonRpc) = do 150 | (T.splitOn "/" -> [_, T.splitOn ":" -> [_, version], _]) <- either fail return . J.parseEither (J..: "useragent") =<< jsonRpcRequest jsonRpc "getversion" ([] :: V.Vector J.Value) 151 | return BlockChainNodeInfo 152 | { bcni_version = version 153 | } 154 | 155 | getCurrentBlockHeight (Neo jsonRpc) = (+ (-1)) <$> jsonRpcRequest jsonRpc "getblockcount" ([] :: V.Vector J.Value) 156 | 157 | getBlockByHeight (Neo jsonRpc) blockHeight = unwrapNeoBlock <$> jsonRpcRequest jsonRpc "getblock" ([J.Number $ fromIntegral blockHeight, J.Number 1] :: V.Vector J.Value) 158 | -------------------------------------------------------------------------------- /coinmetrics-neo/coinmetrics-neo.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-neo 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Neo 17 | build-depends: 18 | aeson 19 | , base 20 | , coinmetrics 21 | , hanalytics-base 22 | , scientific 23 | , text 24 | , time 25 | , vector 26 | ghc-options: -Wall 27 | default-language: Haskell2010 28 | -------------------------------------------------------------------------------- /coinmetrics-ripple/coinmetrics-ripple.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-ripple 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Ripple 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , http-client 24 | , iso8601-time 25 | , memory 26 | , scientific 27 | , text 28 | , time 29 | , unordered-containers 30 | , vector 31 | ghc-options: -Wall 32 | default-language: Haskell2010 33 | -------------------------------------------------------------------------------- /coinmetrics-rosetta/CoinMetrics/Rosetta.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies #-} 2 | 3 | module CoinMetrics.Rosetta 4 | ( Rosetta(..) 5 | , RosettaBlock(..) 6 | , RosettaTransaction(..) 7 | ) where 8 | 9 | import Control.Concurrent.STM 10 | import qualified Data.Aeson as J 11 | import qualified Data.ByteString as B 12 | import Data.Int 13 | import Data.Maybe 14 | import Data.Proxy 15 | import qualified Data.Text as T 16 | import Data.Time.Clock.POSIX 17 | import qualified Data.Vector as V 18 | import qualified Network.HTTP.Client as H 19 | 20 | import CoinMetrics.BlockChain 21 | import CoinMetrics.Schema.Util 22 | import CoinMetrics.Util 23 | import Hanalytics.Schema 24 | 25 | data Rosetta = Rosetta 26 | { rosetta_httpManager :: !H.Manager 27 | , rosetta_httpRequest :: !H.Request 28 | , rosetta_networkIdentifierVar :: !(TVar (Maybe RosettaNetworkIdentifier)) 29 | } 30 | 31 | rosettaRequest :: (J.ToJSON req, J.FromJSON res) => Rosetta -> B.ByteString -> req -> IO res 32 | rosettaRequest Rosetta 33 | { rosetta_httpManager = httpManager 34 | , rosetta_httpRequest = httpRequest 35 | } path request = do 36 | body <- H.responseBody <$> tryWithRepeat (H.httpLbs (httpRequest 37 | { H.path = path 38 | , H.requestBody = H.RequestBodyLBS $ J.encode request 39 | }) httpManager) 40 | either fail return $ J.eitherDecode body 41 | 42 | data RosettaBlock = RosettaBlock 43 | { rb_block_identifier :: {-# UNPACK #-} !RosettaBlockIdentifier 44 | , rb_parent_block_identifier :: {-# UNPACK #-} !RosettaBlockIdentifier 45 | , rb_timestamp :: {-# UNPACK #-} !Int64 46 | , rb_transactions :: !(V.Vector RosettaTransaction) 47 | } 48 | 49 | data RosettaBlockIdentifier = RosettaBlockIdentifier 50 | { rbi_index :: !(Maybe Int64) 51 | , rbi_hash :: !(Maybe HexString) 52 | } 53 | 54 | instance HasBlockHeader RosettaBlock where 55 | getBlockHeader RosettaBlock 56 | { rb_block_identifier = RosettaBlockIdentifier 57 | { rbi_index = maybeHeight 58 | , rbi_hash = maybeHash 59 | } 60 | , rb_parent_block_identifier = RosettaBlockIdentifier 61 | { rbi_hash = maybePrevHash 62 | } 63 | , rb_timestamp = time 64 | } = BlockHeader 65 | { bh_height = fromJust maybeHeight 66 | , bh_hash = fromJust maybeHash 67 | , bh_prevHash = maybePrevHash 68 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral time * 0.001 69 | } 70 | 71 | data RosettaTransaction = RosettaTransaction 72 | { rt_transaction_identifier :: {-# UNPACK #-} !RosettaTransactionIdentifier 73 | } 74 | 75 | data RosettaTransactionIdentifier = RosettaTransactionIdentifier 76 | { rti_hash :: {-# UNPACK #-} !HexString 77 | } 78 | 79 | data RosettaNetworkIdentifier = RosettaNetworkIdentifier 80 | { rni_blockchain :: !T.Text 81 | , rni_network :: !T.Text 82 | } 83 | 84 | data RosettaNetworkListResponse = RosettaNetworkListResponse 85 | { rnlr_network_identifiers :: !(V.Vector RosettaNetworkIdentifier) 86 | } 87 | 88 | data RosettaNetworkStatusRequest = RosettaNetworkStatusRequest 89 | { rnsr_network_identifier :: !RosettaNetworkIdentifier 90 | } 91 | 92 | data RosettaNetworkStatusResponse = RosettaNetworkStatusResponse 93 | { rnsr_current_block_identifier :: !RosettaBlockIdentifier 94 | } 95 | 96 | data RosettaBlockRequest = RosettaBlockRequest 97 | { rbr_network_identifier :: !RosettaNetworkIdentifier 98 | , rbr_block_identifier :: !RosettaBlockIdentifier 99 | } 100 | 101 | data RosettaBlockResponse = RosettaBlockResponse 102 | { rbr_block :: !RosettaBlock 103 | } 104 | 105 | genSchemaInstances 106 | [ ''RosettaBlock, ''RosettaTransaction 107 | , ''RosettaBlockIdentifier, ''RosettaTransactionIdentifier, ''RosettaNetworkIdentifier 108 | , ''RosettaNetworkListResponse 109 | , ''RosettaNetworkStatusRequest, ''RosettaNetworkStatusResponse 110 | , ''RosettaBlockRequest, ''RosettaBlockResponse 111 | ] 112 | 113 | instance BlockChain Rosetta where 114 | type Block Rosetta = RosettaBlock 115 | 116 | getBlockChainInfo _ = BlockChainInfo 117 | { bci_init = \BlockChainParams 118 | { bcp_httpManager = httpManager 119 | , bcp_httpRequest = httpRequest 120 | } -> do 121 | networkIdentifierVar <- newTVarIO Nothing 122 | return Rosetta 123 | { rosetta_httpManager = httpManager 124 | , rosetta_httpRequest = httpRequest 125 | { H.method = "POST" 126 | , H.requestHeaders = ("Content-Type", "application/json") : H.requestHeaders httpRequest 127 | } 128 | , rosetta_networkIdentifierVar = networkIdentifierVar 129 | } 130 | , bci_defaultApiUrls = ["http://127.0.0.1:8080/"] 131 | , bci_defaultBeginBlock = 0 132 | , bci_defaultEndBlock = -100 -- conservative rewrite limit 133 | , bci_heightFieldName = "height" 134 | , bci_schemas = standardBlockChainSchemas 135 | (schemaOf (Proxy :: Proxy RosettaBlock)) 136 | [ schemaOf (Proxy :: Proxy RosettaBlockIdentifier) 137 | , schemaOf (Proxy :: Proxy RosettaTransactionIdentifier) 138 | , schemaOf (Proxy :: Proxy RosettaTransaction) 139 | ] 140 | "CREATE TABLE \"rosetta\" OF \"RosettaBlock\" (PRIMARY KEY (\"height\"));" 141 | } 142 | 143 | getCurrentBlockHeight rosetta = do 144 | networkIdentifier <- getNetworkIdentifier rosetta 145 | RosettaNetworkStatusResponse 146 | { rnsr_current_block_identifier = RosettaBlockIdentifier 147 | { rbi_index = Just blockHeight 148 | } 149 | } <- rosettaRequest rosetta "/network/status" RosettaNetworkStatusRequest 150 | { rnsr_network_identifier = networkIdentifier 151 | } 152 | return blockHeight 153 | 154 | getBlockByHeight rosetta blockHeight = do 155 | networkIdentifier <- getNetworkIdentifier rosetta 156 | RosettaBlockResponse 157 | { rbr_block = block 158 | } <- rosettaRequest rosetta "/block" RosettaBlockRequest 159 | { rbr_network_identifier = networkIdentifier 160 | , rbr_block_identifier = RosettaBlockIdentifier 161 | { rbi_index = Just blockHeight 162 | , rbi_hash = Nothing 163 | } 164 | } 165 | return block 166 | 167 | getNetworkIdentifier :: Rosetta -> IO RosettaNetworkIdentifier 168 | getNetworkIdentifier rosetta@Rosetta 169 | { rosetta_networkIdentifierVar = networkIdentifierVar 170 | } = do 171 | maybeNetworkIdentifier <- readTVarIO networkIdentifierVar 172 | case maybeNetworkIdentifier of 173 | Just networkIdentifier -> return networkIdentifier 174 | Nothing -> do 175 | RosettaNetworkListResponse 176 | { rnlr_network_identifiers = [networkIdentifier] 177 | } <- rosettaRequest rosetta "/network/list" (J.Object []) 178 | atomically $ writeTVar networkIdentifierVar $ Just networkIdentifier 179 | return networkIdentifier 180 | -------------------------------------------------------------------------------- /coinmetrics-rosetta/coinmetrics-rosetta.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-rosetta 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Rosetta 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , http-client 24 | , stm 25 | , text 26 | , time 27 | , vector 28 | ghc-options: -Wall -Wno-tabs 29 | default-language: Haskell2010 30 | -------------------------------------------------------------------------------- /coinmetrics-stellar/coinmetrics-stellar.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 3.0 2 | name: coinmetrics-stellar 3 | version: 0.1.0.0 4 | -- synopsis: 5 | -- description: 6 | license: MIT 7 | author: Alexander Bich 8 | maintainer: quyse0@gmail.com 9 | copyright: (c) Coin Metrics, Inc. and contributors 10 | category: Cryptocurrency 11 | build-type: Simple 12 | -- extra-source-files: 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Stellar 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , cryptonite 23 | , data-default 24 | , hanalytics-base 25 | , http-client 26 | , memory 27 | , stm 28 | , text 29 | , time 30 | , unordered-containers 31 | , vector 32 | , zlib 33 | cxx-sources: 34 | coinmetrics-stellar.cpp 35 | extra-libraries: 36 | xdrpp 37 | stdc++ 38 | ghc-options: -Wall 39 | cxx-options: -Wall -O3 -std=c++17 40 | default-language: Haskell2010 41 | -------------------------------------------------------------------------------- /coinmetrics-stellar/coinmetrics-stellar.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | // Class for encoding to JSON. 6 | struct JsonArchive 7 | { 8 | std::ostream& stream; 9 | bool comma = false; 10 | 11 | JsonArchive(std::ostream& stream) : stream(stream) {} 12 | 13 | void printField(char const* field) 14 | { 15 | if(comma) stream << ','; 16 | else comma = true; 17 | if(field) escapeString(field) << ':'; 18 | } 19 | 20 | void operator()(char const* field, xdr::xdr_void) 21 | { 22 | printField(field); 23 | stream << "null"; 24 | } 25 | 26 | template 27 | void operator()(char const* field, xdr::xstring const& s) 28 | { 29 | printField(field); 30 | escapeString(s); 31 | } 32 | template 33 | void operator()(char const* field, xdr::opaque_array const& v) 34 | { 35 | printField(field); 36 | std::ostream s(stream.rdbuf()); 37 | s << '"'; 38 | for(size_t i = 0; i < v.size(); ++i) 39 | s << std::hex << std::setw(2) << std::setfill('0') << (int)v[i]; 40 | s << '"'; 41 | } 42 | template 43 | void operator()(char const* field, xdr::opaque_vec const& v) 44 | { 45 | printField(field); 46 | std::ostream s(stream.rdbuf()); 47 | s << '"'; 48 | for(size_t i = 0; i < v.size(); ++i) 49 | s << std::hex << std::setw(2) << std::setfill('0') << (int)v[i]; 50 | s << '"'; 51 | } 52 | 53 | template 54 | std::enable_if_t::is_enum> operator()(char const* field, T t) 55 | { 56 | printField(field); 57 | char const* name = xdr::xdr_traits::enum_name(t); 58 | if(name) 59 | escapeString(name); 60 | else 61 | escapeString(std::to_string(t)); 62 | } 63 | 64 | template 65 | std::enable_if_t::is_numeric> operator()(char const* field, T t) 66 | { 67 | printField(field); 68 | stream << std::to_string(t); 69 | } 70 | 71 | template 72 | void operator()(char const* field, std::tuple const& t) 73 | { 74 | xdr::archive(*this, std::get<0>(t), field); 75 | } 76 | 77 | template 78 | void operator()(char const* field, xdr::pointer const& t) 79 | { 80 | if(t) 81 | xdr::archive(*this, *t, field); 82 | else 83 | { 84 | printField(field); 85 | stream << "null"; 86 | } 87 | } 88 | 89 | template 90 | std::enable_if_t::is_class> operator()(char const* field, T const& t) 91 | { 92 | printField(field); 93 | stream << "{"; 94 | comma = false; 95 | xdr::xdr_traits::save(*this, t); 96 | stream << "}"; 97 | comma = true; 98 | } 99 | 100 | template 101 | std::enable_if_t::is_container> operator()(char const* field, T const& t) 102 | { 103 | printField(field); 104 | stream << "["; 105 | comma = false; 106 | for(auto const& o : t) 107 | xdr::archive(*this, o); 108 | stream << "]"; 109 | comma = true; 110 | } 111 | 112 | std::ostream& escapeString(std::string const& s) 113 | { 114 | stream << '\"'; 115 | for(size_t i = 0; i < s.size(); ++i) 116 | { 117 | char c = s[i]; 118 | switch(c) 119 | { 120 | case '"': 121 | stream << "\\\""; 122 | break; 123 | case '\\': 124 | stream << "\\\\"; 125 | break; 126 | case '\b': 127 | stream << "\\b"; 128 | break; 129 | case '\f': 130 | stream << "\\f"; 131 | break; 132 | case '\n': 133 | stream << "\\n"; 134 | break; 135 | case '\r': 136 | stream << "\\r"; 137 | break; 138 | case '\t': 139 | stream << "\\t"; 140 | break; 141 | default: 142 | if(c >= '\x00' && c <= '\x1f') 143 | stream << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)c; 144 | else 145 | stream << c; 146 | break; 147 | } 148 | } 149 | stream << '\"'; 150 | return stream; 151 | } 152 | }; 153 | 154 | namespace xdr 155 | { 156 | template<> 157 | struct archive_adapter 158 | { 159 | template 160 | static void apply(JsonArchive& a, T const& obj, char const* field) 161 | { 162 | a(field, obj); 163 | } 164 | }; 165 | } 166 | 167 | // StablePtr J.Value 168 | struct HsValue; 169 | // StablePtr HexString 170 | struct HsHash; 171 | // StablePtr TransactionHistoryEntry 172 | struct HsTransactionHistoryEntry; 173 | 174 | // Imported functions. 175 | extern "C" HsValue* coinmetrics_stellar_encode_json(char const* str, size_t len); 176 | extern "C" HsHash* coinmetrics_stellar_hash(char const* data, size_t len); 177 | extern "C" HsTransactionHistoryEntry* coinmetrics_stellar_combine_transaction_history_entry_with_hashes(HsValue* value, HsHash** hashes, size_t hashesCount); 178 | 179 | template 180 | HsValue* decode(uint8_t const*& begin, uint8_t const* end) 181 | { 182 | if(begin + 4 >= end) return nullptr; 183 | T t; 184 | { 185 | xdr::xdr_get getter(begin + 4, end); 186 | getter(t); 187 | begin = (uint8_t const*)getter.p_; 188 | } 189 | 190 | std::ostringstream s; 191 | JsonArchive a(s); 192 | xdr::archive(a, t); 193 | std::string json = s.str(); 194 | return coinmetrics_stellar_encode_json(json.data(), json.length()); 195 | } 196 | 197 | // Exported functions. 198 | extern "C" HsValue* coinmetrics_stellar_decode_ledger_history_entry(uint8_t const** begin, uint8_t const* end) 199 | { 200 | return decode(*begin, end); 201 | } 202 | 203 | extern "C" HsTransactionHistoryEntry* coinmetrics_stellar_decode_transaction_history_entry(uint8_t const** begin, uint8_t const* end, uint8_t const* networkIdData) 204 | { 205 | if(*begin + 4 >= end) return nullptr; 206 | 207 | stellar::TransactionHistoryEntry t; 208 | { 209 | xdr::xdr_get getter(*begin + 4, end); 210 | getter(t); 211 | *begin = (uint8_t const*)getter.p_; 212 | } 213 | 214 | stellar::Hash networkId; 215 | memcpy(networkId.data(), networkIdData, + 32); 216 | 217 | auto const& txs = t.txSet.txs; 218 | std::vector hashes; 219 | for(size_t i = 0; i < txs.size(); ++i) 220 | { 221 | xdr::msg_ptr msg; 222 | switch(txs[i].type()) 223 | { 224 | case stellar::ENVELOPE_TYPE_TX_V0: 225 | // yes, it's fucked up 226 | msg = xdr::xdr_to_msg(networkId, stellar::ENVELOPE_TYPE_TX, stellar::ENVELOPE_TYPE_TX_V0, txs[i].v0().tx); 227 | break; 228 | case stellar::ENVELOPE_TYPE_TX: 229 | { 230 | stellar::TransactionSignaturePayload payload; 231 | payload.networkId = networkId; 232 | payload.taggedTransaction.type(stellar::ENVELOPE_TYPE_TX).tx() = txs[i].v1().tx; 233 | msg = xdr::xdr_to_msg(payload); 234 | } 235 | break; 236 | case stellar::ENVELOPE_TYPE_TX_FEE_BUMP: 237 | { 238 | stellar::TransactionSignaturePayload payload; 239 | payload.networkId = networkId; 240 | payload.taggedTransaction.type(stellar::ENVELOPE_TYPE_TX_FEE_BUMP).feeBump() = txs[i].feeBump().tx; 241 | msg = xdr::xdr_to_msg(payload); 242 | } 243 | break; 244 | default: 245 | continue; 246 | } 247 | 248 | hashes.push_back(coinmetrics_stellar_hash(msg->data(), msg->size())); 249 | } 250 | 251 | std::ostringstream s; 252 | JsonArchive a(s); 253 | xdr::archive(a, t); 254 | std::string json = s.str(); 255 | return coinmetrics_stellar_combine_transaction_history_entry_with_hashes(coinmetrics_stellar_encode_json(json.data(), json.length()), &*hashes.begin(), hashes.size()); 256 | } 257 | 258 | extern "C" HsValue* coinmetrics_stellar_decode_result(uint8_t const** begin, uint8_t const* end) 259 | { 260 | return decode(*begin, end); 261 | } 262 | -------------------------------------------------------------------------------- /coinmetrics-stellar/default.nix: -------------------------------------------------------------------------------- 1 | { nixpkgs }: 2 | rec { 3 | xdrpp = with nixpkgs; stdenv.mkDerivation rec { 4 | name = "xdrpp"; 5 | 6 | src = builtins.fetchGit { 7 | url = "https://github.com/xdrpp/xdrpp.git"; 8 | }; 9 | 10 | nativeBuildInputs = [ autoreconfHook bison flex pandoc ]; 11 | 12 | enableParallelBuilding = true; 13 | }; 14 | 15 | stellar-core = builtins.fetchGit { 16 | url = "https://github.com/stellar/stellar-core.git"; 17 | }; 18 | 19 | xdr-headers = nixpkgs.runCommandCC "xdr-headers" {} '' 20 | set -e 21 | mkdir -p $out/include/xdr 22 | cd ${stellar-core}/src/xdr 23 | for i in *.x 24 | do 25 | ${xdrpp}/bin/xdrc -hh $i -o $out/include/xdr/''${i%.*}.h 26 | done 27 | ''; 28 | 29 | extraLibs = { 30 | inherit xdrpp; 31 | }; 32 | 33 | includes = [ xdr-headers ]; 34 | } 35 | -------------------------------------------------------------------------------- /coinmetrics-stellar/shell.nix: -------------------------------------------------------------------------------- 1 | { nixpkgs ? import {} }: 2 | let 3 | packages = import ./default.nix { inherit nixpkgs; }; 4 | in with nixpkgs; mkShell { 5 | buildInputs = with packages; [ xdrpp xdr-headers ]; 6 | } 7 | -------------------------------------------------------------------------------- /coinmetrics-storage/CoinMetrics/Export/Storage.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GADTs, RankNTypes #-} 2 | 3 | module CoinMetrics.Export.Storage 4 | ( ExportStorage(..) 5 | , SomeExportStorage(..) 6 | , ExportStorageOptions(..) 7 | , ExportStorageParams(..) 8 | , evaluatePack 9 | ) where 10 | 11 | import Control.Exception 12 | import qualified Data.Aeson as J 13 | import qualified Data.Avro as A 14 | import qualified Data.Text as T 15 | import qualified Network.HTTP.Client as H 16 | 17 | import CoinMetrics.BlockChain 18 | import Hanalytics.Schema 19 | import Hanalytics.Schema.Postgres 20 | 21 | class ExportStorage s where 22 | 23 | initExportStorage :: ExportStorageOptions a -> IO (s a) 24 | 25 | getExportStorageMaxBlock :: s a -> ExportStorageParams -> Maybe (IO (Maybe BlockHeight)) 26 | getExportStorageMaxBlock _ _ = Nothing 27 | 28 | writeExportStorage :: (Schemable a, A.ToAvro a, ToPostgresText a, J.ToJSON a) => s a -> ExportStorageParams -> [[a]] -> IO () 29 | writeExportStorage s params = writeExportStorageSomeBlocks s params . map (\pack -> [SomeBlocks pack]) 30 | 31 | writeExportStorageSomeBlocks :: s a -> ExportStorageParams -> [[SomeBlocks]] -> IO () 32 | writeExportStorageSomeBlocks _ _ _ = fail "this storage does not support flattened output" 33 | 34 | data SomeExportStorage a where 35 | SomeExportStorage :: ExportStorage s => s a -> SomeExportStorage a 36 | 37 | data ExportStorageOptions a = ExportStorageOptions 38 | { eso_httpManager :: !H.Manager 39 | , eso_tables :: [T.Text] 40 | , eso_primaryField :: !T.Text 41 | , eso_getPrimaryField :: !(a -> T.Text) 42 | , eso_upsert :: !Bool 43 | } 44 | 45 | data ExportStorageParams = ExportStorageParams 46 | { esp_destination :: !String 47 | -- | Function to wrap operations, for time measuring/etc 48 | , esp_wrapOperation :: !(forall a. IO a -> IO a) 49 | } 50 | 51 | -- | Return pack only when it's complete. Does not evaluate items. 52 | evaluatePack :: [a] -> IO [a] 53 | evaluatePack pack = evaluate $ f pack where 54 | f (_:xs) = f xs 55 | f [] = pack 56 | -------------------------------------------------------------------------------- /coinmetrics-storage/CoinMetrics/Export/Storage/AvroFile.hs: -------------------------------------------------------------------------------- 1 | module CoinMetrics.Export.Storage.AvroFile 2 | ( AvroFileExportStorage() 3 | ) where 4 | 5 | import Control.Monad 6 | import qualified Data.Avro as A 7 | import qualified Data.ByteString.Lazy as BL 8 | 9 | import CoinMetrics.Export.Storage 10 | 11 | newtype AvroFileExportStorage a = AvroFileExportStorage (ExportStorageOptions a) 12 | 13 | instance ExportStorage AvroFileExportStorage where 14 | initExportStorage = return . AvroFileExportStorage 15 | 16 | writeExportStorage AvroFileExportStorage {} ExportStorageParams 17 | { esp_destination = destination 18 | } = BL.writeFile destination <=< A.encodeContainer 19 | -------------------------------------------------------------------------------- /coinmetrics-storage/CoinMetrics/Export/Storage/Elastic.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists, OverloadedStrings, ViewPatterns #-} 2 | 3 | module CoinMetrics.Export.Storage.Elastic 4 | ( ElasticExportStorage() 5 | , ElasticFileExportStorage() 6 | ) where 7 | 8 | import Control.Monad 9 | import qualified Data.Aeson as J 10 | import qualified Data.Aeson.Types as J 11 | import qualified Data.ByteString.Lazy as BL 12 | import qualified Data.Text as T 13 | import qualified Data.Text.Encoding as T 14 | import qualified Network.HTTP.Client as H 15 | 16 | import CoinMetrics.Export.Storage 17 | 18 | newtype ElasticExportStorage a = ElasticExportStorage (ExportStorageOptions a) 19 | 20 | instance ExportStorage ElasticExportStorage where 21 | initExportStorage = return . ElasticExportStorage 22 | 23 | getExportStorageMaxBlock (ElasticExportStorage ExportStorageOptions 24 | { eso_httpManager = httpManager 25 | , eso_tables = (head -> table) 26 | , eso_primaryField = primaryField 27 | }) ExportStorageParams 28 | { esp_destination = destination 29 | , esp_wrapOperation = wrapOperation 30 | } = Just $ wrapOperation $ do 31 | httpRequest <- H.parseRequest destination 32 | response <- H.responseBody <$> H.httpLbs httpRequest 33 | { H.method = "GET" 34 | , H.requestHeaders = [("Content-Type", "application/json")] 35 | , H.path = "/" <> T.encodeUtf8 table <> "/_search" 36 | , H.requestBody = H.RequestBodyLBS $ J.encode $ J.Object 37 | [ ("size", J.Number 0) 38 | , ("track_total_hits", J.Bool False) 39 | , ("aggs", J.Object 40 | [ ("max_key", J.Object 41 | [ ("max", J.Object 42 | [ ("field", J.String primaryField) 43 | ]) 44 | ]) 45 | ]) 46 | ] 47 | } httpManager 48 | either fail return . J.parseEither ((J..:? "value") <=< (J..: "max_key") <=< (J..: "aggregations")) =<< either fail return (J.eitherDecode response) 49 | 50 | writeExportStorage (ElasticExportStorage options@ExportStorageOptions 51 | { eso_httpManager = httpManager 52 | , eso_upsert = upsert 53 | }) ExportStorageParams 54 | { esp_destination = destination 55 | , esp_wrapOperation = wrapOperation 56 | } packs = do 57 | httpRequest <- H.parseRequest destination 58 | let 59 | exportPack _ [] = return () 60 | exportPack i pack = if i < (3 :: Int) 61 | then do 62 | response <- H.responseBody <$> H.httpLbs httpRequest 63 | { H.method = "POST" 64 | , H.requestHeaders = [("Content-Type", "application/json")] 65 | , H.path = "/_bulk" 66 | , H.requestBody = H.RequestBodyLBS $ elasticExportStoragePack options pack 67 | } httpManager 68 | items <- either fail return . J.parseEither (J..: "items") =<< either fail return (J.eitherDecode response) 69 | packToRetry <- fmap concat . forM (zip pack items) $ \(block, item) -> do 70 | maybeResult <- either fail return $ J.parseEither ((J..:? "result") <=< (J..: (if upsert then "index" else "create"))) item 71 | return $ if (maybeResult :: Maybe T.Text) == Just "created" || (upsert && maybeResult == Just "updated") 72 | then [] 73 | else [block] 74 | exportPack (i + 1) packToRetry 75 | else fail "too many retries when exporting to ElasticSearch" 76 | mapM_ (wrapOperation . exportPack 0 <=< evaluatePack) packs 77 | 78 | newtype ElasticFileExportStorage a = ElasticFileExportStorage (ExportStorageOptions a) 79 | 80 | instance ExportStorage ElasticFileExportStorage where 81 | initExportStorage = return . ElasticFileExportStorage 82 | 83 | writeExportStorage (ElasticFileExportStorage options) ExportStorageParams 84 | { esp_destination = destination 85 | } = BL.writeFile destination . mconcat . map (elasticExportStoragePack options) 86 | 87 | elasticExportStoragePack :: J.ToJSON a => ExportStorageOptions a -> [a] -> BL.ByteString 88 | elasticExportStoragePack ExportStorageOptions 89 | { eso_tables = (head -> table) 90 | , eso_getPrimaryField = getPrimaryField 91 | , eso_upsert = upsert 92 | } = mconcat . map elasticLine 93 | where 94 | elasticLine block = let 95 | in J.encode (J.Object 96 | [ (if upsert then "index" else "create", J.Object 97 | [ ("_index", J.toJSON table) 98 | , ("_id", J.toJSON $ getPrimaryField block) 99 | ]) 100 | ]) <> "\n" <> J.encode block <> "\n" 101 | -------------------------------------------------------------------------------- /coinmetrics-storage/CoinMetrics/Export/Storage/Postgres.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, ViewPatterns #-} 2 | 3 | module CoinMetrics.Export.Storage.Postgres 4 | ( PostgresExportStorage() 5 | ) where 6 | 7 | import Control.Exception 8 | import Control.Monad 9 | import qualified Database.PostgreSQL.LibPQ as PQ 10 | import qualified Data.Text as T 11 | import qualified Data.Text.Encoding as T 12 | import qualified Data.Text.Lazy as TL 13 | import qualified Data.Text.Lazy.Builder as TL 14 | 15 | import CoinMetrics.Export.Storage 16 | import CoinMetrics.Export.Storage.PostgresFile 17 | 18 | newtype PostgresExportStorage a = PostgresExportStorage (ExportStorageOptions a) 19 | 20 | instance ExportStorage PostgresExportStorage where 21 | initExportStorage = return . PostgresExportStorage 22 | 23 | getExportStorageMaxBlock (PostgresExportStorage ExportStorageOptions 24 | { eso_tables = (head -> table) 25 | , eso_primaryField = primaryField 26 | }) ExportStorageParams 27 | { esp_destination = destination 28 | , esp_wrapOperation = wrapOperation 29 | } = Just $ wrapOperation $ withConnection destination $ \connection -> do 30 | let query = "SELECT MAX(\"" <> primaryField <> "\") FROM \"" <> table <> "\"" 31 | result <- maybe (fail "cannot get latest block from postgres") return =<< PQ.execParams connection (T.encodeUtf8 query) [] PQ.Text 32 | resultStatus <- PQ.resultStatus result 33 | unless (resultStatus == PQ.TuplesOk) $ fail $ "cannot get latest block from postgres: " <> show resultStatus 34 | tuplesCount <- PQ.ntuples result 35 | unless (tuplesCount == 1) $ fail "cannot decode tuples from postgres" 36 | maybeValue <- PQ.getvalue result 0 0 37 | return $ read . T.unpack . T.decodeUtf8 <$> maybeValue 38 | 39 | writeExportStorageSomeBlocks (PostgresExportStorage options) ExportStorageParams 40 | { esp_destination = destination 41 | , esp_wrapOperation = wrapOperation 42 | } = mapM_ (wrapOperation . exportPack <=< evaluatePack) where 43 | exportPack pack = withConnection destination $ \connection -> do 44 | let query = TL.toStrict $ TL.toLazyText $ postgresExportStorageSql options pack 45 | resultStatus <- maybe (return PQ.FatalError) PQ.resultStatus =<< PQ.exec connection (T.encodeUtf8 query) 46 | unless (resultStatus == PQ.CommandOk) $ fail $ "command failed: " <> show resultStatus 47 | 48 | withConnection :: String -> (PQ.Connection -> IO a) -> IO a 49 | withConnection destination = bracket (connectDestination destination) PQ.finish 50 | 51 | connectDestination :: String -> IO PQ.Connection 52 | connectDestination destination = do 53 | connection <- PQ.connectdb $ T.encodeUtf8 $ T.pack destination 54 | connectionStatus <- PQ.status connection 55 | unless (connectionStatus == PQ.ConnectionOk) $ fail $ "postgres connection failed: " <> show connectionStatus 56 | return connection 57 | -------------------------------------------------------------------------------- /coinmetrics-storage/CoinMetrics/Export/Storage/PostgresFile.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module CoinMetrics.Export.Storage.PostgresFile 4 | ( PostgresFileExportStorage() 5 | , postgresExportStorageSql 6 | ) where 7 | 8 | import qualified Data.ByteString.Lazy as BL 9 | import qualified Data.Text.Lazy.Builder as TL 10 | import qualified Data.Text.Lazy.Encoding as TL 11 | 12 | import CoinMetrics.BlockChain 13 | import CoinMetrics.Export.Storage 14 | import Hanalytics.Schema.Postgres 15 | 16 | newtype PostgresFileExportStorage a = PostgresFileExportStorage (ExportStorageOptions a) 17 | 18 | instance ExportStorage PostgresFileExportStorage where 19 | initExportStorage = return . PostgresFileExportStorage 20 | 21 | writeExportStorageSomeBlocks (PostgresFileExportStorage options) ExportStorageParams 22 | { esp_destination = destination 23 | } = BL.writeFile destination . TL.encodeUtf8 . TL.toLazyText . mconcat . map (postgresExportStorageSql options) 24 | 25 | postgresExportStorageSql :: ExportStorageOptions a -> [SomeBlocks] -> TL.Builder 26 | postgresExportStorageSql ExportStorageOptions 27 | { eso_tables = tables 28 | , eso_primaryField = primaryField 29 | , eso_upsert = upsert 30 | } = (if length tables > 1 then wrapWithTx else id) . mconcat . zipWith statementGroup tables 31 | where 32 | statementGroup table (SomeBlocks blocks) = if null blocks 33 | then mempty 34 | else (if upsert then postgresSqlUpsertGroup primaryField else postgresSqlInsertGroup) table blocks 35 | wrapWithTx s = "BEGIN;\n" <> s <> "COMMIT;\n" 36 | -------------------------------------------------------------------------------- /coinmetrics-storage/CoinMetrics/Export/Storage/RabbitMQ.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, RecordWildCards #-} 2 | 3 | module CoinMetrics.Export.Storage.RabbitMQ 4 | ( RabbitMQExportStorage() 5 | ) where 6 | 7 | import qualified Data.Aeson as J 8 | import qualified Network.AMQP as AMQP 9 | import qualified Data.Text as T 10 | import Control.Monad 11 | import Control.Exception 12 | import Data.Maybe 13 | 14 | import CoinMetrics.Export.Storage 15 | import CoinMetrics.BlockChain 16 | 17 | newtype RabbitMQExportStorage a = RabbitMQExportStorage (ExportStorageOptions a) 18 | 19 | 20 | instance ExportStorage RabbitMQExportStorage where 21 | initExportStorage = return . RabbitMQExportStorage 22 | writeExportStorageSomeBlocks (RabbitMQExportStorage ExportStorageOptions{..}) ExportStorageParams 23 | { esp_destination = destination 24 | , esp_wrapOperation = wrapOperation 25 | } blocks = do 26 | when (length queueExchange < 2) cantGetNames 27 | let queueName = queueExchange !! 0 28 | exchangeName = queueExchange !! 1 29 | mapM_ (handleBlock queueName exchangeName) blocks 30 | where 31 | handleBlock queueName exchangeName someBlocks = wrapOperation $ do 32 | let encoded = (\(SomeBlocks b) -> J.encode b) <$> someBlocks 33 | connect = connectToBroker connOpts 34 | mkMessage block = AMQP.newMsg { AMQP.msgBody = block, AMQP.msgDeliveryMode = Just AMQP.Persistent} 35 | close conn chan = AMQP.closeChannel chan >> AMQP.closeConnection conn 36 | send _ chan = do 37 | let queue = AMQP.newQueue {AMQP.queueName = queueName} 38 | exchange = AMQP.newExchange {AMQP.exchangeName = exchangeName, AMQP.exchangeType = "direct"} 39 | _ <- AMQP.declareQueue chan queue 40 | AMQP.bindQueue chan queueName exchangeName "export-block" 41 | AMQP.declareExchange chan exchange 42 | mapM_ (AMQP.publishMsg chan exchangeName "export-block" . mkMessage) encoded 43 | bracket connect (uncurry close) (uncurry send) 44 | cantGetNames = error "Can't get queue and exchange name" 45 | connOpts = parseConnectionOpts $ T.pack destination 46 | queueExchange = maybe cantGetNames (T.splitOn ":") (listToMaybe eso_tables) 47 | 48 | -- format is: amqp://user:pass@host:10000/vhost 49 | parseConnectionOpts :: T.Text -> AMQP.ConnectionOpts 50 | parseConnectionOpts connStr = AMQP.fromURI (T.unpack connStr) 51 | 52 | 53 | connectToBroker :: AMQP.ConnectionOpts -> IO (AMQP.Connection, AMQP.Channel) 54 | connectToBroker opts = do 55 | let openChannel conn = AMQP.openChannel conn >>= (\chan -> pure (conn, chan)) 56 | bracketOnError (AMQP.openConnection'' opts) AMQP.closeConnection openChannel 57 | -------------------------------------------------------------------------------- /coinmetrics-storage/coinmetrics-storage.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-storage 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Export.Storage 17 | CoinMetrics.Export.Storage.AvroFile 18 | CoinMetrics.Export.Storage.Elastic 19 | CoinMetrics.Export.Storage.Postgres 20 | CoinMetrics.Export.Storage.PostgresFile 21 | CoinMetrics.Export.Storage.RabbitMQ 22 | build-depends: 23 | aeson 24 | , avro 25 | , base 26 | , amqp 27 | , bytestring 28 | , coinmetrics 29 | , hanalytics-base 30 | , hanalytics-postgres 31 | , http-client 32 | , postgresql-libpq 33 | , text 34 | ghc-options: -Wall 35 | default-language: Haskell2010 36 | -------------------------------------------------------------------------------- /coinmetrics-tendermint/CoinMetrics/Tendermint.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module: CoinMetrics.Tendermint 3 | Description: Generic Tendermint structures useful for implementing derived blockchains, as well as Blockchain instance for generic non-decoded export. 4 | License: MIT 5 | -} 6 | 7 | {-# LANGUAGE DeriveGeneric, FlexibleInstances, LambdaCase, OverloadedLists, OverloadedStrings, ScopedTypeVariables, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 8 | 9 | module CoinMetrics.Tendermint 10 | ( Tendermint(..) 11 | , TendermintBlock(..) 12 | , TendermintBlockWrapper(..) 13 | , TendermintTx(..) 14 | ) where 15 | 16 | import Control.Monad 17 | import Control.Monad.Fail(MonadFail) 18 | import qualified Data.Aeson as J 19 | import qualified Data.Aeson.Types as J 20 | import qualified Data.Avro as A 21 | import Data.Int 22 | import Data.Maybe 23 | import Data.Proxy 24 | import Data.String 25 | import qualified Data.Text as T 26 | import Data.Time.Clock.POSIX 27 | import Data.Time.ISO8601 28 | import qualified Data.Vector as V 29 | import GHC.Generics(Generic) 30 | import Language.Haskell.TH 31 | import qualified Network.HTTP.Client as H 32 | 33 | import CoinMetrics.BlockChain 34 | import CoinMetrics.Schema.Util 35 | import CoinMetrics.Util 36 | import Hanalytics.Schema 37 | import Hanalytics.Schema.Postgres 38 | 39 | data Tendermint tx = Tendermint 40 | { tendermint_httpManager :: !H.Manager 41 | , tendermint_httpRequest :: !H.Request 42 | } 43 | 44 | data TendermintBlock tx = TendermintBlock 45 | { tb_height :: {-# UNPACK #-} !Int64 46 | , tb_hash :: {-# UNPACK #-} !HexString 47 | , tb_time :: {-# UNPACK #-} !Int64 48 | , tb_transactions :: !(V.Vector tx) 49 | , tb_header :: !J.Value 50 | } 51 | 52 | instance HasBlockHeader (TendermintBlock tx) where 53 | getBlockHeader TendermintBlock 54 | { tb_height = height 55 | , tb_hash = hash 56 | , tb_time = timestamp 57 | } = BlockHeader 58 | { bh_height = height 59 | , bh_hash = hash 60 | , bh_prevHash = Nothing 61 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timestamp * 0.001 62 | } 63 | 64 | newtype TendermintBlockWrapper tx = TendermintBlockWrapper 65 | { unwrapTendermintBlock :: TendermintBlock tx 66 | } 67 | 68 | instance TendermintTx tx => J.FromJSON (TendermintBlockWrapper tx) where 69 | parseJSON = J.withObject "tendermint block" $ \fields -> do 70 | blockMeta <- fields J..: "block_meta" 71 | blockMetaHeader <- blockMeta J..: "header" 72 | fmap TendermintBlockWrapper $ TendermintBlock 73 | <$> (read <$> blockMetaHeader J..: "height") 74 | <*> ((J..: "hash") =<< (blockMeta J..: "block_id")) 75 | <*> (maybe (fail "wrong time") (return . floor . (* 1000) . utcTimeToPOSIXSeconds) . parseISO8601 =<< blockMetaHeader J..: "time") 76 | <*> (mapM decodeTendermintTx . fromMaybe V.empty =<< (J..:? "txs") =<< (J..: "data") =<< (fields J..: "block")) 77 | <*> (return $ J.Object blockMetaHeader) 78 | 79 | data TendermintResult a = TendermintResult 80 | { tr_result :: !a 81 | } deriving Generic 82 | instance J.FromJSON a => J.FromJSON (TendermintResult a) where 83 | parseJSON = J.genericParseJSON schemaJsonOptions 84 | 85 | -- | Typeclass for Tendermint transaction types. 86 | class (SchemableField tx, A.ToAvro tx, ToPostgresText tx, J.ToJSON tx) => TendermintTx tx where 87 | -- | Decode tendermint transaction. 88 | decodeTendermintTx :: MonadFail m => T.Text -> m tx 89 | 90 | -- | Instance for Text, doing nothing. 91 | instance TendermintTx T.Text where 92 | decodeTendermintTx = return 93 | 94 | -- | Instances for tendermint block. 95 | do 96 | tx <- newName "tx" 97 | schemaInstancesCtxDecs [varT tx] [t| TendermintBlock $(varT tx) |] 98 | 99 | instance TendermintTx tx => BlockChain (Tendermint tx) where 100 | type Block (Tendermint tx) = TendermintBlock tx 101 | 102 | getBlockChainInfo _ = BlockChainInfo 103 | { bci_init = \BlockChainParams 104 | { bcp_httpManager = httpManager 105 | , bcp_httpRequest = httpRequest 106 | } -> return Tendermint 107 | { tendermint_httpManager = httpManager 108 | , tendermint_httpRequest = httpRequest 109 | } 110 | , bci_defaultApiUrls = ["http://127.0.0.1:26657/"] 111 | , bci_defaultBeginBlock = 1 112 | , bci_defaultEndBlock = 0 113 | , bci_heightFieldName = "height" 114 | , bci_schemas = standardBlockChainSchemas 115 | (schemaOf (Proxy :: Proxy (TendermintBlock tx))) 116 | [ 117 | ] 118 | "CREATE TABLE \"tendermint\" OF \"TendermintBlock\" (PRIMARY KEY (\"height\"));" 119 | } 120 | 121 | getBlockChainNodeInfo Tendermint 122 | { tendermint_httpManager = httpManager 123 | , tendermint_httpRequest = httpRequest 124 | } = do 125 | response <- tryWithRepeat $ H.httpLbs httpRequest 126 | { H.path = "/status" 127 | } httpManager 128 | result <- either fail (return . tr_result) $ J.eitherDecode' $ H.responseBody response 129 | version <- either fail return $ J.parseEither ((J..: "node_info") >=> (J..: "version")) result 130 | return BlockChainNodeInfo 131 | { bcni_version = version 132 | } 133 | 134 | getCurrentBlockHeight Tendermint 135 | { tendermint_httpManager = httpManager 136 | , tendermint_httpRequest = httpRequest 137 | } = do 138 | response <- tryWithRepeat $ H.httpLbs httpRequest 139 | { H.path = "/block" 140 | } httpManager 141 | either fail (return . tb_height . (id :: TendermintBlock tx -> TendermintBlock tx) . unwrapTendermintBlock . tr_result) $ J.eitherDecode' $ H.responseBody response 142 | 143 | getBlockByHeight Tendermint 144 | { tendermint_httpManager = httpManager 145 | , tendermint_httpRequest = httpRequest 146 | } blockHeight = do 147 | response <- tryWithRepeat $ H.httpLbs httpRequest 148 | { H.path = "/block?height=" <> fromString (show blockHeight) 149 | } httpManager 150 | either fail (return . unwrapTendermintBlock . tr_result) $ J.eitherDecode' $ H.responseBody response 151 | -------------------------------------------------------------------------------- /coinmetrics-tendermint/CoinMetrics/Tendermint/Amino.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE LambdaCase, OverloadedLists #-} 2 | 3 | module CoinMetrics.Tendermint.Amino 4 | ( ReadMsg(..) 5 | , withFields 6 | , readSubStruct 7 | , readRepeated 8 | , readRequired 9 | , readOptional 10 | , withPrefix 11 | , withLen 12 | ) where 13 | 14 | import Control.Applicative 15 | import Control.Monad 16 | import qualified Data.ByteString.Short as BS 17 | import qualified Data.HashMap.Strict as HM 18 | import qualified Data.ProtocolBuffers.Internal as P 19 | import qualified Data.Serialize as S 20 | import qualified Data.Vector as V 21 | import Data.Word 22 | 23 | import CoinMetrics.Util 24 | 25 | class ReadMsg a where 26 | readMsg :: S.Get a 27 | 28 | withFields :: String -> (HM.HashMap P.Tag [P.WireField] -> S.Get a) -> S.Get a 29 | withFields _name g = g . HM.map reverse =<< step HM.empty where 30 | step h = do 31 | f <- Just <$> P.getWireField <|> return Nothing 32 | case f of 33 | Just v -> step $ HM.insertWith (\[x] xs -> x : xs) (P.wireFieldTag v) [v] h 34 | Nothing -> return h 35 | 36 | readSubStruct :: ReadMsg a => P.WireField -> S.Get a 37 | readSubStruct = \case 38 | P.DelimitedField _ bytes -> either fail return $ S.runGet readMsg bytes 39 | f -> fail $ show ("wrong field", f) 40 | 41 | readRepeated :: (P.WireField -> S.Get a) -> P.Tag -> HM.HashMap P.Tag [P.WireField] -> S.Get (V.Vector a) 42 | readRepeated f t h = mapM f $ maybe [] V.fromList $ HM.lookup t h 43 | 44 | readRequired :: (P.WireField -> S.Get a) -> P.Tag -> HM.HashMap P.Tag [P.WireField] -> S.Get a 45 | readRequired f t h = case HM.lookup t h of 46 | Just [a] -> f a 47 | _ -> fail $ show ("can't read required value", t, h) 48 | 49 | readOptional :: (P.WireField -> S.Get a) -> P.Tag -> HM.HashMap P.Tag [P.WireField] -> S.Get (Maybe a) 50 | readOptional f t h = case HM.lookup t h of 51 | Just [a] -> Just <$> f a 52 | Nothing -> return Nothing 53 | _ -> fail $ show ("can't read optional value", t, h) 54 | 55 | withPrefix :: Word32 -> S.Get a -> S.Get a 56 | withPrefix prefix get = do 57 | realPrefix <- S.getWord32be 58 | unless (prefix == realPrefix) $ fail $ show ("wrong prefix", prefix, realPrefix) 59 | get 60 | 61 | withLen :: S.Get a -> S.Get a 62 | withLen get = do 63 | len <- P.getVarInt 64 | S.isolate len get 65 | 66 | instance P.DecodeWire HexString where 67 | decodeWire = fmap (HexString . BS.toShort) . P.decodeWire 68 | -------------------------------------------------------------------------------- /coinmetrics-tendermint/coinmetrics-tendermint.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-tendermint 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Tendermint 17 | CoinMetrics.Tendermint.Amino 18 | build-depends: 19 | aeson 20 | , avro 21 | , base 22 | , bytestring 23 | , cereal 24 | , coinmetrics 25 | , hanalytics-base 26 | , hanalytics-postgres 27 | , http-client 28 | , iso8601-time 29 | , protobuf 30 | , template-haskell 31 | , text 32 | , time 33 | , unordered-containers 34 | , vector 35 | ghc-options: -Wall 36 | default-language: Haskell2010 37 | -------------------------------------------------------------------------------- /coinmetrics-tezos/CoinMetrics/Tezos.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Tezos 4 | ( Tezos(..) 5 | ) where 6 | 7 | import Control.Monad 8 | import qualified Data.Aeson as J 9 | import qualified Data.Aeson.Types as J 10 | import qualified Data.ByteString as B 11 | import qualified Data.ByteString.Short as BS 12 | import Data.Int 13 | import Data.Maybe 14 | import Data.Proxy 15 | import qualified Data.Text as T 16 | import qualified Data.Text.Encoding as T 17 | import qualified Data.Time.Clock.POSIX as Time 18 | import qualified Data.Time.ISO8601 as Time 19 | import qualified Data.Vector as V 20 | import qualified Network.HTTP.Client as H 21 | 22 | import CoinMetrics.BlockChain 23 | import CoinMetrics.Schema.Util 24 | import CoinMetrics.Util 25 | import Hanalytics.Schema 26 | 27 | 28 | data Tezos = Tezos 29 | { tezos_httpManager :: !H.Manager 30 | , tezos_httpRequest :: !H.Request 31 | } 32 | 33 | data TezosBlock = TezosBlock 34 | { tb_level :: {-# UNPACK #-} !Int64 35 | , tb_hash :: !T.Text 36 | , tb_timestamp :: {-# UNPACK #-} !Int64 37 | , tb_baker :: !(Maybe T.Text) 38 | , tb_operations :: !(V.Vector TezosOperation) 39 | , tb_balanceUpdates :: !(V.Vector TezosBalanceUpdate) 40 | } 41 | 42 | instance HasBlockHeader TezosBlock where 43 | getBlockHeader TezosBlock 44 | { tb_level = level 45 | , tb_hash = hash 46 | , tb_timestamp = timestamp 47 | } = BlockHeader 48 | { bh_height = level 49 | , bh_hash = HexString $ BS.toShort $ T.encodeUtf8 hash 50 | , bh_prevHash = Nothing 51 | , bh_timestamp = Time.posixSecondsToUTCTime $ fromIntegral timestamp 52 | } 53 | 54 | newtype TezosBlockWrapper = TezosBlockWrapper 55 | { unwrapTezosBlock :: TezosBlock 56 | } 57 | 58 | instance J.FromJSON TezosBlockWrapper where 59 | parseJSON = J.withObject "tezos block" $ \fields -> do 60 | headerData <- fields J..: "header" 61 | metadata <- fields J..: "metadata" 62 | fmap TezosBlockWrapper $ TezosBlock 63 | <$> (headerData J..: "level") 64 | <*> (fields J..: "hash") 65 | <*> (decodeDate =<< headerData J..: "timestamp") 66 | <*> (metadata J..:? "baker") 67 | <*> (V.map unwrapTezosOperation . V.concat <$> fields J..: "operations") 68 | <*> (V.map unwrapTezosBalanceUpdate . fromMaybe V.empty <$> metadata J..:? "balance_updates") 69 | 70 | data TezosOperation = TezosOperation 71 | { to_hash :: !T.Text 72 | , to_items :: !(V.Vector TezosOperationItem) 73 | } 74 | 75 | newtype TezosOperationWrapper = TezosOperationWrapper 76 | { unwrapTezosOperation :: TezosOperation 77 | } 78 | 79 | instance J.FromJSON TezosOperationWrapper where 80 | parseJSON = J.withObject "tezos operation" $ \fields -> do 81 | fmap TezosOperationWrapper $ TezosOperation 82 | <$> (fields J..: "hash") 83 | <*> (V.map unwrapTezosOperationItem <$> fields J..: "contents") 84 | 85 | data TezosOperationItem = TezosOperationItem 86 | { toi_kind :: !T.Text 87 | , toi_level :: !(Maybe Int64) 88 | , toi_source :: !(Maybe T.Text) 89 | , toi_fee :: !(Maybe Int64) 90 | , toi_counter :: !(Maybe Int64) 91 | , toi_gasLimit :: !(Maybe Int64) 92 | , toi_storageLimit :: !(Maybe Int64) 93 | , toi_amount :: !(Maybe Int64) 94 | , toi_destination :: !(Maybe T.Text) 95 | , toi_publicKey :: !(Maybe T.Text) 96 | , toi_managerPubkey :: !(Maybe T.Text) 97 | , toi_balance :: !(Maybe Int64) 98 | , toi_balanceUpdates :: !(V.Vector TezosBalanceUpdate) 99 | , toi_resultStatus :: !(Maybe T.Text) 100 | , toi_resultBalanceUpdates :: !(V.Vector TezosBalanceUpdate) 101 | , toi_resultConsumedGas :: !(Maybe Int64) 102 | , toi_raw :: !J.Value 103 | } 104 | 105 | newtype TezosOperationItemWrapper = TezosOperationItemWrapper 106 | { unwrapTezosOperationItem :: TezosOperationItem 107 | } 108 | 109 | instance J.FromJSON TezosOperationItemWrapper where 110 | parseJSON = J.withObject "tezos operation item" $ \fields -> do 111 | metadata <- fields J..: "metadata" 112 | maybeResult <- metadata J..:? "operation_result" 113 | fmap TezosOperationItemWrapper $ TezosOperationItem 114 | <$> (fields J..: "kind") 115 | <*> (fields J..:? "level") 116 | <*> (fields J..:? "source") 117 | <*> (traverse decodeReadStr =<< fields J..:? "fee") 118 | <*> (traverse decodeReadStr =<< fields J..:? "counter") 119 | <*> (traverse decodeReadStr =<< fields J..:? "gas_limit") 120 | <*> (traverse decodeReadStr =<< fields J..:? "storage_limit") 121 | <*> (traverse decodeReadStr =<< fields J..:? "amount") 122 | <*> (fields J..:? "destination") 123 | <*> (fields J..:? "public_key") 124 | <*> (fields J..:? "managerPubkey") 125 | <*> (traverse decodeReadStr =<< fields J..:? "balance") 126 | <*> (V.map unwrapTezosBalanceUpdate . fromMaybe V.empty <$> metadata J..:? "balance_updates") 127 | <*> (traverse (J..: "status") maybeResult) 128 | <*> (V.map unwrapTezosBalanceUpdate . fromMaybe V.empty . join <$> traverse (J..:? "balance_updates") maybeResult) 129 | <*> (traverse decodeReadStr . join =<< traverse (J..:? "consumed_gas") maybeResult) 130 | <*> (return $ J.Object fields) 131 | 132 | data TezosBalanceUpdate = TezosBalanceUpdate 133 | { tbu_kind :: !T.Text 134 | , tbu_change :: {-# UNPACK #-} !Int64 135 | , tbu_contract :: !(Maybe T.Text) 136 | , tbu_category :: !(Maybe T.Text) 137 | , tbu_delegate :: !(Maybe T.Text) 138 | , tbu_level :: !(Maybe Int64) 139 | } 140 | 141 | newtype TezosBalanceUpdateWrapper = TezosBalanceUpdateWrapper 142 | { unwrapTezosBalanceUpdate :: TezosBalanceUpdate 143 | } 144 | 145 | instance J.FromJSON TezosBalanceUpdateWrapper where 146 | parseJSON = J.withObject "tezos balance update" $ \fields -> do 147 | fmap TezosBalanceUpdateWrapper $ TezosBalanceUpdate 148 | <$> (fields J..: "kind") 149 | <*> (decodeReadStr =<< fields J..: "change") 150 | <*> (fields J..:? "contract") 151 | <*> (fields J..:? "category") 152 | <*> (fields J..:? "delegate") 153 | <*> (fields J..:? "level") 154 | 155 | decodeDate :: T.Text -> J.Parser Int64 156 | decodeDate (T.unpack -> t) = case Time.parseISO8601 t of 157 | Just date -> return $ floor $ Time.utcTimeToPOSIXSeconds date 158 | Nothing -> fail $ "wrong date: " <> t 159 | 160 | genSchemaInstances [''TezosBlock, ''TezosOperation, ''TezosOperationItem, ''TezosBalanceUpdate] 161 | 162 | 163 | instance BlockChain Tezos where 164 | type Block Tezos = TezosBlock 165 | 166 | getBlockChainInfo _ = BlockChainInfo 167 | { bci_init = \BlockChainParams 168 | { bcp_httpManager = httpManager 169 | , bcp_httpRequest = httpRequest 170 | } -> return Tezos 171 | { tezos_httpManager = httpManager 172 | , tezos_httpRequest = httpRequest 173 | } 174 | , bci_defaultApiUrls = ["http://127.0.0.1:8732/"] 175 | , bci_defaultBeginBlock = 0 176 | , bci_defaultEndBlock = 0 177 | , bci_heightFieldName = "level" 178 | , bci_schemas = standardBlockChainSchemas 179 | (schemaOf (Proxy :: Proxy TezosBlock)) 180 | [ schemaOf (Proxy :: Proxy TezosBalanceUpdate) 181 | , schemaOf (Proxy :: Proxy TezosOperationItem) 182 | , schemaOf (Proxy :: Proxy TezosOperation) 183 | ] 184 | "CREATE TABLE \"tezos\" OF \"TezosBlock\" (PRIMARY KEY (\"level\"));" 185 | } 186 | 187 | getCurrentBlockHeight Tezos 188 | { tezos_httpManager = httpManager 189 | , tezos_httpRequest = httpRequest 190 | } = 191 | tryWithRepeat $ either fail (return . tb_level . unwrapTezosBlock) . J.eitherDecode . H.responseBody =<< H.httpLbs httpRequest 192 | { H.path = "/chains/" <> mainnetChain <> "/blocks/head" 193 | } httpManager 194 | 195 | getBlockByHeight Tezos 196 | { tezos_httpManager = httpManager 197 | , tezos_httpRequest = httpRequest 198 | } blockHeight = do 199 | TezosBlockWrapper TezosBlock 200 | { tb_level = headBlockLevel 201 | , tb_hash = headBlockHash 202 | } <- tryWithRepeat $ either fail return . J.eitherDecode . H.responseBody =<< H.httpLbs httpRequest 203 | { H.path = "/chains/" <> mainnetChain <> "/blocks/head" 204 | } httpManager 205 | tryWithRepeat $ either fail (return . unwrapTezosBlock) . J.eitherDecode . H.responseBody =<< H.httpLbs httpRequest 206 | { H.path = "/chains/" <> mainnetChain <> "/blocks/" <> T.encodeUtf8 headBlockHash <> "~" <> (T.encodeUtf8 $ T.pack $ show $ headBlockLevel - blockHeight) 207 | } httpManager 208 | 209 | mainnetChain :: B.ByteString 210 | mainnetChain = "NetXdQprcVkpaWU" 211 | -------------------------------------------------------------------------------- /coinmetrics-tezos/coinmetrics-tezos.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-tezos 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Tezos 17 | build-depends: 18 | aeson 19 | , base 20 | , bytestring 21 | , coinmetrics 22 | , hanalytics-base 23 | , http-client 24 | , iso8601-time 25 | , text 26 | , time 27 | , vector 28 | ghc-options: -Wall 29 | default-language: Haskell2010 30 | -------------------------------------------------------------------------------- /coinmetrics-tron/CoinMetrics/Tron.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, LambdaCase, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Tron 4 | ( Tron(..) 5 | , TronBlock(..) 6 | , TronTransaction(..) 7 | , TronContract(..) 8 | , TronVote(..) 9 | ) where 10 | 11 | import Control.Monad 12 | import qualified Data.Aeson as J 13 | import qualified Data.Aeson.Types as J 14 | import Data.Int 15 | import Data.Maybe 16 | import Data.Proxy 17 | import qualified Data.Text as T 18 | import qualified Data.Text.Lazy.Encoding as TE 19 | import Data.Time.Clock.POSIX 20 | import qualified Data.Vector as V 21 | import qualified Network.HTTP.Client as H 22 | 23 | import CoinMetrics.BlockChain 24 | import CoinMetrics.Schema.Flatten 25 | import CoinMetrics.Schema.Util 26 | import CoinMetrics.Util 27 | import Hanalytics.Schema 28 | 29 | data Tron = Tron 30 | { tron_httpManager :: !H.Manager 31 | , tron_httpRequest :: !H.Request 32 | } 33 | 34 | data TronBlock = TronBlock 35 | { tb_hash :: {-# UNPACK #-} !HexString 36 | , tb_timestamp :: {-# UNPACK #-} !Int64 37 | , tb_number :: {-# UNPACK #-} !Int64 38 | , tb_witness_address :: {-# UNPACK #-} !HexString 39 | , tb_transactions :: !(V.Vector TronTransaction) 40 | } 41 | 42 | instance HasBlockHeader TronBlock where 43 | getBlockHeader TronBlock 44 | { tb_hash = hash 45 | , tb_timestamp = timestamp 46 | , tb_number = number 47 | } = BlockHeader 48 | { bh_height = number 49 | , bh_hash = hash 50 | , bh_prevHash = Nothing 51 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timestamp * 0.001 52 | } 53 | 54 | newtype TronBlockWrapper = TronBlockWrapper 55 | { unwrapTronBlock :: TronBlock 56 | } 57 | 58 | instance J.FromJSON TronBlockWrapper where 59 | parseJSON = J.withObject "tron block" $ \fields -> do 60 | headerData <- (J..: "raw_data") =<< fields J..: "block_header" 61 | fmap TronBlockWrapper $ TronBlock 62 | <$> (fields J..: "blockID") 63 | <*> (fromMaybe 0 <$> headerData J..:? "timestamp") 64 | <*> (fromMaybe 0 <$> headerData J..:? "number") 65 | <*> (headerData J..: "witness_address") 66 | <*> (V.map unwrapTronTransaction . fromMaybe mempty <$> fields J..:? "transactions") 67 | 68 | data TronTransaction = TronTransaction 69 | { tt_hash :: {-# UNPACK #-} !HexString 70 | , tt_ref_block_bytes :: !(Maybe HexString) 71 | , tt_ref_block_num :: !(Maybe Int64) 72 | , tt_ref_block_hash :: !(Maybe HexString) 73 | , tt_expiration :: !(Maybe Int64) 74 | , tt_timestamp :: !(Maybe Int64) 75 | , tt_contracts :: !(V.Vector TronContract) 76 | , tt_raw :: !J.Value 77 | , tt_raw_info :: !J.Value 78 | } 79 | 80 | newtype TronTransactionWrapper = TronTransactionWrapper 81 | { unwrapTronTransaction :: TronTransaction 82 | } 83 | 84 | instance J.FromJSON TronTransactionWrapper where 85 | parseJSON = J.withObject "tron transaction" $ \fields -> do 86 | rawData <- fields J..: "raw_data" 87 | fmap TronTransactionWrapper $ TronTransaction 88 | <$> (fields J..: "txID") 89 | <*> (rawData J..:? "ref_block_bytes") 90 | <*> (rawData J..:? "ref_block_num") 91 | <*> (rawData J..:? "ref_block_hash") 92 | <*> (rawData J..:? "expiration") 93 | <*> (rawData J..:? "timestamp") 94 | <*> (V.map unwrapTronContract <$> rawData J..: "contract") 95 | <*> (return $ J.Object rawData) 96 | <*> (return J.Null) 97 | 98 | {- 99 | Fields noted for: 100 | AccountUpdateContract 101 | FreezeBalanceContract 102 | ParticipateAssetIssueContract 103 | TransferAssetContract 104 | TransferContract 105 | UnfreezeBalanceContract 106 | VoteWitnessContract 107 | WithdrawBalanceContract 108 | -} 109 | data TronContract = TronContract 110 | { tc_type :: !T.Text 111 | , tc_amount :: !(Maybe Int64) 112 | , tc_account_name :: !(Maybe HexString) 113 | , tc_asset_name :: !(Maybe HexString) 114 | , tc_owner_address :: !(Maybe HexString) 115 | , tc_to_address :: !(Maybe HexString) 116 | , tc_frozen_duration :: !(Maybe Int64) 117 | , tc_frozen_balance :: !(Maybe Int64) 118 | , tc_votes :: !(V.Vector TronVote) 119 | } 120 | 121 | newtype TronContractWrapper = TronContractWrapper 122 | { unwrapTronContract :: TronContract 123 | } 124 | 125 | instance J.FromJSON TronContractWrapper where 126 | parseJSON = J.withObject "tron contract" $ \fields -> do 127 | maybeValue <- (J..:? "value") =<< fields J..: "parameter" 128 | let 129 | getValue :: J.FromJSON a => T.Text -> J.Parser (Maybe a) 130 | getValue = maybe (const (return Nothing)) (J..:?) maybeValue 131 | fmap TronContractWrapper $ TronContract 132 | <$> (fields J..: "type") 133 | <*> (getValue "amount") 134 | <*> (getValue "account_name") 135 | <*> (getValue "asset_name") 136 | <*> (getValue "owner_address") 137 | <*> (getValue "to_address") 138 | <*> (getValue "frozen_duration") 139 | <*> (getValue "frozen_balance") 140 | <*> (V.map unwrapTronVote . fromMaybe mempty <$> getValue "votes") 141 | 142 | data TronVote = TronVote 143 | { tv_address :: {-# UNPACK #-} !HexString 144 | , tv_count :: {-# UNPACK #-} !Int64 145 | } 146 | 147 | newtype TronVoteWrapper = TronVoteWrapper 148 | { unwrapTronVote :: TronVote 149 | } 150 | 151 | instance J.FromJSON TronVoteWrapper where 152 | parseJSON = J.withObject "tron vote" $ \fields -> fmap TronVoteWrapper $ TronVote 153 | <$> (fields J..: "vote_address") 154 | <*> (fields J..: "vote_count") 155 | 156 | genSchemaInstances [''TronBlock, ''TronTransaction, ''TronContract, ''TronVote] 157 | genFlattenedTypes "number" [| tb_number |] [("block", ''TronBlock), ("transaction", ''TronTransaction), ("contract", ''TronContract), ("vote", ''TronVote)] 158 | 159 | instance BlockChain Tron where 160 | type Block Tron = TronBlock 161 | 162 | getBlockChainInfo _ = BlockChainInfo 163 | { bci_init = \BlockChainParams 164 | { bcp_httpManager = httpManager 165 | , bcp_httpRequest = httpRequest 166 | } -> return Tron 167 | { tron_httpManager = httpManager 168 | , tron_httpRequest = httpRequest 169 | } 170 | , bci_defaultApiUrls = ["http://127.0.0.1:8091/"] 171 | , bci_defaultBeginBlock = 0 172 | , bci_defaultEndBlock = 0 -- no need in gap with solidity node 173 | , bci_heightFieldName = "number" 174 | , bci_schemas = standardBlockChainSchemas 175 | (schemaOf (Proxy :: Proxy TronBlock)) 176 | [ schemaOf (Proxy :: Proxy TronVote) 177 | , schemaOf (Proxy :: Proxy TronContract) 178 | , schemaOf (Proxy :: Proxy TronTransaction) 179 | ] 180 | "CREATE TABLE \"tron\" OF \"TronBlock\" (PRIMARY KEY (\"number\"));" 181 | , bci_flattenSuffixes = ["blocks", "transactions", "logs", "actions", "uncles"] 182 | , bci_flattenPack = let 183 | f (blocks, (transactions, (contracts, votes))) = 184 | [ SomeBlocks (blocks :: [TronBlock_flattened]) 185 | , SomeBlocks (transactions :: [TronTransaction_flattened]) 186 | , SomeBlocks (contracts :: [TronContract_flattened]) 187 | , SomeBlocks (votes :: [TronVote_flattened]) 188 | ] 189 | in f . mconcat . map flatten 190 | } 191 | 192 | getBlockChainNodeInfo Tron 193 | { tron_httpManager = httpManager 194 | , tron_httpRequest = httpRequest 195 | } = do 196 | response <- tryWithRepeat $ H.httpLbs httpRequest 197 | { H.path = "/wallet/getnodeinfo" 198 | } httpManager 199 | version <- either fail return $ J.parseEither ((J..: "codeVersion") <=< (J..: "configNodeInfo")) =<< J.eitherDecode' (H.responseBody response) 200 | return BlockChainNodeInfo 201 | { bcni_version = version 202 | } 203 | 204 | getCurrentBlockHeight Tron 205 | { tron_httpManager = httpManager 206 | , tron_httpRequest = httpRequest 207 | } = do 208 | response <- tryWithRepeat $ H.httpLbs httpRequest 209 | { H.path = "/walletsolidity/getnowblock" 210 | } httpManager 211 | either fail return $ J.parseEither ((J..: "number") <=< (J..: "raw_data") <=< (J..: "block_header")) =<< J.eitherDecode' (H.responseBody response) 212 | 213 | getBlockByHeight Tron 214 | { tron_httpManager = httpManager 215 | , tron_httpRequest = httpRequest 216 | } blockHeight = do 217 | response <- tryWithRepeat $ H.httpLbs httpRequest 218 | { H.path = "/walletsolidity/getblockbynum" 219 | , H.requestBody = H.RequestBodyLBS $ J.encode $ J.Object 220 | [ ("num", J.Number $ fromIntegral blockHeight) 221 | ] 222 | , H.method = "POST" 223 | } httpManager 224 | block <- either fail (return . unwrapTronBlock) . J.eitherDecode' . TE.encodeUtf8 . TE.decodeLatin1 . H.responseBody $ response 225 | transactions <- forM (tb_transactions block) $ \transaction@TronTransaction 226 | { tt_hash = txid 227 | } -> do 228 | txRawInfoResponse <- tryWithRepeat $ H.httpLbs httpRequest 229 | { H.path = "/walletsolidity/gettransactioninfobyid" 230 | , H.requestBody = H.RequestBodyLBS $ J.encode $ J.Object 231 | [ ("value", J.toJSON txid) 232 | ] 233 | , H.method = "POST" 234 | } httpManager 235 | txRawInfo <- either fail return $ J.eitherDecode' (H.responseBody txRawInfoResponse) 236 | when (txRawInfo == J.Object [] || txRawInfo == J.Null) $ fail "bad tx raw info" 237 | return transaction 238 | { tt_raw_info = txRawInfo 239 | } 240 | return block 241 | { tb_transactions = transactions 242 | } 243 | -------------------------------------------------------------------------------- /coinmetrics-tron/coinmetrics-tron.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-tron 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Tron 17 | build-depends: 18 | aeson 19 | , base 20 | , coinmetrics 21 | , hanalytics-base 22 | , http-client 23 | , text 24 | , time 25 | , vector 26 | ghc-options: -Wall 27 | default-language: Haskell2010 28 | -------------------------------------------------------------------------------- /coinmetrics-waves/CoinMetrics/Waves.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveGeneric, LambdaCase, OverloadedLists, OverloadedStrings, StandaloneDeriving, TemplateHaskell, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Waves 4 | ( Waves(..) 5 | , WavesBlock(..) 6 | , WavesTransaction(..) 7 | , WavesOrder(..) 8 | , WavesTransfer(..) 9 | ) where 10 | 11 | import qualified Data.Aeson as J 12 | import qualified Data.Aeson.Types as J 13 | import Data.Int 14 | import Data.Maybe 15 | import Data.Proxy 16 | import Data.String 17 | import qualified Data.Text as T 18 | import Data.Time.Clock.POSIX 19 | import qualified Data.Vector as V 20 | import qualified Network.HTTP.Client as H 21 | 22 | import CoinMetrics.BlockChain 23 | import CoinMetrics.Schema.Util 24 | import CoinMetrics.Util 25 | import Hanalytics.Schema 26 | 27 | data Waves = Waves 28 | { waves_httpManager :: !H.Manager 29 | , waves_httpRequest :: !H.Request 30 | } 31 | 32 | data WavesBlock = WavesBlock 33 | { wb_height :: {-# UNPACK #-} !Int64 34 | , wb_version :: {-# UNPACK #-} !Int64 35 | , wb_timestamp :: {-# UNPACK #-} !Int64 36 | , wb_basetarget :: {-# UNPACK #-} !Int64 37 | , wb_generator :: !T.Text 38 | , wb_blocksize :: {-# UNPACK #-} !Int64 39 | , wb_transactions :: !(V.Vector WavesTransaction) 40 | } 41 | 42 | instance HasBlockHeader WavesBlock where 43 | getBlockHeader WavesBlock 44 | { wb_height = height 45 | , wb_timestamp = timestamp 46 | } = BlockHeader 47 | { bh_height = height 48 | , bh_hash = mempty 49 | , bh_prevHash = Nothing 50 | , bh_timestamp = posixSecondsToUTCTime $ fromIntegral timestamp * 0.001 51 | } 52 | 53 | newtype WavesBlockWrapper = WavesBlockWrapper 54 | { unwrapWavesBlock :: WavesBlock 55 | } 56 | 57 | instance J.FromJSON WavesBlockWrapper where 58 | parseJSON = J.withObject "waves block" $ \fields -> fmap WavesBlockWrapper $ WavesBlock 59 | <$> (fields J..: "height") 60 | <*> (fields J..: "version") 61 | <*> (fields J..: "timestamp") 62 | <*> ((J..: "base-target") =<< fields J..: "nxt-consensus") 63 | <*> (fields J..: "generator") 64 | <*> (fields J..: "blocksize") 65 | <*> (V.map unwrapWavesTransaction <$> fields J..: "transactions") 66 | 67 | data WavesTransaction = WavesTransaction 68 | { wt_type :: {-# UNPACK #-} !Int64 69 | , wt_id :: !T.Text 70 | , wt_timestamp :: {-# UNPACK #-} !Int64 71 | , wt_fee :: {-# UNPACK #-} !Int64 72 | , wt_version :: !(Maybe Int64) 73 | , wt_sender :: !(Maybe T.Text) 74 | , wt_recipient :: !(Maybe T.Text) 75 | , wt_amount :: !(Maybe Int64) 76 | , wt_assetId :: !(Maybe T.Text) 77 | , wt_feeAssetId :: !(Maybe T.Text) 78 | , wt_feeAsset :: !(Maybe T.Text) 79 | , wt_name :: !(Maybe T.Text) 80 | , wt_quantity :: !(Maybe Int64) 81 | , wt_reissuable :: !(Maybe Bool) 82 | , wt_decimals :: !(Maybe Int64) 83 | , wt_description :: !(Maybe T.Text) 84 | , wt_price :: !(Maybe Int64) 85 | , wt_buyMatcherFee :: !(Maybe Int64) 86 | , wt_sellMatcherFee :: !(Maybe Int64) 87 | , wt_order1 :: !(Maybe WavesOrder) 88 | , wt_order2 :: !(Maybe WavesOrder) 89 | , wt_leaseId :: !(Maybe T.Text) 90 | , wt_alias :: !(Maybe T.Text) 91 | , wt_transfers :: !(V.Vector WavesTransfer) 92 | } 93 | 94 | newtype WavesTransactionWrapper = WavesTransactionWrapper 95 | { unwrapWavesTransaction :: WavesTransaction 96 | } 97 | 98 | instance J.FromJSON WavesTransactionWrapper where 99 | parseJSON = J.withObject "waves transaction" $ \fields -> fmap WavesTransactionWrapper $ WavesTransaction 100 | <$> (fields J..: "type") 101 | <*> (fields J..: "id") 102 | <*> (fields J..: "timestamp") 103 | <*> (fields J..: "fee") 104 | <*> (fields J..:? "version") 105 | <*> (fields J..:? "sender") 106 | <*> (fields J..:? "recipient") 107 | <*> (fields J..:? "amount") 108 | <*> (fields J..:? "assetId") 109 | <*> (fields J..:? "feeAssetId") 110 | <*> (fields J..:? "feeAsset") 111 | <*> (fields J..:? "name") 112 | <*> (fields J..:? "quantity") 113 | <*> (fields J..:? "reissuable") 114 | <*> (fields J..:? "decimals") 115 | <*> (fields J..:? "description") 116 | <*> (fields J..:? "price") 117 | <*> (fields J..:? "buyMatcherFee") 118 | <*> (fields J..:? "sellMatcherFee") 119 | <*> (fmap unwrapWavesOrder <$> fields J..:? "order1") 120 | <*> (fmap unwrapWavesOrder <$> fields J..:? "order2") 121 | <*> (fields J..:? "leaseId") 122 | <*> (fields J..:? "alias") 123 | <*> (V.map unwrapWavesTransfer . fromMaybe V.empty <$> fields J..:? "transfers") 124 | 125 | data WavesOrder = WavesOrder 126 | { wo_id :: !T.Text 127 | , wo_sender :: !T.Text 128 | , wo_matcherPublicKey :: !T.Text 129 | , wo_amountAsset :: !(Maybe T.Text) 130 | , wo_priceAsset :: !(Maybe T.Text) 131 | , wo_orderType :: !T.Text 132 | , wo_price :: {-# UNPACK #-} !Int64 133 | , wo_amount :: {-# UNPACK #-} !Int64 134 | , wo_timestamp :: {-# UNPACK #-} !Int64 135 | , wo_expiration :: {-# UNPACK #-} !Int64 136 | , wo_matcherFee :: {-# UNPACK #-} !Int64 137 | } 138 | 139 | newtype WavesOrderWrapper = WavesOrderWrapper 140 | { unwrapWavesOrder :: WavesOrder 141 | } 142 | 143 | instance J.FromJSON WavesOrderWrapper where 144 | parseJSON = J.withObject "waves order" $ \fields -> fmap WavesOrderWrapper $ WavesOrder 145 | <$> (fields J..: "id") 146 | <*> (fields J..: "sender") 147 | <*> (fields J..: "matcherPublicKey") 148 | <*> ((J..: "amountAsset") =<< fields J..: "assetPair") 149 | <*> ((J..: "priceAsset") =<< fields J..: "assetPair") 150 | <*> (fields J..: "orderType") 151 | <*> (fields J..: "price") 152 | <*> (fields J..: "amount") 153 | <*> (fields J..: "timestamp") 154 | <*> (fields J..: "expiration") 155 | <*> (fields J..: "matcherFee") 156 | 157 | data WavesTransfer = WavesTransfer 158 | { wtf_recipient :: !T.Text 159 | , wtf_amount :: {-# UNPACK #-} !Int64 160 | } 161 | 162 | newtype WavesTransferWrapper = WavesTransferWrapper 163 | { unwrapWavesTransfer :: WavesTransfer 164 | } 165 | 166 | instance J.FromJSON WavesTransferWrapper where 167 | parseJSON = J.withObject "waves transfer" $ \fields -> fmap WavesTransferWrapper $ WavesTransfer 168 | <$> (fields J..: "recipient") 169 | <*> (fields J..: "amount") 170 | 171 | 172 | genSchemaInstances [''WavesBlock, ''WavesTransaction, ''WavesOrder, ''WavesTransfer] 173 | 174 | 175 | instance BlockChain Waves where 176 | type Block Waves = WavesBlock 177 | 178 | getBlockChainInfo _ = BlockChainInfo 179 | { bci_init = \BlockChainParams 180 | { bcp_httpManager = httpManager 181 | , bcp_httpRequest = httpRequest 182 | } -> return Waves 183 | { waves_httpManager = httpManager 184 | , waves_httpRequest = httpRequest 185 | } 186 | , bci_defaultApiUrls = ["http://127.0.0.1:6869/"] 187 | , bci_defaultBeginBlock = 1 188 | , bci_defaultEndBlock = 0 -- PoS 189 | , bci_heightFieldName = "height" 190 | , bci_schemas = standardBlockChainSchemas 191 | (schemaOf (Proxy :: Proxy WavesBlock)) 192 | [ schemaOf (Proxy :: Proxy WavesTransfer) 193 | , schemaOf (Proxy :: Proxy WavesOrder) 194 | , schemaOf (Proxy :: Proxy WavesTransaction) 195 | ] 196 | "CREATE TABLE \"waves\" OF \"WavesBlock\" (PRIMARY KEY (\"height\"));" 197 | } 198 | 199 | getCurrentBlockHeight Waves 200 | { waves_httpManager = httpManager 201 | , waves_httpRequest = httpRequest 202 | } = do 203 | response <- tryWithRepeat $ H.httpLbs httpRequest 204 | { H.path = "/blocks/height" 205 | } httpManager 206 | either fail return $ J.parseEither (J..: "height") =<< J.eitherDecode' (H.responseBody response) 207 | 208 | getBlockByHeight Waves 209 | { waves_httpManager = httpManager 210 | , waves_httpRequest = httpRequest 211 | } blockHeight = do 212 | response <- tryWithRepeat $ H.httpLbs httpRequest 213 | { H.path = "/blocks/at/" <> fromString (show blockHeight) 214 | } httpManager 215 | either fail (return . unwrapWavesBlock) $ J.eitherDecode' $ H.responseBody response 216 | -------------------------------------------------------------------------------- /coinmetrics-waves/coinmetrics-waves.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics-waves 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.Waves 17 | build-depends: 18 | aeson 19 | , base 20 | , coinmetrics 21 | , hanalytics-base 22 | , http-client 23 | , text 24 | , time 25 | , vector 26 | ghc-options: -Wall 27 | default-language: Haskell2010 28 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/BlockChain.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GADTs, FlexibleContexts, TypeFamilies #-} 2 | 3 | module CoinMetrics.BlockChain 4 | ( BlockChain(..) 5 | , HasBlockHeader(..) 6 | , BlockChainParams(..) 7 | , BlockChainInfo(..) 8 | , BlockChainNodeInfo(..) 9 | , BlockHeader(..) 10 | , BlockHash() 11 | , BlockHeight() 12 | , BlockTimestamp() 13 | , SomeBlockChain(..) 14 | , SomeBlockChainInfo(..) 15 | , SomeBlocks(..) 16 | ) where 17 | 18 | import qualified Data.Aeson as J 19 | import qualified Data.Avro as A 20 | import Data.Int 21 | import qualified Data.HashMap.Strict as HM 22 | import Data.Proxy 23 | import qualified Data.Text as T 24 | import Data.Time.Clock 25 | import qualified Network.HTTP.Client as H 26 | 27 | import Hanalytics.Schema 28 | import Hanalytics.Schema.Postgres 29 | 30 | import CoinMetrics.Util 31 | 32 | class (HasBlockHeader (Block a), Schemable (Block a), A.ToAvro (Block a), ToPostgresText (Block a), J.ToJSON (Block a)) => BlockChain a where 33 | type Block a :: * 34 | 35 | getBlockChainInfo :: Proxy a -> BlockChainInfo a 36 | 37 | getBlockChainNodeInfo :: a -> IO BlockChainNodeInfo 38 | getBlockChainNodeInfo _ = return BlockChainNodeInfo 39 | { bcni_version = T.empty 40 | } 41 | 42 | getCurrentBlockHeight :: a -> IO BlockHeight 43 | 44 | getBlockHeaderByHeight :: a -> BlockHeight -> IO BlockHeader 45 | getBlockHeaderByHeight blockChain blockHeight = getBlockHeader <$> getBlockByHeight blockChain blockHeight 46 | 47 | getBlockByHeight :: a -> BlockHeight -> IO (Block a) 48 | 49 | 50 | class HasBlockHeader a where 51 | getBlockHeader :: a -> BlockHeader 52 | 53 | -- | Params for initializing blockchain. 54 | data BlockChainParams = BlockChainParams 55 | { bcp_httpManager :: !H.Manager 56 | , bcp_httpRequest :: !H.Request 57 | -- | Include transaction trace information. 58 | , bcp_trace :: !Bool 59 | -- | Exclude unaccounted actions from trace information. 60 | , bcp_excludeUnaccountedActions :: !Bool 61 | -- | Specify API flavor. Used to when two blockchains only have minor differences in their API. 62 | , bcp_apiFlavor :: !T.Text 63 | -- | Number of threads working with blockchain. 64 | , bcp_threadsCount :: {-# UNPACK #-} !Int 65 | } 66 | 67 | -- | Information about blockchain. 68 | data BlockChainInfo a = BlockChainInfo 69 | { bci_init :: !(BlockChainParams -> IO a) 70 | , bci_defaultApiUrls :: ![String] 71 | , bci_defaultBeginBlock :: {-# UNPACK #-} !BlockHeight 72 | , bci_defaultEndBlock :: {-# UNPACK #-} !BlockHeight 73 | , bci_heightFieldName :: !T.Text 74 | -- | Schemas referenced by storage type. 75 | , bci_schemas :: !(HM.HashMap T.Text T.Text) 76 | -- | Table suffixes for flattened blocks. 77 | , bci_flattenSuffixes :: [T.Text] 78 | -- | Flatten block function. 79 | , bci_flattenPack :: [Block a] -> [SomeBlocks] 80 | } 81 | 82 | -- | Information about blockchain node. 83 | data BlockChainNodeInfo = BlockChainNodeInfo 84 | { bcni_version :: !T.Text 85 | } 86 | 87 | -- | Information about block. 88 | data BlockHeader = BlockHeader 89 | { bh_height :: {-# UNPACK #-} !BlockHeight 90 | , bh_hash :: {-# UNPACK #-} !BlockHash 91 | , bh_prevHash :: !(Maybe BlockHash) 92 | , bh_timestamp :: !UTCTime 93 | } 94 | 95 | instance HasBlockHeader BlockHeader where 96 | getBlockHeader = id 97 | 98 | type BlockHash = HexString 99 | type BlockHeight = Int64 100 | type BlockTimestamp = UTCTime 101 | 102 | data SomeBlockChain where 103 | SomeBlockChain :: BlockChain a => a -> SomeBlockChain 104 | 105 | data SomeBlockChainInfo where 106 | SomeBlockChainInfo :: BlockChain a => BlockChainInfo a -> SomeBlockChainInfo 107 | 108 | -- | Helper data type to contain stripe of arbitrary blocks. 109 | data SomeBlocks where 110 | SomeBlocks :: (Schemable b, A.ToAvro b, ToPostgresText b, J.ToJSON b) => [b] -> SomeBlocks 111 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/JsonRpc.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists, OverloadedStrings #-} 2 | 3 | module CoinMetrics.JsonRpc 4 | ( JsonRpc() 5 | , newJsonRpc 6 | , jsonRpcRequest 7 | , nonJsonRpcRequest 8 | ) where 9 | 10 | import qualified Data.Aeson as J 11 | import qualified Data.Aeson.Types as J 12 | import qualified Data.ByteString as B 13 | import qualified Data.Text as T 14 | import qualified Data.Text.Encoding as T 15 | import qualified Network.HTTP.Client as H 16 | 17 | import CoinMetrics.Util 18 | 19 | data JsonRpc = JsonRpc 20 | { jsonRpc_httpManager :: !H.Manager 21 | , jsonRpc_httpRequest :: !H.Request 22 | } 23 | 24 | newJsonRpc :: H.Manager -> H.Request -> Maybe (T.Text, T.Text) -> JsonRpc 25 | newJsonRpc httpManager httpRequest maybeCredentials = JsonRpc 26 | { jsonRpc_httpManager = httpManager 27 | , jsonRpc_httpRequest = maybe id (\(authName, authPass) -> H.applyBasicAuth (T.encodeUtf8 authName) (T.encodeUtf8 authPass)) maybeCredentials httpRequest 28 | { H.method = "POST" 29 | , H.requestHeaders = ("Content-Type", "application/json") : H.requestHeaders httpRequest 30 | } 31 | } 32 | 33 | jsonRpcRequest :: (J.FromJSON r, J.ToJSON p) => JsonRpc -> T.Text -> p -> IO r 34 | jsonRpcRequest JsonRpc 35 | { jsonRpc_httpManager = httpManager 36 | , jsonRpc_httpRequest = httpRequest 37 | } method params = do 38 | body <- H.responseBody <$> tryWithRepeat (H.httpLbs httpRequest 39 | { H.requestBody = H.RequestBodyLBS $ J.encode $ J.Object 40 | [ ("jsonrpc", "2.0") 41 | , ("method", J.String method) 42 | , ("params", J.toJSON params) 43 | , ("id", J.String "1") 44 | ] 45 | } httpManager) 46 | case J.eitherDecode' body of 47 | Right obj -> case J.parse (J..: "result") obj of 48 | J.Success result -> return result 49 | J.Error err -> fail err 50 | Left err -> fail err 51 | 52 | nonJsonRpcRequest :: (J.FromJSON r, J.ToJSON p) => JsonRpc -> B.ByteString -> p -> IO r 53 | nonJsonRpcRequest JsonRpc 54 | { jsonRpc_httpManager = httpManager 55 | , jsonRpc_httpRequest = httpRequest 56 | } path params = do 57 | body <- H.responseBody <$> tryWithRepeat (H.httpLbs httpRequest 58 | { H.requestBody = H.RequestBodyLBS $ J.encode $ J.toJSON params 59 | , H.path = path 60 | } httpManager) 61 | either fail return $ J.eitherDecode' body 62 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/Prometheus.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module CoinMetrics.Prometheus 4 | ( OptionPrometheus(..) 5 | , optionPrometheus 6 | , forkPrometheusMetricsServer 7 | , measureTime 8 | ) where 9 | 10 | import Control.Concurrent 11 | import Control.Exception 12 | import Control.Monad 13 | import Data.String 14 | import qualified Network.HTTP.Types as H 15 | import qualified Network.Wai as W 16 | import qualified Network.Wai.Handler.Warp as Warp 17 | import qualified Options.Applicative as O 18 | import qualified Prometheus as P 19 | import qualified System.Clock as Clock 20 | 21 | data OptionPrometheus = OptionPrometheus 22 | { optionPrometheus_host :: !(Maybe String) 23 | , optionPrometheus_port :: !Int 24 | } 25 | 26 | optionPrometheus :: O.Parser OptionPrometheus 27 | optionPrometheus = OptionPrometheus 28 | <$> O.option (O.maybeReader (Just . Just)) 29 | ( O.long "prometheus-host" 30 | <> O.metavar "PROMETHEUS_HOST" 31 | <> O.value Nothing 32 | <> O.help "Prometheus metrics host" 33 | ) 34 | <*> O.option O.auto 35 | ( O.long "prometheus-port" 36 | <> O.metavar "PROMETHEUS_PORT" 37 | <> O.value 8080 38 | <> O.help "Prometheus metrics port" 39 | ) 40 | 41 | forkPrometheusMetricsServer :: OptionPrometheus -> IO () 42 | forkPrometheusMetricsServer OptionPrometheus 43 | { optionPrometheus_host = maybeHost 44 | , optionPrometheus_port = port 45 | } = case maybeHost of 46 | Just host -> void $ forkIO $ 47 | Warp.runSettings (Warp.setHost (fromString host) $ Warp.setPort port Warp.defaultSettings) $ \_request respond -> 48 | respond . W.responseLBS H.status200 49 | [ (H.hContentType, "text/plain; version=0.0.4") 50 | ] =<< P.exportMetricsAsText 51 | Nothing -> return () 52 | 53 | measureTime :: P.Observer o => o -> IO a -> IO a 54 | measureTime observer = bracket start stop . const where 55 | start = Clock.getTime Clock.Monotonic 56 | stop startTime = do 57 | endTime <- Clock.getTime Clock.Monotonic 58 | P.observe observer $ fromTime $ endTime `Clock.diffTimeSpec` startTime 59 | 60 | fromTime :: Clock.TimeSpec -> Double 61 | fromTime Clock.TimeSpec 62 | { Clock.sec = sec 63 | , Clock.nsec = nsec 64 | } = fromIntegral sec + fromIntegral nsec * 0.000000001 65 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/Schema/Flatten.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE FlexibleContexts, TemplateHaskell, TupleSections, TypeFamilies, ViewPatterns #-} 2 | 3 | module CoinMetrics.Schema.Flatten 4 | ( Flattenable(..) 5 | , genFlattenedTypes 6 | ) where 7 | 8 | import Data.Int 9 | import Data.List 10 | import qualified Data.Vector as V 11 | import Language.Haskell.TH 12 | import Language.Haskell.TH.Syntax 13 | 14 | import CoinMetrics.Schema.Util 15 | 16 | class Monoid (Flattened a) => Flattenable a where 17 | type Flattened a :: * 18 | flatten :: a -> Flattened a 19 | 20 | -- | Generates datas and functions for flattened versions of types. 21 | {- 22 | -- decls 23 | data Block { height :: Int64, b :: B, transactions :: Vector Transaction } 24 | data Transaction { t :: T, logs :: Vector Log } 25 | data Log { l :: L } 26 | 27 | -- TH 28 | genFlattenedTypes [''Block, ''Transaction, ''Log] 29 | 30 | -- produces 31 | 32 | data Block_flattened { height :: Int64, b :: B } 33 | data Transaction_flattened { block_height :: Int64, _index :: Int64, t :: T } 34 | data Transaction_flattened { block_height :: Int64, transaction_index :: Int64, _index :: Int64, l :: L } 35 | instance Flattenable Block where 36 | type Flattened Block = ([Block_flattened], ([Transaction_flattened], [Log_flattened])) 37 | flatten Block { height = height', b = b', transactions = transactions' } = 38 | ( [Block_flattened { height = height', b = b' }] 39 | , mconcat $ zipWith (flattenTransaction height') [0..] (V.toList transactions') 40 | ) 41 | flattenTransaction blockHeight i Transaction { t = t', logs = logs' } = 42 | ( [Transaction_flattened { block_height = blockHeight, _index = i, t = t' }] 43 | , mconcat $ zipWith (flattenLog blockHeight i) [0..] (V.toList logs') 44 | ) 45 | flattenLog blockHeight ti i Log { l = l' } = [Log_flattened { block_height = blockHeight, transaction_index = ti, _index = i, l = l' }] 46 | -} 47 | genFlattenedTypes :: String -> ExpQ -> [(String, Name)] -> Q [Dec] 48 | genFlattenedTypes rootKeyName rootKeyExp types@(map snd -> typesNames) = do 49 | (rootDecs, _, _) <- flattenType [] (snd rootType) 50 | return rootDecs 51 | where 52 | 53 | -- generate declarations, lambda and type for given source type 54 | flattenType :: [(String, Name)] -> Name -> Q ([Dec], ExpQ, TypeQ) 55 | flattenType parentIndexNamesAndVars@(unzip -> (parentIndexNames, parentIndexVars)) typeName = do 56 | 57 | -- get information about source type 58 | TyConI (DataD _cxt _dataName _tyVars _kind [RecC typeConName fields] _deriv) <- reify typeName 59 | 60 | let 61 | -- split fields into ones need flattening and others 62 | (fieldsToFlatten, fieldsToKeep) = mconcat . flip map fields $ \field@(_, _, fieldType) -> case fieldType of 63 | AppT (ConT (Name (OccName "Vector") _)) (ConT (flip elem typesNames -> True)) -> 64 | ([field], []) 65 | _ -> ([], [field]) 66 | 67 | -- generate some stuff 68 | Just (typeTitle, _) = find ((== typeName) . snd) types 69 | flattenedTypeName = suffixName "_flattened" typeName 70 | 71 | fieldsToFlattenVars <- mapM genFieldVar fieldsToFlatten 72 | fieldsToKeepVars <- mapM genFieldVar fieldsToKeep 73 | 74 | -- find prefix used 75 | let (takeWhile (/= '_') . nameStr -> fieldPrefix, _, _) = head fieldsToKeep 76 | 77 | -- vars and flags 78 | let isRoot = null parentIndexNamesAndVars 79 | indexVar <- newName "index" 80 | let indexFieldName = prefixName flatFieldPrefix (mkName $ fieldPrefix <> "__index") 81 | rootKeyVar <- newName rootKeyName 82 | rootVar <- newName $ fst rootType 83 | let parentIndexFieldNames = map (prefixName flatFieldPrefix . mkName . (fieldPrefix <>)) parentIndexNames 84 | 85 | -- generate children declarations, lambdas and types 86 | (mconcat -> childrenDecs, childrenLambdas, childrenFlattenedTypes) <- unzip3 <$> sequence 87 | [flattenType (if isRoot then [("_" <> typeTitle <> "_" <> rootKeyName, rootKeyVar)] else parentIndexNamesAndVars <> [("_" <> typeTitle <> "_index", indexVar)]) fieldSubTypeName | (_, _, AppT _ (ConT fieldSubTypeName)) <- fieldsToFlatten] 88 | 89 | let 90 | -- our lambda 91 | lambda = 92 | lamE 93 | ( (if isRoot then [] else [varP indexVar]) 94 | <> [(if isRoot then asP rootVar else id) $ recP typeConName (fieldsPats fieldsToKeep fieldsToKeepVars <> fieldsPats fieldsToFlatten fieldsToFlattenVars)] 95 | ) $ 96 | ( if isRoot 97 | then letE [valD (varP rootKeyVar) (normalB $ appE rootKeyExp (varE rootVar)) []] 98 | else id 99 | ) $ 100 | (if null fieldsToFlatten then head else tupE) 101 | ( listE 102 | [ recConE flattenedTypeName 103 | ( zipWith (\fieldName var -> (fieldName, ) <$> varE var) parentIndexFieldNames parentIndexVars 104 | <> (if isRoot 105 | then [] 106 | else [(indexFieldName, ) <$> varE indexVar] 107 | ) 108 | <> fieldsAssigns fieldsToKeep fieldsToKeepVars 109 | ) 110 | ] 111 | : zipWith (\childVar childLambda -> [| mconcat (zipWith $(childLambda) [0..] (V.toList $(varE childVar))) |]) fieldsToFlattenVars childrenLambdas 112 | ) 113 | 114 | -- our flattened type 115 | flattenedType = 116 | (if null fieldsToFlatten then head else foldl appT (tupleT (1 + length fieldsToFlatten))) 117 | (appT listT (conT flattenedTypeName) : childrenFlattenedTypes) 118 | 119 | -- our declarations 120 | instancesDecs <- schemaInstancesDecs (conT flattenedTypeName) 121 | decs <- sequence $ 122 | [ dataD (pure []) flattenedTypeName [] Nothing 123 | [ recC flattenedTypeName 124 | ( map (\parentIndexFieldName -> varBangType parentIndexFieldName (bangType unpackBang [t| Int64 |])) parentIndexFieldNames 125 | <> ( if isRoot then [] else 126 | [ varBangType indexFieldName (bangType unpackBang [t| Int64 |]) 127 | ] 128 | ) 129 | <> map (pure . prefixFlatField) fieldsToKeep 130 | ) 131 | ] [] 132 | ] 133 | <> 134 | [ instanceD (pure []) [t| Flattenable $(conT typeName) |] 135 | [ tySynInstD (tySynEqn Nothing [t| Flattened $(conT typeName) |] flattenedType) 136 | , funD 'flatten [clause [] (normalB lambda) []] 137 | ] 138 | | isRoot 139 | ] 140 | 141 | return (decs <> instancesDecs <> childrenDecs, lambda, flattenedType) 142 | 143 | rootType = head types 144 | 145 | -- name functions 146 | nameStr (Name (OccName name) _) = name 147 | recreateName f (Name (OccName name) _) = mkName (f name) 148 | prefixName prefix = recreateName (prefix <>) 149 | suffixName suffix = recreateName (<> suffix) 150 | 151 | flatFieldPrefix = "flat'" 152 | 153 | prefixFlatField (fieldName, fieldBang, fieldType) = (prefixName flatFieldPrefix fieldName, fieldBang, fieldType) 154 | 155 | fieldsPats = zipWith $ \(fieldName, _, _) fieldVar -> fieldPat fieldName (varP fieldVar) 156 | fieldsAssigns = zipWith $ \(fieldName, _, _) fieldVar -> pure (prefixName flatFieldPrefix fieldName, VarE fieldVar) 157 | genFieldVar (fieldName, _, _) = newName $ nameStr fieldName 158 | unpackBang = bang sourceUnpack sourceStrict 159 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/Schema/Util.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GeneralizedNewtypeDeriving, LambdaCase, OverloadedStrings, TemplateHaskell #-} 2 | 3 | module CoinMetrics.Schema.Util 4 | ( genSchemaInstances 5 | , schemaInstancesDecs 6 | , schemaInstancesCtxDecs 7 | , genNewtypeSchemaInstances 8 | , genJsonInstances 9 | , schemaJsonOptions 10 | ) where 11 | 12 | import qualified Data.Aeson as J 13 | import qualified Data.Avro as A 14 | import GHC.Generics(Generic) 15 | import Language.Haskell.TH 16 | 17 | import Hanalytics.Schema 18 | import Hanalytics.Schema.Avro 19 | import Hanalytics.Schema.Postgres 20 | 21 | schemaJsonOptions :: J.Options 22 | schemaJsonOptions = J.defaultOptions 23 | { J.fieldLabelModifier = stripBeforeUnderscore 24 | , J.constructorTagModifier = stripBeforeUnderscore 25 | , J.sumEncoding = J.TaggedObject 26 | { J.tagFieldName = "type" 27 | , J.contentsFieldName = "data" 28 | } 29 | } 30 | 31 | genSchemaInstances :: [Name] -> Q [Dec] 32 | genSchemaInstances = fmap mconcat . mapM (schemaInstancesDecs . conT) 33 | 34 | schemaInstancesDecs :: TypeQ -> Q [Dec] 35 | schemaInstancesDecs = schemaInstancesCtxDecs [] 36 | 37 | schemaInstancesCtxDecs :: [TypeQ] -> TypeQ -> Q [Dec] 38 | schemaInstancesCtxDecs preds con = sequence 39 | [ standaloneDerivD (pure []) [t| Generic $con |] 40 | , instanceD (context [ [t| SchemableField |] ]) [t| Schemable $con |] [] 41 | , instanceD (context [ [t| SchemableField |] ]) [t| SchemableField $con |] [] 42 | , instanceD (context [ [t| J.FromJSON |] ]) [t| J.FromJSON $con |] 43 | [ funD 'J.parseJSON [clause [] (normalB [| J.genericParseJSON schemaJsonOptions |] ) []] 44 | ] 45 | , instanceD (context [ [t| J.ToJSON |] ]) [t| J.ToJSON $con |] 46 | [ funD 'J.toJSON [clause [] (normalB [| J.genericToJSON schemaJsonOptions |] ) []] 47 | , funD 'J.toEncoding [clause [] (normalB [| J.genericToEncoding schemaJsonOptions |] ) []] 48 | ] 49 | , instanceD (context [ [t| A.ToAvro |], [t| A.HasAvroSchema |] ]) [t| A.HasAvroSchema $con |] 50 | [ funD 'A.schema [clause [] (normalB [| genericAvroSchema |] ) []] 51 | ] 52 | , instanceD (context [ [t| A.ToAvro |] ]) [t| A.ToAvro $con |] 53 | [ funD 'A.toAvro [clause [] (normalB [| genericToAvro |] ) []] 54 | ] 55 | , instanceD (context [ [t| SchemableField |], [t| ToPostgresText |] ]) [t| ToPostgresText $con |] [] 56 | ] 57 | where 58 | context classes = sequence [cls `appT` prd | cls <- classes, prd <- preds] 59 | 60 | genNewtypeSchemaInstances :: [Name] -> Q [Dec] 61 | genNewtypeSchemaInstances = fmap mconcat . mapM (newtypeSchemaInstancesDecs . conT) 62 | 63 | newtypeSchemaInstancesDecs :: TypeQ -> Q [Dec] 64 | newtypeSchemaInstancesDecs con = sequence 65 | [ standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| Schemable $con |] 66 | , standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| SchemableField $con |] 67 | , standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| J.FromJSON $con |] 68 | , standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| J.ToJSON $con |] 69 | , standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| A.HasAvroSchema $con |] 70 | , standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| A.ToAvro $con |] 71 | , standaloneDerivWithStrategyD (Just NewtypeStrategy) (pure []) [t| ToPostgresText $con |] 72 | ] 73 | 74 | genJsonInstances :: [Name] -> Q [Dec] 75 | genJsonInstances = fmap mconcat . mapM (jsonInstancesDecs . conT) 76 | 77 | jsonInstancesDecs :: TypeQ -> Q [Dec] 78 | jsonInstancesDecs con = sequence 79 | [ standaloneDerivD (pure []) [t| Generic $con |] 80 | , instanceD (pure []) [t| J.FromJSON $con |] 81 | [ funD 'J.parseJSON [clause [] (normalB [| J.genericParseJSON schemaJsonOptions |] ) []] 82 | ] 83 | , instanceD (pure []) [t| J.ToJSON $con |] 84 | [ funD 'J.toJSON [clause [] (normalB [| J.genericToJSON schemaJsonOptions |] ) []] 85 | , funD 'J.toEncoding [clause [] (normalB [| J.genericToEncoding schemaJsonOptions |] ) []] 86 | ] 87 | ] 88 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/Util.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GeneralizedNewtypeDeriving, LambdaCase, OverloadedLists, OverloadedStrings, ViewPatterns #-} 2 | 3 | module CoinMetrics.Util 4 | ( HexString(..) 5 | , decode0xHexBytes 6 | , encode0xHexNumber 7 | , decode0xHexNumber 8 | , decodeReadStr 9 | , decodeMaybeFromText 10 | , tryWithRepeat 11 | , currentLocalTimeToUTC 12 | , standardBlockChainSchemas 13 | ) where 14 | 15 | import Control.Concurrent 16 | import Control.Exception 17 | import Control.Monad 18 | import qualified Data.Aeson as J 19 | import qualified Data.Aeson.Types as J 20 | import qualified Data.Avro as A 21 | import qualified Data.ByteArray.Encoding as BA 22 | import qualified Data.ByteString as B 23 | import qualified Data.ByteString.Short as BS 24 | import qualified Data.ByteString.Lazy as BL 25 | import Data.Hashable 26 | import qualified Data.HashMap.Strict as HM 27 | import qualified Data.Tagged as Tag 28 | import qualified Data.Text as T 29 | import qualified Data.Text.Encoding as T 30 | import qualified Data.Text.Lazy as TL 31 | import qualified Data.Text.Lazy.Builder as TL 32 | import Data.Time.Clock 33 | import Data.Time.LocalTime 34 | import Numeric 35 | import System.IO 36 | import System.IO.Unsafe 37 | 38 | import Hanalytics.Schema 39 | import Hanalytics.Schema.BigQuery 40 | import Hanalytics.Schema.Postgres 41 | 42 | -- | ByteString which serializes to JSON as hex string. 43 | newtype HexString = HexString 44 | { unHexString :: BS.ShortByteString 45 | } deriving (Eq, Ord, Semigroup, Monoid, Hashable) 46 | instance SchemableField HexString where 47 | schemaFieldTypeOf _ = SchemaFieldType_bytes 48 | instance A.HasAvroSchema HexString where 49 | schema = Tag.Tagged $ Tag.unTagged (A.schema :: Tag.Tagged B.ByteString A.Schema) 50 | instance A.ToAvro HexString where 51 | toAvro = A.toAvro . BS.fromShort . unHexString 52 | instance ToPostgresText HexString where 53 | toPostgresText inline = toPostgresText inline . BS.fromShort . unHexString 54 | instance J.FromJSON HexString where 55 | parseJSON = either fail (return . HexString . BS.toShort) . BA.convertFromBase BA.Base16 . T.encodeUtf8 <=< J.parseJSON 56 | instance J.ToJSON HexString where 57 | toJSON = J.toJSON . T.decodeUtf8 . BA.convertToBase BA.Base16 . BS.fromShort . unHexString 58 | toEncoding = J.toEncoding . T.decodeUtf8 . BA.convertToBase BA.Base16 . BS.fromShort . unHexString 59 | 60 | decode0xHexBytes :: T.Text -> J.Parser HexString 61 | decode0xHexBytes = \case 62 | (T.stripPrefix "0x" -> Just (BA.convertFromBase BA.Base16 . T.encodeUtf8 -> Right s)) -> return $ HexString $ BS.toShort s 63 | "" -> return mempty 64 | s -> fail $ "decode0xHexBytes error for: " ++ show s 65 | 66 | encode0xHexNumber :: (Integral a, Show a) => a -> J.Value 67 | encode0xHexNumber = J.String . T.pack . ("0x" <>) . flip showHex "" 68 | 69 | decode0xHexNumber :: Integral a => T.Text -> J.Parser a 70 | decode0xHexNumber = \case 71 | (T.stripPrefix "0x" -> Just (readHex . T.unpack -> [(n, "")])) -> return n 72 | s -> fail $ "decode0xHexNumber error for: " ++ show s 73 | 74 | decodeReadStr :: Read a => T.Text -> J.Parser a 75 | decodeReadStr s = case reads (T.unpack s) of 76 | [(r, "")] -> return r 77 | _ -> fail $ "decodeReadStr error for: " ++ T.unpack s 78 | 79 | decodeMaybeFromText :: (J.FromJSON a, Read a) => J.Value -> J.Parser a 80 | decodeMaybeFromText = \case 81 | J.String s -> decodeReadStr s 82 | v -> J.parseJSON v 83 | 84 | tryWithRepeat :: IO a -> IO a 85 | tryWithRepeat io = let 86 | step i = if i < 5 87 | then do 88 | eitherResult <- try io 89 | case eitherResult of 90 | Right result -> return result 91 | Left (SomeException err) -> do 92 | hPutStrLn stderr $ "error: " ++ show err ++ ", retrying again in 10 seconds" 93 | threadDelay 10000000 94 | step (i + 1) 95 | else fail "repeating failed" 96 | in step (0 :: Int) 97 | 98 | currentLocalTimeToUTC :: LocalTime -> UTCTime 99 | currentLocalTimeToUTC = localTimeToUTC currentTimeZone 100 | 101 | {-# NOINLINE currentTimeZone #-} 102 | currentTimeZone :: TimeZone 103 | currentTimeZone = unsafePerformIO getCurrentTimeZone 104 | 105 | -- | Helper function for initializing blockchain schemas info. 106 | standardBlockChainSchemas :: Schema -> [Schema] -> TL.Builder -> HM.HashMap T.Text T.Text 107 | standardBlockChainSchemas blockSchema additionalSchemas createTableStmt = 108 | [ ( "postgres" 109 | , TL.toStrict $ TL.toLazyText $ mconcat (map postgresSqlCreateType (additionalSchemas ++ [blockSchema])) <> createTableStmt 110 | ) 111 | , ( "bigquery" 112 | , T.decodeUtf8 $ BL.toStrict $ J.encode $ bigQuerySchema blockSchema 113 | ) 114 | ] 115 | -------------------------------------------------------------------------------- /coinmetrics/CoinMetrics/WebCache.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, ViewPatterns, RankNTypes #-} 2 | 3 | module CoinMetrics.WebCache 4 | ( WebCache(..) 5 | , initWebCache 6 | ) where 7 | 8 | import Control.Applicative 9 | import Control.Exception 10 | import Control.Monad 11 | import qualified Data.ByteString.Lazy as BL 12 | import qualified Data.Text as T 13 | import qualified Data.Text.Encoding as T 14 | import qualified Network.URI as NU 15 | import qualified Network.HTTP.Client as H 16 | import qualified Network.HTTP.Types.URI as HU 17 | import System.Environment 18 | import System.Directory 19 | import System.IO 20 | 21 | newtype WebCache = WebCache 22 | { requestWebCache :: forall a. Bool -> H.Request -> (Either String BL.ByteString -> IO (Bool, a)) -> IO a 23 | } 24 | 25 | -- | Init web cache, using environment variables. 26 | initWebCache :: H.Manager -> IO WebCache 27 | initWebCache httpManager = do 28 | maybeVar <- lookupEnv "WEB_CACHE" 29 | rewrite <- maybe False (const True) <$> lookupEnv "WEB_CACHE_REWRITE" 30 | offline <- maybe False (const True) <$> lookupEnv "WEB_CACHE_OFFLINE" 31 | debug <- maybe False (const True) <$> lookupEnv "WEB_CACHE_DEBUG" 32 | case maybeVar of 33 | Just (T.pack -> var) -> case var of 34 | (T.stripPrefix "file:" -> Just filePath) -> do 35 | return WebCache 36 | { requestWebCache = \skipCache httpRequest f -> let 37 | -- uri2 tries the same uri, but with the explicit port 38 | -- that's needed, because old cache may contain explicit port, 39 | -- due to breaking change in URLs 40 | (uri, uri2) = let 41 | u@NU.URI 42 | { NU.uriScheme = scheme 43 | , NU.uriAuthority = maybeAuthority 44 | } = H.getUri httpRequest 45 | in (u, u 46 | { NU.uriAuthority = (\a -> a 47 | { NU.uriPort = case (scheme, NU.uriPort a) of 48 | ("https:", "") -> ":443" 49 | ("http:", "") -> ":80" 50 | _ -> NU.uriPort a 51 | }) <$> maybeAuthority 52 | }) 53 | fileName = T.unpack $ filePath <> "/" <> T.decodeUtf8 (HU.urlEncode False (T.encodeUtf8 $ T.pack $ NU.uriToString id uri "")) 54 | fileName2 = T.unpack $ filePath <> "/" <> T.decodeUtf8 (HU.urlEncode False (T.encodeUtf8 $ T.pack $ NU.uriToString id uri2 "")) 55 | doRequest = do 56 | when debug $ hPutStrLn stderr $ "performing HTTP request: " <> NU.uriToString id uri "" 57 | H.responseBody <$> H.httpLbs httpRequest httpManager 58 | in if skipCache 59 | then snd <$> (f . Right =<< doRequest) 60 | else do 61 | eitherCachedResponse <- if rewrite 62 | then return $ Left $ SomeException (undefined :: SomeException) 63 | else try $ BL.readFile fileName <|> BL.readFile fileName2 -- try both files 64 | case eitherCachedResponse of 65 | Right cachedResponse -> do 66 | when debug $ hPutStrLn stderr $ "using cached response: " <> fileName 67 | snd <$> f (Right cachedResponse) 68 | Left SomeException {} -> 69 | if offline 70 | then do 71 | when debug $ hPutStrLn stderr $ "no cached offline response: " <> fileName 72 | snd <$> f (Left "no cached response in offline mode") 73 | else do 74 | response <- doRequest 75 | (ok, result) <- f (Right response) 76 | if ok 77 | then do 78 | when debug $ hPutStrLn stderr $ "saving to cache: " <> fileName 79 | let tempFileName = fileName <> ".tmp" 80 | BL.writeFile tempFileName response 81 | renameFile tempFileName fileName 82 | else when debug $ hPutStrLn stderr $ "bad response, not saving to cache" 83 | return result 84 | } 85 | _ -> fail "wrong WEB_CACHE" 86 | Nothing -> return WebCache 87 | { requestWebCache = \_skipCache httpRequest f -> fmap snd . f . Right . H.responseBody =<< H.httpLbs httpRequest httpManager 88 | } 89 | -------------------------------------------------------------------------------- /coinmetrics/coinmetrics.cabal: -------------------------------------------------------------------------------- 1 | name: coinmetrics 2 | version: 0.1.0.0 3 | -- synopsis: 4 | -- description: 5 | license: MIT 6 | author: Alexander Bich 7 | maintainer: quyse0@gmail.com 8 | copyright: (c) Coin Metrics, Inc. and contributors 9 | category: Cryptocurrency 10 | build-type: Simple 11 | -- extra-source-files: 12 | cabal-version: >=1.10 13 | 14 | library 15 | exposed-modules: 16 | CoinMetrics.BlockChain 17 | CoinMetrics.JsonRpc 18 | CoinMetrics.Prometheus 19 | CoinMetrics.Schema.Flatten 20 | CoinMetrics.Schema.Util 21 | CoinMetrics.Util 22 | CoinMetrics.WebCache 23 | build-depends: 24 | aeson 25 | , avro 26 | , base 27 | , bytestring 28 | , clock 29 | , directory 30 | , hanalytics-avro 31 | , hanalytics-base 32 | , hanalytics-bigquery 33 | , hanalytics-postgres 34 | , hashable 35 | , http-client 36 | , http-types 37 | , memory 38 | , network-uri 39 | , optparse-applicative 40 | , prometheus-client 41 | , scientific 42 | , tagged 43 | , text 44 | , template-haskell 45 | , time 46 | , unordered-containers 47 | , vector 48 | , wai 49 | , warp 50 | ghc-options: -Wall 51 | default-language: Haskell2010 52 | -------------------------------------------------------------------------------- /coinmetrics/schema/elastic/unified.json: -------------------------------------------------------------------------------- 1 | { "mappings": 2 | { "block": 3 | { "properties": 4 | { "time": { "type": "date" } 5 | , "height": { "type": "integer" } 6 | , "hash": { "type": "keyword" } 7 | , "size": { "type": "integer" } 8 | , "transactions": 9 | { "type": "nested" 10 | , "properties": 11 | { "time": { "type": "date" } 12 | , "hash": { "type": "keyword" } 13 | , "fee": 14 | { "properties": 15 | { "amount": { "type": "double" } 16 | , "asset": { "type": "keyword" } 17 | } 18 | } 19 | , "actions": 20 | { "type": "nested" 21 | , "properties": 22 | { "time": { "type": "date" } 23 | , "from": { "type": "keyword" } 24 | , "to": { "type": "keyword" } 25 | , "value": 26 | { "properties": 27 | { "amount": { "type": "double" } 28 | , "asset": { "type": "keyword" } 29 | } 30 | } 31 | , "effects": 32 | { "type": "nested" 33 | , "properties": 34 | { "type": { "type": "keyword" } 35 | , "account": { "type": "keyword" } 36 | , "value": 37 | { "properties": 38 | { "amount": { "type": "double" } 39 | , "asset": { "type": "keyword" } 40 | } 41 | } 42 | , "contract": { "type": "keyword" } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | } 50 | , "dynamic": "strict" 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { nixpkgs }: 2 | rec { 3 | hanalytics = nixpkgs.fetchFromGitHub { 4 | owner = "quyse"; 5 | repo = "hanalytics"; 6 | rev = "dbe294c4c1524268683764c3cd3ec947105359c9"; 7 | sha256 = "0wpib53843xggnd70vxxivmv9cd0gqgfpia3v3f81q0vdf8mz61k"; 8 | }; 9 | 10 | packages = nixpkgs.haskellPackages.override { 11 | overrides = let 12 | deps = self: super: stellarPackage.extraLibs; 13 | 14 | sourceOverrides = nixpkgs.haskell.lib.packageSourceOverrides { 15 | coinmetrics = ./coinmetrics; 16 | coinmetrics-all-blockchains = ./coinmetrics-all-blockchains; 17 | coinmetrics-binance = ./coinmetrics-binance; 18 | coinmetrics-bitcoin = ./coinmetrics-bitcoin; 19 | coinmetrics-cardano = ./coinmetrics-cardano; 20 | coinmetrics-cosmos = ./coinmetrics-cosmos; 21 | coinmetrics-eos = ./coinmetrics-eos; 22 | coinmetrics-ethereum = ./coinmetrics-ethereum; 23 | coinmetrics-export = ./coinmetrics-export; 24 | coinmetrics-grin = ./coinmetrics-grin; 25 | coinmetrics-iota = ./coinmetrics-iota; 26 | coinmetrics-monero = ./coinmetrics-monero; 27 | coinmetrics-monitor = ./coinmetrics-monitor; 28 | coinmetrics-nem = ./coinmetrics-nem; 29 | coinmetrics-neo = ./coinmetrics-neo; 30 | coinmetrics-ripple = ./coinmetrics-ripple; 31 | coinmetrics-rosetta = ./coinmetrics-rosetta; 32 | coinmetrics-stellar = ./coinmetrics-stellar; 33 | coinmetrics-storage = ./coinmetrics-storage; 34 | coinmetrics-tendermint = ./coinmetrics-tendermint; 35 | coinmetrics-tezos = ./coinmetrics-tezos; 36 | coinmetrics-tron = ./coinmetrics-tron; 37 | coinmetrics-waves = ./coinmetrics-waves; 38 | 39 | hanalytics-avro = "${hanalytics}/hanalytics-avro"; 40 | hanalytics-base = "${hanalytics}/hanalytics-base"; 41 | hanalytics-bigquery = "${hanalytics}/hanalytics-bigquery"; 42 | hanalytics-postgres = "${hanalytics}/hanalytics-postgres"; 43 | 44 | avro = "0.4.7.0"; 45 | diskhash = "0.0.4.0"; 46 | }; 47 | 48 | tweaks = self: super: with nixpkgs.haskell.lib; { 49 | coinmetrics-stellar = super.coinmetrics-stellar.overrideAttrs (attrs: { 50 | buildInputs = attrs.buildInputs ++ stellarPackage.includes; 51 | }); 52 | 53 | diskhash = dontCheck super.diskhash; 54 | }; 55 | 56 | stellarPackage = import ./coinmetrics-stellar/default.nix { inherit nixpkgs; }; 57 | 58 | in nixpkgs.lib.foldl nixpkgs.lib.composeExtensions deps [ sourceOverrides tweaks ]; 59 | }; 60 | 61 | bins = with (builtins.mapAttrs (name: pkg: nixpkgs.haskell.lib.justStaticExecutables pkg) packages); { 62 | inherit coinmetrics-export coinmetrics-monitor; 63 | }; 64 | 65 | env = additionalContents: nixpkgs.buildEnv { 66 | name = "haskell-tools"; 67 | paths = builtins.attrValues bins ++ additionalContents; 68 | }; 69 | 70 | image = { name ? "coinmetrics/haskell-tools", tag ? "latest", additionalContents ? [] }: nixpkgs.dockerTools.buildImage { 71 | inherit name tag; 72 | contents = [ nixpkgs.cacert ]; 73 | config = { 74 | Env = [ "PATH=${env additionalContents}/bin" ]; 75 | User = "1000:1000"; 76 | }; 77 | }; 78 | } 79 | -------------------------------------------------------------------------------- /docs/blockchains.md: -------------------------------------------------------------------------------- 1 | # Blockchains 2 | 3 | This page documents setting blockchain types and endpoints for the tools. 4 | 5 | ## General parameters 6 | 7 | The tools support the following arguments. 8 | 9 | * `--blockchain BLOCKCHAIN` - the type of blockchain to connect to. See below for supported blockchains. 10 | * `--api-url API_URL` - URL of RPC API of the corresponding fullnode, for example: `--api-url http://127.0.0.1:8332/`. Note that every blockchain type sets sensible default for API_URL, using `127.0.0.1` and appropriate port, so specifying `--api-url` manually may be unnecessary. 11 | * `--api-url-username API_URL_USERNAME`, `--api-url-password API_URL_PASSWORD` - allows to set username and password for HTTP authentication for the endpoint. 12 | * `--api-url-insecure` - disable validation of HTTPS certificates (if URL is HTTPS). 13 | 14 | ## Supported blockchains 15 | 16 | | Blockchain | `--blockchain` | 17 | |-----------------------------------------------|----------------| 18 | | Binance Chain | `binance` | 19 | | [Bitcoin](https://bitcoin.org/) | `bitcoin` | 20 | | [Cardano](https://www.cardanohub.org/) | `cardano` | 21 | | [Cosmos](https://cosmos.network/) | `cosmos` | 22 | | [EOS](https://eos.io/) | `eos` | 23 | | [EOS](https://eos.io/) with history plugin | `eos_archive` | 24 | | [Ethereum](https://www.ethereum.org/) | `ethereum` | 25 | | [Grin](https://grin-tech.org/) | `grin` | 26 | | [Monero](https://getmonero.org/) | `monero` | 27 | | [NEM](https://nem.io/) | `nem` | 28 | | [NEO](https://neo.org/) | `neo` | 29 | | [Ripple](https://ripple.com/) | `ripple` | 30 | | [Stellar](https://www.stellar.org/) | `stellar` | 31 | | Generic [Tendermint](https://tendermint.com/) | `tendermint` | 32 | | [Tezos](https://tezos.com/) | `tezos` | 33 | | [Tron](https://tron.network/) (WIP) | `tron` | 34 | | [Waves](https://wavesplatform.com/) | `waves` | 35 | 36 | ## Notes for specific blockchains 37 | 38 | ### Bitcoin and forks 39 | 40 | Most forks of Bitcoin (using the same RPC API) can be exported using `bitcoin` blockchain type. You only need to specify correct port and host with `--api-url`, because default is set for Bitcoin Core (`http://127.0.0.1:8332/`). 41 | 42 | Username and password for RPC API can be set with `--api-url-username` and `--api-url-password` options. 43 | 44 | ### Ethereum 45 | 46 | Tested only with [Ethereum Parity](https://github.com/paritytech/parity-ethereum). Works with both Ethereum and Ethereum Classic. 47 | 48 | If your Parity node [records tracing information](https://wiki.parity.io/JSONRPC-trace-module), you can specify `--trace` to include it in exported output. 49 | 50 | ### EOS 51 | 52 | `eos` is for normal EOS RPC API. `eos_archive` is for exporting data from [State History Plugin](https://developers.eos.io/eosio-nodeos/docs/state_history_plugin). Output data format of `eos` and `eos_archive` is very different. 53 | 54 | ### Ripple 55 | 56 | Ripple mode requires history server, not fullnode. By default it uses official [history server](https://data.ripple.com/). 57 | 58 | ### Stellar 59 | 60 | Stellar mode requires [history server](https://github.com/stellar/stellar-core/tree/master/src/history), not fullnode. By default it uses official [history servers](https://history.stellar.org/). 61 | -------------------------------------------------------------------------------- /docs/coinmetrics-export.md: -------------------------------------------------------------------------------- 1 | # coinmetrics-export 2 | 3 | ## Description 4 | 5 | `coinmetrics-export` exports blockchain data into formats suitable for analysis by other tools (such as PostgreSQL and Google BigQuery). 6 | The intention is to support multitude of blockchains with a single command-line tool. 7 | 8 | ## Supported blockchains 9 | 10 | See [Blockchains](blockchains.md) page. 11 | 12 | ## Output 13 | 14 | Output formats include: 15 | 16 | | Option | Description | 17 | |---------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| 18 | | `--output-postgres CONNECTION_STRING` | Directly insert into PostgreSQL database | 19 | | `--output-postgres-file FILE` | Output textual series of PostgreSQL-compatible SQL statements into file | 20 | | `--output-elastic ENDPOINT` | Directly insert into ElasticSearch cluster | 21 | | `--output-elastic-file FILE` | Output textual series of JSON lines suitable for ElasticSearch ["bulk" API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) | 22 | | `--output-avro-file FILE` | Output to file in AVRO format | 23 | | `--output-rabbitmq CONNECTION_STRING` | Output to RabbitMQ | 24 | 25 | The smallest exportable unit is a blockchain's block. In case of SQL every source block is exported as a database row. Block's transactions are usually stored in `ARRAY`-typed field of block's row. 26 | 27 | For efficiency the tool combines multiple rows into a single SQL statement or Avro block. Number of rows per statement can be adjusted with `--pack-size`. 28 | 29 | ## Tutorial 30 | 31 | Let's say we have full Ethereum Parity node and we want to export Ethereum data continuously into PostgreSQL database for further analysis. 32 | 33 | First of all, we need to create necessary tables in PostgreSQL (`coinmetrics-export` doesn't do that automatically). Run the following command: 34 | 35 | ```bash 36 | coinmetrics-export print-schema --schema ethereum --storage postgres 37 | ``` 38 | 39 | It outputs necessary SQL statements you need to execute manually against your PostgreSQL database. 40 | 41 | Then you can start synchronization: 42 | 43 | ```bash 44 | coinmetrics-export export --blockchain ethereum \ 45 | --continue --threads 16 \ 46 | --output-postgres "host=127.0.0.1 user=postgres" 47 | ``` 48 | 49 | This command will not stop unless something happens (like Ethereum daemon or PostgreSQL server goes down), and will continue synchronizing newly arriving blocks indefinitely. It can be safely interrupted, and on subsequent restart it will query the last synchronized block in database and continue sync from there (the `--continue` option). 50 | 51 | In case you need to export a specific range of blocks you can use `--begin-block` (inclusive) and `--end-block` (exclusive) parameters. 52 | 53 | Higher number of threads talking to blockchain daemon (`--threads` parameter) usually means increase in synchronization speed, but only up to some limit: test what number on your setup works better. It usually makes sense to make it even higher when connecting to blockchain daemon via network (daemon URL is set with `--api-url` parameter). 54 | 55 | No connection pooling for PostgreSQL is performed, new connection is opened and then closed for every write transaction. Usage of external connection pooling middleware such as `pgbouncer` is advised. 56 | 57 | Tip: try small block ranges and file output (`--output-postgres-file`) first to see samples of produced SQL statements before messing up with actual PostgreSQL database. 58 | 59 | ## Options 60 | 61 | The tool tries to have sane defaults for most parameters. Note that as the tool performs single pass only, in order to correctly fetch blockchain's main chain we have to keep distance from top block (usually called "rewrite limit") in case blockchain's daemon temporarly picked up wrong chain; this distance is specified as negative value of `--end-block` parameter. 62 | 63 | `--ignore-missing-blocks` allows to ignore errors when trying to retrieve blocks from node. Mostly useful with Ripple, as their historical server has many gaps. 64 | 65 | By default the tool issues `INSERT` SQL statements and does not overwrite already synchronized records, but this can be changed with `--upsert` flag. 66 | 67 | ## Exporting IOTA 68 | 69 | There is a separate command `export-iota` for exporting IOTA transaction data. Comparing to `export` command, `export-iota` essentially always works in `--continue` mode and there is no way to specify synchronization bounds. Also PostgreSQL database output is required at the moment. Run like this: 70 | 71 | ```bash 72 | coinmetrics-export export-iota \ 73 | --sync-db iota-sync.db \ 74 | --threads 16 \ 75 | --output-postgres "host=127.0.0.1 user=postgres" 76 | ``` 77 | 78 | The tool can be safely interrupted as it resumes synchronization upon restart. It maintains a small single-file internal helper database (denoted by `--sync-db`) which is not needed to be persisted as it is recreated from scratch on every restart. On start the tool can perform (if given `--fill-holes` flag) special one-time discovery query against PostgreSQL in order to get a list of non-synchronized transactions (i.e. ones referenced by other transactions but not yet existing in database), so it is able to "fill holes" left after interruption. 79 | 80 | Normally IOTA node is able to only return transactions happened after latest snapshot. If you have a textual dump file with previous transactions, you can import it using `--read-dump` switch: 81 | 82 | ```bash 83 | coinmetrics-export export-iota \ 84 | --sync-db iota-sync.db \ 85 | --read-dump \ 86 | --output-postgres "host=127.0.0.1 user=postgres" \ 87 | < transactions.dmp 88 | ``` 89 | -------------------------------------------------------------------------------- /docs/coinmetrics-monitor.md: -------------------------------------------------------------------------------- 1 | # coinmetrics-monitor 2 | 3 | ## Description 4 | 5 | `coinmetrics-monitor` polls specified blockchain nodes and provides a HTTP endpoint with Prometheus-compatible metrics. 6 | 7 | ## Global options 8 | 9 | * `--host HOST`, `--port PORT` - endpoint to listen at, `127.0.0.1` and `8080` by default. 10 | * `--global-label NAME=value` - label to be added to all time series. Can be repeated. 11 | 12 | ## Specifying fullnodes to monitor 13 | 14 | See [Blockchains](blockchains.md) page for full nodes specification. 15 | 16 | The tool can monitor multiple fullnodes of different types. It is recommended to use specific order of options. For every node the `--blockchain` option must be specified first, then options related to this node such as `--api-url`, and then `--blockchain` again for the next node (even if it is of the same blockchain type as previous node). 17 | 18 | In addition to generic blockchain options, the tool allows to set per-node Prometheus labels with `--label NAME=VALUE` option (can be repeated). 19 | 20 | ## Metrics & labels 21 | 22 | The tool reports the following metrics: 23 | 24 | * `blockchain_node_sync_height` (override name with `--height-metric`) - height of the latest block 25 | * `blockchain_node_sync_time` (override name with `--time-metric`) - timestamp of the latest block 26 | * `blockchain_node_up` (override name with `--up-metric`) - 1 if node is up, 0 otherwise 27 | 28 | Every metric has the following labels attached: 29 | 30 | * `blockchain` - type of blockchain (equal to `--blockchain` argument) 31 | * `url` - URL of RPC API (equal to `--api-url` argument) 32 | * `version` - version of fullnode, retrieved via RPC API (not all nodes supported) 33 | 34 | ## Example 35 | 36 | Example command line: 37 | 38 | ```bash 39 | coinmetrics-monitor \ 40 | --host=0.0.0.0 \ 41 | --port=8000 \ 42 | --blockchain=ethereum \ 43 | --label=name=ethereum \ 44 | --api-url=http://ethereum:8545/ \ 45 | --blockchain=ethereum \ 46 | --label=name=ethereum_classic \ 47 | --api-url=http://ethereum-classic:8545/ \ 48 | --blockchain=monero 49 | ``` 50 | 51 | This example shows setting host and port for HTTP endpoint and three blockchains to monitor: Ethereum and Ethereum Classic with custom URLs and labels, and Monero blockchain with default settings and no additional labels. 52 | -------------------------------------------------------------------------------- /docs/docker_image.md: -------------------------------------------------------------------------------- 1 | # Docker image for Haskell-based CoinMetrics.io tools 2 | 3 | These tools are used by CoinMetrics.io team for exporting data from blockchains into analytical databases and monitoring full nodes synchronization state. 4 | 5 | This image is Nix-based and built on Coin Metrics infrastructure. 6 | 7 | See full documentation here: https://github.com/coinmetrics/haskell-tools 8 | -------------------------------------------------------------------------------- /release.nix: -------------------------------------------------------------------------------- 1 | { nixpkgs ? import {} 2 | }: 3 | let 4 | package = import ./default.nix { 5 | inherit nixpkgs; 6 | }; 7 | 8 | image = package.image {}; 9 | imagePlus = package.image { additionalContents = with nixpkgs; [ bash coreutils google-cloud-sdk ]; }; 10 | 11 | # script to push images to registry 12 | # depends on runtime vars DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD, and DOCKERHUB_IMAGE 13 | pushImagesScript = nixpkgs.writeScript "push-images" '' 14 | #!${nixpkgs.stdenv.shell} -e 15 | 16 | ${nixpkgs.skopeo}/bin/skopeo --insecure-policy copy --dest-creds $DOCKERHUB_USERNAME:$DOCKERHUB_PASSWORD docker-archive:${image} docker://$DOCKERHUB_IMAGE 17 | ${nixpkgs.skopeo}/bin/skopeo --insecure-policy copy --dest-creds $DOCKERHUB_USERNAME:$DOCKERHUB_PASSWORD docker-archive:${imagePlus} docker://$DOCKERHUB_IMAGE:plus 18 | ''; 19 | 20 | in with package; { 21 | inherit bins image imagePlus pushImagesScript; 22 | touch = bins // { 23 | inherit image imagePlus pushImagesScript; 24 | }; 25 | } 26 | -------------------------------------------------------------------------------- /stack-shell.nix: -------------------------------------------------------------------------------- 1 | { ghc }: 2 | 3 | let 4 | nixpkgs = import {}; 5 | stellar = import ./coinmetrics-stellar/default.nix { inherit nixpkgs; }; 6 | 7 | in with nixpkgs; haskell.lib.buildStackProject { 8 | inherit ghc; 9 | 10 | name = "haskell-tools-env"; 11 | 12 | buildInputs = [zlib postgresql] ++ stellar.includes ++ lib.attrValues stellar.extraLibs; 13 | } 14 | -------------------------------------------------------------------------------- /stack.yaml: -------------------------------------------------------------------------------- 1 | resolver: lts-16.23 2 | 3 | packages: 4 | - coinmetrics 5 | - coinmetrics-all-blockchains 6 | - coinmetrics-binance 7 | - coinmetrics-bitcoin 8 | - coinmetrics-cardano 9 | - coinmetrics-cosmos 10 | - coinmetrics-eos 11 | - coinmetrics-ethereum 12 | - coinmetrics-export 13 | - coinmetrics-grin 14 | - coinmetrics-iota 15 | - coinmetrics-monero 16 | - coinmetrics-monitor 17 | - coinmetrics-nem 18 | - coinmetrics-neo 19 | - coinmetrics-ripple 20 | - coinmetrics-rosetta 21 | - coinmetrics-stellar 22 | - coinmetrics-storage 23 | - coinmetrics-tendermint 24 | - coinmetrics-tezos 25 | - coinmetrics-tron 26 | - coinmetrics-waves 27 | 28 | extra-deps: 29 | - protobuf-0.2.1.3 30 | - diskhash-0.0.4.0 31 | - avro-0.4.7.0 32 | - git: https://github.com/quyse/hanalytics.git 33 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 34 | subdirs: 35 | - hanalytics-avro 36 | - hanalytics-base 37 | - hanalytics-bigquery 38 | - hanalytics-postgres 39 | 40 | nix: 41 | shell-file: stack-shell.nix 42 | -------------------------------------------------------------------------------- /stack.yaml.lock: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by Stack. 2 | # You should not edit this file by hand. 3 | # For more information, please see the documentation at: 4 | # https://docs.haskellstack.org/en/stable/lock_files 5 | 6 | packages: 7 | - completed: 8 | hackage: protobuf-0.2.1.3@sha256:d8ed3c73027c6b7e7583c6a94ee1835beee8a755a875b06bbf51bc667d53ab69,3048 9 | pantry-tree: 10 | size: 986 11 | sha256: b23e85a7cc928c2664b4c162053fae57fa976d5a6c69806f2edbdc7fb62e109f 12 | original: 13 | hackage: protobuf-0.2.1.3 14 | - completed: 15 | hackage: diskhash-0.0.4.0@sha256:a63094c8be9e9a06c2d225630cad84f973507e1b3461ff709a2e877e7e8b86c4,1282 16 | pantry-tree: 17 | size: 666 18 | sha256: b498f95c8c268434338691ffe72c8dd27c7e40aff0c737ac258e82ac0dc922a7 19 | original: 20 | hackage: diskhash-0.0.4.0 21 | - completed: 22 | hackage: avro-0.4.7.0@sha256:22685c55833bee1dbb08e93a1cab5c866254d8c877a0c0ad667cbffa1972b9d8,12403 23 | pantry-tree: 24 | size: 6566 25 | sha256: 86dd3c331759e2dc56f623571e2387a82c775c7a2f246c07c6e256d578f2db56 26 | original: 27 | hackage: avro-0.4.7.0 28 | - completed: 29 | subdir: hanalytics-avro 30 | name: hanalytics-avro 31 | version: 0.1.0.0 32 | git: https://github.com/quyse/hanalytics.git 33 | pantry-tree: 34 | size: 178 35 | sha256: aa06a9a0f7432be35128f7f7b395986fc1b98430f39439ed3d3c1fc45d687b91 36 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 37 | original: 38 | subdir: hanalytics-avro 39 | git: https://github.com/quyse/hanalytics.git 40 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 41 | - completed: 42 | subdir: hanalytics-base 43 | name: hanalytics-base 44 | version: 0.1.0.0 45 | git: https://github.com/quyse/hanalytics.git 46 | pantry-tree: 47 | size: 173 48 | sha256: 04e16388aa9f92122a5e13add2548f319240eab27b3cfb4cd0784f9edb4a2ecd 49 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 50 | original: 51 | subdir: hanalytics-base 52 | git: https://github.com/quyse/hanalytics.git 53 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 54 | - completed: 55 | subdir: hanalytics-bigquery 56 | name: hanalytics-bigquery 57 | version: 0.1.0.0 58 | git: https://github.com/quyse/hanalytics.git 59 | pantry-tree: 60 | size: 186 61 | sha256: e91df7bc379ad8f2021acd820ef7e97344a00983a76226261b1633cec862113f 62 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 63 | original: 64 | subdir: hanalytics-bigquery 65 | git: https://github.com/quyse/hanalytics.git 66 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 67 | - completed: 68 | subdir: hanalytics-postgres 69 | name: hanalytics-postgres 70 | version: 0.1.0.0 71 | git: https://github.com/quyse/hanalytics.git 72 | pantry-tree: 73 | size: 186 74 | sha256: 2e3687e2fd014df31eeaa39d5aad8c0808d808d011c5579eca28d3c5c1b97958 75 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 76 | original: 77 | subdir: hanalytics-postgres 78 | git: https://github.com/quyse/hanalytics.git 79 | commit: dbe294c4c1524268683764c3cd3ec947105359c9 80 | snapshots: 81 | - completed: 82 | size: 532832 83 | url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/16/23.yaml 84 | sha256: fbb2a0519008533924c7753bd7164ddd1009f09504eb06674acad6049b46db09 85 | original: lts-16.23 86 | --------------------------------------------------------------------------------