├── .gitignore ├── LICENSE ├── README.md ├── pom.xml ├── src ├── assembly │ └── assembly.xml └── main │ ├── java │ └── im │ │ └── xiaoyao │ │ └── presto │ │ └── ethereum │ │ ├── EthereumBlockRange.java │ │ ├── EthereumColumnHandle.java │ │ ├── EthereumConnector.java │ │ ├── EthereumConnectorConfig.java │ │ ├── EthereumConnectorFactory.java │ │ ├── EthereumConnectorId.java │ │ ├── EthereumConnectorModule.java │ │ ├── EthereumERC20Token.java │ │ ├── EthereumERC20Utils.java │ │ ├── EthereumHandleResolver.java │ │ ├── EthereumLogLazyIterator.java │ │ ├── EthereumMetadata.java │ │ ├── EthereumPlugin.java │ │ ├── EthereumRecordCursor.java │ │ ├── EthereumRecordSet.java │ │ ├── EthereumRecordSetProvider.java │ │ ├── EthereumSplit.java │ │ ├── EthereumSplitManager.java │ │ ├── EthereumTable.java │ │ ├── EthereumTableHandle.java │ │ ├── EthereumTableLayoutHandle.java │ │ ├── EthereumTransactionHandle.java │ │ ├── EthereumWeb3jProvider.java │ │ └── udfs │ │ ├── EthereumUDFs.java │ │ └── EthereumUnit.java │ └── resources │ └── META-INF │ └── services │ └── com.facebook.presto.spi.Plugin └── use-cases.sql /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.ear 17 | *.zip 18 | *.tar.gz 19 | *.rar 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | 24 | target/ 25 | pom.xml.tag 26 | pom.xml.releaseBackup 27 | pom.xml.versionsBackup 28 | pom.xml.next 29 | release.properties 30 | dependency-reduced-pom.xml 31 | buildNumber.properties 32 | .mvn/timing.properties 33 | 34 | # Avoid ignoring Maven wrapper jar file (.jar files are usually ignored) 35 | !/.mvn/wrapper/maven-wrapper.jar 36 | 37 | .idea/ 38 | *.iml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Presto Ethereum Connector 2 | Unleash the Power of Presto Interactive SQL Querying on Ethereum Blockchain 3 | 4 | ### Introduction 5 | [Presto](https://prestosql.io) is a powerful interactive querying engine that enables running SQL queries on anything -- be it MySQL, HDFS, local file, Kafka -- as long as there exist a connector to the source. 6 | 7 | This is a Presto connector to the Ethereum blockchain data. With this connector, one can get hands on with Ethereum blockchain analytics work without having to know how to play with the nitty gritty Javascript API. 8 | 9 | ### Prerequisites 10 | Have an Ethereum client that you can connect to. There are 2 options: 11 | 1. Run [Geth](https://github.com/ethereum/go-ethereum) or [Parity](https://github.com/paritytech/parity) locally. 12 | 1. Use [Infura](https://infura.io), a hosted Ethereum client in the cloud. 13 | 14 | ### Note 15 | Specify a block range where you can (e.g. `WHERE block.block_number > x AND block.block_number < y`, or `WHERE transaction.tx_blocknumber > x AND transaction.tx_blocknumber < y`, or `WHERE erc20.erc20_blocknumber > x AND erc20.erc20_blocknumber < y`). Block number is the default and only predicate that can push down to narrow down data scan range. Queries without block ranges will cause presto to retrieve blocks all the way from the first block, which takes forever. 16 | 17 | ### Usage 18 | 1. [Install Presto](https://prestosql.io/docs/current/installation/deployment.html). *Follow the instructions on that page to create relevant config files.* 19 | By the end of this step, your presto installation folder structure should look like: 20 | ``` 21 | ├── bin 22 | ├── lib 23 | ├── etc 24 | │   ├── config.properties 25 | │   ├── jvm.config 26 | │   └── node.properties 27 | ├── plugin 28 | ``` 29 | 1. [Install Presto CLI](https://prestosql.io/docs/current/installation/cli.html) 30 | 1. Clone this repo and run `mvn clean package` to build the plugin. You will find the built plugin in the `target` folder. 31 | 1. Load the plugin to Presto 32 | a. Create the ethereum connector config inside of `etc`. 33 | `$ mkdir -p etc/catalog && touch etc/catalog/ethereum.properties` 34 | Paste the following to the ethereum.properties: 35 | ``` 36 | connector.name=ethereum 37 | 38 | # You can connect through Ethereum HTTP JSON RPC endpoint 39 | # IMPORTANT - for local testing start geth with rpcport 40 | # geth --rpc --rpcaddr "127.0.0.1" --rpcport "8545" 41 | ethereum.jsonrpc=http://localhost:8545/ 42 | 43 | 44 | # Or you can connect through IPC socket 45 | # ethereum.ipc=/path/to/ipc_socketfile 46 | 47 | # Or you can connect to Infura 48 | # ethereum.infura=https://mainnet.infura.io/ 49 | ``` 50 | b. Copy and extract the built plugin to your presto plugin folder 51 | ``` 52 | $ mkdir -p plugin/ethereum \ 53 | && cp /target/presto-ethereum-*-plugin.tar.gz . \ 54 | && tar xfz presto-ethereum-*-plugin.tar.gz -C plugin/ethereum --strip-components=1 55 | ``` 56 | 57 | By the end of this step, your presto installation folder structure should look like: 58 | ``` 59 | ├── bin 60 | ├── lib 61 | ├── etc 62 | │   ├── catalog 63 | │   │   └── ethereum.properties 64 | │   ├── config.properties 65 | │   ├── jvm.config 66 | │   └── node.properties 67 | ├── plugin 68 | │   ├── ethereum 69 | │   │   └── 70 | ``` 71 | 1. There you go. You can now start the presto server, and query through presto-cli: 72 | ``` 73 | $ bin/launcher start 74 | $ presto-cli --server localhost:8080 --catalog ethereum --schema default 75 | ``` 76 | 77 | ### Use Cases 78 | Inspired by [An Analysis of the First 100000 Blocks](https://blog.ethereum.org/2015/08/18/frontier-first-100k-blocks/), the following SQL queries capture partially what was depicted in that post. 79 | 80 | - The first 50 block times (in seconds) 81 | ```sql 82 | SELECT b.bn, (b.block_timestamp - a.block_timestamp) AS delta 83 | FROM 84 | (SELECT block_number AS bn, block_timestamp 85 | FROM block 86 | WHERE block_number>=1 AND block_number<=50) AS a 87 | JOIN 88 | (SELECT (block_number-1) AS bn, block_timestamp 89 | FROM block 90 | WHERE block_number>=2 AND block_number<=51) AS b 91 | ON a.bn=b.bn 92 | ORDER BY b.bn; 93 | ``` 94 | - Average block time (every 200th block from genesis to block 10000) 95 | ```sql 96 | WITH 97 | X AS (SELECT b.bn, (b.block_timestamp - a.block_timestamp) AS delta 98 | FROM 99 | (SELECT block_number AS bn, block_timestamp 100 | FROM block 101 | WHERE block_number>=1 AND block_number<=10000) AS a 102 | JOIN 103 | (SELECT (block_number-1) AS bn, block_timestamp 104 | FROM block 105 | WHERE block_number>=2 AND block_number<=10001) AS b 106 | ON a.bn=b.bn 107 | ORDER BY b.bn) 108 | SELECT min(bn) AS chunkStart, avg(delta) 109 | FROM 110 | (SELECT ntile(10000/200) OVER (ORDER BY bn) AS chunk, * FROM X) AS T 111 | GROUP BY chunk 112 | ORDER BY chunkStart; 113 | ``` 114 | - Biggest miners in first 100k blocks (address, blocks, %) 115 | ```sql 116 | SELECT block_miner, count(*) AS num, count(*)/100000.0 AS PERCENT 117 | FROM block 118 | WHERE block_number<=100000 119 | GROUP BY block_miner 120 | ORDER BY num DESC 121 | LIMIT 15; 122 | ``` 123 | - ERC20 Token Movement in the last 100 blocks 124 | ```sql 125 | SELECT erc20_token, SUM(erc20_value) FROM erc20 126 | WHERE erc20_blocknumber >= 4147340 AND erc20_blocknumber<=4147350 127 | GROUP BY erc20_token; 128 | ``` 129 | - Describe the database structure 130 | ```sql 131 | SHOW TABLES; 132 | Table 133 | ------------- 134 | block 135 | erc20 136 | transaction 137 | 138 | DESCRIBE block; 139 | Column | Type | Extra | Comment 140 | ----------------------------------------------------------- 141 | block_number | bigint | | 142 | block_hash | varchar(66) | | 143 | block_parenthash | varchar(66) | | 144 | block_nonce | varchar(18) | | 145 | block_sha3uncles | varchar(66) | | 146 | block_logsbloom | varchar(514) | | 147 | block_transactionsroot | varchar(66) | | 148 | block_stateroot | varchar(66) | | 149 | block_miner | varchar(42) | | 150 | block_difficulty | bigint | | 151 | block_totaldifficulty | bigint | | 152 | block_size | integer | | 153 | block_extradata | varchar | | 154 | block_gaslimit | double | | 155 | block_gasused | double | | 156 | block_timestamp | bigint | | 157 | block_transactions | array(varchar(66)) | | 158 | block_uncles | array(varchar(66)) | | 159 | 160 | 161 | DESCRIBE transaction; 162 | 163 | Column | Type | Extra | Comment 164 | -------------------------------------------------- 165 | tx_hash | varchar(66) | | 166 | tx_nonce | bigint | | 167 | tx_blockhash | varchar(66) | | 168 | tx_blocknumber | bigint | | 169 | tx_transactionindex | integer | | 170 | tx_from | varchar(42) | | 171 | tx_to | varchar(42) | | 172 | tx_value | double | | 173 | tx_gas | double | | 174 | tx_gasprice | double | | 175 | tx_input | varchar | | 176 | 177 | 178 | DESCRIBE erc20; 179 | Column | Type | Extra | Comment 180 | -------------------+-------------+-------+--------- 181 | erc20_token | varchar | | 182 | erc20_from | varchar(42) | | 183 | erc20_to | varchar(42) | | 184 | erc20_value | double | | 185 | erc20_txhash | varchar(66) | | 186 | erc20_blocknumber | bigint | | 187 | ``` 188 | 189 | ### Web3 Functions 190 | In addition to the various built-in [Presto functions](https://prestodb.io/docs/current/functions.html), some web3 functions are ported so that they can be called inline with SQL statements directly. Currently, the supported web3 functions are 191 | 1. [fromWei](https://github.com/ethereum/wiki/wiki/JavaScript-API#web3fromwei) 192 | 1. [toWei](https://github.com/ethereum/wiki/wiki/JavaScript-API#web3towei) 193 | 1. [eth_gasPrice](https://github.com/ethereum/wiki/wiki/JavaScript-API#web3ethgasprice) 194 | 1. [eth_blockNumber](https://github.com/ethereum/wiki/wiki/JavaScript-API#web3ethblocknumber) 195 | 1. [eth_getBalance](https://github.com/ethereum/wiki/wiki/JavaScript-API#web3ethgetbalance) 196 | 1. [eth_getTransactionCount](https://github.com/ethereum/wiki/wiki/JavaScript-API#web3ethgettransactioncount) 197 | 198 | ### Troubleshooting 199 | 200 | * You must use python2. You will get invalid syntax errors if you use Python3. 201 | ``` 202 | -> bin/launcher start 203 | File "/your_path/presto-server-0.196/bin/launcher.py", line 38 204 | except OSError, e: 205 | ^ 206 | SyntaxError: invalid syntax 207 | ``` 208 | 209 | * Use Java 8 only. You might get the following errors if you use the wrong Java version. 210 | 211 | ``` 212 | Unrecognized VM option 'ExitOnOutOfMemoryError' 213 | Did you mean 'OnOutOfMemoryError='? 214 | Error: Could not create the Java Virtual Machine. 215 | Error: A fatal exception has occurred. Program will exit. 216 | ``` 217 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | im.xiaoyao 8 | presto-ethereum 9 | 1.0-SNAPSHOT 10 | jar 11 | 12 | 13 | 0.271 14 | 5.0.0 15 | 4.0 16 | 0.148 17 | 1.16.18 18 | 1.3.2 19 | 20 | 21 | 22 | 23 | com.facebook.presto 24 | presto-spi 25 | provided 26 | ${version.presto} 27 | 28 | 29 | 30 | com.facebook.presto 31 | presto-common 32 | provided 33 | ${version.presto} 34 | 35 | 36 | 37 | org.web3j 38 | core 39 | ${version.web3j} 40 | 41 | 42 | 43 | org.web3j 44 | hosted-providers 45 | ${version.web3j} 46 | 47 | 48 | 49 | com.google.inject 50 | guice 51 | ${version.guice} 52 | 53 | 54 | 55 | com.google.inject.extensions 56 | guice-multibindings 57 | ${version.guice} 58 | 59 | 60 | 61 | io.airlift 62 | configuration 63 | ${version.airlift} 64 | 65 | 66 | 67 | io.airlift 68 | log 69 | ${version.airlift} 70 | 71 | 72 | 73 | io.airlift 74 | bootstrap 75 | ${version.airlift} 76 | 77 | 78 | 79 | io.airlift 80 | json 81 | ${version.airlift} 82 | 83 | 84 | 85 | org.projectlombok 86 | lombok 87 | ${version.lombok} 88 | 89 | 90 | 91 | javax.annotation 92 | javax.annotation-api 93 | ${version.javax} 94 | 95 | 96 | 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-compiler-plugin 102 | 103 | 1.8 104 | 1.8 105 | 106 | 107 | 108 | 109 | org.apache.maven.plugins 110 | maven-assembly-plugin 111 | 112 | 113 | src/assembly/assembly.xml 114 | 115 | 116 | 117 | 118 | package 119 | 120 | single 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /src/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | plugin 5 | 6 | tar.gz 7 | 8 | 9 | 10 | ${project.basedir} 11 | / 12 | 13 | README* 14 | LICENSE* 15 | NOTICE* 16 | 17 | 18 | 19 | ${project.build.directory}/site 20 | docs 21 | 22 | 23 | 24 | 25 | / 26 | true 27 | runtime 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumBlockRange.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.common.predicate.Marker; 4 | import com.fasterxml.jackson.annotation.JsonCreator; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | public class EthereumBlockRange { 8 | private final long startBlock; 9 | private final long endBlock; 10 | 11 | public static EthereumBlockRange fromMarkers(Marker low, Marker high) { 12 | long startBlock; 13 | long endBlock; 14 | if (low.isLowerUnbounded()) { 15 | startBlock = 1L; 16 | } else if (low.getBound() == Marker.Bound.EXACTLY) { 17 | startBlock = (long) low.getValue(); 18 | } else if (low.getBound() == Marker.Bound.ABOVE) { 19 | startBlock = (long) low.getValue() + 1L; 20 | } else { 21 | throw new IllegalArgumentException("Low bound cannot be BELOW"); 22 | } 23 | 24 | if (high.isUpperUnbounded()) { 25 | endBlock = -1L; 26 | } else if (high.getBound() == Marker.Bound.EXACTLY) { 27 | endBlock = (long) high.getValue(); 28 | } else if (high.getBound() == Marker.Bound.BELOW) { 29 | endBlock = (long) high.getValue() - 1L; 30 | } else { 31 | throw new IllegalArgumentException("High bound cannot be ABOVE"); 32 | } 33 | 34 | if (startBlock > endBlock && endBlock != -1L) { 35 | throw new IllegalArgumentException("Low bound is greater than high bound"); 36 | } 37 | 38 | return new EthereumBlockRange(startBlock, endBlock); 39 | } 40 | 41 | @JsonCreator 42 | public EthereumBlockRange( 43 | @JsonProperty("startBlock") long startBlock, 44 | @JsonProperty("endBlock") long endBlock 45 | ) { 46 | this.startBlock = startBlock; 47 | this.endBlock = endBlock; 48 | } 49 | 50 | @JsonProperty 51 | public long getStartBlock() { 52 | return startBlock; 53 | } 54 | 55 | @JsonProperty 56 | public long getEndBlock() { 57 | return endBlock; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumColumnHandle.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ColumnHandle; 4 | import com.facebook.presto.spi.ColumnMetadata; 5 | import com.facebook.presto.common.type.Type; 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | 11 | import static java.util.Objects.requireNonNull; 12 | 13 | @EqualsAndHashCode 14 | @ToString 15 | public final class EthereumColumnHandle implements ColumnHandle { 16 | private final String connectorId; 17 | private final int ordinalPosition; 18 | 19 | private final String name; 20 | private final Type type; 21 | 22 | @JsonCreator 23 | public EthereumColumnHandle( 24 | @JsonProperty("connectorId") String connectorId, 25 | @JsonProperty("ordinalPosition") int ordinalPosition, 26 | @JsonProperty("name") String name, 27 | @JsonProperty("type") Type type 28 | ) { 29 | this.connectorId = requireNonNull(connectorId, "connectorId is null"); 30 | this.ordinalPosition = ordinalPosition; 31 | this.name = requireNonNull(name, "name is null"); 32 | this.type = requireNonNull(type, "type is null"); 33 | } 34 | 35 | @JsonProperty 36 | public String getConnectorId() { 37 | return connectorId; 38 | } 39 | 40 | @JsonProperty 41 | public int getOrdinalPosition() { 42 | return ordinalPosition; 43 | } 44 | 45 | @JsonProperty 46 | public String getName() { 47 | return name; 48 | } 49 | 50 | @JsonProperty 51 | public Type getType() { 52 | return type; 53 | } 54 | 55 | ColumnMetadata getColumnMetadata() { 56 | return new ColumnMetadata(name, type); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumConnector.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.connector.Connector; 4 | import com.facebook.presto.spi.connector.ConnectorMetadata; 5 | import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; 6 | import com.facebook.presto.spi.connector.ConnectorSplitManager; 7 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 8 | import com.facebook.presto.spi.transaction.IsolationLevel; 9 | import io.airlift.bootstrap.LifeCycleManager; 10 | import io.airlift.log.Logger; 11 | 12 | import javax.inject.Inject; 13 | 14 | import static com.facebook.presto.spi.transaction.IsolationLevel.READ_COMMITTED; 15 | import static com.facebook.presto.spi.transaction.IsolationLevel.checkConnectorSupports; 16 | import static java.util.Objects.requireNonNull; 17 | 18 | public class EthereumConnector implements Connector { 19 | private static final Logger log = Logger.get(EthereumConnector.class); 20 | 21 | private final LifeCycleManager lifeCycleManager; 22 | private final EthereumMetadata metadata; 23 | private final EthereumSplitManager splitManager; 24 | private final EthereumRecordSetProvider recordSetProvider; 25 | 26 | @Inject 27 | public EthereumConnector( 28 | LifeCycleManager lifeCycleManager, 29 | EthereumMetadata metadata, 30 | EthereumSplitManager splitManager, 31 | EthereumRecordSetProvider recordSetProvider 32 | ) { 33 | this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); 34 | this.metadata = requireNonNull(metadata, "metadata is null"); 35 | this.splitManager = requireNonNull(splitManager, "splitManager is null"); 36 | this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); 37 | } 38 | 39 | @Override 40 | public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { 41 | checkConnectorSupports(READ_COMMITTED, isolationLevel); 42 | return EthereumTransactionHandle.INSTANCE; 43 | } 44 | 45 | @Override 46 | public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) { 47 | return metadata; 48 | } 49 | 50 | @Override 51 | public ConnectorSplitManager getSplitManager() { 52 | return splitManager; 53 | } 54 | 55 | @Override 56 | public ConnectorRecordSetProvider getRecordSetProvider() { 57 | return recordSetProvider; 58 | } 59 | 60 | @Override 61 | public final void shutdown() { 62 | try { 63 | lifeCycleManager.stop(); 64 | } catch (Exception e) { 65 | log.error("Error shutting down connector"); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import io.airlift.configuration.Config; 4 | 5 | public class EthereumConnectorConfig { 6 | public static final String DEFAULT_JSON_RPC = "http://localhost:8545/"; 7 | private String ethereumJsonRpc; 8 | private String ethereumIpc; 9 | private String infuraRpc; 10 | 11 | @Config("ethereum.jsonrpc") 12 | public EthereumConnectorConfig setEthereumJsonRpc(String ethereumJsonRpc) { 13 | this.ethereumJsonRpc = ethereumJsonRpc; 14 | return this; 15 | } 16 | 17 | public String getEthereumJsonRpc() { 18 | return ethereumJsonRpc; 19 | } 20 | 21 | @Config("ethereum.ipc") 22 | public EthereumConnectorConfig setEthereumIpc(String ethereumIpc) { 23 | this.ethereumIpc = ethereumIpc; 24 | return this; 25 | } 26 | 27 | public String getEthereumIpc() { 28 | return ethereumIpc; 29 | } 30 | 31 | @Config("ethereum.infura") 32 | public EthereumConnectorConfig setInfuraRpc(String infuraRpc) { 33 | this.infuraRpc = infuraRpc; 34 | return this; 35 | } 36 | 37 | public String getInfuraRpc() { 38 | return infuraRpc; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumConnectorFactory.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ConnectorHandleResolver; 4 | import com.facebook.presto.spi.NodeManager; 5 | import com.facebook.presto.spi.connector.Connector; 6 | import com.facebook.presto.spi.connector.ConnectorContext; 7 | import com.facebook.presto.spi.connector.ConnectorFactory; 8 | import com.facebook.presto.common.type.TypeManager; 9 | import com.google.common.base.Throwables; 10 | import com.google.inject.Injector; 11 | import io.airlift.bootstrap.Bootstrap; 12 | 13 | import java.util.Map; 14 | 15 | import static java.util.Objects.requireNonNull; 16 | 17 | public class EthereumConnectorFactory implements ConnectorFactory { 18 | @Override 19 | public String getName() { 20 | return "ethereum"; 21 | } 22 | 23 | @Override 24 | public ConnectorHandleResolver getHandleResolver() { 25 | return new EthereumHandleResolver(); 26 | } 27 | 28 | @Override 29 | public Connector create(String connectorId, Map config, ConnectorContext context) { 30 | requireNonNull(connectorId, "connectorId is null"); 31 | requireNonNull(config, "config is null"); 32 | 33 | try { 34 | Bootstrap app = new Bootstrap( 35 | // new JsonModule(), 36 | new EthereumConnectorModule(), 37 | binder -> { 38 | binder.bind(EthereumConnectorId.class).toInstance(new EthereumConnectorId(connectorId)); 39 | binder.bind(TypeManager.class).toInstance(context.getTypeManager()); 40 | binder.bind(NodeManager.class).toInstance(context.getNodeManager()); 41 | } 42 | ); 43 | 44 | Injector injector = app.strictConfig() 45 | .doNotInitializeLogging() 46 | .setRequiredConfigurationProperties(config) 47 | .initialize(); 48 | 49 | return injector.getInstance(EthereumConnector.class); 50 | } 51 | catch (Exception e) { 52 | throw Throwables.propagate(e); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumConnectorId.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import lombok.EqualsAndHashCode; 4 | import lombok.ToString; 5 | 6 | import static java.util.Objects.requireNonNull; 7 | 8 | @EqualsAndHashCode 9 | @ToString 10 | public class EthereumConnectorId { 11 | private final String connectorId; 12 | 13 | public EthereumConnectorId(String connectorId) { 14 | this.connectorId = requireNonNull(connectorId, "connectorId is null"); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumConnectorModule.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.common.type.Type; 4 | import com.facebook.presto.common.type.TypeManager; 5 | import com.fasterxml.jackson.databind.DeserializationContext; 6 | import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; 7 | import com.google.inject.Binder; 8 | import com.google.inject.Module; 9 | import com.google.inject.Scopes; 10 | 11 | import javax.inject.Inject; 12 | 13 | import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; 14 | import static com.google.common.base.Preconditions.checkArgument; 15 | import static io.airlift.configuration.ConfigBinder.configBinder; 16 | import static io.airlift.json.JsonBinder.jsonBinder; 17 | import static java.util.Objects.requireNonNull; 18 | 19 | public class EthereumConnectorModule implements Module { 20 | @Override 21 | public void configure(Binder binder) { 22 | binder.bind(EthereumConnector.class).in(Scopes.SINGLETON); 23 | binder.bind(EthereumMetadata.class).in(Scopes.SINGLETON); 24 | binder.bind(EthereumWeb3jProvider.class).in(Scopes.SINGLETON); 25 | 26 | binder.bind(EthereumSplitManager.class).in(Scopes.SINGLETON); 27 | binder.bind(EthereumRecordSetProvider.class).in(Scopes.SINGLETON); 28 | 29 | configBinder(binder).bindConfig(EthereumConnectorConfig.class); 30 | jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); 31 | } 32 | 33 | public static final class TypeDeserializer extends FromStringDeserializer { 34 | private static final long serialVersionUID = 1L; 35 | 36 | private final TypeManager typeManager; 37 | 38 | @Inject 39 | public TypeDeserializer(TypeManager typeManager) { 40 | super(Type.class); 41 | this.typeManager = requireNonNull(typeManager, "typeManager is null"); 42 | } 43 | 44 | @Override 45 | protected Type _deserialize(String value, DeserializationContext context) { 46 | Type type = typeManager.getType(parseTypeSignature(value)); 47 | checkArgument(type != null, "Unknown type %s", value); 48 | return type; 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumERC20Token.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import lombok.Getter; 4 | 5 | import java.util.Arrays; 6 | import java.util.Map; 7 | import java.util.stream.Collectors; 8 | 9 | public enum EthereumERC20Token { 10 | QTUM("0x9a642d6b3368ddc662CA244bAdf32cDA716005BC"), 11 | BCAP("0xff3519eeeea3e76f1f699ccce5e23ee0bdda41ac"), 12 | Pluton("0xD8912C10681D8B21Fd3742244f44658dBA12264E"), 13 | NimiqNetwork("0xcfb98637bcae43C13323EAa1731cED2B716962fD"), 14 | SwarmCity("0xb9e7f8568e08d5659f5d29c4997173d84cdf2607"), 15 | Guppy("0xf7b098298f7c69fc14610bf71d5e02c60792894c"), 16 | TIME("0x6531f133e6deebe7f2dce5a0441aa7ef330b4e53"), 17 | SAN("0x7c5a0ce9267ed19b22f8cae653f198e3e8daf098"), 18 | Xaurum("0x4DF812F6064def1e5e029f1ca858777CC98D2D81"), 19 | TAAS("0xe7775a6e9bcf904eb39da2b68c5efb4f9360e08c"), 20 | Trustcoin("0xcb94be6f13a1182e4a4b6140cb7bf2025d28e41b"), 21 | Humaniq("0xcbcc0f036ed4788f63fc0fee32873d6a7487b908"), 22 | TokenCard("0xaaaf91d9b90df800df4f55c205fd6989c977e73a"), 23 | Lunyr("0xfa05A73FfE78ef8f1a739473e462c54bae6567D9"), 24 | Monaco("0xb63b606ac810a52cca15e44bb630fd42d8d1d83d"), 25 | vSlice("0x5c543e7AE0A1104f78406C340E9C64FD9fCE5170"), 26 | Bitquence("0x5af2be193a6abca9c8817001f45744777db30756"), 27 | Edgeless("0x08711d3b02c8758f2fb3ab4e80228418a7f8e39c"), 28 | AdToken("0xd0d6d6c5fe4a677d343cc433536bb717bae167dd"), 29 | district0x("0x0abdace70d3790235af448c88547603b945604ea"), 30 | Melon("0xBEB9eF514a379B997e0798FDcC901Ee474B6D9A1"), 31 | RLC("0x607F4C5BB672230e8672085532f7e901544a7375"), 32 | WINGS("0x667088b212ce3d06a1b553a7221E1fD19000d9aF"), 33 | DICE("0x2e071D2966Aa7D8dECB1005885bA1977D6038A65"), 34 | FirstBlood("0xaf30d2a7e90d7dc361c8c4585e9bb7d2f6f15bc7"), 35 | Aragon("0x960b236A07cf122663c4303350609A66A7B288C0"), 36 | Bancor("0x1f573d6fb3f13d689ff844b4ce37794d79a7ff1c"), 37 | FunFair("0x419d0d8bdd9af5e606ae2232ed285aff190e711b"), 38 | SNGLS("0xaec2e87e0a235266d9c5adc9deb4b2e29b54d009"), 39 | Storj("0xb64ef51c888972c908cfacf59b47c1afbc0ab8ac"), 40 | DGD("0xe0b7927c4af23765cb51314a0e0521a9645f0e2a"), 41 | Civic("0x41e5560054824ea6b0732e656e3ad64e20e94e45"), 42 | BAT("0x0d8775f648430679a709e98d2b0cb6250d2887ef"), 43 | MKR("0xc66ea802717bfb9833400264dd12c2bceaa34a6d"), 44 | Gnosis("0x6810e776880c02933d47db1b9fc05908e5386b96"), 45 | REP("0xe94327d07fc17907b4db788e5adf2ed424addff6"), 46 | StatusNetwork("0x744d70fdbe2ba4cf95131626614a1763df805b9e"), 47 | Golem("0xa74476443119A942dE498590Fe1f2454d7D4aC0d"), 48 | ICONOMI("0x888666CA69E0f178DED6D75b5726Cee99A87D698"), 49 | TenXPay("0xB97048628DB6B661D4C2aA833e95Dbe1A905B280"), 50 | OmiseGo("0xd26114cd6EE289AccF82350c8d8487fedB8A0C07"), 51 | EOS("0x86fa049857e0209aa7d9e616f7eb3b3b78ecfdb0"); 52 | 53 | public static final Map lookup = Arrays.stream(EthereumERC20Token.values()) 54 | .collect(Collectors.toMap(EthereumERC20Token::getTokenContractAddr, ethereumERC20Token -> ethereumERC20Token)); 55 | 56 | @Getter private final String tokenContractAddr; 57 | EthereumERC20Token(String tokenContractAddr) { 58 | this.tokenContractAddr = tokenContractAddr.toLowerCase(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumERC20Utils.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import java.math.BigInteger; 4 | 5 | public class EthereumERC20Utils { 6 | public static final String TRANSFER_EVENT_TOPIC = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; 7 | 8 | public static double hexToDouble(String hex) { 9 | try { 10 | return new BigInteger(hex.substring(2), 16).doubleValue(); 11 | } catch (NumberFormatException e) { 12 | return 0.0; //TEMP FIX in case of ERC-721 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumHandleResolver.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ColumnHandle; 4 | import com.facebook.presto.spi.ConnectorHandleResolver; 5 | import com.facebook.presto.spi.ConnectorSplit; 6 | import com.facebook.presto.spi.ConnectorTableHandle; 7 | import com.facebook.presto.spi.ConnectorTableLayoutHandle; 8 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 9 | 10 | import static com.google.common.base.Preconditions.checkArgument; 11 | import static java.util.Objects.requireNonNull; 12 | 13 | public class EthereumHandleResolver implements ConnectorHandleResolver { 14 | @Override 15 | public Class getTableHandleClass() { 16 | return EthereumTableHandle.class; 17 | } 18 | 19 | @Override 20 | public Class getColumnHandleClass() { 21 | return EthereumColumnHandle.class; 22 | } 23 | 24 | @Override 25 | public Class getSplitClass() { 26 | return EthereumSplit.class; 27 | } 28 | 29 | @Override 30 | public Class getTableLayoutHandleClass() { 31 | return EthereumTableLayoutHandle.class; 32 | } 33 | 34 | @Override 35 | public Class getTransactionHandleClass() { 36 | return EthereumTransactionHandle.class; 37 | } 38 | 39 | static EthereumTableHandle convertTableHandle(ConnectorTableHandle tableHandle) { 40 | requireNonNull(tableHandle, "tableHandle is null"); 41 | checkArgument(tableHandle instanceof EthereumTableHandle, "tableHandle is not an instance of EthereumTableHandle"); 42 | return (EthereumTableHandle) tableHandle; 43 | } 44 | 45 | static EthereumColumnHandle convertColumnHandle(ColumnHandle columnHandle) { 46 | requireNonNull(columnHandle, "columnHandle is null"); 47 | checkArgument(columnHandle instanceof EthereumColumnHandle, "columnHandle is not an instance of EthereumColumnHandle"); 48 | return (EthereumColumnHandle) columnHandle; 49 | } 50 | 51 | static EthereumSplit convertSplit(ConnectorSplit split) { 52 | requireNonNull(split, "split is null"); 53 | checkArgument(split instanceof EthereumSplit, "split is not an instance of EthereumSplit"); 54 | return (EthereumSplit) split; 55 | } 56 | 57 | static EthereumTableLayoutHandle convertLayout(ConnectorTableLayoutHandle layout) { 58 | requireNonNull(layout, "layout is null"); 59 | checkArgument(layout instanceof EthereumTableLayoutHandle, "layout is not an instance of EthereumTableLayoutHandle"); 60 | return (EthereumTableLayoutHandle) layout; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumLogLazyIterator.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import io.airlift.log.Logger; 4 | import org.web3j.protocol.Web3j; 5 | import org.web3j.protocol.core.methods.response.EthBlock; 6 | import org.web3j.protocol.core.methods.response.Log; 7 | import org.web3j.protocol.core.methods.response.TransactionReceipt; 8 | 9 | import java.io.IOException; 10 | import java.util.Iterator; 11 | import java.util.Optional; 12 | 13 | /** 14 | * Created by xiaoyaoqian on 2/14/18. 15 | */ 16 | public class EthereumLogLazyIterator implements Iterator { 17 | private static final Logger log = Logger.get(EthereumRecordCursor.class); 18 | 19 | private final Iterator txIter; 20 | private Iterator logIter; 21 | private final Web3j web3j; 22 | 23 | public EthereumLogLazyIterator(EthBlock block, Web3j web3j) { 24 | this.txIter = block.getBlock().getTransactions().iterator(); 25 | this.web3j = web3j; 26 | } 27 | 28 | @Override 29 | public boolean hasNext() { 30 | if (logIter != null && logIter.hasNext()) { 31 | return true; 32 | } 33 | 34 | while (txIter.hasNext()) { 35 | EthBlock.TransactionResult tr = txIter.next(); 36 | EthBlock.TransactionObject tx = (EthBlock.TransactionObject) tr.get(); 37 | try { 38 | log.info("Getting tx receipts..."); 39 | Optional transactionReceiptOptional = web3j.ethGetTransactionReceipt(tx.getHash()) 40 | .send() 41 | .getTransactionReceipt() 42 | .filter(receipt -> receipt.getLogs() != null && !receipt.getLogs().isEmpty()); 43 | if (!transactionReceiptOptional.isPresent()) { 44 | continue; 45 | } 46 | 47 | this.logIter = transactionReceiptOptional.get().getLogs().iterator(); 48 | return true; 49 | } catch (IOException e) { 50 | throw new IllegalStateException("Unable to get transactionReceipt"); 51 | } 52 | 53 | } 54 | 55 | return false; 56 | } 57 | 58 | @Override 59 | public Log next() { 60 | return logIter.next(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumMetadata.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ColumnHandle; 4 | import com.facebook.presto.spi.ColumnMetadata; 5 | import com.facebook.presto.spi.ConnectorSession; 6 | import com.facebook.presto.spi.ConnectorTableHandle; 7 | import com.facebook.presto.spi.ConnectorTableLayout; 8 | import com.facebook.presto.spi.ConnectorTableLayoutHandle; 9 | import com.facebook.presto.spi.ConnectorTableLayoutResult; 10 | import com.facebook.presto.spi.ConnectorTableMetadata; 11 | import com.facebook.presto.spi.Constraint; 12 | import com.facebook.presto.spi.SchemaTableName; 13 | import com.facebook.presto.spi.SchemaTablePrefix; 14 | import com.facebook.presto.spi.connector.ConnectorMetadata; 15 | import com.facebook.presto.common.predicate.Domain; 16 | import com.facebook.presto.common.predicate.Marker; 17 | import com.facebook.presto.common.predicate.Range; 18 | import com.facebook.presto.common.type.ArrayType; 19 | import com.facebook.presto.common.type.BigintType; 20 | import com.facebook.presto.common.type.DoubleType; 21 | import com.facebook.presto.common.type.IntegerType; 22 | import com.facebook.presto.common.type.VarcharType; 23 | import com.google.common.collect.ImmutableList; 24 | import com.google.common.collect.ImmutableMap; 25 | import io.airlift.log.Logger; 26 | import io.airlift.slice.Slice; 27 | import org.web3j.protocol.Web3j; 28 | import org.web3j.protocol.core.DefaultBlockParameter; 29 | 30 | import javax.inject.Inject; 31 | 32 | import java.io.IOException; 33 | import java.math.BigInteger; 34 | import java.util.Collections; 35 | import java.util.List; 36 | import java.util.Map; 37 | import java.util.Optional; 38 | import java.util.Set; 39 | 40 | import static im.xiaoyao.presto.ethereum.EthereumHandleResolver.convertColumnHandle; 41 | import static im.xiaoyao.presto.ethereum.EthereumHandleResolver.convertTableHandle; 42 | import static java.util.Objects.requireNonNull; 43 | 44 | public class EthereumMetadata implements ConnectorMetadata { 45 | private static final Logger log = Logger.get(EthereumMetadata.class); 46 | 47 | private static final String DEFAULT_SCHEMA = "default"; 48 | public static final int H8_BYTE_HASH_STRING_LENGTH = 2 + 8 * 2; 49 | public static final int H32_BYTE_HASH_STRING_LENGTH = 2 + 32 * 2; 50 | public static final int H256_BYTE_HASH_STRING_LENGTH = 2 + 256 * 2; 51 | public static final int H20_BYTE_HASH_STRING_LENGTH = 2 + 20 * 2; 52 | 53 | private final String connectorId; 54 | private final Web3j web3j; 55 | 56 | @Inject 57 | public EthereumMetadata( 58 | EthereumConnectorId connectorId, 59 | EthereumWeb3jProvider provider 60 | ) { 61 | this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); 62 | this.web3j = requireNonNull(provider, "provider is null").getWeb3j(); 63 | } 64 | 65 | @Override 66 | public List listSchemaNames(ConnectorSession session) { 67 | return Collections.singletonList(DEFAULT_SCHEMA); 68 | } 69 | 70 | @Override 71 | public EthereumTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) { 72 | if (EthereumTable.BLOCK.getName().equals(schemaTableName.getTableName())) { 73 | return new EthereumTableHandle(connectorId, DEFAULT_SCHEMA, EthereumTable.BLOCK.getName()); 74 | } else if (EthereumTable.TRANSACTION.getName().equals(schemaTableName.getTableName())) { 75 | return new EthereumTableHandle(connectorId, DEFAULT_SCHEMA, EthereumTable.TRANSACTION.getName()); 76 | } else if (EthereumTable.ERC20.getName().equals(schemaTableName.getTableName())) { 77 | return new EthereumTableHandle(connectorId, DEFAULT_SCHEMA, EthereumTable.ERC20.getName()); 78 | } else { 79 | throw new IllegalArgumentException("Unknown Table Name " + schemaTableName.getTableName()); 80 | } 81 | } 82 | 83 | @Override 84 | public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) { 85 | return getTableMetadata(convertTableHandle(tableHandle).toSchemaTableName()); 86 | } 87 | 88 | @Override 89 | public List listTables(ConnectorSession session, String schemaNameOrNull) 90 | { 91 | return ImmutableList.of(new SchemaTableName(DEFAULT_SCHEMA, EthereumTable.BLOCK.getName()), 92 | new SchemaTableName(DEFAULT_SCHEMA, EthereumTable.TRANSACTION.getName()), 93 | new SchemaTableName(DEFAULT_SCHEMA, EthereumTable.ERC20.getName())); 94 | } 95 | 96 | @SuppressWarnings("ValueOfIncrementOrDecrementUsed") 97 | @Override 98 | public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { 99 | EthereumTableHandle ethereumTableHandle = convertTableHandle(tableHandle); 100 | String tableName = ethereumTableHandle.getTableName(); 101 | 102 | ImmutableMap.Builder columnHandles = ImmutableMap.builder(); 103 | int index = 0; 104 | if (EthereumTable.BLOCK.getName().equals(tableName)) { 105 | columnHandles.put("block_number", new EthereumColumnHandle(connectorId, index++, "block_number", BigintType.BIGINT)); 106 | columnHandles.put("block_hash", new EthereumColumnHandle(connectorId, index++, "block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 107 | columnHandles.put("block_parentHash", new EthereumColumnHandle(connectorId, index++, "block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 108 | columnHandles.put("block_nonce", new EthereumColumnHandle(connectorId, index++, "block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH))); 109 | columnHandles.put("block_sha3Uncles", new EthereumColumnHandle(connectorId, index++, "block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 110 | columnHandles.put("block_logsBloom", new EthereumColumnHandle(connectorId, index++, "block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH))); 111 | columnHandles.put("block_transactionsRoot", new EthereumColumnHandle(connectorId, index++, "block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 112 | columnHandles.put("block_stateRoot", new EthereumColumnHandle(connectorId, index++, "block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 113 | columnHandles.put("block_miner", new EthereumColumnHandle(connectorId, index++, "block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 114 | columnHandles.put("block_difficulty", new EthereumColumnHandle(connectorId, index++, "block_difficulty", BigintType.BIGINT)); 115 | columnHandles.put("block_totalDifficulty", new EthereumColumnHandle(connectorId, index++, "block_totalDifficulty", BigintType.BIGINT)); 116 | columnHandles.put("block_size", new EthereumColumnHandle(connectorId, index++, "block_size", IntegerType.INTEGER)); 117 | columnHandles.put("block_extraData", new EthereumColumnHandle(connectorId, index++, "block_extraData", VarcharType.VARCHAR)); 118 | columnHandles.put("block_gasLimit", new EthereumColumnHandle(connectorId, index++, "block_gasLimit", DoubleType.DOUBLE)); 119 | columnHandles.put("block_gasUsed", new EthereumColumnHandle(connectorId, index++, "block_gasUsed", DoubleType.DOUBLE)); 120 | columnHandles.put("block_timestamp", new EthereumColumnHandle(connectorId, index++, "block_timestamp", BigintType.BIGINT)); 121 | columnHandles.put("block_transactions", new EthereumColumnHandle(connectorId, index++, "block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)))); 122 | columnHandles.put("block_uncles", new EthereumColumnHandle(connectorId, index++, "block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)))); 123 | } else if (EthereumTable.TRANSACTION.getName().equals(tableName)) { 124 | columnHandles.put("tx_hash", new EthereumColumnHandle(connectorId, index++, "tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 125 | columnHandles.put("tx_nonce", new EthereumColumnHandle(connectorId, index++, "tx_nonce", BigintType.BIGINT)); 126 | columnHandles.put("tx_blockHash", new EthereumColumnHandle(connectorId, index++, "tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 127 | columnHandles.put("tx_blockNumber", new EthereumColumnHandle(connectorId, index++, "tx_blockNumber", BigintType.BIGINT)); 128 | columnHandles.put("tx_transactionIndex", new EthereumColumnHandle(connectorId, index++, "tx_transactionIndex", IntegerType.INTEGER)); 129 | columnHandles.put("tx_from", new EthereumColumnHandle(connectorId, index++, "tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 130 | columnHandles.put("tx_to", new EthereumColumnHandle(connectorId, index++, "tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 131 | columnHandles.put("tx_value", new EthereumColumnHandle(connectorId, index++, "tx_value", DoubleType.DOUBLE)); 132 | columnHandles.put("tx_gas", new EthereumColumnHandle(connectorId, index++, "tx_gas", DoubleType.DOUBLE)); 133 | columnHandles.put("tx_gasPrice", new EthereumColumnHandle(connectorId, index++, "tx_gasPrice", DoubleType.DOUBLE)); 134 | columnHandles.put("tx_input", new EthereumColumnHandle(connectorId, index++, "tx_input", VarcharType.VARCHAR)); 135 | } else if (EthereumTable.ERC20.getName().equals(tableName)) { 136 | columnHandles.put("erc20_token", new EthereumColumnHandle(connectorId, index++, "erc20_token", VarcharType.createUnboundedVarcharType())); 137 | columnHandles.put("erc20_from", new EthereumColumnHandle(connectorId, index++, "erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 138 | columnHandles.put("erc20_to", new EthereumColumnHandle(connectorId, index++, "erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 139 | columnHandles.put("erc20_value", new EthereumColumnHandle(connectorId, index++, "erc20_value", DoubleType.DOUBLE)); 140 | columnHandles.put("erc20_txHash", new EthereumColumnHandle(connectorId, index++, "erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 141 | columnHandles.put("erc20_blockNumber", new EthereumColumnHandle(connectorId, index++, "erc20_blockNumber", BigintType.BIGINT)); 142 | } else { 143 | throw new IllegalArgumentException("Unknown Table Name " + tableName); 144 | } 145 | 146 | return columnHandles.build(); 147 | } 148 | 149 | @Override 150 | public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) 151 | { 152 | requireNonNull(prefix, "prefix is null"); 153 | 154 | ImmutableMap.Builder> columns = ImmutableMap.builder(); 155 | 156 | List tableNames = prefix.getSchemaName() == null ? listTables(session, (String) null) : ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName())); 157 | 158 | for (SchemaTableName tableName : tableNames) { 159 | ConnectorTableMetadata tableMetadata = getTableMetadata(tableName); 160 | // table can disappear during listing operation 161 | if (tableMetadata != null) { 162 | columns.put(tableName, tableMetadata.getColumns()); 163 | } 164 | } 165 | return columns.build(); 166 | } 167 | 168 | @Override 169 | public ColumnMetadata getColumnMetadata( 170 | ConnectorSession session, 171 | ConnectorTableHandle tableHandle, 172 | ColumnHandle columnHandle 173 | ) { 174 | convertTableHandle(tableHandle); 175 | return convertColumnHandle(columnHandle).getColumnMetadata(); 176 | } 177 | 178 | @Override 179 | public List getTableLayouts( 180 | ConnectorSession session, 181 | ConnectorTableHandle table, 182 | Constraint constraint, 183 | Optional> desiredColumns 184 | ) { 185 | ImmutableList.Builder builder = ImmutableList.builder(); 186 | 187 | Optional> domains = constraint.getSummary().getDomains(); 188 | if (domains.isPresent()) { 189 | Map columnHandleDomainMap = domains.get(); 190 | for (Map.Entry entry : columnHandleDomainMap.entrySet()) { 191 | if (entry.getKey() instanceof EthereumColumnHandle 192 | && (((EthereumColumnHandle) entry.getKey()).getName().equals("block_number") 193 | || ((EthereumColumnHandle) entry.getKey()).getName().equals("tx_blockNumber") 194 | || ((EthereumColumnHandle) entry.getKey()).getName().equals("erc20_blockNumber"))) { 195 | entry.getValue().getValues().getRanges().getOrderedRanges().forEach(r -> { 196 | Marker low = r.getLow(); 197 | Marker high = r.getHigh(); 198 | builder.add(EthereumBlockRange.fromMarkers(low, high)); 199 | }); 200 | } else if (entry.getKey() instanceof EthereumColumnHandle 201 | && (((EthereumColumnHandle) entry.getKey()).getName().equals("block_hash") 202 | || ((EthereumColumnHandle) entry.getKey()).getName().equals("tx_blockHash"))) { 203 | entry.getValue().getValues().getRanges().getOrderedRanges().stream() 204 | .filter(Range::isSingleValue).forEach(r -> { 205 | String blockHash = ((Slice) r.getSingleValue()).toStringUtf8(); 206 | try { 207 | long blockNumber = web3j.ethGetBlockByHash(blockHash, true).send().getBlock().getNumber().longValue(); 208 | builder.add(new EthereumBlockRange(blockNumber, blockNumber)); 209 | } 210 | catch (IOException e) { 211 | throw new IllegalStateException("Unable to getting block by hash " + blockHash); 212 | } 213 | }); 214 | log.info(entry.getValue().getValues().toString(null)); 215 | } else if (entry.getKey() instanceof EthereumColumnHandle 216 | && (((EthereumColumnHandle) entry.getKey()).getName().equals("block_timestamp"))) { 217 | entry.getValue().getValues().getRanges().getOrderedRanges().forEach(r -> { 218 | Marker low = r.getLow(); 219 | Marker high = r.getHigh(); 220 | try { 221 | long startBlock = low.isLowerUnbounded() ? 1L : findBlockByTimestamp((Long) low.getValue(), -1L); 222 | long endBlock = high.isUpperUnbounded() ? -1L : findBlockByTimestamp((Long) high.getValue(), 1L); 223 | builder.add(new EthereumBlockRange(startBlock, endBlock)); 224 | } 225 | catch (IOException e) { 226 | throw new IllegalStateException("Unable to find block by timestamp"); 227 | } 228 | }); 229 | log.info(entry.getValue().getValues().toString(null)); 230 | } 231 | } 232 | } 233 | 234 | EthereumTableHandle handle = convertTableHandle(table); 235 | ConnectorTableLayout layout = new ConnectorTableLayout(new EthereumTableLayoutHandle(handle, builder.build())); 236 | return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); 237 | } 238 | 239 | @Override 240 | public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { 241 | return new ConnectorTableLayout(handle); 242 | } 243 | 244 | @SuppressWarnings("ValueOfIncrementOrDecrementUsed") 245 | private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) { 246 | ImmutableList.Builder builder = ImmutableList.builder(); 247 | 248 | if (EthereumTable.BLOCK.getName().equals(schemaTableName.getTableName())) { 249 | builder.add(new ColumnMetadata("block_number", BigintType.BIGINT)); 250 | builder.add(new ColumnMetadata("block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 251 | builder.add(new ColumnMetadata("block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 252 | builder.add(new ColumnMetadata("block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH))); 253 | builder.add(new ColumnMetadata("block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 254 | builder.add(new ColumnMetadata("block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH))); 255 | builder.add(new ColumnMetadata("block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 256 | builder.add(new ColumnMetadata("block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 257 | builder.add(new ColumnMetadata("block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 258 | builder.add(new ColumnMetadata("block_difficulty", BigintType.BIGINT)); 259 | builder.add(new ColumnMetadata("block_totalDifficulty", BigintType.BIGINT)); 260 | builder.add(new ColumnMetadata("block_size", IntegerType.INTEGER)); 261 | builder.add(new ColumnMetadata("block_extraData", VarcharType.VARCHAR)); 262 | builder.add(new ColumnMetadata("block_gasLimit", DoubleType.DOUBLE)); 263 | builder.add(new ColumnMetadata("block_gasUsed", DoubleType.DOUBLE)); 264 | builder.add(new ColumnMetadata("block_timestamp", BigintType.BIGINT)); 265 | builder.add(new ColumnMetadata("block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)))); 266 | builder.add(new ColumnMetadata("block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)))); 267 | } else if (EthereumTable.TRANSACTION.getName().equals(schemaTableName.getTableName())) { 268 | builder.add(new ColumnMetadata("tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 269 | builder.add(new ColumnMetadata("tx_nonce", BigintType.BIGINT)); 270 | builder.add(new ColumnMetadata("tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 271 | builder.add(new ColumnMetadata("tx_blockNumber", BigintType.BIGINT)); 272 | builder.add(new ColumnMetadata("tx_transactionIndex", IntegerType.INTEGER)); 273 | builder.add(new ColumnMetadata("tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 274 | builder.add(new ColumnMetadata("tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 275 | builder.add(new ColumnMetadata("tx_value", DoubleType.DOUBLE)); 276 | builder.add(new ColumnMetadata("tx_gas", DoubleType.DOUBLE)); 277 | builder.add(new ColumnMetadata("tx_gasPrice", DoubleType.DOUBLE)); 278 | builder.add(new ColumnMetadata("tx_input", VarcharType.VARCHAR)); 279 | } else if (EthereumTable.ERC20.getName().equals(schemaTableName.getTableName())) { 280 | builder.add(new ColumnMetadata("erc20_token", VarcharType.createUnboundedVarcharType())); 281 | builder.add(new ColumnMetadata("erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 282 | builder.add(new ColumnMetadata("erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH))); 283 | builder.add(new ColumnMetadata("erc20_value", DoubleType.DOUBLE)); 284 | builder.add(new ColumnMetadata("erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))); 285 | builder.add(new ColumnMetadata("erc20_blockNumber", BigintType.BIGINT)); 286 | } else { 287 | throw new IllegalArgumentException("Unknown Table Name " + schemaTableName.getTableName()); 288 | } 289 | 290 | return new ConnectorTableMetadata(schemaTableName, builder.build()); 291 | } 292 | 293 | private long findBlockByTimestamp(long timestamp, long offset) throws IOException { 294 | long startBlock = 1L; 295 | long currentBlock = web3j.ethBlockNumber().send().getBlockNumber().longValue(); 296 | 297 | if (currentBlock <= 1) { 298 | return currentBlock; 299 | } 300 | 301 | long low = startBlock; 302 | long high = currentBlock; 303 | long middle = low + (high - low) / 2; 304 | 305 | while(low <= high) { 306 | middle = low + (high - low) / 2; 307 | long ts = web3j.ethGetBlockByNumber(DefaultBlockParameter.valueOf(BigInteger.valueOf(middle)), false).send().getBlock().getTimestamp().longValue(); 308 | 309 | if (ts < timestamp) { 310 | low = middle + 1; 311 | } else if (ts > timestamp) { 312 | high = middle - 1; 313 | } else { 314 | return middle; 315 | } 316 | } 317 | return middle + offset; 318 | } 319 | } 320 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumPlugin.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.Plugin; 4 | import com.facebook.presto.spi.connector.ConnectorFactory; 5 | import com.google.common.collect.ImmutableList; 6 | import com.google.common.collect.ImmutableSet; 7 | import im.xiaoyao.presto.ethereum.udfs.EthereumUDFs; 8 | 9 | import java.util.Set; 10 | 11 | public class EthereumPlugin implements Plugin { 12 | @Override 13 | public Iterable getConnectorFactories() { 14 | return ImmutableList.of(new EthereumConnectorFactory()); 15 | } 16 | 17 | @Override 18 | public Set> getFunctions() { 19 | return ImmutableSet.of(EthereumUDFs.class); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumRecordCursor.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.common.block.PageBuilderStatus; 4 | import com.facebook.presto.spi.RecordCursor; 5 | import com.facebook.presto.common.block.Block; 6 | import com.facebook.presto.common.block.BlockBuilder; 7 | import com.facebook.presto.common.block.BlockBuilderStatus; 8 | import com.facebook.presto.common.type.StandardTypes; 9 | import com.facebook.presto.common.type.Type; 10 | import com.google.common.base.Splitter; 11 | import com.google.common.collect.ImmutableList; 12 | import io.airlift.log.Logger; 13 | import io.airlift.slice.Slice; 14 | import io.airlift.slice.Slices; 15 | import org.joda.time.DateTimeZone; 16 | import org.web3j.protocol.Web3j; 17 | import org.web3j.protocol.core.methods.response.EthBlock; 18 | import org.web3j.protocol.core.methods.response.Log; 19 | 20 | import java.sql.Date; 21 | import java.sql.Timestamp; 22 | import java.util.Collections; 23 | import java.util.Iterator; 24 | import java.util.List; 25 | import java.util.Map; 26 | import java.util.Optional; 27 | import java.util.concurrent.TimeUnit; 28 | import java.util.function.Supplier; 29 | import java.util.stream.Collectors; 30 | 31 | import static com.facebook.presto.common.type.BigintType.BIGINT; 32 | import static com.facebook.presto.common.type.BooleanType.BOOLEAN; 33 | import static com.facebook.presto.common.type.Chars.isCharType; 34 | import static com.facebook.presto.common.type.Chars.truncateToLengthAndTrimSpaces; 35 | import static com.facebook.presto.common.type.DateType.DATE; 36 | import static com.facebook.presto.common.type.DoubleType.DOUBLE; 37 | import static com.facebook.presto.common.type.IntegerType.INTEGER; 38 | import static com.facebook.presto.common.type.RealType.REAL; 39 | import static com.facebook.presto.common.type.SmallintType.SMALLINT; 40 | import static com.facebook.presto.common.type.TimestampType.TIMESTAMP; 41 | import static com.facebook.presto.common.type.TinyintType.TINYINT; 42 | import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; 43 | import static com.facebook.presto.common.type.Varchars.isVarcharType; 44 | import static com.facebook.presto.common.type.Varchars.truncateToLength; 45 | import static com.google.common.base.Preconditions.checkArgument; 46 | import static java.lang.Float.floatToRawIntBits; 47 | import static java.util.Objects.requireNonNull; 48 | 49 | public class EthereumRecordCursor implements RecordCursor { 50 | private static final Logger log = Logger.get(EthereumRecordCursor.class); 51 | 52 | private final EthBlock block; 53 | private final Iterator blockIter; 54 | private final Iterator txIter; 55 | private final Iterator logIter; 56 | 57 | private final EthereumTable table; 58 | private final Web3j web3j; 59 | 60 | private final List columnHandles; 61 | private final int[] fieldToColumnIndex; 62 | 63 | private List suppliers; 64 | 65 | public EthereumRecordCursor(List columnHandles, EthBlock block, EthereumTable table, Web3j web3j) { 66 | this.columnHandles = columnHandles; 67 | this.table = table; 68 | this.web3j = web3j; 69 | this.suppliers = Collections.emptyList(); 70 | 71 | fieldToColumnIndex = new int[columnHandles.size()]; 72 | for (int i = 0; i < columnHandles.size(); i++) { 73 | EthereumColumnHandle columnHandle = columnHandles.get(i); 74 | fieldToColumnIndex[i] = columnHandle.getOrdinalPosition(); 75 | } 76 | 77 | // TODO: handle failure upstream 78 | this.block = requireNonNull(block, "block is null"); 79 | this.blockIter = ImmutableList.of(block).iterator(); 80 | this.txIter = block.getBlock().getTransactions().iterator(); 81 | this.logIter = new EthereumLogLazyIterator(block, web3j); 82 | } 83 | 84 | @Override 85 | public long getCompletedBytes() { 86 | return block.getBlock().getSize().longValue(); 87 | } 88 | 89 | @Override 90 | public long getReadTimeNanos() { 91 | return 0; 92 | } 93 | 94 | @Override 95 | public Type getType(int field) { 96 | checkArgument(field < columnHandles.size(), "Invalid field index"); 97 | return columnHandles.get(field).getType(); 98 | } 99 | 100 | @Override 101 | public boolean advanceNextPosition() { 102 | if (table == EthereumTable.BLOCK && !blockIter.hasNext() 103 | || table == EthereumTable.TRANSACTION && !txIter.hasNext() 104 | || table == EthereumTable.ERC20 && !logIter.hasNext()) { 105 | return false; 106 | } 107 | 108 | ImmutableList.Builder builder = ImmutableList.builder(); 109 | if (table == EthereumTable.BLOCK) { 110 | blockIter.next(); 111 | EthBlock.Block blockBlock = this.block.getBlock(); 112 | builder.add(blockBlock::getNumber); 113 | builder.add(blockBlock::getHash); 114 | builder.add(blockBlock::getParentHash); 115 | builder.add(blockBlock::getNonceRaw); 116 | builder.add(blockBlock::getSha3Uncles); 117 | builder.add(blockBlock::getLogsBloom); 118 | builder.add(blockBlock::getTransactionsRoot); 119 | builder.add(blockBlock::getStateRoot); 120 | builder.add(blockBlock::getMiner); 121 | builder.add(blockBlock::getDifficulty); 122 | builder.add(blockBlock::getTotalDifficulty); 123 | builder.add(blockBlock::getSize); 124 | builder.add(blockBlock::getExtraData); 125 | builder.add(blockBlock::getGasLimit); 126 | builder.add(blockBlock::getGasUsed); 127 | builder.add(blockBlock::getTimestamp); 128 | builder.add(() -> { 129 | return blockBlock.getTransactions() 130 | .stream() 131 | .map(tr -> ((EthBlock.TransactionObject) tr.get()).getHash()) 132 | .collect(Collectors.toList()); 133 | }); 134 | builder.add(blockBlock::getUncles); 135 | 136 | } else if (table == EthereumTable.TRANSACTION) { 137 | EthBlock.TransactionResult tr = txIter.next(); 138 | EthBlock.TransactionObject tx = (EthBlock.TransactionObject) tr.get(); 139 | 140 | builder.add(tx::getHash); 141 | builder.add(tx::getNonce); 142 | builder.add(tx::getBlockHash); 143 | builder.add(tx::getBlockNumber); 144 | builder.add(tx::getTransactionIndex); 145 | builder.add(tx::getFrom); 146 | builder.add(tx::getTo); 147 | builder.add(tx::getValue); 148 | builder.add(tx::getGas); 149 | builder.add(tx::getGasPrice); 150 | builder.add(tx::getInput); 151 | } else if (table == EthereumTable.ERC20) { 152 | while (logIter.hasNext()) { 153 | Log l = logIter.next(); 154 | List topics = l.getTopics(); 155 | String data = l.getData(); 156 | 157 | if (topics.get(0).equalsIgnoreCase(EthereumERC20Utils.TRANSFER_EVENT_TOPIC)) { 158 | // Handle unindexed event fields: 159 | // if the number of topics and fields in data part != 4, then it's a weird event 160 | if (topics.size() < 3 && topics.size() + (data.length() - 2) / 64 != 4) { 161 | continue; 162 | } 163 | 164 | if (topics.size() < 3) { 165 | Iterator dataFields = Splitter.fixedLength(64).split(data.substring(2)).iterator(); 166 | while (topics.size() < 3) { 167 | topics.add("0x" + dataFields.next()); 168 | } 169 | data = "0x" + dataFields.next(); 170 | } 171 | 172 | // Token contract address 173 | builder.add(() -> Optional.ofNullable(EthereumERC20Token.lookup.get(l.getAddress().toLowerCase())) 174 | .map(Enum::name).orElse(String.format("ERC20(%s)", l.getAddress()))); 175 | // from address 176 | builder.add(() -> h32ToH20(topics.get(1))); 177 | // to address 178 | builder.add(() -> h32ToH20(topics.get(2))); 179 | // amount value 180 | String finalData = data; 181 | builder.add(() -> EthereumERC20Utils.hexToDouble(finalData)); 182 | builder.add(l::getTransactionHash); 183 | builder.add(l::getBlockNumber); 184 | this.suppliers = builder.build(); 185 | return true; 186 | } 187 | } 188 | 189 | return false; 190 | } else { 191 | return false; 192 | } 193 | 194 | this.suppliers = builder.build(); 195 | return true; 196 | } 197 | 198 | @Override 199 | public boolean getBoolean(int field) { 200 | return (boolean) suppliers.get(fieldToColumnIndex[field]).get(); 201 | } 202 | 203 | @Override 204 | public long getLong(int field) { 205 | return ((Number) suppliers.get(fieldToColumnIndex[field]).get()).longValue(); 206 | } 207 | 208 | @Override 209 | public double getDouble(int field) { 210 | return ((Number) suppliers.get(fieldToColumnIndex[field]).get()).doubleValue(); 211 | } 212 | 213 | @Override 214 | public Slice getSlice(int field) { 215 | return Slices.utf8Slice((String) suppliers.get(fieldToColumnIndex[field]).get()); 216 | } 217 | 218 | @Override 219 | public Object getObject(int field) { 220 | return serializeObject(columnHandles.get(field).getType(), null, suppliers.get(fieldToColumnIndex[field]).get()); 221 | } 222 | 223 | @Override 224 | public boolean isNull(int field) { 225 | return suppliers.get(fieldToColumnIndex[field]).get() == null; 226 | } 227 | 228 | @Override 229 | public void close() { 230 | } 231 | 232 | private static long getLongExpressedValue(Object value) { 233 | if (value instanceof Date) { 234 | long storageTime = ((Date) value).getTime(); 235 | // convert date from VM current time zone to UTC 236 | long utcMillis = storageTime + DateTimeZone.getDefault().getOffset(storageTime); 237 | return TimeUnit.MILLISECONDS.toDays(utcMillis); 238 | } 239 | if (value instanceof Timestamp) { 240 | long parsedJvmMillis = ((Timestamp) value).getTime(); 241 | DateTimeZone jvmTimeZone = DateTimeZone.getDefault(); 242 | long convertedMillis = jvmTimeZone.convertUTCToLocal(parsedJvmMillis); 243 | 244 | return convertedMillis; 245 | } 246 | if (value instanceof Float) { 247 | return floatToRawIntBits(((Float) value)); 248 | } 249 | return ((Number) value).longValue(); 250 | } 251 | 252 | private static Slice getSliceExpressedValue(Object value, Type type) { 253 | Slice sliceValue; 254 | if (value instanceof String) { 255 | sliceValue = Slices.utf8Slice((String) value); 256 | } else if (value instanceof byte[]) { 257 | sliceValue = Slices.wrappedBuffer((byte[]) value); 258 | } else if (value instanceof Integer) { 259 | sliceValue = Slices.utf8Slice(value.toString()); 260 | } else { 261 | throw new IllegalStateException("unsupported string field type: " + value.getClass().getName()); 262 | } 263 | if (isVarcharType(type)) { 264 | sliceValue = truncateToLength(sliceValue, type); 265 | } 266 | if (isCharType(type)) { 267 | sliceValue = truncateToLengthAndTrimSpaces(sliceValue, type); 268 | } 269 | 270 | return sliceValue; 271 | } 272 | 273 | private static Block serializeObject(Type type, BlockBuilder builder, Object object) { 274 | if (!isStructuralType(type)) { 275 | serializePrimitive(type, builder, object); 276 | return null; 277 | } else if (isArrayType(type)) { 278 | return serializeList(type, builder, object); 279 | } else if (isMapType(type)) { 280 | return serializeMap(type, builder, object); 281 | } else if (isRowType(type)) { 282 | return serializeStruct(type, builder, object); 283 | } 284 | throw new RuntimeException("Unknown object type: " + type); 285 | } 286 | 287 | private static Block serializeList(Type type, BlockBuilder builder, Object object) { 288 | List list = (List) object; 289 | if (list == null) { 290 | requireNonNull(builder, "parent builder is null").appendNull(); 291 | return null; 292 | } 293 | 294 | List typeParameters = type.getTypeParameters(); 295 | checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); 296 | Type elementType = typeParameters.get(0); 297 | 298 | BlockBuilder currentBuilder; 299 | if (builder != null) { 300 | currentBuilder = builder.beginBlockEntry(); 301 | } else { 302 | currentBuilder = elementType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), list.size()); 303 | } 304 | 305 | for (Object element : list) { 306 | serializeObject(elementType, currentBuilder, element); 307 | } 308 | 309 | if (builder != null) { 310 | builder.closeEntry(); 311 | return null; 312 | } else { 313 | Block resultBlock = currentBuilder.build(); 314 | return resultBlock; 315 | } 316 | } 317 | 318 | private static Block serializeMap(Type type, BlockBuilder builder, Object object) { 319 | Map map = (Map) object; 320 | if (map == null) { 321 | requireNonNull(builder, "parent builder is null").appendNull(); 322 | return null; 323 | } 324 | 325 | List typeParameters = type.getTypeParameters(); 326 | checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); 327 | Type keyType = typeParameters.get(0); 328 | Type valueType = typeParameters.get(1); 329 | boolean builderSynthesized = false; 330 | 331 | if (builder == null) { 332 | builderSynthesized = true; 333 | builder = type.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); 334 | } 335 | BlockBuilder currentBuilder = builder.beginBlockEntry(); 336 | 337 | for (Map.Entry entry : map.entrySet()) { 338 | // Hive skips map entries with null keys 339 | if (entry.getKey() != null) { 340 | serializeObject(keyType, currentBuilder, entry.getKey()); 341 | serializeObject(valueType, currentBuilder, entry.getValue()); 342 | } 343 | } 344 | 345 | builder.closeEntry(); 346 | if (builderSynthesized) { 347 | return (Block) type.getObject(builder, 0); 348 | } else { 349 | return null; 350 | } 351 | } 352 | 353 | private static Block serializeStruct(Type type, BlockBuilder builder, Object object) { 354 | if (object == null) { 355 | requireNonNull(builder, "parent builder is null").appendNull(); 356 | return null; 357 | } 358 | 359 | List typeParameters = type.getTypeParameters(); 360 | EthBlock.TransactionObject structData = (EthBlock.TransactionObject) object; 361 | boolean builderSynthesized = false; 362 | if (builder == null) { 363 | builderSynthesized = true; 364 | builder = type.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); 365 | } 366 | BlockBuilder currentBuilder = builder.beginBlockEntry(); 367 | 368 | ImmutableList.Builder lstBuilder = ImmutableList.builder(); 369 | lstBuilder.add(structData::getHash); 370 | lstBuilder.add(structData::getNonce); 371 | lstBuilder.add(structData::getBlockHash); 372 | lstBuilder.add(structData::getBlockNumber); 373 | lstBuilder.add(structData::getTransactionIndex); 374 | lstBuilder.add(structData::getFrom); 375 | lstBuilder.add(structData::getTo); 376 | lstBuilder.add(structData::getValue); 377 | lstBuilder.add(structData::getGas); 378 | lstBuilder.add(structData::getGasPrice); 379 | lstBuilder.add(structData::getInput); 380 | ImmutableList txColumns = lstBuilder.build(); 381 | 382 | for (int i = 0; i < typeParameters.size(); i++) { 383 | serializeObject(typeParameters.get(i), currentBuilder, txColumns.get(i).get()); 384 | } 385 | 386 | builder.closeEntry(); 387 | if (builderSynthesized) { 388 | return (Block) type.getObject(builder, 0); 389 | } else { 390 | return null; 391 | } 392 | } 393 | 394 | private static void serializePrimitive(Type type, BlockBuilder builder, Object object) { 395 | requireNonNull(builder, "parent builder is null"); 396 | 397 | if (object == null) { 398 | builder.appendNull(); 399 | return; 400 | } 401 | 402 | if (BOOLEAN.equals(type)) { 403 | BOOLEAN.writeBoolean(builder, (Boolean) object); 404 | } else if (BIGINT.equals(type) || INTEGER.equals(type) || SMALLINT.equals(type) || TINYINT.equals(type) 405 | || REAL.equals(type) || DATE.equals(type) || TIMESTAMP.equals(type)) { 406 | type.writeLong(builder, getLongExpressedValue(object)); 407 | } else if (DOUBLE.equals(type)) { 408 | DOUBLE.writeDouble(builder, ((Number) object).doubleValue()); 409 | } else if (isVarcharType(type) || VARBINARY.equals(type) || isCharType(type)) { 410 | type.writeSlice(builder, getSliceExpressedValue(object, type)); 411 | } else { 412 | throw new UnsupportedOperationException("Unsupported primitive type: " + type); 413 | } 414 | } 415 | 416 | public static boolean isArrayType(Type type) { 417 | return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); 418 | } 419 | 420 | public static boolean isMapType(Type type) { 421 | return type.getTypeSignature().getBase().equals(StandardTypes.MAP); 422 | } 423 | 424 | public static boolean isRowType(Type type) { 425 | return type.getTypeSignature().getBase().equals(StandardTypes.ROW); 426 | } 427 | 428 | public static boolean isStructuralType(Type type) { 429 | String baseName = type.getTypeSignature().getBase(); 430 | return baseName.equals(StandardTypes.MAP) || baseName.equals(StandardTypes.ARRAY) || baseName.equals(StandardTypes.ROW); 431 | } 432 | 433 | private static String h32ToH20(String h32) { 434 | return "0x" + h32.substring(EthereumMetadata.H32_BYTE_HASH_STRING_LENGTH - EthereumMetadata.H20_BYTE_HASH_STRING_LENGTH + 2); 435 | } 436 | } 437 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumRecordSet.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.RecordCursor; 4 | import com.facebook.presto.spi.RecordSet; 5 | import com.facebook.presto.common.type.Type; 6 | import com.google.common.collect.ImmutableList; 7 | import io.airlift.log.Logger; 8 | import org.web3j.protocol.Web3j; 9 | import org.web3j.protocol.core.DefaultBlockParameter; 10 | import org.web3j.protocol.core.methods.response.EthBlock; 11 | 12 | import java.io.IOException; 13 | import java.math.BigInteger; 14 | import java.util.List; 15 | 16 | import static java.util.Objects.requireNonNull; 17 | 18 | public class EthereumRecordSet implements RecordSet { 19 | private static final Logger log = Logger.get(EthereumRecordSet.class); 20 | 21 | private final EthereumSplit split; 22 | private final Web3j web3j; 23 | 24 | private final List columnHandles; 25 | private final List columnTypes; 26 | 27 | EthereumRecordSet(Web3j web3j, List columnHandles, EthereumSplit split) { 28 | this.split = requireNonNull(split, "split is null"); 29 | this.web3j = requireNonNull(web3j, "web3j is null"); 30 | 31 | this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); 32 | 33 | ImmutableList.Builder typeBuilder = ImmutableList.builder(); 34 | 35 | for (EthereumColumnHandle handle : columnHandles) { 36 | typeBuilder.add(handle.getType()); 37 | } 38 | 39 | this.columnTypes = typeBuilder.build(); 40 | } 41 | 42 | @Override 43 | public List getColumnTypes() { 44 | return columnTypes; 45 | } 46 | 47 | @Override 48 | public RecordCursor cursor() { 49 | EthBlock block = null; 50 | try { 51 | block = web3j.ethGetBlockByNumber(DefaultBlockParameter.valueOf(BigInteger.valueOf(split.getBlockId())), true).send(); 52 | } catch (IOException e) { 53 | e.printStackTrace(); 54 | } 55 | return new EthereumRecordCursor(columnHandles, block, split.getTable(), web3j); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumRecordSetProvider.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ColumnHandle; 4 | import com.facebook.presto.spi.ConnectorSession; 5 | import com.facebook.presto.spi.ConnectorSplit; 6 | import com.facebook.presto.spi.RecordSet; 7 | import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; 8 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 9 | import com.google.common.collect.ImmutableList; 10 | import org.web3j.protocol.Web3j; 11 | 12 | import javax.inject.Inject; 13 | 14 | import java.util.List; 15 | 16 | import static im.xiaoyao.presto.ethereum.EthereumHandleResolver.convertColumnHandle; 17 | import static im.xiaoyao.presto.ethereum.EthereumHandleResolver.convertSplit; 18 | 19 | public class EthereumRecordSetProvider implements ConnectorRecordSetProvider { 20 | private final Web3j web3j; 21 | 22 | @Inject 23 | public EthereumRecordSetProvider(EthereumWeb3jProvider web3jProvider) { 24 | this.web3j = web3jProvider.getWeb3j(); 25 | } 26 | 27 | @Override 28 | public RecordSet getRecordSet( 29 | ConnectorTransactionHandle transaction, 30 | ConnectorSession session, 31 | ConnectorSplit split, 32 | List columns 33 | ) { 34 | EthereumSplit ethereumSplit = convertSplit(split); 35 | 36 | ImmutableList.Builder handleBuilder = ImmutableList.builder(); 37 | 38 | for (ColumnHandle handle : columns) { 39 | EthereumColumnHandle columnHandle = convertColumnHandle(handle); 40 | handleBuilder.add(columnHandle); 41 | } 42 | 43 | return new EthereumRecordSet(web3j, handleBuilder.build(), ethereumSplit); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumSplit.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ConnectorSplit; 4 | import com.facebook.presto.spi.HostAddress; 5 | import com.facebook.presto.spi.NodeProvider; 6 | import com.facebook.presto.spi.schedule.NodeSelectionStrategy; 7 | import com.fasterxml.jackson.annotation.JsonCreator; 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | 10 | import java.util.Collections; 11 | import java.util.List; 12 | 13 | public class EthereumSplit implements ConnectorSplit { 14 | private final long blockId; 15 | private final String blockHash; 16 | 17 | private final EthereumTable table; 18 | 19 | @JsonCreator 20 | public EthereumSplit( 21 | @JsonProperty("blockId") long blockId, 22 | @JsonProperty("table") EthereumTable table 23 | ) { 24 | this.blockId = blockId; 25 | this.table = table; 26 | this.blockHash = null; 27 | } 28 | 29 | @JsonProperty 30 | public long getBlockId() { 31 | return blockId; 32 | } 33 | 34 | @JsonProperty 35 | public String getBlockHash() { 36 | return blockHash; 37 | } 38 | 39 | @JsonProperty 40 | public EthereumTable getTable() { 41 | return table; 42 | } 43 | 44 | @Override 45 | public NodeSelectionStrategy getNodeSelectionStrategy() { 46 | return NodeSelectionStrategy.NO_PREFERENCE; 47 | } 48 | 49 | @Override 50 | public List getPreferredNodes(NodeProvider nodeProvider) { 51 | return Collections.emptyList(); 52 | } 53 | 54 | @Override 55 | public Object getInfo() { 56 | return this; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumSplitManager.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ConnectorSession; 4 | import com.facebook.presto.spi.ConnectorSplit; 5 | import com.facebook.presto.spi.ConnectorSplitSource; 6 | import com.facebook.presto.spi.ConnectorTableLayoutHandle; 7 | import com.facebook.presto.spi.FixedSplitSource; 8 | import com.facebook.presto.spi.connector.ConnectorSplitManager; 9 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 10 | import com.google.common.collect.ImmutableList; 11 | import io.airlift.log.Logger; 12 | import org.web3j.protocol.Web3j; 13 | import org.web3j.protocol.core.methods.response.EthBlockNumber; 14 | 15 | import javax.inject.Inject; 16 | 17 | import java.io.IOException; 18 | 19 | import static im.xiaoyao.presto.ethereum.EthereumHandleResolver.convertLayout; 20 | import static java.util.Objects.requireNonNull; 21 | 22 | public class EthereumSplitManager implements ConnectorSplitManager { 23 | private static final Logger log = Logger.get(EthereumSplitManager.class); 24 | 25 | private final String connectorId; 26 | private final Web3j web3j; 27 | 28 | @Inject 29 | public EthereumSplitManager( 30 | EthereumConnectorId connectorId, 31 | EthereumConnectorConfig config, 32 | EthereumWeb3jProvider web3jProvider 33 | ) { 34 | this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); 35 | requireNonNull(web3jProvider, "web3j is null"); 36 | requireNonNull(config, "config is null"); 37 | this.web3j = web3jProvider.getWeb3j(); 38 | } 39 | 40 | @Override 41 | public ConnectorSplitSource getSplits( 42 | ConnectorTransactionHandle transaction, 43 | ConnectorSession session, 44 | ConnectorTableLayoutHandle layout, 45 | SplitSchedulingContext splitSchedulingContext 46 | ) { 47 | EthereumTableLayoutHandle tableLayoutHandle = convertLayout(layout); 48 | EthereumTableHandle tableHandle = tableLayoutHandle.getTable(); 49 | 50 | try { 51 | EthBlockNumber blockNumber = web3j.ethBlockNumber().send(); 52 | log.info("current block number: " + blockNumber.getBlockNumber()); 53 | ImmutableList.Builder splits = ImmutableList.builder(); 54 | 55 | for (EthereumBlockRange blockRange : tableLayoutHandle.getBlockRanges()) { 56 | log.info("start: %d\tend: %d", blockRange.getStartBlock(), blockRange.getEndBlock()); 57 | for (long i = blockRange.getStartBlock(); i <= (blockRange.getEndBlock() == -1 ? blockNumber.getBlockNumber().longValue() : blockRange.getEndBlock()); i++) { 58 | EthereumSplit split = new EthereumSplit(i, EthereumTable.valueOf(tableHandle.getTableName().toUpperCase())); 59 | splits.add(split); 60 | } 61 | } 62 | 63 | if (tableLayoutHandle.getBlockRanges().isEmpty()) { 64 | for (long i = 1; i <= blockNumber.getBlockNumber().longValue(); i++) { 65 | EthereumSplit split = new EthereumSplit(i, EthereumTable.valueOf(tableHandle.getTableName().toUpperCase())); 66 | splits.add(split); 67 | } 68 | } 69 | 70 | ImmutableList connectorSplits = splits.build(); 71 | log.info("Built %d splits", connectorSplits.size()); 72 | return new FixedSplitSource(connectorSplits); 73 | } catch (IOException e) { 74 | e.printStackTrace(); 75 | throw new RuntimeException("Cannot get block number: ", e); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumTable.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | 6 | @AllArgsConstructor 7 | public enum EthereumTable { 8 | BLOCK("block"), 9 | TRANSACTION("transaction"), 10 | ERC20("erc20"); 11 | 12 | @Getter 13 | private final String name; 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumTableHandle.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ConnectorTableHandle; 4 | import com.facebook.presto.spi.SchemaTableName; 5 | import com.fasterxml.jackson.annotation.JsonCreator; 6 | import com.fasterxml.jackson.annotation.JsonProperty; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.ToString; 9 | 10 | import static java.util.Objects.requireNonNull; 11 | 12 | @EqualsAndHashCode 13 | @ToString 14 | public final class EthereumTableHandle implements ConnectorTableHandle { 15 | private final String connectorId; 16 | 17 | private final String schemaName; 18 | 19 | private final String tableName; 20 | 21 | @JsonCreator 22 | public EthereumTableHandle( 23 | @JsonProperty("connectorId") String connectorId, 24 | @JsonProperty("schemaName") String schemaName, 25 | @JsonProperty("tableName") String tableName 26 | ) { 27 | this.connectorId = requireNonNull(connectorId, "connectorId is null"); 28 | this.schemaName = requireNonNull(schemaName, "schemaName is null"); 29 | this.tableName = requireNonNull(tableName, "tableName is null"); 30 | } 31 | 32 | @JsonProperty 33 | public String getConnectorId() { 34 | return connectorId; 35 | } 36 | 37 | @JsonProperty 38 | public String getSchemaName() { 39 | return schemaName; 40 | } 41 | 42 | @JsonProperty 43 | public String getTableName() { 44 | return tableName; 45 | } 46 | 47 | public SchemaTableName toSchemaTableName() { 48 | return new SchemaTableName(schemaName, tableName); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumTableLayoutHandle.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.ConnectorTableLayoutHandle; 4 | import com.fasterxml.jackson.annotation.JsonCreator; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import lombok.ToString; 7 | 8 | import java.util.List; 9 | 10 | import static java.util.Objects.requireNonNull; 11 | 12 | @ToString 13 | public class EthereumTableLayoutHandle implements ConnectorTableLayoutHandle { 14 | private final EthereumTableHandle table; 15 | 16 | private final List blockRanges; 17 | 18 | @JsonCreator 19 | public EthereumTableLayoutHandle( 20 | @JsonProperty("table") EthereumTableHandle table, 21 | @JsonProperty("blockRanges") List blockRanges 22 | ) { 23 | this.table = requireNonNull(table, "table is null"); 24 | this.blockRanges = requireNonNull(blockRanges, "blockRanges is null"); 25 | } 26 | 27 | @JsonProperty 28 | public EthereumTableHandle getTable() { 29 | return table; 30 | } 31 | 32 | @JsonProperty 33 | public List getBlockRanges() { 34 | return blockRanges; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumTransactionHandle.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 4 | 5 | public enum EthereumTransactionHandle implements ConnectorTransactionHandle { 6 | INSTANCE 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/EthereumWeb3jProvider.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum; 2 | 3 | import org.web3j.protocol.Web3j; 4 | import org.web3j.protocol.http.HttpService; 5 | import org.web3j.protocol.infura.InfuraHttpService; 6 | import org.web3j.protocol.ipc.UnixIpcService; 7 | 8 | import javax.inject.Inject; 9 | 10 | public class EthereumWeb3jProvider { 11 | private final Web3j web3j; 12 | 13 | @Inject 14 | public EthereumWeb3jProvider(EthereumConnectorConfig config) { 15 | if (config.getEthereumJsonRpc() == null 16 | && config.getEthereumIpc() == null 17 | && config.getInfuraRpc() == null) { 18 | this.web3j = Web3j.build(new HttpService(EthereumConnectorConfig.DEFAULT_JSON_RPC)); 19 | } else if (config.getEthereumJsonRpc() != null 20 | && config.getEthereumIpc() == null 21 | && config.getInfuraRpc() == null) { 22 | this.web3j = Web3j.build(new HttpService(config.getEthereumJsonRpc())); 23 | } else if (config.getEthereumJsonRpc() == null 24 | && config.getEthereumIpc() != null 25 | && config.getInfuraRpc() == null) { 26 | this.web3j = Web3j.build(new UnixIpcService(config.getEthereumIpc())); 27 | } else if (config.getEthereumJsonRpc() == null 28 | && config.getEthereumIpc() == null 29 | && config.getInfuraRpc() != null) { 30 | this.web3j = Web3j.build(new InfuraHttpService(config.getInfuraRpc())); 31 | } else { 32 | throw new IllegalArgumentException("More than 1 Ethereum service providers found"); 33 | } 34 | } 35 | 36 | public Web3j getWeb3j() { 37 | return web3j; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/udfs/EthereumUDFs.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum.udfs; 2 | 3 | import com.facebook.presto.spi.function.Description; 4 | import com.facebook.presto.spi.function.ScalarFunction; 5 | import com.facebook.presto.spi.function.SqlType; 6 | import com.facebook.presto.common.type.StandardTypes; 7 | import im.xiaoyao.presto.ethereum.EthereumConnectorConfig; 8 | import io.airlift.configuration.ConfigurationLoader; 9 | import io.airlift.log.Logger; 10 | import io.airlift.slice.Slice; 11 | import org.web3j.protocol.Web3j; 12 | import org.web3j.protocol.core.DefaultBlockParameter; 13 | import org.web3j.protocol.http.HttpService; 14 | import org.web3j.protocol.infura.InfuraHttpService; 15 | import org.web3j.protocol.ipc.UnixIpcService; 16 | 17 | import java.io.IOException; 18 | import java.math.BigInteger; 19 | import java.util.Map; 20 | 21 | public class EthereumUDFs { 22 | private static final Logger log = Logger.get(EthereumUDFs.class); 23 | private static final String CONFIG_PATH = "etc/catalog/ethereum.properties"; 24 | private static final String JSONRPC_KEY = "ethereum.jsonrpc"; 25 | private static final String IPC_KEY = "ethereum.ipc"; 26 | private static final String INFURA_KEY = "ethereum.infura"; 27 | private static final String LATEST = "latest"; 28 | private static final String ZERO_X = "0x"; 29 | private static final Web3j web3j; 30 | 31 | // A hack, which I don't like 32 | static { 33 | log.info("Initializing Web3j in UDF..."); 34 | ConfigurationLoader configLoader = new ConfigurationLoader(); 35 | try { 36 | Map config = configLoader.loadPropertiesFrom(CONFIG_PATH); 37 | if (config.get(JSONRPC_KEY) == null 38 | && config.get(IPC_KEY) == null 39 | && config.get(INFURA_KEY) == null) { 40 | web3j = Web3j.build(new HttpService(EthereumConnectorConfig.DEFAULT_JSON_RPC)); 41 | } else if (config.get(JSONRPC_KEY) != null 42 | && config.get(IPC_KEY) == null 43 | && config.get(INFURA_KEY) == null) { 44 | web3j = Web3j.build(new HttpService(config.get(JSONRPC_KEY))); 45 | } else if (config.get(JSONRPC_KEY) == null 46 | && config.get(IPC_KEY) != null 47 | && config.get(INFURA_KEY) == null) { 48 | web3j = Web3j.build(new UnixIpcService(config.get(IPC_KEY))); 49 | } else if (config.get(JSONRPC_KEY) == null 50 | && config.get(IPC_KEY) == null 51 | && config.get(INFURA_KEY) != null) { 52 | web3j = Web3j.build(new InfuraHttpService(config.get(INFURA_KEY))); 53 | } else { 54 | throw new IllegalArgumentException("More than 1 Ethereum service providers found"); 55 | } 56 | } catch (IOException e) { 57 | throw new IllegalStateException("Cannot load config from " + CONFIG_PATH); 58 | } 59 | } 60 | 61 | @ScalarFunction("eth_gasPrice") 62 | @Description("Returns current gas price") 63 | @SqlType(StandardTypes.DOUBLE) 64 | public static double ethGasPrice() throws IOException { 65 | return web3j.ethGasPrice().send().getGasPrice().doubleValue(); 66 | } 67 | 68 | @ScalarFunction("eth_blockNumber") 69 | @Description("Returns current block number") 70 | @SqlType(StandardTypes.BIGINT) 71 | public static long ethBlockNumber() throws IOException { 72 | return web3j.ethBlockNumber().send().getBlockNumber().longValue(); 73 | } 74 | 75 | @ScalarFunction("eth_getBalance") 76 | @Description("Returns the balance of an address") 77 | @SqlType(StandardTypes.DOUBLE) 78 | public static double ethGetBalance(@SqlType(StandardTypes.VARCHAR) Slice address) throws IOException { 79 | return web3j.ethGetBalance(address.toStringUtf8(), DefaultBlockParameter.valueOf(LATEST)).send().getBalance().doubleValue(); 80 | } 81 | 82 | @ScalarFunction("eth_getBalance") 83 | @Description("Returns the balance of an address") 84 | @SqlType(StandardTypes.DOUBLE) 85 | public static double ethGetBalance(@SqlType(StandardTypes.VARCHAR) Slice address, @SqlType(StandardTypes.BIGINT) long blockNumber) throws IOException { 86 | return web3j.ethGetBalance(address.toStringUtf8(), DefaultBlockParameter.valueOf(BigInteger.valueOf(blockNumber))).send().getBalance().doubleValue(); 87 | } 88 | 89 | @ScalarFunction("eth_getBalance") 90 | @Description("Returns the balance of an address") 91 | @SqlType(StandardTypes.DOUBLE) 92 | public static double ethGetBalance(@SqlType(StandardTypes.VARCHAR) Slice address, @SqlType(StandardTypes.VARCHAR) Slice blockName) throws IOException { 93 | return web3j.ethGetBalance(address.toStringUtf8(), DefaultBlockParameter.valueOf(blockName.toStringUtf8())).send().getBalance().doubleValue(); 94 | } 95 | 96 | @ScalarFunction("eth_getTransactionCount") 97 | @Description("Returns the number of transactions from this address") 98 | @SqlType(StandardTypes.BIGINT) 99 | public static long ethGetTransactionCount(@SqlType(StandardTypes.VARCHAR) Slice address) throws IOException { 100 | return web3j.ethGetTransactionCount(address.toStringUtf8(), DefaultBlockParameter.valueOf(LATEST)).send().getTransactionCount().longValue(); 101 | } 102 | 103 | @ScalarFunction("eth_getTransactionCount") 104 | @Description("Returns the number of transactions from this address") 105 | @SqlType(StandardTypes.BIGINT) 106 | public static long ethGetTransactionCount(@SqlType(StandardTypes.VARCHAR) Slice address, @SqlType(StandardTypes.BIGINT) long blockNumber) throws IOException { 107 | return web3j.ethGetTransactionCount(address.toStringUtf8(), DefaultBlockParameter.valueOf(BigInteger.valueOf(blockNumber))).send().getTransactionCount().longValue(); 108 | } 109 | 110 | @ScalarFunction("eth_getTransactionCount") 111 | @Description("Returns the number of transactions from this address") 112 | @SqlType(StandardTypes.BIGINT) 113 | public static long ethGetTransactionCount(@SqlType(StandardTypes.VARCHAR) Slice address, @SqlType(StandardTypes.VARCHAR) Slice blockName) throws IOException { 114 | return web3j.ethGetTransactionCount(address.toStringUtf8(), DefaultBlockParameter.valueOf(blockName.toStringUtf8())).send().getTransactionCount().longValue(); 115 | } 116 | 117 | @ScalarFunction("fromWei") 118 | @Description("fromWei") 119 | @SqlType(StandardTypes.DOUBLE) 120 | public static double fromWei(@SqlType(StandardTypes.DOUBLE) double num, @SqlType(StandardTypes.VARCHAR) Slice unit) { 121 | String unitStr = unit.toStringUtf8().toUpperCase(); 122 | EthereumUnit u = EthereumUnit.valueOf(unitStr); 123 | return u.fromWei(num); 124 | } 125 | 126 | @ScalarFunction("toWei") 127 | @Description("toWei") 128 | @SqlType(StandardTypes.DOUBLE) 129 | public static double toWei(@SqlType(StandardTypes.DOUBLE) double num, @SqlType(StandardTypes.VARCHAR) Slice unit) { 130 | String unitStr = unit.toStringUtf8().toUpperCase(); 131 | EthereumUnit u = EthereumUnit.valueOf(unitStr); 132 | return u.toWei(num); 133 | } 134 | 135 | @ScalarFunction("isContract") 136 | @Description("isContract") 137 | @SqlType(StandardTypes.BOOLEAN) 138 | public static boolean isContract(@SqlType(StandardTypes.VARCHAR) Slice address) throws IOException { 139 | return !web3j.ethGetCode(address.toStringUtf8(), DefaultBlockParameter.valueOf(LATEST)).send().getCode().equals(ZERO_X); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/main/java/im/xiaoyao/presto/ethereum/udfs/EthereumUnit.java: -------------------------------------------------------------------------------- 1 | package im.xiaoyao.presto.ethereum.udfs; 2 | 3 | public enum EthereumUnit { 4 | WEI(1), 5 | KWEI(1E3), ADA(1E3), 6 | MWEI(1E6), BABBAGE(1E6), 7 | GWEI(1E9), SHANNON(1E9), 8 | SZABO(1E12), 9 | FINNEY(1E15), 10 | ETHER(1E18), 11 | KETHER(1E21), GRAND(1E21), EINSTEIN(1E21), 12 | METHER(1E24), 13 | GETHER(1E27), 14 | TETHER(1E30); 15 | 16 | private final double xwei; 17 | EthereumUnit(double v) { 18 | this.xwei = v; 19 | } 20 | 21 | public double fromWei(double num) { 22 | return num / xwei; 23 | } 24 | 25 | public double toWei(double num) { 26 | return num * xwei; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin: -------------------------------------------------------------------------------- 1 | im.xiaoyao.presto.ethereum.EthereumPlugin 2 | -------------------------------------------------------------------------------- /use-cases.sql: -------------------------------------------------------------------------------- 1 | -- Inspired by https://blog.ethereum.org/2015/08/18/frontier-first-100k-blocks/ 2 | -- The following SQL queries capture partially what was depicted in that post. 3 | 4 | -- The first 50 block times (in seconds): 5 | SELECT b.bn, (b.block_timestamp - a.block_timestamp) AS delta 6 | FROM 7 | (SELECT block_number AS bn, block_timestamp 8 | FROM block 9 | WHERE block_number>=1 AND block_number<=50) AS a 10 | JOIN 11 | (SELECT (block_number-1) AS bn, block_timestamp 12 | FROM block 13 | WHERE block_number>=2 AND block_number<=51) AS b 14 | ON a.bn=b.bn 15 | ORDER BY b.bn; 16 | 17 | -- Average block time (every 200th block from genesis to block 10000) 18 | WITH 19 | X AS (SELECT b.bn, (b.block_timestamp - a.block_timestamp) AS delta 20 | FROM 21 | (SELECT block_number AS bn, block_timestamp 22 | FROM block 23 | WHERE block_number>=1 AND block_number<=10000) AS a 24 | JOIN 25 | (SELECT (block_number-1) AS bn, block_timestamp 26 | FROM block 27 | WHERE block_number>=2 AND block_number<=10001) AS b 28 | ON a.bn=b.bn 29 | ORDER BY b.bn) 30 | SELECT min(bn) AS chunkStart, avg(delta) 31 | FROM 32 | (SELECT ntile(10000/200) OVER (ORDER BY bn) AS chunk, * FROM X) AS T 33 | GROUP BY chunk 34 | ORDER BY chunkStart; 35 | 36 | -- Biggest miners in first 100k blocks (address, blocks, %): 37 | SELECT block_miner, count(*) AS num, count(*)/100000.0 AS PERCENT 38 | FROM block 39 | WHERE block_number<=100000 40 | GROUP BY block_miner 41 | ORDER BY num DESC 42 | LIMIT 15; 43 | --------------------------------------------------------------------------------