├── LICENSE ├── README.md ├── config ├── default.json ├── template.callbacks.js └── template.extend-contract.js ├── elastic-ethereum.js ├── extend-contract.js └── package.json /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Nexus Development 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elastic-ethereum 2 | Indexes Ethereum contracts with Elasticsearch. 3 | 4 | Both and Geth and Elasticsearch need to be running. 5 | 6 | In order to configure Elastic Ethereum copy config/default.js to config/production.js and edit the connection configuration if necessary. 7 | 8 | Each contract you want to index needs to be added to the `contracts` section of the configuration. Each contract has two fields: 9 | 10 | * `address` is the address of the Ethereum contract that will have its events watched. 11 | * `index` is the name of the Elasticsearch index that will be used. 12 | 13 | Create config/`contract_key`.abi.json with the JSON ABI from the compiler. 14 | 15 | Copy config/template.callbacks.js to config/`contract_key`.callbacks.js and customize the callbacks. 16 | 17 | * onInit() is run everytime Elastic Ethereum starts or resumes indexing a contract. 18 | * onCreate() is only run when the contract is starting to be indexed the first time or re-indexed from the beginning. 19 | * getDeletes() is passed each event that is detected and should return a list of index ids to delete. 20 | * getDocuments() is passed each event that is detected and should return a list of documents that are new or updated. 21 | 22 | ## Run 23 | ``` 24 | export NODE_ENV=production 25 | node elastic-ethereum.js contract-key 26 | ``` 27 | 28 | ## Extra contract methods 29 | Once your contract is being successfully indexed in Elasticsearch, it makes sense to extend the contract object with some extra methods that will query the index. 30 | 31 | * Install Elastic Ethereum as a submodule of your node program. `npm install --save elastic-ethereum` 32 | * `elasticEthereum = require('elastic-ethereum');` 33 | * `cp -a elastic-ethereum/config .` 34 | * Configure config.production.js as above 35 | * copy config/template.extend-contract.js to config/`contract_key`.extend-contract.js 36 | * Implement additional methods 37 | * After you have obtained the contract object from web3, call elasticEthereum.extendContract(`contractKey`, contractObject) 38 | -------------------------------------------------------------------------------- /config/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "ethereum": { 3 | "provider": "http://localhost:8545" 4 | }, 5 | "elasticsearch": { 6 | "host": "localhost:9200" 7 | }, 8 | "contracts" : { 9 | "example": { 10 | "address": "0x", 11 | "index": "" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /config/template.callbacks.js: -------------------------------------------------------------------------------- 1 | var onInit = function() { 2 | } 3 | 4 | var onCreate = function() { 5 | } 6 | 7 | var getDeletes = function(log) { 8 | 9 | return {} 10 | } 11 | 12 | var getDocuments = function(log) { 13 | 14 | return {} 15 | } 16 | 17 | module.exports = { 18 | onInit: onInit, 19 | onCreate: onCreate, 20 | getDeletes: getDeletes, 21 | getDocuments: getDocuments 22 | }; 23 | -------------------------------------------------------------------------------- /config/template.extend-contract.js: -------------------------------------------------------------------------------- 1 | var extendContract = function(contract) { 2 | } 3 | 4 | module.exports = { 5 | extendContract: extendContract 6 | }; 7 | -------------------------------------------------------------------------------- /elastic-ethereum.js: -------------------------------------------------------------------------------- 1 | var Web3 = require('web3'); 2 | web3 = new Web3(); 3 | var elasticsearch = require('elasticsearch'); 4 | var config = require('config'); 5 | var commandLineArgs = require('command-line-args'); 6 | 7 | var cli = commandLineArgs([ 8 | { name: 'contract', alias: 'c', type: String, defaultOption: true }, 9 | { name: 'reindex', alias: 'r', type: Boolean } 10 | ]); 11 | 12 | var options = cli.parse(); 13 | index = config.contracts[options.contract].index; 14 | 15 | web3.setProvider(new web3.providers.HttpProvider(config.get('ethereum.provider'))); 16 | 17 | var abi = require('./config/' + options.contract + '.abi.json'); 18 | contract = web3.eth.contract(abi).at(config.contracts[options.contract].address); 19 | 20 | client = new elasticsearch.Client({ 21 | host: config.get('elasticsearch.host') 22 | }); 23 | 24 | var callbacks = require('./config/' + options.contract + '.callbacks.js'); 25 | callbacks.onInit(); 26 | 27 | var lastBlockNumber; 28 | var lastLogIndex; 29 | 30 | var queue = []; 31 | var processing = false; 32 | 33 | if (options.reindex) { 34 | client.indices.delete({index: index}).then(function() { 35 | init(); 36 | }, function() { 37 | console.log("error"); 38 | }); 39 | } 40 | else { 41 | init(); 42 | } 43 | 44 | function init() { 45 | // Get last log processed. 46 | client.indices.refresh({ 47 | index: config.index 48 | }, function (error, response) { 49 | console.log(error); 50 | client.get({ 51 | index: index, 52 | type: 'elastic-ethereum', 53 | id: 0 54 | }, function (error, response) { 55 | if (!error) { 56 | lastBlockNumber = response._source.lastBlockNumber; 57 | lastLogIndex = response._source.lastLogIndex; 58 | console.log('Last: ' + lastBlockNumber + ':' + lastLogIndex); 59 | } 60 | else { 61 | lastBlockNumber = 0; 62 | callbacks.onCreate(); 63 | } 64 | 65 | watch(); 66 | }) 67 | }); 68 | } 69 | 70 | function watch() { 71 | var filter = web3.eth.filter({fromBlock: lastBlockNumber, toBlock: 'latest', address: config.contracts[options.contract].address}); 72 | 73 | filter.watch(function(error, result) { 74 | // Check if we have indexed this log before. 75 | if (result.blockNumber == lastBlockNumber && result.logIndex <= lastLogIndex) { 76 | return; 77 | } 78 | 79 | console.log(result.blockNumber + ':' + result.logIndex); 80 | 81 | var deletes = callbacks.getDeletes(result); 82 | var documents = callbacks.getDocuments(result); 83 | 84 | queue.push({deletes: deletes, documents: documents}); 85 | 86 | client.index({ 87 | index: index, 88 | type: 'elastic-ethereum', 89 | id: 0, 90 | body: { 91 | lastBlockNumber: result.blockNumber, 92 | lastLogIndex: result.logIndex 93 | } 94 | }, function (error, response) { 95 | if (!error) { 96 | console.log(response); 97 | } 98 | }); 99 | 100 | if (!processing) { 101 | processQueue(); 102 | } 103 | }); 104 | } 105 | 106 | function processQueue() { 107 | if (queue.length == 0) { 108 | return; 109 | } 110 | processing = true; 111 | 112 | if (queue[0].hasOwnProperty('deletes')) { 113 | for (type in queue[0].deletes) { 114 | for (id in queue[0].deletes[type]) { 115 | client.delete({ 116 | index: index, 117 | type: type, 118 | id: queue[0].deletes[type][id] 119 | }, function (error, response) { 120 | console.log(response); 121 | if (!processing) { 122 | processQueue(); 123 | } 124 | }); 125 | } 126 | } 127 | } 128 | if (queue[0].hasOwnProperty('documents')) { 129 | for (type in queue[0].documents) { 130 | for (id in queue[0].documents[type]) { 131 | client.index({ 132 | index: index, 133 | type: type, 134 | id: id, 135 | body: queue[0].documents[type][id] 136 | }, function (error, response) { 137 | if (!error) { 138 | console.log(response); 139 | if (!processing) { 140 | processQueue(); 141 | } 142 | } 143 | }); 144 | } 145 | } 146 | } 147 | 148 | queue.shift(); 149 | 150 | processing = false; 151 | } 152 | -------------------------------------------------------------------------------- /extend-contract.js: -------------------------------------------------------------------------------- 1 | var elasticsearch = require('elasticsearch'); 2 | var config = require('config'); 3 | 4 | module.exports = function(contractName, contract) { 5 | 6 | elasticsearchClient = new elasticsearch.Client({ 7 | host: config.get('elasticsearch.host') 8 | }); 9 | 10 | var extendContract = require('./config/' + contractName + '.extend-contract.js'); 11 | extendContract(contract); 12 | } 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elastic-ethereum", 3 | "version": "0.0.1", 4 | "description": "Indexes Ethereum contracts with Elasticsearch", 5 | "main": "extra-methods.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/NexusDevelopment/elastic-ethereum.git" 12 | }, 13 | "keywords": [ 14 | "ethereum", 15 | "elasticsearch" 16 | ], 17 | "author": "Jonathan Brown ", 18 | "license": "Unlicense", 19 | "bugs": { 20 | "url": "https://github.com/NexusDevelopment/elastic-ethereum/issues" 21 | }, 22 | "dependencies": { 23 | "command-line-args": "^2.1.1", 24 | "config": "^1.16.0", 25 | "elasticsearch": "^9.0.0", 26 | "web3": "^0.15.1" 27 | } 28 | } 29 | --------------------------------------------------------------------------------