├── .gitignore ├── LICENSE ├── README.md ├── dbwrapper.js ├── debug.js ├── index.js ├── indexes ├── fee.js ├── mediantime.js ├── script.js ├── tx.js ├── txin.js ├── txo.js ├── types.js └── utils.js ├── package.json └── rpc.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017, Daniel Cousens 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # indexd 2 | [![build status](https://secure.travis-ci.org/dcousens/indexd.png)](http://travis-ci.org/dcousens/indexd) 3 | [![Version](https://img.shields.io/npm/v/indexd.svg)](https://www.npmjs.org/package/indexd) 4 | 5 | An external [bitcoind](https://github.com/bitcoin/bitcoin) index management service module. 6 | 7 | 8 | ## Indexes 9 | By default, this module includes a script, spents, transaction block, txout, tx, median time past and fee indexes. 10 | The module uses `getblockheader`, `getblockhash`, `getblock` and `getbestblockhash` RPC methods for blockchain synchronization; and `getrawmempool` for mempool synchronization. 11 | 12 | `-txindex` is not required for this module; but is still useful for individual transaction lookup (aka `txHex`). 13 | See https://github.com/bitcoinjs/indexd/issues/6 if you think an independent transaction index should be added. 14 | 15 | 16 | ## Usage 17 | Assumes [`yajrpc`](https://github.com/dcousens/yajrpc) is used for the provided bitcoind RPC object; and [`leveldown`](https://github.com/level/leveldown) for the database object. 18 | 19 | 20 | ### Conventions 21 | When conveying block height, `-1` represents unconfirmed (in the mempool). 22 | `null` represents unknown or missing. 23 | 24 | For example, the height of the transaction `ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff` in the Bitcoin blockchain is `null` (it doesn't exist!). 25 | 26 | 27 | ## LICENSE [ISC](LICENSE) 28 | -------------------------------------------------------------------------------- /dbwrapper.js: -------------------------------------------------------------------------------- 1 | // let debug = require('debug')('db') 2 | let typeforce = require('typeforce') 3 | let NIL = Buffer.alloc(0) 4 | 5 | function atomic () { 6 | let batch = this.batch() 7 | 8 | // debug('atomic') 9 | return { 10 | del: del.bind(batch), 11 | put: put.bind(batch), 12 | write: (callback) => batch.write(callback) 13 | } 14 | } 15 | 16 | function del (type, key, callback) { 17 | typeforce(type.keyType, key) 18 | key = type.key.encode(key) 19 | 20 | // debug('del', key.length) 21 | return this.del(key, callback) 22 | } 23 | 24 | function get (type, key, callback) { 25 | typeforce(type.keyType, key) 26 | key = type.key.encode(key) 27 | 28 | this.get(key, (err, value) => { 29 | if (err && (/NotFound/).test(err)) return callback() 30 | if (err) return callback(err) 31 | if (!type.value) return callback() 32 | 33 | callback(null, type.value.decode(value)) 34 | }) 35 | } 36 | 37 | function put (type, key, value, callback) { 38 | typeforce(type.keyType, key) 39 | typeforce(type.valueType, value) 40 | 41 | key = type.key.encode(key) 42 | value = type.value ? type.value.encode(value) : NIL 43 | // debug('put', key.length, value.length) 44 | 45 | return this.put(key, value, callback) 46 | } 47 | 48 | function iterator (type, options, forEach, callback) { 49 | typeforce({ 50 | gt: typeforce.maybe(type.keyType), 51 | gte: typeforce.maybe(type.keyType), 52 | lt: typeforce.maybe(type.keyType), 53 | lte: typeforce.maybe(type.keyType), 54 | limit: typeforce.maybe(typeforce.UInt53) 55 | }, options) 56 | 57 | // don't mutate 58 | options = Object.assign({}, options) 59 | options.gt = options.gt && type.key.encode(options.gt) 60 | options.gte = options.gte && type.key.encode(options.gte) 61 | options.lt = options.lt && type.key.encode(options.lt) 62 | options.lte = options.lte && type.key.encode(options.lte) 63 | if (!(options.gt || options.gte)) return callback(new RangeError('Missing minimum')) 64 | if (!(options.lt || options.lte)) return callback(new RangeError('Missing maximum')) 65 | 66 | let iterator = this.iterator(options) 67 | 68 | function loop (err, key, value) { 69 | // NOTE: ignores .end errors, if they occur 70 | if (err) return iterator.end(() => callback(err)) 71 | if (key === undefined || value === undefined) return iterator.end(callback) 72 | 73 | key = type.key.decode(key) 74 | value = type.value ? type.value.decode(value) : null 75 | forEach(key, value, iterator) 76 | 77 | iterator.next(loop) 78 | } 79 | 80 | iterator.next(loop) 81 | } 82 | 83 | module.exports = function wrap (db) { 84 | return { 85 | atomic: atomic.bind(db), 86 | del: del.bind(db), 87 | get: get.bind(db), 88 | iterator: iterator.bind(db), 89 | put: put.bind(db) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /debug.js: -------------------------------------------------------------------------------- 1 | let debug 2 | try { 3 | debug = require('debug') 4 | } catch (e) {} 5 | 6 | function noop () {} 7 | 8 | module.exports = debug || noop 9 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | let debug = require('./debug')('indexd') 2 | let dbwrapper = require('./dbwrapper') 3 | let { EventEmitter } = require('events') 4 | let parallel = require('run-parallel') 5 | let rpcUtil = require('./rpc') 6 | 7 | let FeeIndex = require('./indexes/fee') 8 | let MtpIndex = require('./indexes/mediantime') 9 | let ScriptIndex = require('./indexes/script') 10 | let TxIndex = require('./indexes/tx') 11 | let TxinIndex = require('./indexes/txin') 12 | let TxoIndex = require('./indexes/txo') 13 | 14 | function txoToString ({ txId, vout }) { 15 | return `${txId}:${vout}` 16 | } 17 | 18 | function Indexd (db, rpc) { 19 | this.db = dbwrapper(db) 20 | this.rpc = rpc 21 | this.emitter = new EventEmitter() // TODO: bind to this 22 | this.emitter.setMaxListeners(Infinity) 23 | this.indexes = { 24 | fee: new FeeIndex(), 25 | mtp: new MtpIndex(), 26 | script: new ScriptIndex(), 27 | tx: new TxIndex(), 28 | txin: new TxinIndex(), 29 | txo: new TxoIndex() 30 | } 31 | } 32 | 33 | Indexd.prototype.tips = function (callback) { 34 | let tasks = {} 35 | 36 | for (let indexName in this.indexes) { 37 | let index = this.indexes[indexName] 38 | tasks[indexName] = (next) => index.tip(this.db, next) 39 | } 40 | 41 | parallel(tasks, callback) 42 | } 43 | 44 | // recurses until `nextBlockId` is falsy 45 | Indexd.prototype.connectFrom = function (prevBlockId, blockId, callback) { 46 | this.tips((err, tips) => { 47 | if (err) return callback(err) 48 | 49 | let todo = {} 50 | for (let indexName in tips) { 51 | let tip = tips[indexName] 52 | if (tip && tip.blockId !== prevBlockId) continue 53 | if (indexName === 'fee') { 54 | if (!tips.txo) continue 55 | if (tip && tips.fee.height > tips.txo.height) continue 56 | } 57 | 58 | todo[indexName] = true 59 | } 60 | 61 | let todoList = Object.keys(todo) 62 | if (todoList.length === 0) return callback(new RangeError('Misconfiguration')) 63 | 64 | debug(`Downloading ${blockId} (for ${todoList})`) 65 | 66 | rpcUtil.block(this.rpc, blockId, (err, block) => { 67 | if (err) return callback(err) 68 | 69 | let atomic = this.db.atomic() 70 | let events // TODO 71 | let { height } = block 72 | debug(`Connecting ${blockId} @ ${height}`) 73 | 74 | // connect block to relevant chain tips 75 | for (let indexName in todo) { 76 | let index = this.indexes[indexName] 77 | if (!index.connect) continue 78 | 79 | index.connect(atomic, block, events) 80 | } 81 | 82 | atomic.write((err) => { 83 | if (err) return callback(err) 84 | debug(`Connected ${blockId} @ ${height}`) 85 | 86 | let self = this 87 | function loop (err) { 88 | if (err) return callback(err) 89 | 90 | // recurse until nextBlockId is falsy 91 | if (!block.nextBlockId) return callback(null, true) 92 | self.connectFrom(blockId, block.nextBlockId, callback) 93 | } 94 | 95 | if (!todo.fee) return loop() 96 | 97 | debug(`Connecting ${blockId} (2nd Order)`) 98 | let atomic2 = this.db.atomic() 99 | this.indexes.fee.connect2ndOrder(this.db, this.indexes.txo, atomic2, block, (err) => { 100 | if (err) return loop(err) 101 | 102 | debug(`Connected ${blockId} (2nd Order)`) 103 | atomic2.write(loop) 104 | }) 105 | }) 106 | }) 107 | }) 108 | } 109 | 110 | Indexd.prototype.disconnect = function (blockId, callback) { 111 | debug(`Disconnecting ${blockId}`) 112 | 113 | function fin (err) { 114 | if (err) return callback(err) 115 | debug(`Disconnected ${blockId}`) 116 | callback() 117 | } 118 | 119 | this.tips((err, tips) => { 120 | if (err) return fin(err) 121 | 122 | // TODO: fetch lazily 123 | rpcUtil.block(this.rpc, blockId, (err, block) => { 124 | if (err) return fin(err) 125 | 126 | let atomic = this.db.atomic() 127 | 128 | // disconnect block from relevant chain tips 129 | for (let indexName in this.indexes) { 130 | let index = this.indexes[indexName] 131 | let tip = tips[indexName] 132 | if (!tip) continue 133 | if (tip.blockId !== block.blockId) continue 134 | 135 | index.disconnect(atomic, block) 136 | } 137 | 138 | atomic.write(fin) 139 | }) 140 | }) 141 | } 142 | 143 | // empties the mempool 144 | Indexd.prototype.clear = function () { 145 | for (let indexName in this.indexes) { 146 | this.indexes[indexName].constructor() 147 | } 148 | } 149 | 150 | Indexd.prototype.lowestTip = function(callback) { 151 | this.tips((err, tips) => { 152 | if (err) return callback(err) 153 | 154 | let lowest 155 | for (let key in tips) { 156 | let tip = tips[key] 157 | if (!tip) return callback() 158 | if (!lowest) lowest = tip 159 | if (lowest.height < tip.height) continue 160 | lowest = tip 161 | } 162 | 163 | callback(null, lowest) 164 | }) 165 | } 166 | 167 | Indexd.prototype.__resync = function (done) { 168 | debug('resynchronizing') 169 | 170 | parallel({ 171 | bitcoind: (f) => rpcUtil.tip(this.rpc, f), 172 | indexd: (f) => this.lowestTip(f) 173 | }, (err, r) => { 174 | if (err) return done(err) 175 | 176 | // Step 0, genesis? 177 | if (!r.indexd) { 178 | debug('genesis') 179 | return rpcUtil.blockIdAtHeight(this.rpc, 0, (err, genesisId) => { 180 | if (err) return done(err) 181 | 182 | this.connectFrom(null, genesisId, done) 183 | }) 184 | } 185 | 186 | // Step 1, equal? 187 | debug('...', r) 188 | if (r.bitcoind.blockId === r.indexd.blockId) return done() 189 | 190 | // Step 2, is indexd behind? [aka, does bitcoind have the indexd tip] 191 | rpcUtil.headerJSON(this.rpc, r.indexd.blockId, (err, common) => { 192 | // if (err && /not found/.test(err.message)) return fin(err) // uh, burn it to the ground 193 | if (err) return done(err) 194 | 195 | // forked? 196 | if (common.confirmations === -1) { 197 | debug('forked') 198 | return this.disconnect(r.indexd.blockId, (err) => { 199 | if (err) return done(err) 200 | 201 | this.__resync(done) 202 | }) 203 | } 204 | 205 | // yes, indexd is behind 206 | debug('bitcoind is ahead') 207 | this.connectFrom(common.blockId, common.nextBlockId, done) 208 | }) 209 | }) 210 | } 211 | 212 | Indexd.prototype.tryResync = function (callback) { 213 | if (callback) { 214 | this.emitter.once('resync', callback) 215 | } 216 | 217 | if (this.syncing) return 218 | this.syncing = true 219 | 220 | let self = this 221 | function fin (err) { 222 | self.syncing = false 223 | self.emitter.emit('resync', err) 224 | } 225 | 226 | this.__resync((err, updated) => { 227 | if (err) return fin(err) 228 | if (updated) return this.tryResyncMempool(fin) 229 | fin() 230 | }) 231 | } 232 | 233 | Indexd.prototype.tryResyncMempool = function (callback) { 234 | rpcUtil.mempool(this.rpc, (err, txIds) => { 235 | if (err) return callback(err) 236 | 237 | this.clear() 238 | parallel(txIds.map((txId) => (next) => this.notify(txId, next)), callback) 239 | }) 240 | } 241 | 242 | Indexd.prototype.notify = function (txId, callback) { 243 | rpcUtil.transaction(this.rpc, txId, (err, tx) => { 244 | if (err) return callback(err) 245 | 246 | for (let indexName in this.indexes) { 247 | let index = this.indexes[indexName] 248 | 249 | if (!index.mempool) continue 250 | index.mempool(tx) 251 | } 252 | 253 | callback() 254 | }) 255 | } 256 | 257 | // QUERIES 258 | Indexd.prototype.blockIdByTransactionId = function (txId, callback) { 259 | this.indexes.tx.heightBy(this.db, txId, (err, height) => { 260 | if (err) return callback(err) 261 | if (height === -1) return callback() 262 | 263 | rpcUtil.blockIdAtHeight(this.rpc, height, callback) 264 | }) 265 | } 266 | 267 | Indexd.prototype.latestFeesForNBlocks = function (nBlocks, callback) { 268 | this.indexes.fee.latestFeesFor(this.db, nBlocks, callback) 269 | } 270 | 271 | // returns a txo { txId, vout, value, script }, by key { txId, vout } 272 | Indexd.prototype.txoByTxo = function (txo, callback) { 273 | this.indexes.txo.txoBy(this.db, txo, callback) 274 | } 275 | 276 | // returns the height at scId was first-seen (-1 if unconfirmed) 277 | Indexd.prototype.firstSeenScriptId = function (scId, callback) { 278 | this.indexes.script.firstSeenScriptId(this.db, scId, callback) 279 | } 280 | 281 | // returns a list of txIds with inputs/outputs from/to a { scId, heightRange, ?mempool } 282 | Indexd.prototype.transactionIdsByScriptRange = function (scRange, dbLimit, callback) { 283 | this.txosByScriptRange(scRange, dbLimit, (err, txos) => { 284 | if (err) return callback(err) 285 | 286 | let txIdSet = {} 287 | let tasks = txos.map((txo) => { 288 | txIdSet[txo.txId] = true 289 | return (next) => this.indexes.txin.txinBy(this.db, txo, next) 290 | }) 291 | 292 | parallel(tasks, (err, txins) => { 293 | if (err) return callback(err) 294 | 295 | txins.forEach((txin) => { 296 | if (!txin) return 297 | txIdSet[txin.txId] = true 298 | }) 299 | 300 | callback(null, Object.keys(txIdSet)) 301 | }) 302 | }) 303 | } 304 | 305 | // returns a list of txos { txId, vout, height, value } by { scId, heightRange, ?mempool } 306 | Indexd.prototype.txosByScriptRange = function (scRange, dbLimit, callback) { 307 | this.indexes.script.txosBy(this.db, scRange, dbLimit, callback) 308 | } 309 | 310 | // returns a list of (unspent) txos { txId, vout, height, value }, by { scId, heightRange, ?mempool } 311 | // XXX: despite txo queries being bound by heightRange, the UTXO status is up-to-date 312 | Indexd.prototype.utxosByScriptRange = function (scRange, dbLimit, callback) { 313 | this.txosByScriptRange(scRange, dbLimit, (err, txos) => { 314 | if (err) return callback(err) 315 | 316 | let taskMap = {} 317 | let unspentMap = {} 318 | 319 | txos.forEach((txo) => { 320 | let txoId = txoToString(txo) 321 | unspentMap[txoId] = txo 322 | taskMap[txoId] = (next) => this.indexes.txin.txinBy(this.db, txo, next) 323 | }) 324 | 325 | parallel(taskMap, (err, txinMap) => { 326 | if (err) return callback(err) 327 | 328 | let unspents = [] 329 | for (let txoId in txinMap) { 330 | let txin = txinMap[txoId] 331 | 332 | // has a txin, therefore spent 333 | if (txin) continue 334 | 335 | unspents.push(unspentMap[txoId]) 336 | } 337 | 338 | callback(null, unspents) 339 | }) 340 | }) 341 | } 342 | 343 | module.exports = Indexd 344 | -------------------------------------------------------------------------------- /indexes/fee.js: -------------------------------------------------------------------------------- 1 | let parallel = require('run-parallel') 2 | let typeforce = require('typeforce') 3 | let types = require('./types') 4 | let vstruct = require('varstruct') 5 | 6 | let FEEPREFIX = 0x81 7 | let FEETIP = types.tip(FEEPREFIX) 8 | let FEE = { 9 | keyType: typeforce.compile({ 10 | height: typeforce.UInt32 11 | }), 12 | key: vstruct([ 13 | ['prefix', vstruct.Value(vstruct.UInt8, FEEPREFIX)], 14 | ['height', vstruct.UInt32BE] // big-endian for lexicographical sort 15 | ]), 16 | valueType: typeforce.compile({ 17 | iqr: { 18 | q1: typeforce.UInt53, 19 | median: typeforce.UInt53, 20 | q3: typeforce.UInt53 21 | }, 22 | size: typeforce.UInt32 23 | }), 24 | value: vstruct([ 25 | ['iqr', vstruct([ 26 | ['q1', vstruct.UInt64LE], 27 | ['median', vstruct.UInt64LE], 28 | ['q3', vstruct.UInt64LE] 29 | ])], 30 | ['size', vstruct.UInt32LE] 31 | ]) 32 | } 33 | 34 | function FeeIndex () {} 35 | 36 | FeeIndex.prototype.tip = function (db, callback) { 37 | db.get(FEETIP, {}, callback) 38 | } 39 | 40 | function box (data) { 41 | if (data.length === 0) return { q1: 0, median: 0, q3: 0 } 42 | let quarter = (data.length / 4) | 0 43 | let midpoint = (data.length / 2) | 0 44 | 45 | return { 46 | q1: data[quarter], 47 | median: data[midpoint], 48 | q3: data[midpoint + quarter] 49 | } 50 | } 51 | 52 | FeeIndex.prototype.connect2ndOrder = function (db, txoIndex, atomic, block, callback) { 53 | let { height, transactions } = block 54 | 55 | let txTasks = [] 56 | transactions.forEach((tx) => { 57 | let { ins, outs, vsize } = tx 58 | let inAccum = 0 59 | let outAccum = 0 60 | let txoTasks = [] 61 | let coinbase = false 62 | 63 | ins.forEach((input, vin) => { 64 | if (coinbase) return 65 | if (input.coinbase) { 66 | coinbase = true 67 | return 68 | } 69 | 70 | let { prevTxId, vout } = input 71 | txoTasks.push((next) => { 72 | txoIndex.txoBy(db, { txId: prevTxId, vout }, (err, txo) => { 73 | if (err) return next(err) 74 | if (!txo) return next(new Error(`Missing ${prevTxId}:${vout}`)) 75 | 76 | inAccum += txo.value 77 | next() 78 | }) 79 | }) 80 | }) 81 | 82 | outs.forEach(({ value }, vout) => { 83 | if (coinbase) return 84 | outAccum += value 85 | }) 86 | 87 | txTasks.push((next) => { 88 | if (coinbase) return next(null, 0) 89 | 90 | parallel(txoTasks, (err) => { 91 | if (err) return next(err) 92 | let fee = inAccum - outAccum 93 | let feeRate = Math.floor(fee / vsize) 94 | 95 | next(null, feeRate) 96 | }) 97 | }) 98 | }) 99 | 100 | parallel(txTasks, (err, feeRates) => { 101 | if (err) return callback(err) 102 | feeRates = feeRates.sort((a, b) => a - b) 103 | 104 | atomic.put(FEE, { height }, { 105 | iqr: box(feeRates), 106 | size: block.strippedsize 107 | }) 108 | atomic.put(FEETIP, {}, block) 109 | 110 | callback() 111 | }) 112 | } 113 | 114 | FeeIndex.prototype.disconnect = function (atomic, block) { 115 | let { height } = block 116 | 117 | atomic.del(FEE, { height }) 118 | atomic.put(FEETIP, {}, { blockId: block.prevBlockId, height }) 119 | } 120 | 121 | FeeIndex.prototype.latestFeesFor = function (db, nBlocks, callback) { 122 | db.get(FEETIP, {}, (err, tip) => { 123 | if (err) return callback(err) 124 | if (!tip) return callback(null, []) 125 | 126 | let { height: maxHeight } = tip 127 | let results = [] 128 | 129 | db.iterator(FEE, { 130 | gte: { 131 | height: maxHeight - (nBlocks - 1) 132 | }, 133 | limit: nBlocks 134 | }, ({ height }, { fees, size }) => { 135 | results.push({ height, fees, size }) 136 | }, (err) => callback(err, results)) 137 | }) 138 | } 139 | 140 | module.exports = FeeIndex 141 | module.exports.types = { 142 | data: FEE, 143 | tip: FEETIP 144 | } 145 | -------------------------------------------------------------------------------- /indexes/mediantime.js: -------------------------------------------------------------------------------- 1 | let typeforce = require('typeforce') 2 | let types = require('./types') 3 | let vstruct = require('varstruct') 4 | 5 | let MTPPREFIX = 0x83 6 | let MTPTIP = types.tip(MTPPREFIX) 7 | let MTP = { 8 | keyType: typeforce.compile({ 9 | medianTime: typeforce.UInt32, 10 | height: typeforce.UInt32 11 | }), 12 | key: vstruct([ 13 | ['prefix', vstruct.Value(vstruct.UInt8, MTPPREFIX)], 14 | ['medianTime', vstruct.UInt32BE], // big-endian for lexicographical sort 15 | ['height', vstruct.UInt32LE] 16 | ]), 17 | valueType: typeforce.Null 18 | } 19 | 20 | function MtpIndex () {} 21 | 22 | MtpIndex.prototype.tip = function (db, callback) { 23 | db.get(MTPTIP, {}, callback) 24 | } 25 | 26 | MtpIndex.prototype.connect = function (atomic, block) { 27 | let { height, medianTime } = block 28 | 29 | atomic.put(MTP, { medianTime, height }) 30 | atomic.put(MTPTIP, {}, block) 31 | } 32 | 33 | MtpIndex.prototype.disconnect = function (atomic, block) { 34 | let { height, medianTime } = block 35 | 36 | atomic.del(MTP, { medianTime, height }) 37 | atomic.put(MTPTIP, {}, { blockId: block.prevBlockId, height }) 38 | } 39 | 40 | module.exports = MtpIndex 41 | module.exports.types = { 42 | data: MTP, 43 | tip: MTPTIP 44 | } 45 | -------------------------------------------------------------------------------- /indexes/script.js: -------------------------------------------------------------------------------- 1 | let crypto = require('crypto') 2 | let types = require('./types') 3 | let typeforce = require('typeforce') 4 | let vstruct = require('varstruct') 5 | let utils = require('./utils') 6 | 7 | let SCRIPTPREFIX = 0x33 8 | let SCRIPTTIP = types.tip(SCRIPTPREFIX) 9 | let SCRIPT = { 10 | keyType: typeforce.compile({ 11 | scId: typeforce.HexN(64), 12 | height: typeforce.UInt32, 13 | txId: typeforce.HexN(64), 14 | vout: typeforce.UInt32 15 | }), 16 | key: vstruct([ 17 | ['prefix', vstruct.Value(vstruct.UInt8, SCRIPTPREFIX)], 18 | ['scId', vstruct.String(32, 'hex')], 19 | ['height', vstruct.UInt32BE], // big-endian for lexicographical sort 20 | ['txId', vstruct.String(32, 'hex')], 21 | ['vout', vstruct.UInt32LE] 22 | ]), 23 | valueType: typeforce.compile({ 24 | value: typeforce.UInt53, 25 | coinbase: typeforce.UInt8 26 | }), 27 | value: vstruct([ 28 | ['value', vstruct.UInt64LE], 29 | ['coinbase', vstruct.Byte] 30 | ]) 31 | } 32 | 33 | function sha256 (buffer) { 34 | return crypto.createHash('sha256') 35 | .update(buffer) 36 | .digest('hex') 37 | } 38 | 39 | function ScriptIndex () { 40 | this.scripts = {} 41 | } 42 | 43 | ScriptIndex.prototype.tip = function (db, callback) { 44 | db.get(SCRIPTTIP, {}, callback) 45 | } 46 | 47 | ScriptIndex.prototype.mempool = function (tx, events) { 48 | let { txId, outs } = tx 49 | 50 | outs.forEach(({ vout, script, value }) => { 51 | let scId = sha256(script) 52 | utils.getOrSetDefault(this.scripts, scId, []) 53 | .push({ txId, vout, height: -1, value }) 54 | 55 | if (events) events.push(['script', scId, null, txId, vout, value]) 56 | }) 57 | } 58 | 59 | ScriptIndex.prototype.connect = function (atomic, block, events) { 60 | let { height, transactions } = block 61 | 62 | transactions.forEach((tx) => { 63 | let { txId, outs } = tx 64 | 65 | let coinbase = (tx.ins.reduce((cb, txin) => cb || ('coinbase' in txin), false))?1:0 66 | 67 | outs.forEach(({ vout, script, value }) => { 68 | let scId = sha256(script) 69 | atomic.put(SCRIPT, { scId, height, txId, vout }, { value, coinbase }) 70 | 71 | if (events) events.push(['script', scId, height, txId, vout, value]) 72 | }) 73 | }) 74 | 75 | atomic.put(SCRIPTTIP, {}, block) 76 | } 77 | 78 | ScriptIndex.prototype.disconnect = function (atomic, block) { 79 | let { height, transactions } = block 80 | 81 | transactions.forEach((tx) => { 82 | let { txId, outs } = tx 83 | 84 | outs.forEach(({ vout, script }) => { 85 | let scId = sha256(script) 86 | atomic.del(SCRIPT, { scId, height, txId, vout }) 87 | }) 88 | }) 89 | 90 | atomic.put(SCRIPTTIP, {}, { blockId: block.prevBlockId, height }) 91 | } 92 | 93 | let ZERO64 = '0000000000000000000000000000000000000000000000000000000000000000' 94 | let MAX64 = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' 95 | 96 | // returns the height at scId was first-seen (-1 if unconfirmed, null if unknown) 97 | ScriptIndex.prototype.firstSeenScriptId = function (db, scId, callback) { 98 | let result = null 99 | db.iterator(SCRIPT, { 100 | gte: { scId, height: 0, txId: ZERO64, vout: 0 }, 101 | lt: { scId, height: 0xffffffff, txId: ZERO64, vout: 0 }, 102 | limit: 1 103 | }, ({ height }) => { 104 | result = height 105 | }, (err) => { 106 | if (err) return callback(err) 107 | if (result !== null) return callback(null, result) 108 | 109 | let mem = this.scripts[scId] 110 | if (mem) return callback(null, -1) 111 | callback(null, null) 112 | }) 113 | } 114 | 115 | // XXX: if heightRange distance is < 2, the limit is ignored 116 | // -- could be rectified by supporting a minimum txId value (aka, last retrieved) 117 | // 118 | // returns a list of { txId, vout, height, value } by { scId, heightRange: [from, to] } 119 | ScriptIndex.prototype.txosBy = function (db, { scId, heightRange, mempool }, maxRows, callback) { 120 | let [fromHeight, toHeight] = heightRange 121 | let distance = toHeight - fromHeight 122 | if (distance < 0) return callback(null, []) 123 | if (distance < 2) maxRows = Infinity 124 | fromHeight = Math.min(Math.max(0, fromHeight), 0xffffffff) 125 | toHeight = Math.min(Math.max(0, toHeight), 0xffffffff) 126 | 127 | let results = [] 128 | if (mempool && (scId in this.scripts)) { 129 | results = this.scripts[scId].concat() 130 | } 131 | 132 | db.iterator(SCRIPT, { 133 | gte: { scId, height: fromHeight, txId: ZERO64, vout: 0 }, 134 | lt: { scId, height: toHeight, txId: MAX64, vout: 0xffffffff }, 135 | limit: maxRows + 1 136 | }, ({ height, txId, vout }, { value, coinbase }, __iterator) => { 137 | results.push({ 138 | txId, vout, height, value, coinbase 139 | }) 140 | 141 | if (results.length > maxRows) return __iterator.end((err) => callback(err || new RangeError('Exceeded Limit'))) 142 | }, (err) => callback(err, results)) 143 | } 144 | 145 | module.exports = ScriptIndex 146 | module.exports.types = { 147 | data: SCRIPT, 148 | tip: SCRIPTTIP 149 | } 150 | -------------------------------------------------------------------------------- /indexes/tx.js: -------------------------------------------------------------------------------- 1 | let types = require('./types') 2 | let typeforce = require('typeforce') 3 | let vstruct = require('varstruct') 4 | 5 | let TXPREFIX = 0x35 6 | let TXTIP = types.tip(TXPREFIX) 7 | let TX = { 8 | keyType: typeforce.compile({ 9 | txId: typeforce.HexN(64) 10 | }), 11 | key: vstruct([ 12 | ['prefix', vstruct.Value(vstruct.UInt8, TXPREFIX)], 13 | ['txId', vstruct.String(32, 'hex')] 14 | ]), 15 | valueType: typeforce.compile({ 16 | height: typeforce.UInt32 17 | }), 18 | value: vstruct([ 19 | ['height', vstruct.UInt32LE] 20 | ]) 21 | } 22 | 23 | function TxIndex () { 24 | this.txs = {} 25 | } 26 | 27 | TxIndex.prototype.tip = function (db, callback) { 28 | db.get(TXTIP, {}, callback) 29 | } 30 | 31 | TxIndex.prototype.mempool = function (tx, events) { 32 | let { txId } = tx 33 | 34 | this.txs[txId] = true 35 | } 36 | 37 | TxIndex.prototype.connect = function (atomic, block, events) { 38 | let { height, transactions } = block 39 | 40 | transactions.forEach((tx) => { 41 | let { txId } = tx 42 | atomic.put(TX, { txId }, { height }) 43 | }) 44 | 45 | atomic.put(TXTIP, {}, block) 46 | } 47 | 48 | TxIndex.prototype.disconnect = function (atomic, block) { 49 | let { height, transactions } = block 50 | 51 | transactions.forEach((tx) => { 52 | let { txId } = tx 53 | 54 | atomic.del(TX, { txId }) 55 | }) 56 | 57 | atomic.put(TXTIP, {}, { blockId: block.prevBlockId, height }) 58 | } 59 | 60 | // returns the height (-1 if unconfirmed, null if unknown) of a transaction, by txId 61 | TxIndex.prototype.heightBy = function (db, txId, callback) { 62 | let mem = this.txs[txId] 63 | if (mem) return callback(null, -1) 64 | 65 | db.get(TX, { txId }, (err, result) => { 66 | if (err) return callback(err) 67 | if (!result) return callback(null, null) 68 | 69 | callback(null, result.height) 70 | }) 71 | } 72 | 73 | module.exports = TxIndex 74 | module.exports.types = { 75 | data: TX, 76 | tip: TXTIP 77 | } 78 | -------------------------------------------------------------------------------- /indexes/txin.js: -------------------------------------------------------------------------------- 1 | let types = require('./types') 2 | let typeforce = require('typeforce') 3 | let vstruct = require('varstruct') 4 | let utils = require('./utils') 5 | 6 | let TXINPREFIX = 0x32 7 | let TXINTIP = types.tip(TXINPREFIX) 8 | let TXIN = { 9 | keyType: typeforce.compile({ 10 | txId: typeforce.HexN(64), 11 | vout: typeforce.UInt32 12 | }), 13 | key: vstruct([ 14 | ['prefix', vstruct.Value(vstruct.UInt8, TXINPREFIX)], 15 | ['txId', vstruct.String(32, 'hex')], 16 | ['vout', vstruct.UInt32LE] 17 | ]), 18 | valueType: typeforce.compile({ 19 | txId: typeforce.HexN(64), 20 | vin: typeforce.UInt32, 21 | coinbase: typeforce.UInt8 22 | }), 23 | value: vstruct([ 24 | ['txId', vstruct.String(32, 'hex')], 25 | ['vin', vstruct.UInt32LE], 26 | ['coinbase', vstruct.Byte] 27 | ]) 28 | } 29 | 30 | function TxinIndex () { 31 | this.txins = {} 32 | } 33 | 34 | TxinIndex.prototype.tip = function (db, callback) { 35 | db.get(TXINTIP, {}, callback) 36 | } 37 | 38 | TxinIndex.prototype.mempool = function (tx, events) { 39 | let { txId, ins } = tx 40 | 41 | ins.forEach((input, vin) => { 42 | if (input.coinbase) return 43 | let { prevTxId, vout } = input 44 | 45 | utils.getOrSetDefault(this.txins, `${prevTxId}:${vout}`, []) 46 | .push({ txId, vin }) 47 | 48 | if (events) events.push(['txin', `${prevTxId}:${vout}`, txId, vin]) 49 | }) 50 | } 51 | 52 | TxinIndex.prototype.connect = function (atomic, block, events) { 53 | let { transactions } = block 54 | 55 | transactions.forEach((tx) => { 56 | let { txId, ins } = tx 57 | 58 | ins.forEach((input, vin) => { 59 | let coinbase = ('coinbase' in input)?1:0 60 | 61 | let { prevTxId, vout } = input 62 | 63 | if (!prevTxId) { 64 | prevTxId = '0000000000000000000000000000000000000000000000000000000000000000' 65 | vout = 0xffffffff 66 | } 67 | 68 | atomic.put(TXIN, { txId: prevTxId, vout }, { txId, vin, coinbase }) 69 | 70 | if (events) events.push(['txin', `${prevTxId}:${vout}`, txId, vin]) 71 | }) 72 | }) 73 | 74 | atomic.put(TXINTIP, {}, block) 75 | } 76 | 77 | TxinIndex.prototype.disconnect = function (atomic, block) { 78 | let { height, transactions } = block 79 | 80 | transactions.forEach((tx) => { 81 | let { txId, outs } = tx 82 | 83 | outs.forEach(({ value, vout }) => { 84 | atomic.del(TXIN, { txId, vout }) 85 | }) 86 | }) 87 | 88 | atomic.put(TXINTIP, {}, { blockId: block.prevBlockId, height }) 89 | } 90 | 91 | // returns a txin { txId, vin } by { txId, vout } 92 | TxinIndex.prototype.txinBy = function (db, txo, callback) { 93 | let { txId, vout } = txo 94 | let mem = this.txins[`${txId}:${vout}`] 95 | if (mem) return callback(null, mem[0]) // XXX: returns first-seen only 96 | 97 | db.get(TXIN, txo, callback) 98 | } 99 | 100 | module.exports = TxinIndex 101 | module.exports.types = { 102 | data: TXIN, 103 | tip: TXINTIP 104 | } 105 | -------------------------------------------------------------------------------- /indexes/txo.js: -------------------------------------------------------------------------------- 1 | let types = require('./types') 2 | let typeforce = require('typeforce') 3 | let varuint = require('varuint-bitcoin') 4 | let vstruct = require('varstruct') 5 | 6 | let TXOPREFIX = 0x34 7 | let TXOTIP = types.tip(TXOPREFIX) 8 | let TXO = { 9 | keyType: typeforce.compile({ 10 | txId: typeforce.HexN(64), 11 | vout: typeforce.UInt32 12 | }), 13 | key: vstruct([ 14 | ['prefix', vstruct.Value(vstruct.UInt8, TXOPREFIX)], 15 | ['txId', vstruct.String(32, 'hex')], 16 | ['vout', vstruct.UInt32LE] 17 | ]), 18 | valueType: typeforce.compile({ 19 | value: typeforce.UInt53, 20 | script: typeforce.Buffer, 21 | coinbase: typeforce.UInt8 22 | }), 23 | value: vstruct([ 24 | ['value', vstruct.UInt64LE], 25 | ['script', vstruct.VarBuffer(varuint)], 26 | ['coinbase', vstruct.Byte] 27 | ]) 28 | } 29 | 30 | function TxoIndex () { 31 | this.txos = {} 32 | } 33 | 34 | TxoIndex.prototype.tip = function (db, callback) { 35 | db.get(TXOTIP, {}, callback) 36 | } 37 | 38 | TxoIndex.prototype.mempool = function (tx) { 39 | let { txId, outs } = tx 40 | 41 | outs.forEach(({ script, value, vout }) => { 42 | this.txos[`${txId}:${vout}`] = { script, value } 43 | }) 44 | } 45 | 46 | TxoIndex.prototype.connect = function (atomic, block) { 47 | let { transactions } = block 48 | 49 | transactions.forEach((tx) => { 50 | let { txId, outs } = tx 51 | 52 | let coinbase = (tx.ins.reduce((cb, txin) => cb || ('coinbase' in txin), false))?1:0 53 | 54 | outs.forEach(({ script, value, vout }) => { 55 | atomic.put(TXO, { txId, vout }, { value, script, coinbase }) 56 | }) 57 | }) 58 | 59 | atomic.put(TXOTIP, {}, block) 60 | } 61 | 62 | TxoIndex.prototype.disconnect = function (atomic, block) { 63 | let { height, transactions } = block 64 | 65 | transactions.forEach((tx) => { 66 | let { txId, outs } = tx 67 | 68 | outs.forEach(({ value, vout }) => { 69 | atomic.del(TXO, { txId, vout }) 70 | }) 71 | }) 72 | 73 | atomic.put(TXOTIP, {}, { blockId: block.prevBlockId, height }) 74 | } 75 | 76 | // returns a txo { txId, vout, value, script } by { txId, vout } 77 | TxoIndex.prototype.txoBy = function (db, txo, callback) { 78 | let { txId, vout } = txo 79 | let mem = this.txos[`${txId}:${vout}`] 80 | if (mem) return callback(null, mem) 81 | 82 | db.get(TXO, txo, callback) 83 | } 84 | 85 | module.exports = TxoIndex 86 | module.exports.types = { 87 | data: TXO, 88 | tip: TXOTIP 89 | } 90 | -------------------------------------------------------------------------------- /indexes/types.js: -------------------------------------------------------------------------------- 1 | let typeforce = require('typeforce') 2 | let vstruct = require('varstruct') 3 | 4 | function tip (prefix) { 5 | return { 6 | keyType: {}, 7 | key: vstruct([ 8 | ['prefix', vstruct.Value(vstruct.UInt8, prefix)] 9 | ]), 10 | valueType: { 11 | blockId: typeforce.HexN(64), 12 | height: typeforce.UInt32 13 | }, 14 | value: vstruct([ 15 | ['blockId', vstruct.String(32, 'hex')], 16 | ['height', vstruct.UInt32LE] 17 | ]) 18 | } 19 | } 20 | 21 | module.exports = { tip } 22 | -------------------------------------------------------------------------------- /indexes/utils.js: -------------------------------------------------------------------------------- 1 | function getOrSetDefault (object, key, defaultValue) { 2 | let existing = object[key] 3 | if (existing !== undefined) return existing 4 | object[key] = defaultValue 5 | return defaultValue 6 | } 7 | 8 | module.exports = { getOrSetDefault } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "indexd", 3 | "version": "0.9.1", 4 | "description": "", 5 | "main": "index.js", 6 | "dependencies": { 7 | "run-parallel": "^1.1.6", 8 | "typeforce": "^1.10.6", 9 | "varstruct": "^6.1.1", 10 | "varuint-bitcoin": "^1.1.0" 11 | }, 12 | "optionalDependencies": { 13 | "debug": "*" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/dcousens/indexd.git" 18 | }, 19 | "scripts": {}, 20 | "author": "Daniel Cousens", 21 | "license": "ISC" 22 | } 23 | -------------------------------------------------------------------------------- /rpc.js: -------------------------------------------------------------------------------- 1 | let debug = require('./debug')('indexd:rpc') 2 | 3 | function rpcd (rpc, method, params, done) { 4 | debug(method, params) 5 | rpc(method, params, (err, result) => { 6 | if (err) debug(method, params, err) 7 | if (err) return done(err) 8 | 9 | done(null, result) 10 | }) 11 | } 12 | 13 | function augment (tx) { 14 | delete tx.hex 15 | tx.txId = tx.txid 16 | delete tx.txid 17 | tx.vin.forEach((input) => { 18 | input.prevTxId = input.txid 19 | delete input.txid 20 | }) 21 | tx.vout.forEach((output) => { 22 | output.script = Buffer.from(output.scriptPubKey.hex, 'hex') 23 | delete output.scriptPubKey 24 | output.value = Math.round(output.value * 1e8) 25 | output.vout = output.n 26 | delete output.n 27 | }) 28 | tx.ins = tx.vin 29 | tx.outs = tx.vout 30 | delete tx.vin 31 | delete tx.vout 32 | return tx 33 | } 34 | 35 | function block (rpc, blockId, done) { 36 | rpcd(rpc, 'getblock', [blockId, 2], (err, block) => { 37 | if (err) return done(err) 38 | 39 | block.blockId = blockId 40 | delete block.hash 41 | block.nextBlockId = block.nextblockhash 42 | delete block.nextblockhash 43 | block.prevBlockId = block.previousblockhash 44 | delete block.prevblockhash 45 | block.medianTime = block.mediantime 46 | delete block.mediantime 47 | 48 | block.transactions = block.tx.map(t => augment(t)) 49 | delete block.tx 50 | 51 | done(null, block) 52 | }) 53 | } 54 | 55 | function blockIdAtHeight (rpc, height, done) { 56 | rpcd(rpc, 'getblockhash', [height], done) 57 | } 58 | 59 | function headerJSON (rpc, blockId, done) { 60 | rpcd(rpc, 'getblockheader', [blockId, true], (err, header) => { 61 | if (err) return done(err) 62 | 63 | header.blockId = blockId 64 | delete header.hash 65 | header.nextBlockId = header.nextblockhash 66 | delete header.nextblockhash 67 | 68 | done(null, header) 69 | }) 70 | } 71 | 72 | function mempool (rpc, done) { 73 | rpcd(rpc, 'getrawmempool', [false], done) 74 | } 75 | 76 | function tip (rpc, done) { 77 | rpcd(rpc, 'getchaintips', [], (err, tips) => { 78 | if (err) return done(err) 79 | 80 | let { 81 | hash: blockId, 82 | height 83 | } = tips.filter(x => x.status === 'active').pop() 84 | 85 | done(null, { blockId, height }) 86 | }) 87 | } 88 | 89 | function transaction (rpc, txId, next, forgiving) { 90 | rpcd(rpc, 'getrawtransaction', [txId, true], (err, tx) => { 91 | if (err) { 92 | if (forgiving && /No such mempool or blockchain transaction/.test(err)) return next() 93 | return next(err) 94 | } 95 | 96 | next(null, augment(tx)) 97 | }) 98 | } 99 | 100 | module.exports = { 101 | block, 102 | blockIdAtHeight, 103 | headerJSON, 104 | mempool, 105 | tip, 106 | transaction 107 | } 108 | --------------------------------------------------------------------------------