├── .gitignore ├── LICENSE ├── README.md ├── index.js ├── package.json └── test └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Commenting this out is preferred by some people, see 24 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 25 | node_modules 26 | 27 | # Users Environment Variables 28 | .lock-wscript 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Daniel Cousens 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # blkdat-stream 2 | 3 | [![Version](http://img.shields.io/npm/v/blkdat-stream.svg)](https://www.npmjs.org/package/blkdat-stream) 4 | 5 | A lite blk\*.dat streaming module, useful for parsing the Bitcoin blockchain 6 | 7 | **Note**: For a high performance C++ parser, see https://github.com/dcousens/fast-dat-parser 8 | 9 | 10 | ### Example 11 | 12 | ``` javascript 13 | // usage: cat blk*.dat | node this.js 14 | 15 | var BlockStream = require('blkdat-stream') 16 | var blockStream = new BlockStream() // for testnet3: new BlockStream(0x0709110b) 17 | 18 | process.stdin.pipe(new BlockStream()).on('data', function (blockBuffer) { 19 | // ... now, parse the block data buffer (is an atomic block) 20 | }) 21 | ``` 22 | 23 | To parse the returned block data, use a library such as [bitcoinjs-lib](https://github.com/bitcoinjs/bitcoinjs-lib) or [bitcore](https://github.com/bitpay/bitcore). 24 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var through = require('through') 2 | 3 | module.exports = function BlkDatStream (blkMagicInt) { 4 | if (blkMagicInt === undefined) blkMagicInt = 0xd9b4bef9 5 | 6 | var buffers = [] 7 | var buffer = new Buffer(0) 8 | 9 | var blockLength 10 | var remaining = 0 11 | var needed = 0 12 | 13 | return through(function write (data) { 14 | remaining += data.length 15 | buffers.push(data) 16 | 17 | // early exit if we have a header, but not enough data 18 | if (needed !== 0 && remaining < needed) return 19 | 20 | // merge buffers 21 | buffer = Buffer.concat([buffer].concat(buffers), remaining) 22 | buffers = [] 23 | 24 | do { 25 | // do we need to parse a magic header? 26 | if (needed === 0) { 27 | // do we have enough for a magic header? 28 | if (remaining < 8) break 29 | 30 | // read magic number from the magic header 31 | var magicInt = buffer.readUInt32LE(0) 32 | if (magicInt !== blkMagicInt) throw new Error('Unexpected data') 33 | 34 | // read block length from the magic header 35 | blockLength = buffer.readUInt32LE(4) 36 | 37 | // include the magic header 38 | needed = 8 + blockLength 39 | 40 | // and loop 41 | continue 42 | } 43 | 44 | // read the block 45 | var block = buffer.slice(8, needed) 46 | 47 | // process block 48 | this.queue(block) 49 | 50 | // update stream information 51 | buffer = buffer.slice(needed) 52 | remaining = buffer.length 53 | needed = 0 54 | 55 | // do we [still] have enough data? 56 | } while (remaining >= needed) 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "blkdat-stream", 3 | "version": "1.0.2", 4 | "description": "A streamline blk*.dat stream module, useful for parsing the Bitcoin blockchain", 5 | "main": "index.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "scripts": { 10 | "standard": "standard", 11 | "test": "npm run standard && npm run unit", 12 | "unit": "echo \"Error: no test specified\"" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/dcousens/blkdat-stream.git" 17 | }, 18 | "keywords": [ 19 | "blockchain", 20 | "bitcoin", 21 | "block", 22 | "blkdat", 23 | "dat", 24 | "blk*.dat", 25 | "chain", 26 | "parser", 27 | "streams", 28 | "streaming", 29 | "stream" 30 | ], 31 | "author": "Daniel Cousens", 32 | "license": "MIT", 33 | "bugs": { 34 | "url": "https://github.com/dcousens/blkdat-stream/issues" 35 | }, 36 | "homepage": "https://github.com/dcousens/blkdat-stream", 37 | "dependencies": { 38 | "through": "^2.3.6" 39 | }, 40 | "devDependencies": { 41 | "combined-stream": "0.0.7", 42 | "glob": "^5.0.3", 43 | "standard": "^6.0.4" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var glob = require('glob') 3 | 4 | var CombinedStream = require('combined-stream') 5 | var BlkDatStream = require('../') 6 | 7 | glob(process.env.BLOCKS_DIR + '/blk*.dat', function (err, fileNames) { 8 | if (err) throw err 9 | 10 | var cs = new CombinedStream() 11 | 12 | fileNames.forEach(function (fileName) { 13 | cs.append(fs.createReadStream(fileName, { mode: '0444' })) 14 | }) 15 | 16 | var bds = new BlkDatStream() 17 | var j = 0 18 | bds.on('data', function () { 19 | console.log('>> Read block (', j, ')') 20 | j++ 21 | }) 22 | 23 | cs.pipe(bds) 24 | }) 25 | --------------------------------------------------------------------------------