├── go.mod
├── .gitignore
├── LICENSE
├── main.go
├── README.md
├── lib
    ├── types.go
    └── reader.go
└── go.sum


/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/0xb10c/mempool-dat
2 | 
3 | go 1.12
4 | 
5 | require (
6 | 	github.com/btcsuite/btcd v0.0.0-20190427004231-96897255fd17
7 | 	github.com/gizak/termui/v3 v3.0.0
8 | )
9 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Binaries for programs and plugins
 2 | *.exe
 3 | *.exe~
 4 | *.dll
 5 | *.so
 6 | *.dylib
 7 | 
 8 | # Test binary, build with `go test -c`
 9 | *.test
10 | 
11 | # Output of the go coverage tool, specifically when used with LiteIDE
12 | *.out
13 | 
14 | main
15 | *.dat
16 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 0xB10C
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 
 6 | 	mempoolDat "github.com/0xb10c/mempool-dat/lib"
 7 | )
 8 | 
 9 | const path string = "mempool.dat"
10 | 
11 | func main() {
12 | 	// please handle errors, omitted for brevity 
13 | 	mempool, _ := mempoolDat.ReadMempoolFromPath(path)
14 | 
15 | 	// prints the version and number of tx
16 | 	fmt.Println(mempool.GetFileHeader())
17 | 
18 | 	// get all mempool entries with GetMempoolEntries (here only the first three are used)
19 | 	for _, e := range mempool.GetMempoolEntries()[:3]{
20 | 		fmt.Println(e.Info())
21 | 	}
22 | 
23 | 	// get the firstSeen timestamp of a transaction
24 | 	fmt.Println(mempool.GetMempoolEntries()[4].GetFirstSeen())
25 | 
26 | 	/* Furthermore you can use the full functionality of MsgTx (https://godoc.org/github.com/btcsuite/btcd/wire#MsgTx):
27 | 		  - transaction has a witness (spends a SegWit output)? with <entry>.transaction.HasWitness() 
28 | 		  - access input and outputs
29 | 		  - ...
30 | 		 But there is no way to get feerates for transactions, because they are (rightfully) not stored in the `mempool.dat`.
31 | 		 You'd have to the current UTXO set to get input amounts (which you need to calculate the fees)
32 | 	*/
33 | }
34 | 
35 | func readFile(path string) (mempool mempoolDat.Mempool){
36 | 	mempool, err := mempoolDat.ReadMempoolFromPath(path)
37 | 	if err != nil {
38 | 		fmt.Println(err.Error())
39 | 	}
40 | 	return 
41 | }
42 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # mempool-dat
 2 | 
 3 | [![GoDoc](https://godoc.org/github.com/0xB10C/mempool-dat/lib?status.svg)](https://godoc.org/github.com/0xB10C/mempool-dat/lib)
 4 | 
 5 | This Go package parses bitcoin mempool `mempool.dat` files.
 6 | These are wirtten by Bitcoin Core v0.14+ on shutdown and since v0.16.0 with the RPC `savemempool`.
 7 | 
 8 | The package offers access to the `mempool.dat`
 9 | - header: version and number of transactions
10 | - mempool entries: raw transaction (here parsed as [btcsuite/btcd/wire MsgTx](https://godoc.org/github.com/btcsuite/btcd/wire#MsgTx)), first seen timestamp and the feeDelta  
11 | - and the not-parsed mapDeltas as byte slices 
12 | 
13 | You may see this package as Work-In-Progress. There are no tests yet. 
14 | 
15 | ## Example usage
16 | 
17 | For a runnable version of this snippet see [main.go](./main.go).
18 | 
19 | 
20 | ```go
21 | import (
22 |   "fmt"
23 |   mempoolDat "github.com/0xb10c/mempool-dat/lib"
24 | )
25 | 
26 | [...]
27 | 
28 | const path string = "mempool.dat"
29 | 
30 | [...]
31 | 
32 | // please handle errors, omitted for brevity 
33 | mempool, _ := mempoolDat.ReadMempoolFromPath(path)
34 | 
35 | // prints the version and number of tx
36 | fmt.Println(mempool.GetFileHeader())
37 | 
38 | // get all mempool entries with GetMempoolEntries (here only the first three are used)
39 | for _, e := range mempool.GetMempoolEntries()[:3]{
40 |   fmt.Println(e.Info())
41 | }
42 | 
43 | // get the firstSeen timestamp of a transaction
44 | fmt.Println(mempool.GetMempoolEntries()[4].GetFirstSeen())
45 | 
46 | /* Furthermore you can use the full functionality of MsgTx (https://godoc.org/github.com/btcsuite/btcd/wire#MsgTx):
47 |     - transaction has a witness (spends a SegWit output)? with <entry>.transaction.HasWitness() 
48 |     - access input and outputs
49 |     - ...
50 |     But there is no way to get feerates for transactions, because they are (rightfully) not stored in the `mempool.dat`.
51 |     You'd have to the current UTXO set to get input amounts (which you need to calculate the fees)
52 | */
53 | 
54 | ```
55 | 
56 | The full documentation can be fund on https://godoc.org/github.com/0xB10C/mempool-dat/lib.
57 | 


--------------------------------------------------------------------------------
/lib/types.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 
 6 | 	"github.com/btcsuite/btcd/wire"
 7 | )
 8 | 
 9 | /*  FileHeader  */
10 | 
11 | // FileHeader represents the mempool file header
12 | type FileHeader struct {
13 | 	version int64
14 | 	numTx   int64
15 | }
16 | 
17 | // returns the version and the number of transactions in the file
18 | func (header FileHeader) String() string {
19 | 	return fmt.Sprintf("Version %d, contains %d transactions", header.version, header.numTx)
20 | }
21 | 
22 | // GetVersion returns the mempool file version
23 | func (header FileHeader) GetVersion() int64 {
24 | 	return header.version
25 | }
26 | 
27 | // GetTxCount returns the number of transactions in the corresponding file
28 | func (header FileHeader) GetTxCount() int64 {
29 | 	return header.numTx
30 | }
31 | 
32 | /*  MempoolEntry  */
33 | 
34 | // MempoolEntry represents a mempool entry
35 | type MempoolEntry struct {
36 | 	transaction   *wire.MsgTx
37 | 	firstSeenTime int64
38 | 	feeDelta      int64
39 | }
40 | 
41 | // returns the transaction hash of the entry
42 | func (entry MempoolEntry) String() string {
43 | 	return entry.transaction.TxHash().String()
44 | }
45 | 
46 | // GetFirstSeen returns the firstSeen time of the entry as timestamp
47 | func (entry MempoolEntry) GetFirstSeen() int64 {
48 | 	return entry.firstSeenTime
49 | }
50 | 
51 | // GetFeeDelta returns feeDelta of the entry
52 | func (entry MempoolEntry) GetFeeDelta() int64 {
53 | 	return entry.feeDelta
54 | }
55 | 
56 | // Info returns a string with information for a given MempoolEntry
57 | func (entry MempoolEntry) Info() string {
58 | 	firstSeen := entry.firstSeenTime
59 | 	numInputs := len(entry.transaction.TxIn)
60 | 	numOutputs := len(entry.transaction.TxOut)
61 | 	isSegWit := entry.transaction.HasWitness()
62 | 	hash := entry.transaction.TxHash()
63 | 
64 | 	return fmt.Sprintf("txid: %v, in: %d, out: %d, firstSeen: %d, isSegWit %t", hash, numInputs, numOutputs, firstSeen, isSegWit)
65 | }
66 | 
67 | /*  Mempool  */
68 | 
69 | // Mempool represents a parsed mempool.dat file
70 | type Mempool struct {
71 | 	header    FileHeader
72 | 	entries   []MempoolEntry
73 | 	mapDeltas []byte // not parsed
74 | }
75 | 
76 | // GetMempoolEntries returns a slice with mempool entries
77 | func (mempool Mempool) GetMempoolEntries() []MempoolEntry {
78 | 	return mempool.entries
79 | }
80 | 
81 | // GetFileHeader returns a the mempool file header
82 | func (mempool Mempool) GetFileHeader() FileHeader {
83 | 	return mempool.header
84 | }
85 | 
86 | // GetMapDeltas returns a byte slice of not parsed mapDelta entries
87 | func (mempool Mempool) GetMapDeltas() []byte {
88 | 	return mempool.mapDeltas
89 | }
90 | 


--------------------------------------------------------------------------------
/lib/reader.go:
--------------------------------------------------------------------------------
  1 | // Package lib provides mempool file primitives and functionality to load a mempool.dat file.
  2 | //
  3 | // You'd want to be calling ReadMempoolFromPath( ) to read a mempool.dat file.
  4 | // See https://github.com/0xB10C/mempool-dat for some usage examples.
  5 | package lib
  6 | 
  7 | import (
  8 | 	"bufio"
  9 | 	"encoding/binary"
 10 | 	"errors"
 11 | 	"io"
 12 | 	"os"
 13 | 
 14 | 	"github.com/btcsuite/btcd/wire"
 15 | )
 16 | 
 17 | // ReadMempoolFromPath reads a mempool file from a given path and returns a Mempool type
 18 | func ReadMempoolFromPath(path string) (mem Mempool, err error) {
 19 | 
 20 | 	file, err := os.Open(path)
 21 | 	if err != nil {
 22 | 		return mem, errors.New("Could not read mempool file: " + err.Error())
 23 | 	}
 24 | 	defer file.Close()
 25 | 
 26 | 	r := bufio.NewReader(file)
 27 | 	header, err := readFileHeader(r)
 28 | 	if err != nil {
 29 | 		return mem, errors.New("Could not read header: " + err.Error())
 30 | 	}
 31 | 	mem.header = header
 32 | 
 33 | 	var entries []MempoolEntry
 34 | 	// for the number of entries specified in the header
 35 | 	for i := int64(0); i < header.numTx; i++ {
 36 | 		mementry, err := readMempoolEntry(r)
 37 | 		if err != nil {
 38 | 			return mem, errors.New("Could not read mempoolEntry at index " + string(i) + " : " + err.Error())
 39 | 		}
 40 | 		entries = append(entries, mementry)
 41 | 	}
 42 | 	mem.entries = entries
 43 | 
 44 | 	var feeDelta []byte
 45 | 	for {
 46 | 		remainingBytes, err := r.ReadByte()
 47 | 		if err == io.EOF {
 48 | 			break
 49 | 		} else if err != nil {
 50 | 			return mem, errors.New("Could not read feeDelta: " + err.Error())
 51 | 		}
 52 | 		feeDelta = append(feeDelta, remainingBytes)
 53 | 	}
 54 | 
 55 | 	return
 56 | }
 57 | 
 58 | func readFileHeader(r *bufio.Reader) (header FileHeader, err error) {
 59 | 	fileVersion, err := readLEint64(r)
 60 | 	if err != nil {
 61 | 		return header, err
 62 | 	}
 63 | 
 64 | 	numberOfTx, err := readLEint64(r)
 65 | 	if err != nil {
 66 | 		return header, err
 67 | 	}
 68 | 
 69 | 	header = FileHeader{fileVersion, numberOfTx}
 70 | 	return
 71 | }
 72 | 
 73 | func readMempoolEntry(r *bufio.Reader) (mementry MempoolEntry, err error) {
 74 | 	msgTx := wire.NewMsgTx(1) // TODO: check if version 2 is correct
 75 | 	err = msgTx.Deserialize(r)
 76 | 	if err != nil {
 77 | 		return mementry, err
 78 | 	}
 79 | 
 80 | 	timestamp, err := readLEint64(r)
 81 | 	if err != nil {
 82 | 		return mementry, err
 83 | 	}
 84 | 
 85 | 	feeDelta, err := readLEint64(r)
 86 | 	if err != nil {
 87 | 		return mementry, err
 88 | 	}
 89 | 
 90 | 	mementry = MempoolEntry{msgTx, timestamp, feeDelta}
 91 | 	return
 92 | }
 93 | 
 94 | // reads the next 64bit in Little Endian and returns a int64
 95 | // used here to get the mempoolEntry's timestamp and feeDelta
 96 | func readLEint64(r *bufio.Reader) (res int64, err error) {
 97 | 	next64Bit := make([]byte, 8, 8)
 98 | 	_, err = io.ReadFull(r, next64Bit)
 99 | 	if err != nil {
100 | 		return 0, err
101 | 	}
102 | 
103 | 	res = int64(binary.LittleEndian.Uint64(next64Bit))
104 | 	return
105 | }
106 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 2 | github.com/btcsuite/btcd v0.0.0-20190427004231-96897255fd17 h1:m0N5Vg5nP3zEz8TREZpwX3gt4Biw3/8fbIf4A3hO96g=
 3 | github.com/btcsuite/btcd v0.0.0-20190427004231-96897255fd17/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 4 | github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 5 | github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
 6 | github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 7 | github.com/btcsuite/btcwire v0.0.0-20150205222541-f4c2644fdf0b h1:zr7tO4/72KTqRmkIX1XHSxeniI8TzbQPLeXETHFGVt4=
 8 | github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 9 | github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
10 | github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
11 | github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
12 | github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
13 | github.com/cjbassi/drawille-go v0.0.0-20190126131713-27dc511fe6fd h1:XtfPmj9tQRilnrEmI1HjQhxXWRhEM+m8CACtaMJE/kM=
14 | github.com/cjbassi/drawille-go v0.0.0-20190126131713-27dc511fe6fd/go.mod h1:vjcQJUZJYD3MeVGhtZXSMnCHfUNZxsyYzJt90eCYxK4=
15 | github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495 h1:6IyqGr3fnd0tM3YxipK27TUskaOVUjU2nG45yzwcQKY=
16 | github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
17 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
18 | github.com/gizak/termui/v3 v3.0.0 h1:NYTUG6ig/sJK05O5FyhWemwlVPO8ilNpvS/PgRtrKAE=
19 | github.com/gizak/termui/v3 v3.0.0/go.mod h1:uinu2dMdtMI+FTIdEFUJQT5y+KShnhQRshvPblXq3lY=
20 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
21 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
22 | github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
23 | github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
24 | github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
25 | github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o=
26 | github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
27 | github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
28 | github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
29 | github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d h1:x3S6kxmy49zXVVyhcnrFqxvNVCBPb2KZ9hV2RBdS840=
30 | github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
31 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
32 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
33 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
34 | golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
35 | golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
36 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
37 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
38 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
39 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
40 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
41 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
42 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
43 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
44 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
45 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
46 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
47 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
48 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
49 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
50 | 


--------------------------------------------------------------------------------