├── .github └── workflows │ └── pr.yaml ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── scripts └── setup-ci.sh ├── store ├── boltdb │ ├── bolt_store.go │ └── bolt_store_test.go ├── inmem │ ├── inmem_store.go │ └── inmem_store_test.go ├── postgresql │ ├── postgresql_store.go │ └── postgresql_store_test.go ├── store.go └── testing.go ├── tracker.go └── tracker_test.go /.github/workflows/pr.yaml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | on: [pull_request] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | name: Go test 7 | steps: 8 | - uses: actions/checkout@v2 9 | - name: Setup go 10 | uses: actions/setup-go@v1 11 | with: 12 | go-version: '1.15.10' 13 | - name: "Setup" 14 | run: ./scripts/setup-ci.sh 15 | - name: Go test 16 | run: go test -v ./... -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Eth-event-tracker 3 | 4 | Tracker of Ethereum events. 5 | 6 | ### Example 7 | 8 | ``` 9 | package main 10 | 11 | import ( 12 | "context" 13 | "encoding/binary" 14 | "flag" 15 | "fmt" 16 | "os" 17 | "os/signal" 18 | "syscall" 19 | 20 | web3 "github.com/umbracle/go-web3" 21 | "github.com/umbracle/go-web3/abi" 22 | "github.com/umbracle/go-web3/jsonrpc" 23 | "github.com/umbracle/go-web3/tracker" 24 | 25 | boltdbStore "github.com/umbracle/go-web3/tracker/store/boltdb" 26 | ) 27 | 28 | var depositEvent = abi.MustNewEvent(`DepositEvent( 29 | bytes pubkey, 30 | bytes whitdrawalcred, 31 | bytes amount, 32 | bytes signature, 33 | bytes index 34 | )`) 35 | 36 | func main() { 37 | var endpoint string 38 | var target string 39 | 40 | flag.StringVar(&endpoint, "endpoint", "", "") 41 | flag.StringVar(&target, "target", "", "") 42 | 43 | flag.Parse() 44 | 45 | provider, err := jsonrpc.NewClient(endpoint) 46 | if err != nil { 47 | fmt.Printf("[ERR]: %v", err) 48 | os.Exit(1) 49 | } 50 | 51 | store, err := boltdbStore.New("deposit.db") 52 | if err != nil { 53 | fmt.Printf("[ERR]: failted to start store %v", err) 54 | os.Exit(1) 55 | } 56 | 57 | tt, err := tracker.NewTracker(provider.Eth(), 58 | tracker.WithBatchSize(20000), 59 | tracker.WithStore(store), 60 | tracker.WithEtherscan(os.Getenv("ETHERSCAN_APIKEY")), 61 | tracker.WithFilter(&tracker.FilterConfig{ 62 | Async: true, 63 | Address: []web3.Address{ 64 | web3.HexToAddress(target), 65 | }, 66 | }), 67 | ) 68 | if err != nil { 69 | fmt.Printf("[ERR]: failed to create the tracker %v", err) 70 | os.Exit(1) 71 | } 72 | 73 | lastBlock, err := tt.GetLastBlock() 74 | if err != nil { 75 | fmt.Printf("[ERR]: failed to get last block %v", err) 76 | os.Exit(1) 77 | } 78 | if lastBlock != nil { 79 | fmt.Printf("Last block processed: %d\n", lastBlock.Number) 80 | } 81 | 82 | ctx, cancelFn := context.WithCancel(context.Background()) 83 | go func() { 84 | go func() { 85 | if err := tt.Sync(ctx); err != nil { 86 | fmt.Printf("[ERR]: %v", err) 87 | } 88 | }() 89 | 90 | go func() { 91 | for { 92 | select { 93 | case evnt := <-tt.EventCh: 94 | for _, log := range evnt.Added { 95 | if depositEvent.Match(log) { 96 | vals, err := depositEvent.ParseLog(log) 97 | if err != nil { 98 | panic(err) 99 | } 100 | 101 | index := binary.LittleEndian.Uint64(vals["index"].([]byte)) 102 | amount := binary.LittleEndian.Uint64(vals["amount"].([]byte)) 103 | 104 | fmt.Printf("Deposit: Block %d Index %d Amount %d\n", log.BlockNumber, index, amount) 105 | } 106 | } 107 | case <-tt.DoneCh: 108 | fmt.Println("historical sync done") 109 | } 110 | } 111 | }() 112 | 113 | }() 114 | 115 | handleSignals(cancelFn) 116 | } 117 | 118 | func handleSignals(cancelFn context.CancelFunc) int { 119 | signalCh := make(chan os.Signal, 4) 120 | signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) 121 | 122 | <-signalCh 123 | 124 | gracefulCh := make(chan struct{}) 125 | go func() { 126 | cancelFn() 127 | close(gracefulCh) 128 | }() 129 | 130 | select { 131 | case <-signalCh: 132 | return 1 133 | case <-gracefulCh: 134 | return 0 135 | } 136 | } 137 | ``` 138 | 139 | You can query the ETH2.0 Deposit contract like so: 140 | 141 | ``` 142 | go run main.go --endpoint https://mainnet.infura.io/v3/... --target 0x00000000219ab540356cbb839cbe05303d7705fa 143 | ``` 144 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/umbracle/eth-event-tracker 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/boltdb/bolt v1.3.1 7 | github.com/jmoiron/sqlx v1.3.4 8 | github.com/lib/pq v1.10.3 9 | github.com/ory/dockertest v3.3.5+incompatible 10 | github.com/stretchr/testify v1.7.0 11 | github.com/umbracle/go-web3 v0.0.0-20211029073105-5da9ae17a7ad 12 | ) 13 | 14 | require ( 15 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect 16 | github.com/Microsoft/go-winio v0.4.13 // indirect 17 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect 18 | github.com/cenkalti/backoff v2.2.1+incompatible // indirect 19 | github.com/containerd/continuity v0.0.0-20191214063359-1097c8bae83b // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/docker/go-connections v0.4.0 // indirect 22 | github.com/docker/go-units v0.4.0 // indirect 23 | github.com/gorilla/websocket v1.4.1 // indirect 24 | github.com/klauspost/compress v1.4.1 // indirect 25 | github.com/klauspost/cpuid v1.2.0 // indirect 26 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect 27 | github.com/mitchellh/mapstructure v1.1.2 // indirect 28 | github.com/opencontainers/go-digest v1.0.0-rc1 // indirect 29 | github.com/opencontainers/image-spec v1.0.1 // indirect 30 | github.com/opencontainers/runc v0.1.1 // indirect 31 | github.com/pkg/errors v0.8.1 // indirect 32 | github.com/pmezard/go-difflib v1.0.0 // indirect 33 | github.com/sirupsen/logrus v1.4.2 // indirect 34 | github.com/umbracle/fastrlp v0.0.0-20210128110402-41364ca56ca8 // indirect 35 | github.com/valyala/bytebufferpool v1.0.0 // indirect 36 | github.com/valyala/fasthttp v1.4.0 // indirect 37 | github.com/valyala/fastjson v1.4.1 // indirect 38 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect 39 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect 40 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 // indirect 41 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 42 | ) 43 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= 2 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= 3 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= 4 | github.com/Microsoft/go-winio v0.4.13 h1:Hmi80lzZuI/CaYmlJp/b+FjZdRZhKu9c2mDVqKlLWVs= 5 | github.com/Microsoft/go-winio v0.4.13/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= 6 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= 7 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= 8 | github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= 9 | github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= 10 | github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= 11 | github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= 12 | github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= 13 | github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= 14 | github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= 15 | github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= 16 | github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= 17 | github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= 18 | github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= 19 | github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= 20 | github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= 21 | github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= 22 | github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= 23 | github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= 24 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 25 | github.com/containerd/continuity v0.0.0-20191214063359-1097c8bae83b h1:pik3LX++5O3UiNWv45wfP/WT81l7ukBJzd3uUiifbSU= 26 | github.com/containerd/continuity v0.0.0-20191214063359-1097c8bae83b/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= 27 | github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 28 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 29 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 30 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 31 | github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= 32 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= 33 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 34 | github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= 35 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 36 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 37 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 38 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 39 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 40 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= 41 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 42 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 43 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 44 | github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= 45 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 46 | github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= 47 | github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 48 | github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= 49 | github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= 50 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 51 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 52 | github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 53 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 54 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 55 | github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= 56 | github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= 57 | github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= 58 | github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= 59 | github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 60 | github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= 61 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 62 | github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 63 | github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= 64 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 65 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 66 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 67 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 68 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 69 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 70 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 71 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 72 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 73 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 74 | github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= 75 | github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 76 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 77 | github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= 78 | github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 79 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 80 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 81 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 82 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 83 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 84 | github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 85 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 86 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 87 | github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= 88 | github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 89 | github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= 90 | github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 91 | github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= 92 | github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= 93 | github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= 94 | github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= 95 | github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 96 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 97 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 98 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 99 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 100 | github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 101 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 102 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 103 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 104 | github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 105 | github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 106 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 107 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 108 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 109 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 110 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 111 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 112 | github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= 113 | github.com/umbracle/fastrlp v0.0.0-20210128110402-41364ca56ca8 h1:lWWKP+Oi7FSORlB3Y8rLz1Q7OOxtD8vecmtYOSNkIpo= 114 | github.com/umbracle/fastrlp v0.0.0-20210128110402-41364ca56ca8/go.mod h1:z0AyVhz/7VbuYSaCB+tFgypZKD1DJL76ATih6XqFlig= 115 | github.com/umbracle/go-web3 v0.0.0-20211029073105-5da9ae17a7ad h1:OWXjnF/hX/Y9vYot9/BhO7nff1s2IQn6ZZ9jlwvKYm0= 116 | github.com/umbracle/go-web3 v0.0.0-20211029073105-5da9ae17a7ad/go.mod h1:ewphLEPDU44NJLSf5My9H367fW/qNksvjjKhWaS+2Dg= 117 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 118 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 119 | github.com/valyala/fasthttp v1.4.0 h1:PuaTGZIw3mjYhhhbVbCQp8aciRZN9YdoB7MGX9Ko76A= 120 | github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= 121 | github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/yE= 122 | github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o= 123 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 124 | golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 125 | golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 126 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 127 | golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 128 | golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 129 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 130 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= 131 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 132 | golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 133 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 134 | golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 135 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 136 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 137 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= 138 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 139 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 140 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 141 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 142 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 143 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 144 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 145 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 146 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= 147 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 148 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 149 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 150 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 151 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 152 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 153 | gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= 154 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 155 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 156 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 157 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 158 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= 159 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 160 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 161 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 162 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 163 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 164 | gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= 165 | gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 166 | -------------------------------------------------------------------------------- /scripts/setup-ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # set -o errexit 4 | 5 | install_solidity() { 6 | VERSION="0.5.5" 7 | DOWNLOAD=https://github.com/ethereum/solidity/releases/download/v${VERSION}/solc-static-linux 8 | 9 | curl -L $DOWNLOAD > /tmp/solc 10 | chmod +x /tmp/solc 11 | mv /tmp/solc /usr/local/bin/solc 12 | } 13 | 14 | install_solidity -------------------------------------------------------------------------------- /store/boltdb/bolt_store.go: -------------------------------------------------------------------------------- 1 | package trackerboltdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | 7 | "github.com/boltdb/bolt" 8 | "github.com/umbracle/go-web3" 9 | "github.com/umbracle/go-web3/tracker/store" 10 | ) 11 | 12 | var _ store.Store = (*BoltStore)(nil) 13 | 14 | var ( 15 | dbLogs = []byte("logs") 16 | dbConf = []byte("conf") 17 | ) 18 | 19 | // BoltStore is a tracker store implementation. 20 | type BoltStore struct { 21 | conn *bolt.DB 22 | } 23 | 24 | // New creates a new boltdbstore 25 | func New(path string) (*BoltStore, error) { 26 | db, err := bolt.Open(path, 0600, nil) 27 | if err != nil { 28 | return nil, err 29 | } 30 | store := &BoltStore{ 31 | conn: db, 32 | } 33 | if err := store.setupDB(); err != nil { 34 | store.Close() 35 | return nil, err 36 | } 37 | return store, nil 38 | } 39 | 40 | func (b *BoltStore) setupDB() error { 41 | txn, err := b.conn.Begin(true) 42 | if err != nil { 43 | return err 44 | } 45 | defer txn.Rollback() 46 | 47 | if _, err := txn.CreateBucketIfNotExists(dbConf); err != nil { 48 | return err 49 | } 50 | return txn.Commit() 51 | } 52 | 53 | // Close implements the store interface 54 | func (b *BoltStore) Close() error { 55 | return b.conn.Close() 56 | } 57 | 58 | // Get implements the store interface 59 | func (b *BoltStore) Get(k string) (string, error) { 60 | txn, err := b.conn.Begin(false) 61 | if err != nil { 62 | return "", err 63 | } 64 | defer txn.Rollback() 65 | 66 | bucket := txn.Bucket(dbConf) 67 | val := bucket.Get([]byte(k)) 68 | 69 | return string(val), nil 70 | } 71 | 72 | // ListPrefix implements the store interface 73 | func (b *BoltStore) ListPrefix(prefix string) ([]string, error) { 74 | txn, err := b.conn.Begin(false) 75 | if err != nil { 76 | return nil, err 77 | } 78 | defer txn.Rollback() 79 | 80 | res := []string{} 81 | c := txn.Bucket(dbConf).Cursor() 82 | for k, v := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() { 83 | res = append(res, string(v)) 84 | } 85 | return res, nil 86 | } 87 | 88 | // Set implements the store interface 89 | func (b *BoltStore) Set(k, v string) error { 90 | txn, err := b.conn.Begin(true) 91 | if err != nil { 92 | return err 93 | } 94 | defer txn.Rollback() 95 | 96 | bucket := txn.Bucket(dbConf) 97 | if err := bucket.Put([]byte(k), []byte(v)); err != nil { 98 | return err 99 | } 100 | return txn.Commit() 101 | } 102 | 103 | // GetEntry implements the store interface 104 | func (b *BoltStore) GetEntry(hash string) (store.Entry, error) { 105 | txn, err := b.conn.Begin(true) 106 | if err != nil { 107 | return nil, err 108 | } 109 | defer txn.Rollback() 110 | 111 | bucketName := append(dbLogs, []byte(hash)...) 112 | if _, err := txn.CreateBucketIfNotExists(bucketName); err != nil { 113 | return nil, err 114 | } 115 | if err := txn.Commit(); err != nil { 116 | return nil, err 117 | } 118 | e := &Entry{ 119 | conn: b.conn, 120 | bucket: bucketName, 121 | } 122 | return e, nil 123 | } 124 | 125 | // Entry is an store.Entry implementation 126 | type Entry struct { 127 | conn *bolt.DB 128 | bucket []byte 129 | } 130 | 131 | // LastIndex implements the store interface 132 | func (e *Entry) LastIndex() (uint64, error) { 133 | tx, err := e.conn.Begin(false) 134 | if err != nil { 135 | return 0, err 136 | } 137 | defer tx.Rollback() 138 | 139 | curs := tx.Bucket(e.bucket).Cursor() 140 | if last, _ := curs.Last(); last != nil { 141 | return bytesToUint64(last) + 1, nil 142 | } 143 | return 0, nil 144 | } 145 | 146 | // StoreLog implements the store interface 147 | func (e *Entry) StoreLog(log *web3.Log) error { 148 | return e.StoreLogs([]*web3.Log{log}) 149 | } 150 | 151 | // StoreLogs implements the store interface 152 | func (e *Entry) StoreLogs(logs []*web3.Log) error { 153 | tx, err := e.conn.Begin(true) 154 | if err != nil { 155 | return err 156 | } 157 | defer tx.Rollback() 158 | 159 | indx, err := e.LastIndex() 160 | if err != nil { 161 | return err 162 | } 163 | 164 | bucket := tx.Bucket(e.bucket) 165 | for logIndx, log := range logs { 166 | key := uint64ToBytes(indx + uint64(logIndx)) 167 | 168 | val, err := log.MarshalJSON() 169 | if err != nil { 170 | return err 171 | } 172 | if err := bucket.Put(key, val); err != nil { 173 | return err 174 | } 175 | } 176 | return tx.Commit() 177 | } 178 | 179 | // RemoveLogs implements the store interface 180 | func (e *Entry) RemoveLogs(indx uint64) error { 181 | indxKey := uint64ToBytes(indx) 182 | 183 | tx, err := e.conn.Begin(true) 184 | if err != nil { 185 | return err 186 | } 187 | defer tx.Rollback() 188 | 189 | curs := tx.Bucket(e.bucket).Cursor() 190 | for k, _ := curs.Seek(indxKey); k != nil; k, _ = curs.Next() { 191 | if err := curs.Delete(); err != nil { 192 | return err 193 | } 194 | } 195 | 196 | return tx.Commit() 197 | } 198 | 199 | // GetLog implements the store interface 200 | func (e *Entry) GetLog(indx uint64, log *web3.Log) error { 201 | txn, err := e.conn.Begin(false) 202 | if err != nil { 203 | return err 204 | } 205 | defer txn.Rollback() 206 | 207 | bucket := txn.Bucket(e.bucket) 208 | val := bucket.Get(uint64ToBytes(indx)) 209 | 210 | if err := log.UnmarshalJSON(val); err != nil { 211 | return err 212 | } 213 | return nil 214 | } 215 | 216 | func bytesToUint64(b []byte) uint64 { 217 | return binary.BigEndian.Uint64(b) 218 | } 219 | 220 | func uint64ToBytes(u uint64) []byte { 221 | buf := make([]byte, 8) 222 | binary.BigEndian.PutUint64(buf, u) 223 | return buf 224 | } 225 | -------------------------------------------------------------------------------- /store/boltdb/bolt_store_test.go: -------------------------------------------------------------------------------- 1 | package trackerboltdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/umbracle/go-web3/tracker/store" 10 | ) 11 | 12 | func setupDB(t *testing.T) (store.Store, func()) { 13 | dir, err := ioutil.TempDir("/tmp", "boltdb-test") 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | path := filepath.Join(dir, "test.db") 19 | store, err := New(path) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | close := func() { 25 | if err := os.RemoveAll(dir); err != nil { 26 | t.Fatal(err) 27 | } 28 | } 29 | return store, close 30 | } 31 | 32 | func TestBoltDBStore(t *testing.T) { 33 | store.TestStore(t, setupDB) 34 | } 35 | -------------------------------------------------------------------------------- /store/inmem/inmem_store.go: -------------------------------------------------------------------------------- 1 | package inmem 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | 7 | web3 "github.com/umbracle/go-web3" 8 | "github.com/umbracle/go-web3/tracker/store" 9 | ) 10 | 11 | var _ store.Store = (*InmemStore)(nil) 12 | 13 | // InmemStore implements the Store interface. 14 | type InmemStore struct { 15 | l sync.RWMutex 16 | entries map[string]*Entry 17 | kv map[string]string 18 | } 19 | 20 | // NewInmemStore returns a new in-memory store. 21 | func NewInmemStore() *InmemStore { 22 | return &InmemStore{ 23 | entries: map[string]*Entry{}, 24 | kv: map[string]string{}, 25 | } 26 | } 27 | 28 | // Close implements the store interface 29 | func (i *InmemStore) Close() error { 30 | return nil 31 | } 32 | 33 | // Get implements the store interface 34 | func (i *InmemStore) Get(k string) (string, error) { 35 | i.l.Lock() 36 | defer i.l.Unlock() 37 | return i.kv[string(k)], nil 38 | } 39 | 40 | // ListPrefix implements the store interface 41 | func (i *InmemStore) ListPrefix(prefix string) ([]string, error) { 42 | i.l.Lock() 43 | defer i.l.Unlock() 44 | 45 | res := []string{} 46 | for k, v := range i.kv { 47 | if strings.HasPrefix(k, prefix) { 48 | res = append(res, v) 49 | } 50 | } 51 | return res, nil 52 | } 53 | 54 | // Set implements the store interface 55 | func (i *InmemStore) Set(k, v string) error { 56 | i.l.Lock() 57 | defer i.l.Unlock() 58 | i.kv[string(k)] = v 59 | return nil 60 | } 61 | 62 | // GetEntry implements the store interface 63 | func (i *InmemStore) GetEntry(hash string) (store.Entry, error) { 64 | i.l.Lock() 65 | defer i.l.Unlock() 66 | e, ok := i.entries[hash] 67 | if ok { 68 | return e, nil 69 | } 70 | e = &Entry{ 71 | logs: []*web3.Log{}, 72 | } 73 | i.entries[hash] = e 74 | return e, nil 75 | } 76 | 77 | // Entry is a store.Entry implementation 78 | type Entry struct { 79 | l sync.RWMutex 80 | logs []*web3.Log 81 | } 82 | 83 | // LastIndex implements the store interface 84 | func (e *Entry) LastIndex() (uint64, error) { 85 | e.l.Lock() 86 | defer e.l.Unlock() 87 | return uint64(len(e.logs)), nil 88 | } 89 | 90 | // Logs returns the logs of the inmemory store 91 | func (e *Entry) Logs() []*web3.Log { 92 | return e.logs 93 | } 94 | 95 | // StoreLogs implements the store interface 96 | func (e *Entry) StoreLogs(logs []*web3.Log) error { 97 | e.l.Lock() 98 | defer e.l.Unlock() 99 | for _, log := range logs { 100 | e.logs = append(e.logs, log) 101 | } 102 | return nil 103 | } 104 | 105 | // RemoveLogs implements the store interface 106 | func (e *Entry) RemoveLogs(indx uint64) error { 107 | e.l.Lock() 108 | defer e.l.Unlock() 109 | e.logs = e.logs[:indx] 110 | return nil 111 | } 112 | 113 | // GetLog implements the store interface 114 | func (e *Entry) GetLog(indx uint64, log *web3.Log) error { 115 | *log = *e.logs[indx] 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /store/inmem/inmem_store_test.go: -------------------------------------------------------------------------------- 1 | package inmem 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/umbracle/go-web3/tracker/store" 7 | ) 8 | 9 | func TestInMemoryStore(t *testing.T) { 10 | store.TestStore(t, func(t *testing.T) (store.Store, func()) { 11 | return NewInmemStore(), func() {} 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /store/postgresql/postgresql_store.go: -------------------------------------------------------------------------------- 1 | package trackerpostgresql 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/hex" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/jmoiron/sqlx" 10 | "github.com/umbracle/go-web3" 11 | "github.com/umbracle/go-web3/tracker/store" 12 | 13 | // Enable postgres for sqlx 14 | _ "github.com/lib/pq" 15 | ) 16 | 17 | var _ store.Store = (*PostgreSQLStore)(nil) 18 | 19 | // PostgreSQLStore is a tracker store implementation that uses PostgreSQL as a backend. 20 | type PostgreSQLStore struct { 21 | db *sqlx.DB 22 | } 23 | 24 | // NewPostgreSQLStore creates a new PostgreSQL store 25 | func NewPostgreSQLStore(endpoint string) (*PostgreSQLStore, error) { 26 | db, err := sql.Open("postgres", endpoint) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return NewSQLStore(db, "postgres") 31 | } 32 | 33 | // NewSQLStore creates a new store with an sql driver 34 | func NewSQLStore(db *sql.DB, driver string) (*PostgreSQLStore, error) { 35 | sqlxDB := sqlx.NewDb(db, driver) 36 | 37 | // create the kv database if it does not exists 38 | if _, err := db.Exec(kvSQLSchema); err != nil { 39 | return nil, err 40 | } 41 | return &PostgreSQLStore{db: sqlxDB}, nil 42 | } 43 | 44 | // Close implements the store interface 45 | func (p *PostgreSQLStore) Close() error { 46 | return p.db.Close() 47 | } 48 | 49 | // Get implements the store interface 50 | func (p *PostgreSQLStore) Get(k string) (string, error) { 51 | var out string 52 | if err := p.db.Get(&out, "SELECT val FROM kv WHERE key=$1", string(k)); err != nil { 53 | if err == sql.ErrNoRows { 54 | return "", nil 55 | } 56 | return "", err 57 | } 58 | return out, nil 59 | } 60 | 61 | // ListPrefix implements the store interface 62 | func (p *PostgreSQLStore) ListPrefix(prefix string) ([]string, error) { 63 | var out []string 64 | if err := p.db.Select(&out, "SELECT val FROM kv WHERE key LIKE $1", string(prefix)+"%"); err != nil { 65 | return nil, err 66 | } 67 | return out, nil 68 | } 69 | 70 | // Set implements the store interface 71 | func (p *PostgreSQLStore) Set(k, v string) error { 72 | if _, err := p.db.Exec("INSERT INTO kv (key, val) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET val = $2", k, v); err != nil { 73 | return err 74 | } 75 | return nil 76 | } 77 | 78 | // GetEntry implements the store interface 79 | func (p *PostgreSQLStore) GetEntry(hash string) (store.Entry, error) { 80 | tableName := "logs_" + hash 81 | if _, err := p.db.Exec(logSQLSchema(tableName)); err != nil { 82 | return nil, err 83 | } 84 | e := &Entry{ 85 | table: tableName, 86 | db: p.db, 87 | } 88 | return e, nil 89 | } 90 | 91 | // Entry is an store.Entry implementation 92 | type Entry struct { 93 | table string 94 | db *sqlx.DB 95 | } 96 | 97 | // LastIndex implements the store interface 98 | func (e *Entry) LastIndex() (uint64, error) { 99 | var index uint64 100 | if err := e.db.Get(&index, "SELECT indx FROM "+e.table+" ORDER BY indx DESC LIMIT 1"); err != nil { 101 | if err == sql.ErrNoRows { 102 | return 0, nil 103 | } 104 | return 0, err 105 | } 106 | return index + 1, nil 107 | } 108 | 109 | // StoreLogs implements the store interface 110 | func (e *Entry) StoreLogs(logs []*web3.Log) error { 111 | lastIndex, err := e.LastIndex() 112 | if err != nil { 113 | return err 114 | } 115 | 116 | tx, err := e.db.Beginx() 117 | if err != nil { 118 | return err 119 | } 120 | defer tx.Rollback() 121 | 122 | query := "INSERT INTO " + e.table + " (indx, tx_index, tx_hash, block_num, block_hash, address, data, topics) VALUES (:indx, :tx_index, :tx_hash, :block_num, :block_hash, :address, :data, :topics)" 123 | 124 | for indx, log := range logs { 125 | topics := []string{} 126 | for _, topic := range log.Topics { 127 | topics = append(topics, topic.String()) 128 | } 129 | obj := &logObj{ 130 | Index: lastIndex + uint64(indx), 131 | TxIndex: log.TransactionIndex, 132 | TxHash: log.TransactionHash.String(), 133 | BlockNum: log.BlockNumber, 134 | BlockHash: log.BlockHash.String(), 135 | Address: log.Address.String(), 136 | Topics: strings.Join(topics, ","), 137 | } 138 | if log.Data != nil { 139 | obj.Data = "0x" + hex.EncodeToString(log.Data) 140 | } 141 | 142 | if _, err := tx.NamedExec(query, obj); err != nil { 143 | return err 144 | } 145 | } 146 | if err := tx.Commit(); err != nil { 147 | return err 148 | } 149 | return nil 150 | } 151 | 152 | // RemoveLogs implements the store interface 153 | func (e *Entry) RemoveLogs(indx uint64) error { 154 | if _, err := e.db.Exec("DELETE FROM "+e.table+" WHERE indx >= $1", indx); err != nil { 155 | return err 156 | } 157 | return nil 158 | } 159 | 160 | // GetLog implements the store interface 161 | func (e *Entry) GetLog(indx uint64, log *web3.Log) error { 162 | obj := logObj{} 163 | if err := e.db.Get(&obj, "SELECT * FROM "+e.table+" WHERE indx=$1", indx); err != nil { 164 | return err 165 | } 166 | 167 | log.TransactionIndex = obj.TxIndex 168 | if err := log.TransactionHash.UnmarshalText([]byte(obj.TxHash)); err != nil { 169 | return err 170 | } 171 | log.BlockNumber = obj.BlockNum 172 | if err := log.BlockHash.UnmarshalText([]byte(obj.BlockHash)); err != nil { 173 | return err 174 | } 175 | if err := log.Address.UnmarshalText([]byte(obj.Address)); err != nil { 176 | return err 177 | } 178 | 179 | if obj.Topics != "" { 180 | log.Topics = []web3.Hash{} 181 | for _, item := range strings.Split(obj.Topics, ",") { 182 | var topic web3.Hash 183 | if err := topic.UnmarshalText([]byte(item)); err != nil { 184 | return err 185 | } 186 | log.Topics = append(log.Topics, topic) 187 | } 188 | } else { 189 | log.Topics = nil 190 | } 191 | 192 | if obj.Data != "" { 193 | if !strings.HasPrefix(obj.Data, "0x") { 194 | return fmt.Errorf("0x prefix not found in data") 195 | } 196 | buf, err := hex.DecodeString(obj.Data[2:]) 197 | if err != nil { 198 | return err 199 | } 200 | log.Data = buf 201 | } else { 202 | log.Data = nil 203 | } 204 | 205 | return nil 206 | } 207 | 208 | type logObj struct { 209 | Index uint64 `db:"indx"` 210 | TxIndex uint64 `db:"tx_index"` 211 | TxHash string `db:"tx_hash"` 212 | BlockNum uint64 `db:"block_num"` 213 | BlockHash string `db:"block_hash"` 214 | Address string `db:"address"` 215 | Topics string `db:"topics"` 216 | Data string `db:"data"` 217 | } 218 | 219 | var kvSQLSchema = ` 220 | CREATE TABLE IF NOT EXISTS kv ( 221 | key text unique, 222 | val text 223 | ); 224 | ` 225 | 226 | func logSQLSchema(name string) string { 227 | return ` 228 | CREATE TABLE IF NOT EXISTS ` + name + ` ( 229 | indx numeric, 230 | tx_index numeric, 231 | tx_hash text, 232 | block_num numeric, 233 | block_hash text, 234 | address text, 235 | topics text, 236 | data text 237 | ); 238 | ` 239 | } 240 | -------------------------------------------------------------------------------- /store/postgresql/postgresql_store_test.go: -------------------------------------------------------------------------------- 1 | package trackerpostgresql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/ory/dockertest" 9 | "github.com/umbracle/go-web3/tracker/store" 10 | ) 11 | 12 | func setupDB(t *testing.T) (store.Store, func()) { 13 | pool, err := dockertest.NewPool("") 14 | if err != nil { 15 | t.Fatalf("Could not connect to docker: %s", err) 16 | } 17 | 18 | resource, err := pool.Run("postgres", "latest", []string{"POSTGRES_HOST_AUTH_METHOD=trust"}) 19 | if err != nil { 20 | t.Fatalf("Could not start resource: %s", err) 21 | } 22 | 23 | endpoint := fmt.Sprintf("postgres://postgres@localhost:%s/postgres?sslmode=disable", resource.GetPort("5432/tcp")) 24 | 25 | // wait for the db to be running 26 | if err := pool.Retry(func() error { 27 | db, err := sql.Open("postgres", endpoint) 28 | if err != nil { 29 | return err 30 | } 31 | return db.Ping() 32 | }); err != nil { 33 | t.Fatal(err) 34 | } 35 | 36 | cleanup := func() { 37 | if err := pool.Purge(resource); err != nil { 38 | t.Fatalf("Could not purge resource: %s", err) 39 | } 40 | } 41 | 42 | store, err := NewPostgreSQLStore(endpoint) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | return store, cleanup 47 | } 48 | 49 | func TestPostgreSQLStore(t *testing.T) { 50 | store.TestStore(t, setupDB) 51 | } 52 | -------------------------------------------------------------------------------- /store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import web3 "github.com/umbracle/go-web3" 4 | 5 | // Store is a datastore for the tracker 6 | type Store interface { 7 | // Get gets a value 8 | Get(k string) (string, error) 9 | 10 | // ListPrefix lists values by prefix 11 | ListPrefix(prefix string) ([]string, error) 12 | 13 | // Set sets a value 14 | Set(k, v string) error 15 | 16 | // Close closes the store 17 | Close() error 18 | 19 | // GetEntry returns a specific entry 20 | GetEntry(hash string) (Entry, error) 21 | } 22 | 23 | // Entry is a filter entry in the store 24 | type Entry interface { 25 | // LastIndex returns index of the last stored event 26 | LastIndex() (uint64, error) 27 | 28 | // StoreLogs stores the web3 logs of the event 29 | StoreLogs(logs []*web3.Log) error 30 | 31 | // RemoveLogs all the logs starting at index 'indx' 32 | RemoveLogs(indx uint64) error 33 | 34 | // GetLog returns the log at indx 35 | GetLog(indx uint64, log *web3.Log) error 36 | } 37 | -------------------------------------------------------------------------------- /store/testing.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | web3 "github.com/umbracle/go-web3" 8 | ) 9 | 10 | // SetupDB is a function that creates a backend 11 | type SetupDB func(t *testing.T) (Store, func()) 12 | 13 | // TestStore tests a tracker store 14 | func TestStore(t *testing.T, setup SetupDB) { 15 | testMultipleStores(t, setup) 16 | testGetSet(t, setup) 17 | testRemoveLogs(t, setup) 18 | testStoreLogs(t, setup) 19 | testPrefix(t, setup) 20 | } 21 | 22 | func testMultipleStores(t *testing.T, setup SetupDB) { 23 | store, close := setup(t) 24 | defer close() 25 | 26 | entry0, err := store.GetEntry("0") 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | log := web3.Log{ 31 | BlockNumber: 10, 32 | } 33 | if err := entry0.StoreLogs([]*web3.Log{&log}); err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | entry1, err := store.GetEntry("1") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | log = web3.Log{ 42 | BlockNumber: 15, 43 | } 44 | if err := entry1.StoreLogs([]*web3.Log{&log}); err != nil { 45 | t.Fatal(err) 46 | } 47 | 48 | index0, err := entry0.LastIndex() 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | if index0 != 1 { 53 | t.Fatal("bad") 54 | } 55 | 56 | index1, err := entry1.LastIndex() 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | if index1 != 1 { 61 | t.Fatal("bad") 62 | } 63 | } 64 | 65 | func testPrefix(t *testing.T, setup SetupDB) { 66 | store, close := setup(t) 67 | defer close() 68 | 69 | v1 := "val1" 70 | v2 := "val2" 71 | v3 := "val3" 72 | 73 | // add same prefix values 74 | if err := store.Set(v1, v1); err != nil { 75 | t.Fatal(err) 76 | } 77 | if err := store.Set(v2, v2); err != nil { 78 | t.Fatal(err) 79 | } 80 | if err := store.Set(v3, v3); err != nil { 81 | t.Fatal(err) 82 | } 83 | 84 | // add distinct value 85 | if err := store.Set("a", "b"); err != nil { 86 | t.Fatal(err) 87 | } 88 | 89 | checkPrefix := func(prefix string, expected int) { 90 | res, err := store.ListPrefix(prefix) 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | if len(res) != expected { 95 | t.Fatalf("%d values expected for prefix '%s' but %d found", expected, string(prefix), len(res)) 96 | } 97 | } 98 | 99 | checkPrefix("val", 3) 100 | checkPrefix("a", 1) 101 | checkPrefix("b", 0) 102 | } 103 | 104 | func testGetSet(t *testing.T, setup SetupDB) { 105 | store, close := setup(t) 106 | defer close() 107 | 108 | k1 := "k1" 109 | v1 := "v1" 110 | v2 := "v2" 111 | 112 | res, err := store.Get(k1) 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | if len(res) != 0 { 117 | t.Fatal("expected empty") 118 | } 119 | 120 | // set the entry 121 | if err := store.Set(k1, v1); err != nil { 122 | t.Fatal(err) 123 | } 124 | res, err = store.Get(k1) 125 | if err != nil { 126 | t.Fatal(err) 127 | } 128 | if res != v1 { 129 | t.Fatal("bad") 130 | } 131 | 132 | // update the entry 133 | if err := store.Set(k1, v2); err != nil { 134 | t.Fatal(err) 135 | } 136 | res, err = store.Get(k1) 137 | if err != nil { 138 | t.Fatal(err) 139 | } 140 | if res != v2 { 141 | t.Fatal("bad") 142 | } 143 | } 144 | 145 | func testStoreLogs(t *testing.T, setup SetupDB) { 146 | store, close := setup(t) 147 | defer close() 148 | 149 | entry, err := store.GetEntry("1") 150 | if err != nil { 151 | t.Fatal(err) 152 | } 153 | 154 | indx, err := entry.LastIndex() 155 | if err != nil { 156 | t.Fatal(err) 157 | } 158 | if indx != 0 { 159 | t.Fatal("index should be zero") 160 | } 161 | 162 | log := web3.Log{ 163 | BlockNumber: 10, 164 | } 165 | if err := entry.StoreLogs([]*web3.Log{&log}); err != nil { 166 | t.Fatal(err) 167 | } 168 | 169 | indx, err = entry.LastIndex() 170 | if err != nil { 171 | t.Fatal(err) 172 | } 173 | if indx != 1 { 174 | t.Fatal("index should be one") 175 | } 176 | 177 | var log2 web3.Log 178 | if err := entry.GetLog(0, &log2); err != nil { 179 | t.Fatal(err) 180 | } 181 | if !reflect.DeepEqual(log, log2) { 182 | t.Fatal("bad") 183 | } 184 | 185 | // retrieve entry again 186 | entry1, err := store.GetEntry("1") 187 | if err != nil { 188 | t.Fatal(err) 189 | } 190 | indx1, err := entry1.LastIndex() 191 | if err != nil { 192 | t.Fatal(err) 193 | } 194 | if indx1 != indx { 195 | t.Fatal("bad last index") 196 | } 197 | } 198 | 199 | func testRemoveLogs(t *testing.T, setup SetupDB) { 200 | store, close := setup(t) 201 | defer close() 202 | 203 | logs := []*web3.Log{} 204 | for i := uint64(0); i < 10; i++ { 205 | logs = append(logs, &web3.Log{ 206 | BlockNumber: i, 207 | }) 208 | } 209 | 210 | entry, err := store.GetEntry("1") 211 | if err != nil { 212 | t.Fatal(err) 213 | } 214 | 215 | if err := entry.StoreLogs(logs); err != nil { 216 | t.Fatal(err) 217 | } 218 | 219 | if err := entry.RemoveLogs(5); err != nil { 220 | t.Fatal(err) 221 | } 222 | 223 | indx, err := entry.LastIndex() 224 | if err != nil { 225 | t.Fatal(err) 226 | } 227 | if indx != 5 { 228 | t.Fatal("bad") 229 | } 230 | 231 | // add again the values 232 | if err := entry.StoreLogs(logs[5:]); err != nil { 233 | t.Fatal(err) 234 | } 235 | indx, err = entry.LastIndex() 236 | if err != nil { 237 | t.Fatal(err) 238 | } 239 | if indx != 10 { 240 | t.Fatal("bad") 241 | } 242 | } 243 | -------------------------------------------------------------------------------- /tracker.go: -------------------------------------------------------------------------------- 1 | package tracker 2 | 3 | import ( 4 | "context" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "log" 11 | "math/big" 12 | "strconv" 13 | "strings" 14 | "sync" 15 | "sync/atomic" 16 | "time" 17 | 18 | web3 "github.com/umbracle/go-web3" 19 | "github.com/umbracle/go-web3/blocktracker" 20 | "github.com/umbracle/go-web3/etherscan" 21 | "github.com/umbracle/go-web3/jsonrpc/codec" 22 | "github.com/umbracle/go-web3/tracker/store" 23 | "github.com/umbracle/go-web3/tracker/store/inmem" 24 | ) 25 | 26 | var ( 27 | dbGenesis = "genesis" 28 | dbChainID = "chainID" 29 | dbLastBlock = "lastBlock" 30 | dbFilter = "filter" 31 | ) 32 | 33 | const ( 34 | defaultMaxBlockBacklog = 10 35 | defaultBatchSize = 100 36 | ) 37 | 38 | // FilterConfig is a tracker filter configuration 39 | type FilterConfig struct { 40 | Address []web3.Address `json:"address"` 41 | Topics []*web3.Hash `json:"topics"` 42 | Start uint64 43 | Hash string 44 | Async bool 45 | } 46 | 47 | func (f *FilterConfig) buildHash() { 48 | h := sha256.New() 49 | for _, i := range f.Address { 50 | h.Write([]byte(i.String())) 51 | } 52 | for _, i := range f.Topics { 53 | if i == nil { 54 | h.Write([]byte("empty")) 55 | } else { 56 | h.Write([]byte(i.String())) 57 | } 58 | } 59 | f.Hash = hex.EncodeToString(h.Sum(nil)) 60 | } 61 | 62 | func (f *FilterConfig) getFilterSearch() *web3.LogFilter { 63 | filter := &web3.LogFilter{} 64 | if len(f.Address) != 0 { 65 | filter.Address = f.Address 66 | } 67 | if len(f.Topics) != 0 { 68 | filter.Topics = f.Topics 69 | } 70 | return filter 71 | } 72 | 73 | // Config is the configuration of the tracker 74 | type Config struct { 75 | BatchSize uint64 76 | BlockTracker *blocktracker.BlockTracker // move to interface 77 | EtherscanAPIKey string 78 | Filter *FilterConfig 79 | Store store.Store 80 | } 81 | 82 | type ConfigOption func(*Config) 83 | 84 | func WithBatchSize(b uint64) ConfigOption { 85 | return func(c *Config) { 86 | c.BatchSize = b 87 | } 88 | } 89 | 90 | func WithBlockTracker(b *blocktracker.BlockTracker) ConfigOption { 91 | return func(c *Config) { 92 | c.BlockTracker = b 93 | } 94 | } 95 | 96 | func WithStore(s store.Store) ConfigOption { 97 | return func(c *Config) { 98 | c.Store = s 99 | } 100 | } 101 | 102 | func WithFilter(f *FilterConfig) ConfigOption { 103 | return func(c *Config) { 104 | c.Filter = f 105 | } 106 | } 107 | 108 | func WithEtherscan(k string) ConfigOption { 109 | return func(c *Config) { 110 | c.EtherscanAPIKey = k 111 | } 112 | } 113 | 114 | // DefaultConfig returns the default tracker config 115 | func DefaultConfig() *Config { 116 | return &Config{ 117 | BatchSize: defaultBatchSize, 118 | Store: inmem.NewInmemStore(), 119 | Filter: &FilterConfig{}, 120 | EtherscanAPIKey: "", 121 | } 122 | } 123 | 124 | // Provider are the eth1x methods required by the tracker 125 | type Provider interface { 126 | BlockNumber() (uint64, error) 127 | GetBlockByHash(hash web3.Hash, full bool) (*web3.Block, error) 128 | GetBlockByNumber(i web3.BlockNumber, full bool) (*web3.Block, error) 129 | GetLogs(filter *web3.LogFilter) ([]*web3.Log, error) 130 | ChainID() (*big.Int, error) 131 | } 132 | 133 | // Tracker is a contract event tracker 134 | type Tracker struct { 135 | logger *log.Logger 136 | provider Provider 137 | config *Config 138 | store store.Store 139 | entry store.Entry 140 | preSyncOnce sync.Once 141 | blockTracker *blocktracker.BlockTracker 142 | synced int32 143 | BlockCh chan *blocktracker.BlockEvent 144 | ReadyCh chan struct{} 145 | SyncCh chan uint64 146 | EventCh chan *Event 147 | DoneCh chan struct{} 148 | } 149 | 150 | // NewTracker creates a new tracker 151 | func NewTracker(provider Provider, opts ...ConfigOption) (*Tracker, error) { 152 | config := DefaultConfig() 153 | for _, opt := range opts { 154 | opt(config) 155 | } 156 | 157 | t := &Tracker{ 158 | provider: provider, 159 | config: config, 160 | BlockCh: make(chan *blocktracker.BlockEvent, 1), 161 | logger: log.New(ioutil.Discard, "", log.LstdFlags), 162 | ReadyCh: make(chan struct{}), 163 | store: config.Store, 164 | blockTracker: config.BlockTracker, 165 | DoneCh: make(chan struct{}, 1), 166 | EventCh: make(chan *Event), 167 | SyncCh: make(chan uint64, 1), 168 | synced: 0, 169 | } 170 | if err := t.setupFilter(); err != nil { 171 | return nil, err 172 | } 173 | return t, nil 174 | } 175 | 176 | // NewFilter creates a new log filter 177 | func (t *Tracker) setupFilter() error { 178 | if t.config.Filter == nil { 179 | // generic config 180 | t.config.Filter = &FilterConfig{} 181 | } 182 | 183 | // generate a random hash if not provided 184 | if t.config.Filter.Hash == "" { 185 | t.config.Filter.buildHash() 186 | } 187 | 188 | entry, err := t.store.GetEntry(t.config.Filter.Hash) 189 | if err != nil { 190 | return err 191 | } 192 | t.entry = entry 193 | 194 | // insert the filter config in the db 195 | filterKey := dbFilter + "_" + t.config.Filter.Hash 196 | data, err := t.store.Get(filterKey) 197 | if err != nil { 198 | return err 199 | } 200 | if data == "" { 201 | raw, err := json.Marshal(t.config.Filter) 202 | if err != nil { 203 | return err 204 | } 205 | rawStr := hex.EncodeToString(raw) 206 | if err := t.store.Set(filterKey, rawStr); err != nil { 207 | return err 208 | } 209 | } 210 | return nil 211 | } 212 | 213 | func (t *Tracker) Entry() store.Entry { 214 | return t.entry 215 | } 216 | 217 | // GetLastBlock returns the last block processed for this filter 218 | func (t *Tracker) GetLastBlock() (*web3.Block, error) { 219 | buf, err := t.store.Get(dbLastBlock + "_" + t.config.Filter.Hash) 220 | if err != nil { 221 | return nil, err 222 | } 223 | if len(buf) == 0 { 224 | return nil, nil 225 | } 226 | raw, err := hex.DecodeString(buf) 227 | if err != nil { 228 | return nil, err 229 | } 230 | b := &web3.Block{} 231 | if err := b.UnmarshalJSON(raw); err != nil { 232 | return nil, err 233 | } 234 | return b, nil 235 | } 236 | 237 | func (t *Tracker) storeLastBlock(b *web3.Block) error { 238 | if b.Difficulty == nil { 239 | b.Difficulty = big.NewInt(0) 240 | } 241 | buf, err := b.MarshalJSON() 242 | if err != nil { 243 | return err 244 | } 245 | raw := hex.EncodeToString(buf) 246 | return t.store.Set(dbLastBlock+"_"+t.config.Filter.Hash, raw) 247 | } 248 | 249 | func (t *Tracker) emitEvent(evnt *Event) { 250 | if evnt == nil { 251 | return 252 | } 253 | if t.config.Filter.Async { 254 | select { 255 | case t.EventCh <- evnt: 256 | default: 257 | } 258 | } else { 259 | t.EventCh <- evnt 260 | } 261 | } 262 | 263 | // IsSynced returns true if the filter is synced to head 264 | func (t *Tracker) IsSynced() bool { 265 | return atomic.LoadInt32(&t.synced) != 0 266 | } 267 | 268 | // Wait waits the filter to finish 269 | func (t *Tracker) Wait() { 270 | t.WaitDuration(0) 271 | } 272 | 273 | // WaitDuration waits for the filter to finish up to duration 274 | func (t *Tracker) WaitDuration(dur time.Duration) error { 275 | if t.IsSynced() { 276 | return nil 277 | } 278 | 279 | var waitCh <-chan time.Time 280 | if dur == 0 { 281 | waitCh = time.After(dur) 282 | } 283 | select { 284 | case <-waitCh: 285 | return fmt.Errorf("timeout") 286 | case <-t.DoneCh: 287 | } 288 | return nil 289 | } 290 | 291 | func (t *Tracker) findAncestor(block, pivot *web3.Block) (uint64, error) { 292 | // block is part of a fork that is not the current head, find a common ancestor 293 | // both block and pivot are at the same height 294 | var err error 295 | 296 | for i := uint64(0); i < t.blockTracker.MaxBlockBacklog(); i++ { 297 | if block.Number != pivot.Number { 298 | return 0, fmt.Errorf("block numbers do not match") 299 | } 300 | if block.Hash == pivot.Hash { 301 | // this is the common ancestor in both 302 | return block.Number, nil 303 | } 304 | block, err = t.provider.GetBlockByHash(block.ParentHash, false) 305 | if err != nil { 306 | return 0, err 307 | } 308 | pivot, err = t.provider.GetBlockByHash(pivot.ParentHash, false) 309 | if err != nil { 310 | return 0, err 311 | } 312 | } 313 | return 0, fmt.Errorf("the reorg is bigger than maxBlockBacklog %d", t.blockTracker.MaxBlockBacklog()) 314 | } 315 | 316 | func (t *Tracker) emitLogs(typ EventType, logs []*web3.Log) { 317 | evnt := &Event{} 318 | if typ == EventAdd { 319 | evnt.Added = logs 320 | } 321 | if typ == EventDel { 322 | evnt.Removed = logs 323 | } 324 | t.emitEvent(evnt) 325 | } 326 | 327 | func tooMuchDataRequestedError(err error) bool { 328 | obj, ok := err.(*codec.ErrorObject) 329 | if !ok { 330 | return false 331 | } 332 | if obj.Message == "query returned more than 10000 results" { 333 | return true 334 | } 335 | return false 336 | } 337 | 338 | func (t *Tracker) syncBatch(ctx context.Context, from, to uint64) error { 339 | query := t.config.Filter.getFilterSearch() 340 | 341 | batchSize := t.config.BatchSize 342 | additiveFactor := uint64(float64(batchSize) * 0.10) 343 | 344 | i := from 345 | 346 | START: 347 | dst := min(to, i+batchSize) 348 | 349 | query.SetFromUint64(i) 350 | query.SetToUint64(dst) 351 | 352 | logs, err := t.provider.GetLogs(query) 353 | if err != nil { 354 | if tooMuchDataRequestedError(err) { 355 | // multiplicative decrease 356 | batchSize = batchSize / 2 357 | goto START 358 | } 359 | return err 360 | } 361 | 362 | if t.SyncCh != nil { 363 | select { 364 | case t.SyncCh <- dst: 365 | default: 366 | } 367 | } 368 | 369 | // add logs to the store 370 | if err := t.entry.StoreLogs(logs); err != nil { 371 | return err 372 | } 373 | t.emitLogs(EventAdd, logs) 374 | 375 | // update the last block entry 376 | block, err := t.provider.GetBlockByNumber(web3.BlockNumber(dst), false) 377 | if err != nil { 378 | return err 379 | } 380 | if err := t.storeLastBlock(block); err != nil { 381 | return err 382 | } 383 | 384 | // check if the execution is over after each query batch 385 | if err := ctx.Err(); err != nil { 386 | return err 387 | } 388 | 389 | i += batchSize + 1 390 | 391 | // update the batchSize with additive increase 392 | if batchSize < t.config.BatchSize { 393 | batchSize = min(t.config.BatchSize, batchSize+additiveFactor) 394 | } 395 | 396 | if i <= to { 397 | goto START 398 | } 399 | return nil 400 | } 401 | 402 | func (t *Tracker) preSyncCheck() error { 403 | var err error 404 | t.preSyncOnce.Do(func() { 405 | err = t.preSyncCheckImpl() 406 | }) 407 | return err 408 | } 409 | 410 | func (t *Tracker) preSyncCheckImpl() error { 411 | rGenesis, err := t.provider.GetBlockByNumber(0, false) 412 | if err != nil { 413 | return err 414 | } 415 | rChainID, err := t.provider.ChainID() 416 | if err != nil { 417 | return err 418 | } 419 | 420 | genesis, err := t.store.Get(dbGenesis) 421 | if err != nil { 422 | return err 423 | } 424 | chainID, err := t.store.Get(dbChainID) 425 | if err != nil { 426 | return err 427 | } 428 | if len(genesis) != 0 { 429 | if genesis != rGenesis.Hash.String() { 430 | return fmt.Errorf("bad genesis") 431 | } 432 | if chainID != rChainID.String() { 433 | return fmt.Errorf("bad genesis") 434 | } 435 | } else { 436 | if err := t.store.Set(dbGenesis, rGenesis.Hash.String()); err != nil { 437 | return err 438 | } 439 | if err := t.store.Set(dbChainID, rChainID.String()); err != nil { 440 | return err 441 | } 442 | } 443 | return nil 444 | } 445 | 446 | func (t *Tracker) fastTrack(filterConfig *FilterConfig) (*web3.Block, error) { 447 | // Try to use first the user provided block if any 448 | if filterConfig.Start != 0 { 449 | bb, err := t.provider.GetBlockByNumber(web3.BlockNumber(filterConfig.Start), false) 450 | if err != nil { 451 | return nil, err 452 | } 453 | return bb, nil 454 | } 455 | 456 | // Only possible if we filter addresses 457 | if len(filterConfig.Address) == 0 { 458 | return nil, nil 459 | } 460 | 461 | if t.config.EtherscanAPIKey != "" { 462 | chainID, err := t.provider.ChainID() 463 | if err != nil { 464 | return nil, err 465 | } 466 | 467 | // get the etherscan instance for this chainID 468 | e, err := etherscan.NewEtherscanFromNetwork(web3.Network(chainID.Uint64()), t.config.EtherscanAPIKey) 469 | if err != nil { 470 | // there is no etherscan api for this specific chainid 471 | return nil, nil 472 | } 473 | 474 | getAddress := func(addr web3.Address) (uint64, error) { 475 | params := map[string]string{ 476 | "address": addr.String(), 477 | "fromBlock": "0", 478 | "toBlock": "latest", 479 | } 480 | var out []map[string]interface{} 481 | if err := e.Query("logs", "getLogs", &out, params); err != nil { 482 | return 0, err 483 | } 484 | if len(out) == 0 { 485 | return 0, nil 486 | } 487 | 488 | cc, ok := out[0]["blockNumber"].(string) 489 | if !ok { 490 | return 0, fmt.Errorf("failed to cast blocknumber") 491 | } 492 | 493 | num, err := parseUint64orHex(cc) 494 | if err != nil { 495 | return 0, err 496 | } 497 | return num, nil 498 | } 499 | 500 | minBlock := ^uint64(0) // max uint64 501 | for _, addr := range filterConfig.Address { 502 | num, err := getAddress(addr) 503 | if err != nil { 504 | return nil, err 505 | } 506 | if num < minBlock { 507 | minBlock = num 508 | } 509 | } 510 | 511 | bb, err := t.provider.GetBlockByNumber(web3.BlockNumber(minBlock-1), false) 512 | if err != nil { 513 | return nil, err 514 | } 515 | return bb, nil 516 | } 517 | 518 | return nil, nil 519 | } 520 | 521 | func (t *Tracker) BatchSync(ctx context.Context) error { 522 | if err := t.preSyncCheck(); err != nil { 523 | return err 524 | } 525 | 526 | if t.blockTracker == nil { 527 | // run a specfic block tracker 528 | t.blockTracker = blocktracker.NewBlockTracker(t.provider) 529 | if err := t.blockTracker.Init(); err != nil { 530 | return err 531 | } 532 | go t.blockTracker.Start() 533 | go func() { 534 | // track our stop 535 | <-ctx.Done() 536 | t.blockTracker.Close() 537 | }() 538 | } else { 539 | // just try to init 540 | if err := t.blockTracker.Init(); err != nil { 541 | return err 542 | } 543 | } 544 | 545 | close(t.ReadyCh) 546 | 547 | if err := t.syncImpl(ctx); err != nil { 548 | return err 549 | } 550 | 551 | select { 552 | case t.DoneCh <- struct{}{}: 553 | default: 554 | } 555 | 556 | atomic.StoreInt32(&t.synced, 1) 557 | return nil 558 | } 559 | 560 | // Sync syncs a specific filter 561 | func (t *Tracker) Sync(ctx context.Context) error { 562 | if err := t.BatchSync(ctx); err != nil { 563 | return err 564 | } 565 | 566 | // subscribe and sync 567 | sub := t.blockTracker.Subscribe() 568 | go func() { 569 | for { 570 | select { 571 | case evnt := <-sub: 572 | t.handleBlockEvnt(evnt) 573 | case <-ctx.Done(): 574 | return 575 | } 576 | } 577 | }() 578 | 579 | return nil 580 | } 581 | 582 | func (t *Tracker) syncImpl(ctx context.Context) error { 583 | if err := t.preSyncCheck(); err != nil { 584 | return err 585 | } 586 | 587 | lock := t.blockTracker.AcquireLock() 588 | defer func() { 589 | if lock.Locked { 590 | lock.Unlock() 591 | } 592 | }() 593 | 594 | // We only hold the lock when we sync the head (last MaxBackLogs) 595 | // because we want to avoid changes in the head while we sync. 596 | // We will only release the lock if we do a bulk sync since it can 597 | // move the target block for the sync. 598 | 599 | lock.Lock() 600 | if t.blockTracker.Len() == 0 { 601 | return nil 602 | } 603 | 604 | // get the current target 605 | target := t.blockTracker.LastBlocked() 606 | if target == nil { 607 | return nil 608 | } 609 | targetNum := target.Number 610 | 611 | last, err := t.GetLastBlock() 612 | if err != nil { 613 | return err 614 | } 615 | if last == nil { 616 | // Try to fast track to the valid block (if possible) 617 | last, err = t.fastTrack(t.config.Filter) 618 | if err != nil { 619 | return fmt.Errorf("failed to fasttrack: %v", err) 620 | } 621 | if last != nil { 622 | if err := t.storeLastBlock(last); err != nil { 623 | return err 624 | } 625 | } 626 | } else { 627 | if last.Hash == target.Hash { 628 | return nil 629 | } 630 | } 631 | 632 | // There might been a reorg when we stopped syncing last time, 633 | // check that our 'beacon' block matches the one in the chain. 634 | // If that is not the case, we consider beacon-maxBackLog our 635 | // real origin point and remove any logs ahead of that point. 636 | 637 | var origin uint64 638 | if last != nil { 639 | if last.Number > targetNum { 640 | return fmt.Errorf("store is more advanced than the chain") 641 | } 642 | 643 | pivot, err := t.provider.GetBlockByNumber(web3.BlockNumber(last.Number), false) 644 | if err != nil { 645 | return err 646 | } 647 | 648 | if last.Number == targetNum { 649 | origin = last.Number 650 | } else { 651 | origin = last.Number + 1 652 | } 653 | 654 | if pivot.Hash != last.Hash { 655 | ancestor, err := t.findAncestor(last, pivot) 656 | if err != nil { 657 | return err 658 | } 659 | 660 | origin = ancestor + 1 661 | logs, err := t.removeLogs(ancestor+1, nil) 662 | if err != nil { 663 | return err 664 | } 665 | t.emitLogs(EventDel, logs) 666 | } 667 | } 668 | 669 | step := targetNum - origin + 1 670 | if step > t.blockTracker.MaxBlockBacklog() { 671 | // we are far (more than maxBackLog) from the target block 672 | // Do a bulk sync with the eth_getLogs endpoint and get closer 673 | // to the target block. 674 | 675 | for { 676 | if origin > targetNum { 677 | return fmt.Errorf("from (%d) higher than to (%d)", origin, targetNum) 678 | } 679 | if targetNum-origin+1 <= t.blockTracker.MaxBlockBacklog() { 680 | break 681 | } 682 | 683 | // release the lock 684 | lock.Unlock() 685 | 686 | limit := targetNum - t.blockTracker.MaxBlockBacklog() 687 | if err := t.syncBatch(ctx, origin, limit); err != nil { 688 | return err 689 | } 690 | 691 | origin = limit + 1 692 | 693 | // lock again to reset the target block 694 | lock.Lock() 695 | targetNum = t.blockTracker.LastBlocked().Number 696 | } 697 | } 698 | 699 | // we are still holding the lock on the blocksLock so that we are sure 700 | // that the targetNum has not changed 701 | trackerBlocks := t.blockTracker.BlocksBlocked() 702 | added := trackerBlocks[uint64(len(trackerBlocks))-1-(targetNum-origin):] 703 | 704 | evnt, err := t.doFilter(added, nil) 705 | if err != nil { 706 | return err 707 | } 708 | if evnt != nil { 709 | t.emitEvent(evnt) 710 | } 711 | 712 | // release the lock on the blocks 713 | lock.Unlock() 714 | return nil 715 | } 716 | 717 | func (t *Tracker) removeLogs(number uint64, hash *web3.Hash) ([]*web3.Log, error) { 718 | index, err := t.entry.LastIndex() 719 | if err != nil { 720 | return nil, err 721 | } 722 | if index == 0 { 723 | return nil, nil 724 | } 725 | 726 | var remove []*web3.Log 727 | for { 728 | elemIndex := index - 1 729 | 730 | var log web3.Log 731 | if err := t.entry.GetLog(elemIndex, &log); err != nil { 732 | return nil, err 733 | } 734 | if log.BlockNumber == number { 735 | if hash != nil && log.BlockHash != *hash { 736 | break 737 | } 738 | } 739 | if log.BlockNumber < number { 740 | break 741 | } 742 | remove = append(remove, &log) 743 | if elemIndex == 0 { 744 | index = 0 745 | break 746 | } 747 | index = elemIndex 748 | } 749 | 750 | if err := t.entry.RemoveLogs(index); err != nil { 751 | return nil, err 752 | } 753 | return remove, nil 754 | } 755 | 756 | func revertLogs(in []*web3.Log) (out []*web3.Log) { 757 | for i := len(in) - 1; i >= 0; i-- { 758 | out = append(out, in[i]) 759 | } 760 | return 761 | } 762 | 763 | func (t *Tracker) handleBlockEvnt(blockEvnt *blocktracker.BlockEvent) error { 764 | if blockEvnt == nil { 765 | return nil 766 | } 767 | 768 | // emit the block event 769 | select { 770 | case t.BlockCh <- blockEvnt: 771 | default: 772 | } 773 | 774 | if t.IsSynced() { 775 | evnt, err := t.doFilter(blockEvnt.Added, blockEvnt.Removed) 776 | if err != nil { 777 | return err 778 | } 779 | if evnt != nil { 780 | t.emitEvent(evnt) 781 | } 782 | } 783 | return nil 784 | } 785 | 786 | func (t *Tracker) doFilter(added []*web3.Block, removed []*web3.Block) (*Event, error) { 787 | evnt := &Event{} 788 | if len(removed) != 0 { 789 | pivot := removed[0] 790 | logs, err := t.removeLogs(pivot.Number, &pivot.Hash) 791 | if err != nil { 792 | return nil, err 793 | } 794 | evnt.Removed = append(evnt.Removed, revertLogs(logs)...) 795 | } 796 | 797 | for _, block := range added { 798 | // check logs for this blocks 799 | query := t.config.Filter.getFilterSearch() 800 | query.BlockHash = &block.Hash 801 | 802 | // We check the hash, we need to do a retry to let unsynced nodes get the block 803 | var logs []*web3.Log 804 | var err error 805 | 806 | for i := 0; i < 5; i++ { 807 | logs, err = t.provider.GetLogs(query) 808 | if err == nil { 809 | break 810 | } 811 | time.Sleep(500 * time.Millisecond) 812 | } 813 | if err != nil { 814 | return nil, err 815 | } 816 | 817 | // add logs to the store 818 | if err := t.entry.StoreLogs(logs); err != nil { 819 | return nil, err 820 | } 821 | evnt.Added = append(evnt.Added, logs...) 822 | } 823 | 824 | // store the last block as the new index 825 | if err := t.storeLastBlock(added[len(added)-1]); err != nil { 826 | return nil, err 827 | } 828 | return evnt, nil 829 | } 830 | 831 | // EventType is the type of the event 832 | type EventType int 833 | 834 | const ( 835 | // EventAdd happens when a new event is included in the chain 836 | EventAdd EventType = iota 837 | // EventDel may happen when there is a reorg and a past event is deleted 838 | EventDel 839 | ) 840 | 841 | // Event is an event emitted when a new log is included 842 | type Event struct { 843 | Type EventType 844 | Added []*web3.Log 845 | Removed []*web3.Log 846 | } 847 | 848 | // BlockEvent is an event emitted when a new block is included 849 | type BlockEvent struct { 850 | Type EventType 851 | Added []*web3.Block 852 | Removed []*web3.Block 853 | } 854 | 855 | func min(i, j uint64) uint64 { 856 | if i < j { 857 | return i 858 | } 859 | return j 860 | } 861 | 862 | func parseUint64orHex(str string) (uint64, error) { 863 | base := 10 864 | if strings.HasPrefix(str, "0x") { 865 | str = str[2:] 866 | base = 16 867 | } 868 | return strconv.ParseUint(str, base, 64) 869 | } 870 | -------------------------------------------------------------------------------- /tracker_test.go: -------------------------------------------------------------------------------- 1 | package tracker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/big" 7 | "math/rand" 8 | "reflect" 9 | "strconv" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/assert" 14 | web3 "github.com/umbracle/go-web3" 15 | "github.com/umbracle/go-web3/abi" 16 | "github.com/umbracle/go-web3/blocktracker" 17 | "github.com/umbracle/go-web3/jsonrpc" 18 | "github.com/umbracle/go-web3/jsonrpc/codec" 19 | "github.com/umbracle/go-web3/testutil" 20 | "github.com/umbracle/go-web3/tracker/store/inmem" 21 | ) 22 | 23 | func testConfig() ConfigOption { 24 | return func(c *Config) { 25 | c.BatchSize = 10 26 | } 27 | } 28 | 29 | func testFilter(t *testing.T, provider Provider, filterConfig *FilterConfig) []*web3.Log { 30 | filterConfig.Async = true 31 | tt, _ := NewTracker(provider, WithFilter(filterConfig)) 32 | 33 | ctx, cancelFn := context.WithCancel(context.Background()) 34 | defer cancelFn() 35 | 36 | if err := tt.Sync(ctx); err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | return tt.entry.(*inmem.Entry).Logs() 41 | } 42 | 43 | func TestPolling(t *testing.T) { 44 | s := testutil.NewTestServer(t, nil) 45 | defer s.Close() 46 | 47 | client, _ := jsonrpc.NewClient(s.HTTPAddr()) 48 | 49 | c0 := &testutil.Contract{} 50 | c0.AddEvent(testutil.NewEvent("A").Add("uint256", true).Add("uint256", true)) 51 | c0.EmitEvent("setA1", "A", "1", "2") 52 | 53 | _, addr0 := s.DeployContract(c0) 54 | 55 | // send 5 txns 56 | for i := 0; i < 5; i++ { 57 | s.TxnTo(addr0, "setA1") 58 | } 59 | 60 | tt, err := NewTracker(client.Eth()) 61 | assert.NoError(t, err) 62 | 63 | ctx, cancelFn := context.WithCancel(context.Background()) 64 | defer cancelFn() 65 | 66 | go func() { 67 | if err := tt.Sync(ctx); err != nil { 68 | panic(err) 69 | } 70 | }() 71 | 72 | // wait for the bulk sync to finish 73 | for { 74 | select { 75 | case <-tt.EventCh: 76 | case <-tt.DoneCh: 77 | goto EXIT 78 | case <-time.After(1 * time.Second): 79 | t.Fatal("timeout to sync") 80 | } 81 | } 82 | EXIT: 83 | 84 | // send another 5 transactions, we have to have another log each time 85 | for i := 0; i < 5; i++ { 86 | receipt := s.TxnTo(addr0, "setA1") 87 | 88 | select { 89 | case evnt := <-tt.EventCh: 90 | if !reflect.DeepEqual(evnt.Added, receipt.Logs) { 91 | t.Fatal("bad") 92 | } 93 | case <-time.After(2 * time.Second): // wait at least the polling interval 94 | t.Fatal("event expected") 95 | } 96 | } 97 | } 98 | 99 | func TestFilterIntegration(t *testing.T) { 100 | s := testutil.NewTestServer(t, nil) 101 | defer s.Close() 102 | 103 | client, _ := jsonrpc.NewClient(s.HTTPAddr()) 104 | 105 | c0 := &testutil.Contract{} 106 | c0.AddEvent(testutil.NewEvent("A").Add("uint256", true).Add("uint256", true)) 107 | c0.EmitEvent("setA1", "A", "1", "2") 108 | 109 | _, addr0 := s.DeployContract(c0) 110 | _, addr1 := s.DeployContract(c0) 111 | 112 | for i := 0; i < 20; i++ { 113 | if i%2 == 0 { 114 | s.TxnTo(addr0, "setA1") 115 | } else { 116 | s.TxnTo(addr1, "setA1") 117 | } 118 | } 119 | 120 | // sync all the logs 121 | logs := testFilter(t, client.Eth(), &FilterConfig{}) 122 | if len(logs) != 20 { 123 | t.Fatal("bad") 124 | } 125 | 126 | // filter by address 127 | logs = testFilter(t, client.Eth(), &FilterConfig{Address: []web3.Address{addr0}}) 128 | if len(logs) != 10 { 129 | t.Fatal("bad") 130 | } 131 | 132 | // filter by value 133 | typ, _ := abi.NewType("uint256") 134 | topic, _ := abi.EncodeTopic(typ, 1) 135 | 136 | logs = testFilter(t, client.Eth(), &FilterConfig{Topics: []*web3.Hash{nil, &topic}}) 137 | if len(logs) != 20 { 138 | t.Fatal("bad") 139 | } 140 | } 141 | 142 | func TestFilterIntegrationEventHash(t *testing.T) { 143 | s := testutil.NewTestServer(t, nil) 144 | defer s.Close() 145 | 146 | client, _ := jsonrpc.NewClient(s.HTTPAddr()) 147 | 148 | c0 := &testutil.Contract{} 149 | c0.AddEvent(testutil.NewEvent("A").Add("uint256", true).Add("uint256", true)) 150 | c0.EmitEvent("setA1", "A", "1", "2") 151 | 152 | c1 := &testutil.Contract{} 153 | c1.AddEvent(testutil.NewEvent("B").Add("uint256", true).Add("uint256", true)) 154 | c1.EmitEvent("setB1", "B", "1", "2") 155 | 156 | artifacts0, addr0 := s.DeployContract(c0) 157 | _, addr1 := s.DeployContract(c0) 158 | 159 | abi0, _ := abi.NewABI(artifacts0.Abi) 160 | 161 | for i := 0; i < 20; i++ { 162 | if i%2 == 0 { 163 | s.TxnTo(addr0, "setA1") 164 | } else { 165 | s.TxnTo(addr1, "setB1") 166 | } 167 | } 168 | 169 | eventTopicID := abi0.Events["A"].ID() 170 | logs := testFilter(t, client.Eth(), &FilterConfig{Topics: []*web3.Hash{&eventTopicID}}) 171 | if len(logs) != 10 { 172 | t.Fatal("bad") 173 | } 174 | 175 | eventTopicID[1] = 1 176 | logs = testFilter(t, client.Eth(), &FilterConfig{Topics: []*web3.Hash{&eventTopicID}}) 177 | if len(logs) != 0 { 178 | t.Fatal("bad") 179 | } 180 | } 181 | 182 | func TestPreflight(t *testing.T) { 183 | store := inmem.NewInmemStore() 184 | 185 | l := testutil.MockList{} 186 | l.Create(0, 100, func(b *testutil.MockBlock) {}) 187 | 188 | m := &testutil.MockClient{} 189 | m.AddScenario(l) 190 | 191 | tt0, _ := NewTracker(m, testConfig(), WithStore(store)) 192 | if err := tt0.preSyncCheckImpl(); err != nil { 193 | t.Fatal(err) 194 | } 195 | 196 | // change the genesis hash 197 | 198 | l0 := testutil.MockList{} 199 | l0.Create(0, 100, func(b *testutil.MockBlock) { 200 | b.Extra("1") 201 | }) 202 | 203 | m.AddScenario(l0) 204 | 205 | tt1, _ := NewTracker(m, testConfig(), WithStore(store)) 206 | if err := tt1.preSyncCheckImpl(); err == nil { 207 | t.Fatal("it should fail") 208 | } 209 | 210 | // change the chainID 211 | 212 | m.AddScenario(l) 213 | m.SetChainID(big.NewInt(1)) 214 | 215 | tt2, _ := NewTracker(m, testConfig(), WithStore(store)) 216 | if err := tt2.preSyncCheckImpl(); err == nil { 217 | t.Fatal("it should fail") 218 | } 219 | } 220 | 221 | func TestTrackerSyncerRestarts(t *testing.T) { 222 | store := inmem.NewInmemStore() 223 | m := &testutil.MockClient{} 224 | l := testutil.MockList{} 225 | 226 | advance := func(first, last int, void ...bool) { 227 | if len(void) == 0 { 228 | l.Create(first, last, func(b *testutil.MockBlock) { 229 | if b.GetNum()%5 == 0 { 230 | b.Log("0x1") 231 | } 232 | }) 233 | m.AddScenario(l) 234 | } 235 | 236 | tt, err := NewTracker(m, 237 | testConfig(), 238 | WithStore(store), 239 | WithFilter(&FilterConfig{Async: true}), 240 | ) 241 | assert.NoError(t, err) 242 | 243 | go func() { 244 | if err := tt.Sync(context.Background()); err != nil { 245 | panic(err) 246 | } 247 | }() 248 | 249 | if err := tt.WaitDuration(2 * time.Second); err != nil { 250 | t.Fatal(err) 251 | } 252 | 253 | if tt.blockTracker.BlocksBlocked()[0].Number != uint64(last-10) { 254 | t.Fatal("bad") 255 | } 256 | if tt.blockTracker.BlocksBlocked()[9].Number != uint64(last-1) { 257 | t.Fatal("bad") 258 | } 259 | if !testutil.CompareLogs(l.GetLogs(), tt.entry.(*inmem.Entry).Logs()) { 260 | t.Fatal("bad") 261 | } 262 | } 263 | 264 | // initial range 265 | advance(0, 100) 266 | 267 | // dont advance 268 | advance(0, 100, true) 269 | 270 | // advance less than backlog 271 | advance(100, 105) 272 | 273 | // advance more than backlog 274 | advance(105, 150) 275 | } 276 | 277 | func testSyncerReconcile(t *testing.T, iniLen, forkNum, endLen int) { 278 | // test that the syncer can reconcile if there is a fork in the saved state 279 | l := testutil.MockList{} 280 | l.Create(0, iniLen, func(b *testutil.MockBlock) { 281 | b.Log("0x01") 282 | }) 283 | 284 | m := &testutil.MockClient{} 285 | m.AddScenario(l) 286 | 287 | store := inmem.NewInmemStore() 288 | 289 | tt0, err := NewTracker(m, 290 | testConfig(), 291 | WithStore(store), 292 | WithFilter(&FilterConfig{Async: true}), 293 | ) 294 | assert.NoError(t, err) 295 | 296 | go func() { 297 | if err := tt0.Sync(context.Background()); err != nil { 298 | panic(err) 299 | } 300 | }() 301 | tt0.WaitDuration(2 * time.Second) 302 | 303 | // create a fork at 'forkNum' and continue to 'endLen' 304 | l1 := testutil.MockList{} 305 | l1.Create(0, endLen, func(b *testutil.MockBlock) { 306 | if b.GetNum() < forkNum { 307 | b.Log("0x01") // old fork 308 | } else { 309 | if b.GetNum() == forkNum { 310 | b = b.Log("0x02") 311 | } else { 312 | b = b.Log("0x03") 313 | } 314 | b.Extra("123") // used to set the new fork 315 | } 316 | }) 317 | 318 | m1 := &testutil.MockClient{} 319 | m1.AddScenario(l) 320 | m1.AddScenario(l1) 321 | 322 | tt1, _ := NewTracker(m1, 323 | testConfig(), 324 | WithStore(store), 325 | WithFilter(&FilterConfig{Async: true}), 326 | ) 327 | go func() { 328 | if err := tt1.Sync(context.Background()); err != nil { 329 | panic(err) 330 | } 331 | }() 332 | tt1.WaitDuration(2 * time.Second) 333 | 334 | logs := tt1.entry.(*inmem.Entry).Logs() 335 | 336 | if !testutil.CompareLogs(l1.GetLogs(), logs) { 337 | t.Fatal("bad") 338 | } 339 | 340 | // check the content of the logs 341 | 342 | // first half 343 | for i := 0; i < forkNum; i++ { 344 | if logs[i].Data[0] != 0x1 { 345 | t.Fatal("bad") 346 | } 347 | } 348 | // fork point 349 | if logs[forkNum].Data[0] != 0x2 { 350 | t.Fatal("bad") 351 | } 352 | // second half 353 | for i := forkNum + 1; i < endLen; i++ { 354 | if logs[i].Data[0] != 0x3 { 355 | t.Fatal("bad") 356 | } 357 | } 358 | } 359 | 360 | func TestTrackerSyncerReconcile(t *testing.T) { 361 | t.Run("Backlog", func(t *testing.T) { 362 | testSyncerReconcile(t, 50, 45, 55) 363 | }) 364 | t.Run("Historical", func(t *testing.T) { 365 | testSyncerReconcile(t, 50, 45, 100) 366 | }) 367 | } 368 | 369 | func randomInt(min, max int) int { 370 | return min + rand.Intn(max-min) 371 | } 372 | 373 | func testTrackerSyncerRandom(t *testing.T, n int, backlog uint64) { 374 | m := &testutil.MockClient{} 375 | c := 0 // current block 376 | f := 0 // current fork 377 | 378 | store := inmem.NewInmemStore() 379 | 380 | for i := 0; i < n; i++ { 381 | // fmt.Println("########################################") 382 | 383 | // create the new batch of blocks 384 | var forkSize int 385 | if randomInt(0, 10) < 3 && c > 10 { 386 | // add a fork, go back at most maxBacklogSize 387 | forkSize = randomInt(1, int(backlog)) 388 | c = c - forkSize 389 | f++ 390 | } 391 | 392 | forkID := strconv.Itoa(f) 393 | 394 | // add new blocks 395 | l := testutil.MockList{} 396 | 397 | // we have to create at least the blocks removed by the fork, otherwise 398 | // we may end up going backwards if the forks remove more data than the 399 | // advance includes 400 | 401 | start := forkSize 402 | if start == 0 && i == 0 { 403 | start = 1 // at least advance one block on the first iteration 404 | } 405 | num := randomInt(start, 20) 406 | count := 0 407 | 408 | for j := c; j < c+num; j++ { 409 | bb := testutil.Mock(j).Extra(forkID) 410 | if j != 0 { 411 | count++ 412 | bb = bb.Log(forkID) 413 | } 414 | l = append(l, bb) 415 | } 416 | 417 | m.AddScenario(l) 418 | 419 | // use a custom block tracker to add specific backlog 420 | tracker := blocktracker.NewBlockTracker(m, blocktracker.WithBlockMaxBacklog(backlog)) 421 | 422 | tt, _ := NewTracker(m, 423 | testConfig(), 424 | WithStore(store), 425 | WithBlockTracker(tracker), 426 | ) 427 | 428 | go func() { 429 | if err := tt.Sync(context.Background()); err != nil { 430 | panic(err) 431 | } 432 | }() 433 | 434 | var added, removed []*web3.Log 435 | for { 436 | select { 437 | case evnt := <-tt.EventCh: 438 | added = append(added, evnt.Added...) 439 | removed = append(removed, evnt.Removed...) 440 | 441 | case <-tt.DoneCh: 442 | // no more events to read 443 | goto EXIT 444 | } 445 | } 446 | EXIT: 447 | 448 | // validate the included logs 449 | if len(added) != count { 450 | t.Fatal("bad added logs") 451 | } 452 | // validate the removed logs 453 | if len(removed) != forkSize { 454 | t.Fatal("bad removed logs") 455 | } 456 | 457 | // validate blocks 458 | if blocks := m.GetLastBlocks(backlog); !testutil.CompareBlocks(tt.blockTracker.BlocksBlocked(), blocks) { 459 | // tracker does not consider block 0 but getLastBlocks does return it, this is only a problem 460 | // with syncs on chains lower than maxBacklog 461 | if !testutil.CompareBlocks(blocks[1:], tt.blockTracker.BlocksBlocked()) { 462 | t.Fatal("bad blocks") 463 | } 464 | } 465 | // validate logs 466 | if logs := m.GetAllLogs(); !testutil.CompareLogs(tt.entry.(*inmem.Entry).Logs(), logs) { 467 | t.Fatal("bad logs") 468 | } 469 | 470 | c += num 471 | } 472 | } 473 | 474 | func TestTrackerSyncerRandom(t *testing.T) { 475 | rand.Seed(time.Now().UTC().UnixNano()) 476 | 477 | for i := 0; i < 100; i++ { 478 | t.Run("", func(t *testing.T) { 479 | testTrackerSyncerRandom(t, 100, uint64(randomInt(2, 10))) 480 | }) 481 | } 482 | } 483 | 484 | func TestTrackerReconcile(t *testing.T) { 485 | type TestEvent struct { 486 | Added testutil.MockList 487 | Removed testutil.MockList 488 | } 489 | 490 | type Reconcile struct { 491 | block *testutil.MockBlock 492 | event *TestEvent 493 | } 494 | 495 | cases := []struct { 496 | Name string 497 | Scenario testutil.MockList 498 | History testutil.MockList 499 | Reconcile []Reconcile 500 | Expected testutil.MockList 501 | }{ 502 | { 503 | Name: "Empty history", 504 | Reconcile: []Reconcile{ 505 | { 506 | block: testutil.Mock(0x1).Log("0x1"), 507 | event: &TestEvent{ 508 | Added: testutil.MockList{ 509 | testutil.Mock(0x1).Log("0x1"), 510 | }, 511 | }, 512 | }, 513 | }, 514 | Expected: []*testutil.MockBlock{ 515 | testutil.Mock(1).Log("0x1"), 516 | }, 517 | }, 518 | { 519 | Name: "Repeated header", 520 | History: []*testutil.MockBlock{ 521 | testutil.Mock(0x1), 522 | }, 523 | Reconcile: []Reconcile{ 524 | { 525 | block: testutil.Mock(0x1), 526 | }, 527 | }, 528 | Expected: []*testutil.MockBlock{ 529 | testutil.Mock(0x1), 530 | }, 531 | }, 532 | { 533 | Name: "New head", 534 | History: testutil.MockList{ 535 | testutil.Mock(0x1), 536 | }, 537 | Reconcile: []Reconcile{ 538 | { 539 | block: testutil.Mock(0x2), 540 | event: &TestEvent{ 541 | Added: testutil.MockList{ 542 | testutil.Mock(0x2), 543 | }, 544 | }, 545 | }, 546 | }, 547 | Expected: testutil.MockList{ 548 | testutil.Mock(0x1), 549 | testutil.Mock(0x2), 550 | }, 551 | }, 552 | { 553 | Name: "Ignore block already on history", 554 | History: testutil.MockList{ 555 | testutil.Mock(0x1), 556 | testutil.Mock(0x2), 557 | testutil.Mock(0x3), 558 | }, 559 | Reconcile: []Reconcile{ 560 | { 561 | block: testutil.Mock(0x2), 562 | }, 563 | }, 564 | Expected: testutil.MockList{ 565 | testutil.Mock(0x1), 566 | testutil.Mock(0x2), 567 | testutil.Mock(0x3), 568 | }, 569 | }, 570 | { 571 | Name: "Multi Roll back", 572 | History: testutil.MockList{ 573 | testutil.Mock(0x1), 574 | testutil.Mock(0x2), 575 | testutil.Mock(0x3).Log("0x3"), 576 | testutil.Mock(0x4).Log("0x4"), 577 | }, 578 | Reconcile: []Reconcile{ 579 | { 580 | block: testutil.Mock(0x30).Parent(0x2).Log("0x30"), 581 | event: &TestEvent{ 582 | Added: testutil.MockList{ 583 | testutil.Mock(0x30).Parent(0x2).Log("0x30"), 584 | }, 585 | Removed: testutil.MockList{ 586 | testutil.Mock(0x3).Log("0x3"), 587 | testutil.Mock(0x4).Log("0x4"), 588 | }, 589 | }, 590 | }, 591 | }, 592 | Expected: testutil.MockList{ 593 | testutil.Mock(0x1), 594 | testutil.Mock(0x2), 595 | testutil.Mock(0x30).Parent(0x2).Log("0x30"), 596 | }, 597 | }, 598 | { 599 | Name: "Backfills missing blocks", 600 | Scenario: testutil.MockList{ 601 | testutil.Mock(0x3), 602 | testutil.Mock(0x4).Log("0x2"), 603 | }, 604 | History: testutil.MockList{ 605 | testutil.Mock(0x1).Log("0x1"), 606 | testutil.Mock(0x2), 607 | }, 608 | Reconcile: []Reconcile{ 609 | { 610 | block: testutil.Mock(0x5).Log("0x3"), 611 | event: &TestEvent{ 612 | Added: testutil.MockList{ 613 | testutil.Mock(0x3), 614 | testutil.Mock(0x4).Log("0x2"), 615 | testutil.Mock(0x5).Log("0x3"), 616 | }, 617 | }, 618 | }, 619 | }, 620 | Expected: testutil.MockList{ 621 | testutil.Mock(0x1).Log("0x1"), 622 | testutil.Mock(0x2), 623 | testutil.Mock(0x3), 624 | testutil.Mock(0x4).Log("0x2"), 625 | testutil.Mock(0x5).Log("0x3"), 626 | }, 627 | }, 628 | { 629 | Name: "Rolls back and backfills", 630 | Scenario: testutil.MockList{ 631 | testutil.Mock(0x30).Parent(0x2).Num(3).Log("0x5"), 632 | testutil.Mock(0x40).Parent(0x30).Num(4), 633 | }, 634 | History: testutil.MockList{ 635 | testutil.Mock(0x1), 636 | testutil.Mock(0x2).Log("0x3"), 637 | testutil.Mock(0x3).Log("0x2"), 638 | testutil.Mock(0x4).Log("0x1"), 639 | }, 640 | Reconcile: []Reconcile{ 641 | { 642 | block: testutil.Mock(0x50).Parent(0x40).Num(5), 643 | event: &TestEvent{ 644 | Added: testutil.MockList{ 645 | testutil.Mock(0x30).Parent(0x2).Num(3).Log("0x5"), 646 | testutil.Mock(0x40).Parent(0x30).Num(4), 647 | testutil.Mock(0x50).Parent(0x40).Num(5), 648 | }, 649 | Removed: testutil.MockList{ 650 | testutil.Mock(0x3).Log("0x2"), 651 | testutil.Mock(0x4).Log("0x1"), 652 | }, 653 | }, 654 | }, 655 | }, 656 | Expected: testutil.MockList{ 657 | testutil.Mock(0x1), 658 | testutil.Mock(0x2).Log("0x3"), 659 | testutil.Mock(0x30).Parent(0x2).Num(3).Log("0x5"), 660 | testutil.Mock(0x40).Parent(0x30).Num(4), 661 | testutil.Mock(0x50).Parent(0x40).Num(5), 662 | }, 663 | }, 664 | } 665 | 666 | for _, c := range cases { 667 | t.Run(c.Name, func(t *testing.T) { 668 | // safe check for now, we ma need to restart the tracker and mock client for every reconcile scenario? 669 | if len(c.Reconcile) != 1 { 670 | t.Fatal("only one reconcile supported so far") 671 | } 672 | 673 | m := &testutil.MockClient{} 674 | 675 | // add the full scenario with the logs 676 | m.AddScenario(c.Scenario) 677 | 678 | // add the logs of the reconcile block because those are also unknown for the tracker 679 | m.AddLogs(c.Reconcile[0].block.GetLogs()) 680 | 681 | store := inmem.NewInmemStore() 682 | 683 | btracker := blocktracker.NewBlockTracker(m) 684 | 685 | tt, err := NewTracker(m, WithStore(store), WithBlockTracker(btracker)) 686 | if err != nil { 687 | t.Fatal(err) 688 | } 689 | 690 | // important to set a buffer here, otherwise everything is blocked 691 | tt.EventCh = make(chan *Event, 1) 692 | 693 | // set the filter as synced since we only want to 694 | // try reconciliation 695 | tt.synced = 1 696 | 697 | // build past block history 698 | for _, b := range c.History.ToBlocks() { 699 | tt.blockTracker.AddBlockLocked(b) 700 | } 701 | // add the history to the store 702 | for _, b := range c.History { 703 | tt.entry.StoreLogs(b.GetLogs()) 704 | } 705 | 706 | for _, b := range c.Reconcile { 707 | aux, err := tt.blockTracker.HandleBlockEvent(b.block.Block()) 708 | if err != nil { 709 | t.Fatal(err) 710 | } 711 | if aux == nil { 712 | continue 713 | } 714 | if err := tt.handleBlockEvnt(aux); err != nil { 715 | t.Fatal(err) 716 | } 717 | 718 | var evnt *Event 719 | select { 720 | case evnt = <-tt.EventCh: 721 | case <-time.After(1 * time.Second): 722 | t.Fatal("log event timeout") 723 | } 724 | 725 | // check logs 726 | if !testutil.CompareLogs(b.event.Added.GetLogs(), evnt.Added) { 727 | t.Fatal("err") 728 | } 729 | if !testutil.CompareLogs(b.event.Removed.GetLogs(), evnt.Removed) { 730 | t.Fatal("err") 731 | } 732 | 733 | var blockEvnt *blocktracker.BlockEvent 734 | select { 735 | case blockEvnt = <-tt.BlockCh: 736 | case <-time.After(1 * time.Second): 737 | t.Fatal("block event timeout") 738 | } 739 | 740 | // check blocks 741 | if !testutil.CompareBlocks(b.event.Added.ToBlocks(), blockEvnt.Added) { 742 | t.Fatal("err") 743 | } 744 | if !testutil.CompareBlocks(b.event.Removed.ToBlocks(), blockEvnt.Removed) { 745 | t.Fatal("err") 746 | } 747 | } 748 | 749 | // check the post state (logs and blocks) after all the reconcile events 750 | if !testutil.CompareLogs(tt.entry.(*inmem.Entry).Logs(), c.Expected.GetLogs()) { 751 | t.Fatal("bad3") 752 | } 753 | if !testutil.CompareBlocks(tt.blockTracker.BlocksBlocked(), c.Expected.ToBlocks()) { 754 | t.Fatal("bad") 755 | } 756 | }) 757 | } 758 | } 759 | 760 | type mockClientWithLimit struct { 761 | limit uint64 762 | *testutil.MockClient 763 | } 764 | 765 | func (m *mockClientWithLimit) GetLogs(filter *web3.LogFilter) ([]*web3.Log, error) { 766 | if filter.BlockHash != nil { 767 | return m.MockClient.GetLogs(filter) 768 | } 769 | from, to := uint64(*filter.From), uint64(*filter.To) 770 | if from > to { 771 | return nil, fmt.Errorf("from higher than to") 772 | } 773 | if to-from > m.limit { 774 | return nil, &codec.ErrorObject{Message: "query returned more than 10000 results"} 775 | } 776 | // fallback to the client 777 | return m.MockClient.GetLogs(filter) 778 | } 779 | 780 | func TestTooMuchDataRequested(t *testing.T) { 781 | count := 0 782 | 783 | // create 100 blocks with 2 (even) or 5 (odd) logs each 784 | l := testutil.MockList{} 785 | l.Create(0, 100, func(b *testutil.MockBlock) { 786 | var numLogs int 787 | if b.GetNum()%2 == 0 { 788 | numLogs = 2 789 | } else { 790 | numLogs = 5 791 | } 792 | for i := 0; i < numLogs; i++ { 793 | count++ 794 | b.Log("0x1") 795 | } 796 | }) 797 | 798 | m := &testutil.MockClient{} 799 | m.AddScenario(l) 800 | 801 | mm := &mockClientWithLimit{ 802 | limit: 3, 803 | MockClient: m, 804 | } 805 | 806 | config := DefaultConfig() 807 | config.BatchSize = 11 808 | 809 | tt, _ := NewTracker(mm, 810 | WithFilter(&FilterConfig{Async: true}), 811 | ) 812 | if err := tt.Sync(context.Background()); err != nil { 813 | t.Fatal(err) 814 | } 815 | if count != len(tt.entry.(*inmem.Entry).Logs()) { 816 | t.Fatal("not the same count") 817 | } 818 | } 819 | --------------------------------------------------------------------------------