├── LICENSE ├── README.md ├── fixtures └── transactions.json ├── go.mod ├── go.sum ├── iterator.go ├── store.go ├── store_ancient.go ├── store_leveldb.go ├── types.go └── types_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | # Mozilla Public License Version 2.0 2 | 3 | 1. Definitions 4 | 5 | --- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | 88 | --- 89 | 90 | 2.1. Grants 91 | 92 | Each Contributor hereby grants You a world-wide, royalty-free, 93 | non-exclusive license: 94 | 95 | (a) under intellectual property rights (other than patent or trademark) 96 | Licensable by such Contributor to use, reproduce, make available, 97 | modify, display, perform, distribute, and otherwise exploit its 98 | Contributions, either on an unmodified basis, with Modifications, or 99 | as part of a Larger Work; and 100 | 101 | (b) under Patent Claims of such Contributor to make, use, sell, offer 102 | for sale, have made, import, and otherwise transfer either its 103 | Contributions or its Contributor Version. 104 | 105 | 2.2. Effective Date 106 | 107 | The licenses granted in Section 2.1 with respect to any Contribution 108 | become effective for each Contribution on the date the Contributor first 109 | distributes such Contribution. 110 | 111 | 2.3. Limitations on Grant Scope 112 | 113 | The licenses granted in this Section 2 are the only rights granted under 114 | this License. No additional rights or licenses will be implied from the 115 | distribution or licensing of Covered Software under this License. 116 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 117 | Contributor: 118 | 119 | (a) for any code that a Contributor has removed from Covered Software; 120 | or 121 | 122 | (b) for infringements caused by: (i) Your and any other third party's 123 | modifications of Covered Software, or (ii) the combination of its 124 | Contributions with other software (except as part of its Contributor 125 | Version); or 126 | 127 | (c) under Patent Claims infringed by Covered Software in the absence of 128 | its Contributions. 129 | 130 | This License does not grant any rights in the trademarks, service marks, 131 | or logos of any Contributor (except as may be necessary to comply with 132 | the notice requirements in Section 3.4). 133 | 134 | 2.4. Subsequent Licenses 135 | 136 | No Contributor makes additional grants as a result of Your choice to 137 | distribute the Covered Software under a subsequent version of this 138 | License (see Section 10.2) or under the terms of a Secondary License (if 139 | permitted under the terms of Section 3.3). 140 | 141 | 2.5. Representation 142 | 143 | Each Contributor represents that the Contributor believes its 144 | Contributions are its original creation(s) or it has sufficient rights 145 | to grant the rights to its Contributions conveyed by this License. 146 | 147 | 2.6. Fair Use 148 | 149 | This License is not intended to limit any rights You have under 150 | applicable copyright doctrines of fair use, fair dealing, or other 151 | equivalents. 152 | 153 | 2.7. Conditions 154 | 155 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 156 | in Section 2.1. 157 | 158 | 3. Responsibilities 159 | 160 | --- 161 | 162 | 3.1. Distribution of Source Form 163 | 164 | All distribution of Covered Software in Source Code Form, including any 165 | Modifications that You create or to which You contribute, must be under 166 | the terms of this License. You must inform recipients that the Source 167 | Code Form of the Covered Software is governed by the terms of this 168 | License, and how they can obtain a copy of this License. You may not 169 | attempt to alter or restrict the recipients' rights in the Source Code 170 | Form. 171 | 172 | 3.2. Distribution of Executable Form 173 | 174 | If You distribute Covered Software in Executable Form then: 175 | 176 | (a) such Covered Software must also be made available in Source Code 177 | Form, as described in Section 3.1, and You must inform recipients of 178 | the Executable Form how they can obtain a copy of such Source Code 179 | Form by reasonable means in a timely manner, at a charge no more 180 | than the cost of distribution to the recipient; and 181 | 182 | (b) You may distribute such Executable Form under the terms of this 183 | License, or sublicense it under different terms, provided that the 184 | license for the Executable Form does not attempt to limit or alter 185 | the recipients' rights in the Source Code Form under this License. 186 | 187 | 3.3. Distribution of a Larger Work 188 | 189 | You may create and distribute a Larger Work under terms of Your choice, 190 | provided that You also comply with the requirements of this License for 191 | the Covered Software. If the Larger Work is a combination of Covered 192 | Software with a work governed by one or more Secondary Licenses, and the 193 | Covered Software is not Incompatible With Secondary Licenses, this 194 | License permits You to additionally distribute such Covered Software 195 | under the terms of such Secondary License(s), so that the recipient of 196 | the Larger Work may, at their option, further distribute the Covered 197 | Software under the terms of either this License or such Secondary 198 | License(s). 199 | 200 | 3.4. Notices 201 | 202 | You may not remove or alter the substance of any license notices 203 | (including copyright notices, patent notices, disclaimers of warranty, 204 | or limitations of liability) contained within the Source Code Form of 205 | the Covered Software, except that You may alter any license notices to 206 | the extent required to remedy known factual inaccuracies. 207 | 208 | 3.5. Application of Additional Terms 209 | 210 | You may choose to offer, and to charge a fee for, warranty, support, 211 | indemnity or liability obligations to one or more recipients of Covered 212 | Software. However, You may do so only on Your own behalf, and not on 213 | behalf of any Contributor. You must make it absolutely clear that any 214 | such warranty, support, indemnity, or liability obligation is offered by 215 | You alone, and You hereby agree to indemnify every Contributor for any 216 | liability incurred by such Contributor as a result of warranty, support, 217 | indemnity or liability terms You offer. You may include additional 218 | disclaimers of warranty and limitations of liability specific to any 219 | jurisdiction. 220 | 221 | 4. Inability to Comply Due to Statute or Regulation 222 | 223 | --- 224 | 225 | If it is impossible for You to comply with any of the terms of this 226 | License with respect to some or all of the Covered Software due to 227 | statute, judicial order, or regulation then You must: (a) comply with 228 | the terms of this License to the maximum extent possible; and (b) 229 | describe the limitations and the code they affect. Such description must 230 | be placed in a text file included with all distributions of the Covered 231 | Software under this License. Except to the extent prohibited by statute 232 | or regulation, such description must be sufficiently detailed for a 233 | recipient of ordinary skill to be able to understand it. 234 | 235 | 5. Termination 236 | 237 | --- 238 | 239 | 5.1. The rights granted under this License will terminate automatically 240 | if You fail to comply with any of its terms. However, if You become 241 | compliant, then the rights granted under this License from a particular 242 | Contributor are reinstated (a) provisionally, unless and until such 243 | Contributor explicitly and finally terminates Your grants, and (b) on an 244 | ongoing basis, if such Contributor fails to notify You of the 245 | non-compliance by some reasonable means prior to 60 days after You have 246 | come back into compliance. Moreover, Your grants from a particular 247 | Contributor are reinstated on an ongoing basis if such Contributor 248 | notifies You of the non-compliance by some reasonable means, this is the 249 | first time You have received notice of non-compliance with this License 250 | from such Contributor, and You become compliant prior to 30 days after 251 | Your receipt of the notice. 252 | 253 | 5.2. If You initiate litigation against any entity by asserting a patent 254 | infringement claim (excluding declaratory judgment actions, 255 | counter-claims, and cross-claims) alleging that a Contributor Version 256 | directly or indirectly infringes any patent, then the rights granted to 257 | You by any and all Contributors for the Covered Software under Section 258 | 2.1 of this License shall terminate. 259 | 260 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 261 | end user license agreements (excluding distributors and resellers) which 262 | have been validly granted by You or Your distributors under this License 263 | prior to termination shall survive termination. 264 | 265 | --- 266 | 267 | - * 268 | - 6. Disclaimer of Warranty \* 269 | - ------------------------- \* 270 | - * 271 | - Covered Software is provided under this License on an "as is" \* 272 | - basis, without warranty of any kind, either expressed, implied, or \* 273 | - statutory, including, without limitation, warranties that the \* 274 | - Covered Software is free of defects, merchantable, fit for a \* 275 | - particular purpose or non-infringing. The entire risk as to the \* 276 | - quality and performance of the Covered Software is with You. \* 277 | - Should any Covered Software prove defective in any respect, You \* 278 | - (not any Contributor) assume the cost of any necessary servicing, \* 279 | - repair, or correction. This disclaimer of warranty constitutes an \* 280 | - essential part of this License. No use of any Covered Software is \* 281 | - authorized under this License except under this disclaimer. \* 282 | - * 283 | 284 | --- 285 | 286 | --- 287 | 288 | - * 289 | - 7. Limitation of Liability \* 290 | - -------------------------- \* 291 | - * 292 | - Under no circumstances and under no legal theory, whether tort \* 293 | - (including negligence), contract, or otherwise, shall any \* 294 | - Contributor, or anyone who distributes Covered Software as \* 295 | - permitted above, be liable to You for any direct, indirect, \* 296 | - special, incidental, or consequential damages of any character \* 297 | - including, without limitation, damages for lost profits, loss of \* 298 | - goodwill, work stoppage, computer failure or malfunction, or any \* 299 | - and all other commercial damages or losses, even if such party \* 300 | - shall have been informed of the possibility of such damages. This \* 301 | - limitation of liability shall not apply to liability for death or \* 302 | - personal injury resulting from such party's negligence to the \* 303 | - extent applicable law prohibits such limitation. Some \* 304 | - jurisdictions do not allow the exclusion or limitation of \* 305 | - incidental or consequential damages, so this exclusion and \* 306 | - limitation may not apply to You. \* 307 | - * 308 | 309 | --- 310 | 311 | 8. Litigation 312 | 313 | --- 314 | 315 | Any litigation relating to this License may be brought only in the 316 | courts of a jurisdiction where the defendant maintains its principal 317 | place of business and such litigation shall be governed by laws of that 318 | jurisdiction, without reference to its conflict-of-law provisions. 319 | Nothing in this Section shall prevent a party's ability to bring 320 | cross-claims or counter-claims. 321 | 322 | 9. Miscellaneous 323 | 324 | --- 325 | 326 | This License represents the complete agreement concerning the subject 327 | matter hereof. If any provision of this License is held to be 328 | unenforceable, such provision shall be reformed only to the extent 329 | necessary to make it enforceable. Any law or regulation which provides 330 | that the language of a contract shall be construed against the drafter 331 | shall not be used to construe this License against a Contributor. 332 | 333 | 10. Versions of the License 334 | 335 | --- 336 | 337 | 10.1. New Versions 338 | 339 | Mozilla Foundation is the license steward. Except as provided in Section 340 | 10.3, no one other than the license steward has the right to modify or 341 | publish new versions of this License. Each version will be given a 342 | distinguishing version number. 343 | 344 | 10.2. Effect of New Versions 345 | 346 | You may distribute the Covered Software under the terms of the version 347 | of the License under which You originally received the Covered Software, 348 | or under the terms of any subsequent version published by the license 349 | steward. 350 | 351 | 10.3. Modified Versions 352 | 353 | If you create software not governed by this License, and you want to 354 | create a new license for such software, you may create and use a 355 | modified version of this License if you rename the license and remove 356 | any references to the name of the license steward (except to note that 357 | such modified license differs from this License). 358 | 359 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 360 | Licenses 361 | 362 | If You choose to distribute Source Code Form that is Incompatible With 363 | Secondary Licenses under the terms of this version of the License, the 364 | notice described in Exhibit B of this License must be attached. 365 | 366 | ## Exhibit A - Source Code Form License Notice 367 | 368 | This Source Code Form is subject to the terms of the Mozilla Public 369 | License, v. 2.0. If a copy of the MPL was not distributed with this 370 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 371 | 372 | If it is not possible or desirable to put the notice in a particular 373 | file, then You may include the notice in a location (such as a LICENSE 374 | file in a relevant directory) where a recipient would be likely to look 375 | for such a notice. 376 | 377 | You may add additional accurate notices of copyright ownership. 378 | 379 | ## Exhibit B - "Incompatible With Secondary Licenses" Notice 380 | 381 | This Source Code Form is "Incompatible With Secondary Licenses", as 382 | defined by the Mozilla Public License, v. 2.0. 383 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Geth-data-layer 2 | 3 | Go library to access the [`geth`](https://github.com/ethereum/go-ethereum) stored data. 4 | 5 | ## Usage 6 | 7 | ```go 8 | package main 9 | 10 | import ( 11 | "fmt" 12 | 13 | gethdatalayer "github.com/umbracle/geth-data-layer" 14 | ) 15 | 16 | func main() { 17 | path := "..../chaindata" // path to the storage data 18 | 19 | store, err := gethdatalayer.NewStore(path) 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | iter := store.Iterator() 25 | // iter.Seek(1000000) 26 | 27 | for iter.Next() { 28 | val, _ := iter.Value() 29 | fmt.Println(val.Number) 30 | } 31 | } 32 | ``` 33 | 34 | There are three storage interaces: 35 | 36 | - `NewAncientStore`: Access the `ancient` store data. 37 | - `NewLevelDbStore`: Access the `leveldb` store data. 38 | - `NewStore`: Abstraction on top of the `leveldb` and `ancient` data. 39 | -------------------------------------------------------------------------------- /fixtures/transactions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "raw": "b902bd02f902b9010b843b9aca0085047074344c8302114e9468b3465833fb72a70ecdf485e0e4c7bd8665fc458737d2ba67af24bab90244ac9650d800000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000001a45ae401dc000000000000000000000000000000000000000000000000000000006322cb2b00000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000e404e45aaf000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000f1b99e3e573a1a9c5e6b2ce818b617f0e664e86b0000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000002159fadfe8ae234c7e155a1a487c76f657d295de0000000000000000000000000000000000000000000000000037d2ba67af24ba00000000000000000000000000000000000000000000000002ced37f210aef3800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c080a0e5b8c0ddbbdd8d22e2c0e4f8bcecd9b2502a8f2f98f1e2fb1be5b5ff9e1bc589a0732b5ff8e65ee18dc30a4254679de6a8710e6982bbc9f594cf75ef8b74d50f6c", 4 | "hash": "716a3d37d9bc8fdde74a0d196fdbedd97b7fedb4af0c6844bc0a1b5a4c49cf5f" 5 | }, 6 | { 7 | "raw": "b89b02f898017d84540ae4808523d923d77983016795944dbd4fc535ac27206064b68ffcf827b0a60bab3f88016345785d8a0000a40f4d14e9000000000000000000000000000000000000000000000000000000a7fa8e9daec001a0958e492a16870cf805abaa1320da1a0b6272f3ce7e516168f5389dc798467725a06625cac11fffa553e82dc60d42fc0c3ab2c5c503517bba0f6e8f5cd340a33f04", 8 | "hash": "cb9d5443bd1956d63a28c53a2a15d5b97dc08d765a06c866f66847f233a08277" 9 | } 10 | ] -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/umbracle/geth-data-layer 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/golang/snappy v0.0.4 7 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 8 | github.com/umbracle/fastrlp v0.0.0-20220705090633-9adaa99b7668 9 | ) 10 | 11 | require ( 12 | github.com/google/gofuzz v1.2.0 // indirect 13 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect 14 | golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect 15 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 16 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect 17 | gopkg.in/yaml.v2 v2.4.0 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 2 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 3 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 4 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 5 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 6 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 7 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 8 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 9 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 10 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 11 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 12 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 13 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 14 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 15 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 16 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 17 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 18 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 19 | github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= 20 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 21 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 22 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 23 | github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= 24 | github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 25 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 26 | github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= 27 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 28 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= 29 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= 30 | github.com/umbracle/fastrlp v0.0.0-20220705090633-9adaa99b7668 h1:1+HhIsmtvkxxiNkvsPFSp/usy5DEB72qjc1MJ0vwYNw= 31 | github.com/umbracle/fastrlp v0.0.0-20220705090633-9adaa99b7668/go.mod h1:5RHgqiFjd4vLJESMWagP/E7su+5Gzk0iqqmrotR8WdA= 32 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 33 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 34 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= 35 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 36 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 37 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 38 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 39 | golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 40 | golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= 41 | golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 42 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 43 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 44 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 45 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 46 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 47 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 48 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 49 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 50 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 51 | golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 52 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= 53 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 54 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 55 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 56 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 57 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 58 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 59 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 60 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 61 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= 62 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= 63 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 64 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 65 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 66 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 67 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 68 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 69 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 70 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 71 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 72 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 73 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 74 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 75 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 76 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 77 | -------------------------------------------------------------------------------- /iterator.go: -------------------------------------------------------------------------------- 1 | package gethdatalayer 2 | 3 | type Iterator interface { 4 | Seek(num uint64) 5 | Next() bool 6 | Value() (*Block, error) 7 | } 8 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package gethdatalayer 2 | 3 | import ( 4 | "path/filepath" 5 | ) 6 | 7 | type Store struct { 8 | leveldbStore *LevelDbStore 9 | ancientStore *AncientStore 10 | } 11 | 12 | func NewStore(path string) (*Store, error) { 13 | // load the kv store 14 | leveldbStore, err := NewLevelDBStore(path) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | // load the ancient store 20 | ancientStore, err := NewAncientStore(filepath.Join(path, "ancient/chain")) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | // figure out if there is a stream between the last block we can 26 | // read from ancient store and the leveldb 27 | lastAncientBlock := ancientStore.LastNum() 28 | 29 | if _, err := decodeBlock(leveldbStore, lastAncientBlock+1); err != nil { 30 | return nil, err 31 | } 32 | 33 | s := &Store{ 34 | leveldbStore: leveldbStore, 35 | ancientStore: ancientStore, 36 | } 37 | return s, nil 38 | } 39 | 40 | func (s *Store) Iterator() Iterator { 41 | iter := &storeIterator{ 42 | store: s, 43 | lastAncientNum: s.ancientStore.LastNum(), 44 | } 45 | return iter 46 | } 47 | 48 | type storeIterator struct { 49 | store *Store 50 | num uint64 51 | lastAncientNum uint64 52 | iterLevelDb Iterator 53 | iterAncient Iterator 54 | } 55 | 56 | func (s *storeIterator) Seek(num uint64) { 57 | s.num = num 58 | } 59 | 60 | func (s *storeIterator) Next() bool { 61 | if s.num+1 >= s.lastAncientNum { 62 | // use the leveldb store 63 | if s.iterLevelDb == nil { 64 | s.iterLevelDb = s.store.leveldbStore.Iterator() 65 | s.iterLevelDb.Seek(s.num) 66 | } 67 | if s.iterAncient != nil { 68 | // reset the ancient iterator (TODO: Close it) 69 | s.iterAncient = nil 70 | } 71 | s.num++ 72 | return s.iterLevelDb.Next() 73 | } 74 | 75 | // use the ancient store 76 | if s.iterAncient == nil { 77 | s.iterAncient = s.store.ancientStore.Iterator() 78 | s.iterAncient.Seek(s.num) 79 | } 80 | s.num++ 81 | return s.iterAncient.Next() 82 | } 83 | 84 | func (s *storeIterator) Value() (*Block, error) { 85 | if s.iterAncient != nil { 86 | return s.iterAncient.Value() 87 | } 88 | return s.iterLevelDb.Value() 89 | } 90 | -------------------------------------------------------------------------------- /store_ancient.go: -------------------------------------------------------------------------------- 1 | package gethdatalayer 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/golang/snappy" 10 | ) 11 | 12 | var _ Iterator = &ancientIterator{} 13 | 14 | type AncientStore struct { 15 | receipts *ancientTable 16 | headers *ancientTable 17 | bodies *ancientTable 18 | } 19 | 20 | func NewAncientStore(path string) (*AncientStore, error) { 21 | receiptsTable, err := newAncientTable(path, "receipts") 22 | if err != nil { 23 | return nil, err 24 | } 25 | headerTable, err := newAncientTable(path, "headers") 26 | if err != nil { 27 | return nil, err 28 | } 29 | bodiesTable, err := newAncientTable(path, "bodies") 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | // all the tables should have the same number of items 35 | if headerTable.numItems != bodiesTable.numItems { 36 | return nil, fmt.Errorf("header and bodies table do not have same num of items") 37 | } 38 | if headerTable.numItems != receiptsTable.numItems { 39 | return nil, fmt.Errorf("header and receipts table do not have same num of items") 40 | } 41 | 42 | store := &AncientStore{ 43 | receipts: receiptsTable, 44 | headers: headerTable, 45 | bodies: bodiesTable, 46 | } 47 | return store, nil 48 | } 49 | 50 | func (a *AncientStore) LastNum() uint64 { 51 | return a.headers.numItems 52 | } 53 | 54 | func (a *AncientStore) Iterator() Iterator { 55 | iter := &ancientIterator{ 56 | rIter: a.receipts.Iter(), 57 | hIter: a.headers.Iter(), 58 | bIter: a.bodies.Iter(), 59 | } 60 | return iter 61 | } 62 | 63 | type ancientIterator struct { 64 | rIter *ancientTableIterator 65 | hIter *ancientTableIterator 66 | bIter *ancientTableIterator 67 | } 68 | 69 | func (i *ancientIterator) Seek(num uint64) { 70 | i.rIter.Seek(num) 71 | i.hIter.Seek(num) 72 | i.bIter.Seek(num) 73 | } 74 | 75 | func (i *ancientIterator) Next() bool { 76 | next := i.rIter.Next() 77 | i.hIter.Next() 78 | i.bIter.Next() 79 | 80 | return next 81 | } 82 | 83 | func (i *ancientIterator) Value() (*Block, error) { 84 | receipts := Receipts{} 85 | if err := i.rIter.Value(&receipts); err != nil { 86 | return nil, err 87 | } 88 | header := Header{} 89 | if err := i.hIter.Value(&header); err != nil { 90 | return nil, err 91 | } 92 | body := Body{} 93 | if err := i.bIter.Value(&body); err != nil { 94 | return nil, err 95 | } 96 | if len(body.Transactions) != len(receipts) { 97 | return nil, fmt.Errorf("incorrect match") 98 | } 99 | 100 | block := &Block{ 101 | Number: header.Number, 102 | Header: &header, 103 | Body: &body, 104 | Receipts: receipts, 105 | } 106 | return block, nil 107 | } 108 | 109 | type ancientTable struct { 110 | path string 111 | name string 112 | compressed bool 113 | 114 | // store the file of index and offsets 115 | index *os.File 116 | 117 | // data files 118 | data map[uint16]*os.File 119 | 120 | // number of items in the table 121 | numItems uint64 122 | } 123 | 124 | func newAncientTable(path, name string) (*ancientTable, error) { 125 | t := &ancientTable{ 126 | path: path, 127 | name: name, 128 | data: map[uint16]*os.File{}, 129 | } 130 | err := t.checkIndex() 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | // open index file 136 | if t.index, err = os.Open(t.getIndexName(t.compressed)); err != nil { 137 | return nil, err 138 | } 139 | 140 | stat, err := t.index.Stat() 141 | if err != nil { 142 | return nil, err 143 | } 144 | 145 | t.numItems = uint64(stat.Size() / indexEntrySize) 146 | 147 | // preopen all the data files 148 | if err := t.openDataFiles(); err != nil { 149 | return nil, err 150 | } 151 | return t, nil 152 | } 153 | 154 | func (a *ancientTable) readTable(fileNum uint16, from uint32, size uint32) []byte { 155 | buf := make([]byte, size) 156 | _, err := a.data[fileNum].ReadAt(buf, int64(from)) 157 | if err != nil { 158 | panic(err) 159 | } 160 | 161 | if buf, err = snappy.Decode(nil, buf); err != nil { 162 | panic(err) 163 | } 164 | return buf 165 | } 166 | 167 | func (a *ancientTable) checkIndex() error { 168 | hasCompr, err := exists(a.getIndexName(true)) 169 | if err != nil { 170 | return err 171 | } 172 | hasNormal, err := exists(a.getIndexName(false)) 173 | if err != nil { 174 | return err 175 | } 176 | if !hasCompr && !hasNormal { 177 | return fmt.Errorf("table not found") 178 | } 179 | if hasCompr && hasNormal { 180 | return fmt.Errorf("both compress and uncompress index found") 181 | } 182 | a.compressed = hasCompr 183 | return nil 184 | } 185 | 186 | func (a *ancientTable) getDataName(indx uint16, compressed bool) string { 187 | ext := "" 188 | if compressed { 189 | ext = "cdat" 190 | } else { 191 | ext = "rdat" 192 | } 193 | return filepath.Join(a.path, fmt.Sprintf("%s.%04d.%s", a.name, indx, ext)) 194 | } 195 | 196 | func (a *ancientTable) getIndexName(compressed bool) string { 197 | ext := "" 198 | if compressed { 199 | ext = "cidx" 200 | } else { 201 | ext = "ridx" 202 | } 203 | return filepath.Join(a.path, a.name+"."+ext) 204 | } 205 | 206 | func (a *ancientTable) openDataFiles() error { 207 | stat, err := a.index.Stat() 208 | if err != nil { 209 | return err 210 | } 211 | 212 | var firstEntry, lastEntry indexEntry 213 | buf := make([]byte, indexEntrySize) 214 | 215 | readAt := func(entry *indexEntry, pos int64) error { 216 | if _, err := a.index.ReadAt(buf, pos); err != nil { 217 | return err 218 | } 219 | entry.Unmarshal(buf) 220 | return nil 221 | } 222 | 223 | // read the first entry 224 | if err := readAt(&firstEntry, 0); err != nil { 225 | return err 226 | } 227 | // read last entry 228 | if err := readAt(&lastEntry, stat.Size()-indexEntrySize); err != nil { 229 | return err 230 | } 231 | 232 | // open the files 233 | for i := firstEntry.FileNum; i <= lastEntry.FileNum; i++ { 234 | f, err := os.Open(a.getDataName(i, a.compressed)) 235 | if err != nil { 236 | return err 237 | } 238 | a.data[i] = f 239 | } 240 | return nil 241 | } 242 | 243 | func (a *ancientTable) Iter() *ancientTableIterator { 244 | i := &ancientTableIterator{ 245 | table: a, 246 | indexFile: a.index, 247 | } 248 | i.Seek(0) 249 | return i 250 | } 251 | 252 | type ancientTableIterator struct { 253 | table *ancientTable 254 | indexFile *os.File 255 | ptr indexEntry 256 | val []byte 257 | } 258 | 259 | func (i *ancientTableIterator) readEntry() indexEntry { 260 | buf := make([]byte, indexEntrySize) 261 | 262 | if _, err := i.indexFile.Read(buf); err != nil { 263 | panic(err) 264 | } 265 | 266 | var entry indexEntry 267 | entry.Unmarshal(buf) 268 | 269 | return entry 270 | } 271 | 272 | func (i *ancientTableIterator) Seek(num uint64) { 273 | i.indexFile.Seek(int64(num)*indexEntrySize, 0) 274 | i.ptr = i.readEntry() 275 | } 276 | 277 | func (i *ancientTableIterator) Next() bool { 278 | // read next entry 279 | next := i.readEntry() 280 | 281 | if i.ptr.FileNum != next.FileNum { 282 | // start from the next item 283 | i.val = i.table.readTable(next.FileNum, 0, next.Offset) 284 | } else { 285 | // follow the sequence 286 | i.val = i.table.readTable(i.ptr.FileNum, i.ptr.Offset, next.Offset-i.ptr.Offset) 287 | } 288 | 289 | i.ptr = next 290 | return true 291 | } 292 | 293 | type rlpObj interface { 294 | UnmarshalRLP(v []byte) error 295 | } 296 | 297 | func (i *ancientTableIterator) Value(obj rlpObj) error { 298 | return obj.UnmarshalRLP(i.val) 299 | } 300 | 301 | const indexEntrySize = int64(6) 302 | 303 | type indexEntry struct { 304 | FileNum uint16 // 2 bytes 305 | Offset uint32 // 4 bytes 306 | } 307 | 308 | func (i *indexEntry) Unmarshal(b []byte) { 309 | i.FileNum = binary.BigEndian.Uint16(b[:2]) 310 | i.Offset = binary.BigEndian.Uint32(b[2:]) 311 | } 312 | -------------------------------------------------------------------------------- /store_leveldb.go: -------------------------------------------------------------------------------- 1 | package gethdatalayer 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | 7 | "github.com/syndtr/goleveldb/leveldb" 8 | "github.com/syndtr/goleveldb/leveldb/opt" 9 | ) 10 | 11 | type LevelDbStore struct { 12 | db *leveldb.DB 13 | } 14 | 15 | func NewLevelDBStore(path string) (*LevelDbStore, error) { 16 | db, err := leveldb.OpenFile(path, &opt.Options{ReadOnly: true}) 17 | if err != nil { 18 | return nil, fmt.Errorf("failed to load leveldb: %v", err) 19 | } 20 | store := &LevelDbStore{ 21 | db: db, 22 | } 23 | return store, nil 24 | } 25 | 26 | func (l *LevelDbStore) Iterator() Iterator { 27 | iter := &levelDbIterator{ 28 | db: l, 29 | } 30 | iter.Seek(0) 31 | return iter 32 | } 33 | 34 | type levelDbIterator struct { 35 | db *LevelDbStore 36 | num uint64 37 | block *Block 38 | } 39 | 40 | type kvDb interface { 41 | Get([]byte) ([]byte, error) 42 | } 43 | 44 | func decodeBlock(db kvDb, num uint64) (*Block, error) { 45 | // find the canonical chain for 'num' to resolve 46 | // the hash 47 | hashB, err := db.Get(headerHashKey(num)) 48 | if err != nil { 49 | return nil, err 50 | } 51 | if len(hashB) != 32 { 52 | return nil, fmt.Errorf("incorrect hash length: %d", len(hashB)) 53 | } 54 | 55 | // header 56 | headerRaw, err := db.Get(headerKey(num, hashB)) 57 | if err != nil { 58 | return nil, err 59 | } 60 | header := new(Header) 61 | if err := header.UnmarshalRLP(headerRaw); err != nil { 62 | return nil, fmt.Errorf("failed to decode header: %v", err) 63 | } 64 | 65 | // body 66 | bodyRaw, err := db.Get(blockBodyKey(num, hashB)) 67 | if err != nil { 68 | return nil, err 69 | } 70 | body := new(Body) 71 | if err := body.UnmarshalRLP(bodyRaw); err != nil { 72 | return nil, fmt.Errorf("failed to decode body: %v", err) 73 | } 74 | 75 | // receipts 76 | receiptsRaw, err := db.Get(blockReceiptsKey(num, hashB)) 77 | if err != nil { 78 | return nil, err 79 | } 80 | receipts := new(Receipts) 81 | if err := receipts.UnmarshalRLP(receiptsRaw); err != nil { 82 | return nil, fmt.Errorf("failed to decode receipts: %v", err) 83 | } 84 | 85 | if len(body.Transactions) != len(*receipts) { 86 | return nil, fmt.Errorf("incorrect match") 87 | } 88 | 89 | resp := &Block{ 90 | Number: num, 91 | Header: header, 92 | Body: body, 93 | Receipts: *receipts, 94 | } 95 | return resp, nil 96 | } 97 | 98 | func (l *LevelDbStore) Get(k []byte) ([]byte, error) { 99 | return l.db.Get(k, nil) 100 | } 101 | 102 | func (l *levelDbIterator) Seek(num uint64) { 103 | l.num = num 104 | } 105 | 106 | func (l *levelDbIterator) Next() bool { 107 | block, err := decodeBlock(l.db, l.num) 108 | if err != nil { 109 | return false 110 | } 111 | 112 | l.block = block 113 | l.num++ 114 | 115 | return true 116 | } 117 | 118 | func (l *levelDbIterator) Value() (*Block, error) { 119 | return l.block, nil 120 | } 121 | 122 | var ( 123 | // headerPrefix + num (uint64 big endian) + hash -> header 124 | headerPrefix = []byte("h") 125 | 126 | // blockBodyPrefix + num (uint64 big endian) + hash -> block body 127 | blockBodyPrefix = []byte("b") 128 | 129 | // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts 130 | blockReceiptsPrefix = []byte("r") 131 | 132 | // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash 133 | headerHashSuffix = []byte("n") 134 | 135 | headBlockKey = []byte("LastBlock") 136 | 137 | headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian) 138 | ) 139 | 140 | func marshalUint64(num uint64) []byte { 141 | buf := make([]byte, 8) 142 | binary.BigEndian.PutUint64(buf, num) 143 | return buf 144 | } 145 | 146 | func unmarshalUint64(b []byte) uint64 { 147 | return binary.BigEndian.Uint64(b) 148 | } 149 | 150 | func decodeKey(b []byte) (uint64, []byte) { 151 | return unmarshalUint64(b[1:9]), b[9:] 152 | } 153 | 154 | func headerKey(number uint64, hash []byte) []byte { 155 | return append(append(headerPrefix, marshalUint64(number)...), hash...) 156 | } 157 | 158 | func blockBodyKey(number uint64, hash []byte) []byte { 159 | return append(append(blockBodyPrefix, marshalUint64(number)...), hash[:]...) 160 | } 161 | 162 | func blockReceiptsKey(number uint64, hash []byte) []byte { 163 | return append(append(blockReceiptsPrefix, marshalUint64(number)...), hash[:]...) 164 | } 165 | 166 | func headerHashKey(number uint64) []byte { 167 | return append(append(headerPrefix, marshalUint64(number)...), headerHashSuffix...) 168 | } 169 | 170 | func headerNumberKey(hash []byte) []byte { 171 | return append(headerNumberPrefix, hash...) 172 | } 173 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package gethdatalayer 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/hex" 6 | "errors" 7 | "fmt" 8 | "math/big" 9 | "os" 10 | 11 | "github.com/umbracle/fastrlp" 12 | ) 13 | 14 | type Block struct { 15 | Number uint64 16 | Header *Header 17 | Body *Body 18 | Receipts Receipts 19 | } 20 | 21 | type hash [32]byte 22 | 23 | func (h hash) String() string { 24 | return "0x" + hex.EncodeToString(h[:]) 25 | } 26 | 27 | type address [20]byte 28 | 29 | func (a address) String() string { 30 | return "0x" + hex.EncodeToString(a[:]) 31 | } 32 | 33 | type Receipt struct { 34 | PostStateOrStatus []byte 35 | CumulativeGasUsed uint64 36 | Logs []*Log 37 | } 38 | 39 | type Log struct { 40 | Address address 41 | Topics []hash 42 | Data []byte 43 | } 44 | 45 | type Receipts []*Receipt 46 | 47 | func (r *Receipts) UnmarshalRLP(input []byte) error { 48 | return unmarshalRlp(r.UnmarshalRLPFrom, input) 49 | } 50 | 51 | func (r *Receipts) UnmarshalRLPFrom(p *fastrlp.Parser, v *fastrlp.Value) error { 52 | elems, err := v.GetElems() 53 | if err != nil { 54 | return err 55 | } 56 | for _, elem := range elems { 57 | rr := &Receipt{} 58 | if err := rr.UnmarshalRLPFrom(p, elem); err != nil { 59 | return err 60 | } 61 | (*r) = append(*r, rr) 62 | } 63 | return nil 64 | } 65 | 66 | func (r *Receipt) UnmarshalRLP(input []byte) error { 67 | return unmarshalRlp(r.UnmarshalRLPFrom, input) 68 | } 69 | 70 | // UnmarshalRLP unmarshals a Receipt in RLP format 71 | func (r *Receipt) UnmarshalRLPFrom(p *fastrlp.Parser, v *fastrlp.Value) error { 72 | elems, err := v.GetElems() 73 | if err != nil { 74 | return err 75 | } 76 | if len(elems) != 3 { 77 | return fmt.Errorf("expected 3 elements") 78 | } 79 | 80 | // root or status 81 | buf, err := elems[0].Bytes() 82 | if err != nil { 83 | return err 84 | } 85 | r.PostStateOrStatus = buf 86 | 87 | // cumulativeGasUsed 88 | if r.CumulativeGasUsed, err = elems[1].GetUint64(); err != nil { 89 | return err 90 | } 91 | 92 | // logs 93 | logsElems, err := v.Get(2).GetElems() 94 | if err != nil { 95 | return err 96 | } 97 | for _, elem := range logsElems { 98 | log := &Log{} 99 | if err := log.UnmarshalRLPFrom(p, elem); err != nil { 100 | return err 101 | } 102 | r.Logs = append(r.Logs, log) 103 | } 104 | return nil 105 | } 106 | 107 | func (l *Log) UnmarshalRLPFrom(p *fastrlp.Parser, v *fastrlp.Value) error { 108 | elems, err := v.GetElems() 109 | if err != nil { 110 | return err 111 | } 112 | if len(elems) != 3 { 113 | return fmt.Errorf("bad elems") 114 | } 115 | 116 | // address 117 | if err := elems[0].GetAddr(l.Address[:]); err != nil { 118 | return err 119 | } 120 | // topics 121 | topicElems, err := elems[1].GetElems() 122 | if err != nil { 123 | return err 124 | } 125 | l.Topics = make([]hash, len(topicElems)) 126 | for indx, topic := range topicElems { 127 | if err := topic.GetHash(l.Topics[indx][:]); err != nil { 128 | return err 129 | } 130 | } 131 | // data 132 | if l.Data, err = elems[2].GetBytes(l.Data[:0]); err != nil { 133 | return err 134 | } 135 | return nil 136 | } 137 | 138 | type unmarshalRLPFunc func(p *fastrlp.Parser, v *fastrlp.Value) error 139 | 140 | func unmarshalRlp(obj unmarshalRLPFunc, input []byte) error { 141 | pr := &fastrlp.Parser{} 142 | 143 | v, err := pr.Parse(input) 144 | if err != nil { 145 | return err 146 | } 147 | if err := obj(pr, v); err != nil { 148 | return err 149 | } 150 | return nil 151 | } 152 | 153 | func exists(name string) (bool, error) { 154 | _, err := os.Stat(name) 155 | if err == nil { 156 | return true, nil 157 | } 158 | if errors.Is(err, os.ErrNotExist) { 159 | return false, nil 160 | } 161 | return false, err 162 | } 163 | 164 | type Body struct { 165 | Transactions []*Transaction 166 | Uncles []*Header 167 | } 168 | 169 | func (b *Body) UnmarshalRLP(input []byte) error { 170 | return unmarshalRlp(b.UnmarshalRLPFrom, input) 171 | } 172 | 173 | func (b *Body) UnmarshalRLPFrom(p *fastrlp.Parser, v *fastrlp.Value) error { 174 | tuple, err := v.GetElems() 175 | if err != nil { 176 | return err 177 | } 178 | if len(tuple) != 2 { 179 | return fmt.Errorf("not enough elements to decode header, expected 15 but found %d", len(tuple)) 180 | } 181 | 182 | // transactions 183 | txns, err := tuple[0].GetElems() 184 | if err != nil { 185 | return err 186 | } 187 | for _, txn := range txns { 188 | bTxn := &Transaction{} 189 | if err := bTxn.UnmarshalRLPFrom(p, txn); err != nil { 190 | return err 191 | } 192 | b.Transactions = append(b.Transactions, bTxn) 193 | } 194 | 195 | // uncles 196 | uncles, err := tuple[1].GetElems() 197 | if err != nil { 198 | return err 199 | } 200 | for _, uncle := range uncles { 201 | bUncle := &Header{} 202 | if err := bUncle.UnmarshalRLPFrom(p, uncle); err != nil { 203 | return err 204 | } 205 | b.Uncles = append(b.Uncles, bUncle) 206 | } 207 | 208 | return nil 209 | } 210 | 211 | type TransactionType int 212 | 213 | const ( 214 | TransactionLegacy TransactionType = 0 215 | // eip-2930 216 | TransactionAccessList TransactionType = 1 217 | // eip-1559 218 | TransactionDynamicFee TransactionType = 2 219 | ) 220 | 221 | type Transaction struct { 222 | Type TransactionType 223 | 224 | // legacy values 225 | Hash hash 226 | From address 227 | To *address 228 | Input []byte 229 | GasPrice uint64 230 | Gas uint64 231 | Value *big.Int 232 | Nonce uint64 233 | V []byte 234 | R []byte 235 | S []byte 236 | 237 | // eip-2930 values 238 | ChainID *big.Int 239 | AccessList AccessList 240 | 241 | // eip-1559 values 242 | MaxPriorityFeePerGas *big.Int 243 | MaxFeePerGas *big.Int 244 | } 245 | 246 | type AccessEntry struct { 247 | Address address 248 | Storage []hash 249 | } 250 | 251 | type AccessList []AccessEntry 252 | 253 | func (t *Transaction) UnmarshalRLP(buf []byte) error { 254 | return unmarshalRlp(t.UnmarshalRLPFrom, buf) 255 | } 256 | 257 | func (t *Transaction) UnmarshalRLPFrom(p *fastrlp.Parser, v *fastrlp.Value) error { 258 | keccak := fastrlp.NewKeccak256() 259 | 260 | if v.Type() == fastrlp.TypeBytes { 261 | // typed transaction 262 | buf, err := v.Bytes() 263 | if err != nil { 264 | return err 265 | } 266 | 267 | switch typ := buf[0]; typ { 268 | case 1: 269 | t.Type = TransactionAccessList 270 | case 2: 271 | t.Type = TransactionDynamicFee 272 | default: 273 | return fmt.Errorf("type byte %d not found", typ) 274 | } 275 | buf = buf[1:] 276 | 277 | pp := fastrlp.Parser{} 278 | if v, err = pp.Parse(buf); err != nil { 279 | return err 280 | } 281 | 282 | keccak.Write([]byte{byte(t.Type)}) 283 | keccak.Write(pp.Raw(v)) 284 | } else { 285 | keccak.Write(p.Raw(v)) 286 | } 287 | 288 | keccak.Sum(t.Hash[:0]) 289 | 290 | elems, err := v.GetElems() 291 | if err != nil { 292 | return err 293 | } 294 | 295 | getElem := func() *fastrlp.Value { 296 | v := elems[0] 297 | elems = elems[1:] 298 | return v 299 | } 300 | 301 | var num int 302 | switch t.Type { 303 | case TransactionLegacy: 304 | num = 9 305 | case TransactionAccessList: 306 | // legacy + chain id + access list 307 | num = 11 308 | case TransactionDynamicFee: 309 | // access list txn + gas fee 1 + gas fee 2 - gas price 310 | num = 12 311 | default: 312 | return fmt.Errorf("transaction type %d not found", t.Type) 313 | } 314 | if numElems := len(elems); numElems != num { 315 | return fmt.Errorf("not enough elements to decode transaction, expected %d but found %d", num, numElems) 316 | } 317 | 318 | if t.Type != 0 { 319 | t.ChainID = new(big.Int) 320 | if err := getElem().GetBigInt(t.ChainID); err != nil { 321 | return err 322 | } 323 | } 324 | 325 | // nonce 326 | if t.Nonce, err = getElem().GetUint64(); err != nil { 327 | return err 328 | } 329 | 330 | if t.Type == TransactionDynamicFee { 331 | // dynamic fee uses 332 | t.MaxPriorityFeePerGas = new(big.Int) 333 | if err := getElem().GetBigInt(t.MaxPriorityFeePerGas); err != nil { 334 | return err 335 | } 336 | t.MaxFeePerGas = new(big.Int) 337 | if err := getElem().GetBigInt(t.MaxFeePerGas); err != nil { 338 | return err 339 | } 340 | } else { 341 | // legacy and access type use gas price 342 | if t.GasPrice, err = getElem().GetUint64(); err != nil { 343 | return err 344 | } 345 | } 346 | 347 | // gas 348 | if t.Gas, err = getElem().GetUint64(); err != nil { 349 | return err 350 | } 351 | // to 352 | vv, _ := getElem().Bytes() 353 | if len(vv) == 20 { 354 | // address 355 | var addr address 356 | copy(addr[:], vv) 357 | t.To = &addr 358 | } else { 359 | // reset To 360 | t.To = nil 361 | } 362 | // value 363 | t.Value = new(big.Int) 364 | if err := getElem().GetBigInt(t.Value); err != nil { 365 | return err 366 | } 367 | // input 368 | if t.Input, err = getElem().GetBytes(t.Input[:0]); err != nil { 369 | return err 370 | } 371 | 372 | if t.Type != 0 { 373 | if err := t.AccessList.UnmarshalRLPWith(getElem()); err != nil { 374 | return err 375 | } 376 | } 377 | 378 | // V 379 | if t.V, err = getElem().GetBytes(t.V); err != nil { 380 | return err 381 | } 382 | // R 383 | if t.R, err = getElem().GetBytes(t.R); err != nil { 384 | return err 385 | } 386 | // S 387 | if t.S, err = getElem().GetBytes(t.S); err != nil { 388 | return err 389 | } 390 | 391 | return nil 392 | } 393 | 394 | func (a *AccessList) UnmarshalRLPWith(v *fastrlp.Value) error { 395 | if v.Type() == fastrlp.TypeArrayNull { 396 | // empty 397 | return nil 398 | } 399 | 400 | elems, err := v.GetElems() 401 | if err != nil { 402 | return err 403 | } 404 | for _, elem := range elems { 405 | entry := AccessEntry{} 406 | 407 | acctElems, err := elem.GetElems() 408 | if err != nil { 409 | return err 410 | } 411 | if len(acctElems) != 2 { 412 | return fmt.Errorf("two elems expected but %d found", len(acctElems)) 413 | } 414 | 415 | // decode 'address' 416 | if err = acctElems[0].GetAddr(entry.Address[:]); err != nil { 417 | return err 418 | } 419 | 420 | // decode 'storage' 421 | if acctElems[1].Type() != fastrlp.TypeArrayNull { 422 | storageElems, err := acctElems[1].GetElems() 423 | if err != nil { 424 | return err 425 | } 426 | 427 | entry.Storage = make([]hash, len(storageElems)) 428 | for indx, storage := range storageElems { 429 | // decode storage 430 | if err = storage.GetHash(entry.Storage[indx][:]); err != nil { 431 | return err 432 | } 433 | } 434 | } 435 | (*a) = append((*a), entry) 436 | } 437 | return nil 438 | } 439 | 440 | type Header struct { 441 | Hash hash 442 | ParentHash hash 443 | Sha3Uncles hash 444 | Miner address 445 | StateRoot hash 446 | TxRoot hash 447 | ReceiptsRoot hash 448 | LogsBloom [32]byte 449 | Difficulty uint64 450 | Number uint64 451 | GasLimit uint64 452 | GasUsed uint64 453 | Timestamp uint64 454 | ExtraData []byte 455 | MixHash hash 456 | Nonce [8]byte 457 | BaseFee *big.Int 458 | } 459 | 460 | func (h *Header) UnmarshalRLP(input []byte) error { 461 | return unmarshalRlp(h.UnmarshalRLPFrom, input) 462 | } 463 | 464 | func (h *Header) UnmarshalRLPFrom(p *fastrlp.Parser, v *fastrlp.Value) error { 465 | elems, err := v.GetElems() 466 | if err != nil { 467 | return err 468 | } 469 | num := len(elems) 470 | if num != 15 && num != 16 { 471 | return fmt.Errorf("not enough elements to decode header, expected 15 or 16 but found %d", num) 472 | } 473 | 474 | p.Hash(h.Hash[:0], v) 475 | 476 | // parentHash 477 | if err = elems[0].GetHash(h.ParentHash[:]); err != nil { 478 | return err 479 | } 480 | // sha3uncles 481 | if err = elems[1].GetHash(h.Sha3Uncles[:]); err != nil { 482 | return err 483 | } 484 | // miner 485 | if err = elems[2].GetAddr(h.Miner[:]); err != nil { 486 | return err 487 | } 488 | // stateroot 489 | if err = elems[3].GetHash(h.StateRoot[:]); err != nil { 490 | return err 491 | } 492 | // txroot 493 | if err = elems[4].GetHash(h.TxRoot[:]); err != nil { 494 | return err 495 | } 496 | // receiptroot 497 | if err = elems[5].GetHash(h.ReceiptsRoot[:]); err != nil { 498 | return err 499 | } 500 | // logsBloom 501 | if _, err = elems[6].GetBytes(h.LogsBloom[:0], 256); err != nil { 502 | return err 503 | } 504 | // difficulty 505 | if h.Difficulty, err = elems[7].GetUint64(); err != nil { 506 | return err 507 | } 508 | // number 509 | if h.Number, err = elems[8].GetUint64(); err != nil { 510 | return err 511 | } 512 | // gasLimit 513 | if h.GasLimit, err = elems[9].GetUint64(); err != nil { 514 | return err 515 | } 516 | // gasused 517 | if h.GasUsed, err = elems[10].GetUint64(); err != nil { 518 | return err 519 | } 520 | // timestamp 521 | if h.Timestamp, err = elems[11].GetUint64(); err != nil { 522 | return err 523 | } 524 | // extraData 525 | if h.ExtraData, err = elems[12].GetBytes(h.ExtraData[:0]); err != nil { 526 | return err 527 | } 528 | // mixHash 529 | if err = elems[13].GetHash(h.MixHash[:0]); err != nil { 530 | return err 531 | } 532 | // nonce 533 | nonce, err := elems[14].GetUint64() 534 | if err != nil { 535 | return err 536 | } 537 | binary.BigEndian.PutUint64(h.Nonce[:], nonce) 538 | 539 | if num == 16 { 540 | // base fee 541 | h.BaseFee = new(big.Int) 542 | if err := elems[15].GetBigInt(h.BaseFee); err != nil { 543 | return err 544 | } 545 | } 546 | 547 | return err 548 | } 549 | -------------------------------------------------------------------------------- /types_test.go: -------------------------------------------------------------------------------- 1 | package gethdatalayer 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "encoding/hex" 7 | "encoding/json" 8 | "testing" 9 | ) 10 | 11 | //go:embed fixtures/transactions.json 12 | var transactionsFixtures string 13 | 14 | func TestTypesTxn(t *testing.T) { 15 | var cases []struct { 16 | Raw string 17 | Hash string 18 | } 19 | if err := json.Unmarshal([]byte(transactionsFixtures), &cases); err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | for _, c := range cases { 24 | rlpHex, _ := hex.DecodeString(c.Raw) 25 | hashHex, _ := hex.DecodeString(c.Hash) 26 | 27 | var txn Transaction 28 | if err := txn.UnmarshalRLP(rlpHex); err != nil { 29 | t.Fatal(err) 30 | } 31 | 32 | if !bytes.Equal(hashHex, txn.Hash[:]) { 33 | t.Fatal("not equal") 34 | } 35 | } 36 | } 37 | --------------------------------------------------------------------------------