├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.txt ├── Makefile ├── README.md ├── cmd ├── README.md ├── check_construction.go ├── check_data.go ├── check_perf.go ├── check_spec.go ├── check_spec_utils.go ├── configuration_create.go ├── configuration_validate.go ├── key_gen.go ├── key_sign.go ├── key_verify.go ├── root.go ├── utils_asserter_configuration.go ├── utils_asserter_configuration_test.go ├── utils_shared.go ├── utils_train_zstd.go ├── validate_asserter_config.go ├── validate_asserter_config_test.go ├── view_balance.go ├── view_block.go └── view_networks.go ├── configuration ├── configuration.go ├── configuration_test.go ├── testdata │ └── test.ros └── types.go ├── examples ├── bootstrap_balances.json ├── configuration │ ├── default.json │ ├── sign.json │ ├── simple.json │ └── verify.json ├── exempt_accounts.json └── interesting_accounts.json ├── go.mod ├── go.sum ├── main.go ├── pkg ├── errors │ └── errors.go ├── logger │ └── logger.go ├── processor │ ├── balance_storage_handler.go │ ├── balance_storage_helper.go │ ├── balance_storage_helper_test.go │ ├── broadcast_storage_handler.go │ ├── broadcast_storage_helper.go │ ├── coin_storage_helper.go │ ├── coordinator_handler.go │ ├── coordinator_helper.go │ ├── reconciler_handler.go │ └── reconciler_helper.go ├── results │ ├── construction_results.go │ ├── data_results.go │ ├── data_results_test.go │ ├── perf_results.go │ ├── types.go │ ├── utils.go │ └── utils_test.go └── tester │ ├── benchmark_utils.go │ ├── construction.go │ ├── data.go │ ├── data_perf.go │ └── general.go └── scripts ├── compile.sh └── install.sh /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 12 | 13 | **To Reproduce** 14 | 15 | 16 | **Expected behavior** 17 | 18 | 19 | **Additional context** 20 | 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'enhancement' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | 12 | 13 | **Describe the solution you'd like** 14 | 15 | 16 | **Describe alternatives you've considered** 17 | 18 | 19 | **Additional context** 20 | 21 | 22 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes # . 2 | 3 | ### Motivation 4 | 7 | 8 | ### Solution 9 | 12 | 13 | ### Open questions 14 | 17 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Coinbase, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | version: 2 16 | 17 | updates: 18 | - package-ecosystem: "gomod" 19 | directory: "/" 20 | schedule: 21 | interval: "daily" 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | env: 10 | go_version: 1.16 11 | GO111MODULE: on 12 | COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }} 13 | 14 | jobs: 15 | Build: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - uses: actions/setup-go@v3 20 | with: 21 | go-version: ${{ env.go_version }} 22 | - run: make build 23 | Test: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v3 27 | - uses: actions/setup-go@v3 28 | with: 29 | go-version: ${{ env.go_version }} 30 | - run: make test 31 | Lint: 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v3 35 | - uses: actions/setup-go@v3 36 | with: 37 | go-version: ${{ env.go_version }} 38 | - uses: golangci/golangci-lint-action@v3 39 | with: 40 | version: latest 41 | args: --timeout 3m 42 | 43 | Check-License: 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: actions/checkout@v3 47 | with: 48 | version: latest 49 | - uses: actions/setup-go@v3 50 | with: 51 | go-version: ${{ env.go_version }} 52 | - run: make check-license 53 | 54 | Check-format: 55 | runs-on: ubuntu-latest 56 | steps: 57 | - uses: actions/checkout@v3 58 | with: 59 | version: latest 60 | - run: make check-format 61 | 62 | Coverage: 63 | runs-on: ubuntu-latest 64 | steps: 65 | - uses: actions/checkout@v3 66 | with: 67 | version: latest 68 | - uses: actions/setup-go@v3 69 | with: 70 | go-version: ${{ env.go_version }} 71 | - run: make test-cover COVERALLS_TOKEN="$COVERALLS_TOKEN" 72 | 73 | # Salus: 74 | # runs-on: ubuntu-latest 75 | # steps: 76 | # - uses: actions/checkout@v3 77 | # with: 78 | # version: latest 79 | # - run: make salus 80 | 81 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | rosetta-cli 2 | bin/ 3 | /.vscode 4 | /cli-data 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to mesh-cli 2 | 3 | ## Code of Conduct 4 | 5 | All interactions with this project follow our [Code of Conduct](https://github.com/coinbase/code-of-conduct). By participating, you are expected to honor this code. Violators can be banned from further participation in this project, or potentially all Coinbase projects. 6 | 7 | ## How to Contribute 8 | 9 | You can contribute to this repository by asking questions, providing feedback, and reporting issues. 10 | 11 | ### Asking Questions 12 | 13 | Submit your questions via the [Mesh Community boards][13]. 14 | 15 | ### Providing Feedback 16 | 17 | You can also use the [Mesh Community boards][13] to provide feedback. 18 | 19 | ### Reporting Issues 20 | 21 | You can report issues by submitting bug reports, feature requests, or pull requests via GitHub. You **must** submit [security issues](#security-issues) and [support requests](#support-requests) through the links provided. 22 | 23 | #### Bug Reports 24 | 25 | Before filing a bug report, ensure that your issue [has not already been reported][1]. It may already be fixed! 26 | 27 | If your bug hasn’t been fixed, follow these steps to file a bug report: 28 | 29 | 1. [Open an issue in GitHub][10]. 30 | 2. Add a title for your bug report. It should briefly describe the problem. 31 | 3. Follow the template that appears in the Write text box. This is the best way to describe the bug. 32 | 4. Click _Submit new issue_ to finish filing the bug report. 33 | 34 | #### Feature Requests 35 | 36 | We welcome feedback with or without pull requests. If you have an idea for how to improve the project, great! All we ask is that you take the time to write a clear and concise explanation of the need you are trying to solve. If you have thoughts on _how_ it can be solved, include those too! 37 | 38 | To submit a feature request, follow these steps: 39 | 40 | 1. [Open an issue in GitHub][10]. 41 | 2. Add a title for your feature request. It should briefly describe your requested feature. 42 | 3. Follow the template that appears in the Write text box. This is the best way to explain your request. Be clear and concise in your responses. 43 | 4. Click _Submit new issue_ to submit the feature request. 44 | 45 | The best way to see a feature added, however, is to submit a pull request. 46 | 47 | #### Pull Requests 48 | 49 | Before creating your pull request, it's usually worth asking whether the code you're planning on writing will be considered for merging. You can do this by [opening an issue][1] and asking. It may also help give the maintainers context for when the time comes to review your code. 50 | 51 | Ensure that your [commit messages are well-written][2]. This can double as your pull request message, so it pays to take the time to write a clear message. 52 | 53 | Additionally, make sure that you have written unit tests for your changes. If you're unsure as to what to test, don't hesitate to [open an issue][1] and ask! 54 | 55 | To submit your pull request, follow these steps: 56 | 57 | 1. Follow these instructions on how to [open a pull request in GitHub][11]. 58 | 2. Click _Create pull request_ to submit your pull request. 59 | 60 | Once you submit your pull request, a reviewer will revise it, and either approve it or offer suggestions. 61 | 62 | #### Security Issues 63 | 64 | You can send a report through Coinbase's [H1 program][12]. Check out the [Security][14] tab for more information. 65 | 66 | #### Support Requests 67 | 68 | All support requests must be made via [our support team][3]. 69 | 70 | **For security reasons, any communication referencing support tickets for Coinbase products will be ignored.** The request will have its content redacted and will be locked to prevent further discussion. 71 | 72 | © 2022 Coinbase 73 | 74 | 75 | [1]: https://github.com/coinbase/mesh-cli/issues 76 | [2]: https://chris.beams.io/posts/git-commit/#seven-rules 77 | [3]: https://support.coinbase.com/customer/en/portal/articles/2288496-how-can-i-contact-coinbase-support- 78 | 79 | [5]: https://github.com/coinbase/mesh-cli/issues/new/choose 80 | [6]: https://github.com/coinbase/mesh-cli/issues/new?assignees=&labels=bug&template=bug_report.md&title= 81 | [7]: https://github.com/coinbase/mesh-cli/issues/new?assignees=&labels=enhancement&template=feature_request.md&title= 82 | [8]: https://github.com/coinbase/mesh-cli/pulls 83 | [9]: https://github.com/coinbase/mesh-cli/compare 84 | [10]: https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/working-with-your-remote-repository-on-github-or-github-enterprise/creating-an-issue-or-pull-request#creating-an-issue 85 | [11]: https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/working-with-your-remote-repository-on-github-or-github-enterprise/creating-an-issue-or-pull-request#creating-a-pull-request 86 | [12]: https://hackerone.com/coinbase 87 | [13]: https://github.com/coinbase/mesh-cli/security 88 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Coinbase, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Compile golang 16 | FROM ubuntu:20.04 as cli 17 | 18 | RUN apt-get update && apt-get install -y curl make gcc g++ git 19 | ENV GOLANG_VERSION 1.17.9 20 | ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz 21 | ENV GOLANG_DOWNLOAD_SHA256 9dacf782028fdfc79120576c872dee488b81257b1c48e9032d122cfdb379cca6 22 | 23 | RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ 24 | && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ 25 | && tar -C /usr/local -xzf golang.tar.gz \ 26 | && rm golang.tar.gz 27 | 28 | ENV GOPATH /go 29 | ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH 30 | RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" 31 | 32 | WORKDIR /go/src 33 | 34 | ARG VERSION=v0.10.4 35 | RUN git clone https://github.com/coinbase/rosetta-cli.git && \ 36 | cd rosetta-cli && \ 37 | git fetch --all --tags && \ 38 | git checkout $VERSION && \ 39 | make install 40 | 41 | FROM ubuntu:20.04 42 | 43 | RUN apt-get update -y && apt-get install -y \ 44 | curl 45 | 46 | # Copy all the binaries 47 | COPY --from=cli /go/bin/ /usr/local/bin/ 48 | 49 | WORKDIR /app 50 | ENTRYPOINT ["rosetta-cli"] -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 Coinbase, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deps lint format check-format test test-cover add-license \ 2 | check-license shorten-lines salus validate watch-blocks \ 3 | watch-transactions watch-balances watch-reconciliations \ 4 | view-block-benchmarks view-account-benchmarks mocks 5 | 6 | # To run the the following packages as commands, 7 | # it is necessary to use `go run `. Running `go get` does 8 | # not install any binaries that could be used to run 9 | # the commands directly. 10 | ADDLICENSE_INSTALL=go install github.com/google/addlicense@latest 11 | ADDLICENSE_CMD=addlicense 12 | ADDLICENSE_IGNORE=-ignore ".github/**/*" -ignore ".idea/**/*" 13 | ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v ${ADDLICENSE_IGNORE} 14 | GOLINES_INSTALL=go install github.com/segmentio/golines@latest 15 | GOLINES_CMD=golines 16 | GOVERALLS_INSTALL=go install github.com/mattn/goveralls@latest 17 | GOVERALLS_CMD=goveralls 18 | COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/results/... \ 19 | ./pkg/logger/... ./cmd 20 | TEST_SCRIPT=go test -v ./pkg/... ./configuration/... ./cmd 21 | COVERAGE_TEST_SCRIPT=go test -v ${COVERAGE_TEST_DIRECTORIES} 22 | 23 | deps: 24 | go get ./... 25 | 26 | lint: 27 | golangci-lint run --timeout 2m0s -v \ 28 | -E golint,misspell,gocyclo,whitespace,goconst,gocritic,gocognit,bodyclose,unconvert,lll,unparam,gomnd; 29 | 30 | format: 31 | gofmt -s -w -l .; 32 | 33 | check-format: 34 | ! gofmt -s -l . | read; 35 | 36 | validate-configuration-files: 37 | go run main.go configuration:validate examples/configuration/simple.json; 38 | go run main.go configuration:create examples/configuration/default.json; 39 | go run main.go configuration:validate examples/configuration/default.json; 40 | git diff --exit-code; 41 | 42 | test: | validate-configuration-files 43 | ${TEST_SCRIPT} 44 | 45 | test-cover: 46 | ${GOVERALLS_INSTALL} 47 | if [ "${COVERALLS_TOKEN}" ]; then ${COVERAGE_TEST_SCRIPT} -coverprofile=c.out -covermode=count; ${GOVERALLS_CMD} -coverprofile=c.out -repotoken ${COVERALLS_TOKEN}; fi 48 | 49 | add-license: 50 | ${ADDLICENSE_INSTALL} 51 | ${ADDLICENCE_SCRIPT} . 52 | 53 | check-license: 54 | ${ADDLICENSE_INSTALL} 55 | ${ADDLICENCE_SCRIPT} -check . 56 | 57 | shorten-lines: 58 | ${GOLINES_INSTALL} 59 | ${GOLINES_CMD} -w --shorten-comments pkg cmd configuration 60 | 61 | salus: 62 | docker run --rm -t -v ${PWD}:/home/repo coinbase/salus 63 | 64 | release: add-license shorten-lines format test lint salus 65 | 66 | # This command is to generate multi-platform binaries. 67 | compile: 68 | ./scripts/compile.sh $(version) 69 | 70 | build: 71 | go build ./... 72 | 73 | install: 74 | go install ./... 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Mesh CLI 3 |

4 |

5 | CLI to validate the correctness of Mesh API implementations 6 |

7 |

8 | 9 | 10 | 11 | 12 |

13 | 14 | The `mesh-cli` tool is used by developers to test the correctness of their Mesh API implementations. The CLI also provides the ability to look up block contents and account balances. 15 | 16 | ## Installation 17 | 18 | To download a binary for the latest release, run: 19 | ``` 20 | curl -sSfL https://raw.githubusercontent.com/coinbase/mesh-cli/master/scripts/install.sh | sh -s 21 | ``` 22 | 23 | The binary will be installed inside the `./bin` directory (relative to the directory where you ran the installation command). 24 | 25 | _Downloading binaries from the Github UI will cause permission errors on Mac._ 26 | 27 | ### Installing in a Custom Location 28 | To download the binary into a specific directory, run: 29 | ``` 30 | curl -sSfL https://raw.githubusercontent.com/coinbase/mesh-cli/master/scripts/install.sh | sh -s -- -b 31 | ``` 32 | 33 | ### Run via Docker 34 | Running the following command will start a Docker container and present CLI for the Mesh API. 35 | ``` 36 | docker run -it [image-name] [command] 37 | ``` 38 | 39 | Example: To validate that the Data API implementation is correct, running the following command will start a Docker container with a data directory at ``. 40 | ``` 41 | docker run -v "$(pwd):/data" -it [image-name] check:data --configuration-file /data/config.json 42 | ``` 43 | 44 | ## Key Sign Tool 45 | Mesh CLI comes with a handy key sign tool for local testing. Please refer to this [README](./cmd/README.md) on how to use it. 46 | 47 | ## Updates and Releases 48 | 49 | We recommend that you continually update your installation to the latest release as soon as possible. 50 | 51 | You can also view releases and change log information in the [Releases](https://github.com/coinbase/mesh-cli/releases) section of this repository. 52 | 53 | ## Documentation 54 | 55 | You can find the Mesh API documentation [here](https://docs.cdp.coinbase.com/mesh/docs/welcome/) 56 | 57 | For more information on the mesh-cli tool, read our [The mesh-cli tool](https://docs.cdp.coinbase.com/mesh/docs/mesh-cli/) documentation. 58 | 59 | For more information on how to test your implementation file with the `mesh-cli` tool, read our [How to Test Your Mesh Implementation](https://docs.cdp.coinbase.com/mesh/docs/mesh-test/) documentation. 60 | 61 | ## Contributing 62 | 63 | You may contribute to the `mesh-cli` project in various ways: 64 | 65 | * [Asking Questions](CONTRIBUTING.md/#asking-questions) 66 | * [Providing Feedback](CONTRIBUTING.md/#providing-feedback) 67 | * [Reporting Issues](CONTRIBUTING.md/#reporting-issues) 68 | 69 | Read our [Contributing](CONTRIBUTING.MD) documentation for more information. 70 | 71 | ## mesh-cli Tool Development 72 | 73 | While working on improvements to this repository, we recommend that you use these commands to check your code: 74 | 75 | * `make deps` to install dependencies 76 | * `make test` to run tests 77 | * `make lint` to lint the source code (included generated code) 78 | * `make release` to run one last check before opening a PR 79 | * `make compile version=RELEASE_TAG` to generate binaries 80 | 81 | If you are developing on both the `mesh-cli` and `mesh-sdk-go` repositories, use [go.mod replace](https://golang.org/ref/mod#go-mod-file-replace) to reference local changes: 82 | ``` 83 | replace "github.com/coinbase/mesh-sdk-go" v0.6.8 => "" 84 | ``` 85 | ### Release 86 | * When we release a new mesh-cli version, please update the version number to follow [PR](https://github.com/coinbase/mesh-cli/pull/334) so that `mesh-cli version` command can print the correct value. 87 | * Create binaries and upload all the binaries in the new release tag (e.g. https://github.com/coinbase/mesh-cli/releases/tag/v0.7.7) 88 | * Ensure `$GOPATH/bin` is added to `$PATH` 89 | * Run `make compile version=` 90 | * All the binaries will be created in the `bin` folder and should have extension as `tar.gz` and new version number 91 | 92 | ### Helper/Handler 93 | Many of the packages use a `Helper/Handler` interface pattern to acquire required information or to send events to some client implementation. An example of this is in the `reconciler` package where a `Helper` is used to get the account balance and the `Handler` is called to indicate whether the reconciliation of an account was successful. 94 | 95 | ### Repo Structure 96 | ``` 97 | cmd 98 | examples // examples of different config files 99 | pkg 100 | logger // logic to write syncing information to stdout/files 101 | processor // Helper/Handler implementations for reconciler, storage, and syncer 102 | tester // test orchestrators 103 | ``` 104 | 105 | ### Troubleshooting 106 | 107 | While running the `check:data` or `check:construction` option, if you get the following error: 108 | 109 | ```dial tcp 127.0.0.1:8080: socket: too many open files: unable to sync to 1902533: unable to sync to 1902533``` 110 | 111 | Please run the `ulimit -n 10000` command to increase the max concurrent opened file limit. 112 | 113 | _Note: MacOS users, if you face `ulimit: setrlimit failed: invalid argument` error while setting `ulimit`, please run `sudo launchctl limit maxfiles 10000 200000` before setting the `ulimit`._ 114 | 115 | ## Related Projects 116 | 117 | * [`mesh-sdk-go`](https://github.com/coinbase/mesh-sdk-go) — The `mesh-sdk-go` SDK provides a collection of packages used for interaction with the Mesh API specification. Much of the SDK code is generated from this, the [`mesh-specifications`](https://github.com/coinbase/mesh-specifications) repository. 118 | * [`mesh-specifications`](https://github.com/coinbase/mesh-specifications) — The `mesh-specifications` repository generates the SDK code in the [`mesh-sdk-go`](https://github.com/coinbase/mesh-sdk-go) repository. 119 | 120 | ### Reference Implementations 121 | 122 | To help you with examples, we developed complete Mesh API reference implementations for [Bitcoin](https://github.com/coinbase/mesh-bitcoin) and [Ethereum](https://github.com/coinbase/mesh-ethereum). Developers of Bitcoin-like or Ethereum-like blockchains may find it easier to fork these reference implementations than to write an implementation from scratch. 123 | 124 | You can also find community implementations for a variety of blockchains in the [mesh-ecosystem](https://github.com/coinbase/mesh-ecosystem) repository. 125 | 126 | ## License 127 | This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). 128 | © 2022 Coinbase -------------------------------------------------------------------------------- /cmd/README.md: -------------------------------------------------------------------------------- 1 | ## Key Sign Tool 2 | 3 | Mesh CLI has a key sign tool, which you can use to sign and verify various curves supported 4 | by mesh-specifications. This should only be used for local development. Never share private keys anywhere. 5 | 6 | ### Usage 7 | #### Key Generate 8 | ``` 9 | mesh-cli key:gen --curve-type secp256k1 10 | ``` 11 | Curve Type options are specified by [mesh-specifications](https://github.com/coinbase/mesh-specifications/blob/master/models/CurveType.yaml) 12 | #### Sign 13 | ``` 14 | mesh-cli key:sign --configuration-file config.json 15 | ``` 16 | 17 | A sample config file is located [here](../examples/configuration/sign.json) 18 | 19 | Required fields includes 20 | - `pub_key` 21 | - `private_key` 22 | - `signing_payload` 23 | 24 | 25 | #### Verify 26 | ``` 27 | mesh-cli key:verify --configuration-file verify.json 28 | ``` 29 | A sample config file is located [here](../examples/configuration/verify.json) 30 | 31 | Required fields includes 32 | - `pub_key` 33 | - `signing_payload` 34 | - `signature` 35 | 36 | ### Troubleshoot 37 | - `account_identifier` field in `signing_payload` field should've a dummy address for providing valid payload. -------------------------------------------------------------------------------- /cmd/check_construction.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 23 | "github.com/coinbase/rosetta-cli/pkg/logger" 24 | "github.com/fatih/color" 25 | 26 | "github.com/coinbase/rosetta-cli/pkg/results" 27 | "github.com/coinbase/rosetta-cli/pkg/tester" 28 | 29 | "github.com/coinbase/rosetta-sdk-go/fetcher" 30 | "github.com/coinbase/rosetta-sdk-go/types" 31 | "github.com/coinbase/rosetta-sdk-go/utils" 32 | "github.com/spf13/cobra" 33 | "golang.org/x/sync/errgroup" 34 | ) 35 | 36 | var ( 37 | checkConstructionCmd = &cobra.Command{ 38 | Use: "check:construction", 39 | Short: "Check the correctness of a Rosetta Construction API Implementation", 40 | Long: `The check:construction command runs an automated test of a 41 | Construction API implementation by creating and broadcasting transactions 42 | on a blockchain. In short, this tool generates new addresses, requests 43 | funds, constructs transactions, signs transactions, broadcasts transactions, 44 | and confirms transactions land on-chain. At each phase, a series of tests 45 | are run to ensure that intermediate representations are correct (for example, does 46 | an unsigned transaction return a superset of operations provided during 47 | construction?). 48 | 49 | Check out the https://github.com/coinbase/rosetta-cli/tree/master/examples 50 | directory for examples of how to configure this test for Bitcoin and 51 | Ethereum. 52 | 53 | Right now, this tool only supports transfer testing (for both account-based 54 | and UTXO-based blockchains). However, we plan to add support for testing 55 | arbitrary scenarios (for example, staking and governance).`, 56 | RunE: runCheckConstructionCmd, 57 | } 58 | constructionMetadata string 59 | ) 60 | 61 | func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { 62 | if Config.Construction == nil { 63 | return results.ExitConstruction( 64 | Config, 65 | nil, 66 | nil, 67 | cliErrs.ErrConstructionConfigMissing, 68 | ) 69 | } 70 | 71 | metadataMap := logger.ConvertStringToMap(Config.InfoMetaData) 72 | metadataMap = logger.AddRequestUUIDToMap(metadataMap, Config.RequestUUID) 73 | constructionMetadata = logger.ConvertMapToString(metadataMap) 74 | 75 | ensureDataDirectoryExists() 76 | ctx, cancel := context.WithCancel(Context) 77 | 78 | fetcherOpts := []fetcher.Option{ 79 | fetcher.WithMaxConnections(Config.MaxOnlineConnections), 80 | fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), 81 | fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), 82 | fetcher.WithMaxRetries(Config.MaxRetries), 83 | fetcher.WithMetaData(metadata), 84 | } 85 | if Config.ForceRetry { 86 | fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) 87 | } 88 | 89 | fetcher := fetcher.New( 90 | Config.OnlineURL, 91 | fetcherOpts..., 92 | ) 93 | 94 | _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) 95 | if fetchErr != nil { 96 | cancel() 97 | err := fmt.Errorf("unable to initialize asserter for fetcher: %w%s", fetchErr.Err, constructionMetadata) 98 | color.Red(err.Error()) 99 | return results.ExitConstruction( 100 | Config, 101 | nil, 102 | nil, 103 | err, 104 | ) 105 | } 106 | 107 | _, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) 108 | if err != nil { 109 | cancel() 110 | err = fmt.Errorf("unable to confirm network %s is supported: %w%s", types.PrintStruct(Config.Network), err, constructionMetadata) 111 | color.Red(err.Error()) 112 | return results.ExitConstruction( 113 | Config, 114 | nil, 115 | nil, 116 | err, 117 | ) 118 | } 119 | 120 | if asserterConfigurationFile != "" { 121 | if err := validateNetworkOptionsMatchesAsserterConfiguration( 122 | ctx, fetcher, Config.Network, asserterConfigurationFile, 123 | ); err != nil { 124 | cancel() 125 | err = fmt.Errorf("network options don't match asserter configuration file %s: %w%s", asserterConfigurationFile, err, constructionMetadata) 126 | color.Red(err.Error()) 127 | return results.ExitConstruction( 128 | Config, 129 | nil, 130 | nil, 131 | err, 132 | ) 133 | } 134 | } 135 | 136 | constructionTester, err := tester.InitializeConstruction( 137 | ctx, 138 | Config, 139 | Config.Network, 140 | fetcher, 141 | cancel, 142 | &SignalReceived, 143 | ) 144 | if err != nil { 145 | err = fmt.Errorf("unable to initialize construction tester: %w%s", err, constructionMetadata) 146 | color.Red(err.Error()) 147 | return results.ExitConstruction( 148 | Config, 149 | nil, 150 | nil, 151 | err, 152 | ) 153 | } 154 | defer constructionTester.CloseDatabase(ctx) 155 | 156 | if err := constructionTester.PerformBroadcasts(ctx); err != nil { 157 | err = fmt.Errorf("unable to perform broadcasts: %w%s", err, constructionMetadata) 158 | color.Red(err.Error()) 159 | return results.ExitConstruction( 160 | Config, 161 | nil, 162 | nil, 163 | err, 164 | ) 165 | } 166 | 167 | g, ctx := errgroup.WithContext(ctx) 168 | ctx = logger.AddMetadataMapToContext(ctx, metadataMap) 169 | 170 | g.Go(func() error { 171 | return constructionTester.StartPeriodicLogger(ctx) 172 | }) 173 | 174 | g.Go(func() error { 175 | return constructionTester.StartSyncer(ctx, cancel) 176 | }) 177 | 178 | g.Go(func() error { 179 | return constructionTester.StartConstructor(ctx) 180 | }) 181 | 182 | g.Go(func() error { 183 | return constructionTester.WatchEndConditions(ctx) 184 | }) 185 | 186 | g.Go(func() error { 187 | return tester.LogMemoryLoop(ctx) 188 | }) 189 | 190 | g.Go(func() error { 191 | return tester.StartServer( 192 | ctx, 193 | "check:construction status", 194 | constructionTester, 195 | Config.Construction.StatusPort, 196 | ) 197 | }) 198 | 199 | sigListeners := []context.CancelFunc{cancel} 200 | go handleSignals(&sigListeners) 201 | 202 | return constructionTester.HandleErr(g.Wait(), &sigListeners) 203 | } 204 | -------------------------------------------------------------------------------- /cmd/check_data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/coinbase/rosetta-cli/pkg/logger" 23 | "github.com/coinbase/rosetta-cli/pkg/results" 24 | "github.com/coinbase/rosetta-cli/pkg/tester" 25 | "github.com/coinbase/rosetta-sdk-go/fetcher" 26 | "github.com/coinbase/rosetta-sdk-go/types" 27 | "github.com/coinbase/rosetta-sdk-go/utils" 28 | "github.com/fatih/color" 29 | "github.com/spf13/cobra" 30 | "golang.org/x/sync/errgroup" 31 | ) 32 | 33 | var ( 34 | checkDataCmd = &cobra.Command{ 35 | Use: "check:data", 36 | Short: "Check the correctness of a Rosetta Data API Implementation", 37 | Long: `Check all server responses are 38 | properly constructed, that there are no duplicate blocks and transactions, that blocks can be processed 39 | from genesis to the current block (re-orgs handled automatically), and that 40 | computed balance changes are equal to balance changes reported by the node. 41 | 42 | When re-running this command, it will start where it left off if you specify 43 | some data directory. Otherwise, it will create a new temporary directory and start 44 | again from the genesis block. If you want to discard some number of blocks 45 | populate the --start flag with some block index. Starting from a given index 46 | can be useful to debug a small range of blocks for issues but it is highly 47 | recommended you sync from start to finish to ensure all correctness checks 48 | are performed. 49 | 50 | By default, account balances are looked up at specific heights (instead of 51 | only at the current block). If your node does not support this functionality 52 | set historical balance disabled to true. This will make reconciliation much 53 | less efficient but it will still work. 54 | 55 | If check fails due to an INACTIVE reconciliation error (balance changed without 56 | any corresponding operation), the cli will automatically try to find the block 57 | missing an operation. If historical balance disabled is true, this automatic 58 | debugging tool does not work. 59 | 60 | To debug an INACTIVE account reconciliation error without historical balance lookup, 61 | set the interesting accounts to the path of a JSON file containing 62 | accounts that will be actively checked for balance changes at each block. This 63 | will return an error at the block where a balance change occurred with no 64 | corresponding operations. 65 | 66 | If your blockchain has a genesis allocation of funds and you set 67 | historical balance disabled to true, you must provide an 68 | absolute path to a JSON file containing initial balances with the 69 | bootstrap balance config. You can look at the examples folder for an example 70 | of what one of these files looks like.`, 71 | RunE: runCheckDataCmd, 72 | } 73 | metadata string 74 | ) 75 | 76 | func runCheckDataCmd(_ *cobra.Command, _ []string) error { 77 | ensureDataDirectoryExists() 78 | ctx, cancel := context.WithCancel(Context) 79 | 80 | metadataMap := logger.ConvertStringToMap(Config.InfoMetaData) 81 | metadataMap = logger.AddRequestUUIDToMap(metadataMap, Config.RequestUUID) 82 | metadata = logger.ConvertMapToString(metadataMap) 83 | 84 | fetcherOpts := []fetcher.Option{ 85 | fetcher.WithMaxConnections(Config.MaxOnlineConnections), 86 | fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), 87 | fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), 88 | fetcher.WithMaxRetries(Config.MaxRetries), 89 | fetcher.WithMetaData(metadata), 90 | } 91 | if Config.ForceRetry { 92 | fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) 93 | } 94 | 95 | fetcher := fetcher.New( 96 | Config.OnlineURL, 97 | fetcherOpts..., 98 | ) 99 | 100 | _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) 101 | if fetchErr != nil { 102 | cancel() 103 | err := fmt.Errorf("unable to initialize asserter for fetcher: %w%s", fetchErr.Err, metadata) 104 | color.Red(err.Error()) 105 | return results.ExitData( 106 | Config, 107 | nil, 108 | nil, 109 | err, 110 | "", 111 | "", 112 | ) 113 | } 114 | 115 | networkStatus, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) 116 | if err != nil { 117 | cancel() 118 | err = fmt.Errorf("unable to confirm network %s is supported: %w%s", types.PrintStruct(Config.Network), err, metadata) 119 | color.Red(err.Error()) 120 | return results.ExitData( 121 | Config, 122 | nil, 123 | nil, 124 | err, 125 | "", 126 | "", 127 | ) 128 | } 129 | 130 | if asserterConfigurationFile != "" { 131 | if err := validateNetworkOptionsMatchesAsserterConfiguration( 132 | ctx, fetcher, Config.Network, asserterConfigurationFile, 133 | ); err != nil { 134 | cancel() 135 | err = fmt.Errorf("network options don't match asserter configuration file %s: %w%s", asserterConfigurationFile, err, metadata) 136 | color.Red(err.Error()) 137 | return results.ExitData( 138 | Config, 139 | nil, 140 | nil, 141 | err, 142 | "", 143 | "", 144 | ) 145 | } 146 | } 147 | 148 | dataTester, err := tester.InitializeData( 149 | ctx, 150 | Config, 151 | Config.Network, 152 | fetcher, 153 | cancel, 154 | networkStatus.GenesisBlockIdentifier, 155 | nil, // only populated when doing recursive search 156 | &SignalReceived, 157 | ) 158 | if err != nil { 159 | err = fmt.Errorf("unable to initialize data tester: %w%s", err, metadata) 160 | color.Red(err.Error()) 161 | return results.ExitData( 162 | Config, 163 | nil, 164 | nil, 165 | err, 166 | "", 167 | "", 168 | ) 169 | } 170 | defer dataTester.CloseDatabase(ctx) 171 | 172 | g, ctx := errgroup.WithContext(ctx) 173 | ctx = logger.AddMetadataMapToContext(ctx, metadataMap) 174 | 175 | g.Go(func() error { 176 | return dataTester.StartPeriodicLogger(ctx) 177 | }) 178 | 179 | g.Go(func() error { 180 | return dataTester.StartReconciler(ctx) 181 | }) 182 | 183 | g.Go(func() error { 184 | return dataTester.StartSyncing(ctx) 185 | }) 186 | 187 | g.Go(func() error { 188 | return dataTester.StartPruning(ctx) 189 | }) 190 | 191 | g.Go(func() error { 192 | return dataTester.WatchEndConditions(ctx) 193 | }) 194 | 195 | g.Go(func() error { 196 | return dataTester.StartReconcilerCountUpdater(ctx) 197 | }) 198 | 199 | g.Go(func() error { 200 | return tester.LogMemoryLoop(ctx) 201 | }) 202 | 203 | g.Go(func() error { 204 | return tester.StartServer( 205 | ctx, 206 | "check:data status", 207 | dataTester, 208 | Config.Data.StatusPort, 209 | ) 210 | }) 211 | 212 | sigListeners := []context.CancelFunc{cancel} 213 | go handleSignals(&sigListeners) 214 | 215 | // HandleErr will exit if we should not attempt 216 | // to find missing operations. 217 | return dataTester.HandleErr(g.Wait(), &sigListeners) 218 | } 219 | -------------------------------------------------------------------------------- /cmd/check_perf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/coinbase/rosetta-cli/pkg/results" 23 | t "github.com/coinbase/rosetta-cli/pkg/tester" 24 | "github.com/spf13/cobra" 25 | "golang.org/x/sync/errgroup" 26 | ) 27 | 28 | var ( 29 | checkPerfCmd = &cobra.Command{ 30 | Use: "check:perf", 31 | Short: "Benchmark performance of time-critical endpoints of Asset Issuer's Rosetta Implementation", 32 | Long: `This command can be used to benchmark the performance of time critical methods for a Rosetta server. 33 | This is useful for ensuring that there are no performance degradations in the rosetta-server.`, 34 | RunE: runCheckPerfCmd, 35 | } 36 | ) 37 | 38 | func runCheckPerfCmd(_ *cobra.Command, _ []string) error { 39 | ctx, cancel := context.WithCancel(Context) 40 | defer cancel() 41 | g, ctx := errgroup.WithContext(ctx) 42 | 43 | TotalNumEndpoints := int64(Config.Perf.NumTimesToHitEndpoints) * (Config.Perf.EndBlock - Config.Perf.StartBlock) 44 | perfRawStats := &results.CheckPerfRawStats{AccountBalanceEndpointTotalTime: -1, BlockEndpointTotalTime: -1} 45 | 46 | fmt.Printf("Running Check:Perf for %s:%s for blocks %d-%d \n", Config.Network.Blockchain, Config.Network.Network, Config.Perf.StartBlock, Config.Perf.EndBlock) 47 | 48 | fetcher, timer, elapsed := t.SetupBenchmarking(Config) 49 | blockEndpointTimeConstraint := time.Duration(Config.Perf.BlockEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond 50 | blockEndpointCtx, blockEndpointCancel := context.WithTimeout(ctx, blockEndpointTimeConstraint) 51 | g.Go(func() error { 52 | return t.BmarkBlock(blockEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats) 53 | }) 54 | defer blockEndpointCancel() 55 | 56 | fetcher, timer, elapsed = t.SetupBenchmarking(Config) 57 | accountBalanceEndpointTimeConstraint := time.Duration(Config.Perf.AccountBalanceEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond 58 | accountBalanceEndpointCtx, accountBalanceEndpointCancel := context.WithTimeout(ctx, accountBalanceEndpointTimeConstraint) 59 | g.Go(func() error { 60 | return t.BmarkAccountBalance(accountBalanceEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats) 61 | }) 62 | defer accountBalanceEndpointCancel() 63 | 64 | return results.ExitPerf(Config.Perf, g.Wait(), perfRawStats) 65 | } 66 | -------------------------------------------------------------------------------- /cmd/check_spec_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | 21 | "github.com/coinbase/rosetta-sdk-go/fetcher" 22 | "github.com/fatih/color" 23 | 24 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 25 | ) 26 | 27 | type checkSpecAPI string 28 | type checkSpecRequirement string 29 | 30 | const ( 31 | networkList checkSpecAPI = "/network/list" 32 | networkOptions checkSpecAPI = "/network/options" 33 | accountCoins checkSpecAPI = "/account/coins" 34 | block checkSpecAPI = "/block" 35 | errorObject checkSpecAPI = "error object" 36 | modes checkSpecAPI = "modes" 37 | 38 | networkIDs checkSpecRequirement = "network_identifiers is required" 39 | offlineMode checkSpecRequirement = "endpoint should work in offline mode" 40 | staticNetworkID checkSpecRequirement = "network_identifier must be static" 41 | version checkSpecRequirement = "field version is required" 42 | allow checkSpecRequirement = "field allow is required" 43 | 44 | blockID checkSpecRequirement = "block_identifier is required" 45 | coins checkSpecRequirement = "field coins is required" 46 | idempotent checkSpecRequirement = "same hash should return the same block" 47 | defaultTip checkSpecRequirement = "tip should be returned if block_identifier is not specified" 48 | 49 | errorCode checkSpecRequirement = "error code is required" 50 | errorMessage checkSpecRequirement = "error message is required" 51 | diffURLs checkSpecRequirement = "offline_url should be different from offline_url and not empty" 52 | 53 | checkSpecSuccess string = "Success" 54 | checkSpecFailure string = "Failure" 55 | cbSpec bool = true 56 | ) 57 | 58 | type checkSpecStatus struct { 59 | status string 60 | coinbaseSpec bool 61 | } 62 | 63 | type checkSpecOutput struct { 64 | api checkSpecAPI 65 | validation map[checkSpecRequirement]checkSpecStatus 66 | } 67 | 68 | func twoModes() checkSpecOutput { 69 | output := checkSpecOutput{ 70 | api: modes, 71 | validation: map[checkSpecRequirement]checkSpecStatus{ 72 | diffURLs: { 73 | status: checkSpecSuccess, 74 | coinbaseSpec: cbSpec, 75 | }, 76 | }, 77 | } 78 | 79 | if isEmpty(Config.OnlineURL) || 80 | isEmpty(Config.Construction.OfflineURL) || 81 | isEqual(Config.OnlineURL, Config.Construction.OfflineURL) { 82 | setValidationStatusFailed(output, diffURLs) 83 | } 84 | 85 | return output 86 | } 87 | 88 | func markAllValidationsFailed(output checkSpecOutput) { 89 | for k, v := range output.validation { 90 | output.validation[k] = checkSpecStatus{ 91 | status: checkSpecFailure, 92 | coinbaseSpec: v.coinbaseSpec, 93 | } 94 | } 95 | } 96 | 97 | func setValidationStatusFailed(output checkSpecOutput, req checkSpecRequirement) { 98 | output.validation[req] = checkSpecStatus{ 99 | status: checkSpecFailure, 100 | coinbaseSpec: output.validation[req].coinbaseSpec, 101 | } 102 | } 103 | 104 | func validateErrorObject(err *fetcher.Error, output checkSpecOutput) { 105 | if err != nil { 106 | if err.ClientErr != nil && isNegative(int64(err.ClientErr.Code)) { 107 | printError("%v\n", cliErrs.ErrErrorNegativeCode) 108 | setValidationStatusFailed(output, errorCode) 109 | } 110 | 111 | if err.ClientErr != nil && isEmpty(err.ClientErr.Message) { 112 | printError("%v\n", cliErrs.ErrErrorEmptyMessage) 113 | setValidationStatusFailed(output, errorMessage) 114 | } 115 | } 116 | } 117 | 118 | func printInfo(format string, a ...interface{}) { 119 | fmt.Printf(format, a...) 120 | } 121 | 122 | func printError(format string, a ...interface{}) { 123 | fmt.Print(color.RedString(format, a...)) 124 | } 125 | 126 | func printSuccess(format string, a ...interface{}) { 127 | fmt.Print(color.GreenString(format, a...)) 128 | } 129 | 130 | func printValidationResult(format string, css checkSpecStatus, a ...interface{}) { 131 | if css.status == checkSpecFailure { 132 | printError(format, a...) 133 | } else { 134 | printSuccess(format, a...) 135 | } 136 | } 137 | 138 | func printCheckSpecOutputHeader() { 139 | printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+-----------------+") 140 | printInfo("%v\n", "| API | Requirement | Status | Coinbase Spec |") 141 | printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+-----------------+") 142 | } 143 | 144 | func printCheckSpecOutputBody(output checkSpecOutput) { 145 | for k, v := range output.validation { 146 | // print api 147 | printInfo("%v", "| ") 148 | printValidationResult("%v", v, output.api) 149 | for j := 0; j < 24-len(output.api); j++ { 150 | printInfo("%v", " ") 151 | } 152 | 153 | // print requirement description 154 | printInfo("%v", "| ") 155 | printValidationResult("%v", v, k) 156 | for j := 0; j < 65-len(k); j++ { 157 | printInfo(" ") 158 | } 159 | 160 | // print validation status 161 | printInfo("%v", "| ") 162 | printValidationResult("%v", v, v.status) 163 | for j := 0; j < 9-len(v.status); j++ { 164 | printInfo("%v", " ") 165 | } 166 | 167 | // print coinbase spec flag 168 | printInfo("%v", "| ") 169 | printValidationResult("%v", v, v.coinbaseSpec) 170 | for j := 0; j < 11-len(strconv.FormatBool(v.coinbaseSpec)); j++ { 171 | printInfo("%v", " ") 172 | } 173 | 174 | printInfo("%v\n", "|") 175 | printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+-----------------+") 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /cmd/configuration_create.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/coinbase/rosetta-cli/configuration" 21 | 22 | "github.com/coinbase/rosetta-sdk-go/utils" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | var ( 27 | configurationCreateCmd = &cobra.Command{ 28 | Use: "configuration:create", 29 | Short: "Create a default configuration file at the provided path", 30 | RunE: runConfigurationCreateCmd, 31 | Args: cobra.ExactArgs(1), 32 | } 33 | ) 34 | 35 | func runConfigurationCreateCmd(cmd *cobra.Command, args []string) error { 36 | if err := utils.SerializeAndWrite(args[0], configuration.DefaultConfiguration()); err != nil { 37 | return fmt.Errorf("unable to save configuration file to %s: %w", args[0], err) 38 | } 39 | 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /cmd/configuration_validate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/coinbase/rosetta-cli/configuration" 21 | 22 | "github.com/fatih/color" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | var ( 27 | configurationValidateCmd = &cobra.Command{ 28 | Use: "configuration:validate", 29 | Short: "Ensure a configuration file at the provided path is formatted correctly", 30 | RunE: runConfigurationValidateCmd, 31 | Args: cobra.ExactArgs(1), 32 | } 33 | ) 34 | 35 | func runConfigurationValidateCmd(cmd *cobra.Command, args []string) error { 36 | _, err := configuration.LoadConfiguration(Context, args[0]) 37 | if err != nil { 38 | return fmt.Errorf("configuration validation failed %s: %w", args[0], err) 39 | } 40 | 41 | color.Green("Configuration file validated!") 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /cmd/key_gen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "encoding/hex" 19 | "errors" 20 | 21 | "github.com/coinbase/rosetta-sdk-go/keys" 22 | "github.com/coinbase/rosetta-sdk-go/types" 23 | "github.com/fatih/color" 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | var ( 28 | keyGenCmd = &cobra.Command{ 29 | Use: "key:gen", 30 | Short: "Used to generate a public private key pair", 31 | Long: `Used to generate a public private key pair 32 | It supports Keypair specified by https://github.com/coinbase/rosetta-specifications 33 | Please provide valid CurveType`, 34 | RunE: runKeyGenCmd, 35 | } 36 | ) 37 | 38 | func runKeyGenCmd(_ *cobra.Command, _ []string) error { 39 | if len(curveType) == 0 { 40 | color.Red("please provide a non-empty curve type") 41 | return errors.New("invalid curve-type string") 42 | } 43 | 44 | curve := types.CurveType(curveType) 45 | 46 | color.Yellow("Generating new %s keypair...", curve) 47 | keyPair, err := keys.GenerateKeypair(curve) 48 | if err != nil { 49 | color.Red("failed to generate keypair with error %#v", err) 50 | } 51 | 52 | color.Green("CurveType: %s", curve) 53 | color.Green("Public Key (hex): %s", hex.EncodeToString(keyPair.PublicKey.Bytes)) 54 | color.Green("Private Key (hex): %s", hex.EncodeToString(keyPair.PrivateKey)) 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /cmd/key_sign.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "encoding/hex" 19 | "errors" 20 | 21 | "github.com/coinbase/rosetta-sdk-go/keys" 22 | "github.com/fatih/color" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | var ( 27 | keySignCmd = &cobra.Command{ 28 | Use: "key:sign", 29 | Short: "Sign an unsigned payload with given private key", 30 | Long: `Sign an unsigned payload with given private key 31 | It supports Keypair specified by https://github.com/coinbase/rosetta-specifications 32 | Please provide valid PrivateKey, CurveType, SignaturePayload`, 33 | RunE: runKeySignCmd, 34 | } 35 | ) 36 | 37 | func runKeySignCmd(_ *cobra.Command, _ []string) error { 38 | if Config.Sign == nil { 39 | return errors.New("sign configuration is missing") 40 | } 41 | 42 | if len(Config.Sign.PrivateKey) == 0 || 43 | Config.Sign.PubKey.CurveType == "" || 44 | Config.Sign.SigningPayload == nil || 45 | Config.Sign.SigningPayload.SignatureType == "" { 46 | color.Red("invalid sign input") 47 | } 48 | 49 | keyPair, err := keys.ImportPrivateKey(Config.Sign.PrivateKey, Config.Sign.PubKey.CurveType) 50 | if err != nil { 51 | color.Red("unable to import private keys %#v", err) 52 | return err 53 | } 54 | 55 | err = keyPair.IsValid() 56 | if err != nil { 57 | color.Red("keypair invalid with err %#v", err) 58 | return err 59 | } 60 | 61 | signer, err := keyPair.Signer() 62 | if err != nil { 63 | color.Red("signer invalid with err %#v", err) 64 | return err 65 | } 66 | 67 | signingPayload := Config.Sign.SigningPayload 68 | signatureType := Config.Sign.SigningPayload.SignatureType 69 | 70 | sign, err := signer.Sign(signingPayload, signatureType) 71 | if err != nil { 72 | color.Red("unable to sign with err %#v", err) 73 | return err 74 | } 75 | 76 | hexSig := hex.EncodeToString(sign.Bytes) 77 | color.Green("Signature: %s", hexSig) 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /cmd/key_verify.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "errors" 19 | "github.com/coinbase/rosetta-sdk-go/keys" 20 | "github.com/fatih/color" 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | var ( 25 | keyVerifyCmd = &cobra.Command{ 26 | Use: "key:verify", 27 | Short: "Verify the signature using the public key", 28 | Long: `Verify the signature using the public key 29 | It supports Keypair specified by https://github.com/coinbase/rosetta-specifications`, 30 | RunE: runKeyVerifyCmd, 31 | } 32 | ) 33 | 34 | func runKeyVerifyCmd(_ *cobra.Command, _ []string) error { 35 | if Config.Sign == nil { 36 | return errors.New("sign configuration is missing") 37 | } 38 | 39 | if len(Config.Sign.Signature.Bytes) == 0 || 40 | Config.Sign.SigningPayload == nil || 41 | Config.Sign.SigningPayload.SignatureType == "" || 42 | Config.Sign.PubKey == nil { 43 | color.Red("invalid verify input") 44 | } 45 | 46 | keyPair := keys.KeyPair{ 47 | PublicKey: Config.Sign.PubKey, 48 | } 49 | 50 | signer, err := keyPair.Signer() 51 | if err != nil { 52 | color.Red("signer invalid with err %#v", err) 53 | return err 54 | } 55 | 56 | signature := Config.Sign.Signature 57 | signature.SignatureType = Config.Sign.SigningPayload.SignatureType 58 | signature.SigningPayload = Config.Sign.SigningPayload 59 | signature.PublicKey = Config.Sign.PubKey 60 | 61 | err = signer.Verify(signature) 62 | if err != nil { 63 | color.Red("invalid signature with err %#v", err) 64 | return err 65 | } 66 | 67 | color.Green("Signature Verified.") 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /cmd/utils_asserter_configuration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | "time" 21 | 22 | "github.com/coinbase/rosetta-sdk-go/asserter" 23 | "github.com/coinbase/rosetta-sdk-go/fetcher" 24 | "github.com/coinbase/rosetta-sdk-go/utils" 25 | "github.com/fatih/color" 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | var ( 30 | utilsAsserterConfigurationCmd = &cobra.Command{ 31 | Use: "utils:asserter-configuration", 32 | Short: "Generate a static configuration file for the Asserter", 33 | Long: `In production deployments, it is useful to initialize the response 34 | Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using 35 | a static configuration instead of initializing a configuration dynamically 36 | from the node. This allows a client to error on new types/statuses that may 37 | have been added in an update instead of silently erroring. 38 | 39 | To use this command, simply provide an absolute path as the argument for where 40 | the configuration file should be saved (in JSON).`, 41 | RunE: runCreateConfigurationCmd, 42 | Args: cobra.ExactArgs(1), 43 | } 44 | ) 45 | 46 | func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { 47 | // Create a new fetcher 48 | newFetcher := fetcher.New( 49 | Config.OnlineURL, 50 | fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), 51 | fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), 52 | fetcher.WithMaxRetries(Config.MaxRetries), 53 | ) 54 | 55 | // Initialize the fetcher's asserter 56 | _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) 57 | if fetchErr != nil { 58 | return fmt.Errorf("failed to initialize asserter for fetcher: %w", fetchErr.Err) 59 | } 60 | 61 | configuration, err := newFetcher.Asserter.ClientConfiguration() 62 | if err != nil { 63 | return fmt.Errorf("unable to generate asserter configuration: %w", err) 64 | } 65 | 66 | sortArrayFieldsOnConfiguration(configuration) 67 | 68 | if err := utils.SerializeAndWrite(args[0], configuration); err != nil { 69 | return fmt.Errorf("unable to serialize asserter configuration: %w", err) 70 | } 71 | 72 | color.Green("Configuration file saved!") 73 | return nil 74 | } 75 | 76 | func sortArrayFieldsOnConfiguration(configuration *asserter.Configuration) { 77 | sort.Strings(configuration.AllowedOperationTypes) 78 | sort.Slice(configuration.AllowedOperationStatuses, func(i, j int) bool { 79 | return configuration.AllowedOperationStatuses[i].Status < configuration.AllowedOperationStatuses[j].Status 80 | }) 81 | sort.Slice(configuration.AllowedErrors, func(i, j int) bool { 82 | return configuration.AllowedErrors[i].Code < configuration.AllowedErrors[j].Code 83 | }) 84 | } 85 | -------------------------------------------------------------------------------- /cmd/utils_asserter_configuration_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import "github.com/coinbase/rosetta-sdk-go/types" 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | 24 | "github.com/coinbase/rosetta-sdk-go/asserter" 25 | ) 26 | 27 | var ( 28 | basicNetwork = &types.NetworkIdentifier{ 29 | Blockchain: "blockchain", 30 | Network: "network", 31 | } 32 | 33 | basicBlock = &types.BlockIdentifier{ 34 | Index: 10, 35 | Hash: "block 10", 36 | } 37 | 38 | allowedOperationTypes = []string{"OUTPUT", "INPUT", "TRANSFER"} 39 | 40 | allowedOperationStatuses = []*types.OperationStatus{ 41 | { 42 | Status: "SUCCESS", 43 | Successful: true, 44 | }, 45 | { 46 | Status: "SKIPPED", 47 | Successful: true, 48 | }, 49 | } 50 | 51 | allowedErrors = []*types.Error{ 52 | { 53 | Code: 4, 54 | Message: "Block not found", 55 | Retriable: false, 56 | }, 57 | { 58 | Code: 0, 59 | Message: "Endpoint not implemented", 60 | Retriable: false, 61 | }, 62 | { 63 | Code: 3, 64 | Message: "Bitcoind error", 65 | Retriable: false, 66 | }, 67 | } 68 | 69 | timestampStartIndex = int64(6) 70 | ) 71 | 72 | func TestSortArrayFields(t *testing.T) { 73 | var clientConfiguration = &asserter.Configuration{ 74 | NetworkIdentifier: basicNetwork, 75 | GenesisBlockIdentifier: basicBlock, 76 | AllowedOperationTypes: allowedOperationTypes, 77 | AllowedOperationStatuses: allowedOperationStatuses, 78 | AllowedErrors: allowedErrors, 79 | AllowedTimestampStartIndex: timestampStartIndex, 80 | } 81 | var assert = assert.New(t) 82 | sortArrayFieldsOnConfiguration(clientConfiguration) 83 | assert.Equal([]string{"INPUT", "OUTPUT", "TRANSFER"}, clientConfiguration.AllowedOperationTypes) 84 | assert.Equal([]*types.OperationStatus{ 85 | { 86 | Status: "SKIPPED", 87 | Successful: true, 88 | }, 89 | { 90 | Status: "SUCCESS", 91 | Successful: true, 92 | }, 93 | }, clientConfiguration.AllowedOperationStatuses) 94 | assert.Equal([]*types.Error{ 95 | { 96 | Code: 0, 97 | Message: "Endpoint not implemented", 98 | Retriable: false, 99 | }, 100 | { 101 | Code: 3, 102 | Message: "Bitcoind error", 103 | Retriable: false, 104 | }, 105 | { 106 | Code: 4, 107 | Message: "Block not found", 108 | Retriable: false, 109 | }, 110 | }, clientConfiguration.AllowedErrors) 111 | } 112 | -------------------------------------------------------------------------------- /cmd/utils_shared.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | func isEmpty(s string) bool { 18 | return s == "" 19 | } 20 | 21 | func isNegative(n int64) bool { 22 | return n < 0 23 | } 24 | 25 | func isEqual(s1 string, s2 string) bool { 26 | return s1 == s2 27 | } 28 | 29 | func isUTXO() bool { 30 | return Config.CoinSupported 31 | } 32 | -------------------------------------------------------------------------------- /cmd/utils_train_zstd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "path" 21 | "strconv" 22 | 23 | "github.com/coinbase/rosetta-sdk-go/storage/database" 24 | "github.com/coinbase/rosetta-sdk-go/storage/encoder" 25 | "github.com/fatih/color" 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | const ( 30 | trainArgs = 4 31 | ) 32 | 33 | var ( 34 | utilsTrainZstdCmd = &cobra.Command{ 35 | Use: "utils:train-zstd", 36 | Short: "Generate a zstd dictionary for enhanced compression performance", 37 | Long: `Zstandard (https://github.com/facebook/zstd) is used by 38 | rosetta-sdk-go/storage to compress data stored to disk. It is possible 39 | to improve compression performance by training a dictionary on a particular 40 | storage namespace. This command runs this training and outputs a dictionary 41 | that can be used with rosetta-sdk-go/storage. 42 | 43 | The arguments for this command are: 44 | () 45 | 46 | You can learn more about dictionary compression on the Zstandard 47 | website: https://github.com/facebook/zstd#the-case-for-small-data-compression`, 48 | RunE: runTrainZstdCmd, 49 | Args: cobra.MinimumNArgs(trainArgs), 50 | } 51 | ) 52 | 53 | func runTrainZstdCmd(cmd *cobra.Command, args []string) error { 54 | namespace := args[0] 55 | databasePath := path.Clean(args[1]) 56 | dictionaryPath := path.Clean(args[2]) 57 | maxItems, err := strconv.Atoi(args[3]) 58 | if err != nil { 59 | return fmt.Errorf("unable to convert max items to integer: %w", err) 60 | } 61 | 62 | compressorEntries := []*encoder.CompressorEntry{} 63 | if len(args) > trainArgs { 64 | compressorEntries = append(compressorEntries, &encoder.CompressorEntry{ 65 | Namespace: namespace, 66 | DictionaryPath: args[4], 67 | }) 68 | 69 | log.Printf("found dictionary path %s\n", args[4]) 70 | } 71 | 72 | log.Printf("Running zstd training (this could take a while)...") 73 | 74 | _, _, err = database.BadgerTrain( 75 | Context, 76 | namespace, 77 | databasePath, 78 | dictionaryPath, 79 | maxItems, 80 | compressorEntries, 81 | ) 82 | if err != nil { 83 | return fmt.Errorf("badger training failed: %w", err) 84 | } 85 | 86 | color.Green("Training successful!") 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /cmd/validate_asserter_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "reflect" 21 | "sort" 22 | "strings" 23 | 24 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 25 | "github.com/coinbase/rosetta-sdk-go/asserter" 26 | "github.com/coinbase/rosetta-sdk-go/fetcher" 27 | "github.com/coinbase/rosetta-sdk-go/types" 28 | "github.com/coinbase/rosetta-sdk-go/utils" 29 | ) 30 | 31 | // Common helper across Construction and Data 32 | // Issues an RPC to fetch /network/options, and extracts the `Allow` 33 | // Reads the JSON file at `asserterConfigurationFile` and loads into a Go object 34 | // Validates the `Allow`s across both objects match 35 | func validateNetworkOptionsMatchesAsserterConfiguration( 36 | ctx context.Context, f *fetcher.Fetcher, network *types.NetworkIdentifier, 37 | asserterConfigurationFile string, 38 | ) error { 39 | var asserterConfiguration asserter.Configuration 40 | if err := utils.LoadAndParse(asserterConfigurationFile, &asserterConfiguration); err != nil { 41 | return fmt.Errorf("failed to load and parse asserter configuration file %s: %w", asserterConfigurationFile, err) 42 | } 43 | 44 | resp, fetchErr := f.NetworkOptions(ctx, network, nil) 45 | if fetchErr != nil { 46 | return fmt.Errorf("failed to get network options: %w", fetchErr.Err) 47 | } 48 | 49 | err := validateNetworkAndAsserterAllowMatch(resp.Allow, &asserterConfiguration) 50 | if err != nil { 51 | return fmt.Errorf("failed to validate network options allowlist with asserter configuration: %w", err) 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func validateNetworkAndAsserterAllowMatch( 58 | networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, 59 | ) error { 60 | if networkAllow == nil { 61 | return cliErrs.ErrNetworkOptionsAllowlistIsNil 62 | } 63 | if asserterConfiguration == nil { 64 | return cliErrs.ErrAsserterConfigurationIsNil 65 | } 66 | 67 | if err := verifyTimestampStartIndex( 68 | networkAllow.TimestampStartIndex, asserterConfiguration.AllowedTimestampStartIndex, 69 | ); err != nil { 70 | return fmt.Errorf("failed to verify timestamp start index: %w", err) 71 | } 72 | 73 | if err := verifyOperationTypes( 74 | networkAllow.OperationTypes, asserterConfiguration.AllowedOperationTypes, 75 | ); err != nil { 76 | return fmt.Errorf("failed to verify operation types: %w", err) 77 | } 78 | 79 | if err := verifyOperationStatuses( 80 | networkAllow.OperationStatuses, asserterConfiguration.AllowedOperationStatuses, 81 | ); err != nil { 82 | return fmt.Errorf("failed to verify operation statuses: %w", err) 83 | } 84 | 85 | if err := verifyErrors( 86 | networkAllow.Errors, asserterConfiguration.AllowedErrors, 87 | ); err != nil { 88 | return fmt.Errorf("failed to verify errors: %w", err) 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { 95 | var networkTsiVal int64 = 1 96 | if networkTsi != nil { // This field is optional and defaults to all allowed 97 | networkTsiVal = *networkTsi 98 | } 99 | if networkTsiVal != assertTsi { 100 | return fmt.Errorf("network options timestamp start index %d, asserter configuration timestamp start index %d: %w", networkTsiVal, assertTsi, cliErrs.ErrTimestampStartIndexMismatch) 101 | } 102 | 103 | return nil 104 | } 105 | 106 | func verifyOperationTypes(networkOt, asserterOt []string) error { 107 | if len(networkOt) != len(asserterOt) { 108 | return fmt.Errorf("network options operation type length %d, asserter configuration operation type length %d: %w", len(networkOt), len(asserterOt), cliErrs.ErrOperationTypeLengthMismatch) 109 | } 110 | 111 | sort.Strings(networkOt) 112 | sort.Strings(asserterOt) 113 | 114 | for i, networkOperationType := range networkOt { 115 | asserterOperationType := asserterOt[i] 116 | if networkOperationType != asserterOperationType { 117 | return fmt.Errorf("network options operation type %s, asserter configuration operation type %s: %w", networkOperationType, asserterOperationType, cliErrs.ErrOperationTypeMismatch) 118 | } 119 | } 120 | 121 | return nil 122 | } 123 | 124 | func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) error { 125 | if len(networkOs) != len(asserterOs) { 126 | return fmt.Errorf("network options operation status length %d, asserter configuration operation status length %d: %w", len(networkOs), len(asserterOs), cliErrs.ErrOperationStatusLengthMismatch) 127 | } 128 | 129 | sort.Slice(networkOs, func(i, j int) bool { 130 | return strings.Compare(networkOs[i].Status, networkOs[j].Status) < 0 131 | }) 132 | sort.Slice(asserterOs, func(i, j int) bool { 133 | return strings.Compare(asserterOs[i].Status, asserterOs[j].Status) < 0 134 | }) 135 | 136 | for i, networkOperationStatus := range networkOs { 137 | asserterOperationStatus := asserterOs[i] 138 | if !reflect.DeepEqual(networkOperationStatus, asserterOperationStatus) { 139 | return fmt.Errorf("network options operation type %s, asserter configuration operation type %s: %w", types.PrintStruct(networkOperationStatus), types.PrintStruct(asserterOperationStatus), cliErrs.ErrOperationStatusMismatch) 140 | } 141 | } 142 | 143 | return nil 144 | } 145 | 146 | func verifyErrors(networkErrors, asserterErrors []*types.Error) error { 147 | if len(networkErrors) != len(asserterErrors) { 148 | return fmt.Errorf("network options error length %d, asserter configuration error length %d: %w", len(networkErrors), len(asserterErrors), cliErrs.ErrErrorLengthMismatch) 149 | } 150 | 151 | sort.Slice(networkErrors, func(i, j int) bool { 152 | return networkErrors[i].Code < networkErrors[j].Code 153 | }) 154 | sort.Slice(asserterErrors, func(i, j int) bool { 155 | return asserterErrors[i].Code < asserterErrors[j].Code 156 | }) 157 | 158 | for i, networkError := range networkErrors { 159 | asserterError := asserterErrors[i] 160 | if !reflect.DeepEqual(networkError, asserterError) { 161 | return fmt.Errorf("network options error %s, asserter configuration error %s: %w", types.PrintStruct(networkError), types.PrintStruct(asserterError), cliErrs.ErrErrorMismatch) 162 | } 163 | } 164 | 165 | return nil 166 | } 167 | -------------------------------------------------------------------------------- /cmd/validate_asserter_config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "github.com/coinbase/rosetta-sdk-go/asserter" 19 | "github.com/coinbase/rosetta-sdk-go/types" 20 | "github.com/stretchr/testify/assert" 21 | "testing" 22 | ) 23 | 24 | func TestMatch(t *testing.T) { 25 | networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() 26 | confirmSuccess(t, networkAllow, asserterConfiguration) 27 | } 28 | 29 | func TestNil(t *testing.T) { 30 | networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() 31 | confirmError(t, nil, asserterConfiguration) 32 | confirmError(t, networkAllow, nil) 33 | } 34 | 35 | func TestTsi(t *testing.T) { 36 | // Confirm nil defaults to 1 37 | networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() 38 | networkAllow.TimestampStartIndex = nil 39 | confirmError(t, networkAllow, asserterConfiguration) 40 | asserterConfiguration.AllowedTimestampStartIndex = 1 41 | confirmSuccess(t, networkAllow, asserterConfiguration) 42 | 43 | networkAllow, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() 44 | asserterConfiguration.AllowedTimestampStartIndex = 567 45 | confirmError(t, networkAllow, asserterConfiguration) 46 | } 47 | 48 | func TestOperationTypes(t *testing.T) { 49 | networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() 50 | networkAllow.OperationTypes[1] = "mismatchType" 51 | confirmError(t, networkAllow, asserterConfiguration) 52 | 53 | networkAllow.OperationTypes = append(generateOperationTypes(), "extra") 54 | confirmError(t, networkAllow, asserterConfiguration) 55 | 56 | networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() 57 | asserterConfiguration.AllowedOperationTypes = nil 58 | confirmError(t, networkAllow, asserterConfiguration) 59 | } 60 | 61 | func TestOperationStatuses(t *testing.T) { 62 | networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() 63 | networkAllow.OperationStatuses[0].Successful = !networkAllow.OperationStatuses[0].Successful 64 | confirmError(t, networkAllow, asserterConfiguration) 65 | 66 | networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() 67 | asserterConfiguration.AllowedOperationStatuses[1].Status = "mismatchStatus" 68 | confirmError(t, networkAllow, asserterConfiguration) 69 | 70 | _, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() 71 | asserterConfiguration.AllowedOperationStatuses = append(generateOperationStatuses(), 72 | &types.OperationStatus{Status: "extra"}) 73 | confirmError(t, networkAllow, asserterConfiguration) 74 | 75 | _, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() 76 | networkAllow.OperationStatuses = nil 77 | confirmError(t, networkAllow, asserterConfiguration) 78 | } 79 | 80 | func TestErrors(t *testing.T) { 81 | networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() 82 | networkAllow.Errors[0].Code = 123 83 | confirmError(t, networkAllow, asserterConfiguration) 84 | 85 | networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() 86 | asserterConfiguration.AllowedErrors[1].Message = "mismatchMessage" 87 | confirmError(t, networkAllow, asserterConfiguration) 88 | 89 | _, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() 90 | networkAllow.Errors[0].Details = map[string]interface{}{"key": "value"} 91 | asserterConfiguration.AllowedErrors[0].Details = map[string]interface{}{"key": "differentValue"} 92 | confirmError(t, networkAllow, asserterConfiguration) 93 | 94 | networkAllow, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() 95 | asserterConfiguration.AllowedErrors = append(asserterConfiguration.AllowedErrors, 96 | &types.Error{Code: 123, Message: "extra"}) 97 | confirmError(t, networkAllow, asserterConfiguration) 98 | } 99 | 100 | // Generate simple configs for testing 101 | // Generators used internally below are so they are logically equal but can be mutated separately 102 | func generateNetworkAllowAndAsserterConfiguration() ( 103 | *types.Allow, *asserter.Configuration, 104 | ) { 105 | var tsi int64 = 5 106 | allow := &types.Allow{ 107 | OperationStatuses: generateOperationStatuses(), 108 | OperationTypes: generateOperationTypes(), 109 | Errors: generateErrors(), 110 | TimestampStartIndex: &tsi, 111 | } 112 | config := &asserter.Configuration{ 113 | AllowedOperationStatuses: generateOperationStatuses(), 114 | AllowedOperationTypes: generateOperationTypes(), 115 | AllowedErrors: generateErrors(), 116 | AllowedTimestampStartIndex: tsi, 117 | } 118 | 119 | return allow, config 120 | } 121 | 122 | func generateOperationTypes() []string { 123 | return []string{"type0", "type1"} 124 | } 125 | 126 | func generateOperationStatuses() []*types.OperationStatus { 127 | return []*types.OperationStatus{ 128 | { 129 | Successful: true, 130 | Status: "status0", 131 | }, 132 | { 133 | // Successful: false 134 | Status: "status1", 135 | }, 136 | } 137 | } 138 | 139 | func generateErrors() []*types.Error { 140 | return []*types.Error{ 141 | { 142 | Code: 1, 143 | Message: "message1", 144 | }, 145 | { 146 | Code: 2, 147 | Message: "message2", 148 | }, 149 | } 150 | } 151 | 152 | func confirmSuccess( 153 | t *testing.T, networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, 154 | ) { 155 | assert.NoError(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) 156 | } 157 | 158 | func confirmError( 159 | t *testing.T, networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, 160 | ) { 161 | assert.Error(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) 162 | } 163 | -------------------------------------------------------------------------------- /cmd/view_balance.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "log" 21 | "strconv" 22 | "time" 23 | 24 | "github.com/coinbase/rosetta-sdk-go/asserter" 25 | "github.com/coinbase/rosetta-sdk-go/fetcher" 26 | "github.com/coinbase/rosetta-sdk-go/types" 27 | "github.com/coinbase/rosetta-sdk-go/utils" 28 | "github.com/spf13/cobra" 29 | ) 30 | 31 | var ( 32 | viewAccountCmd = &cobra.Command{ 33 | Use: "view:balance", 34 | Short: "View an account balance", 35 | Long: `While debugging, it is often useful to inspect the state 36 | of an account at a certain block. This command allows you to look up 37 | any account by providing a JSON representation of a types.AccountIdentifier 38 | (and optionally a height to perform the query). 39 | 40 | For example, you could run view:balance '{"address":"interesting address"}' 1000 41 | to lookup the balance of an interesting address at block 1000. Allowing the 42 | address to specified as JSON allows for querying by SubAccountIdentifier.`, 43 | RunE: runViewBalanceCmd, 44 | Args: cobra.MinimumNArgs(1), 45 | } 46 | ) 47 | 48 | func runViewBalanceCmd(cmd *cobra.Command, args []string) error { 49 | account := &types.AccountIdentifier{} 50 | if err := json.Unmarshal([]byte(args[0]), account); err != nil { 51 | return fmt.Errorf("unable to unmarshal account %s: %w", args[0], err) 52 | } 53 | 54 | if err := asserter.AccountIdentifier(account); err != nil { 55 | return fmt.Errorf("invalid account identifier %s: %w", types.PrintStruct(account), err) 56 | } 57 | 58 | // Create a new fetcher 59 | fetcherOpts := []fetcher.Option{ 60 | fetcher.WithMaxConnections(Config.MaxOnlineConnections), 61 | fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), 62 | fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), 63 | fetcher.WithMaxRetries(Config.MaxRetries), 64 | } 65 | if Config.ForceRetry { 66 | fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) 67 | } 68 | 69 | newFetcher := fetcher.New( 70 | Config.OnlineURL, 71 | fetcherOpts..., 72 | ) 73 | 74 | // Initialize the fetcher's asserter 75 | _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) 76 | if fetchErr != nil { 77 | return fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err) 78 | } 79 | 80 | _, err := utils.CheckNetworkSupported(Context, Config.Network, newFetcher) 81 | if err != nil { 82 | return fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err) 83 | } 84 | 85 | var lookupBlock *types.PartialBlockIdentifier 86 | if len(args) > 1 { 87 | index, err := strconv.ParseInt(args[1], 10, 64) 88 | if err != nil { 89 | return fmt.Errorf("unable to parse index %s: %w", args[0], err) 90 | } 91 | 92 | lookupBlock = &types.PartialBlockIdentifier{Index: &index} 93 | } 94 | 95 | block, amounts, metadata, fetchErr := newFetcher.AccountBalanceRetry( 96 | Context, 97 | Config.Network, 98 | account, 99 | lookupBlock, 100 | nil, 101 | ) 102 | if fetchErr != nil { 103 | return fmt.Errorf("unable to fetch account balance for account %s: %w", types.PrintStruct(account), fetchErr.Err) 104 | } 105 | 106 | log.Printf("Amounts: %s\n", types.PrettyPrintStruct(amounts)) 107 | log.Printf("Metadata: %s\n", types.PrettyPrintStruct(metadata)) 108 | log.Printf("Balance Fetched At: %s\n", types.PrettyPrintStruct(block)) 109 | 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /cmd/view_block.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | "time" 21 | 22 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 23 | 24 | "github.com/coinbase/rosetta-sdk-go/fetcher" 25 | "github.com/coinbase/rosetta-sdk-go/parser" 26 | "github.com/coinbase/rosetta-sdk-go/types" 27 | "github.com/coinbase/rosetta-sdk-go/utils" 28 | "github.com/fatih/color" 29 | "github.com/spf13/cobra" 30 | ) 31 | 32 | var ( 33 | viewBlockCmd = &cobra.Command{ 34 | Use: "view:block", 35 | Short: "View a block", 36 | Long: `While debugging a Data API implementation, it can be very 37 | useful to inspect block contents. This command allows you to fetch any 38 | block by index to inspect its contents. It uses the 39 | fetcher (https://github.com/coinbase/rosetta-sdk-go/tree/master/fetcher) package 40 | to automatically get all transactions in the block and assert the format 41 | of the block is correct before printing. 42 | 43 | If this command errors, it is likely because the block you are trying to 44 | fetch is formatted incorrectly.`, 45 | RunE: runViewBlockCmd, 46 | Args: cobra.ExactArgs(1), 47 | } 48 | ) 49 | 50 | func printChanges(balanceChanges []*parser.BalanceChange) error { 51 | for _, balanceChange := range balanceChanges { 52 | parsedDiff, err := types.BigInt(balanceChange.Difference) 53 | if err != nil { 54 | return fmt.Errorf("unable to parse balance change difference: %w", err) 55 | } 56 | 57 | if parsedDiff.Sign() == 0 { 58 | continue 59 | } 60 | 61 | fmt.Println( 62 | types.PrintStruct(balanceChange.Account), 63 | "->", 64 | utils.PrettyAmount(parsedDiff, balanceChange.Currency), 65 | ) 66 | } 67 | 68 | return nil 69 | } 70 | 71 | func runViewBlockCmd(_ *cobra.Command, args []string) error { 72 | index, err := strconv.ParseInt(args[0], 10, 64) 73 | if err != nil { 74 | return fmt.Errorf("unable to parse index %s: %w", args[0], err) 75 | } 76 | 77 | // Create a new fetcher 78 | fetcherOpts := []fetcher.Option{ 79 | fetcher.WithMaxConnections(Config.MaxOnlineConnections), 80 | fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), 81 | fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), 82 | fetcher.WithMaxRetries(Config.MaxRetries), 83 | } 84 | if Config.ForceRetry { 85 | fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) 86 | } 87 | 88 | newFetcher := fetcher.New( 89 | Config.OnlineURL, 90 | fetcherOpts..., 91 | ) 92 | 93 | // Initialize the fetcher's asserter 94 | // 95 | // Behind the scenes this makes a call to get the 96 | // network status and uses the response to inform 97 | // the asserter what are valid responses. 98 | _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) 99 | if fetchErr != nil { 100 | return fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err) 101 | } 102 | 103 | _, err = utils.CheckNetworkSupported(Context, Config.Network, newFetcher) 104 | if err != nil { 105 | return fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err) 106 | } 107 | 108 | // Fetch the specified block with retries (automatically 109 | // asserted for correctness) 110 | // 111 | // On another note, notice that fetcher.BlockRetry 112 | // automatically fetches all transactions that are 113 | // returned in BlockResponse.OtherTransactions. If you use 114 | // the client directly, you will need to implement a mechanism 115 | // to fully populate the block by fetching all these 116 | // transactions. 117 | block, fetchErr := newFetcher.BlockRetry( 118 | Context, 119 | Config.Network, 120 | &types.PartialBlockIdentifier{ 121 | Index: &index, 122 | }, 123 | ) 124 | if fetchErr != nil { 125 | return fmt.Errorf("unable to fetch block %d: %w", index, fetchErr.Err) 126 | } 127 | // It's valid for a block to be omitted without triggering an error 128 | if block == nil { 129 | return cliErrs.ErrBlockNotFound 130 | } 131 | 132 | fmt.Printf("\n") 133 | if !OnlyChanges { 134 | color.Cyan("Current Block:") 135 | fmt.Println(types.PrettyPrintStruct(block)) 136 | } 137 | 138 | // Print out all balance changes in a given block. This does NOT exempt 139 | // any operations/accounts from parsing. 140 | color.Cyan("Balance Changes:") 141 | p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }, nil) 142 | balanceChanges, err := p.BalanceChanges(Context, block, false) 143 | if err != nil { 144 | return fmt.Errorf("unable to calculate balance changes: %w", err) 145 | } 146 | 147 | fmt.Println("Cumulative:", block.BlockIdentifier.Hash) 148 | 149 | if err := printChanges(balanceChanges); err != nil { 150 | return err 151 | } 152 | 153 | fmt.Printf("\n") 154 | 155 | // Print out balance changes by transaction hash 156 | // 157 | // TODO: modify parser to allow for calculating balance 158 | // changes for a single transaction. 159 | for _, tx := range block.Transactions { 160 | balanceChanges, err := p.BalanceChanges(Context, &types.Block{ 161 | Transactions: []*types.Transaction{ 162 | tx, 163 | }, 164 | }, false) 165 | if err != nil { 166 | return fmt.Errorf("unable to calculate balance changes: %w", err) 167 | } 168 | 169 | fmt.Println("Transaction:", tx.TransactionIdentifier.Hash) 170 | 171 | if err := printChanges(balanceChanges); err != nil { 172 | return err 173 | } 174 | fmt.Printf("\n") 175 | } 176 | 177 | if !OnlyChanges { 178 | // Print out all OperationGroups for each transaction in a block. 179 | color.Cyan("Operation Groups:") 180 | for _, tx := range block.Transactions { 181 | fmt.Printf( 182 | "Transaction %s Operation Groups: %s\n", 183 | tx.TransactionIdentifier.Hash, 184 | types.PrettyPrintStruct(parser.GroupOperations(tx)), 185 | ) 186 | } 187 | } 188 | 189 | return nil 190 | } 191 | -------------------------------------------------------------------------------- /cmd/view_networks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "time" 21 | 22 | "github.com/coinbase/rosetta-cli/pkg/errors" 23 | 24 | "github.com/coinbase/rosetta-sdk-go/fetcher" 25 | "github.com/coinbase/rosetta-sdk-go/types" 26 | "github.com/fatih/color" 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | var ( 31 | viewNetworksCmd = &cobra.Command{ 32 | Use: "view:networks", 33 | Short: "View all network statuses", 34 | Long: `While debugging a Data API implementation, it can be very 35 | useful to view network(s) status. This command fetches the network 36 | status from all available networks and prints it to the terminal. 37 | 38 | If this command errors, it is likely because the /network/* endpoints are 39 | not formatted correctly.`, 40 | RunE: runViewNetworksCmd, 41 | } 42 | ) 43 | 44 | func runViewNetworksCmd(cmd *cobra.Command, args []string) error { 45 | fetcherOpts := []fetcher.Option{ 46 | fetcher.WithMaxConnections(Config.MaxOnlineConnections), 47 | fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), 48 | fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), 49 | fetcher.WithMaxRetries(Config.MaxRetries), 50 | } 51 | if Config.ForceRetry { 52 | fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) 53 | } 54 | 55 | f := fetcher.New( 56 | Config.OnlineURL, 57 | fetcherOpts..., 58 | ) 59 | 60 | // Attempt to fetch network list 61 | networkList, fetchErr := f.NetworkListRetry(Context, nil) 62 | if fetchErr != nil { 63 | return fmt.Errorf("unable to get network list: %w", fetchErr.Err) 64 | } 65 | 66 | if len(networkList.NetworkIdentifiers) == 0 { 67 | return errors.ErrNoAvailableNetwork 68 | } 69 | 70 | for _, network := range networkList.NetworkIdentifiers { 71 | color.Cyan(types.PrettyPrintStruct(network)) 72 | networkOptions, fetchErr := f.NetworkOptions( 73 | Context, 74 | network, 75 | nil, 76 | ) 77 | if fetchErr != nil { 78 | return fmt.Errorf("unable to get network options: %w", fetchErr.Err) 79 | } 80 | 81 | log.Printf("Network options: %s\n", types.PrettyPrintStruct(networkOptions)) 82 | 83 | networkStatus, fetchErr := f.NetworkStatusRetry( 84 | Context, 85 | network, 86 | nil, 87 | ) 88 | if fetchErr != nil { 89 | return fmt.Errorf("unable to get network status: %w", fetchErr.Err) 90 | } 91 | 92 | log.Printf("Network status: %s\n", types.PrettyPrintStruct(networkStatus)) 93 | } 94 | 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /configuration/configuration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package configuration 16 | 17 | import ( 18 | "context" 19 | "encoding/hex" 20 | "fmt" 21 | "log" 22 | "path" 23 | "runtime" 24 | "strings" 25 | 26 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 27 | "github.com/coinbase/rosetta-sdk-go/asserter" 28 | "github.com/coinbase/rosetta-sdk-go/constructor/dsl" 29 | "github.com/coinbase/rosetta-sdk-go/constructor/job" 30 | "github.com/coinbase/rosetta-sdk-go/types" 31 | "github.com/coinbase/rosetta-sdk-go/utils" 32 | "github.com/fatih/color" 33 | ) 34 | 35 | // DefaultDataConfiguration returns the default *DataConfiguration 36 | // for running `check:data`. 37 | func DefaultDataConfiguration() *DataConfiguration { 38 | return &DataConfiguration{ 39 | ActiveReconciliationConcurrency: DefaultActiveReconciliationConcurrency, 40 | InactiveReconciliationConcurrency: DefaultInactiveReconciliationConcurrency, 41 | InactiveReconciliationFrequency: DefaultInactiveReconciliationFrequency, 42 | StatusPort: DefaultStatusPort, 43 | } 44 | } 45 | 46 | // DefaultPerfConfiguration returns the default *CheckPerfConfiguration 47 | // for running `check:perf`. 48 | func DefaultPerfConfiguration() *CheckPerfConfiguration { 49 | return &CheckPerfConfiguration{ 50 | StartBlock: 10, 51 | BlockEndpointTimeConstraintMs: 50000000, 52 | AccountBalanceEndpointTimeConstraintMs: 50000000, 53 | EndBlock: 50, 54 | NumTimesToHitEndpoints: 1, 55 | StatsOutputFile: "./check_perf_stats.json", 56 | } 57 | } 58 | 59 | // DefaultConfiguration returns a *Configuration with the 60 | // EthereumNetwork, DefaultURL, DefaultTimeout, and DefaultDataConfiguration. 61 | func DefaultConfiguration() *Configuration { 62 | return &Configuration{ 63 | Network: EthereumNetwork, 64 | OnlineURL: DefaultURL, 65 | MaxOnlineConnections: DefaultMaxOnlineConnections, 66 | HTTPTimeout: DefaultTimeout, 67 | MaxRetries: DefaultMaxRetries, 68 | MaxSyncConcurrency: DefaultMaxSyncConcurrency, 69 | TipDelay: DefaultTipDelay, 70 | MaxReorgDepth: DefaultMaxReorgDepth, 71 | Data: DefaultDataConfiguration(), 72 | } 73 | } 74 | 75 | func populatePerfMissingFields( 76 | perfConfig *CheckPerfConfiguration, 77 | ) *CheckPerfConfiguration { 78 | if perfConfig == nil { 79 | return nil 80 | } 81 | 82 | if len(perfConfig.StatsOutputFile) == 0 { 83 | perfConfig.StatsOutputFile = DefaultOutputFile 84 | } 85 | 86 | if perfConfig.NumTimesToHitEndpoints == 0 { 87 | perfConfig.NumTimesToHitEndpoints = DefaultNumTimesToHitEndpoints 88 | } 89 | 90 | return perfConfig 91 | } 92 | 93 | func populateConstructionMissingFields( 94 | constructionConfig *ConstructionConfiguration, 95 | ) *ConstructionConfiguration { 96 | if constructionConfig == nil { 97 | return nil 98 | } 99 | 100 | if len(constructionConfig.OfflineURL) == 0 { 101 | constructionConfig.OfflineURL = DefaultURL 102 | } 103 | 104 | if constructionConfig.MaxOfflineConnections == 0 { 105 | constructionConfig.MaxOfflineConnections = DefaultMaxOfflineConnections 106 | } 107 | 108 | if constructionConfig.StaleDepth == 0 { 109 | constructionConfig.StaleDepth = DefaultStaleDepth 110 | } 111 | 112 | if constructionConfig.BroadcastLimit == 0 { 113 | constructionConfig.BroadcastLimit = DefaultBroadcastLimit 114 | } 115 | 116 | if constructionConfig.BlockBroadcastLimit == 0 { 117 | constructionConfig.BlockBroadcastLimit = DefaultBlockBroadcastLimit 118 | } 119 | 120 | if constructionConfig.StatusPort == 0 { 121 | constructionConfig.StatusPort = DefaultStatusPort 122 | } 123 | 124 | return constructionConfig 125 | } 126 | 127 | func populateDataMissingFields(dataConfig *DataConfiguration) *DataConfiguration { 128 | if dataConfig == nil { 129 | return DefaultDataConfiguration() 130 | } 131 | 132 | if dataConfig.ActiveReconciliationConcurrency == 0 { 133 | dataConfig.ActiveReconciliationConcurrency = DefaultActiveReconciliationConcurrency 134 | } 135 | 136 | if dataConfig.InactiveReconciliationConcurrency == 0 { 137 | dataConfig.InactiveReconciliationConcurrency = DefaultInactiveReconciliationConcurrency 138 | } 139 | 140 | if dataConfig.InactiveReconciliationFrequency == 0 { 141 | dataConfig.InactiveReconciliationFrequency = DefaultInactiveReconciliationFrequency 142 | } 143 | 144 | if dataConfig.StatusPort == 0 { 145 | dataConfig.StatusPort = DefaultStatusPort 146 | } 147 | 148 | return dataConfig 149 | } 150 | 151 | func populateMissingFields(config *Configuration) *Configuration { 152 | if config == nil { 153 | return DefaultConfiguration() 154 | } 155 | 156 | if config.Network == nil { 157 | config.Network = EthereumNetwork 158 | } 159 | 160 | if len(config.OnlineURL) == 0 { 161 | config.OnlineURL = DefaultURL 162 | } 163 | 164 | if config.HTTPTimeout == 0 { 165 | config.HTTPTimeout = DefaultTimeout 166 | } 167 | 168 | if config.MaxRetries == 0 { 169 | config.MaxRetries = DefaultMaxRetries 170 | } 171 | 172 | if config.MaxOnlineConnections == 0 { 173 | config.MaxOnlineConnections = DefaultMaxOnlineConnections 174 | } 175 | 176 | if config.MaxSyncConcurrency == 0 { 177 | config.MaxSyncConcurrency = DefaultMaxSyncConcurrency 178 | } 179 | 180 | if config.TipDelay == 0 { 181 | config.TipDelay = DefaultTipDelay 182 | } 183 | 184 | if config.MaxReorgDepth == 0 { 185 | config.MaxReorgDepth = DefaultMaxReorgDepth 186 | } 187 | 188 | numCPU := runtime.NumCPU() 189 | if config.SeenBlockWorkers == 0 { 190 | config.SeenBlockWorkers = numCPU 191 | } 192 | 193 | if config.SerialBlockWorkers == 0 { 194 | config.SerialBlockWorkers = numCPU 195 | } 196 | 197 | if len(strings.TrimSpace(config.ValidationFile)) == 0 { 198 | config.ValidationFile = "" 199 | } 200 | 201 | config.Construction = populateConstructionMissingFields(config.Construction) 202 | config.Data = populateDataMissingFields(config.Data) 203 | config.Perf = populatePerfMissingFields(config.Perf) 204 | 205 | return config 206 | } 207 | 208 | func assertConstructionConfiguration(ctx context.Context, config *ConstructionConfiguration) error { 209 | if config == nil { 210 | return nil 211 | } 212 | 213 | if len(config.Workflows) > 0 && len(config.ConstructorDSLFile) > 0 { 214 | return cliErrs.ErrMultipleDSLFiles 215 | } 216 | 217 | if len(config.Workflows) == 0 && len(config.ConstructorDSLFile) == 0 { 218 | return cliErrs.ErrNoDSLFile 219 | } 220 | 221 | // Compile ConstructorDSLFile and save to Workflows 222 | if len(config.ConstructorDSLFile) > 0 { 223 | compiledWorkflows, err := dsl.Parse(ctx, config.ConstructorDSLFile) 224 | if err != nil { 225 | err.Log() 226 | return fmt.Errorf("DSL file is invalid, line %d, line contents %s: %w", err.Line, err.LineContents, err.Err) 227 | } 228 | 229 | config.Workflows = compiledWorkflows 230 | } 231 | 232 | // Parse provided Workflows 233 | for _, workflow := range config.Workflows { 234 | if workflow.Name == string(job.CreateAccount) || workflow.Name == string(job.RequestFunds) { 235 | if workflow.Concurrency != job.ReservedWorkflowConcurrency { 236 | return fmt.Errorf( 237 | "DSL file is invalid, reserved workflow %s must have concurrency %d: %w", 238 | workflow.Name, 239 | job.ReservedWorkflowConcurrency, 240 | cliErrs.ErrWrongWorkflowConcurrency, 241 | ) 242 | } 243 | } 244 | } 245 | 246 | for _, account := range config.PrefundedAccounts { 247 | // Checks that privkey is hex encoded 248 | _, err := hex.DecodeString(account.PrivateKeyHex) 249 | if err != nil { 250 | return fmt.Errorf( 251 | "private key %s is not hex encoded for prefunded account: %w", 252 | account.PrivateKeyHex, 253 | err, 254 | ) 255 | } 256 | 257 | // Checks if valid CurveType 258 | if err := asserter.CurveType(account.CurveType); err != nil { 259 | return fmt.Errorf("prefunded account curve type %s is invalid: %w", types.PrintStruct(account.CurveType), err) 260 | } 261 | 262 | // Checks if valid AccountIdentifier 263 | if err := asserter.AccountIdentifier(account.AccountIdentifier); err != nil { 264 | return fmt.Errorf("prefunded account identifier %s is invalid: %w", types.PrintStruct(account.AccountIdentifier), err) 265 | } 266 | 267 | // Check if valid Currency when Currency is specified 268 | // If Currency is not specified, the balances of all available currencies 269 | // for the specific pre-funded account will be stored in the balance storage 270 | if account.Currency != nil { 271 | err = asserter.Currency(account.Currency) 272 | if err != nil { 273 | return fmt.Errorf("prefunded account currency %s is invalid: %w", types.PrintStruct(account.Currency), err) 274 | } 275 | } 276 | } 277 | 278 | return nil 279 | } 280 | 281 | func assertDataConfiguration(config *DataConfiguration) error { // nolint:gocognit 282 | if config.StartIndex != nil && *config.StartIndex < 0 { 283 | return fmt.Errorf("start index %d is invalid: %w", *config.StartIndex, cliErrs.ErrNegativeStartIndex) 284 | } 285 | 286 | if !config.ReconciliationDisabled && config.BalanceTrackingDisabled { 287 | return cliErrs.ErrBalanceTrackingIsDisabledForReconciliation 288 | } 289 | 290 | if config.EndConditions == nil { 291 | return nil 292 | } 293 | 294 | if config.EndConditions.Index != nil { 295 | if *config.EndConditions.Index < 0 { 296 | return fmt.Errorf("end index %d is invalid: %w", *config.EndConditions.Index, cliErrs.ErrNegativeEndIndex) 297 | } 298 | } 299 | 300 | if config.EndConditions.ReconciliationCoverage != nil { 301 | coverage := config.EndConditions.ReconciliationCoverage.Coverage 302 | if coverage < 0 || coverage > 1 { 303 | return fmt.Errorf("reconciliation coverage %f is invalid: %w", coverage, cliErrs.ErrReconciliationOutOfRange) 304 | } 305 | 306 | index := config.EndConditions.ReconciliationCoverage.Index 307 | if index != nil && *index < 0 { 308 | return fmt.Errorf("reconciliation coverage index %d is invalid: %w", *index, cliErrs.ErrNegativeReconciliationCoverageIndex) 309 | } 310 | 311 | accountCount := config.EndConditions.ReconciliationCoverage.AccountCount 312 | if accountCount != nil && *accountCount < 0 { 313 | return fmt.Errorf( 314 | "reconciliation coverage account count %d is invalid: %w", 315 | *accountCount, 316 | cliErrs.ErrNegativeReconciliationCoverageAccountCount, 317 | ) 318 | } 319 | 320 | if config.BalanceTrackingDisabled { 321 | return cliErrs.ErrBalanceTrackingIsDisabledForReconciliationCoverageEndCondition 322 | } 323 | 324 | if config.IgnoreReconciliationError { 325 | return cliErrs.ErrReconciliationErrorIsIgnoredForReconciliationCoverageEndCondition 326 | } 327 | 328 | if config.ReconciliationDisabled { 329 | return cliErrs.ErrReconciliationIsDisabledForReconciliationCoverageEndCondition 330 | } 331 | } 332 | 333 | return nil 334 | } 335 | 336 | func assertConfiguration(ctx context.Context, config *Configuration) error { 337 | if err := asserter.NetworkIdentifier(config.Network); err != nil { 338 | return fmt.Errorf("invalid network identifier %s: %w", types.PrintStruct(config.Network), err) 339 | } 340 | 341 | if config.SeenBlockWorkers <= 0 { 342 | return fmt.Errorf("the number of seen block workers %d is invalid: %w", config.SeenBlockWorkers, cliErrs.ErrNegativeSeenBlockWorkers) 343 | } 344 | 345 | if config.SerialBlockWorkers <= 0 { 346 | return fmt.Errorf("the number of serial block workers %d is invalid: %w", config.SerialBlockWorkers, cliErrs.ErrNegativeSerialBlockWorkers) 347 | } 348 | 349 | if config.TableSize != nil && (*config.TableSize < 1 || *config.TableSize > 100) { 350 | return fmt.Errorf("table size %d is invalid: %w", *config.TableSize, cliErrs.ErrTableSizeIsOutOfRange) 351 | } 352 | 353 | if config.ValueLogFileSize != nil && (*config.ValueLogFileSize < 128 || *config.ValueLogFileSize > 2048) { 354 | return fmt.Errorf("value log file size %d is invalid: %w", *config.ValueLogFileSize, cliErrs.ErrValueLogFileSizeIsOutOfRange) 355 | } 356 | 357 | if err := assertDataConfiguration(config.Data); err != nil { 358 | return fmt.Errorf("data configuration is invalid: %w", err) 359 | } 360 | 361 | if err := assertConstructionConfiguration(ctx, config.Construction); err != nil { 362 | return fmt.Errorf("construction configuration is invalid: %w", err) 363 | } 364 | 365 | return nil 366 | } 367 | 368 | // modifyFilePaths modifies a collection of filepaths in a *Configuration 369 | // file to make them relative to the configuration file (this makes it a lot easier 370 | // to store all config-related files in the same directory and to run the rosetta-cli 371 | // from a different directory). 372 | func modifyFilePaths(config *Configuration, fileDir string) { 373 | if config.Data != nil { 374 | if len(config.Data.BootstrapBalances) > 0 { 375 | config.Data.BootstrapBalances = path.Join(fileDir, config.Data.BootstrapBalances) 376 | } 377 | 378 | if len(config.Data.InterestingAccounts) > 0 { 379 | config.Data.InterestingAccounts = path.Join(fileDir, config.Data.InterestingAccounts) 380 | } 381 | 382 | if len(config.Data.ExemptAccounts) > 0 { 383 | config.Data.ExemptAccounts = path.Join(fileDir, config.Data.ExemptAccounts) 384 | } 385 | } 386 | 387 | if config.Construction != nil { 388 | if len(config.Construction.ConstructorDSLFile) > 0 { 389 | config.Construction.ConstructorDSLFile = path.Join( 390 | fileDir, 391 | config.Construction.ConstructorDSLFile, 392 | ) 393 | } 394 | } 395 | 396 | if len(config.ValidationFile) > 0 { 397 | config.ValidationFile = path.Join(fileDir, config.ValidationFile) 398 | } 399 | } 400 | 401 | // LoadConfiguration returns a parsed and asserted Configuration for running 402 | // tests. 403 | func LoadConfiguration(ctx context.Context, filePath string) (*Configuration, error) { 404 | var configRaw Configuration 405 | if err := utils.LoadAndParse(filePath, &configRaw); err != nil { 406 | return nil, fmt.Errorf("unable to load and parse configuration file: %w", err) 407 | } 408 | 409 | config := populateMissingFields(&configRaw) 410 | 411 | // Get the configuration file directory so we can load all files 412 | // relative to the location of the configuration file. 413 | fileDir := path.Dir(filePath) 414 | modifyFilePaths(config, fileDir) 415 | 416 | if err := assertConfiguration(ctx, config); err != nil { 417 | return nil, fmt.Errorf("configuration is invalid: %w", err) 418 | } 419 | 420 | color.Cyan( 421 | "loaded configuration file: %s\n", 422 | filePath, 423 | ) 424 | 425 | if config.LogConfiguration { 426 | log.Println(types.PrettyPrintStruct(config)) 427 | } 428 | 429 | return config, nil 430 | } 431 | -------------------------------------------------------------------------------- /configuration/configuration_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package configuration 16 | 17 | import ( 18 | "context" 19 | "os/exec" 20 | "path" 21 | "runtime" 22 | "testing" 23 | 24 | "github.com/coinbase/rosetta-sdk-go/constructor/job" 25 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 26 | "github.com/coinbase/rosetta-sdk-go/types" 27 | "github.com/coinbase/rosetta-sdk-go/utils" 28 | "github.com/stretchr/testify/assert" 29 | ) 30 | 31 | var ( 32 | startIndex = int64(89) 33 | badStartIndex = int64(-10) 34 | goodCoverage = float64(0.33) 35 | badCoverage = float64(-2) 36 | endTip = false 37 | historicalDisabled = false 38 | fakeWorkflows = []*job.Workflow{ 39 | { 40 | Name: string(job.CreateAccount), 41 | Concurrency: job.ReservedWorkflowConcurrency, 42 | Scenarios: []*job.Scenario{ 43 | { 44 | Name: "blah", 45 | Actions: []*job.Action{}, 46 | }, 47 | }, 48 | }, 49 | { 50 | Name: string(job.RequestFunds), 51 | Concurrency: job.ReservedWorkflowConcurrency, 52 | Scenarios: []*job.Scenario{ 53 | { 54 | Name: "blah", 55 | Actions: []*job.Action{}, 56 | }, 57 | }, 58 | }, 59 | } 60 | whackyConfig = &Configuration{ 61 | Network: &types.NetworkIdentifier{ 62 | Blockchain: "sweet", 63 | Network: "sweeter", 64 | }, 65 | OnlineURL: "http://hasudhasjkdk", 66 | MaxOnlineConnections: 10, 67 | HTTPTimeout: 21, 68 | MaxRetries: 1000, 69 | MaxSyncConcurrency: 12, 70 | TipDelay: 1231, 71 | MaxReorgDepth: 12, 72 | SeenBlockWorkers: 300, 73 | SerialBlockWorkers: 200, 74 | ErrorStackTraceDisabled: false, 75 | Construction: &ConstructionConfiguration{ 76 | OfflineURL: "https://ashdjaksdkjshdk", 77 | MaxOfflineConnections: 21, 78 | StaleDepth: 12, 79 | BroadcastLimit: 200, 80 | BlockBroadcastLimit: 992, 81 | StatusPort: 21, 82 | Workflows: append( 83 | fakeWorkflows, 84 | &job.Workflow{ 85 | Name: "transfer", 86 | Concurrency: 100, 87 | }, 88 | ), 89 | }, 90 | Data: &DataConfiguration{ 91 | ActiveReconciliationConcurrency: 100, 92 | InactiveReconciliationConcurrency: 2938, 93 | InactiveReconciliationFrequency: 3, 94 | ReconciliationDisabled: false, 95 | HistoricalBalanceDisabled: &historicalDisabled, 96 | StartIndex: &startIndex, 97 | StatusPort: 123, 98 | EndConditions: &DataEndConditions{ 99 | ReconciliationCoverage: &ReconciliationCoverage{ 100 | Coverage: goodCoverage, 101 | }, 102 | }, 103 | }, 104 | } 105 | invalidNetwork = &Configuration{ 106 | Network: &types.NetworkIdentifier{ 107 | Blockchain: "?", 108 | }, 109 | } 110 | invalidPrefundedAccounts = &Configuration{ 111 | Construction: &ConstructionConfiguration{ 112 | PrefundedAccounts: []*modules.PrefundedAccount{ 113 | { 114 | PrivateKeyHex: "hello", 115 | }, 116 | }, 117 | }, 118 | } 119 | invalidStartIndex = &Configuration{ 120 | Data: &DataConfiguration{ 121 | StartIndex: &badStartIndex, 122 | }, 123 | } 124 | multipleEndConditions = &Configuration{ 125 | Data: &DataConfiguration{ 126 | EndConditions: &DataEndConditions{ 127 | Index: &startIndex, 128 | Tip: &endTip, 129 | }, 130 | }, 131 | } 132 | invalidEndIndex = &Configuration{ 133 | Data: &DataConfiguration{ 134 | EndConditions: &DataEndConditions{ 135 | Index: &badStartIndex, 136 | }, 137 | }, 138 | } 139 | invalidReconciliationCoverage = &Configuration{ 140 | Data: &DataConfiguration{ 141 | EndConditions: &DataEndConditions{ 142 | ReconciliationCoverage: &ReconciliationCoverage{ 143 | Coverage: badCoverage, 144 | }, 145 | }, 146 | }, 147 | } 148 | ) 149 | 150 | func TestLoadConfiguration(t *testing.T) { 151 | var ( 152 | goodAccountCount = int64(10) 153 | badAccountCount = int64(-10) 154 | ) 155 | var tests = map[string]struct { 156 | provided *Configuration 157 | expected *Configuration 158 | 159 | err bool 160 | }{ 161 | "nothing provided": { 162 | provided: &Configuration{}, 163 | expected: func() *Configuration { 164 | cfg := DefaultConfiguration() 165 | cfg.SeenBlockWorkers = runtime.NumCPU() 166 | cfg.SerialBlockWorkers = runtime.NumCPU() 167 | 168 | return cfg 169 | }(), 170 | }, 171 | "no overwrite": { 172 | provided: whackyConfig, 173 | expected: whackyConfig, 174 | }, 175 | "overwrite missing": { 176 | provided: &Configuration{ 177 | Construction: &ConstructionConfiguration{ 178 | Workflows: fakeWorkflows, 179 | }, 180 | Data: &DataConfiguration{}, 181 | }, 182 | expected: func() *Configuration { 183 | cfg := DefaultConfiguration() 184 | cfg.SeenBlockWorkers = runtime.NumCPU() 185 | cfg.SerialBlockWorkers = runtime.NumCPU() 186 | cfg.Construction = &ConstructionConfiguration{ 187 | OfflineURL: DefaultURL, 188 | MaxOfflineConnections: DefaultMaxOfflineConnections, 189 | StaleDepth: DefaultStaleDepth, 190 | BroadcastLimit: DefaultBroadcastLimit, 191 | BlockBroadcastLimit: DefaultBlockBroadcastLimit, 192 | StatusPort: DefaultStatusPort, 193 | Workflows: fakeWorkflows, 194 | } 195 | 196 | return cfg 197 | }(), 198 | }, 199 | "overwrite missing with DSL": { 200 | provided: &Configuration{ 201 | Construction: &ConstructionConfiguration{ 202 | ConstructorDSLFile: "test.ros", 203 | }, 204 | Data: &DataConfiguration{}, 205 | }, 206 | expected: func() *Configuration { 207 | cfg := DefaultConfiguration() 208 | cfg.SeenBlockWorkers = runtime.NumCPU() 209 | cfg.SerialBlockWorkers = runtime.NumCPU() 210 | cfg.Construction = &ConstructionConfiguration{ 211 | OfflineURL: DefaultURL, 212 | MaxOfflineConnections: DefaultMaxOfflineConnections, 213 | StaleDepth: DefaultStaleDepth, 214 | BroadcastLimit: DefaultBroadcastLimit, 215 | BlockBroadcastLimit: DefaultBlockBroadcastLimit, 216 | StatusPort: DefaultStatusPort, 217 | Workflows: fakeWorkflows, 218 | ConstructorDSLFile: "test.ros", 219 | } 220 | 221 | return cfg 222 | }(), 223 | }, 224 | "transfer workflow": { 225 | provided: &Configuration{ 226 | Construction: &ConstructionConfiguration{ 227 | Workflows: []*job.Workflow{ 228 | { 229 | Name: "transfer", 230 | Concurrency: 10, 231 | }, 232 | }, 233 | }, 234 | Data: &DataConfiguration{}, 235 | }, 236 | expected: func() *Configuration { 237 | cfg := DefaultConfiguration() 238 | cfg.SeenBlockWorkers = runtime.NumCPU() 239 | cfg.SerialBlockWorkers = runtime.NumCPU() 240 | cfg.Construction = &ConstructionConfiguration{ 241 | OfflineURL: DefaultURL, 242 | MaxOfflineConnections: DefaultMaxOfflineConnections, 243 | StaleDepth: DefaultStaleDepth, 244 | BroadcastLimit: DefaultBroadcastLimit, 245 | BlockBroadcastLimit: DefaultBlockBroadcastLimit, 246 | StatusPort: DefaultStatusPort, 247 | Workflows: []*job.Workflow{ 248 | { 249 | Name: "transfer", 250 | Concurrency: 10, 251 | }, 252 | }, 253 | } 254 | 255 | return cfg 256 | }(), 257 | }, 258 | "invalid network": { 259 | provided: invalidNetwork, 260 | err: true, 261 | }, 262 | "invalid prefunded accounts": { 263 | provided: invalidPrefundedAccounts, 264 | err: true, 265 | }, 266 | "invalid start index": { 267 | provided: invalidStartIndex, 268 | err: true, 269 | }, 270 | "invalid end index": { 271 | provided: invalidEndIndex, 272 | err: true, 273 | }, 274 | "invalid reconciliation coverage": { 275 | provided: invalidReconciliationCoverage, 276 | err: true, 277 | }, 278 | "invalid reconciliation coverage (reconciliation disabled)": { 279 | provided: &Configuration{ 280 | Data: &DataConfiguration{ 281 | ReconciliationDisabled: true, 282 | EndConditions: &DataEndConditions{ 283 | ReconciliationCoverage: &ReconciliationCoverage{ 284 | Coverage: goodCoverage, 285 | }, 286 | }, 287 | }, 288 | }, 289 | err: true, 290 | }, 291 | "invalid reconciliation coverage (balance tracking disabled)": { 292 | provided: &Configuration{ 293 | Data: &DataConfiguration{ 294 | BalanceTrackingDisabled: true, 295 | EndConditions: &DataEndConditions{ 296 | ReconciliationCoverage: &ReconciliationCoverage{ 297 | Coverage: goodCoverage, 298 | }, 299 | }, 300 | }, 301 | }, 302 | err: true, 303 | }, 304 | "invalid reconciliation coverage (ignore reconciliation error)": { 305 | provided: &Configuration{ 306 | Data: &DataConfiguration{ 307 | IgnoreReconciliationError: true, 308 | EndConditions: &DataEndConditions{ 309 | ReconciliationCoverage: &ReconciliationCoverage{ 310 | Coverage: goodCoverage, 311 | }, 312 | }, 313 | }, 314 | }, 315 | err: true, 316 | }, 317 | "valid reconciliation coverage (with account count)": { 318 | provided: &Configuration{ 319 | Data: &DataConfiguration{ 320 | EndConditions: &DataEndConditions{ 321 | ReconciliationCoverage: &ReconciliationCoverage{ 322 | Coverage: goodCoverage, 323 | AccountCount: &goodAccountCount, 324 | Index: &goodAccountCount, 325 | }, 326 | }, 327 | }, 328 | }, 329 | expected: func() *Configuration { 330 | cfg := DefaultConfiguration() 331 | cfg.SeenBlockWorkers = runtime.NumCPU() 332 | cfg.SerialBlockWorkers = runtime.NumCPU() 333 | cfg.Data.EndConditions = &DataEndConditions{ 334 | ReconciliationCoverage: &ReconciliationCoverage{ 335 | Coverage: goodCoverage, 336 | AccountCount: &goodAccountCount, 337 | Index: &goodAccountCount, 338 | }, 339 | } 340 | 341 | return cfg 342 | }(), 343 | }, 344 | "invalid reconciliation coverage (with account count)": { 345 | provided: &Configuration{ 346 | Data: &DataConfiguration{ 347 | EndConditions: &DataEndConditions{ 348 | ReconciliationCoverage: &ReconciliationCoverage{ 349 | Coverage: goodCoverage, 350 | AccountCount: &badAccountCount, 351 | }, 352 | }, 353 | }, 354 | }, 355 | err: true, 356 | }, 357 | "invalid reconciliation coverage (with index)": { 358 | provided: &Configuration{ 359 | Data: &DataConfiguration{ 360 | EndConditions: &DataEndConditions{ 361 | ReconciliationCoverage: &ReconciliationCoverage{ 362 | Coverage: goodCoverage, 363 | Index: &badAccountCount, 364 | }, 365 | }, 366 | }, 367 | }, 368 | err: true, 369 | }, 370 | "empty workflows": { 371 | provided: &Configuration{ 372 | Construction: &ConstructionConfiguration{ 373 | Workflows: []*job.Workflow{}, 374 | }, 375 | }, 376 | err: true, 377 | }, 378 | "non-existent dsl file": { 379 | provided: &Configuration{ 380 | Construction: &ConstructionConfiguration{ 381 | ConstructorDSLFile: "blah.ros", 382 | }, 383 | }, 384 | err: true, 385 | }, 386 | "multiple end conditions": { 387 | provided: multipleEndConditions, 388 | expected: func() *Configuration { 389 | def := DefaultConfiguration() 390 | def.SeenBlockWorkers = runtime.NumCPU() 391 | def.SerialBlockWorkers = runtime.NumCPU() 392 | def.Data.EndConditions = multipleEndConditions.Data.EndConditions 393 | 394 | return def 395 | }(), 396 | err: false, 397 | }, 398 | } 399 | 400 | for name, test := range tests { 401 | t.Run(name, func(t *testing.T) { 402 | dir, err := utils.CreateTempDir() 403 | assert.NoError(t, err) 404 | defer utils.RemoveTempDir(dir) 405 | 406 | filePath := path.Join(dir, "test.json") 407 | err = utils.SerializeAndWrite(filePath, test.provided) 408 | assert.NoError(t, err) 409 | 410 | // Copy test.ros to temp dir 411 | cmd := exec.Command("cp", "testdata/test.ros", path.Join(dir, "test.ros")) 412 | assert.NoError(t, cmd.Run()) 413 | 414 | // Check if expected fields populated 415 | config, err := LoadConfiguration(context.Background(), filePath) 416 | if test.err { 417 | assert.Error(t, err) 418 | assert.Nil(t, config) 419 | } else { 420 | assert.NoError(t, err) 421 | 422 | // Ensure test.ros expected file path is right 423 | if test.expected.Construction != nil && len(test.expected.Construction.ConstructorDSLFile) > 0 { 424 | test.expected.Construction.ConstructorDSLFile = path.Join(dir, test.expected.Construction.ConstructorDSLFile) 425 | } 426 | assert.Equal(t, test.expected, config) 427 | } 428 | }) 429 | } 430 | } 431 | -------------------------------------------------------------------------------- /configuration/testdata/test.ros: -------------------------------------------------------------------------------- 1 | create_account(1){ 2 | blah{ 3 | } 4 | } 5 | 6 | request_funds(1){ 7 | blah{ 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/bootstrap_balances.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "account_identifier": { 4 | "address":"1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa" 5 | }, 6 | "currency":{ 7 | "symbol":"BTC", 8 | "decimals":8 9 | }, 10 | "value": "5000000000" 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /examples/configuration/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "network": { 3 | "blockchain": "Ethereum", 4 | "network": "Ropsten" 5 | }, 6 | "online_url": "http://localhost:8080", 7 | "data_directory": "", 8 | "http_timeout": 10, 9 | "max_retries": 5, 10 | "retry_elapsed_time": 0, 11 | "max_online_connections": 120, 12 | "max_sync_concurrency": 64, 13 | "tip_delay": 300, 14 | "max_reorg_depth": 100, 15 | "log_configuration": false, 16 | "compression_disabled": false, 17 | "l0_in_memory_enabled": false, 18 | "all_in_memory_enabled": false, 19 | "error_stack_trace_disabled": false, 20 | "coin_supported": false, 21 | "construction": null, 22 | "data": { 23 | "active_reconciliation_concurrency": 16, 24 | "inactive_reconciliation_concurrency": 4, 25 | "inactive_reconciliation_frequency": 250, 26 | "log_blocks": false, 27 | "log_transactions": false, 28 | "log_balance_changes": false, 29 | "log_reconciliations": false, 30 | "ignore_reconciliation_error": false, 31 | "exempt_accounts": "", 32 | "bootstrap_balances": "", 33 | "interesting_accounts": "", 34 | "reconciliation_disabled": false, 35 | "reconciliation_drain_disabled": false, 36 | "inactive_discrepancy_search_disabled": false, 37 | "balance_tracking_disabled": false, 38 | "coin_tracking_disabled": false, 39 | "status_port": 9090, 40 | "results_output_file": "", 41 | "pruning_block_disabled": false, 42 | "pruning_balance_disabled": false, 43 | "initial_balance_fetch_disabled": false 44 | }, 45 | "perf": null, 46 | "sign": null 47 | } -------------------------------------------------------------------------------- /examples/configuration/sign.json: -------------------------------------------------------------------------------- 1 | { 2 | "sign": { 3 | "pub_key": { 4 | "curve_type": "secp256k1", 5 | "hex_bytes": "03c7e625aa08cad8f257d9ee2b9b7a0214f19f981afd5b498c728ad7ed6c0c3df6" 6 | }, 7 | "private_key": "", 8 | "signing_payload": { 9 | "hex_bytes": "370e74254e8cbaa343af3564901456082ec7af967e45ff24ba061233b1a1b04f", 10 | "signature_type": "ecdsa", 11 | "account_identifier": { 12 | "address": "dummy" 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /examples/configuration/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "network": { 3 | "blockchain": "Ethereum", 4 | "network": "Ropsten" 5 | }, 6 | "online_url": "http://localhost:8080", 7 | "data_directory": "", 8 | "http_timeout": 10, 9 | "tip_delay": 300, 10 | "data": { 11 | "historical_balance_disabled": true, 12 | "reconciliation_disabled": true, 13 | "inactive_discrepancy_search_disabled": true, 14 | "balance_tracking_disabled": true, 15 | "end_conditions": { 16 | "tip": true 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/configuration/verify.json: -------------------------------------------------------------------------------- 1 | { 2 | "sign": { 3 | "pub_key": { 4 | "curve_type": "secp256k1", 5 | "hex_bytes": "03c7e625aa08cad8f257d9ee2b9b7a0214f19f981afd5b498c728ad7ed6c0c3df6" 6 | }, 7 | "signing_payload": { 8 | "hex_bytes": "370e74254e8cbaa343af3564901456082ec7af967e45ff24ba061233b1a1b04f", 9 | "signature_type": "ecdsa", 10 | "account_identifier": { 11 | "address": "dummy" 12 | } 13 | }, 14 | "signature": { 15 | "hex_bytes": "c80547470b7e4d3fc17c988b2244dfebc909b3e9f7fd0c1387763263cc70d16d24f326b9c12ba2ea278164c0b30f128a809585fc503eda43de429aadb9f893ef" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /examples/exempt_accounts.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "account_identifier": { 4 | "address":"0x379fC39D8744ED0C1c8BCfd86771338b9086660D" 5 | }, 6 | "currency": { 7 | "symbol": "ETH", 8 | "decimals": 18 9 | } 10 | } 11 | ] 12 | -------------------------------------------------------------------------------- /examples/interesting_accounts.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "account_identifier": { 4 | "address":"0x379fC39D8744ED0C1c8BCfd86771338b9086660D" 5 | }, 6 | "currency": { 7 | "symbol": "ETH", 8 | "decimals": 18 9 | } 10 | } 11 | ] 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/coinbase/rosetta-cli 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/coinbase/rosetta-sdk-go v0.8.3 7 | github.com/coinbase/rosetta-sdk-go/types v1.0.0 8 | github.com/fatih/color v1.13.0 9 | github.com/google/go-cmp v0.5.6 // indirect 10 | github.com/mattn/go-colorable v0.1.12 // indirect 11 | github.com/olekukonko/tablewriter v0.0.5 12 | github.com/pkg/errors v0.9.1 13 | github.com/spf13/cobra v1.4.0 14 | github.com/stretchr/testify v1.7.2 15 | go.uber.org/zap v1.21.0 16 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 17 | google.golang.org/protobuf v1.27.1 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "os" 19 | 20 | "github.com/coinbase/rosetta-cli/cmd" 21 | 22 | "github.com/fatih/color" 23 | ) 24 | 25 | func main() { 26 | err := cmd.Execute() 27 | if err != nil { 28 | color.Red("Command Failed: %s", err.Error()) 29 | os.Exit(1) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pkg/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors 16 | 17 | import ( 18 | "errors" 19 | ) 20 | 21 | var ( 22 | // Configuration errors 23 | ErrMultipleDSLFiles = errors.New("multiple DSL files are found") 24 | ErrNoDSLFile = errors.New("no DSL file") 25 | ErrWrongWorkflowConcurrency = errors.New("reserved workflow concurrency doesn't match") 26 | ErrNegativeStartIndex = errors.New("start index is negative") 27 | ErrNegativeEndIndex = errors.New("end index is negative") 28 | ErrNegativeReconciliationCoverageIndex = errors.New("reconciliation coverage index is negative") 29 | ErrNegativeReconciliationCoverageAccountCount = errors.New("reconciliation coverage account is negative") 30 | ErrNegativeSeenBlockWorkers = errors.New("the number of seen block workers is negative") 31 | ErrNegativeSerialBlockWorkers = errors.New("the number of serial block workers is negative") 32 | ErrReconciliationOutOfRange = errors.New("reconciliation is out of range, it must be in the range [0, 1]") 33 | ErrTableSizeIsOutOfRange = errors.New("table size is out of range, it must be in the range [1, 100]") 34 | ErrValueLogFileSizeIsOutOfRange = errors.New("value log file size is out of range, it must be in the range [128, 2048]") 35 | ErrBalanceTrackingIsDisabledForReconciliation = errors.New("balance tracking cannot be disabled for reconciliation") 36 | ErrBalanceTrackingIsDisabledForReconciliationCoverageEndCondition = errors.New("balance tracking cannot be disabled for reconciliation coverage end condition") 37 | ErrReconciliationErrorIsIgnoredForReconciliationCoverageEndCondition = errors.New("reconciliation error cannot be ignored for reconciliation coverage end condition") 38 | ErrReconciliationIsDisabledForReconciliationCoverageEndCondition = errors.New("reconciliation cannot be disabled for reconciliation coverage end condition") 39 | ErrConstructionConfigMissing = errors.New("construction configuration is missing") 40 | 41 | // Data check errors 42 | ErrDataCheckHalt = errors.New("data check halted") 43 | ErrReconciliationFailure = errors.New("reconciliation failure") 44 | ErrInitDataTester = errors.New("unexpected error occurred while trying to initialize data tester") 45 | ErrReconcilerDrainHalt = errors.New("reconciler queue drain halted") 46 | ErrMissingOps = errors.New("search for block with missing ops halted") 47 | ErrUnableToFindMissingOps = errors.New("unable to find missing ops") 48 | 49 | // Spec check errors 50 | ErrErrorEmptyMessage = errors.New("error object can't have empty message") 51 | ErrErrorNegativeCode = errors.New("error object can't have negative code") 52 | ErrAccountNullPointer = errors.New("account is nil") 53 | ErrBlockNotIdempotent = errors.New("multiple calls with the same hash don't return the same block") 54 | ErrBlockTip = errors.New("unspecified block_identifier doesn't give the tip block") 55 | 56 | // Construction check errors 57 | ErrConstructionCheckHalt = errors.New("construction check halted") 58 | ErrBalanceExemptionsWithInitialBalanceFetchDisabled = errors.New("found balance exemptions but initial balance fetch disabled") 59 | 60 | // Command errors 61 | ErrBlockNotFound = errors.New("block not found") 62 | ErrNoAvailableNetwork = errors.New("no networks available") 63 | ErrNetworkOptionsAllowlistIsNil = errors.New("network options allowlist is nil") 64 | ErrAsserterConfigurationIsNil = errors.New("asserter configuration is nil") 65 | ErrTimestampStartIndexMismatch = errors.New("timestamp start index mismatch") 66 | ErrOperationTypeLengthMismatch = errors.New("operation type length mismatch") 67 | ErrOperationTypeMismatch = errors.New("operation type mismatch") 68 | ErrOperationStatusLengthMismatch = errors.New("operation status length mismatch") 69 | ErrOperationStatusMismatch = errors.New("operation status mismatch") 70 | ErrErrorLengthMismatch = errors.New("error length mismatch") 71 | ErrErrorMismatch = errors.New("error mismatch") 72 | ErrAsserterConfigError = errors.New("asserter configuration validation failed") 73 | ErrNoHeadBlock = errors.New("no head block") 74 | ErrBlockBenchmarkTimeout = errors.New("/block endpoint benchmarking timed out") 75 | ErrAccountBalanceBenchmarkTimeout = errors.New("/account/balance endpoint benchmarking timed out") 76 | ) 77 | -------------------------------------------------------------------------------- /pkg/processor/balance_storage_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "math/big" 21 | 22 | "github.com/coinbase/rosetta-cli/pkg/logger" 23 | 24 | "github.com/coinbase/rosetta-sdk-go/parser" 25 | "github.com/coinbase/rosetta-sdk-go/reconciler" 26 | "github.com/coinbase/rosetta-sdk-go/storage/database" 27 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 28 | "github.com/coinbase/rosetta-sdk-go/types" 29 | ) 30 | 31 | var _ modules.BalanceStorageHandler = (*BalanceStorageHandler)(nil) 32 | 33 | // BalanceStorageHandler is invoked whenever a block is added 34 | // or removed from block storage so that balance changes 35 | // can be sent to other functions (ex: reconciler). 36 | type BalanceStorageHandler struct { 37 | logger *logger.Logger 38 | reconciler *reconciler.Reconciler 39 | counterStorage *modules.CounterStorage 40 | 41 | reconcile bool 42 | interestingAccount *types.AccountCurrency 43 | } 44 | 45 | // NewBalanceStorageHandler returns a new *BalanceStorageHandler. 46 | func NewBalanceStorageHandler( 47 | logger *logger.Logger, 48 | reconciler *reconciler.Reconciler, 49 | counterStorage *modules.CounterStorage, 50 | reconcile bool, 51 | interestingAccount *types.AccountCurrency, 52 | ) *BalanceStorageHandler { 53 | return &BalanceStorageHandler{ 54 | logger: logger, 55 | reconciler: reconciler, 56 | counterStorage: counterStorage, 57 | reconcile: reconcile, 58 | interestingAccount: interestingAccount, 59 | } 60 | } 61 | 62 | // BlockAdded is called whenever a block is committed to BlockStorage. 63 | func (h *BalanceStorageHandler) BlockAdded( 64 | ctx context.Context, 65 | block *types.Block, 66 | changes []*parser.BalanceChange, 67 | ) error { 68 | _ = h.logger.BalanceStream(ctx, changes) 69 | 70 | // When testing, it can be useful to not run any reconciliations to just check 71 | // if blocks are well formatted and balances don't go negative. 72 | if !h.reconcile { 73 | return nil 74 | } 75 | 76 | // When an interesting account is provided, only reconcile 77 | // balance changes affecting that account. This makes finding missing 78 | // ops much faster. 79 | if h.interestingAccount != nil { 80 | var interestingChange *parser.BalanceChange 81 | for _, change := range changes { 82 | if types.Hash(&types.AccountCurrency{ 83 | Account: change.Account, 84 | Currency: change.Currency, 85 | }) == types.Hash(h.interestingAccount) { 86 | interestingChange = change 87 | break 88 | } 89 | } 90 | 91 | if interestingChange != nil { 92 | changes = []*parser.BalanceChange{interestingChange} 93 | } else { 94 | changes = []*parser.BalanceChange{} 95 | } 96 | } 97 | 98 | // Mark accounts for reconciliation...this may be 99 | // blocking 100 | return h.reconciler.QueueChanges(ctx, block.BlockIdentifier, changes) 101 | } 102 | 103 | // BlockRemoved is called whenever a block is removed from BlockStorage. 104 | func (h *BalanceStorageHandler) BlockRemoved( 105 | ctx context.Context, 106 | block *types.Block, 107 | changes []*parser.BalanceChange, 108 | ) error { 109 | _ = h.logger.BalanceStream(ctx, changes) 110 | 111 | // We only attempt to reconciler changes when blocks are added, 112 | // not removed 113 | return nil 114 | } 115 | 116 | // AccountsReconciled updates the total accounts reconciled by count. 117 | func (h *BalanceStorageHandler) AccountsReconciled( 118 | ctx context.Context, 119 | dbTx database.Transaction, 120 | count int, 121 | ) error { 122 | _, err := h.counterStorage.UpdateTransactional( 123 | ctx, 124 | dbTx, 125 | modules.ReconciledAccounts, 126 | big.NewInt(int64(count)), 127 | ) 128 | if err != nil { 129 | return fmt.Errorf("failed to update the total accounts reconciled by count: %w", err) 130 | } 131 | return nil 132 | } 133 | 134 | // AccountsSeen updates the total accounts seen by count. 135 | func (h *BalanceStorageHandler) AccountsSeen( 136 | ctx context.Context, 137 | dbTx database.Transaction, 138 | count int, 139 | ) error { 140 | _, err := h.counterStorage.UpdateTransactional( 141 | ctx, 142 | dbTx, 143 | modules.SeenAccounts, 144 | big.NewInt(int64(count)), 145 | ) 146 | if err != nil { 147 | return fmt.Errorf("failed to update the total accounts seen by count: %w", err) 148 | } 149 | return nil 150 | } 151 | -------------------------------------------------------------------------------- /pkg/processor/balance_storage_helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "math/big" 21 | 22 | "github.com/coinbase/rosetta-sdk-go/asserter" 23 | "github.com/coinbase/rosetta-sdk-go/fetcher" 24 | "github.com/coinbase/rosetta-sdk-go/parser" 25 | "github.com/coinbase/rosetta-sdk-go/storage/database" 26 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 27 | "github.com/coinbase/rosetta-sdk-go/syncer" 28 | "github.com/coinbase/rosetta-sdk-go/types" 29 | "github.com/coinbase/rosetta-sdk-go/utils" 30 | ) 31 | 32 | var _ modules.BalanceStorageHelper = (*BalanceStorageHelper)(nil) 33 | 34 | // BalanceStorageHelper implements the storage.Helper 35 | // interface. 36 | type BalanceStorageHelper struct { 37 | network *types.NetworkIdentifier 38 | fetcher *fetcher.Fetcher 39 | counterStorage *modules.CounterStorage 40 | 41 | // Configuration settings 42 | lookupBalanceByBlock bool 43 | exemptAccounts map[string]struct{} 44 | balanceExemptions []*types.BalanceExemption 45 | initialFetchDisabled bool 46 | 47 | // Interesting-only Parsing 48 | interestingOnly bool 49 | interestingAddresses map[string]struct{} 50 | } 51 | 52 | // NewBalanceStorageHelper returns a new BalanceStorageHelper. 53 | func NewBalanceStorageHelper( 54 | network *types.NetworkIdentifier, 55 | fetcher *fetcher.Fetcher, 56 | counterStorage *modules.CounterStorage, 57 | lookupBalanceByBlock bool, 58 | exemptAccounts []*types.AccountCurrency, 59 | interestingOnly bool, 60 | balanceExemptions []*types.BalanceExemption, 61 | initialFetchDisabled bool, 62 | ) *BalanceStorageHelper { 63 | exemptMap := map[string]struct{}{} 64 | 65 | // Pre-process exemptAccounts on initialization 66 | // to provide fast lookup while syncing. 67 | for _, account := range exemptAccounts { 68 | // if users do not specify Currency, we add the address 69 | // by this, all the Currencies in this address will be skipped 70 | if account.Currency == nil { 71 | exemptMap[account.Account.Address] = struct{}{} 72 | } else { 73 | exemptMap[types.Hash(account)] = struct{}{} 74 | } 75 | } 76 | 77 | return &BalanceStorageHelper{ 78 | network: network, 79 | fetcher: fetcher, 80 | counterStorage: counterStorage, 81 | lookupBalanceByBlock: lookupBalanceByBlock, 82 | exemptAccounts: exemptMap, 83 | interestingAddresses: map[string]struct{}{}, 84 | interestingOnly: interestingOnly, 85 | balanceExemptions: balanceExemptions, 86 | initialFetchDisabled: initialFetchDisabled, 87 | } 88 | } 89 | 90 | // AccountBalance attempts to fetch the balance 91 | // for a missing account in storage. This is necessary 92 | // for running the "check" command at an arbitrary height 93 | // instead of syncing from genesis. 94 | func (h *BalanceStorageHelper) AccountBalance( 95 | ctx context.Context, 96 | account *types.AccountIdentifier, 97 | currency *types.Currency, 98 | lookupBlock *types.BlockIdentifier, 99 | ) (*types.Amount, error) { 100 | if !h.lookupBalanceByBlock || h.initialFetchDisabled { 101 | return &types.Amount{ 102 | Value: "0", 103 | Currency: currency, 104 | }, nil 105 | } 106 | 107 | // In the case that we are syncing from arbitrary height, 108 | // we may need to recover the balance of an account to 109 | // perform validations. 110 | amount, block, err := utils.CurrencyBalance( 111 | ctx, 112 | h.network, 113 | h.fetcher, 114 | account, 115 | currency, 116 | lookupBlock.Index, 117 | ) 118 | if err != nil { 119 | return nil, fmt.Errorf("unable to get balance of currency %s for account %s: %w", types.PrintStruct(currency), types.PrintStruct(account), err) 120 | } 121 | 122 | // If the returned balance block does not match the intended 123 | // block a re-org could've occurred. 124 | if types.Hash(lookupBlock) != types.Hash(block) { 125 | return nil, syncer.ErrOrphanHead 126 | } 127 | 128 | return &types.Amount{ 129 | Value: amount.Value, 130 | Currency: currency, 131 | }, nil 132 | } 133 | 134 | // Asserter returns a *asserter.Asserter. 135 | func (h *BalanceStorageHelper) Asserter() *asserter.Asserter { 136 | return h.fetcher.Asserter 137 | } 138 | 139 | // AddInterestingAddress adds an address to track the balance of. 140 | // This is often done after generating an account. 141 | func (h *BalanceStorageHelper) AddInterestingAddress(address string) { 142 | h.interestingAddresses[address] = struct{}{} 143 | } 144 | 145 | // ExemptFunc returns a parser.ExemptOperation. 146 | func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { 147 | return func(op *types.Operation) bool { 148 | if h.interestingOnly { 149 | if _, exists := h.interestingAddresses[op.Account.Address]; !exists { 150 | return true 151 | } 152 | } 153 | // if exemptAccounts have the Account address means all the 154 | // currencies in this Account address need to be skipped 155 | _, existsAddress := h.exemptAccounts[op.Account.Address] 156 | if existsAddress { 157 | return existsAddress 158 | } 159 | 160 | thisAcct := types.Hash(&types.AccountCurrency{ 161 | Account: op.Account, 162 | Currency: op.Amount.Currency, 163 | }) 164 | 165 | _, exists := h.exemptAccounts[thisAcct] 166 | return exists 167 | } 168 | } 169 | 170 | // BalanceExemptions returns a list of *types.BalanceExemption. 171 | func (h *BalanceStorageHelper) BalanceExemptions() []*types.BalanceExemption { 172 | return h.balanceExemptions 173 | } 174 | 175 | // AccountsReconciled returns the total accounts reconciled by count. 176 | func (h *BalanceStorageHelper) AccountsReconciled( 177 | ctx context.Context, 178 | dbTx database.Transaction, 179 | ) (*big.Int, error) { 180 | return h.counterStorage.GetTransactional(ctx, dbTx, modules.ReconciledAccounts) 181 | } 182 | 183 | // AccountsSeen returns the total accounts seen by count. 184 | func (h *BalanceStorageHelper) AccountsSeen( 185 | ctx context.Context, 186 | dbTx database.Transaction, 187 | ) (*big.Int, error) { 188 | return h.counterStorage.GetTransactional(ctx, dbTx, modules.SeenAccounts) 189 | } 190 | -------------------------------------------------------------------------------- /pkg/processor/balance_storage_helper_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/coinbase/rosetta-sdk-go/types" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | var ( 25 | opAmountCurrency = &types.AccountCurrency{ 26 | Account: &types.AccountIdentifier{ 27 | Address: "hello", 28 | }, 29 | Currency: &types.Currency{ 30 | Symbol: "BTC", 31 | Decimals: 8, 32 | }, 33 | } 34 | ) 35 | 36 | func TestExemptFuncExemptAccounts(t *testing.T) { 37 | var tests = map[string]struct { 38 | exemptAccounts []*types.AccountCurrency 39 | exempt bool 40 | }{ 41 | "no exempt accounts": {}, 42 | "account not exempt": { 43 | exemptAccounts: []*types.AccountCurrency{ 44 | { 45 | Account: &types.AccountIdentifier{ 46 | Address: "addr1", 47 | }, 48 | Currency: opAmountCurrency.Currency, 49 | }, 50 | { 51 | Account: &types.AccountIdentifier{ 52 | Address: "addr2", 53 | }, 54 | Currency: opAmountCurrency.Currency, 55 | }, 56 | }, 57 | }, 58 | "account is exempt": { 59 | exemptAccounts: []*types.AccountCurrency{ 60 | { 61 | Account: &types.AccountIdentifier{ 62 | Address: "addr1", 63 | }, 64 | Currency: opAmountCurrency.Currency, 65 | }, 66 | { 67 | Account: &types.AccountIdentifier{ 68 | Address: "addr2", 69 | }, 70 | Currency: opAmountCurrency.Currency, 71 | }, 72 | opAmountCurrency, 73 | }, 74 | exempt: true, 75 | }, 76 | } 77 | 78 | for name, test := range tests { 79 | t.Run(name, func(t *testing.T) { 80 | helper := NewBalanceStorageHelper( 81 | nil, 82 | nil, 83 | nil, 84 | false, 85 | test.exemptAccounts, 86 | false, 87 | nil, 88 | false, 89 | ) 90 | 91 | result := helper.ExemptFunc()(&types.Operation{ 92 | Account: opAmountCurrency.Account, 93 | Amount: &types.Amount{ 94 | Value: "100", 95 | Currency: opAmountCurrency.Currency, 96 | }, 97 | }) 98 | 99 | assert.Equal(t, test.exempt, result) 100 | }) 101 | } 102 | } 103 | 104 | func TestExemptFuncInterestingParsing(t *testing.T) { 105 | var tests = map[string]struct { 106 | interestingAddresses []string 107 | exempt bool 108 | }{ 109 | "no interesting accounts": { 110 | exempt: true, 111 | }, 112 | "account interesting": { 113 | interestingAddresses: []string{opAmountCurrency.Account.Address, "addr 3"}, 114 | exempt: false, 115 | }, 116 | "account not interesting": { 117 | interestingAddresses: []string{"addr2"}, 118 | exempt: true, 119 | }, 120 | } 121 | 122 | for name, test := range tests { 123 | t.Run(name, func(t *testing.T) { 124 | helper := NewBalanceStorageHelper( 125 | nil, 126 | nil, 127 | nil, 128 | false, 129 | nil, 130 | true, 131 | nil, 132 | false, 133 | ) 134 | 135 | for _, addr := range test.interestingAddresses { 136 | helper.AddInterestingAddress(addr) 137 | } 138 | 139 | result := helper.ExemptFunc()(&types.Operation{ 140 | Account: opAmountCurrency.Account, 141 | Amount: &types.Amount{ 142 | Value: "100", 143 | Currency: opAmountCurrency.Currency, 144 | }, 145 | }) 146 | 147 | assert.Equal(t, test.exempt, result) 148 | }) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /pkg/processor/broadcast_storage_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "math/big" 21 | "reflect" 22 | 23 | "github.com/coinbase/rosetta-cli/configuration" 24 | 25 | "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" 26 | "github.com/coinbase/rosetta-sdk-go/parser" 27 | "github.com/coinbase/rosetta-sdk-go/storage/database" 28 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 29 | "github.com/coinbase/rosetta-sdk-go/types" 30 | ) 31 | 32 | var _ modules.BroadcastStorageHandler = (*BroadcastStorageHandler)(nil) 33 | 34 | // BroadcastStorageHandler is invoked whenever a block is added 35 | // or removed from block storage so that balance changes 36 | // can be sent to other functions (ex: reconciler). 37 | type BroadcastStorageHandler struct { 38 | config *configuration.Configuration 39 | blockStorage *modules.BlockStorage 40 | counterStorage *modules.CounterStorage 41 | coordinator *coordinator.Coordinator 42 | parser *parser.Parser 43 | } 44 | 45 | // NewBroadcastStorageHandler returns a new *BroadcastStorageHandler. 46 | func NewBroadcastStorageHandler( 47 | config *configuration.Configuration, 48 | blockStorage *modules.BlockStorage, 49 | counterStorage *modules.CounterStorage, 50 | coordinator *coordinator.Coordinator, 51 | parser *parser.Parser, 52 | ) *BroadcastStorageHandler { 53 | return &BroadcastStorageHandler{ 54 | config: config, 55 | blockStorage: blockStorage, 56 | counterStorage: counterStorage, 57 | coordinator: coordinator, 58 | parser: parser, 59 | } 60 | } 61 | 62 | // TransactionConfirmed is called when a transaction is observed on-chain for the 63 | // last time at a block height < current block height - confirmationDepth. 64 | func (h *BroadcastStorageHandler) TransactionConfirmed( 65 | ctx context.Context, 66 | dbTx database.Transaction, 67 | identifier string, 68 | blockIdentifier *types.BlockIdentifier, 69 | transaction *types.Transaction, 70 | intent []*types.Operation, 71 | intentMetadata map[string]interface{}, 72 | ) error { 73 | _, _, relatedTransactions, err := h.blockStorage.FindRelatedTransactions(ctx, transaction.TransactionIdentifier, dbTx) 74 | if err != nil { 75 | return fmt.Errorf("failed to find related transactions %s: %w", types.PrintStruct(transaction.TransactionIdentifier), err) 76 | } 77 | 78 | observed := transaction.Operations 79 | for _, relatedTransaction := range relatedTransactions { 80 | observed = append(observed, relatedTransaction.Operations...) 81 | } 82 | 83 | if err := h.parser.ExpectedOperations(intent, observed, false, true); err != nil { 84 | errMsg := fmt.Errorf("confirmed transaction did not match intent: %w", err) 85 | if !isValidStakingOperation(intent, intentMetadata) { 86 | return errMsg 87 | } 88 | } 89 | 90 | // Validate destination memo if it's needed 91 | if intentMemo, found := intentMetadata["memo"]; found { 92 | if observedMemo, found := transaction.Metadata["memo"]; found { 93 | if !reflect.DeepEqual(intentMemo, observedMemo) { 94 | return fmt.Errorf("observed destination memo did not match intent destination memo, observed destination memo: %v, intent destination memo: %v", observedMemo, intentMemo) 95 | } 96 | } else { 97 | return fmt.Errorf("observed destination memo did not found, observed destination memo: %v, intent destination memo: %v", observedMemo, intentMemo) 98 | } 99 | } 100 | 101 | _, _ = h.counterStorage.UpdateTransactional( 102 | ctx, 103 | dbTx, 104 | modules.TransactionsConfirmedCounter, 105 | big.NewInt(1), 106 | ) 107 | 108 | if err := h.coordinator.BroadcastComplete( 109 | ctx, 110 | dbTx, 111 | identifier, 112 | transaction, 113 | ); err != nil { 114 | return fmt.Errorf("coordinator could not handle transaction: %w", err) 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func isValidStakingOperation(intent []*types.Operation, metadata map[string]interface{}) bool { 121 | stakingOpsTypes := map[string]bool{ 122 | "stake": true, 123 | "unstake": true, 124 | "withdraw": true, 125 | "restake": true, 126 | } 127 | 128 | if _, found := metadata["validator_src_address"]; found { 129 | if len(intent) == 1 { 130 | _, found := stakingOpsTypes[intent[0].Type] 131 | return found 132 | } 133 | } 134 | return false 135 | } 136 | 137 | // TransactionStale is called when a transaction has not yet been 138 | // seen on-chain and is considered stale. This occurs when 139 | // current block height - last broadcast > staleDepth. 140 | func (h *BroadcastStorageHandler) TransactionStale( 141 | ctx context.Context, 142 | dbTx database.Transaction, 143 | identifier string, 144 | transactionIdentifier *types.TransactionIdentifier, 145 | ) error { 146 | _, _ = h.counterStorage.UpdateTransactional( 147 | ctx, 148 | dbTx, 149 | modules.StaleBroadcastsCounter, 150 | big.NewInt(1), 151 | ) 152 | 153 | return nil 154 | } 155 | 156 | // BroadcastFailed is called when another transaction broadcast would 157 | // put it over the provided broadcast limit. 158 | func (h *BroadcastStorageHandler) BroadcastFailed( 159 | ctx context.Context, 160 | dbTx database.Transaction, 161 | identifier string, 162 | transactionIdentifier *types.TransactionIdentifier, 163 | intent []*types.Operation, 164 | ) error { 165 | _, _ = h.counterStorage.UpdateTransactional( 166 | ctx, 167 | dbTx, 168 | modules.FailedBroadcastsCounter, 169 | big.NewInt(1), 170 | ) 171 | 172 | if err := h.coordinator.BroadcastComplete( 173 | ctx, 174 | dbTx, 175 | identifier, 176 | nil, 177 | ); err != nil { 178 | return fmt.Errorf("coordinator could not handle transaction: %w", err) 179 | } 180 | 181 | if h.config.Construction.IgnoreBroadcastFailures { 182 | return nil 183 | } 184 | 185 | return fmt.Errorf( 186 | "broadcast failed for transaction %s with intent %s", 187 | transactionIdentifier.Hash, 188 | types.PrettyPrintStruct(intent), 189 | ) 190 | } 191 | -------------------------------------------------------------------------------- /pkg/processor/broadcast_storage_helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/coinbase/rosetta-sdk-go/utils" 22 | 23 | "github.com/coinbase/rosetta-sdk-go/fetcher" 24 | "github.com/coinbase/rosetta-sdk-go/storage/database" 25 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 26 | "github.com/coinbase/rosetta-sdk-go/types" 27 | ) 28 | 29 | var _ modules.BroadcastStorageHelper = (*BroadcastStorageHelper)(nil) 30 | 31 | // BroadcastStorageHelper implements the storage.Helper 32 | // interface. 33 | type BroadcastStorageHelper struct { 34 | network *types.NetworkIdentifier 35 | blockStorage *modules.BlockStorage 36 | fetcher *fetcher.Fetcher 37 | } 38 | 39 | // NewBroadcastStorageHelper returns a new BroadcastStorageHelper. 40 | func NewBroadcastStorageHelper( 41 | network *types.NetworkIdentifier, 42 | blockStorage *modules.BlockStorage, 43 | fetcher *fetcher.Fetcher, 44 | ) *BroadcastStorageHelper { 45 | return &BroadcastStorageHelper{ 46 | network: network, 47 | blockStorage: blockStorage, 48 | fetcher: fetcher, 49 | } 50 | } 51 | 52 | // AtTip is called before transaction broadcast to determine if we are at tip. 53 | func (h *BroadcastStorageHelper) AtTip( 54 | ctx context.Context, 55 | tipDelay int64, 56 | ) (bool, error) { 57 | atTip, _, err := utils.CheckStorageTip(ctx, h.network, tipDelay, h.fetcher, h.blockStorage) 58 | if err != nil { 59 | return false, fmt.Errorf("failed to check storage tip: %w", err) 60 | } 61 | 62 | return atTip, nil 63 | } 64 | 65 | // CurrentBlockIdentifier is called before transaction broadcast and is used 66 | // to determine if a transaction broadcast is stale. 67 | func (h *BroadcastStorageHelper) CurrentBlockIdentifier( 68 | ctx context.Context, 69 | ) (*types.BlockIdentifier, error) { 70 | blockIdentifier, err := h.blockStorage.GetHeadBlockIdentifier(ctx) 71 | if err != nil { 72 | return nil, fmt.Errorf("unable to get head block identifier: %w", err) 73 | } 74 | 75 | return blockIdentifier, nil 76 | } 77 | 78 | // FindTransaction looks for the provided TransactionIdentifier in processed 79 | // blocks and returns the block identifier containing the most recent sighting 80 | // and the transaction seen in that block. 81 | func (h *BroadcastStorageHelper) FindTransaction( 82 | ctx context.Context, 83 | transactionIdentifier *types.TransactionIdentifier, 84 | txn database.Transaction, 85 | ) (*types.BlockIdentifier, *types.Transaction, error) { 86 | newestBlock, transaction, err := h.blockStorage.FindTransaction(ctx, transactionIdentifier, txn) 87 | if err != nil { 88 | return nil, nil, fmt.Errorf("unable to perform transaction search for transaction %s: %w", types.PrintStruct(transactionIdentifier), err) 89 | } 90 | 91 | return newestBlock, transaction, nil 92 | } 93 | 94 | // BroadcastTransaction broadcasts a transaction to a Rosetta implementation 95 | // and returns the *types.TransactionIdentifier returned by the implementation. 96 | func (h *BroadcastStorageHelper) BroadcastTransaction( 97 | ctx context.Context, 98 | networkIdentifier *types.NetworkIdentifier, 99 | networkTransaction string, 100 | ) (*types.TransactionIdentifier, error) { 101 | transactionIdentifier, _, fetchErr := h.fetcher.ConstructionSubmit( 102 | ctx, 103 | networkIdentifier, 104 | networkTransaction, 105 | ) 106 | if fetchErr != nil { 107 | return nil, fmt.Errorf("unable to broadcast transaction %s: %w", networkTransaction, fetchErr.Err) 108 | } 109 | 110 | return transactionIdentifier, nil 111 | } 112 | -------------------------------------------------------------------------------- /pkg/processor/coin_storage_helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/coinbase/rosetta-sdk-go/storage/database" 21 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 22 | "github.com/coinbase/rosetta-sdk-go/types" 23 | ) 24 | 25 | var _ modules.CoinStorageHelper = (*CoinStorageHelper)(nil) 26 | 27 | // CoinStorageHelper implements the storage.CoinStorageHelper 28 | // interface. 29 | type CoinStorageHelper struct { 30 | blockStorage *modules.BlockStorage 31 | } 32 | 33 | // NewCoinStorageHelper returns a new *CoinStorageHelper. 34 | func NewCoinStorageHelper(blockStorage *modules.BlockStorage) *CoinStorageHelper { 35 | return &CoinStorageHelper{blockStorage: blockStorage} 36 | } 37 | 38 | // CurrentBlockIdentifier returns the head *types.BlockIdentifier in 39 | // the context of a storage.DatabaseTransaction. 40 | func (c *CoinStorageHelper) CurrentBlockIdentifier( 41 | ctx context.Context, 42 | transaction database.Transaction, 43 | ) (*types.BlockIdentifier, error) { 44 | return c.blockStorage.GetHeadBlockIdentifierTransactional(ctx, transaction) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/processor/coordinator_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "math/big" 20 | 21 | "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" 22 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 23 | "github.com/coinbase/rosetta-sdk-go/types" 24 | ) 25 | 26 | var _ coordinator.Handler = (*CoordinatorHandler)(nil) 27 | 28 | // CoordinatorHandler is invoked by the Coordinator 29 | // when addresses are created or transactions are created. 30 | type CoordinatorHandler struct { 31 | counterStorage *modules.CounterStorage 32 | } 33 | 34 | // NewCoordinatorHandler returns a new 35 | // *CoordinatorHandler. 36 | func NewCoordinatorHandler( 37 | counterStorage *modules.CounterStorage, 38 | ) *CoordinatorHandler { 39 | return &CoordinatorHandler{ 40 | counterStorage: counterStorage, 41 | } 42 | } 43 | 44 | // TransactionCreated increments the TransactionsCreatedCounter in 45 | // CounterStorage. 46 | func (h *CoordinatorHandler) TransactionCreated( 47 | ctx context.Context, 48 | jobIdentifier string, 49 | transactionIdentifier *types.TransactionIdentifier, 50 | ) error { 51 | _, _ = h.counterStorage.Update( 52 | ctx, 53 | modules.TransactionsCreatedCounter, 54 | big.NewInt(1), 55 | ) 56 | 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /pkg/processor/reconciler_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "math/big" 21 | "sync" 22 | "time" 23 | 24 | "github.com/coinbase/rosetta-cli/pkg/logger" 25 | 26 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 27 | "github.com/coinbase/rosetta-sdk-go/reconciler" 28 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 29 | "github.com/coinbase/rosetta-sdk-go/types" 30 | ) 31 | 32 | const ( 33 | updateFrequency = 10 * time.Second 34 | ) 35 | 36 | var _ reconciler.Handler = (*ReconcilerHandler)(nil) 37 | 38 | var ( 39 | countKeys = []string{ 40 | modules.FailedReconciliationCounter, 41 | modules.SkippedReconciliationsCounter, 42 | modules.ExemptReconciliationCounter, 43 | modules.ActiveReconciliationCounter, 44 | modules.InactiveReconciliationCounter, 45 | } 46 | ) 47 | 48 | // ReconcilerHandler implements the Reconciler.Handler interface. 49 | type ReconcilerHandler struct { 50 | logger *logger.Logger 51 | counterStorage *modules.CounterStorage 52 | balanceStorage *modules.BalanceStorage 53 | haltOnReconciliationError bool 54 | 55 | InactiveFailure *types.AccountCurrency 56 | InactiveFailureBlock *types.BlockIdentifier 57 | 58 | ActiveFailureBlock *types.BlockIdentifier 59 | 60 | counterLock sync.Mutex 61 | counts map[string]int64 62 | } 63 | 64 | // NewReconcilerHandler creates a new ReconcilerHandler. 65 | func NewReconcilerHandler( 66 | logger *logger.Logger, 67 | counterStorage *modules.CounterStorage, 68 | balanceStorage *modules.BalanceStorage, 69 | haltOnReconciliationError bool, 70 | ) *ReconcilerHandler { 71 | counts := map[string]int64{} 72 | for _, key := range countKeys { 73 | counts[key] = 0 74 | } 75 | 76 | return &ReconcilerHandler{ 77 | logger: logger, 78 | counterStorage: counterStorage, 79 | balanceStorage: balanceStorage, 80 | haltOnReconciliationError: haltOnReconciliationError, 81 | counts: counts, 82 | } 83 | } 84 | 85 | // Updater periodically updates modules.with cached counts. 86 | func (h *ReconcilerHandler) Updater(ctx context.Context) error { 87 | tc := time.NewTicker(updateFrequency) 88 | defer tc.Stop() 89 | 90 | for { 91 | select { 92 | case <-tc.C: 93 | if err := h.UpdateCounts(ctx); err != nil { 94 | return err 95 | } 96 | case <-ctx.Done(): 97 | return ctx.Err() 98 | } 99 | } 100 | } 101 | 102 | // UpdateCounts forces cached counts to be written to modules. 103 | func (h *ReconcilerHandler) UpdateCounts(ctx context.Context) error { 104 | for _, key := range countKeys { 105 | h.counterLock.Lock() 106 | count := h.counts[key] 107 | h.counts[key] = 0 108 | h.counterLock.Unlock() 109 | 110 | if count == 0 { 111 | continue 112 | } 113 | 114 | if _, err := h.counterStorage.Update(ctx, key, big.NewInt(count)); err != nil { 115 | return fmt.Errorf("failed to key %s in counter storage: %w", key, err) 116 | } 117 | } 118 | 119 | return nil 120 | } 121 | 122 | // ReconciliationFailed is called each time a reconciliation fails. 123 | // In this Handler implementation, we halt if haltOnReconciliationError 124 | // was set to true. We also cancel the context. 125 | func (h *ReconcilerHandler) ReconciliationFailed( 126 | ctx context.Context, 127 | reconciliationType string, 128 | account *types.AccountIdentifier, 129 | currency *types.Currency, 130 | computedBalance string, 131 | liveBalance string, 132 | block *types.BlockIdentifier, 133 | ) error { 134 | h.counterLock.Lock() 135 | h.counts[modules.FailedReconciliationCounter]++ 136 | h.counterLock.Unlock() 137 | 138 | err := h.logger.ReconcileFailureStream( 139 | ctx, 140 | reconciliationType, 141 | account, 142 | currency, 143 | computedBalance, 144 | liveBalance, 145 | block, 146 | ) 147 | if err != nil { 148 | return fmt.Errorf("failed to log reconciliation checks when reconciliation is failed: %w", err) 149 | } 150 | 151 | if h.haltOnReconciliationError { 152 | // Update counts before exiting 153 | _ = h.UpdateCounts(ctx) 154 | 155 | if reconciliationType == reconciler.InactiveReconciliation { 156 | // Populate inactive failure information so we can try to find block with 157 | // missing ops. 158 | h.InactiveFailure = &types.AccountCurrency{ 159 | Account: account, 160 | Currency: currency, 161 | } 162 | h.InactiveFailureBlock = block 163 | return fmt.Errorf( 164 | "inactive reconciliation error for account address %s at block index %d (computed: %s%s, live: %s%s): %w", 165 | account.Address, 166 | block.Index, 167 | computedBalance, 168 | currency.Symbol, 169 | liveBalance, 170 | currency.Symbol, 171 | cliErrs.ErrReconciliationFailure, 172 | ) 173 | } 174 | 175 | // If we halt on an active reconciliation error, store in the handler. 176 | h.ActiveFailureBlock = block 177 | return fmt.Errorf( 178 | "active reconciliation error for account address %s at block index %d (computed: %s%s, live: %s%s): %w", 179 | account.Address, 180 | block.Index, 181 | computedBalance, 182 | currency.Symbol, 183 | liveBalance, 184 | currency.Symbol, 185 | cliErrs.ErrReconciliationFailure, 186 | ) 187 | } 188 | 189 | return nil 190 | } 191 | 192 | // ReconciliationExempt is called each time a reconciliation fails 193 | // but is considered exempt because of provided []*types.BalanceExemption. 194 | func (h *ReconcilerHandler) ReconciliationExempt( 195 | ctx context.Context, 196 | reconciliationType string, 197 | account *types.AccountIdentifier, 198 | currency *types.Currency, 199 | computedBalance string, 200 | liveBalance string, 201 | block *types.BlockIdentifier, 202 | exemption *types.BalanceExemption, 203 | ) error { 204 | h.counterLock.Lock() 205 | h.counts[modules.ExemptReconciliationCounter]++ 206 | h.counterLock.Unlock() 207 | 208 | // Although the reconciliation was exempt (non-zero difference that was ignored), 209 | // we still mark the account as being reconciled because the balance was in the range 210 | // specified by exemption. 211 | if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { 212 | return fmt.Errorf("unable to store updated reconciliation currency %s of account %s at block %s: %w", types.PrintStruct(currency), types.PrintStruct(account), types.PrintStruct(block), err) 213 | } 214 | 215 | return nil 216 | } 217 | 218 | // ReconciliationSkipped is called each time a reconciliation is skipped. 219 | func (h *ReconcilerHandler) ReconciliationSkipped( 220 | ctx context.Context, 221 | reconciliationType string, 222 | account *types.AccountIdentifier, 223 | currency *types.Currency, 224 | cause string, 225 | ) error { 226 | h.counterLock.Lock() 227 | h.counts[modules.SkippedReconciliationsCounter]++ 228 | h.counterLock.Unlock() 229 | 230 | return nil 231 | } 232 | 233 | // ReconciliationSucceeded is called each time a reconciliation succeeds. 234 | func (h *ReconcilerHandler) ReconciliationSucceeded( 235 | ctx context.Context, 236 | reconciliationType string, 237 | account *types.AccountIdentifier, 238 | currency *types.Currency, 239 | balance string, 240 | block *types.BlockIdentifier, 241 | ) error { 242 | // Update counters 243 | counter := modules.ActiveReconciliationCounter 244 | if reconciliationType == reconciler.InactiveReconciliation { 245 | counter = modules.InactiveReconciliationCounter 246 | } 247 | 248 | h.counterLock.Lock() 249 | h.counts[counter]++ 250 | h.counterLock.Unlock() 251 | 252 | if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { 253 | return fmt.Errorf("unable to store updated reconciliation currency %s of account %s at block %s: %w", types.PrintStruct(currency), types.PrintStruct(account), types.PrintStruct(block), err) 254 | } 255 | 256 | return h.logger.ReconcileSuccessStream( 257 | ctx, 258 | reconciliationType, 259 | account, 260 | currency, 261 | balance, 262 | block, 263 | ) 264 | } 265 | -------------------------------------------------------------------------------- /pkg/processor/reconciler_helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package processor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/coinbase/rosetta-cli/configuration" 22 | 23 | "github.com/coinbase/rosetta-sdk-go/fetcher" 24 | "github.com/coinbase/rosetta-sdk-go/reconciler" 25 | "github.com/coinbase/rosetta-sdk-go/storage/database" 26 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 27 | "github.com/coinbase/rosetta-sdk-go/types" 28 | "github.com/coinbase/rosetta-sdk-go/utils" 29 | ) 30 | 31 | var _ reconciler.Helper = (*ReconcilerHelper)(nil) 32 | 33 | // ReconcilerHelper implements the Reconciler.Helper 34 | // interface. 35 | type ReconcilerHelper struct { 36 | config *configuration.Configuration 37 | 38 | network *types.NetworkIdentifier 39 | fetcher *fetcher.Fetcher 40 | 41 | database database.Database 42 | blockStorage *modules.BlockStorage 43 | balanceStorage *modules.BalanceStorage 44 | forceInactiveReconciliation *bool 45 | } 46 | 47 | // NewReconcilerHelper returns a new ReconcilerHelper. 48 | func NewReconcilerHelper( 49 | config *configuration.Configuration, 50 | network *types.NetworkIdentifier, 51 | fetcher *fetcher.Fetcher, 52 | database database.Database, 53 | blockStorage *modules.BlockStorage, 54 | balanceStorage *modules.BalanceStorage, 55 | forceInactiveReconciliation *bool, 56 | ) *ReconcilerHelper { 57 | return &ReconcilerHelper{ 58 | config: config, 59 | network: network, 60 | fetcher: fetcher, 61 | database: database, 62 | blockStorage: blockStorage, 63 | balanceStorage: balanceStorage, 64 | forceInactiveReconciliation: forceInactiveReconciliation, 65 | } 66 | } 67 | 68 | // DatabaseTransaction returns a new read-only database.Transaction. 69 | func (h *ReconcilerHelper) DatabaseTransaction( 70 | ctx context.Context, 71 | ) database.Transaction { 72 | return h.database.ReadTransaction(ctx) 73 | } 74 | 75 | // CanonicalBlock returns a boolean indicating if a block 76 | // is in the canonical chain. This is necessary to reconcile across 77 | // reorgs. If the block returned on an account balance fetch 78 | // does not exist, reconciliation will be skipped. 79 | func (h *ReconcilerHelper) CanonicalBlock( 80 | ctx context.Context, 81 | dbTx database.Transaction, 82 | block *types.BlockIdentifier, 83 | ) (bool, error) { 84 | return h.blockStorage.CanonicalBlockTransactional(ctx, block, dbTx) 85 | } 86 | 87 | // IndexAtTip returns a boolean indicating if a block 88 | // index is at tip (provided some acceptable 89 | // tip delay). If the index is ahead of the head block 90 | // and the head block is at tip, we consider the 91 | // index at tip. 92 | func (h *ReconcilerHelper) IndexAtTip( 93 | ctx context.Context, 94 | index int64, 95 | ) (bool, error) { 96 | return h.blockStorage.IndexAtTip( 97 | ctx, 98 | h.config.TipDelay, 99 | index, 100 | ) 101 | } 102 | 103 | // CurrentBlock returns the last processed block and is used 104 | // to determine which block to check account balances at during 105 | // inactive reconciliation. 106 | func (h *ReconcilerHelper) CurrentBlock( 107 | ctx context.Context, 108 | dbTx database.Transaction, 109 | ) (*types.BlockIdentifier, error) { 110 | return h.blockStorage.GetHeadBlockIdentifierTransactional(ctx, dbTx) 111 | } 112 | 113 | // ComputedBalance returns the balance of an account in block storage. 114 | // It is necessary to perform this check outside of the Reconciler 115 | // package to allow for separation from a default storage backend. 116 | func (h *ReconcilerHelper) ComputedBalance( 117 | ctx context.Context, 118 | dbTx database.Transaction, 119 | account *types.AccountIdentifier, 120 | currency *types.Currency, 121 | index int64, 122 | ) (*types.Amount, error) { 123 | return h.balanceStorage.GetBalanceTransactional(ctx, dbTx, account, currency, index) 124 | } 125 | 126 | // LiveBalance returns the live balance of an account. 127 | func (h *ReconcilerHelper) LiveBalance( 128 | ctx context.Context, 129 | account *types.AccountIdentifier, 130 | currency *types.Currency, 131 | index int64, 132 | ) (*types.Amount, *types.BlockIdentifier, error) { 133 | amt, block, err := utils.CurrencyBalance( 134 | ctx, 135 | h.network, 136 | h.fetcher, 137 | account, 138 | currency, 139 | index, 140 | ) 141 | if err != nil { 142 | return nil, nil, fmt.Errorf("failed to get current balance of currency %s of account %s: %w", types.PrintStruct(currency), types.PrintStruct(account), err) 143 | } 144 | return amt, block, nil 145 | } 146 | 147 | // PruneBalances removes all historical balance states 148 | // <= some index. This can significantly reduce storage 149 | // usage in scenarios where historical balances are only 150 | // retrieved once (like reconciliation). 151 | func (h *ReconcilerHelper) PruneBalances( 152 | ctx context.Context, 153 | account *types.AccountIdentifier, 154 | currency *types.Currency, 155 | index int64, 156 | ) error { 157 | if h.config.Data.PruningBalanceDisabled { 158 | return nil 159 | } 160 | 161 | return h.balanceStorage.PruneBalances( 162 | ctx, 163 | account, 164 | currency, 165 | index, 166 | ) 167 | } 168 | 169 | // ForceInactiveReconciliation overrides the default 170 | // calculation to determine if an account should be 171 | // reconciled inactively. 172 | func (h *ReconcilerHelper) ForceInactiveReconciliation( 173 | ctx context.Context, 174 | account *types.AccountIdentifier, 175 | currency *types.Currency, 176 | lastChecked *types.BlockIdentifier, 177 | ) bool { 178 | if h.forceInactiveReconciliation == nil { 179 | return false 180 | } 181 | 182 | return *h.forceInactiveReconciliation 183 | } 184 | -------------------------------------------------------------------------------- /pkg/results/construction_results.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package results 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "log" 21 | "os" 22 | "strconv" 23 | 24 | pkgError "github.com/pkg/errors" 25 | 26 | "github.com/coinbase/rosetta-cli/configuration" 27 | 28 | "github.com/coinbase/rosetta-sdk-go/storage/modules" 29 | "github.com/coinbase/rosetta-sdk-go/types" 30 | "github.com/coinbase/rosetta-sdk-go/utils" 31 | "github.com/fatih/color" 32 | "github.com/olekukonko/tablewriter" 33 | ) 34 | 35 | // CheckConstructionResults contains any error that 36 | // occurred on a check:construction run and a collection 37 | // of interesting stats. 38 | type CheckConstructionResults struct { 39 | Error string `json:"error"` 40 | EndConditions map[string]int `json:"end_conditions"` 41 | Stats *CheckConstructionStats `json:"stats"` 42 | // TODO: add test output (like check data) 43 | } 44 | 45 | // Print logs CheckConstructionResults to the console. 46 | func (c *CheckConstructionResults) Print() { 47 | if len(c.Error) > 0 { 48 | fmt.Printf("\n") 49 | color.Red("Error: %s", c.Error) 50 | } else { 51 | fmt.Printf("\n") 52 | color.Green("Success: %s", types.PrintStruct(c.EndConditions)) 53 | } 54 | 55 | fmt.Printf("\n") 56 | if c.Stats != nil { 57 | c.Stats.Print() 58 | fmt.Printf("\n") 59 | } 60 | } 61 | 62 | // Output writes CheckConstructionResults to the provided 63 | // path. 64 | func (c *CheckConstructionResults) Output(path string) { 65 | if len(path) > 0 { 66 | writeErr := utils.SerializeAndWrite(path, c) 67 | if writeErr != nil { 68 | log.Printf("unable to save results: %s\n", writeErr.Error()) 69 | } 70 | } 71 | } 72 | 73 | // ComputeCheckConstructionResults returns a populated 74 | // CheckConstructionResults. 75 | func ComputeCheckConstructionResults( 76 | cfg *configuration.Configuration, 77 | err error, 78 | counterStorage *modules.CounterStorage, 79 | jobStorage *modules.JobStorage, 80 | ) *CheckConstructionResults { 81 | ctx := context.Background() 82 | stats := ComputeCheckConstructionStats(ctx, cfg, counterStorage, jobStorage) 83 | results := &CheckConstructionResults{ 84 | Stats: stats, 85 | } 86 | 87 | if err != nil { 88 | results.Error = fmt.Sprintf("%+v", err) 89 | 90 | // We never want to populate an end condition 91 | // if there was an error! 92 | return results 93 | } 94 | 95 | results.EndConditions = cfg.Construction.EndConditions 96 | 97 | return results 98 | } 99 | 100 | // CheckConstructionStats contains interesting stats 101 | // that are tracked while running check:construction. 102 | type CheckConstructionStats struct { 103 | TransactionsConfirmed int64 `json:"transactions_confirmed"` 104 | TransactionsCreated int64 `json:"transactions_created"` 105 | StaleBroadcasts int64 `json:"stale_broadcasts"` 106 | FailedBroadcasts int64 `json:"failed_broadcasts"` 107 | AddressesCreated int64 `json:"addresses_created"` 108 | 109 | WorkflowsCompleted map[string]int64 `json:"workflows_completed"` 110 | } 111 | 112 | // PrintCounts logs counter-related stats to the console. 113 | func (c *CheckConstructionStats) PrintCounts() { 114 | table := tablewriter.NewWriter(os.Stdout) 115 | table.SetRowLine(true) 116 | table.SetRowSeparator("-") 117 | table.SetHeader([]string{"check:construction Stats", "Description", "Value"}) 118 | table.Append([]string{ 119 | "Addresses Created", 120 | "# of addresses created", 121 | strconv.FormatInt(c.AddressesCreated, 10), 122 | }) 123 | table.Append([]string{ 124 | "Transactions Created", 125 | "# of transactions created", 126 | strconv.FormatInt(c.TransactionsCreated, 10), 127 | }) 128 | table.Append([]string{ 129 | "Stale Broadcasts", 130 | "# of broadcasts missing after stale depth", 131 | strconv.FormatInt(c.StaleBroadcasts, 10), 132 | }) 133 | table.Append([]string{ 134 | "Transactions Confirmed", 135 | "# of transactions seen on-chain", 136 | strconv.FormatInt(c.TransactionsConfirmed, 10), 137 | }) 138 | table.Append([]string{ 139 | "Failed Broadcasts", 140 | "# of transactions that exceeded broadcast limit", 141 | strconv.FormatInt(c.FailedBroadcasts, 10), 142 | }) 143 | 144 | table.Render() 145 | } 146 | 147 | // PrintWorkflows logs workflow counts to the console. 148 | func (c *CheckConstructionStats) PrintWorkflows() { 149 | table := tablewriter.NewWriter(os.Stdout) 150 | table.SetRowLine(true) 151 | table.SetRowSeparator("-") 152 | table.SetHeader([]string{"check:construction Workflows", "Count"}) 153 | for workflow, count := range c.WorkflowsCompleted { 154 | table.Append([]string{ 155 | workflow, 156 | strconv.FormatInt(count, 10), 157 | }) 158 | } 159 | 160 | table.Render() 161 | } 162 | 163 | // Print calls PrintCounts and PrintWorkflows. 164 | func (c *CheckConstructionStats) Print() { 165 | c.PrintCounts() 166 | c.PrintWorkflows() 167 | } 168 | 169 | // ComputeCheckConstructionStats returns a populated 170 | // CheckConstructionStats. 171 | func ComputeCheckConstructionStats( 172 | ctx context.Context, 173 | config *configuration.Configuration, 174 | counters *modules.CounterStorage, 175 | jobs *modules.JobStorage, 176 | ) *CheckConstructionStats { 177 | if counters == nil || jobs == nil { 178 | return nil 179 | } 180 | 181 | transactionsCreated, err := counters.Get(ctx, modules.TransactionsCreatedCounter) 182 | if err != nil { 183 | log.Printf("%s cannot get transactions created counter\n", err.Error()) 184 | return nil 185 | } 186 | 187 | transactionsConfirmed, err := counters.Get(ctx, modules.TransactionsConfirmedCounter) 188 | if err != nil { 189 | log.Printf("%s cannot get transactions confirmed counter\n", err.Error()) 190 | return nil 191 | } 192 | 193 | staleBroadcasts, err := counters.Get(ctx, modules.StaleBroadcastsCounter) 194 | if err != nil { 195 | log.Printf("%s cannot get stale broadcasts counter\n", err) 196 | return nil 197 | } 198 | 199 | failedBroadcasts, err := counters.Get(ctx, modules.FailedBroadcastsCounter) 200 | if err != nil { 201 | log.Printf("%s cannot get failed broadcasts counter\n", err.Error()) 202 | return nil 203 | } 204 | 205 | addressesCreated, err := counters.Get(ctx, modules.AddressesCreatedCounter) 206 | if err != nil { 207 | log.Printf("%s cannot get addresses created counter\n", err.Error()) 208 | return nil 209 | } 210 | 211 | workflowsCompleted := map[string]int64{} 212 | for _, workflow := range config.Construction.Workflows { 213 | completed, err := jobs.Completed(ctx, workflow.Name) 214 | if err != nil { 215 | log.Printf("%s cannot get completed count for %s\n", err.Error(), workflow.Name) 216 | return nil 217 | } 218 | 219 | workflowsCompleted[workflow.Name] = int64(len(completed)) 220 | } 221 | 222 | return &CheckConstructionStats{ 223 | TransactionsCreated: transactionsCreated.Int64(), 224 | TransactionsConfirmed: transactionsConfirmed.Int64(), 225 | StaleBroadcasts: staleBroadcasts.Int64(), 226 | FailedBroadcasts: failedBroadcasts.Int64(), 227 | AddressesCreated: addressesCreated.Int64(), 228 | WorkflowsCompleted: workflowsCompleted, 229 | } 230 | } 231 | 232 | // CheckConstructionProgress contains the number of 233 | // currently broadcasting transactions and processing 234 | // jobs. 235 | type CheckConstructionProgress struct { 236 | Broadcasting int `json:"broadcasting"` 237 | Processing int `json:"processing"` 238 | } 239 | 240 | // ComputeCheckConstructionProgress computes 241 | // *CheckConstructionProgress. 242 | func ComputeCheckConstructionProgress( 243 | ctx context.Context, 244 | broadcasts *modules.BroadcastStorage, 245 | jobs *modules.JobStorage, 246 | ) *CheckConstructionProgress { 247 | inflight, err := broadcasts.GetAllBroadcasts(ctx) 248 | if err != nil { 249 | log.Printf("%s cannot get all broadcasts\n", err.Error()) 250 | return nil 251 | } 252 | 253 | processing, err := jobs.AllProcessing(ctx) 254 | if err != nil { 255 | log.Printf("%s cannot get all jobs\n", err.Error()) 256 | return nil 257 | } 258 | 259 | return &CheckConstructionProgress{ 260 | Broadcasting: len(inflight), 261 | Processing: len(processing), 262 | } 263 | } 264 | 265 | // CheckConstructionStatus contains CheckConstructionStats. 266 | type CheckConstructionStatus struct { 267 | Stats *CheckConstructionStats `json:"stats"` 268 | Progress *CheckConstructionProgress `json:"progress"` 269 | } 270 | 271 | // ComputeCheckConstructionStatus returns a populated 272 | // *CheckConstructionStatus. 273 | func ComputeCheckConstructionStatus( 274 | ctx context.Context, 275 | config *configuration.Configuration, 276 | counters *modules.CounterStorage, 277 | broadcasts *modules.BroadcastStorage, 278 | jobs *modules.JobStorage, 279 | ) *CheckConstructionStatus { 280 | return &CheckConstructionStatus{ 281 | Stats: ComputeCheckConstructionStats(ctx, config, counters, jobs), 282 | Progress: ComputeCheckConstructionProgress(ctx, broadcasts, jobs), 283 | } 284 | } 285 | 286 | // FetchCheckConstructionStatus fetches *CheckConstructionStatus. 287 | func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) { 288 | var status CheckConstructionStatus 289 | if err := JSONFetch(url, &status); err != nil { 290 | return nil, fmt.Errorf("unable to fetch check construction status: %w", err) 291 | } 292 | 293 | return &status, nil 294 | } 295 | 296 | // ExitConstruction exits check:construction, logs the test results to the console, 297 | // and to a provided output path. 298 | func ExitConstruction( 299 | config *configuration.Configuration, 300 | counterStorage *modules.CounterStorage, 301 | jobStorage *modules.JobStorage, 302 | err error, 303 | ) error { 304 | if !config.ErrorStackTraceDisabled { 305 | err = pkgError.WithStack(err) 306 | } 307 | 308 | results := ComputeCheckConstructionResults( 309 | config, 310 | err, 311 | counterStorage, 312 | jobStorage, 313 | ) 314 | if results != nil { 315 | results.Print() 316 | if config.Construction != nil { 317 | results.Output(config.Construction.ResultsOutputFile) 318 | } 319 | } 320 | 321 | return err 322 | } 323 | -------------------------------------------------------------------------------- /pkg/results/perf_results.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package results 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "os" 21 | "strconv" 22 | "time" 23 | 24 | "github.com/coinbase/rosetta-cli/configuration" 25 | "github.com/coinbase/rosetta-sdk-go/utils" 26 | "github.com/olekukonko/tablewriter" 27 | ) 28 | 29 | // Output writes *CheckPerfResults to the provided 30 | // path. 31 | func (c *CheckPerfStats) Output(path string) { 32 | if len(path) > 0 { 33 | writeErr := utils.SerializeAndWrite(path, c) 34 | if writeErr != nil { 35 | log.Printf("unable to save results: %s\n", writeErr.Error()) 36 | } 37 | } 38 | } 39 | 40 | type CheckPerfRawStats struct { 41 | BlockEndpointTotalTime time.Duration 42 | BlockEndpointNumErrors int64 43 | AccountBalanceEndpointTotalTime time.Duration 44 | AccountBalanceNumErrors int64 45 | } 46 | 47 | // CheckPerfStats contains interesting stats that 48 | // are counted while running the check:perf. 49 | type CheckPerfStats struct { 50 | StartBlock int64 `json:"start_block"` 51 | EndBlock int64 `json:"end_block"` 52 | NumTimesHitEachEndpoint int `json:"num_times_hit_each_endpoint"` 53 | AccountBalanceEndpointAverageTimeMs int64 `json:"account_balance_endpoint_average_time_ms"` 54 | AccountBalanceEndpointTotalTimeMs int64 `json:"account_balance_endpoint_total_time_ms"` 55 | AccountBalanceEndpointNumErrors int64 `json:"account_balance_endpoint_num_errors"` 56 | BlockEndpointAverageTimeMs int64 `json:"block_endpoint_average_time_ms"` 57 | BlockEndpointTotalTimeMs int64 `json:"block_endpoint_total_time_ms"` 58 | BlockEndpointNumErrors int64 `json:"block_endpoint_num_errors"` 59 | } 60 | 61 | // Print logs CheckPerfStats to the console. 62 | func (c *CheckPerfStats) Print() { 63 | table := tablewriter.NewWriter(os.Stdout) 64 | table.SetRowLine(true) 65 | table.SetRowSeparator("-") 66 | table.SetHeader([]string{"check:perf Stats", "Description", "Value"}) 67 | table.Append([]string{"Start Block", "The Starting Block", strconv.FormatInt(c.StartBlock, 10)}) 68 | table.Append([]string{"End Block", "The Ending Block", strconv.FormatInt(c.EndBlock, 10)}) 69 | table.Append([]string{"Num Times Each Endpoint", "Number of times that each endpoint was hit", strconv.FormatInt(int64(c.NumTimesHitEachEndpoint), 10)}) 70 | table.Append( 71 | []string{ 72 | "/Block Endpoint Total Time", 73 | "Total elapsed time taken to fetch all blocks (ms)", 74 | strconv.FormatInt(c.BlockEndpointTotalTimeMs, 10), 75 | }, 76 | ) 77 | table.Append( 78 | []string{ 79 | "/Block Endpoint Average Time", 80 | "Average time taken to fetch each block (ms)", 81 | strconv.FormatInt(c.BlockEndpointAverageTimeMs, 10), 82 | }, 83 | ) 84 | table.Append( 85 | []string{ 86 | "/Block Endpoint Num Errors", 87 | "Total num errors occurred while fetching blocks", 88 | strconv.FormatInt(c.BlockEndpointNumErrors, 10), 89 | }, 90 | ) 91 | table.Append( 92 | []string{ 93 | "/Account/Balance Endpoint Average Time", 94 | "Average time taken to fetch each account balance (ms)", 95 | strconv.FormatInt(c.AccountBalanceEndpointAverageTimeMs, 10), 96 | }, 97 | ) 98 | table.Append( 99 | []string{ 100 | "/Account/Balance Endpoint Total Time", 101 | "Total elapsed time taken to fetch all account balances (ms)", 102 | strconv.FormatInt(c.AccountBalanceEndpointTotalTimeMs, 10), 103 | }, 104 | ) 105 | table.Append( 106 | []string{ 107 | "/Account/Balance Endpoint Num Errors", 108 | "Total num errors occurred while fetching account balances", 109 | strconv.FormatInt(c.AccountBalanceEndpointNumErrors, 10), 110 | }, 111 | ) 112 | 113 | table.Render() 114 | } 115 | 116 | // ComputeCheckPerfStats returns a populated CheckPerfStats. 117 | func ComputeCheckPerfStats( 118 | config *configuration.CheckPerfConfiguration, 119 | rawStats *CheckPerfRawStats, 120 | ) *CheckPerfStats { 121 | totalNumEndpointsHit := (config.EndBlock - config.StartBlock) * int64(config.NumTimesToHitEndpoints) 122 | stats := &CheckPerfStats{ 123 | BlockEndpointAverageTimeMs: rawStats.BlockEndpointTotalTime.Milliseconds() / totalNumEndpointsHit, 124 | BlockEndpointTotalTimeMs: rawStats.BlockEndpointTotalTime.Milliseconds(), 125 | BlockEndpointNumErrors: rawStats.BlockEndpointNumErrors, 126 | AccountBalanceEndpointAverageTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds() / totalNumEndpointsHit, 127 | AccountBalanceEndpointTotalTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds(), 128 | AccountBalanceEndpointNumErrors: rawStats.AccountBalanceNumErrors, 129 | StartBlock: config.StartBlock, 130 | EndBlock: config.EndBlock, 131 | NumTimesHitEachEndpoint: config.NumTimesToHitEndpoints, 132 | } 133 | 134 | return stats 135 | } 136 | 137 | // ExitPerf exits check:perf, logs the test results to the console, 138 | // and to a provided output path. 139 | func ExitPerf( 140 | config *configuration.CheckPerfConfiguration, 141 | err error, 142 | rawStats *CheckPerfRawStats, 143 | ) error { 144 | if err != nil { 145 | log.Fatal(fmt.Errorf("Check:Perf Failed!: %w", err)) 146 | } 147 | 148 | stats := ComputeCheckPerfStats( 149 | config, 150 | rawStats, 151 | ) 152 | 153 | stats.Print() 154 | stats.Output(config.StatsOutputFile) 155 | 156 | return err 157 | } 158 | -------------------------------------------------------------------------------- /pkg/results/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package results 16 | 17 | const ( 18 | // TimeElapsedCounter tracks the total time elapsed in seconds. 19 | TimeElapsedCounter = "time_elapsed" 20 | ) 21 | -------------------------------------------------------------------------------- /pkg/results/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package results 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "io/ioutil" 21 | "net/http" 22 | ) 23 | 24 | // JSONFetch makes a GET request to the URL and unmarshal 25 | // the response into output. 26 | func JSONFetch(url string, output interface{}) error { 27 | resp, err := http.Get(url) // #nosec 28 | if err != nil { 29 | return fmt.Errorf("unable to fetch url %s: %w", url, err) 30 | } 31 | defer resp.Body.Close() 32 | 33 | body, err := ioutil.ReadAll(resp.Body) 34 | if err != nil { 35 | return fmt.Errorf("unable to read body: %w", err) 36 | } 37 | 38 | if resp.StatusCode != http.StatusOK { 39 | return fmt.Errorf("received %d status with body %s", resp.StatusCode, string(body)) 40 | } 41 | 42 | if err := json.Unmarshal(body, output); err != nil { 43 | return fmt.Errorf("unable to unmarshal: %w", err) 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/results/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package results 16 | 17 | import ( 18 | "fmt" 19 | "net/http" 20 | "net/http/httptest" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | func TestJSONFetch(t *testing.T) { 27 | var tests = map[string]struct { 28 | status int 29 | body string 30 | 31 | expectedResult map[string]interface{} 32 | expectedError string 33 | }{ 34 | "simple 200": { 35 | status: http.StatusOK, 36 | body: `{"test":"123"}`, 37 | expectedResult: map[string]interface{}{ 38 | "test": "123", 39 | }, 40 | }, 41 | "not 200": { 42 | status: http.StatusUnsupportedMediaType, 43 | body: `hello`, 44 | expectedError: "received 415 status with body hello\n", 45 | }, 46 | "not JSON": { 47 | status: http.StatusOK, 48 | body: `hello`, 49 | expectedError: "unable to unmarshal: invalid character 'h' looking for beginning of value", 50 | }, 51 | } 52 | 53 | for name, test := range tests { 54 | t.Run(name, func(t *testing.T) { 55 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 56 | assert.Equal(t, "GET", r.Method) 57 | 58 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 59 | w.WriteHeader(test.status) 60 | fmt.Fprintln(w, test.body) 61 | })) 62 | defer ts.Close() 63 | 64 | var obj map[string]interface{} 65 | err := JSONFetch(ts.URL, &obj) 66 | if len(test.expectedError) > 0 { 67 | assert.EqualError(t, err, test.expectedError) 68 | assert.Len(t, obj, 0) 69 | } else { 70 | assert.NoError(t, err) 71 | assert.Equal(t, test.expectedResult, obj) 72 | } 73 | }) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/tester/benchmark_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tester 16 | 17 | import "time" 18 | 19 | func timerFactory() func() time.Duration { 20 | start := time.Now() 21 | return func() time.Duration { 22 | return time.Since(start) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /pkg/tester/data_perf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tester 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/coinbase/rosetta-cli/configuration" 22 | cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" 23 | "github.com/coinbase/rosetta-cli/pkg/results" 24 | "github.com/coinbase/rosetta-sdk-go/fetcher" 25 | "github.com/coinbase/rosetta-sdk-go/types" 26 | ) 27 | 28 | func SetupBenchmarking(config *configuration.Configuration) (*fetcher.Fetcher, func() time.Duration, chan time.Duration) { 29 | // Create a new fetcher 30 | fetcher := fetcher.New( 31 | config.OnlineURL, 32 | fetcher.WithMaxRetries(0), 33 | ) 34 | timer := timerFactory() 35 | elapsed := make(chan time.Duration, 1) 36 | return fetcher, timer, elapsed 37 | } 38 | 39 | // Benchmark the asset issuer's /block endpoint 40 | func BmarkBlock(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error { 41 | total_errors := 0 42 | go func() { 43 | for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ { 44 | for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ { 45 | partialBlockId := &types.PartialBlockIdentifier{ 46 | Hash: nil, 47 | Index: &m, 48 | } 49 | _, err := fetcher.Block(ctx, config.Network, partialBlockId) 50 | if err != nil { 51 | total_errors++ 52 | } 53 | } 54 | } 55 | elapsed <- timer() 56 | }() 57 | select { 58 | case <-ctx.Done(): 59 | return cliErrs.ErrBlockBenchmarkTimeout 60 | case timeTaken := <-elapsed: 61 | rawStats.BlockEndpointTotalTime = timeTaken 62 | rawStats.BlockEndpointNumErrors = int64(total_errors) 63 | return nil 64 | } 65 | } 66 | 67 | // Benchmark the asset issuers /account/balance endpoint 68 | func BmarkAccountBalance(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error { 69 | total_errors := 0 70 | go func() { 71 | for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ { 72 | for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ { 73 | account := &types.AccountIdentifier{ 74 | Address: "address", 75 | } 76 | partialBlockId := &types.PartialBlockIdentifier{ 77 | Hash: nil, 78 | Index: &m, 79 | } 80 | _, _, _, err := fetcher.AccountBalance(ctx, config.Network, account, partialBlockId, nil) 81 | if err != nil { 82 | total_errors++ 83 | } 84 | } 85 | } 86 | elapsed <- timer() 87 | }() 88 | select { 89 | case <-ctx.Done(): 90 | return cliErrs.ErrAccountBalanceBenchmarkTimeout 91 | case timeTaken := <-elapsed: 92 | rawStats.AccountBalanceEndpointTotalTime = timeTaken 93 | rawStats.AccountBalanceNumErrors = int64(total_errors) 94 | return nil 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pkg/tester/general.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Coinbase, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tester 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "log" 21 | "net/http" 22 | "time" 23 | 24 | "github.com/coinbase/rosetta-cli/pkg/logger" 25 | ) 26 | 27 | const ( 28 | // MemoryLoggingFrequency is the frequency that memory 29 | // usage stats are logged to the terminal. 30 | MemoryLoggingFrequency = 10 * time.Second 31 | 32 | // ReadHeaderTimeout is the header timeout for server 33 | ReadHeaderTimeout = 5 * time.Second 34 | ) 35 | 36 | // LogMemoryLoop runs a loop that logs memory usage. 37 | func LogMemoryLoop( 38 | ctx context.Context, 39 | ) error { 40 | ticker := time.NewTicker(MemoryLoggingFrequency) 41 | defer ticker.Stop() 42 | 43 | for { 44 | select { 45 | case <-ctx.Done(): 46 | logger.LogMemoryStats(ctx) 47 | return ctx.Err() 48 | case <-ticker.C: 49 | logger.LogMemoryStats(ctx) 50 | } 51 | } 52 | } 53 | 54 | // StartServer stats a server at a port with a particular handler. 55 | // This is often used to support a status endpoint for a particular test. 56 | func StartServer( 57 | ctx context.Context, 58 | name string, 59 | handler http.Handler, 60 | port uint, 61 | ) error { 62 | server := &http.Server{ 63 | Addr: fmt.Sprintf(":%d", port), 64 | Handler: handler, 65 | ReadHeaderTimeout: ReadHeaderTimeout, 66 | } 67 | 68 | go func() { 69 | log.Printf("%s server running on port %d\n", name, port) 70 | _ = server.ListenAndServe() 71 | }() 72 | 73 | go func() { 74 | // If we don't shutdown server, it will 75 | // never stop because server.ListenAndServe doesn't 76 | // take any context. 77 | <-ctx.Done() 78 | log.Printf("%s server shutting down", name) 79 | 80 | _ = server.Shutdown(ctx) 81 | }() 82 | 83 | return ctx.Err() 84 | } 85 | -------------------------------------------------------------------------------- /scripts/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2020 Coinbase, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | VERSION=$1; 18 | 19 | go install github.com/crazy-max/xgo@latest 20 | 21 | MAC_TARGETS="darwin/amd64,darwin/arm64" 22 | LINUX_TARGETS="linux/amd64,linux/arm64,linux/mips64,linux/mips64le,linux/ppc64le,linux/s390x" 23 | WINDOWS_TARGET="windows/amd64" 24 | TARGETS="${MAC_TARGETS},${LINUX_TARGETS},${WINDOWS_TARGET}" 25 | 26 | xgo -go 1.16.3 --targets=${TARGETS} -out "bin/rosetta-cli-${VERSION}" .; 27 | 28 | # Rename some files 29 | mv "bin/rosetta-cli-${VERSION}-darwin-10.16-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" 30 | mv "bin/rosetta-cli-${VERSION}-darwin-10.16-arm64" "bin/rosetta-cli-${VERSION}-darwin-arm64" 31 | mv "bin/rosetta-cli-${VERSION}-windows-4.0-amd64.exe" "bin/rosetta-cli-${VERSION}-windows-amd64" 32 | 33 | # Tar all files 34 | cd bin || exit; 35 | for i in *; do tar -czf "$i.tar.gz" "$i" && rm "$i"; done 36 | 37 | go mod tidy 38 | -------------------------------------------------------------------------------- /scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2020 Coinbase, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -e 17 | 18 | # Inspired by: https://github.com/golangci/golangci-lint/blob/master/install.sh 19 | 20 | usage() { 21 | this=$1 22 | cat </dev/null 118 | } 119 | echoerr() { 120 | echo "$@" 1>&2 121 | } 122 | log_prefix() { 123 | echo "$0" 124 | } 125 | _logp=6 126 | log_set_priority() { 127 | _logp="$1" 128 | } 129 | log_priority() { 130 | if test -z "$1"; then 131 | echo "$_logp" 132 | return 133 | fi 134 | [ "$1" -le "$_logp" ] 135 | } 136 | log_tag() { 137 | case $1 in 138 | 0) echo "emerg" ;; 139 | 1) echo "alert" ;; 140 | 2) echo "crit" ;; 141 | 3) echo "err" ;; 142 | 4) echo "warning" ;; 143 | 5) echo "notice" ;; 144 | 6) echo "info" ;; 145 | 7) echo "debug" ;; 146 | *) echo "$1" ;; 147 | esac 148 | } 149 | log_debug() { 150 | log_priority 7 || return 0 151 | echoerr "$(log_prefix)" "$(log_tag 7)" "$@" 152 | } 153 | log_info() { 154 | log_priority 6 || return 0 155 | echoerr "$(log_prefix)" "$(log_tag 6)" "$@" 156 | } 157 | log_err() { 158 | log_priority 3 || return 0 159 | echoerr "$(log_prefix)" "$(log_tag 3)" "$@" 160 | } 161 | log_crit() { 162 | log_priority 2 || return 0 163 | echoerr "$(log_prefix)" "$(log_tag 2)" "$@" 164 | } 165 | uname_os() { 166 | os=$(uname -s | tr '[:upper:]' '[:lower:]') 167 | case "$os" in 168 | cygwin_nt*) os="windows" ;; 169 | mingw*) os="windows" ;; 170 | msys_nt*) os="windows" ;; 171 | esac 172 | echo "$os" 173 | } 174 | uname_arch() { 175 | arch=$(uname -m) 176 | case $arch in 177 | x86_64) arch="amd64" ;; 178 | x86) arch="386" ;; 179 | i686) arch="386" ;; 180 | i386) arch="386" ;; 181 | aarch64) arch="arm64" ;; 182 | armv5*) arch="armv5" ;; 183 | armv6*) arch="armv6" ;; 184 | armv7*) arch="armv7" ;; 185 | esac 186 | echo ${arch} 187 | } 188 | uname_os_check() { 189 | os=$(uname_os) 190 | case "$os" in 191 | darwin) return 0 ;; 192 | dragonfly) return 0 ;; 193 | freebsd) return 0 ;; 194 | linux) return 0 ;; 195 | android) return 0 ;; 196 | nacl) return 0 ;; 197 | netbsd) return 0 ;; 198 | openbsd) return 0 ;; 199 | plan9) return 0 ;; 200 | solaris) return 0 ;; 201 | windows) return 0 ;; 202 | esac 203 | log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" 204 | return 1 205 | } 206 | uname_arch_check() { 207 | arch=$(uname_arch) 208 | case "$arch" in 209 | 386) return 0 ;; 210 | amd64) return 0 ;; 211 | arm64) return 0 ;; 212 | armv5) return 0 ;; 213 | armv6) return 0 ;; 214 | armv7) return 0 ;; 215 | ppc64) return 0 ;; 216 | ppc64le) return 0 ;; 217 | mips) return 0 ;; 218 | mipsle) return 0 ;; 219 | mips64) return 0 ;; 220 | mips64le) return 0 ;; 221 | s390x) return 0 ;; 222 | amd64p32) return 0 ;; 223 | esac 224 | log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" 225 | return 1 226 | } 227 | untar() { 228 | tarball=$1 229 | case "${tarball}" in 230 | *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; 231 | *.tar) tar --no-same-owner -xf "${tarball}" ;; 232 | *.zip) unzip "${tarball}" ;; 233 | *) 234 | log_err "untar unknown archive format for ${tarball}" 235 | return 1 236 | ;; 237 | esac 238 | } 239 | http_download_curl() { 240 | local_file=$1 241 | source_url=$2 242 | header=$3 243 | if [ -z "$header" ]; then 244 | code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") 245 | else 246 | code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") 247 | fi 248 | if [ "$code" != "200" ]; then 249 | log_debug "http_download_curl received HTTP status $code" 250 | return 1 251 | fi 252 | return 0 253 | } 254 | http_download_wget() { 255 | local_file=$1 256 | source_url=$2 257 | header=$3 258 | if [ -z "$header" ]; then 259 | wget -q -O "$local_file" "$source_url" 260 | else 261 | wget -q --header "$header" -O "$local_file" "$source_url" 262 | fi 263 | } 264 | http_download() { 265 | log_debug "http_download $2" 266 | if is_command curl; then 267 | http_download_curl "$@" 268 | return 269 | elif is_command wget; then 270 | http_download_wget "$@" 271 | return 272 | fi 273 | log_crit "http_download unable to find wget or curl" 274 | return 1 275 | } 276 | http_copy() { 277 | tmp=$(mktemp) 278 | http_download "${tmp}" "$1" "$2" || return 1 279 | body=$(cat "$tmp") 280 | rm -f "${tmp}" 281 | echo "$body" 282 | } 283 | github_release() { 284 | owner_repo=$1 285 | version=$2 286 | test -z "$version" && version="latest" 287 | giturl="https://github.com/${owner_repo}/releases/${version}" 288 | json=$(http_copy "$giturl" "Accept:application/json") 289 | test -z "$json" && return 1 290 | version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') 291 | test -z "$version" && return 1 292 | echo "$version" 293 | } 294 | cat /dev/null <