├── .env.template ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature-request-or-epic.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── build-deploy-prod.yml │ ├── release.yml │ └── tests.yml ├── .gitignore ├── .golangci.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── config │ └── config.dev.yaml └── main.go ├── copyright.txt ├── crawler ├── crawl │ ├── crawl.go │ ├── service.go │ └── udpv5.go ├── p2p │ └── host.go ├── rpc │ ├── methods │ │ └── status.go │ └── request │ │ ├── buf_limit_read.go │ │ ├── compression.go │ │ ├── encode.go │ │ ├── handle_request.go │ │ ├── handle_response.go │ │ ├── request.go │ │ ├── request_test.go │ │ └── rpc_method.go ├── service.go └── util │ ├── util.go │ └── util_test.go ├── docker-compose.yaml ├── go.mod ├── go.sum ├── gqlgen.yml ├── graph ├── generated │ └── generated.go ├── model │ ├── helpers.go │ └── models_gen.go ├── resolver.go ├── schema.graphqls └── schema.resolvers.go ├── infra └── aws-ecs │ └── task_definition_PROD.json ├── models ├── data.go ├── history.go ├── marshallable_epoch.go └── peer.go ├── resolver ├── ipdata │ └── ipdata.go ├── ipgeolocation │ └── ipgeolocation.go └── resolver.go ├── store ├── peerstore │ ├── error.go │ ├── mongo │ │ └── mongo.go │ └── store.go └── record │ ├── mongo │ └── mongo.go │ └── store.go └── utils ├── config └── config.go └── server └── server.go /.env.template: -------------------------------------------------------------------------------- 1 | #url of mongodb connection 2 | MONGODB_URI= 3 | 4 | # ipdata api key 5 | # get it from https://dashboard.ipdata.co/ 6 | RESOLVER_API_KEY= -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | 12 | ## Expected Behavior 13 | 14 | 15 | 16 | ## Current Behavior 17 | 18 | 19 | 20 | ## Possible Solution 21 | 22 | 23 | 24 | ## Steps to Reproduce (for bugs) 25 | 26 | 27 | 1. 28 | 2. 29 | 3. 30 | 4. 31 | 32 | ## Versions 33 | Eth2-Crawler commit (or docker tag): 34 | Go version: -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request-or-epic.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request or epic 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Story 11 | As a 12 | I want 13 | So that I can 14 | 15 | ## Background 16 | 17 | 18 | ## Details 19 | 20 | 21 | ## Scenarios 22 | Scenario: 23 | Given I am 24 | When 25 | And 26 | Then 27 | 28 | ## Implementation details 29 | 30 | 31 | ## Testing details 32 | 33 | 34 | ## Acceptance criteria 35 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | ## Description 7 | 8 | 9 | 10 | ## Changes 11 | 12 | - 13 | - 14 | - 15 | 16 | 17 | 22 | #### Closes: #? 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | version: 2 5 | updates: 6 | # Maintain dependencies for GitHub Actions 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | 12 | - package-ecosystem: "gomod" 13 | directory: "/" 14 | schedule: 15 | interval: "weekly" -------------------------------------------------------------------------------- /.github/workflows/build-deploy-prod.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | name: Build & Deploy PROD 5 | 6 | on: 7 | workflow_call: 8 | inputs: 9 | tag: 10 | required: true 11 | type: string 12 | secrets: 13 | aws_region: 14 | required: true 15 | aws_account_id: 16 | required: true 17 | ecr_repo: 18 | required: true 19 | ecs_cluster: 20 | required: true 21 | ecs_service: 22 | required: true 23 | 24 | env: 25 | ENVIRONMENT: PROD 26 | 27 | jobs: 28 | build: 29 | name: build_deploy_prod 30 | runs-on: ubuntu-latest 31 | 32 | permissions: 33 | contents: read 34 | id-token: write 35 | 36 | steps: 37 | # download the source code into the runner 38 | - name: checkout 39 | uses: actions/checkout@v2 40 | 41 | - name: Set output 42 | id: vars 43 | run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} 44 | 45 | - name: Configure AWS credentials 46 | uses: aws-actions/configure-aws-credentials@v4 47 | with: 48 | role-to-assume: arn:aws:iam::${{ secrets.aws_account_id }}:role/github-actions-role 49 | aws-region: ${{ secrets.aws_region }} 50 | role-session-name: GithubActions 51 | 52 | # gather metadata from git & github actions to reference in docker 53 | - name: git & github metadata 54 | id: metadata 55 | uses: docker/metadata-action@v3 56 | with: 57 | images: ${{ secrets.aws_account_id }}.dkr.ecr.${{ secrets.aws_region }}.amazonaws.com/${{ secrets.ecr_repo }} 58 | 59 | # login in docker repository 60 | - name: docker login 61 | uses: aws-actions/amazon-ecr-login@v1 62 | 63 | # build a docker image 64 | - name: docker & push image 65 | uses: docker/build-push-action@v2 66 | with: 67 | context: . 68 | file: ./Dockerfile 69 | push: true 70 | tags: | 71 | ${{ secrets.aws_account_id }}.dkr.ecr.${{ secrets.aws_region }}.amazonaws.com/${{ secrets.ecr_repo }}:latest 72 | ${{ secrets.aws_account_id }}.dkr.ecr.${{ secrets.aws_region }}.amazonaws.com/${{ secrets.ecr_repo }}:${{ steps.vars.outputs.tag }} 73 | 74 | # deploy to AWS ECS 75 | - name: Deploy to Amazon ECS 76 | uses: aws-actions/amazon-ecs-deploy-task-definition@v1 77 | with: 78 | task-definition: infra/aws-ecs/task_definition_${{ env.ENVIRONMENT }}.json 79 | service: ${{ secrets.ecs_service }} 80 | cluster: ${{ secrets.ecs_cluster }} 81 | wait-for-service-stability: true 82 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | name: Deploy Release 4 | on: 5 | push: 6 | branches: 7 | - main 8 | jobs: 9 | release: 10 | permissions: write-all 11 | runs-on: ubuntu-latest 12 | outputs: 13 | release_created: ${{ steps.release.outputs.release_created }} 14 | tag: ${{ steps.release.outputs.tag_name }} 15 | steps: 16 | - uses: GoogleCloudPlatform/release-please-action@v3.1 17 | id: release 18 | with: 19 | release-type: go 20 | token: ${{secrets.GITHUB_TOKEN}} 21 | 22 | deploy-services: 23 | needs: release 24 | uses: ChainSafe/nodewatch-api/.github/workflows/build-deploy-prod.yml@main 25 | if: ${{ needs.release.outputs.release_created }} 26 | with: 27 | tag: ${{ needs.release.outputs.tag }} 28 | secrets: 29 | aws_region: ${{ secrets.AWS_REGION }} 30 | aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} 31 | ecr_repo: ${{ secrets.AWS_ECR_REPO }} 32 | ecs_cluster: ${{ secrets.AWS_ECS_CLUSTER }} 33 | ecs_service: ${{ secrets.AWS_ECS_SERVICE }} 34 | 35 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | name: Tests 5 | 6 | on: 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | 10 | jobs: 11 | test: 12 | name: Test 13 | strategy: 14 | matrix: 15 | go-version: [ 1.20.x ] 16 | platform: [ ubuntu-latest ] 17 | runs-on: ${{ matrix.platform }} 18 | steps: 19 | - name: Install Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: ${{ matrix.go-version }} 23 | - name: Checkout code 24 | uses: actions/checkout@v3 25 | - uses: actions/cache@v2.1.7 26 | with: 27 | path: ~/go/pkg/mod 28 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 29 | restore-keys: | 30 | ${{ runner.os }}-go- 31 | - name: Test 32 | run: | 33 | make test 34 | 35 | lint: 36 | name: Lint & License Check 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/setup-go@v3 40 | with: 41 | go-version: '1.20.7' 42 | - uses: actions/checkout@v3 43 | - name: golangci-lint 44 | uses: golangci/golangci-lint-action@v3 45 | with: 46 | version: v1.54.1 47 | skip-cache: false 48 | skip-pkg-cache: false 49 | skip-build-cache: false 50 | - name: License Check 51 | run: make license-check 52 | 53 | build: 54 | name: Docker Build 55 | runs-on: ubuntu-latest 56 | steps: 57 | - uses: actions/checkout@v3 58 | - name: Set up Docker Buildx 59 | uses: docker/setup-buildx-action@v1 60 | - name: Build 61 | uses: docker/build-push-action@v2 62 | with: 63 | context: . 64 | file: ./Dockerfile 65 | push: false 66 | tags: ${{ github.repository }}:latest 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | .gitconfig 8 | 9 | # Test binary, build with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | coverage.txt 15 | 16 | .DS_Store 17 | 18 | # IDEs and Editors 19 | .idea/ 20 | .vscode/ 21 | logs/ 22 | 23 | # Vendored Dependencies 24 | vendor/ 25 | 26 | bin/ 27 | docs/ 28 | .env 29 | 30 | helm/stage.yaml 31 | helm/prod.yaml -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | # options for analysis running 5 | run: 6 | # default concurrency is a available CPU number 7 | concurrency: 6 8 | 9 | # timeout for analysis, e.g. 30s, 5m, default is 1m 10 | timeout: 5m 11 | 12 | # exit code when at least one issue was found, default is 1 13 | issues-exit-code: 1 14 | 15 | # include test files or not, default is true 16 | tests: true 17 | 18 | # list of build tags, all linters use it. Default is empty list. 19 | build-tags: 20 | - mytag 21 | 22 | # which dirs to skip: issues from them won't be reported; 23 | # can use regexp here: generated.*, regexp is applied on full path; 24 | # default value is empty list, but default dirs are skipped independently 25 | # from this option's value (see skip-dirs-use-default). 26 | skip-dirs: 27 | - src/external_libs 28 | - autogenerated_by_my_lib 29 | 30 | # default is true. Enables skipping of directories: 31 | # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 32 | skip-dirs-use-default: true 33 | 34 | # which files to skip: they will be analyzed, but issues from them 35 | # won't be reported. Default value is empty list, but there is 36 | # no need to include all autogenerated files, we confidently recognize 37 | # autogenerated files. If it's not please let us know. 38 | skip-files: 39 | - ".*\\.my\\.go$" 40 | - lib/bad.go 41 | 42 | # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": 43 | # If invoked with -mod=readonly, the go command is disallowed from the implicit 44 | # automatic updating of go.mod described above. Instead, it fails when any changes 45 | # to go.mod are needed. This setting is most useful to check that go.mod does 46 | # not need updates, such as in a continuous integration and testing system. 47 | # If invoked with -mod=vendor, the go command assumes that the vendor 48 | # directory holds the correct copies of dependencies and ignores 49 | # the dependency descriptions in go.mod. 50 | # modules-download-mode: readonly|release|vendor 51 | 52 | 53 | # output configuration options 54 | output: 55 | # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" 56 | format: colored-line-number 57 | 58 | # print lines of code with issue, default is true 59 | print-issued-lines: true 60 | 61 | # print linter name in the end of issue text, default is true 62 | print-linter-name: true 63 | 64 | 65 | # all available settings of specific linters 66 | linters-settings: 67 | errcheck: 68 | # report about not checking of errors in type assetions: `a := b.(MyStruct)`; 69 | # default is false: such cases aren't reported by default. 70 | check-type-assertions: false 71 | 72 | # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; 73 | # default is false: such cases aren't reported by default. 74 | check-blank: false 75 | 76 | # [deprecated] comma-separated list of pairs of the form pkg:regex 77 | # the regex is used to ignore names within pkg. (default "fmt:.*"). 78 | # see https://github.com/kisielk/errcheck#the-deprecated-method for details 79 | ignore: fmt:.*,io/ioutil:^Read.* 80 | 81 | # path to a file containing a list of functions to exclude from checking 82 | # see https://github.com/kisielk/errcheck#excluding-functions for details 83 | #exclude: /path/to/file.txt 84 | 85 | funlen: 86 | lines: 200 87 | statements: 100 88 | 89 | govet: 90 | # report about shadowed variables 91 | check-shadowing: true 92 | 93 | # settings per analyzer 94 | settings: 95 | printf: # analyzer name, run `go tool vet help` to see all analyzers 96 | funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer 97 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof 98 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf 99 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf 100 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf 101 | 102 | # enable or disable analyzers by name 103 | enable: 104 | - atomicalign 105 | enable-all: false 106 | disable: 107 | - shadow 108 | disable-all: false 109 | gofmt: 110 | # simplify code: gofmt with `-s` option, true by default 111 | simplify: false 112 | goimports: 113 | # put imports beginning with prefix after 3rd-party packages; 114 | # it's a comma-separated list of prefixes 115 | local-prefixes: github.com/org/project 116 | gocyclo: 117 | # minimal code complexity to report, 30 by default (but we recommend 10-20) 118 | min-complexity: 20 119 | gocognit: 120 | # minimal code complexity to report, 30 by default (but we recommend 10-20) 121 | min-complexity: 10 122 | maligned: 123 | # print struct with more effective memory layout or not, false by default 124 | suggest-new: true 125 | dupl: 126 | # tokens count to trigger issue, 150 by default 127 | threshold: 100 128 | goconst: 129 | # minimal length of string constant, 3 by default 130 | min-len: 3 131 | # minimal occurrences count to trigger, 3 by default 132 | min-occurrences: 3 133 | depguard: 134 | rules: 135 | main: 136 | allow: 137 | - $gostd 138 | - eth2-crawler 139 | - github.com 140 | - $all # List of file globs that will match this list of settings to compare against. 141 | misspell: 142 | # Correct spellings using locale preferences for US or UK. 143 | # Default is to use a neutral variety of English. 144 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 145 | locale: US 146 | ignore-words: 147 | - someword 148 | lll: 149 | # max line length, lines longer will be reported. Default is 120. 150 | # '\t' is counted as 1 character by default, and can be changed with the tab-width option 151 | line-length: 200 152 | # tab width in spaces. Default to 1. 153 | tab-width: 1 154 | unused: 155 | # treat code as a program (not a library) and report unused exported identifiers; default is false. 156 | # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: 157 | # if it's called for subdir of a project it can't find funcs usages. All text editor integrations 158 | # with golangci-lint call it on a directory with the changed file. 159 | check-exported: false 160 | unparam: 161 | # Inspect exported functions, default is false. Set to true if no external program/library imports your code. 162 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 163 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 164 | # with golangci-lint call it on a directory with the changed file. 165 | check-exported: false 166 | nakedret: 167 | # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 168 | max-func-lines: 30 169 | prealloc: 170 | # XXX: we don't recommend using this linter before doing performance profiling. 171 | # For most programs usage of prealloc will be a premature optimization. 172 | 173 | # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. 174 | # True by default. 175 | simple: true 176 | range-loops: true # Report preallocation suggestions on range loops, true by default 177 | for-loops: false # Report preallocation suggestions on for loops, false by default 178 | gocritic: 179 | # Which checks should be enabled; can't be combined with 'disabled-checks'; 180 | # See https://go-critic.github.io/overview#checks-overview 181 | # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` 182 | # By default list of stable checks is used. 183 | enabled-checks: 184 | - commentedOutCode 185 | 186 | # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty 187 | disabled-checks: 188 | - regexpMust 189 | 190 | # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. 191 | # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". 192 | enabled-tags: 193 | - performance 194 | 195 | settings: # settings passed to gocritic 196 | captLocal: # must be valid enabled check name 197 | paramsOnly: true 198 | rangeValCopy: 199 | sizeThreshold: 32 200 | godox: 201 | # report any comments starting with keywords, this is useful for TODO or FIXME comments that 202 | # might be left in the code accidentally and should be resolved before merging 203 | keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting 204 | - NOTE 205 | - FIXME 206 | dogsled: 207 | # checks assignments with too many blank identifiers; default is 2 208 | max-blank-identifiers: 2 209 | 210 | whitespace: 211 | multi-if: false # Enforces newlines (or comments) after every multi-line if statement 212 | multi-func: false # Enforces newlines (or comments) after every multi-line function signature 213 | wsl: 214 | # If true append is only allowed to be cuddled if appending value is 215 | # matching variables, fields or types on line above. Default is true. 216 | strict-append: true 217 | # Allow calls and assignments to be cuddled as long as the lines have any 218 | # matching variables, fields or types. Default is true. 219 | allow-assign-and-call: true 220 | # Allow multiline assignments to be cuddled. Default is true. 221 | allow-multiline-assign: true 222 | # Allow case blocks to end with a whitespace. 223 | allow-case-traling-whitespace: true 224 | # Allow declarations (var) to be cuddled. 225 | allow-cuddle-declarations: false 226 | exhaustive: 227 | # indicates that switch statements are to be considered exhaustive if a 228 | # 'default' case is present, even if all enum members aren't listed in the 229 | # switch 230 | default-signifies-exhaustive: true 231 | linters: 232 | enable: 233 | - bodyclose 234 | - deadcode 235 | - depguard 236 | - dogsled 237 | - errcheck 238 | - funlen 239 | - goconst 240 | - gocritic 241 | - gocyclo 242 | - godox 243 | - gofmt 244 | - goimports 245 | - gosec 246 | - gosimple 247 | - govet 248 | - ineffassign 249 | - lll 250 | - misspell 251 | - nakedret 252 | - exportloopref 253 | - staticcheck 254 | - structcheck 255 | - stylecheck 256 | - typecheck 257 | - unconvert 258 | - unparam 259 | - unused 260 | - varcheck 261 | - whitespace 262 | disable: 263 | - gochecknoinits 264 | - gochecknoglobals 265 | - gocognit 266 | - maligned 267 | - prealloc 268 | - dupl 269 | - noctx 270 | presets: 271 | - bugs 272 | - unused 273 | fast: false 274 | 275 | 276 | issues: 277 | # List of regexps of issue texts to exclude, empty list by default. 278 | # But independently from this option we use default exclude patterns, 279 | # it can be disabled by `exclude-use-default: false`. To list all 280 | # excluded by default patterns execute `golangci-lint run --help` 281 | exclude: 282 | - G304 283 | - G107 284 | 285 | # Excluding configuration per-path, per-linter, per-text and per-source 286 | exclude-rules: 287 | # Exclude some linters from running on tests files. 288 | - path: _test\.go 289 | linters: 290 | - gocyclo 291 | - errcheck 292 | - dupl 293 | - gosec 294 | - funlen 295 | - gochecknoinits 296 | 297 | # Exclude known linters from partially hard-vendored code, 298 | # which is impossible to exclude via "nolint" comments. 299 | - path: internal/hmac/ 300 | text: "weak cryptographic primitive" 301 | linters: 302 | - gosec 303 | 304 | # Exclude some staticcheck messages 305 | - linters: 306 | - staticcheck 307 | text: "SA9003:" 308 | 309 | # Exclude lll issues for long lines with go:generate 310 | - linters: 311 | - lll 312 | source: "^//go:generate " 313 | 314 | # Independently from option `exclude` we use default exclude patterns, 315 | # it can be disabled by this option. To list all 316 | # excluded by default patterns execute `golangci-lint run --help`. 317 | # Default value for this option is true. 318 | exclude-use-default: false 319 | 320 | # Maximum issues count per one linter. Set to 0 to disable. Default is 50. 321 | max-issues-per-linter: 0 322 | 323 | # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. 324 | max-same-issues: 0 325 | 326 | # Show only new issues: if there are unstaged changes or untracked files, 327 | # only those changes are analyzed, else only changes in HEAD~ are analyzed. 328 | # It's a super-useful option for integration of golangci-lint into existing 329 | # large codebase. It's not practical to fix all existing issues at the moment 330 | # of integration: much better don't allow issues in new code. 331 | # Default is false. 332 | new: false 333 | 334 | # Show only new issues created after git revision `REV` 335 | # new-from-rev: REV 336 | 337 | # Show only new issues created in git patch with set file path. 338 | # new-from-patch: path/to/patch/file -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [1.5.0](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.7...v1.5.0) (2024-01-24) 4 | 5 | 6 | ### Features 7 | 8 | * Added api for checking altair upgrade percentage ([#110](https://github.com/ChainSafe/nodewatch-api/issues/110)) ([dc99a2b](https://github.com/ChainSafe/nodewatch-api/commit/dc99a2b61eac1a6359a54c3bdef4b336f20896b0)) 9 | * added raw agent version in db ([#88](https://github.com/ChainSafe/nodewatch-api/issues/88)) ([38cf181](https://github.com/ChainSafe/nodewatch-api/commit/38cf181ee789927d7f55c2ec067248eb3e62387b)) 10 | * ECS task deployment config file added ([#217](https://github.com/ChainSafe/nodewatch-api/issues/217)) ([bf3c570](https://github.com/ChainSafe/nodewatch-api/commit/bf3c57036ca29fdb4ce360b8616ed3345829227f)) 11 | * eth2 crawler configured with basic node information ([#84](https://github.com/ChainSafe/nodewatch-api/issues/84)) ([7d17887](https://github.com/ChainSafe/nodewatch-api/commit/7d17887b0f042b06de5268a26015d25ffdb661f4)) 12 | * loadstart parser added ([#99](https://github.com/ChainSafe/nodewatch-api/issues/99)) ([d5da38a](https://github.com/ChainSafe/nodewatch-api/commit/d5da38aaecdc36b7d852a8ce60be68156c6a8dc0)) 13 | 14 | 15 | ### Bug Fixes 16 | 17 | * actions deployment ([#259](https://github.com/ChainSafe/nodewatch-api/issues/259)) ([2f61383](https://github.com/ChainSafe/nodewatch-api/commit/2f6138311b69efc06fdbc8ea3e94cb20fd2f0fee)) 18 | * actions deployment ([#262](https://github.com/ChainSafe/nodewatch-api/issues/262)) ([b08ba21](https://github.com/ChainSafe/nodewatch-api/commit/b08ba21b64c596b9796caaffb4fe03e983a6e2c3)) 19 | * bump github.com/ethereum/go-ethereum from 1.10.10 to 1.10.11 ([#118](https://github.com/ChainSafe/nodewatch-api/issues/118)) ([d556e00](https://github.com/ChainSafe/nodewatch-api/commit/d556e00426457b6e6946e08c9bd3c24b7d3add61)) 20 | * ci pipeline ([#228](https://github.com/ChainSafe/nodewatch-api/issues/228)) ([9216648](https://github.com/ChainSafe/nodewatch-api/commit/9216648296620e7187300a03b5b043fb94c55d0f)) 21 | * crawler update ([#230](https://github.com/ChainSafe/nodewatch-api/issues/230)) ([d253c50](https://github.com/ChainSafe/nodewatch-api/commit/d253c5063f91c678bfef9c48235acea5f5a16ad0)) 22 | * deployment gh action ([#219](https://github.com/ChainSafe/nodewatch-api/issues/219)) ([c14b72a](https://github.com/ChainSafe/nodewatch-api/commit/c14b72abaef78a5c8da50caa9980eb7b6f29b5b7)) 23 | * ecs fixed, release ci fixed ([#236](https://github.com/ChainSafe/nodewatch-api/issues/236)) ([6da48aa](https://github.com/ChainSafe/nodewatch-api/commit/6da48aa4689c495a68cb1a9fd00b0232291a1e97)) 24 | * gh action ([#221](https://github.com/ChainSafe/nodewatch-api/issues/221)) ([e71105e](https://github.com/ChainSafe/nodewatch-api/commit/e71105e292c7a479b257b9417c8eab5c237e2753)) 25 | * go-multiaddress updated ([#258](https://github.com/ChainSafe/nodewatch-api/issues/258)) ([6750656](https://github.com/ChainSafe/nodewatch-api/commit/67506566e289bd9c4d962d6549d377fde1709c53)) 26 | * node stats ([#96](https://github.com/ChainSafe/nodewatch-api/issues/96)) ([ec95a2b](https://github.com/ChainSafe/nodewatch-api/commit/ec95a2b298c681bdfb58273c7bdbd610962ca5c8)) 27 | * prod deployment ([#225](https://github.com/ChainSafe/nodewatch-api/issues/225)) ([ec7154d](https://github.com/ChainSafe/nodewatch-api/commit/ec7154d7b1de72357865794c8a0b7a80d1584905)) 28 | * Revert "fix: ecs fixed, release ci fixed" ([#237](https://github.com/ChainSafe/nodewatch-api/issues/237)) ([7caa71f](https://github.com/ChainSafe/nodewatch-api/commit/7caa71fc133327d403dfbe38483f3fe36b43ee79)) 29 | * scale ecs cpu & ram ([#234](https://github.com/ChainSafe/nodewatch-api/issues/234)) ([a526299](https://github.com/ChainSafe/nodewatch-api/commit/a5262992fe0408be3cece09983c544ba412a3388)) 30 | * syntext fixed ([#227](https://github.com/ChainSafe/nodewatch-api/issues/227)) ([d0a1880](https://github.com/ChainSafe/nodewatch-api/commit/d0a1880ef126a5df97060c87f167a12c0b39e028)) 31 | * update peer with latest data ([#202](https://github.com/ChainSafe/nodewatch-api/issues/202)) ([135548c](https://github.com/ChainSafe/nodewatch-api/commit/135548c21895f37d1b208fe50ad64b911a3102f9)) 32 | * Updated Config ([#94](https://github.com/ChainSafe/nodewatch-api/issues/94)) ([8e9adf5](https://github.com/ChainSafe/nodewatch-api/commit/8e9adf5049e34bbddd520218ee4bd5a43e042c72)) 33 | 34 | ### [1.4.7](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.6...v1.4.7) (2024-01-18) 35 | 36 | 37 | ### Bug Fixes 38 | 39 | * actions deployment ([#259](https://github.com/ChainSafe/nodewatch-api/issues/259)) ([2f61383](https://github.com/ChainSafe/nodewatch-api/commit/2f6138311b69efc06fdbc8ea3e94cb20fd2f0fee)) 40 | 41 | ### [1.4.6](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.5...v1.4.6) (2024-01-15) 42 | 43 | 44 | ### Bug Fixes 45 | 46 | * ecs fixed, release ci fixed ([#236](https://github.com/ChainSafe/nodewatch-api/issues/236)) ([6da48aa](https://github.com/ChainSafe/nodewatch-api/commit/6da48aa4689c495a68cb1a9fd00b0232291a1e97)) 47 | * go-multiaddress updated ([#258](https://github.com/ChainSafe/nodewatch-api/issues/258)) ([6750656](https://github.com/ChainSafe/nodewatch-api/commit/67506566e289bd9c4d962d6549d377fde1709c53)) 48 | * Revert "fix: ecs fixed, release ci fixed" ([#237](https://github.com/ChainSafe/nodewatch-api/issues/237)) ([7caa71f](https://github.com/ChainSafe/nodewatch-api/commit/7caa71fc133327d403dfbe38483f3fe36b43ee79)) 49 | 50 | ### [1.4.5](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.4...v1.4.5) (2022-11-01) 51 | 52 | 53 | ### Bug Fixes 54 | 55 | * scale ecs cpu & ram ([#234](https://github.com/ChainSafe/nodewatch-api/issues/234)) ([a526299](https://github.com/ChainSafe/nodewatch-api/commit/a5262992fe0408be3cece09983c544ba412a3388)) 56 | 57 | ### [1.4.4](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.3...v1.4.4) (2022-10-31) 58 | 59 | 60 | ### Bug Fixes 61 | 62 | * ci pipeline ([#228](https://github.com/ChainSafe/nodewatch-api/issues/228)) ([9216648](https://github.com/ChainSafe/nodewatch-api/commit/9216648296620e7187300a03b5b043fb94c55d0f)) 63 | * crawler update ([#230](https://github.com/ChainSafe/nodewatch-api/issues/230)) ([d253c50](https://github.com/ChainSafe/nodewatch-api/commit/d253c5063f91c678bfef9c48235acea5f5a16ad0)) 64 | * prod deployment ([#225](https://github.com/ChainSafe/nodewatch-api/issues/225)) ([ec7154d](https://github.com/ChainSafe/nodewatch-api/commit/ec7154d7b1de72357865794c8a0b7a80d1584905)) 65 | * syntext fixed ([#227](https://github.com/ChainSafe/nodewatch-api/issues/227)) ([d0a1880](https://github.com/ChainSafe/nodewatch-api/commit/d0a1880ef126a5df97060c87f167a12c0b39e028)) 66 | 67 | ### [1.4.3](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.2...v1.4.3) (2022-10-26) 68 | 69 | 70 | ### Bug Fixes 71 | 72 | * update peer with latest data ([#202](https://github.com/ChainSafe/nodewatch-api/issues/202)) ([135548c](https://github.com/ChainSafe/nodewatch-api/commit/135548c21895f37d1b208fe50ad64b911a3102f9)) 73 | 74 | ### [1.4.2](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.1...v1.4.2) (2022-10-26) 75 | 76 | 77 | ### Bug Fixes 78 | 79 | * gh action ([#221](https://github.com/ChainSafe/nodewatch-api/issues/221)) ([e71105e](https://github.com/ChainSafe/nodewatch-api/commit/e71105e292c7a479b257b9417c8eab5c237e2753)) 80 | 81 | ### [1.4.1](https://github.com/ChainSafe/nodewatch-api/compare/v1.4.0...v1.4.1) (2022-10-26) 82 | 83 | 84 | ### Bug Fixes 85 | 86 | * deployment gh action ([#219](https://github.com/ChainSafe/nodewatch-api/issues/219)) ([c14b72a](https://github.com/ChainSafe/nodewatch-api/commit/c14b72abaef78a5c8da50caa9980eb7b6f29b5b7)) 87 | 88 | ## [1.4.0](https://github.com/ChainSafe/nodewatch-api/compare/v1.3.1...v1.4.0) (2022-10-26) 89 | 90 | 91 | ### Features 92 | 93 | * ECS task deployment config file added ([#217](https://github.com/ChainSafe/nodewatch-api/issues/217)) ([bf3c570](https://github.com/ChainSafe/nodewatch-api/commit/bf3c57036ca29fdb4ce360b8616ed3345829227f)) 94 | 95 | ### [1.3.1](https://www.github.com/ChainSafe/eth2-crawler/compare/v1.3.0...v1.3.1) (2021-10-23) 96 | 97 | 98 | ### Bug Fixes 99 | 100 | * bump github.com/ethereum/go-ethereum from 1.10.10 to 1.10.11 ([#118](https://www.github.com/ChainSafe/eth2-crawler/issues/118)) ([d556e00](https://www.github.com/ChainSafe/eth2-crawler/commit/d556e00426457b6e6946e08c9bd3c24b7d3add61)) 101 | 102 | ## [1.3.0](https://www.github.com/ChainSafe/eth2-crawler/compare/v1.2.0...v1.3.0) (2021-10-18) 103 | 104 | 105 | ### Features 106 | 107 | * Added api for checking altair upgrade percentage ([#110](https://www.github.com/ChainSafe/eth2-crawler/issues/110)) ([dc99a2b](https://www.github.com/ChainSafe/eth2-crawler/commit/dc99a2b61eac1a6359a54c3bdef4b336f20896b0)) 108 | 109 | ## [1.2.0](https://www.github.com/ChainSafe/eth2-crawler/compare/v1.1.1...v1.2.0) (2021-09-08) 110 | 111 | 112 | ### Features 113 | 114 | * loadstart parser added ([#99](https://www.github.com/ChainSafe/eth2-crawler/issues/99)) ([d5da38a](https://www.github.com/ChainSafe/eth2-crawler/commit/d5da38aaecdc36b7d852a8ce60be68156c6a8dc0)) 115 | 116 | ### [1.1.1](https://www.github.com/ChainSafe/eth2-crawler/compare/v1.1.0...v1.1.1) (2021-09-06) 117 | 118 | 119 | ### Bug Fixes 120 | 121 | * node stats ([#96](https://www.github.com/ChainSafe/eth2-crawler/issues/96)) ([ec95a2b](https://www.github.com/ChainSafe/eth2-crawler/commit/ec95a2b298c681bdfb58273c7bdbd610962ca5c8)) 122 | * Updated Config ([#94](https://www.github.com/ChainSafe/eth2-crawler/issues/94)) ([8e9adf5](https://www.github.com/ChainSafe/eth2-crawler/commit/8e9adf5049e34bbddd520218ee4bd5a43e042c72)) 123 | 124 | ## [1.1.0](https://www.github.com/ChainSafe/eth2-crawler/compare/v1.0.0...v1.1.0) (2021-08-30) 125 | 126 | 127 | ### Features 128 | 129 | * added raw agent version in db ([#88](https://www.github.com/ChainSafe/eth2-crawler/issues/88)) ([38cf181](https://www.github.com/ChainSafe/eth2-crawler/commit/38cf181ee789927d7f55c2ec067248eb3e62387b)) 130 | 131 | ## 1.0.0 (2021-08-23) 132 | 133 | 134 | ### Features 135 | 136 | * eth2 crawler configured with basic node information ([#84](https://www.github.com/ChainSafe/eth2-crawler/issues/84)) ([7d17887](https://www.github.com/ChainSafe/eth2-crawler/commit/7d17887b0f042b06de5268a26015d25ffdb661f4)) 137 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | FROM golang:1.20-alpine AS builder 5 | 6 | RUN apk add build-base 7 | WORKDIR /code 8 | COPY go.mod . 9 | COPY go.sum . 10 | RUN go mod download && go mod tidy 11 | 12 | # build the binary 13 | ADD . . 14 | RUN env GOOS=linux GOARCH=amd64 go build -o /crawler cmd/main.go 15 | 16 | # final stage 17 | FROM alpine:3.14.0 18 | 19 | RUN apk add build-base 20 | ARG env=dev 21 | 22 | RUN apk add curl 23 | COPY --from=builder /crawler / 24 | COPY cmd/config/config.$env.yaml /config.yaml 25 | 26 | RUN chmod +x /crawler 27 | ENTRYPOINT ["/crawler", "-p", "/config.yaml"] 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECTNAME=$(shell basename "$(PWD)") 2 | GOLANGCI := $(GOPATH)/bin/golangci-lint 3 | 4 | .PHONY: help lint test run 5 | all: help 6 | help: Makefile 7 | @echo 8 | @echo " Choose a make command to run in "$(PROJECTNAME)":" 9 | @echo 10 | @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' 11 | @echo 12 | 13 | .PHONY: get-lint 14 | get-lint: 15 | if [ ! -f ./bin/golangci-lint ]; then \ 16 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.54.2; \ 17 | fi; 18 | 19 | .PHONY: lint 20 | lint: get-lint 21 | @echo " > \033[32mRunning lint...\033[0m " 22 | ./bin/golangci-lint run --config=./.golangci.yml 23 | 24 | ## license: Adds license header to missing files. 25 | license: 26 | @echo " > \033[32mAdding license headers...\033[0m " 27 | GO111MODULE=off go get -u github.com/google/addlicense 28 | addlicense -c "ChainSafe Systems" -f ./copyright.txt -y 2021 . 29 | 30 | ## license-check: Checks for missing license headers 31 | license-check: 32 | @echo " > \033[32mChecking for license headers...\033[0m " 33 | GO111MODULE=off go get -u github.com/google/addlicense 34 | addlicense -check -c "ChainSafe Systems" -f ./copyright.txt -y 2021 . 35 | 36 | test: 37 | go test ./... 38 | 39 | build: 40 | go build -o ./bin/crawler cmd/main.go 41 | 42 | run: 43 | @echo " > \033[32mUsing Docker Container for development...\033[0m " 44 | @echo " > \033[32mRemoving old User Service stuff...\033[0m " 45 | docker-compose -f docker-compose.yaml down -v 46 | @echo " > \033[32mStarting Crawler Service w/o build...\033[0m " 47 | docker-compose -f docker-compose.yaml up --build $$scale 48 | 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Eth2 Crawler 2 | Eth2 Crawler is Ethereum blockchain project that extracts eth2 node information from the network save it to the datastore. It also exposes a graphQL interface to access the information saved in the datastore. 3 | 4 | ## Getting Started 5 | There are three main components in the project: 6 | 1. Crawler: crawls the network for eth2 nodes, extract additional information about the node and save it to the datastore 7 | 2. MongoDB: datastore to save eth2 nodes information 8 | 3. GraphQL Interface: provide access the stored information 9 | 10 | ### Prerequisites 11 | * docker 12 | * docker-compose 13 | 14 | ### Environment Setup 15 | Before building, please make sure environment variables `RESOLVER_API_KEY`(which is used to fetch information about node using IP) is setup properly. You can get your key from [IP data dashboard](https://dashboard.ipdata.co). To setup the variable, create an `.env` in the same folder as of `docker-compose.yaml` 16 | 17 | Example `.env` File 18 | ```shell 19 | RESOLVER_API_KEY=your_ip_data_key 20 | ``` 21 | 22 | ### Configs and Flags 23 | Eth2 crawler support config through yaml files. Default yaml config is provided at `cmd/config/config.dev.yaml`. You can use your own config file by providing it's path using the `-p` flag 24 | 25 | ### Usage 26 | We use docker-compose for testing locally. Once you have defined the environment variable in the `.env` file, you can start the server using: 27 | ```shell 28 | make run 29 | ``` 30 | 31 | ## Additional Commands 32 | * `make run` - run the crawler service 33 | * `make lint` - run linter 34 | * `make test` - runs the test cases 35 | * `make license` - add license to the missing files 36 | * `make license-check` - checks for missing license headers 37 | 38 | ## LICENSE 39 | See the [LICENSE](https://github.com/ChainSafe/eth2-crawler/blob/main/LICENSE) file for license rights and limitations (lgpl-3.0). 40 | -------------------------------------------------------------------------------- /cmd/config/config.dev.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | server: 5 | port: 8080 6 | debug: true 7 | read_timeout_seconds: 360 8 | read_header_timeout_seconds: 360 9 | write_timeout_seconds: 360 10 | cors: ["*"] 11 | 12 | database: 13 | request_timeout_sec: 5 14 | database: crawler 15 | collection: peers 16 | history_collection: history 17 | 18 | resolver: 19 | request_timeout_sec: 3 -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "flag" 9 | "fmt" 10 | "log" 11 | "net/http" 12 | "time" 13 | 14 | "eth2-crawler/crawler" 15 | "eth2-crawler/graph" 16 | "eth2-crawler/graph/generated" 17 | "eth2-crawler/resolver/ipdata" 18 | peerStore "eth2-crawler/store/peerstore/mongo" 19 | recordStore "eth2-crawler/store/record/mongo" 20 | "eth2-crawler/utils/config" 21 | "eth2-crawler/utils/server" 22 | 23 | "github.com/99designs/gqlgen/graphql/handler" 24 | "github.com/99designs/gqlgen/graphql/playground" 25 | ) 26 | 27 | func main() { 28 | cfgPath := flag.String("p", "./cmd/config/config.dev.yaml", "The configuration path") 29 | flag.Parse() 30 | 31 | cfg, err := config.Load(*cfgPath) 32 | if err != nil { 33 | log.Fatalf("error loading configuration: %s", err.Error()) 34 | } 35 | 36 | peerStore, err := peerStore.New(cfg.Database) 37 | if err != nil { 38 | log.Fatalf("error Initializing the peer store: %s", err.Error()) 39 | } 40 | 41 | historyStore, err := recordStore.New(cfg.Database) 42 | if err != nil { 43 | log.Fatalf("error Initializing the record store: %s", err.Error()) 44 | } 45 | 46 | resolverService, err := ipdata.New(cfg.Resolver.APIKey, time.Duration(cfg.Resolver.Timeout)*time.Second) 47 | if err != nil { 48 | log.Fatalf("error Initializing the ip resolver: %s", err.Error()) 49 | } 50 | 51 | // TODO collect config from a config files or from command args and pass to Start() 52 | go crawler.Start(peerStore, historyStore, resolverService) 53 | 54 | srv := handler.NewDefaultServer(generated.NewExecutableSchema(generated.Config{Resolvers: graph.NewResolver(peerStore, historyStore)})) 55 | 56 | router := http.NewServeMux() 57 | // TODO: make playground accessible only in Dev mode 58 | router.Handle("/", playground.Handler("GraphQL playground", "/query")) 59 | router.Handle("/query", srv) 60 | // TODO: setup proper status handler 61 | router.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) { 62 | fmt.Fprintf(w, "{ \"status\": \"up\" }") 63 | }) 64 | 65 | server.Start(context.TODO(), cfg.Server, router) 66 | } 67 | -------------------------------------------------------------------------------- /copyright.txt: -------------------------------------------------------------------------------- 1 | Copyright 2021 ChainSafe Systems 2 | SPDX-License-Identifier: LGPL-3.0-only 3 | -------------------------------------------------------------------------------- /crawler/crawl/crawl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package crawl 5 | 6 | import ( 7 | "context" 8 | "crypto/ecdsa" 9 | "eth2-crawler/crawler/p2p" 10 | reqresp "eth2-crawler/crawler/rpc/request" 11 | "eth2-crawler/crawler/util" 12 | "eth2-crawler/graph/model" 13 | "eth2-crawler/models" 14 | ipResolver "eth2-crawler/resolver" 15 | "eth2-crawler/store/peerstore" 16 | "eth2-crawler/store/record" 17 | "time" 18 | 19 | "github.com/protolambda/zrnt/eth2/beacon/common" 20 | 21 | "github.com/ethereum/go-ethereum/log" 22 | "github.com/ethereum/go-ethereum/p2p/enode" 23 | ) 24 | 25 | type crawler struct { 26 | disc resolver 27 | peerStore peerstore.Provider 28 | historyStore record.Provider 29 | ipResolver ipResolver.Provider 30 | iter enode.Iterator 31 | nodeCh chan *enode.Node 32 | privateKey *ecdsa.PrivateKey 33 | host p2p.Host 34 | jobs chan *models.Peer 35 | jobsConcurrency int 36 | } 37 | 38 | // resolver holds methods of discovery v5 39 | type resolver interface { 40 | Ping(n *enode.Node) error 41 | } 42 | 43 | // newCrawler inits new crawler service 44 | func newCrawler(disc resolver, peerStore peerstore.Provider, historyStore record.Provider, 45 | ipResolver ipResolver.Provider, privateKey *ecdsa.PrivateKey, iter enode.Iterator, 46 | host p2p.Host, jobConcurrency int) *crawler { 47 | c := &crawler{ 48 | disc: disc, 49 | peerStore: peerStore, 50 | historyStore: historyStore, 51 | ipResolver: ipResolver, 52 | privateKey: privateKey, 53 | iter: iter, 54 | nodeCh: make(chan *enode.Node), 55 | host: host, 56 | jobs: make(chan *models.Peer, jobConcurrency), 57 | jobsConcurrency: jobConcurrency, 58 | } 59 | return c 60 | } 61 | 62 | // start runs the crawler 63 | func (c *crawler) start(ctx context.Context) { 64 | doneCh := make(chan enode.Iterator) 65 | go c.runIterator(ctx, doneCh, c.iter) 66 | for { 67 | select { 68 | case n := <-c.nodeCh: 69 | c.storePeer(ctx, n) 70 | case <-doneCh: 71 | // crawling finished 72 | log.Info("finished iterator") 73 | return 74 | } 75 | } 76 | } 77 | 78 | // runIterator uses the node iterator and sends node data through channel 79 | func (c *crawler) runIterator(ctx context.Context, doneCh chan enode.Iterator, it enode.Iterator) { 80 | defer func() { doneCh <- it }() 81 | for it.Next() { 82 | select { 83 | case c.nodeCh <- it.Node(): 84 | case <-ctx.Done(): 85 | return 86 | } 87 | } 88 | } 89 | 90 | func (c *crawler) storePeer(ctx context.Context, node *enode.Node) { 91 | // only consider the node having tcp port exported 92 | if node.TCP() == 0 { 93 | return 94 | } 95 | // filter only eth2 nodes 96 | eth2Data, err := util.ParseEnrEth2Data(node) 97 | if err != nil { // not eth2 nodes 98 | return 99 | } 100 | log.Debug("found a eth2 node", log.Ctx{"node": node}) 101 | 102 | // get basic info 103 | peer, err := models.NewPeer(node, eth2Data) 104 | if err != nil { 105 | return 106 | } 107 | // save to db if not exists 108 | err = c.peerStore.Create(ctx, peer) 109 | if err != nil { 110 | log.Error("err inserting peer", log.Ctx{"err": err, "peer": peer.String()}) 111 | } 112 | } 113 | 114 | func (c *crawler) updatePeer(ctx context.Context) { 115 | c.runBGWorkersPool(ctx) 116 | for { 117 | select { 118 | case <-ctx.Done(): 119 | log.Error("update peer job context was canceled", log.Ctx{"err": ctx.Err()}) 120 | default: 121 | c.selectPendingAndExecute(ctx) 122 | } 123 | time.Sleep(5 * time.Second) 124 | } 125 | } 126 | 127 | func (c *crawler) selectPendingAndExecute(ctx context.Context) { 128 | // get peers that was updated 24 hours ago 129 | reqs, err := c.peerStore.ListForJob(ctx, time.Hour*24, c.jobsConcurrency) 130 | if err != nil { 131 | log.Error("error getting list from peerstore", log.Ctx{"err": err}) 132 | return 133 | } 134 | for _, req := range reqs { 135 | // update the pr, so it won't be picked again in 24 hours 136 | // We have to update the LastUpdated field here and cannot rety on the worker to update it 137 | // That is because the same request will be picked again when it is in worker. 138 | req.LastUpdated = time.Now().Unix() 139 | err = c.peerStore.Update(ctx, req) 140 | if err != nil { 141 | log.Error("error updating request", log.Ctx{"err": err}) 142 | continue 143 | } 144 | select { 145 | case <-ctx.Done(): 146 | log.Error("update selector stopped", log.Ctx{"err": ctx.Err()}) 147 | return 148 | default: 149 | c.jobs <- req 150 | } 151 | } 152 | } 153 | 154 | func (c *crawler) runBGWorkersPool(ctx context.Context) { 155 | for i := 0; i < c.jobsConcurrency; i++ { 156 | go c.bgWorker(ctx) 157 | } 158 | } 159 | 160 | func (c *crawler) bgWorker(ctx context.Context) { 161 | for { 162 | select { 163 | case <-ctx.Done(): 164 | log.Error("context canceled", log.Ctx{"err": ctx.Err()}) 165 | return 166 | case req := <-c.jobs: 167 | c.updatePeerInfo(ctx, req) 168 | } 169 | } 170 | } 171 | 172 | func (c *crawler) updatePeerInfo(ctx context.Context, peer *models.Peer) { 173 | // update connection status, agent version, sync status 174 | isConnectable := c.collectNodeInfoRetryer(ctx, peer) 175 | if isConnectable { 176 | peer.SetConnectionStatus(true) 177 | peer.Score = models.ScoreGood 178 | peer.LastConnected = time.Now().Unix() 179 | // update geolocation 180 | if peer.GeoLocation == nil { 181 | c.updateGeolocation(ctx, peer) 182 | } 183 | } else { 184 | peer.Score-- 185 | } 186 | // remove the node if it has bad score 187 | if peer.Score <= models.ScoreBad { 188 | log.Info("deleting node for bad score", log.Ctx{"peer_id": peer.ID}) 189 | err := c.peerStore.Delete(ctx, peer) 190 | if err != nil { 191 | log.Error("failed on deleting from peerstore", log.Ctx{"err": err}) 192 | } 193 | return 194 | } 195 | err := c.peerStore.Update(ctx, peer) 196 | if err != nil { 197 | log.Error("failed on updating peerstore", log.Ctx{"err": err}) 198 | } 199 | } 200 | 201 | func (c *crawler) collectNodeInfoRetryer(ctx context.Context, peer *models.Peer) bool { 202 | count := 0 203 | var err error 204 | var ag, pv string 205 | for count < 20 { 206 | time.Sleep(time.Second * 5) 207 | count++ 208 | 209 | err = c.host.Connect(ctx, *peer.GetPeerInfo()) 210 | if err != nil { 211 | continue 212 | } 213 | // get status 214 | var status *common.Status 215 | status, err = c.host.FetchStatus(c.host.NewStream, ctx, peer, new(reqresp.SnappyCompression)) 216 | if err != nil || status == nil { 217 | continue 218 | } 219 | ag, err = c.host.GetAgentVersion(peer.ID) 220 | if err != nil { 221 | continue 222 | } else { 223 | peer.SetUserAgent(ag) 224 | } 225 | 226 | pv, err = c.host.GetProtocolVersion(peer.ID) 227 | if err != nil { 228 | continue 229 | } else { 230 | peer.SetProtocolVersion(pv) 231 | } 232 | // set sync status 233 | peer.SetSyncStatus(int64(status.HeadSlot)) 234 | log.Info("successfully collected all info", peer.Log()) 235 | return true 236 | } 237 | // unsuccessful 238 | log.Error("failed on retryer", log.Ctx{ 239 | "attempt": count, 240 | "error": err, 241 | }) 242 | return false 243 | } 244 | 245 | func (c *crawler) updateGeolocation(ctx context.Context, peer *models.Peer) { 246 | geoLoc, err := c.ipResolver.GetGeoLocation(ctx, peer.IP) 247 | if err != nil { 248 | log.Error("unable to get geo information", log.Ctx{ 249 | "error": err, 250 | "ip_addr": peer.IP, 251 | }) 252 | return 253 | } 254 | peer.SetGeoLocation(geoLoc) 255 | } 256 | 257 | func (c *crawler) insertToHistory() { 258 | ctx := context.Background() 259 | // get count 260 | aggregateData, err := c.peerStore.AggregateBySyncStatus(ctx, &model.PeerFilter{}) 261 | if err != nil { 262 | log.Error("error getting sync status", log.Ctx{"err": err}) 263 | } 264 | 265 | history := models.NewHistory(aggregateData.Synced, aggregateData.Total) 266 | err = c.historyStore.Create(ctx, history) 267 | if err != nil { 268 | log.Error("error inserting sync status", log.Ctx{"err": err}) 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /crawler/crawl/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package crawl holds the eth2 node discovery utilities 5 | package crawl 6 | 7 | import ( 8 | "context" 9 | "crypto/ecdsa" 10 | "eth2-crawler/store/peerstore" 11 | "eth2-crawler/store/record" 12 | "fmt" 13 | "net" 14 | 15 | "github.com/robfig/cron/v3" 16 | 17 | "eth2-crawler/crawler/p2p" 18 | ipResolver "eth2-crawler/resolver" 19 | 20 | "github.com/ethereum/go-ethereum/crypto" 21 | "github.com/libp2p/go-libp2p" 22 | ic "github.com/libp2p/go-libp2p-core/crypto" 23 | noise "github.com/libp2p/go-libp2p-noise" 24 | "github.com/libp2p/go-tcp-transport" 25 | ma "github.com/multiformats/go-multiaddr" 26 | ) 27 | 28 | // listenConfig holds configuration for running v5discovry node 29 | type listenConfig struct { 30 | bootNodeAddrs []string 31 | listenAddress net.IP 32 | listenPORT int 33 | dbPath string 34 | privateKey *ecdsa.PrivateKey 35 | } 36 | 37 | // Initialize initializes the core crawler component 38 | func Initialize(peerStore peerstore.Provider, historyStore record.Provider, ipResolver ipResolver.Provider, bootNodeAddrs []string) error { 39 | ctx := context.Background() 40 | pkey, _ := crypto.GenerateKey() 41 | listenCfg := &listenConfig{ 42 | bootNodeAddrs: bootNodeAddrs, 43 | listenAddress: net.IPv4zero, 44 | listenPORT: 30304, 45 | dbPath: "", 46 | privateKey: pkey, 47 | } 48 | disc, err := startV5(listenCfg) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | listenAddrs, err := multiAddressBuilder(listenCfg.listenAddress, listenCfg.listenPORT) 54 | if err != nil { 55 | return err 56 | } 57 | host, err := p2p.NewHost( 58 | libp2p.Identity(convertToInterfacePrivkey(listenCfg.privateKey)), 59 | libp2p.ListenAddrs(listenAddrs), 60 | libp2p.UserAgent("Eth2-Crawler"), 61 | libp2p.Transport(tcp.NewTCPTransport), 62 | libp2p.Security(noise.ID, noise.New), 63 | libp2p.NATPortMap(), 64 | ) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | c := newCrawler(disc, peerStore, historyStore, ipResolver, listenCfg.privateKey, disc.RandomNodes(), host, 200) 70 | go c.start(ctx) 71 | // scheduler for updating peer 72 | go c.updatePeer(ctx) 73 | 74 | // add scheduler for updating history store 75 | scheduler := cron.New() 76 | _, err = scheduler.AddFunc("@daily", c.insertToHistory) 77 | if err != nil { 78 | return err 79 | } 80 | scheduler.Start() 81 | return nil 82 | } 83 | 84 | func convertToInterfacePrivkey(privkey *ecdsa.PrivateKey) ic.PrivKey { 85 | typeAssertedKey := ic.PrivKey((*ic.Secp256k1PrivateKey)(privkey)) 86 | return typeAssertedKey 87 | } 88 | 89 | func multiAddressBuilder(ipAddr net.IP, port int) (ma.Multiaddr, error) { 90 | if ipAddr.To4() == nil && ipAddr.To16() == nil { 91 | return nil, fmt.Errorf("invalid ip address provided: %s", ipAddr) 92 | } 93 | if ipAddr.To4() != nil { 94 | return ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d", ipAddr.String(), port)) 95 | } 96 | return ma.NewMultiaddr(fmt.Sprintf("/ip6/%s/tcp/%d", ipAddr.String(), port)) 97 | } 98 | -------------------------------------------------------------------------------- /crawler/crawl/udpv5.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package crawl 5 | 6 | import ( 7 | "bytes" 8 | "encoding/base64" 9 | "encoding/hex" 10 | "fmt" 11 | "net" 12 | "strings" 13 | 14 | "github.com/ethereum/go-ethereum/p2p/discover" 15 | "github.com/ethereum/go-ethereum/p2p/enode" 16 | "github.com/ethereum/go-ethereum/p2p/enr" 17 | "github.com/ethereum/go-ethereum/rlp" 18 | ) 19 | 20 | // startV5 starts an ephemeral discovery v5 node. 21 | func startV5(listenCfg *listenConfig) (*discover.UDPv5, error) { 22 | ln, config, err := getDiscoveryConfig(listenCfg) 23 | if err != nil { 24 | return nil, err 25 | } 26 | socket, err := listen(listenCfg) 27 | if err != nil { 28 | return nil, err 29 | } 30 | disc, err := discover.ListenV5(socket, ln, *config) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return disc, nil 35 | } 36 | 37 | // getDiscoveryConfig returns config for listening v5 node for peer discovery 38 | func getDiscoveryConfig(listenCfg *listenConfig) (*enode.LocalNode, *discover.Config, error) { 39 | cfg := new(discover.Config) 40 | 41 | cfg.PrivateKey = listenCfg.privateKey 42 | bootNodes, err := parseBootNodes(listenCfg.bootNodeAddrs) 43 | if err != nil { 44 | return nil, nil, fmt.Errorf("error parsing bootnodes: %w", err) 45 | } 46 | cfg.Bootnodes = bootNodes 47 | 48 | db, err := enode.OpenDB(listenCfg.dbPath) 49 | if err != nil { 50 | return nil, nil, fmt.Errorf("error opening db: %w", err) 51 | } 52 | ln := enode.NewLocalNode(db, cfg.PrivateKey) 53 | return ln, cfg, nil 54 | } 55 | 56 | // listen opens an udp connections on given address 57 | func listen(cfg *listenConfig) (*net.UDPConn, error) { 58 | udpAddr := &net.UDPAddr{ 59 | IP: cfg.listenAddress, 60 | Port: cfg.listenPORT, 61 | } 62 | conn, err := net.ListenUDP("udp", udpAddr) 63 | if err != nil { 64 | return nil, fmt.Errorf("error listening to udp: %w", err) 65 | } 66 | return conn, nil 67 | } 68 | 69 | // parseBootNodes parse bootnodes from []string 70 | func parseBootNodes(nodeStr []string) ([]*enode.Node, error) { 71 | nodes := make([]*enode.Node, len(nodeStr)) 72 | var err error 73 | for i, record := range nodeStr { 74 | nodes[i], err = parseNode(record) 75 | if err != nil { 76 | return nil, fmt.Errorf("invalid bootstrap node: %w", err) 77 | } 78 | } 79 | return nodes, nil 80 | } 81 | 82 | // parseNode parses a node record and verifies its signature. 83 | func parseNode(source string) (*enode.Node, error) { 84 | if strings.HasPrefix(source, "enode://") { 85 | return enode.ParseV4(source) 86 | } 87 | r, err := parseRecord(source) 88 | if err != nil { 89 | return nil, err 90 | } 91 | return enode.New(enode.ValidSchemes, r) 92 | } 93 | 94 | // parseRecord parses a node record from hex, base64, or raw binary input. 95 | func parseRecord(source string) (*enr.Record, error) { 96 | bin := []byte(source) 97 | if d, ok := decodeRecordHex(bytes.TrimSpace(bin)); ok { 98 | bin = d 99 | } else if d, ok := decodeRecordBase64(bytes.TrimSpace(bin)); ok { 100 | bin = d 101 | } 102 | var r enr.Record 103 | err := rlp.DecodeBytes(bin, &r) 104 | return &r, err 105 | } 106 | 107 | func decodeRecordHex(b []byte) ([]byte, bool) { 108 | if bytes.HasPrefix(b, []byte("0x")) { 109 | b = b[2:] 110 | } 111 | dec := make([]byte, hex.DecodedLen(len(b))) 112 | _, err := hex.Decode(dec, b) 113 | return dec, err == nil 114 | } 115 | 116 | func decodeRecordBase64(b []byte) ([]byte, bool) { 117 | if bytes.HasPrefix(b, []byte("enr:")) { 118 | b = b[4:] 119 | } 120 | dec := make([]byte, base64.RawURLEncoding.DecodedLen(len(b))) 121 | n, err := base64.RawURLEncoding.Decode(dec, b) 122 | return dec[:n], err == nil 123 | } 124 | -------------------------------------------------------------------------------- /crawler/p2p/host.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package p2p represent p2p host service 5 | package p2p 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "eth2-crawler/crawler/rpc/methods" 11 | reqresp "eth2-crawler/crawler/rpc/request" 12 | "eth2-crawler/models" 13 | "fmt" 14 | 15 | beacon "github.com/protolambda/zrnt/eth2/beacon/common" 16 | 17 | "github.com/libp2p/go-libp2p" 18 | "github.com/libp2p/go-libp2p-core/host" 19 | "github.com/libp2p/go-libp2p-core/network" 20 | "github.com/libp2p/go-libp2p-core/peer" 21 | "github.com/libp2p/go-libp2p/p2p/protocol/identify" 22 | ) 23 | 24 | // Client represent custom p2p client 25 | type Client struct { 26 | host.Host 27 | idSvc idService 28 | } 29 | 30 | // Host represent p2p services 31 | type Host interface { 32 | host.Host 33 | IdentifyRequest(ctx context.Context, peerInfo *peer.AddrInfo) error 34 | GetProtocolVersion(peer.ID) (string, error) 35 | GetAgentVersion(peer.ID) (string, error) 36 | FetchStatus(sFn reqresp.NewStreamFn, ctx context.Context, peer *models.Peer, comp reqresp.Compression) ( 37 | *beacon.Status, error) 38 | } 39 | 40 | type idService interface { 41 | IdentifyWait(c network.Conn) <-chan struct{} 42 | } 43 | 44 | // NewHost initializes custom host 45 | func NewHost(opt ...libp2p.Option) (Host, error) { 46 | h, err := libp2p.New(context.Background(), opt...) 47 | if err != nil { 48 | return nil, err 49 | } 50 | idService, err := identify.NewIDService(h) 51 | if err != nil { 52 | return nil, err 53 | } 54 | return &Client{Host: h, idSvc: idService}, nil 55 | } 56 | 57 | // IdentifyRequest performs libp2p identify request after connecting to peer. 58 | // It disconnects to peer after request is done 59 | func (c *Client) IdentifyRequest(ctx context.Context, peerInfo *peer.AddrInfo) error { 60 | // Connect to peer first 61 | err := c.Connect(ctx, *peerInfo) 62 | if err != nil { 63 | return fmt.Errorf("error connecting to peer: %w", err) 64 | } 65 | defer func() { 66 | _ = c.Network().ClosePeer(peerInfo.ID) 67 | }() 68 | if conns := c.Network().ConnsToPeer(peerInfo.ID); len(conns) > 0 { 69 | select { 70 | case <-c.idSvc.IdentifyWait(conns[0]): 71 | case <-ctx.Done(): 72 | } 73 | } else { 74 | return errors.New("not connected to peer, cannot await connection identify") 75 | } 76 | return nil 77 | } 78 | 79 | // GetProtocolVersion returns peer protocol version from peerstore. 80 | // Need to call IdentifyRequest first for a peer. 81 | func (c *Client) GetProtocolVersion(peerID peer.ID) (string, error) { 82 | key := "ProtocolVersion" 83 | value, err := c.Peerstore().Get(peerID, key) 84 | if err != nil { 85 | return "", fmt.Errorf("error getting protocol version:%w", err) 86 | } 87 | version, ok := value.(string) 88 | if !ok { 89 | return "", fmt.Errorf("error converting interface to string") 90 | } 91 | return version, nil 92 | } 93 | 94 | // GetAgentVersion returns peer agent version from peerstore. 95 | // Need to call IdentifyRequest first for a peer. 96 | func (c *Client) GetAgentVersion(peerID peer.ID) (string, error) { 97 | key := "AgentVersion" 98 | value, err := c.Peerstore().Get(peerID, key) 99 | if err != nil { 100 | return "", fmt.Errorf("error getting protocol version:%w", err) 101 | } 102 | version, ok := value.(string) 103 | if !ok { 104 | return "", fmt.Errorf("error converting interface to string") 105 | } 106 | return version, nil 107 | } 108 | 109 | func (c *Client) FetchStatus(sFn reqresp.NewStreamFn, ctx context.Context, peer *models.Peer, comp reqresp.Compression) ( 110 | *beacon.Status, error) { 111 | // use the fork digest same of peer to avoid stream reset 112 | status := &beacon.Status{ 113 | ForkDigest: peer.ForkDigest, 114 | FinalizedRoot: beacon.Root{}, 115 | FinalizedEpoch: 0, 116 | HeadRoot: beacon.Root{}, 117 | HeadSlot: 0, 118 | } 119 | resCode := reqresp.ServerErrCode // error by default 120 | var data *beacon.Status 121 | err := methods.StatusRPCv1.RunRequest(ctx, sFn, peer.ID, comp, 122 | reqresp.RequestSSZInput{Obj: status}, 1, 123 | func() error { 124 | return nil 125 | }, 126 | func(chunk reqresp.ChunkedResponseHandler) error { 127 | resCode = chunk.ResultCode() 128 | switch resCode { 129 | case reqresp.ServerErrCode, reqresp.InvalidReqCode: 130 | msg, err := chunk.ReadErrMsg() 131 | if err != nil { 132 | return fmt.Errorf("%s: %w", msg, err) 133 | } 134 | case reqresp.SuccessCode: 135 | var stat beacon.Status 136 | if err := chunk.ReadObj(&stat); err != nil { 137 | return err 138 | } 139 | data = &stat 140 | default: 141 | return errors.New("unexpected result code") 142 | } 143 | return nil 144 | }) 145 | return data, err 146 | } 147 | -------------------------------------------------------------------------------- /crawler/rpc/methods/status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package methods holds eth2 rpc methods 5 | package methods 6 | 7 | import ( 8 | reqresp "eth2-crawler/crawler/rpc/request" 9 | 10 | beacon "github.com/protolambda/zrnt/eth2/beacon/common" 11 | ) 12 | 13 | var StatusRPCv1 = reqresp.RPCMethod{ 14 | Protocol: "/eth2/beacon_chain/req/status/1/ssz", 15 | RequestCodec: reqresp.NewSSZCodec(func() reqresp.SerDes { return new(beacon.Status) }, beacon.StatusByteLen, beacon.StatusByteLen), 16 | ResponseChunkCodec: reqresp.NewSSZCodec(func() reqresp.SerDes { return new(beacon.Status) }, beacon.StatusByteLen, beacon.StatusByteLen), 17 | DefaultResponseChunkCount: 1, 18 | } 19 | -------------------------------------------------------------------------------- /crawler/rpc/request/buf_limit_read.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "io" 10 | ) 11 | 12 | // BufLimitReader implements buffering for an io.Reader object. 13 | type BufLimitReader struct { 14 | buf []byte 15 | rd io.Reader // reader provided by the client 16 | r, w int // buf read and write positions 17 | N int // max bytes remaining 18 | PerRead bool // Limit applies per read, i.e. it is not affected at the end of the read. 19 | } 20 | 21 | // NewBufLimitReader returns a new Reader whose buffer has the specified size. 22 | // The reader will return an error if Read crosses the limit. 23 | func NewBufLimitReader(rd io.Reader, size int, limit int) *BufLimitReader { 24 | r := &BufLimitReader{ 25 | buf: make([]byte, size), 26 | rd: rd, 27 | N: limit, 28 | } 29 | return r 30 | } 31 | 32 | var errNegativeRead = errors.New("reader returned negative count from Read") 33 | 34 | // Read reads data into p. 35 | // It returns the number of bytes read into p. 36 | // The bytes are taken from at most one Read on the underlying Reader, 37 | // hence N may be less than len(p). 38 | // At EOF, the count will be zero and err will be io.EOF. 39 | func (b *BufLimitReader) Read(p []byte) (n int, err error) { 40 | defer func() { 41 | if !b.PerRead { 42 | b.N -= n 43 | } 44 | }() 45 | if b.N <= 0 { 46 | return 0, io.EOF 47 | } 48 | if len(p) > b.N { 49 | if b.N == 0 { 50 | return 0, fmt.Errorf("reader BufLimitReader tried to read %d bytes, but limit was reached", len(p)) 51 | } 52 | p = p[:b.N] 53 | } 54 | n = len(p) 55 | if n == 0 { 56 | return 0, nil 57 | } 58 | // if all buffered bytes have been written 59 | if b.r == b.w { 60 | if len(p) >= len(b.buf) { 61 | // Large read, empty buffer. 62 | // Read directly into p to avoid copy. 63 | n, err = b.rd.Read(p) 64 | if n < 0 { 65 | panic(errNegativeRead) 66 | } 67 | return n, err 68 | } 69 | b.r = 0 70 | b.w = 0 71 | to := b.N // read no more than allowed. 72 | if to > len(b.buf) { 73 | to = len(b.buf) 74 | } 75 | n, err = b.rd.Read(b.buf[:to]) 76 | if n < 0 { 77 | panic(errNegativeRead) 78 | } 79 | b.w += n 80 | if err != nil { 81 | return n, err 82 | } 83 | } 84 | 85 | // copy as much as we can 86 | n = copy(p, b.buf[b.r:b.w]) 87 | b.r += n 88 | return n, nil 89 | } 90 | 91 | func (b *BufLimitReader) ReadByte() (byte, error) { 92 | out := [1]byte{} 93 | n, err := b.Read(out[:]) 94 | if n == 0 && err == nil { 95 | return 0, errors.New("failed to read single byte, but no error") 96 | } 97 | return out[0], err 98 | } 99 | -------------------------------------------------------------------------------- /crawler/rpc/request/compression.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "fmt" 8 | "io" 9 | 10 | "github.com/golang/snappy" 11 | ) 12 | 13 | type Compression interface { 14 | // Decompress Wraps a reader to decompress data as reads happen. 15 | Decompress(r io.Reader) io.Reader 16 | // Compress Wraps a writer to compress data as writes happen. 17 | Compress(w io.WriteCloser) io.WriteCloser 18 | // MaxEncodedLen Returns an error when the input size is too large to encode. 19 | MaxEncodedLen(msgLen uint64) (uint64, error) 20 | // Name is the name of the compression that is suffixed to the actual encoding. E.g. "snappy", w.r.t. "ssz_snappy". 21 | Name() string 22 | } 23 | 24 | type SnappyCompression struct{} 25 | 26 | func (c SnappyCompression) Decompress(reader io.Reader) io.Reader { 27 | return snappy.NewReader(reader) 28 | } 29 | 30 | func (c SnappyCompression) Compress(w io.WriteCloser) io.WriteCloser { 31 | return snappy.NewBufferedWriter(w) 32 | } 33 | 34 | func (c SnappyCompression) MaxEncodedLen(msgLen uint64) (uint64, error) { 35 | if msgLen&(1<<63) != 0 { 36 | return 0, fmt.Errorf("message length %d is too large to compress with snappy", msgLen) 37 | } 38 | m := snappy.MaxEncodedLen(int(msgLen)) 39 | if m < 0 { 40 | return 0, fmt.Errorf("message length %d is too large to compress with snappy", msgLen) 41 | } 42 | return uint64(m), nil 43 | } 44 | 45 | func (c SnappyCompression) Name() string { 46 | return "snappy" 47 | } 48 | -------------------------------------------------------------------------------- /crawler/rpc/request/encode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "bytes" 8 | "encoding/binary" 9 | "fmt" 10 | "io" 11 | ) 12 | 13 | type payloadBuffer bytes.Buffer 14 | 15 | func (p *payloadBuffer) Close() error { 16 | return nil 17 | } 18 | 19 | func (p *payloadBuffer) Write(b []byte) (n int, err error) { 20 | return (*bytes.Buffer)(p).Write(b) 21 | } 22 | 23 | func (p *payloadBuffer) OutputSizeVarint(w io.Writer) error { 24 | size := (*bytes.Buffer)(p).Len() 25 | sizeBytes := [binary.MaxVarintLen64]byte{} 26 | sizeByteLen := binary.PutUvarint(sizeBytes[:], uint64(size)) 27 | _, err := w.Write(sizeBytes[:sizeByteLen]) 28 | return err 29 | } 30 | 31 | func (p *payloadBuffer) WriteTo(w io.Writer) (n int64, err error) { 32 | return (*bytes.Buffer)(p).WriteTo(w) 33 | } 34 | 35 | type noCloseWriter struct { 36 | w io.Writer 37 | } 38 | 39 | func (nw *noCloseWriter) Write(p []byte) (n int, err error) { 40 | return nw.w.Write(p) 41 | } 42 | 43 | func (nw *noCloseWriter) Close() error { 44 | return nil 45 | } 46 | 47 | // EncodeHeaderAndPayload reads a payload, buffers (and optionally compresses) the payload, 48 | // then computes the header-data (varint of byte size). And then writes header and payload. 49 | func EncodeHeaderAndPayload(r io.Reader, w io.Writer, comp Compression) error { 50 | var buf payloadBuffer 51 | if _, err := io.Copy(&buf, r); err != nil { 52 | return err 53 | } 54 | if err := buf.OutputSizeVarint(w); err != nil { 55 | return err 56 | } 57 | if comp != nil { 58 | compressedWriter := comp.Compress(&noCloseWriter{w: w}) 59 | defer func() { 60 | _ = compressedWriter.Close() 61 | }() 62 | if _, err := buf.WriteTo(compressedWriter); err != nil { 63 | return err 64 | } 65 | } else if _, err := buf.WriteTo(w); err != nil { 66 | return err 67 | } 68 | return nil 69 | } 70 | 71 | // StreamHeaderAndPayload reads a payload and streams (and optionally compresses) it to the writer. 72 | // To do so, it requires the (uncompressed) payload length to be known in advance. 73 | func StreamHeaderAndPayload(size uint64, r io.Reader, w io.Writer, comp Compression) error { 74 | sizeBytes := [binary.MaxVarintLen64]byte{} 75 | sizeByteLen := binary.PutUvarint(sizeBytes[:], size) 76 | n, err := w.Write(sizeBytes[:sizeByteLen]) 77 | if err != nil { 78 | return fmt.Errorf("failed to write size bytes: %w", err) 79 | } 80 | if n != sizeByteLen { 81 | return fmt.Errorf("failed to write size bytes fully: %d/%d", n, sizeByteLen) 82 | } 83 | if comp != nil { 84 | compressedWriter := comp.Compress(&noCloseWriter{w: w}) 85 | defer func() { 86 | _ = compressedWriter.Close() 87 | }() 88 | if _, err := io.Copy(compressedWriter, r); err != nil { 89 | return fmt.Errorf("failed to write payload through compressed writer: %w", err) 90 | } 91 | return nil 92 | } else { 93 | if _, err := io.Copy(w, r); err != nil { 94 | return fmt.Errorf("failed to write payload: %w", err) 95 | } 96 | return nil 97 | } 98 | } 99 | 100 | // EncodeResult writes the result code to the output writer. 101 | func EncodeResult(result ResponseCode, w io.Writer) error { 102 | _, err := w.Write([]byte{uint8(result)}) 103 | return err 104 | } 105 | 106 | // EncodeChunk reads (decompressed) response message from the msg io.Reader, 107 | // and writes it as a chunk with given result code to the output writer. The compression is optional and may be nil. 108 | func EncodeChunk(result ResponseCode, r io.Reader, w io.Writer, comp Compression) error { 109 | if err := EncodeResult(result, w); err != nil { 110 | return err 111 | } 112 | return EncodeHeaderAndPayload(r, w, comp) 113 | } 114 | 115 | // StreamChunk reads (decompressed) response message from the msg io.Reader, 116 | // and writes it as a chunk with given result code to the output writer. The compression is optional and may be nil. 117 | func StreamChunk(result ResponseCode, size uint64, r io.Reader, w io.Writer, comp Compression) error { 118 | if err := EncodeResult(result, w); err != nil { 119 | return err 120 | } 121 | return StreamHeaderAndPayload(size, r, w, comp) 122 | } 123 | -------------------------------------------------------------------------------- /crawler/rpc/request/handle_request.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "context" 8 | "encoding/binary" 9 | "fmt" 10 | "io" 11 | 12 | "github.com/libp2p/go-libp2p-core/network" 13 | "github.com/libp2p/go-libp2p-core/peer" 14 | ) 15 | 16 | const requestBufferSize = 2048 17 | 18 | // RequestPayloadHandler processes a request (decompressed if previously compressed), read from r. 19 | // The handler can respond by writing to w. After returning the writer will automatically be closed. 20 | // If the input is already known to be invalid, e.g. the request size is invalid, then `invalidInputErr != nil`, and r will not read anything more. 21 | type RequestPayloadHandler func(ctx context.Context, peerId peer.ID, requestLen uint64, r io.Reader, w io.Writer, comp Compression, invalidInputErr error) 22 | 23 | type StreamCtxFn func() context.Context 24 | 25 | // MakeStreamHandler startReqRPC registers a request handler for the given protocol. Compression is optional and may be nil. 26 | func (handle RequestPayloadHandler) MakeStreamHandler(newCtx StreamCtxFn, comp Compression, minRequestContentSize, maxRequestContentSize uint64) network.StreamHandler { 27 | return func(stream network.Stream) { 28 | peerID := stream.Conn().RemotePeer() 29 | ctx, cancel := context.WithCancel(newCtx()) 30 | defer cancel() 31 | 32 | go func() { 33 | <-ctx.Done() 34 | // TODO: should this be a stream reset? 35 | _ = stream.Close() // Close stream after ctx closes. 36 | }() 37 | 38 | w := io.WriteCloser(stream) 39 | // If no request data, then do not even read a length from the stream. 40 | if maxRequestContentSize == 0 { 41 | handle(ctx, peerID, 0, nil, w, comp, nil) 42 | return 43 | } 44 | 45 | var invalidInputErr error 46 | 47 | // TODO: pool this 48 | blr := NewBufLimitReader(stream, requestBufferSize, 0) 49 | blr.N = 1 // var ints need to be read byte by byte 50 | blr.PerRead = true 51 | reqLen, err := binary.ReadUvarint(blr) 52 | blr.PerRead = false 53 | switch { 54 | case err != nil: 55 | invalidInputErr = err 56 | case reqLen < minRequestContentSize: 57 | // Check against raw content size minimum (without compression applied) 58 | invalidInputErr = fmt.Errorf("request length %d is unexpectedly small, request size minimum is %d", reqLen, minRequestContentSize) 59 | case reqLen > maxRequestContentSize: 60 | // Check against raw content size limit (without compression applied) 61 | invalidInputErr = fmt.Errorf("request length %d exceeds request size limit %d", reqLen, maxRequestContentSize) 62 | case comp != nil: 63 | // Now apply compression adjustment for size limit, and use that as the limit for the buffered-limited-reader. 64 | s, err := comp.MaxEncodedLen(maxRequestContentSize) 65 | if err != nil { 66 | invalidInputErr = err 67 | } else { 68 | maxRequestContentSize = s 69 | } 70 | } 71 | switch { 72 | case invalidInputErr != nil: // If the input is invalid, never read it. 73 | maxRequestContentSize = 0 74 | case comp == nil: 75 | blr.N = int(maxRequestContentSize) 76 | default: 77 | v, err := comp.MaxEncodedLen(maxRequestContentSize) 78 | if err != nil { 79 | blr.N = int(maxRequestContentSize) 80 | } else { 81 | blr.N = int(v) 82 | } 83 | } 84 | r := io.Reader(blr) 85 | handle(ctx, peerID, reqLen, r, w, comp, invalidInputErr) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /crawler/rpc/request/handle_response.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "context" 8 | "encoding/binary" 9 | "errors" 10 | "fmt" 11 | "io" 12 | ) 13 | 14 | // ResponseChunkHandler is a function that processes a response chunk. The index, size and result-code are already parsed. 15 | // The contents (decompressed if previously compressed) can be read from r. Optionally an answer can be written back to w. 16 | // If the response chunk could not be processed, an error may be returned. 17 | type ResponseChunkHandler func(ctx context.Context, chunkIndex uint64, chunkSize uint64, result ResponseCode, r io.Reader, w io.Writer) error 18 | 19 | // ResponseHandler processes a response by internally processing chunks, any error is propagated up. 20 | type ResponseHandler func(ctx context.Context, r io.Reader, w io.WriteCloser) error 21 | 22 | type OnRequested func() 23 | 24 | // MakeResponseHandler builds a ResponseHandler, which won't take more than maxChunkCount chunks, or chunk contents larger than maxChunkContentSize. 25 | // Compression is optional and may be nil. Chunks are processed by the given ResponseChunkHandler. 26 | func (handleChunk ResponseChunkHandler) MakeResponseHandler(maxChunkCount uint64, maxChunkContentSize uint64, comp Compression) ResponseHandler { 27 | // response ::= * 28 | // response_chunk ::= | | 29 | // result ::= “0” | “1” | “2” | [“128” ... ”255”] 30 | return func(ctx context.Context, r io.Reader, w io.WriteCloser) error { 31 | defer func() { 32 | _ = w.Close() 33 | }() 34 | if maxChunkCount == 0 { 35 | return nil 36 | } 37 | blr := NewBufLimitReader(r, 1024, 0) 38 | for chunkIndex := uint64(0); chunkIndex < maxChunkCount; chunkIndex++ { 39 | blr.N = 1 40 | resByte, err := blr.ReadByte() 41 | if errors.Is(err, io.EOF) { // no more chunks left. 42 | return nil 43 | } 44 | if err != nil { 45 | return fmt.Errorf("failed to read chunk %d result byte: %w", chunkIndex, err) 46 | } 47 | // varints need to be read byte by byte. 48 | blr.N = 1 49 | blr.PerRead = true 50 | chunkSize, err := binary.ReadUvarint(blr) 51 | blr.PerRead = false 52 | // TODO when input is incorrect, return a different type of error. 53 | if err != nil { 54 | return err 55 | } 56 | if resByte == byte(InvalidReqCode) || resByte == byte(ServerErrCode) { 57 | if chunkSize > MaxErrSize { 58 | return fmt.Errorf("chunk size %d of chunk %d exceeds error size limit %d", chunkSize, chunkIndex, MaxErrSize) 59 | } 60 | blr.N = MaxErrSize 61 | } else { 62 | if chunkSize > maxChunkContentSize { 63 | return fmt.Errorf("chunk size %d of chunk %d exceeds chunk limit %d", chunkSize, chunkIndex, maxChunkContentSize) 64 | } 65 | blr.N = int(maxChunkContentSize) 66 | } 67 | cr := io.Reader(blr) 68 | cw := w 69 | if comp != nil { 70 | cr = comp.Decompress(cr) 71 | cw = comp.Compress(cw) 72 | } 73 | if err := handleChunk(ctx, chunkIndex, chunkSize, ResponseCode(resByte), cr, cw); err != nil { 74 | _ = cw.Close() 75 | return err 76 | } 77 | if comp != nil { 78 | if err := cw.Close(); err != nil { 79 | return fmt.Errorf("failed to close response writer for chunk") 80 | } 81 | } 82 | } 83 | return nil 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /crawler/rpc/request/request.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "io" 10 | 11 | "github.com/libp2p/go-libp2p-core/network" 12 | "github.com/libp2p/go-libp2p-core/peer" 13 | "github.com/libp2p/go-libp2p-core/protocol" 14 | ) 15 | 16 | type NewStreamFn func(ctx context.Context, peerId peer.ID, protocolId ...protocol.ID) (network.Stream, error) 17 | 18 | func (newStreamFn NewStreamFn) Request(ctx context.Context, peerID peer.ID, protocolID protocol.ID, r io.Reader, comp Compression, handle ResponseHandler) error { 19 | stream, err := newStreamFn(ctx, peerID, protocolID) 20 | if err != nil { 21 | return err 22 | } 23 | defer func() { 24 | _ = stream.Close() 25 | }() 26 | 27 | var buf bytes.Buffer 28 | if err := EncodeHeaderAndPayload(r, &buf, comp); err != nil { 29 | return err 30 | } 31 | if _, err := stream.Write(buf.Bytes()); err != nil { 32 | return err 33 | } 34 | return handle(ctx, stream, stream) 35 | } 36 | -------------------------------------------------------------------------------- /crawler/rpc/request/request_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package reqresp 5 | 6 | import ( 7 | "bytes" 8 | "encoding/hex" 9 | "testing" 10 | ) 11 | 12 | func TestEncodeHeaderAndPayloadSnappy(t *testing.T) { 13 | input, _ := hex.DecodeString("aabb1234") 14 | r := bytes.NewReader(input) 15 | var buf bytes.Buffer 16 | err := EncodeHeaderAndPayload(r, &buf, SnappyCompression{}) 17 | if err != nil { 18 | t.Error(err) 19 | } 20 | expected, _ := hex.DecodeString("04ff060000734e6150705901080000e5310030aabb1234") 21 | if !bytes.Equal(expected, buf.Bytes()) { 22 | t.Error("unexpected encoding output") 23 | } 24 | } 25 | 26 | func TestEncodeHeaderAndPayload(t *testing.T) { 27 | input, _ := hex.DecodeString("aabb1234") 28 | r := bytes.NewReader(input) 29 | var buf bytes.Buffer 30 | err := EncodeHeaderAndPayload(r, &buf, nil) // no compression here 31 | if err != nil { 32 | t.Error(err) 33 | } 34 | expected, _ := hex.DecodeString("04aabb1234") 35 | if !bytes.Equal(expected, buf.Bytes()) { 36 | t.Error("unexpected encoding output") 37 | } 38 | } 39 | 40 | func TestStreamHeaderAndPayloadSnappy(t *testing.T) { 41 | input, _ := hex.DecodeString("aabb1234") 42 | r := bytes.NewReader(input) 43 | var buf bytes.Buffer 44 | err := StreamHeaderAndPayload(uint64(len(input)), r, &buf, SnappyCompression{}) 45 | if err != nil { 46 | t.Error(err) 47 | } 48 | expected, _ := hex.DecodeString("04ff060000734e6150705901080000e5310030aabb1234") 49 | if !bytes.Equal(expected, buf.Bytes()) { 50 | t.Error("unexpected encoding output") 51 | } 52 | } 53 | 54 | func TestStreamHeaderAndPayload(t *testing.T) { 55 | input, _ := hex.DecodeString("aabb1234") 56 | r := bytes.NewReader(input) 57 | var buf bytes.Buffer 58 | err := StreamHeaderAndPayload(uint64(len(input)), r, &buf, nil) // no compression here 59 | if err != nil { 60 | t.Error(err) 61 | } 62 | expected, _ := hex.DecodeString("04aabb1234") 63 | if !bytes.Equal(expected, buf.Bytes()) { 64 | t.Error("unexpected encoding output") 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /crawler/rpc/request/rpc_method.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package reqresp holds stream request service 5 | package reqresp 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "errors" 11 | "fmt" 12 | "io" 13 | 14 | "github.com/libp2p/go-libp2p-core/network" 15 | "github.com/libp2p/go-libp2p-core/peer" 16 | "github.com/libp2p/go-libp2p-core/protocol" 17 | "github.com/protolambda/ztyp/codec" 18 | ) 19 | 20 | type Request interface { 21 | fmt.Stringer 22 | } 23 | 24 | type Codec interface { 25 | MinByteLen() uint64 26 | MaxByteLen() uint64 27 | Encode(w io.Writer, input codec.Serializable) error 28 | Decode(r io.Reader, bytesLen uint64, dest codec.Deserializable) error 29 | Alloc() SerDes 30 | } 31 | 32 | type SerDes interface { 33 | codec.Serializable 34 | codec.Deserializable 35 | } 36 | 37 | type SSZCodec struct { 38 | alloc func() SerDes 39 | minByteLen uint64 40 | maxByteLen uint64 41 | } 42 | 43 | func NewSSZCodec(alloc func() SerDes, minByteLen uint64, maxByteLen uint64) *SSZCodec { 44 | return &SSZCodec{ 45 | alloc: alloc, 46 | minByteLen: minByteLen, 47 | maxByteLen: maxByteLen, 48 | } 49 | } 50 | 51 | func (c *SSZCodec) MinByteLen() uint64 { 52 | if c == nil { 53 | return 0 54 | } 55 | return c.minByteLen 56 | } 57 | 58 | func (c *SSZCodec) MaxByteLen() uint64 { 59 | if c == nil { 60 | return 0 61 | } 62 | return c.maxByteLen 63 | } 64 | 65 | func (c *SSZCodec) Encode(w io.Writer, input codec.Serializable) error { 66 | if c == nil { 67 | if input != nil { 68 | return errors.New("expected empty data, nil input. This codec is no-op") 69 | } 70 | return nil 71 | } 72 | return input.Serialize(codec.NewEncodingWriter(w)) 73 | } 74 | 75 | func (c *SSZCodec) Decode(r io.Reader, bytesLen uint64, dest codec.Deserializable) error { 76 | if c == nil { 77 | if bytesLen != 0 { 78 | return errors.New("expected 0 bytes, no definition") 79 | } 80 | return nil 81 | } 82 | return dest.Deserialize(codec.NewDecodingReader(r, bytesLen)) 83 | } 84 | 85 | func (c *SSZCodec) Alloc() SerDes { 86 | if c == nil { 87 | return nil 88 | } 89 | return c.alloc() 90 | } 91 | 92 | type RPCMethod struct { 93 | Protocol protocol.ID 94 | RequestCodec Codec 95 | ResponseChunkCodec Codec 96 | DefaultResponseChunkCount uint64 97 | } 98 | 99 | type ResponseCode uint8 100 | 101 | const ( 102 | SuccessCode ResponseCode = iota 103 | InvalidReqCode 104 | ServerErrCode 105 | ) 106 | 107 | // MaxErrSize holds maximum err size 108 | const MaxErrSize = 256 109 | 110 | type OnResponseListener func(chunk ChunkedResponseHandler) error 111 | 112 | type RequestInput interface { 113 | Reader(c Codec) (io.Reader, error) 114 | } 115 | 116 | type RequestSSZInput struct { 117 | Obj codec.Serializable 118 | } 119 | 120 | func (v RequestSSZInput) Reader(c Codec) (io.Reader, error) { 121 | var buf bytes.Buffer 122 | if err := c.Encode(&buf, v.Obj); err != nil { 123 | return nil, err 124 | } 125 | return &buf, nil 126 | } 127 | 128 | type RequestBytesInput []byte 129 | 130 | func (v RequestBytesInput) Reader(_ Codec) (io.Reader, error) { 131 | return bytes.NewReader(v), nil 132 | } 133 | 134 | type ChunkedResponseHandler interface { 135 | ChunkSize() uint64 136 | ChunkIndex() uint64 137 | ResultCode() ResponseCode 138 | ReadRaw() ([]byte, error) 139 | ReadErrMsg() (string, error) 140 | ReadObj(dest codec.Deserializable) error 141 | } 142 | 143 | type chRespHandler struct { 144 | m *RPCMethod 145 | r io.Reader 146 | result ResponseCode 147 | chunkSize uint64 148 | chunkIndex uint64 149 | } 150 | 151 | func (c *chRespHandler) ChunkSize() uint64 { 152 | return c.chunkSize 153 | } 154 | 155 | func (c *chRespHandler) ChunkIndex() uint64 { 156 | return c.chunkIndex 157 | } 158 | 159 | func (c *chRespHandler) ResultCode() ResponseCode { 160 | return c.result 161 | } 162 | 163 | func (c *chRespHandler) ReadRaw() ([]byte, error) { 164 | var buf bytes.Buffer 165 | _, err := buf.ReadFrom(io.LimitReader(c.r, int64(c.chunkSize))) 166 | return buf.Bytes(), err 167 | } 168 | 169 | func (c *chRespHandler) ReadErrMsg() (string, error) { 170 | var buf bytes.Buffer 171 | _, err := buf.ReadFrom(io.LimitReader(c.r, int64(c.chunkSize))) 172 | return buf.String(), err 173 | } 174 | 175 | func (c *chRespHandler) ReadObj(dest codec.Deserializable) error { 176 | return c.m.ResponseChunkCodec.Decode(c.r, c.chunkSize, dest) 177 | } 178 | 179 | func (m *RPCMethod) RunRequest(ctx context.Context, newStreamFn NewStreamFn, 180 | peerID peer.ID, comp Compression, req RequestInput, maxRespChunks uint64, madeRequest func() error, 181 | onResponse OnResponseListener) error { 182 | handleChunks := ResponseChunkHandler(func(ctx context.Context, chunkIndex uint64, chunkSize uint64, result ResponseCode, r io.Reader, w io.Writer) error { 183 | return onResponse(&chRespHandler{ 184 | m: m, 185 | r: r, 186 | result: result, 187 | chunkSize: chunkSize, 188 | chunkIndex: chunkIndex, 189 | }) 190 | }) 191 | 192 | reqR, err := req.Reader(m.RequestCodec) 193 | if err != nil { 194 | return err 195 | } 196 | 197 | protocolID := m.Protocol 198 | maxChunkContentSize := m.ResponseChunkCodec.MaxByteLen() 199 | if comp != nil { 200 | protocolID += protocol.ID("_" + comp.Name()) 201 | if s, err := comp.MaxEncodedLen(maxChunkContentSize); err != nil { 202 | return err 203 | } else { 204 | maxChunkContentSize = s 205 | } 206 | } 207 | 208 | respHandler := handleChunks.MakeResponseHandler(maxRespChunks, maxChunkContentSize, comp) 209 | 210 | handler := ResponseHandler(func(ctx context.Context, r io.Reader, w io.WriteCloser) error { 211 | if err := madeRequest(); err != nil { 212 | return fmt.Errorf("made request, but could not continue: %w", err) 213 | } 214 | return respHandler(ctx, r, w) 215 | }) 216 | 217 | // Runs the request in sync, which processes responses, 218 | // and then finally closes the channel through the earlier deferred close. 219 | return newStreamFn.Request(ctx, peerID, protocolID, reqR, comp, handler) 220 | } 221 | 222 | type ReadRequestFn func(dest interface{}) error 223 | type WriteSuccessChunkFn func(data interface{}) error 224 | type WriteMsgFn func(msg string) error 225 | 226 | type RequestReader interface { 227 | InvalidInput() error 228 | ReadRequest(dest codec.Deserializable) error 229 | RawRequest() ([]byte, error) 230 | } 231 | 232 | type RequestResponder interface { 233 | WriteResponseChunk(code ResponseCode, data codec.Serializable) error 234 | WriteRawResponseChunk(code ResponseCode, chunk []byte) error 235 | StreamResponseChunk(code ResponseCode, size uint64, r io.Reader) error 236 | WriteErrorChunk(code ResponseCode, msg string) error 237 | } 238 | 239 | type ChunkedRequestHandler interface { 240 | RequestReader 241 | RequestResponder 242 | } 243 | 244 | type chReqHandler struct { 245 | m *RPCMethod 246 | comp Compression 247 | respBuf bytes.Buffer 248 | reqLen uint64 249 | r io.Reader 250 | w io.Writer 251 | invalidInputErr error 252 | } 253 | 254 | func (h *chReqHandler) InvalidInput() error { 255 | return h.invalidInputErr 256 | } 257 | 258 | func (h *chReqHandler) ReadRequest(dest codec.Deserializable) error { 259 | if h.invalidInputErr != nil { 260 | return h.invalidInputErr 261 | } 262 | r := h.r 263 | if h.comp != nil { 264 | r = h.comp.Decompress(r) 265 | } 266 | return h.m.RequestCodec.Decode(r, h.reqLen, dest) 267 | } 268 | 269 | func (h *chReqHandler) RawRequest() ([]byte, error) { 270 | if h.invalidInputErr != nil { 271 | return nil, h.invalidInputErr 272 | } 273 | var buf bytes.Buffer 274 | r := h.r 275 | if h.comp != nil { 276 | r = h.comp.Decompress(r) 277 | } 278 | if _, err := buf.ReadFrom(io.LimitReader(r, int64(h.reqLen))); err != nil { 279 | return nil, err 280 | } 281 | return buf.Bytes(), nil 282 | } 283 | 284 | func (h *chReqHandler) WriteResponseChunk(code ResponseCode, data codec.Serializable) error { 285 | h.respBuf.Reset() // re-use buffer for each response chunk 286 | if err := h.m.ResponseChunkCodec.Encode(&h.respBuf, data); err != nil { 287 | return err 288 | } 289 | b := h.respBuf.Bytes() 290 | return StreamChunk(code, uint64(len(b)), bytes.NewReader(b), h.w, h.comp) 291 | } 292 | 293 | func (h *chReqHandler) WriteRawResponseChunk(code ResponseCode, chunk []byte) error { 294 | return StreamChunk(code, uint64(len(chunk)), bytes.NewReader(chunk), h.w, h.comp) 295 | } 296 | 297 | func (h *chReqHandler) StreamResponseChunk(code ResponseCode, size uint64, r io.Reader) error { 298 | return StreamChunk(code, size, r, h.w, h.comp) 299 | } 300 | 301 | func (h *chReqHandler) WriteErrorChunk(code ResponseCode, msg string) error { 302 | if len(msg) > MaxErrSize { 303 | msg = msg[:MaxErrSize-3] 304 | msg += "..." 305 | } 306 | b := []byte(msg) 307 | return StreamChunk(code, uint64(len(b)), bytes.NewReader(b), h.w, h.comp) 308 | } 309 | 310 | type OnRequestListener func(ctx context.Context, peerId peer.ID, handler ChunkedRequestHandler) 311 | 312 | func (m *RPCMethod) MakeStreamHandler(newCtx StreamCtxFn, comp Compression, listener OnRequestListener) network.StreamHandler { 313 | return RequestPayloadHandler(func(ctx context.Context, peerId peer.ID, requestLen uint64, r io.Reader, w io.Writer, comp Compression, invalidInputErr error) { 314 | listener(ctx, peerId, &chReqHandler{ 315 | m: m, comp: comp, reqLen: requestLen, r: r, w: w, invalidInputErr: invalidInputErr, 316 | }) 317 | }).MakeStreamHandler(newCtx, comp, m.RequestCodec.MinByteLen(), m.RequestCodec.MaxByteLen()) 318 | } 319 | -------------------------------------------------------------------------------- /crawler/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package crawler holds the whole crawler service. It includes crawler, db component and GraphQL 5 | package crawler 6 | 7 | import ( 8 | "eth2-crawler/crawler/crawl" 9 | ipResolver "eth2-crawler/resolver" 10 | "eth2-crawler/store/peerstore" 11 | "eth2-crawler/store/record" 12 | 13 | "github.com/ethereum/go-ethereum/log" 14 | 15 | "github.com/ethereum/go-ethereum/params" 16 | ) 17 | 18 | // Start starts the crawler service 19 | func Start(peerStore peerstore.Provider, historyStore record.Provider, ipResolver ipResolver.Provider) { 20 | h := log.CallerFileHandler(log.StdoutHandler) 21 | log.Root().SetHandler(h) 22 | 23 | handler := log.MultiHandler( 24 | log.LvlFilterHandler(log.LvlInfo, h), 25 | ) 26 | log.Root().SetHandler(handler) 27 | 28 | err := crawl.Initialize(peerStore, historyStore, ipResolver, params.V5Bootnodes) 29 | if err != nil { 30 | panic(err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /crawler/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package util holds utility functions for different conversion 5 | package util 6 | 7 | import ( 8 | "bytes" 9 | "encoding/hex" 10 | "fmt" 11 | "net" 12 | "time" 13 | 14 | "github.com/ethereum/go-ethereum/p2p/enode" 15 | "github.com/libp2p/go-libp2p-core/crypto" 16 | "github.com/libp2p/go-libp2p-core/peer" 17 | "github.com/multiformats/go-multiaddr" 18 | beacon "github.com/protolambda/zrnt/eth2/beacon/common" 19 | "github.com/protolambda/ztyp/codec" 20 | ) 21 | 22 | func AddrsFromEnode(node *enode.Node) (*peer.AddrInfo, error) { 23 | madds, err := EnodeToMultiAddr(node) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | if len(madds) == 0 { 29 | return nil, nil 30 | } 31 | 32 | peerInfo, err := peer.AddrInfoFromP2pAddr(madds[0]) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | for i := 1; i < len(madds); i++ { 38 | transport, _ := peer.SplitAddr(madds[i]) 39 | peerInfo.Addrs = append(peerInfo.Addrs, transport) 40 | } 41 | 42 | return peerInfo, nil 43 | } 44 | 45 | func EnodeToMultiAddr(node *enode.Node) ([]multiaddr.Multiaddr, error) { 46 | multiAddrs := []multiaddr.Multiaddr{} 47 | 48 | ipScheme := "ip4" 49 | if len(node.IP()) == net.IPv6len { 50 | ipScheme = "ip6" 51 | } 52 | pubkey := node.Pubkey() 53 | peerID, err := peer.IDFromPublicKey(crypto.PubKey((*crypto.Secp256k1PublicKey)(pubkey))) 54 | if err != nil { 55 | return nil, err 56 | } 57 | tcpMultiAddrStr := fmt.Sprintf("/%s/%s/tcp/%d/p2p/%s", ipScheme, node.IP().String(), node.TCP(), peerID) 58 | tcpMultiAddr, err := multiaddr.NewMultiaddr(tcpMultiAddrStr) 59 | if err != nil { 60 | return nil, err 61 | } 62 | multiAddrs = append(multiAddrs, tcpMultiAddr) 63 | 64 | udpMultiAddrStr := fmt.Sprintf("/%s/%s/udp/%d/p2p/%s", ipScheme, node.IP().String(), node.UDP(), peerID) 65 | udpMultiAddr, err := multiaddr.NewMultiaddr(udpMultiAddrStr) 66 | if err != nil { 67 | return nil, err 68 | } 69 | multiAddrs = append(multiAddrs, udpMultiAddr) 70 | 71 | return multiAddrs, nil 72 | } 73 | 74 | type Eth2ENREntry []byte 75 | 76 | func (eee Eth2ENREntry) ENRKey() string { 77 | return "eth2" 78 | } 79 | 80 | func (eee Eth2ENREntry) Eth2Data() (*beacon.Eth2Data, error) { 81 | var dat beacon.Eth2Data 82 | if err := dat.Deserialize(codec.NewDecodingReader(bytes.NewReader(eee), uint64(len(eee)))); err != nil { 83 | return nil, err 84 | } 85 | return &dat, nil 86 | } 87 | 88 | func (eee Eth2ENREntry) String() string { 89 | dat, err := eee.Eth2Data() 90 | if err != nil { 91 | return fmt.Sprintf("invalid eth2 data! Raw: %x", eee[:]) 92 | } 93 | return fmt.Sprintf("digest: %s, next fork version: %s, next fork epoch: %d", 94 | dat.ForkDigest, dat.NextForkVersion, dat.NextForkEpoch) 95 | } 96 | 97 | func ParseEnrEth2Data(n *enode.Node) (*beacon.Eth2Data, error) { 98 | var eth2 Eth2ENREntry 99 | if err := n.Load(ð2); err != nil { 100 | return nil, err 101 | } 102 | dat, err := eth2.Eth2Data() 103 | if err != nil { 104 | return nil, fmt.Errorf("failed parsing eth2 bytes: %w", err) 105 | } 106 | return dat, nil 107 | } 108 | 109 | func ParseEnrAttnets(n *enode.Node) (*beacon.AttnetBits, error) { 110 | var attnets AttnetsENREntry 111 | if err := n.Load(&attnets); err != nil { 112 | return nil, err 113 | } 114 | dat, err := attnets.AttnetBits() 115 | if err != nil { 116 | return nil, fmt.Errorf("failed parsing attnets bytes: %w", err) 117 | } 118 | return &dat, nil 119 | } 120 | 121 | type AttnetsENREntry []byte 122 | 123 | func (aee AttnetsENREntry) ENRKey() string { 124 | return "attnets" 125 | } 126 | 127 | func (aee AttnetsENREntry) AttnetBits() (beacon.AttnetBits, error) { 128 | var dat beacon.AttnetBits 129 | if err := dat.Deserialize(codec.NewDecodingReader(bytes.NewReader(aee), uint64(len(aee)))); err != nil { 130 | return beacon.AttnetBits{}, err 131 | } 132 | return dat, nil 133 | } 134 | 135 | func (aee AttnetsENREntry) String() string { 136 | return hex.EncodeToString(aee) 137 | } 138 | 139 | func getGenesisTime() time.Time { 140 | t, _ := time.Parse(time.RFC822, "01 Dec 20 12:00 GMT") 141 | return t 142 | } 143 | 144 | func CurrentBlock() int64 { 145 | duration := time.Since(getGenesisTime()) 146 | return int64((duration / time.Second) / 12) 147 | } 148 | -------------------------------------------------------------------------------- /crawler/util/util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package util 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestCurrentBlock(t *testing.T) { 14 | block1 := CurrentBlock() 15 | assert.Greater(t, block1, int64(0)) 16 | time.Sleep(12 * time.Second) 17 | block2 := CurrentBlock() 18 | assert.Equal(t, block1, block2-1) 19 | } 20 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | version: '3.7' 5 | 6 | volumes: 7 | mongo_data: {} 8 | 9 | networks: 10 | crawler_net: 11 | driver: bridge 12 | 13 | services: 14 | mongo-db: 15 | image: library/mongo:bionic 16 | ports: 17 | - "27017:27017/tcp" 18 | volumes: 19 | - mongo_data:/data/db 20 | environment: 21 | MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USR:-mongoUsr} 22 | MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PWD:-mongoPwd} 23 | networks: 24 | - crawler_net 25 | healthcheck: 26 | test: echo 'db.runCommand("ping").ok' | mongo mongo-db:27017/test --quiet 27 | interval: 10s 28 | timeout: 10s 29 | retries: 5 30 | start_period: 40s 31 | restart: on-failure 32 | 33 | crawler: 34 | image: "crawler-dev:latest" 35 | build: 36 | context: . 37 | dockerfile: Dockerfile 38 | env_file: 39 | - .env 40 | environment: 41 | MONGODB_URI: mongodb://${MONGODB_USR:-mongoUsr}:${MONGODB_PWD:-mongoPwd}@mongo-db:27017 42 | ports: 43 | - "8080:8080/tcp" 44 | depends_on: 45 | - mongo-db 46 | networks: 47 | - crawler_net 48 | restart: on-failure -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module eth2-crawler 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/99designs/gqlgen v0.17.13 7 | github.com/ethereum/go-ethereum v1.10.12 8 | github.com/golang/snappy v0.0.4 9 | github.com/google/uuid v1.3.0 10 | github.com/hashicorp/go-version v1.4.0 11 | github.com/ipdata/go v0.7.2 12 | github.com/libp2p/go-libp2p v0.15.1 13 | github.com/libp2p/go-libp2p-core v0.9.0 14 | github.com/libp2p/go-libp2p-noise v0.2.2 15 | github.com/libp2p/go-tcp-transport v0.2.8 16 | github.com/multiformats/go-multiaddr v0.12.1 17 | github.com/protolambda/zrnt v0.25.0 18 | github.com/protolambda/ztyp v0.2.1 19 | github.com/robfig/cron/v3 v3.0.1 20 | github.com/rs/cors v1.8.2 21 | github.com/stretchr/testify v1.7.1 22 | github.com/vektah/gqlparser/v2 v2.4.6 23 | go.mongodb.org/mongo-driver v1.8.3 24 | gopkg.in/yaml.v2 v2.4.0 25 | ) 26 | 27 | require ( 28 | github.com/agnivade/levenshtein v1.1.1 // indirect 29 | github.com/beorn7/perks v1.0.1 // indirect 30 | github.com/btcsuite/btcd v0.22.0-beta // indirect 31 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect 34 | github.com/flynn/noise v1.0.0 // indirect 35 | github.com/go-stack/stack v1.8.0 // indirect 36 | github.com/gogo/protobuf v1.3.2 // indirect 37 | github.com/golang/protobuf v1.5.2 // indirect 38 | github.com/google/gopacket v1.1.19 // indirect 39 | github.com/gorilla/websocket v1.5.0 // indirect 40 | github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect 41 | github.com/holiman/uint256 v1.2.0 // indirect 42 | github.com/huin/goupnp v1.0.2 // indirect 43 | github.com/ipfs/go-cid v0.0.7 // indirect 44 | github.com/ipfs/go-ipfs-util v0.0.2 // indirect 45 | github.com/ipfs/go-log v1.0.5 // indirect 46 | github.com/ipfs/go-log/v2 v2.3.0 // indirect 47 | github.com/jackpal/go-nat-pmp v1.0.2 // indirect 48 | github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect 49 | github.com/jbenet/goprocess v0.1.4 // indirect 50 | github.com/kilic/bls12-381 v0.1.0 // indirect 51 | github.com/klauspost/compress v1.13.6 // indirect 52 | github.com/klauspost/cpuid/v2 v2.2.3 // indirect 53 | github.com/koron/go-ssdp v0.0.2 // indirect 54 | github.com/libp2p/go-addr-util v0.1.0 // indirect 55 | github.com/libp2p/go-buffer-pool v0.0.2 // indirect 56 | github.com/libp2p/go-conn-security-multistream v0.2.1 // indirect 57 | github.com/libp2p/go-eventbus v0.2.1 // indirect 58 | github.com/libp2p/go-flow-metrics v0.0.3 // indirect 59 | github.com/libp2p/go-libp2p-autonat v0.4.2 // indirect 60 | github.com/libp2p/go-libp2p-blankhost v0.2.0 // indirect 61 | github.com/libp2p/go-libp2p-circuit v0.4.0 // indirect 62 | github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect 63 | github.com/libp2p/go-libp2p-mplex v0.4.1 // indirect 64 | github.com/libp2p/go-libp2p-nat v0.0.6 // indirect 65 | github.com/libp2p/go-libp2p-peerstore v0.2.8 // indirect 66 | github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect 67 | github.com/libp2p/go-libp2p-swarm v0.5.3 // indirect 68 | github.com/libp2p/go-libp2p-tls v0.2.0 // indirect 69 | github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect 70 | github.com/libp2p/go-libp2p-yamux v0.5.4 // indirect 71 | github.com/libp2p/go-maddr-filter v0.1.0 // indirect 72 | github.com/libp2p/go-mplex v0.3.0 // indirect 73 | github.com/libp2p/go-msgio v0.0.6 // indirect 74 | github.com/libp2p/go-nat v0.0.5 // indirect 75 | github.com/libp2p/go-netroute v0.1.6 // indirect 76 | github.com/libp2p/go-openssl v0.0.7 // indirect 77 | github.com/libp2p/go-reuseport v0.0.2 // indirect 78 | github.com/libp2p/go-reuseport-transport v0.0.5 // indirect 79 | github.com/libp2p/go-sockaddr v0.1.1 // indirect 80 | github.com/libp2p/go-stream-muxer-multistream v0.3.0 // indirect 81 | github.com/libp2p/go-ws-transport v0.5.0 // indirect 82 | github.com/libp2p/go-yamux/v2 v2.2.0 // indirect 83 | github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect 84 | github.com/mattn/go-isatty v0.0.14 // indirect 85 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 86 | github.com/miekg/dns v1.1.43 // indirect 87 | github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect 88 | github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect 89 | github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect 90 | github.com/minio/sha256-simd v1.0.1 // indirect 91 | github.com/mitchellh/mapstructure v1.4.1 // indirect 92 | github.com/mr-tron/base58 v1.2.0 // indirect 93 | github.com/multiformats/go-base32 v0.0.3 // indirect 94 | github.com/multiformats/go-base36 v0.1.0 // indirect 95 | github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect 96 | github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect 97 | github.com/multiformats/go-multibase v0.0.3 // indirect 98 | github.com/multiformats/go-multihash v0.0.15 // indirect 99 | github.com/multiformats/go-multistream v0.2.2 // indirect 100 | github.com/multiformats/go-varint v0.0.6 // indirect 101 | github.com/opentracing/opentracing-go v1.2.0 // indirect 102 | github.com/pkg/errors v0.9.1 // indirect 103 | github.com/pmezard/go-difflib v1.0.0 // indirect 104 | github.com/prometheus/client_golang v1.11.0 // indirect 105 | github.com/prometheus/client_model v0.2.0 // indirect 106 | github.com/prometheus/common v0.30.0 // indirect 107 | github.com/prometheus/procfs v0.7.3 // indirect 108 | github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e // indirect 109 | github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect 110 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect 111 | github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect 112 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 113 | github.com/xdg-go/scram v1.0.2 // indirect 114 | github.com/xdg-go/stringprep v1.0.2 // indirect 115 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 116 | go.uber.org/atomic v1.9.0 // indirect 117 | go.uber.org/multierr v1.7.0 // indirect 118 | go.uber.org/zap v1.19.0 // indirect 119 | golang.org/x/crypto v0.17.0 // indirect 120 | golang.org/x/exp v0.0.0-20230725012225-302865e7556b // indirect 121 | golang.org/x/net v0.10.0 // indirect 122 | golang.org/x/sync v0.1.0 // indirect 123 | golang.org/x/sys v0.15.0 // indirect 124 | golang.org/x/text v0.14.0 // indirect 125 | google.golang.org/protobuf v1.28.0 // indirect 126 | gopkg.in/yaml.v3 v3.0.0 // indirect 127 | ) 128 | -------------------------------------------------------------------------------- /gqlgen.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 ChainSafe Systems 2 | # SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | # Where are all the schema files located? globs are supported eg src/**/*.graphqls 5 | schema: 6 | - graph/*.graphqls 7 | 8 | # Where should the generated server code go? 9 | exec: 10 | filename: graph/generated/generated.go 11 | package: generated 12 | 13 | # Uncomment to enable federation 14 | # federation: 15 | # filename: graph/generated/federation.go 16 | # package: generated 17 | 18 | # Where should any generated models go? 19 | model: 20 | filename: graph/model/models_gen.go 21 | package: model 22 | 23 | # Where should the resolver implementations go? 24 | resolver: 25 | layout: follow-schema 26 | dir: graph 27 | package: graph 28 | 29 | # Optional: turn on use `gqlgen:"fieldName"` tags in your models 30 | # struct_tag: json 31 | 32 | # Optional: turn on to use []Thing instead of []*Thing 33 | # omit_slice_element_pointers: false 34 | 35 | # Optional: set to speed up generation time by not performing a final validation pass. 36 | # skip_validation: true 37 | 38 | # gqlgen will search for any type names in the schema in these go packages 39 | # if they match it will use them, otherwise it will generate them. 40 | autobind: 41 | - "eth2-crawler/graph/model" 42 | 43 | # This section declares type mapping between the GraphQL and go type systems 44 | # 45 | # The first line in each type will be used as defaults for resolver arguments and 46 | # modelgen, the others will be allowed when binding to fields. Configure them to 47 | # your liking 48 | models: 49 | ID: 50 | model: 51 | - github.com/99designs/gqlgen/graphql.ID 52 | - github.com/99designs/gqlgen/graphql.Int 53 | - github.com/99designs/gqlgen/graphql.Int64 54 | - github.com/99designs/gqlgen/graphql.Int32 55 | Int: 56 | model: 57 | - github.com/99designs/gqlgen/graphql.Int 58 | - github.com/99designs/gqlgen/graphql.Int64 59 | - github.com/99designs/gqlgen/graphql.Int32 60 | -------------------------------------------------------------------------------- /graph/model/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package model contains auto-generated files as well as helper funtions 5 | package model 6 | 7 | import ( 8 | svcModels "eth2-crawler/models" 9 | "fmt" 10 | 11 | "github.com/hashicorp/go-version" 12 | ) 13 | 14 | func SortByCount(data []*NextHardforkAggregation) []*NextHardforkAggregation { 15 | for i := 0; i < len(data); i++ { 16 | for j := i + 1; j < len(data); j++ { 17 | if data[i].Count < data[j].Count { 18 | data[i], data[j] = data[j], data[i] 19 | } 20 | } 21 | } 22 | return data 23 | } 24 | 25 | func GroupByHardforkSchedule(allPeers []*svcModels.Peer) map[string]*NextHardforkAggregation { 26 | result := map[string]*NextHardforkAggregation{} 27 | for _, peer := range allPeers { 28 | key := fmt.Sprintf("%s-%s", peer.NextForkVersion.String(), peer.NextForkEpoch.String()) 29 | if _, ok := result[key]; !ok { 30 | result[key] = &NextHardforkAggregation{ 31 | Epoch: peer.NextForkEpoch.String(), 32 | Version: peer.NextForkVersion.String(), 33 | Count: 1, 34 | } 35 | } else { 36 | result[key].Count++ 37 | } 38 | } 39 | return result 40 | } 41 | 42 | func SupportAltairUpgrade(clientName, ver string) bool { 43 | if len(ver) != 0 && ver[0:1] != "v" { 44 | ver = "v" + ver 45 | } 46 | clientVersion, err := version.NewVersion(ver) 47 | if err != nil { 48 | return false 49 | } 50 | 51 | switch svcModels.ClientName(clientName) { 52 | case svcModels.PrysmClient: 53 | v, _ := version.NewVersion("v2.0.0") 54 | if clientVersion.GreaterThanOrEqual(v) { 55 | return true 56 | } 57 | case svcModels.TekuClient: 58 | v, _ := version.NewVersion("v21.9.2") 59 | if clientVersion.GreaterThanOrEqual(v) { 60 | return true 61 | } 62 | case svcModels.LighthouseClient: 63 | v, _ := version.NewVersion("v2.0.0") 64 | if clientVersion.GreaterThanOrEqual(v) { 65 | return true 66 | } 67 | case svcModels.NimbusClient: 68 | v, _ := version.NewVersion("v1.5.0") 69 | if clientVersion.GreaterThanOrEqual(v) { 70 | return true 71 | } 72 | case svcModels.LodestarClient: 73 | v, _ := version.NewVersion("v0.31.0") 74 | if clientVersion.GreaterThanOrEqual(v) { 75 | return true 76 | } 77 | default: 78 | return false 79 | } 80 | return false 81 | } 82 | -------------------------------------------------------------------------------- /graph/model/models_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by github.com/99designs/gqlgen, DO NOT EDIT. 2 | 3 | package model 4 | 5 | type AggregateData struct { 6 | Name string `json:"name"` 7 | Count int `json:"count"` 8 | } 9 | 10 | type ClientVersionAggregation struct { 11 | Client string `json:"client"` 12 | Count int `json:"count"` 13 | Versions []*AggregateData `json:"versions"` 14 | } 15 | 16 | type HeatmapData struct { 17 | NetworkType string `json:"networkType"` 18 | ClientType string `json:"clientType"` 19 | SyncStatus string `json:"syncStatus"` 20 | Latitude float64 `json:"latitude"` 21 | Longitude float64 `json:"longitude"` 22 | City string `json:"city"` 23 | Country string `json:"country"` 24 | } 25 | 26 | type NextHardforkAggregation struct { 27 | Version string `json:"version"` 28 | Epoch string `json:"epoch"` 29 | Count int `json:"count"` 30 | } 31 | 32 | type NodeStats struct { 33 | TotalNodes int `json:"totalNodes"` 34 | NodeSyncedPercentage float64 `json:"nodeSyncedPercentage"` 35 | NodeUnsyncedPercentage float64 `json:"nodeUnsyncedPercentage"` 36 | } 37 | 38 | type NodeStatsOverTime struct { 39 | Time float64 `json:"time"` 40 | TotalNodes int `json:"totalNodes"` 41 | SyncedNodes int `json:"syncedNodes"` 42 | UnsyncedNodes int `json:"unsyncedNodes"` 43 | } 44 | 45 | type PeerFilter struct { 46 | ForkDigest *string `json:"forkDigest"` 47 | } 48 | 49 | type RegionalStats struct { 50 | TotalParticipatingCountries int `json:"totalParticipatingCountries"` 51 | HostedNodePercentage float64 `json:"hostedNodePercentage"` 52 | NonhostedNodePercentage float64 `json:"nonhostedNodePercentage"` 53 | } 54 | -------------------------------------------------------------------------------- /graph/resolver.go: -------------------------------------------------------------------------------- 1 | //go:generate go run github.com/99designs/gqlgen generate 2 | // Copyright 2021 ChainSafe Systems 3 | // SPDX-License-Identifier: LGPL-3.0-only 4 | 5 | // Package graph contans graph related code 6 | package graph 7 | 8 | import ( 9 | "eth2-crawler/store/peerstore" 10 | "eth2-crawler/store/record" 11 | ) 12 | 13 | // This file will not be regenerated automatically. 14 | // 15 | // It serves as dependency injection for your app, add any dependencies you require here. 16 | 17 | type Resolver struct { 18 | peerStore peerstore.Provider 19 | historyStore record.Provider 20 | } 21 | 22 | func NewResolver(peerStore peerstore.Provider, historyStore record.Provider) *Resolver { 23 | return &Resolver{peerStore: peerStore, historyStore: historyStore} 24 | } 25 | -------------------------------------------------------------------------------- /graph/schema.graphqls: -------------------------------------------------------------------------------- 1 | type AggregateData { 2 | name: String! 3 | count: Int! 4 | } 5 | 6 | type NextHardforkAggregation { 7 | version: String! 8 | epoch: String! 9 | count: Int! 10 | } 11 | 12 | type ClientVersionAggregation { 13 | client: String! 14 | count: Int! 15 | versions: [AggregateData!]! 16 | } 17 | 18 | type NodeStats { 19 | totalNodes: Int! 20 | nodeSyncedPercentage: Float! 21 | nodeUnsyncedPercentage: Float! 22 | } 23 | 24 | type NodeStatsOverTime { 25 | time: Float! 26 | totalNodes: Int! 27 | syncedNodes: Int! 28 | unsyncedNodes: Int! 29 | } 30 | 31 | type RegionalStats { 32 | totalParticipatingCountries: Int! 33 | hostedNodePercentage: Float! 34 | nonhostedNodePercentage: Float! 35 | } 36 | 37 | type HeatmapData { 38 | networkType: String! 39 | clientType: String! 40 | syncStatus: String! 41 | latitude: Float! 42 | longitude: Float! 43 | city: String! 44 | country: String! 45 | } 46 | 47 | input PeerFilter { 48 | forkDigest: String 49 | } 50 | 51 | type Query { 52 | aggregateByAgentName(peerFilter: PeerFilter): [AggregateData!]! 53 | aggregateByCountry(peerFilter: PeerFilter): [AggregateData!]! 54 | aggregateByOperatingSystem(peerFilter: PeerFilter): [AggregateData!]! 55 | aggregateByNetwork(peerFilter: PeerFilter): [AggregateData!]! 56 | aggregateByHardforkSchedule(peerFilter: PeerFilter): [NextHardforkAggregation!]! 57 | aggregateByClientVersion(peerFilter: PeerFilter): [ClientVersionAggregation!]! 58 | getHeatmapData(peerFilter: PeerFilter): [HeatmapData!]! 59 | getNodeStats(peerFilter: PeerFilter): NodeStats! 60 | getNodeStatsOverTime(start: Float!, end: Float!, peerFilter: PeerFilter): [NodeStatsOverTime!]! 61 | getRegionalStats(peerFilter: PeerFilter): RegionalStats! 62 | getAltairUpgradePercentage(peerFilter: PeerFilter): Float! 63 | } -------------------------------------------------------------------------------- /graph/schema.resolvers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package graph 5 | 6 | // This file will be automatically regenerated based on the schema, any resolver implementations 7 | // will be copied through when generating and any unknown code will be moved to the end. 8 | 9 | import ( 10 | "context" 11 | "eth2-crawler/graph/generated" 12 | "eth2-crawler/graph/model" 13 | svcModels "eth2-crawler/models" 14 | ) 15 | 16 | // AggregateByAgentName is the resolver for the aggregateByAgentName field. 17 | func (r *queryResolver) AggregateByAgentName(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.AggregateData, error) { 18 | aggregateData, err := r.peerStore.AggregateByAgentName(ctx, peerFilter) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | result := []*model.AggregateData{} 24 | for i := range aggregateData { 25 | result = append(result, &model.AggregateData{ 26 | Name: aggregateData[i].Name, 27 | Count: aggregateData[i].Count, 28 | }) 29 | } 30 | return result, nil 31 | } 32 | 33 | // AggregateByCountry is the resolver for the aggregateByCountry field. 34 | func (r *queryResolver) AggregateByCountry(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.AggregateData, error) { 35 | aggregateData, err := r.peerStore.AggregateByCountry(ctx, peerFilter) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | result := []*model.AggregateData{} 41 | for i := range aggregateData { 42 | result = append(result, &model.AggregateData{ 43 | Name: aggregateData[i].Name, 44 | Count: aggregateData[i].Count, 45 | }) 46 | } 47 | return result, nil 48 | } 49 | 50 | // AggregateByOperatingSystem is the resolver for the aggregateByOperatingSystem field. 51 | func (r *queryResolver) AggregateByOperatingSystem(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.AggregateData, error) { 52 | aggregateData, err := r.peerStore.AggregateByOperatingSystem(ctx, peerFilter) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | result := []*model.AggregateData{} 58 | for i := range aggregateData { 59 | result = append(result, &model.AggregateData{ 60 | Name: aggregateData[i].Name, 61 | Count: aggregateData[i].Count, 62 | }) 63 | } 64 | return result, nil 65 | } 66 | 67 | // AggregateByNetwork is the resolver for the aggregateByNetwork field. 68 | func (r *queryResolver) AggregateByNetwork(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.AggregateData, error) { 69 | aggregateData, err := r.peerStore.AggregateByNetworkType(ctx, peerFilter) 70 | if err != nil { 71 | return nil, err 72 | } 73 | result := []*model.AggregateData{} 74 | for i := range aggregateData { 75 | result = append(result, &model.AggregateData{ 76 | Name: aggregateData[i].Name, 77 | Count: aggregateData[i].Count, 78 | }) 79 | } 80 | return result, nil 81 | } 82 | 83 | // AggregateByHardforkSchedule is the resolver for the aggregateByHardforkSchedule field. 84 | func (r *queryResolver) AggregateByHardforkSchedule(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.NextHardforkAggregation, error) { 85 | allPeers, err := r.peerStore.ViewAll(ctx, peerFilter) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | result := []*model.NextHardforkAggregation{} 91 | 92 | for _, group := range model.GroupByHardforkSchedule(allPeers) { 93 | result = append(result, &model.NextHardforkAggregation{ 94 | Version: group.Version, 95 | Epoch: group.Epoch, 96 | Count: group.Count, 97 | }) 98 | } 99 | return model.SortByCount(result), nil 100 | } 101 | 102 | // AggregateByClientVersion is the resolver for the aggregateByClientVersion field. 103 | func (r *queryResolver) AggregateByClientVersion(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.ClientVersionAggregation, error) { 104 | aggregateData, err := r.peerStore.AggregateByClientVersion(ctx, peerFilter) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | result := []*model.ClientVersionAggregation{} 110 | for i := range aggregateData { 111 | versions := []*model.AggregateData{} 112 | for j := range aggregateData[i].Versions { 113 | versions = append(versions, &model.AggregateData{ 114 | Name: aggregateData[i].Versions[j].Name, 115 | Count: aggregateData[i].Versions[j].Count, 116 | }) 117 | } 118 | result = append(result, &model.ClientVersionAggregation{ 119 | Client: aggregateData[i].Client, 120 | Count: aggregateData[i].Count, 121 | Versions: versions, 122 | }) 123 | } 124 | return result, nil 125 | } 126 | 127 | // GetHeatmapData is the resolver for the getHeatmapData field. 128 | func (r *queryResolver) GetHeatmapData(ctx context.Context, peerFilter *model.PeerFilter) ([]*model.HeatmapData, error) { 129 | peers, err := r.peerStore.ViewAll(ctx, peerFilter) 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | result := []*model.HeatmapData{} 135 | for i := range peers { 136 | if peers[i].GeoLocation != nil && 137 | (peers[i].GeoLocation.Latitude != 0 || 138 | peers[i].GeoLocation.Longitude != 0) { 139 | var syncStatus string 140 | if peers[i].Sync != nil { 141 | syncStatus = peers[i].Sync.String() 142 | } 143 | result = append(result, &model.HeatmapData{ 144 | NetworkType: string(peers[i].GeoLocation.ASN.Type), 145 | ClientType: string(peers[i].UserAgent.Name), 146 | SyncStatus: syncStatus, 147 | Latitude: peers[i].GeoLocation.Latitude, 148 | Longitude: peers[i].GeoLocation.Longitude, 149 | City: peers[i].GeoLocation.City, 150 | Country: peers[i].GeoLocation.Country, 151 | }) 152 | } 153 | } 154 | return result, nil 155 | } 156 | 157 | // GetNodeStats is the resolver for the getNodeStats field. 158 | func (r *queryResolver) GetNodeStats(ctx context.Context, peerFilter *model.PeerFilter) (*model.NodeStats, error) { 159 | aggregateData, err := r.peerStore.AggregateBySyncStatus(ctx, peerFilter) 160 | if err != nil { 161 | return nil, err 162 | } 163 | return &model.NodeStats{ 164 | TotalNodes: aggregateData.Total, 165 | NodeSyncedPercentage: (float64(aggregateData.Synced) / float64(aggregateData.Total)) * 100, 166 | NodeUnsyncedPercentage: (float64(aggregateData.Unsynced) / float64(aggregateData.Total)) * 100, 167 | }, nil 168 | } 169 | 170 | // GetNodeStatsOverTime is the resolver for the getNodeStatsOverTime field. 171 | func (r *queryResolver) GetNodeStatsOverTime(ctx context.Context, start float64, end float64, peerFilter *model.PeerFilter) ([]*model.NodeStatsOverTime, error) { 172 | data, err := r.historyStore.GetHistory(ctx, int64(start), int64(end), peerFilter) 173 | if err != nil { 174 | return nil, err 175 | } 176 | result := make([]*model.NodeStatsOverTime, 0) 177 | for _, v := range data { 178 | result = append(result, &model.NodeStatsOverTime{ 179 | Time: float64(v.Time), 180 | TotalNodes: v.TotalNodes, 181 | SyncedNodes: v.SyncedNodes, 182 | UnsyncedNodes: v.TotalNodes - v.SyncedNodes, 183 | }) 184 | } 185 | return result, nil 186 | } 187 | 188 | // GetRegionalStats is the resolver for the getRegionalStats field. 189 | func (r *queryResolver) GetRegionalStats(ctx context.Context, peerFilter *model.PeerFilter) (*model.RegionalStats, error) { 190 | countryAggrData, err := r.peerStore.AggregateByCountry(ctx, peerFilter) 191 | if err != nil { 192 | return nil, err 193 | } 194 | 195 | networkAggrData, err := r.peerStore.AggregateByNetworkType(ctx, peerFilter) 196 | if err != nil { 197 | return nil, err 198 | } 199 | 200 | var hostedCount, nonhostedCount, total int 201 | for i := range networkAggrData { 202 | total += networkAggrData[i].Count 203 | if networkAggrData[i].Name == string(svcModels.UsageTypeHosting) { 204 | hostedCount += networkAggrData[i].Count 205 | } else { 206 | nonhostedCount += networkAggrData[i].Count 207 | } 208 | } 209 | 210 | result := &model.RegionalStats{ 211 | TotalParticipatingCountries: len(countryAggrData), 212 | HostedNodePercentage: (float64(hostedCount) / float64(total)) * 100, 213 | NonhostedNodePercentage: (float64(nonhostedCount) / float64(total)) * 100, 214 | } 215 | return result, nil 216 | } 217 | 218 | // GetAltairUpgradePercentage is the resolver for the getAltairUpgradePercentage field. 219 | func (r *queryResolver) GetAltairUpgradePercentage(ctx context.Context, peerFilter *model.PeerFilter) (float64, error) { 220 | aggregateData, err := r.peerStore.AggregateByClientVersion(ctx, peerFilter) 221 | if err != nil { 222 | return 0, err 223 | } 224 | // check altair upgrade 225 | count := 0 226 | total := 0 227 | for _, client := range aggregateData { 228 | for _, v := range client.Versions { 229 | total += v.Count 230 | if model.SupportAltairUpgrade(client.Client, v.Name) { 231 | count += v.Count 232 | } 233 | } 234 | } 235 | percentage := float64(count) / float64(total) * 100 236 | return percentage, nil 237 | } 238 | 239 | // Query returns generated.QueryResolver implementation. 240 | func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} } 241 | 242 | type queryResolver struct{ *Resolver } 243 | -------------------------------------------------------------------------------- /infra/aws-ecs/task_definition_PROD.json: -------------------------------------------------------------------------------- 1 | { 2 | "taskDefinitionArn": "arn:aws:ecs:us-east-2:381177214925:task-definition/nodewatch-prod-task:4", 3 | "containerDefinitions": [ 4 | { 5 | "name": "nodewatch-prod-container", 6 | "image": "381177214925.dkr.ecr.us-east-2.amazonaws.com/nodewatch-prod-ecr", 7 | "cpu": 256, 8 | "portMappings": [ 9 | { 10 | "containerPort": 8080, 11 | "hostPort": 8080, 12 | "protocol": "tcp" 13 | } 14 | ], 15 | "essential": true, 16 | "environment": [ 17 | { 18 | "name": "env", 19 | "value": "prod" 20 | } 21 | ], 22 | "mountPoints": [], 23 | "volumesFrom": [], 24 | "secrets": [ 25 | { 26 | "name": "MONGODB_URI", 27 | "valueFrom": "arn:aws:secretsmanager:us-east-2:381177214925:secret:nodewatch-prod-8FCCGw:MONGODB_URI::" 28 | }, 29 | { 30 | "name": "RESOLVER_API_KEY", 31 | "valueFrom": "arn:aws:secretsmanager:us-east-2:381177214925:secret:nodewatch-prod-8FCCGw:RESOLVER_API_KEY::" 32 | } 33 | ], 34 | "logConfiguration": { 35 | "logDriver": "awslogs", 36 | "options": { 37 | "awslogs-group": "nodewatch-prod-logs", 38 | "awslogs-region": "us-east-2", 39 | "awslogs-stream-prefix": "ecs" 40 | } 41 | } 42 | } 43 | ], 44 | "family": "nodewatch-prod-task", 45 | "executionRoleArn": "arn:aws:iam::381177214925:role/nodewatch-prod-role", 46 | "networkMode": "awsvpc", 47 | "revision": 4, 48 | "volumes": [], 49 | "status": "ACTIVE", 50 | "requiresAttributes": [ 51 | { 52 | "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" 53 | }, 54 | { 55 | "name": "ecs.capability.execution-role-awslogs" 56 | }, 57 | { 58 | "name": "com.amazonaws.ecs.capability.ecr-auth" 59 | }, 60 | { 61 | "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" 62 | }, 63 | { 64 | "name": "ecs.capability.secrets.asm.environment-variables" 65 | }, 66 | { 67 | "name": "ecs.capability.execution-role-ecr-pull" 68 | }, 69 | { 70 | "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" 71 | }, 72 | { 73 | "name": "ecs.capability.task-eni" 74 | } 75 | ], 76 | "placementConstraints": [], 77 | "compatibilities": [ 78 | "EC2", 79 | "FARGATE" 80 | ], 81 | "requiresCompatibilities": [ 82 | "FARGATE" 83 | ], 84 | "cpu": "1024", 85 | "memory": "2048", 86 | "registeredAt": "2022-04-27T15:23:12.288Z", 87 | "registeredBy": "arn:aws:sts::381177214925:assumed-role/AWSReservedSSO_AWSAdministratorAccess_8acb862b989cc854/faith@chainsafe.io", 88 | "tags": [ 89 | { 90 | "key": "Terraform", 91 | "value": "true" 92 | }, 93 | { 94 | "key": "Env", 95 | "value": "PROD" 96 | }, 97 | { 98 | "key": "Project", 99 | "value": "Nodewatch" 100 | } 101 | ] 102 | } 103 | -------------------------------------------------------------------------------- /models/data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package models 5 | 6 | const ( 7 | SyncTypeSynced = "synced" 8 | SyncTypeUnsynced = "unsynced" 9 | ) 10 | 11 | // AggregateData represents data of group by queries 12 | type AggregateData struct { 13 | Name string `json:"name"` 14 | Count int `json:"count"` 15 | } 16 | 17 | // ClientVersionAggregation represents aggregation data for client and client version 18 | type ClientVersionAggregation struct { 19 | Client string `json:"client"` 20 | Count int `json:"count"` 21 | Versions []*AggregateData `json:"versions"` 22 | } 23 | 24 | type HistoryCount struct { 25 | Time int64 `json:"time"` 26 | TotalNodes int `json:"total_nodes"` 27 | SyncedNodes int `json:"synced_nodes"` 28 | } 29 | 30 | type SyncAggregateData struct { 31 | Total int `json:"total" bson:"total"` 32 | Synced int `json:"synced" bson:"synced"` 33 | Unsynced int `json:"unsynced" bson:"unsynced"` 34 | } 35 | -------------------------------------------------------------------------------- /models/history.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package models 5 | 6 | import ( 7 | "time" 8 | 9 | "github.com/google/uuid" 10 | ) 11 | 12 | type History struct { 13 | ID uuid.UUID `bson:"_id" json:"id"` 14 | Time int64 `json:"time" bson:"time"` 15 | SyncNodes int `bson:"sync_nodes" json:"sync_nodes"` 16 | Eth2Nodes int `bson:"eth_2_nodes" json:"eth_2_nodes"` 17 | } 18 | 19 | func NewHistory(syncNodes int, eth2Nodes int) *History { 20 | t := time.Now() 21 | return &History{ 22 | ID: uuid.New(), 23 | Time: t.Unix(), 24 | SyncNodes: syncNodes, 25 | Eth2Nodes: eth2Nodes, 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /models/marshallable_epoch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package models 5 | 6 | import ( 7 | "fmt" 8 | "strconv" 9 | 10 | "github.com/protolambda/zrnt/eth2/beacon/common" 11 | "go.mongodb.org/mongo-driver/bson" 12 | "go.mongodb.org/mongo-driver/bson/bsontype" 13 | ) 14 | 15 | // Epoch type is needed to support proper BSON marshaling and unmarshaling to/from MongoDB. 16 | type Epoch common.Epoch 17 | 18 | func (e *Epoch) MarshalBSONValue() (bsontype.Type, []byte, error) { 19 | return bson.MarshalValue(toHexString(uint64(*e))) 20 | } 21 | 22 | func (e *Epoch) UnmarshalBSONValue(t bsontype.Type, b []byte) error { 23 | var container string 24 | rv := bson.RawValue{Type: t, Value: b} 25 | err := rv.Unmarshal(&container) 26 | if err != nil { 27 | return err 28 | } 29 | val, err := fromHexString(container) 30 | if err != nil { 31 | return err 32 | } 33 | *e = Epoch(val) 34 | return nil 35 | } 36 | 37 | func (e *Epoch) String() string { 38 | return common.Epoch(*e).String() 39 | } 40 | 41 | func toHexString(i uint64) string { 42 | return fmt.Sprintf("0x%x", i) 43 | } 44 | 45 | func fromHexString(s string) (uint64, error) { 46 | return strconv.ParseUint(s[2:], 16, 64) 47 | } 48 | -------------------------------------------------------------------------------- /models/peer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package models represent the models for the service 5 | package models 6 | 7 | import ( 8 | "encoding/hex" 9 | "encoding/json" 10 | "strings" 11 | "time" 12 | 13 | "eth2-crawler/crawler/util" 14 | 15 | "github.com/ethereum/go-ethereum/log" 16 | "github.com/ethereum/go-ethereum/p2p/enode" 17 | ic "github.com/libp2p/go-libp2p-core/crypto" 18 | "github.com/libp2p/go-libp2p-core/peer" 19 | ma "github.com/multiformats/go-multiaddr" 20 | "github.com/protolambda/zrnt/eth2/beacon/common" 21 | ) 22 | 23 | // 256 epochs 24 | const blockIgnoreThreshold = 8192 25 | 26 | // ClientName defines the type for eth2 client name 27 | type ClientName string 28 | 29 | const ( 30 | PrysmClient ClientName = "prysm" 31 | LighthouseClient ClientName = "lighthouse" 32 | TekuClient ClientName = "teku" 33 | CortexClient ClientName = "cortex" 34 | LodestarClient ClientName = "lodestar" 35 | NimbusClient ClientName = "nimbus" 36 | TrinityClient ClientName = "trinity" 37 | GrandineClient ClientName = "grandine" 38 | OthersClient ClientName = "others" 39 | ) 40 | 41 | // clients contains mapping for client name - possible identity names mapping 42 | var clients map[ClientName][]string 43 | 44 | func init() { 45 | clients = map[ClientName][]string{ 46 | PrysmClient: {"prysm"}, 47 | LighthouseClient: {"lighthouse"}, 48 | TekuClient: {"teku"}, 49 | CortexClient: {"cortex"}, 50 | LodestarClient: {"lodestar", "js-libp2p"}, 51 | NimbusClient: {"nimbus"}, 52 | TrinityClient: {"trinity"}, 53 | GrandineClient: {"grandine", "rust-libp2p"}, 54 | } 55 | } 56 | 57 | // OS defines the type of os of agent 58 | 59 | type OS string 60 | 61 | const ( 62 | OSLinux OS = "linux" 63 | OSMAC OS = "mac" 64 | OSWindows OS = "windows" 65 | OSUnknown OS = "unknown" 66 | 67 | VersionUnknown = "unknown" 68 | StatusSynced = "synced" 69 | StatusUnsynced = "unsynced" 70 | ) 71 | 72 | // UserAgent holds peer's client related info 73 | type UserAgent struct { 74 | Name ClientName `json:"name" bson:"name"` 75 | Version string `json:"version" bson:"version"` 76 | OS OS `json:"os" bson:"os"` 77 | } 78 | 79 | // UsageType defines the ASN usage type 80 | type UsageType string 81 | 82 | const ( 83 | UsageTypeNil UsageType = "" 84 | UsageTypeHosting UsageType = "hosting" 85 | UsageTypeResidential UsageType = "residential" 86 | UsageTypeNonResidential UsageType = "non-residential" 87 | UsageTypeBusiness UsageType = "business" 88 | UsageTypeEducation UsageType = "education" 89 | UsageTypeGovernment UsageType = "government" 90 | UsageTypeMilitary UsageType = "military" 91 | ) 92 | 93 | type Score int 94 | 95 | const ( 96 | ScoreGood Score = 3 97 | ScoreBad Score = 0 98 | ) 99 | 100 | // ASN holds the Autonomous system details 101 | type ASN struct { 102 | ID string `json:"id" bson:"id"` 103 | Name string `json:"name" bson:"name"` 104 | Domain string `json:"domain" bson:"domain"` 105 | Route string `json:"route" bson:"route"` 106 | Type UsageType `json:"type" bson:"type"` 107 | } 108 | 109 | // GeoLocation holds peer's geo location related info 110 | type GeoLocation struct { 111 | ASN ASN `json:"asn" nson:"asn"` 112 | Country string `json:"country_name" bson:"country"` 113 | State string `json:"state" bson:"state"` 114 | City string `json:"city" bson:"city"` 115 | Latitude float64 `json:"latitude" bson:"latitude"` 116 | Longitude float64 `json:"longitude" bson:"longitude"` 117 | } 118 | 119 | // Sync holds peer sync related info 120 | type Sync struct { 121 | Status bool `json:"status" bson:"status"` 122 | Distance int `json:"distance" bson:"distance"` // sync distance in percentage 123 | } 124 | 125 | // String returns the sync status 126 | func (s *Sync) String() string { 127 | if s.Status { 128 | return StatusSynced 129 | } 130 | return StatusUnsynced 131 | } 132 | 133 | // Peer holds all information of an eth2 peer 134 | type Peer struct { 135 | ID peer.ID `json:"id" bson:"_id"` 136 | NodeID string `json:"node_id" bson:"node_id"` 137 | Pubkey string `json:"pubkey" bson:"pubkey"` 138 | 139 | IP string `json:"ip" bson:"ip"` 140 | TCPPort int `json:"tcp_port" bson:"tcp_port"` 141 | UDPPort int `json:"udp_port" bson:"udp_port"` 142 | Addrs []string `json:"addrs,omitempty" bson:"addrs"` 143 | 144 | Attnets common.AttnetBits `json:"enr_attnets,omitempty" bson:"attnets"` 145 | 146 | ForkDigest common.ForkDigest `json:"fork_digest" bson:"fork_digest"` 147 | ForkDigestStr string `json:"fork_digest_str" bson:"fork_digest_str"` 148 | NextForkEpoch Epoch `json:"next_fork_epoch" bson:"next_fork_epoch"` 149 | NextForkVersion common.Version `json:"next_fork_version" bson:"next_fork_version"` 150 | 151 | ProtocolVersion string `json:"protocol_version,omitempty" bson:"protocol_version"` 152 | UserAgent *UserAgent `json:"user_agent,omitempty" bson:"user_agent"` 153 | UserAgentRaw string `json:"user_agent_raw" bson:"user_agent_raw"` 154 | GeoLocation *GeoLocation `json:"geo_location" bson:"geo_location"` 155 | 156 | Sync *Sync `json:"sync" bson:"sync"` 157 | Score Score `json:"score" bson:"score"` 158 | 159 | IsConnectable bool `json:"is_connectable" bson:"is_connectable"` 160 | LastConnected int64 `json:"last_connected" bson:"last_connected"` 161 | LastUpdated int64 `json:"last_updated" bson:"last_updated"` 162 | } 163 | 164 | // NewPeer initializes new peer 165 | func NewPeer(node *enode.Node, eth2Data *common.Eth2Data) (*Peer, error) { 166 | pk := ic.PubKey((*ic.Secp256k1PublicKey)(node.Pubkey())) 167 | pkByte, err := pk.Raw() 168 | if err != nil { 169 | return nil, err 170 | } 171 | addr, err := util.AddrsFromEnode(node) 172 | if err != nil { 173 | return nil, err 174 | } 175 | addrStr := make([]string, 0) 176 | for _, madd := range addr.Addrs { 177 | addrStr = append(addrStr, madd.String()) 178 | } 179 | 180 | attnetsVal := common.AttnetBits{} 181 | attnets, err := util.ParseEnrAttnets(node) 182 | if err == nil { 183 | attnetsVal = *attnets 184 | } 185 | return &Peer{ 186 | ID: addr.ID, 187 | NodeID: node.ID().String(), 188 | Pubkey: hex.EncodeToString(pkByte), 189 | IP: node.IP().String(), 190 | TCPPort: node.TCP(), 191 | UDPPort: node.UDP(), 192 | Addrs: addrStr, 193 | ForkDigest: eth2Data.ForkDigest, 194 | ForkDigestStr: eth2Data.ForkDigest.String(), 195 | NextForkVersion: eth2Data.NextForkVersion, 196 | NextForkEpoch: Epoch(eth2Data.NextForkEpoch), 197 | Attnets: attnetsVal, 198 | Score: ScoreGood, 199 | }, nil 200 | } 201 | 202 | // SetProtocolVersion sets peer's protocol version 203 | func (p *Peer) SetProtocolVersion(pv string) { 204 | p.ProtocolVersion = pv 205 | } 206 | 207 | // SetUserAgent sets peer's agent info 208 | func (p *Peer) SetUserAgent(ag string) { 209 | // split the ag based on this format. might not be identical with each type of node 210 | // ag = Name/Version/OS(or git commit hash for Prysm) 211 | 212 | userAgent := new(UserAgent) 213 | parts := strings.Split(ag, "/") 214 | 215 | nameChecker: 216 | for name, identityNames := range clients { 217 | for i := range identityNames { 218 | if strings.EqualFold(identityNames[i], parts[0]) { 219 | userAgent.Name = name 220 | break nameChecker 221 | } 222 | } 223 | } 224 | 225 | if userAgent.Name == "" { 226 | userAgent.Name = OthersClient 227 | } 228 | 229 | var os = "" 230 | switch userAgent.Name { 231 | case TekuClient: 232 | if len(parts) > 2 { 233 | userAgent.Version = parts[2] 234 | } 235 | if len(parts) > 3 { 236 | os = parts[3] 237 | } 238 | case PrysmClient: 239 | if len(parts) > 1 { 240 | userAgent.Version = parts[1] 241 | } 242 | default: 243 | if len(parts) > 1 { 244 | userAgent.Version = parts[1] 245 | } 246 | if len(parts) > 2 { 247 | os = parts[2] 248 | } 249 | } 250 | // update the version and os to standard form 251 | versions := strings.Split(userAgent.Version, "-") 252 | versions = strings.Split(versions[0], "+") 253 | userAgent.Version = versions[0] 254 | if userAgent.Version == "" { 255 | userAgent.Version = VersionUnknown 256 | } 257 | 258 | var validOS = []OS{OSLinux, OSMAC, OSWindows} 259 | for _, vos := range validOS { 260 | if strings.Contains(strings.ToLower(os), strings.ToLower(string(vos))) { 261 | userAgent.OS = vos 262 | } 263 | } 264 | if userAgent.OS == "" { 265 | userAgent.OS = OSUnknown 266 | } 267 | p.UserAgent = userAgent 268 | p.UserAgentRaw = ag 269 | } 270 | 271 | // SetConnectionStatus sets connection status and date 272 | func (p *Peer) SetConnectionStatus(status bool) { 273 | p.IsConnectable = status 274 | if status { 275 | p.LastConnected = time.Now().Unix() 276 | } 277 | } 278 | 279 | // SetSyncStatus sets the sync status of a peer 280 | func (p *Peer) SetSyncStatus(block int64) { 281 | cb := util.CurrentBlock() 282 | if cb-block <= blockIgnoreThreshold { 283 | p.Sync = &Sync{ 284 | Status: true, 285 | Distance: 0, 286 | } 287 | } else { 288 | p.Sync = &Sync{ 289 | Status: false, 290 | Distance: int(((cb - block) * 100) / cb), 291 | } 292 | } 293 | } 294 | 295 | // SetGeoLocation sets the geolocation information 296 | func (p *Peer) SetGeoLocation(geoLocation *GeoLocation) { 297 | p.GeoLocation = geoLocation 298 | } 299 | 300 | // GetPeerInfo returns peer's AddrInfo 301 | func (p *Peer) GetPeerInfo() *peer.AddrInfo { 302 | maddrs := make([]ma.Multiaddr, 0) 303 | for _, v := range p.Addrs { 304 | madd, _ := ma.NewMultiaddr(v) 305 | maddrs = append(maddrs, madd) 306 | } 307 | return &peer.AddrInfo{ 308 | ID: p.ID, 309 | Addrs: maddrs, 310 | } 311 | } 312 | 313 | // String returns peer object's json form in string 314 | func (p *Peer) String() string { 315 | if p == nil { 316 | return "no data available" 317 | } else { 318 | dat, err := json.Marshal(p) 319 | if err != nil { 320 | return "failed to format peer data" 321 | } 322 | return string(dat) 323 | } 324 | } 325 | 326 | // Log returns log ctx from peer 327 | func (p *Peer) Log() log.Ctx { 328 | dat, err := json.Marshal(p) 329 | if err != nil { 330 | return log.Ctx{} 331 | } 332 | val := log.Ctx{} 333 | err = json.Unmarshal(dat, &val) 334 | if err != nil { 335 | return log.Ctx{} 336 | } 337 | return val 338 | } 339 | -------------------------------------------------------------------------------- /resolver/ipdata/ipdata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package ipdata implements the ip resolver using ipdata APIs 5 | package ipdata 6 | 7 | import ( 8 | "context" 9 | "eth2-crawler/models" 10 | "eth2-crawler/resolver" 11 | "time" 12 | 13 | ipdata "github.com/ipdata/go" 14 | ) 15 | 16 | type client struct { 17 | ipdataClient ipdata.Client 18 | defaultTimeout time.Duration 19 | } 20 | 21 | func New(apiKey string, defaultTimeout time.Duration) (resolver.Provider, error) { 22 | ipdataClient, err := ipdata.NewClient(apiKey) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | return &client{ 28 | ipdataClient: ipdataClient, 29 | defaultTimeout: defaultTimeout, 30 | }, nil 31 | } 32 | 33 | func (c *client) GetGeoLocation(ctx context.Context, ipAddr string) (*models.GeoLocation, error) { 34 | ctx, cancel := context.WithTimeout(ctx, c.defaultTimeout) 35 | defer cancel() 36 | 37 | data, err := c.ipdataClient.LookupWithContext(ctx, ipAddr) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | var asnType models.UsageType 43 | switch data.ASN.Type { 44 | case "hosting": 45 | asnType = models.UsageTypeHosting 46 | case "isp": 47 | asnType = models.UsageTypeResidential 48 | case "business": 49 | asnType = models.UsageTypeBusiness 50 | case "edu": 51 | asnType = models.UsageTypeEducation 52 | case "gov": 53 | asnType = models.UsageTypeGovernment 54 | case "mil": 55 | asnType = models.UsageTypeMilitary 56 | default: 57 | asnType = models.UsageTypeNil 58 | } 59 | 60 | geoLoc := &models.GeoLocation{ 61 | ASN: models.ASN{ 62 | ID: data.ASN.ASN, 63 | Name: data.ASN.Name, 64 | Domain: data.ASN.Domain, 65 | Route: data.ASN.Route, 66 | Type: asnType, 67 | }, 68 | Country: data.CountryName, 69 | State: data.Region, 70 | City: data.City, 71 | Latitude: data.Latitude, 72 | Longitude: data.Longitude, 73 | } 74 | return geoLoc, nil 75 | } 76 | -------------------------------------------------------------------------------- /resolver/ipgeolocation/ipgeolocation.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package ipgeolocation implements the ip resolver using ipgeolocation APIs 5 | package ipgeolocation 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "strconv" 14 | "time" 15 | 16 | "eth2-crawler/models" 17 | "eth2-crawler/resolver" 18 | ) 19 | 20 | const ( 21 | url = "https://api.ipgeolocation.io/ipgeo" 22 | ) 23 | 24 | type client struct { 25 | httpClient *http.Client 26 | apiKey string 27 | } 28 | 29 | type geoInformation struct { 30 | ISP string `json:"isp"` 31 | Organization string `json:"organization"` 32 | Country string `json:"country_name"` 33 | State string `json:"state_prov"` 34 | City string `json:"city"` 35 | Latitude string `json:"latitude"` 36 | Longitude string `json:"longitude"` 37 | } 38 | 39 | func New(apiKey string, defaultTimeout time.Duration) resolver.Provider { 40 | return &client{ 41 | httpClient: &http.Client{Timeout: defaultTimeout}, 42 | apiKey: apiKey, 43 | } 44 | } 45 | 46 | func (c *client) GetGeoLocation(ctx context.Context, ipAddr string) (*models.GeoLocation, error) { 47 | req, err := http.NewRequest("GET", url, nil) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | q := req.URL.Query() 53 | q.Add("apiKey", c.apiKey) 54 | q.Add("ip", ipAddr) 55 | req.URL.RawQuery = q.Encode() 56 | 57 | res, err := c.httpClient.Do(req) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | // nolint 63 | defer res.Body.Close() 64 | resBody, err := io.ReadAll(res.Body) 65 | if err != nil { 66 | return nil, fmt.Errorf("unable to read response body") 67 | } 68 | 69 | if res.StatusCode != http.StatusOK { 70 | return nil, fmt.Errorf("invalid status code returned. statusCode::%d response::%s", res.StatusCode, string(resBody)) 71 | } 72 | 73 | result := &geoInformation{} 74 | err = json.Unmarshal(resBody, result) 75 | if err != nil { 76 | return nil, fmt.Errorf("unable to unmarshal body. error::%w", err) 77 | } 78 | 79 | lat, _ := strconv.ParseFloat(result.Latitude, 64) 80 | long, _ := strconv.ParseFloat(result.Longitude, 64) 81 | 82 | geoLoc := &models.GeoLocation{ 83 | Country: result.Country, 84 | State: result.State, 85 | City: result.City, 86 | Latitude: lat, 87 | Longitude: long, 88 | } 89 | return geoLoc, nil 90 | } 91 | -------------------------------------------------------------------------------- /resolver/resolver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package resolver implements the ip resolver 5 | package resolver 6 | 7 | import ( 8 | "context" 9 | 10 | "eth2-crawler/models" 11 | ) 12 | 13 | type Provider interface { 14 | GetGeoLocation(ctx context.Context, ipAddr string) (*models.GeoLocation, error) 15 | } 16 | -------------------------------------------------------------------------------- /store/peerstore/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | package peerstore 5 | 6 | import "errors" 7 | 8 | var ( 9 | ErrPeerNotFound = errors.New("unable to find the node") 10 | ) 11 | -------------------------------------------------------------------------------- /store/peerstore/mongo/mongo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package mongo represent store driver for mongodb 5 | package mongo 6 | 7 | import ( 8 | "context" 9 | "encoding/hex" 10 | "errors" 11 | "fmt" 12 | "time" 13 | 14 | "eth2-crawler/graph/model" 15 | "eth2-crawler/models" 16 | "eth2-crawler/store/peerstore" 17 | "eth2-crawler/utils/config" 18 | 19 | "github.com/libp2p/go-libp2p-core/peer" 20 | "go.mongodb.org/mongo-driver/bson" 21 | "go.mongodb.org/mongo-driver/mongo" 22 | "go.mongodb.org/mongo-driver/mongo/options" 23 | "go.mongodb.org/mongo-driver/mongo/readpref" 24 | ) 25 | 26 | type mongoStore struct { 27 | client *mongo.Client 28 | coll *mongo.Collection 29 | timeout time.Duration 30 | } 31 | 32 | func (s *mongoStore) Create(ctx context.Context, peer *models.Peer) error { 33 | _, err := s.View(ctx, peer.ID) 34 | if err != nil { 35 | if errors.Is(err, peerstore.ErrPeerNotFound) { 36 | _, err = s.coll.InsertOne(ctx, peer) 37 | return err 38 | } 39 | return err 40 | } 41 | return nil 42 | } 43 | 44 | func (s *mongoStore) Update(ctx context.Context, peer *models.Peer) error { 45 | filter := bson.D{ 46 | {Key: "_id", Value: peer.ID}, 47 | } 48 | _, err := s.coll.ReplaceOne(ctx, filter, peer) 49 | if err != nil { 50 | return err 51 | } 52 | return nil 53 | } 54 | 55 | func (s *mongoStore) Delete(ctx context.Context, peer *models.Peer) error { 56 | filter := bson.D{ 57 | {Key: "_id", Value: peer.ID}, 58 | } 59 | _, err := s.coll.DeleteOne(ctx, filter) 60 | if err != nil { 61 | return err 62 | } 63 | return nil 64 | } 65 | 66 | func (s *mongoStore) View(ctx context.Context, peerID peer.ID) (*models.Peer, error) { 67 | filter := bson.D{ 68 | {Key: "_id", Value: peerID}, 69 | } 70 | res := new(models.Peer) 71 | err := s.coll.FindOne(ctx, filter).Decode(res) 72 | if err != nil { 73 | if errors.Is(err, mongo.ErrNoDocuments) { 74 | return nil, peerstore.ErrPeerNotFound 75 | } 76 | return nil, err 77 | } 78 | 79 | return res, nil 80 | } 81 | 82 | func AddForkDigestFilterToQueryPipeline(query mongo.Pipeline, peerFilter *model.PeerFilter) (mongo.Pipeline, error) { 83 | forkDigest := *(peerFilter.ForkDigest) 84 | if forkDigest[0:2] == "0x" { 85 | forkDigest = forkDigest[2:] 86 | } 87 | forkDigestBytes, err := hex.DecodeString(forkDigest) 88 | if err != nil { 89 | return nil, err 90 | } 91 | query = append(query, bson.D{ 92 | {Key: "$match", Value: bson.D{{Key: "fork_digest", Value: forkDigestBytes}}}, 93 | }) 94 | 95 | return query, nil 96 | } 97 | 98 | func AddPeerFilterToQueryPipeline(query mongo.Pipeline, peerFilter *model.PeerFilter) (mongo.Pipeline, error) { 99 | var err error 100 | if peerFilter != nil && 101 | peerFilter.ForkDigest != nil { 102 | query, err = AddForkDigestFilterToQueryPipeline(query, peerFilter) 103 | if err != nil { 104 | return nil, err 105 | } 106 | } 107 | 108 | return query, nil 109 | } 110 | 111 | // Todo: accept filter and find options to get limited information 112 | func (s *mongoStore) ViewAll(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.Peer, error) { 113 | var peers []*models.Peer 114 | 115 | query := mongo.Pipeline{ 116 | bson.D{ 117 | {Key: "$match", Value: bson.D{{Key: "is_connectable", Value: true}}}, 118 | }, 119 | } 120 | 121 | var err error 122 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | cursor, err := s.coll.Aggregate(ctx, query) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | for cursor.Next(ctx) { 133 | // create a value into which the single document can be decoded 134 | peer := new(models.Peer) 135 | err := cursor.Decode(peer) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | peers = append(peers, peer) 141 | } 142 | return peers, nil 143 | } 144 | 145 | func (s *mongoStore) ListForJob(ctx context.Context, lastUpdated time.Duration, limit int) ([]*models.Peer, error) { 146 | var peers []*models.Peer 147 | timeToSkip := time.Now().Add(-lastUpdated).Unix() 148 | opts := options.Find() 149 | opts.SetLimit(int64(limit)) 150 | opts.SetSort(bson.D{{Key: "last_updated", Value: 1}}) 151 | filter := bson.D{{Key: "last_updated", Value: bson.D{{Key: "$lt", Value: timeToSkip}}}} 152 | cursor, err := s.coll.Find(ctx, filter, opts) 153 | if err != nil { 154 | return nil, err 155 | } 156 | 157 | for cursor.Next(ctx) { 158 | // create a value into which the single document can be decoded 159 | peer := new(models.Peer) 160 | err := cursor.Decode(peer) 161 | if err != nil { 162 | return nil, err 163 | } 164 | 165 | peers = append(peers, peer) 166 | } 167 | return peers, nil 168 | } 169 | 170 | type aggregateData struct { 171 | ID string `json:"_id" bson:"_id"` 172 | Count int `json:"count" bson:"count"` 173 | } 174 | 175 | func (s *mongoStore) AggregateByAgentName(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) { 176 | query := mongo.Pipeline{ 177 | bson.D{ 178 | {Key: "$match", Value: bson.D{ 179 | {Key: "is_connectable", Value: bson.D{{Key: "$eq", Value: true}}}, 180 | }}, 181 | }, 182 | } 183 | 184 | var err error 185 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | query = append(query, bson.D{ 191 | {Key: "$group", Value: bson.D{ 192 | {Key: "_id", Value: "$user_agent.name"}, 193 | {Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}}, 194 | }}, 195 | }) 196 | 197 | cursor, err := s.coll.Aggregate(ctx, query) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | var result []*models.AggregateData 203 | for cursor.Next(ctx) { 204 | // create a value into which the single document can be decoded 205 | data := new(aggregateData) 206 | err := cursor.Decode(data) 207 | if err != nil { 208 | return nil, err 209 | } 210 | 211 | result = append(result, &models.AggregateData{Name: data.ID, Count: data.Count}) 212 | } 213 | return result, nil 214 | } 215 | 216 | type clientVersionAggregation struct { 217 | ID string `json:"_id" bson:"_id"` 218 | Count int `json:"count" bson:"count"` 219 | Versions []*models.AggregateData `json:"versions" bson:"versions"` 220 | } 221 | 222 | func (s *mongoStore) AggregateByClientVersion(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.ClientVersionAggregation, error) { 223 | query := mongo.Pipeline{ 224 | bson.D{ 225 | {Key: "$match", Value: bson.D{ 226 | {Key: "is_connectable", Value: bson.D{{Key: "$eq", Value: true}}}, 227 | }}, 228 | }, 229 | } 230 | 231 | var err error 232 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 233 | if err != nil { 234 | return nil, err 235 | } 236 | 237 | query = append(query, bson.D{ 238 | {Key: "$group", Value: bson.D{ 239 | {Key: "_id", Value: bson.D{ 240 | {Key: "client", Value: "$user_agent.name"}, 241 | {Key: "version", Value: "$user_agent.version"}, 242 | }}, 243 | {Key: "versionCount", Value: bson.D{{Key: "$sum", Value: 1}}}, 244 | }}, 245 | }, 246 | 247 | bson.D{ 248 | {Key: "$group", Value: bson.D{ 249 | {Key: "_id", Value: "$_id.client"}, 250 | {Key: "versions", Value: bson.D{ 251 | {Key: "$push", Value: bson.D{ 252 | {Key: "name", Value: "$_id.version"}, 253 | {Key: "count", Value: "$versionCount"}, 254 | }}, 255 | }}, 256 | {Key: "count", Value: bson.D{ 257 | {Key: "$sum", Value: "$versionCount"}, 258 | }}, 259 | }}, 260 | }) 261 | 262 | cursor, err := s.coll.Aggregate(ctx, query) 263 | if err != nil { 264 | return nil, err 265 | } 266 | 267 | var result []*models.ClientVersionAggregation 268 | for cursor.Next(ctx) { 269 | // create a value into which the single document can be decoded 270 | data := new(clientVersionAggregation) 271 | err := cursor.Decode(data) 272 | if err != nil { 273 | return nil, err 274 | } 275 | 276 | result = append(result, &models.ClientVersionAggregation{ 277 | Client: data.ID, 278 | Count: data.Count, 279 | Versions: data.Versions, 280 | }) 281 | } 282 | return result, nil 283 | } 284 | 285 | func (s *mongoStore) AggregateByOperatingSystem(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) { 286 | query := mongo.Pipeline{ 287 | bson.D{ 288 | {Key: "$match", Value: bson.D{ 289 | {Key: "is_connectable", Value: bson.D{{Key: "$eq", Value: true}}}, 290 | }}, 291 | }, 292 | } 293 | 294 | var err error 295 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 296 | if err != nil { 297 | return nil, err 298 | } 299 | 300 | query = append(query, bson.D{ 301 | {Key: "$group", Value: bson.D{ 302 | {Key: "_id", Value: "$user_agent.os"}, 303 | {Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}}, 304 | }}, 305 | }) 306 | 307 | cursor, err := s.coll.Aggregate(ctx, query) 308 | if err != nil { 309 | return nil, err 310 | } 311 | 312 | var result []*models.AggregateData 313 | for cursor.Next(ctx) { 314 | // create a value into which the single document can be decoded 315 | data := new(aggregateData) 316 | err := cursor.Decode(data) 317 | if err != nil { 318 | return nil, err 319 | } 320 | 321 | result = append(result, &models.AggregateData{Name: data.ID, Count: data.Count}) 322 | } 323 | return result, nil 324 | } 325 | 326 | func (s *mongoStore) AggregateByCountry(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) { 327 | query := mongo.Pipeline{ 328 | bson.D{ 329 | {Key: "$match", Value: bson.D{ 330 | {Key: "is_connectable", Value: bson.D{{Key: "$eq", Value: true}}}, 331 | }}, 332 | }, 333 | } 334 | 335 | var err error 336 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 337 | if err != nil { 338 | return nil, err 339 | } 340 | 341 | query = append(query, bson.D{ 342 | {Key: "$group", Value: bson.D{ 343 | {Key: "_id", Value: "$geo_location.country"}, 344 | {Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}}, 345 | }}, 346 | }) 347 | 348 | cursor, err := s.coll.Aggregate(ctx, query) 349 | if err != nil { 350 | return nil, err 351 | } 352 | 353 | var result []*models.AggregateData 354 | for cursor.Next(ctx) { 355 | // create a value into which the single document can be decoded 356 | data := new(aggregateData) 357 | err := cursor.Decode(data) 358 | if err != nil { 359 | return nil, err 360 | } 361 | 362 | result = append(result, &models.AggregateData{Name: data.ID, Count: data.Count}) 363 | } 364 | return result, nil 365 | } 366 | 367 | func (s *mongoStore) AggregateByNetworkType(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) { 368 | query := mongo.Pipeline{ 369 | bson.D{ 370 | // avoid aggregation of entries without geolocation information 371 | {Key: "$match", Value: bson.D{ 372 | {Key: "$and", Value: bson.A{ 373 | bson.D{{Key: "is_connectable", Value: bson.D{{Key: "$eq", Value: true}}}}, 374 | bson.D{{Key: "geo_location", Value: bson.D{{Key: "$ne", Value: nil}}}}, 375 | }}, 376 | }}, 377 | }, 378 | } 379 | 380 | var err error 381 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 382 | if err != nil { 383 | return nil, err 384 | } 385 | 386 | query = append(query, bson.D{ 387 | {Key: "$group", Value: bson.D{ 388 | {Key: "_id", Value: "$geo_location.asn.type"}, 389 | {Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}}, 390 | }}, 391 | }) 392 | 393 | cursor, err := s.coll.Aggregate(ctx, query) 394 | if err != nil { 395 | return nil, err 396 | } 397 | 398 | var result []*models.AggregateData 399 | for cursor.Next(ctx) { 400 | // create a value into which the single document can be decoded 401 | data := new(aggregateData) 402 | err := cursor.Decode(data) 403 | if err != nil { 404 | return nil, err 405 | } 406 | 407 | result = append(result, &models.AggregateData{Name: data.ID, Count: data.Count}) 408 | } 409 | return result, nil 410 | } 411 | 412 | type count struct { 413 | Count int `json:"count" bson:"count"` 414 | } 415 | 416 | type aggregateSyncData struct { 417 | Total []count `json:"total" bson:"total"` 418 | Synced []count `json:"synced" bson:"synced"` 419 | Unsynced []count `json:"unsynced" bson:"unsynced"` 420 | } 421 | 422 | func (s *mongoStore) AggregateBySyncStatus(ctx context.Context, peerFilter *model.PeerFilter) (*models.SyncAggregateData, error) { 423 | query := mongo.Pipeline{ 424 | bson.D{ 425 | {Key: "$match", Value: bson.D{ 426 | {Key: "is_connectable", Value: bson.D{{Key: "$eq", Value: true}}}, 427 | }}, 428 | }, 429 | } 430 | 431 | var err error 432 | query, err = AddPeerFilterToQueryPipeline(query, peerFilter) 433 | if err != nil { 434 | return nil, err 435 | } 436 | 437 | total := bson.A{bson.D{{Key: "$count", Value: "count"}}} 438 | synced := bson.A{bson.D{{Key: "$match", Value: bson.D{{Key: "sync.status", Value: true}}}}, bson.D{{Key: "$count", Value: "count"}}} 439 | unsynced := bson.A{bson.D{{Key: "$match", Value: bson.D{{Key: "sync.status", Value: false}}}}, bson.D{{Key: "$count", Value: "count"}}} 440 | query = append(query, bson.D{{Key: "$facet", Value: bson.D{ 441 | {Key: "total", Value: total}, 442 | {Key: "synced", Value: synced}, 443 | {Key: "unsynced", Value: unsynced}}}}) 444 | 445 | cursor, err := s.coll.Aggregate(ctx, query) 446 | if err != nil { 447 | return nil, err 448 | } 449 | 450 | result := new(models.SyncAggregateData) 451 | for cursor.Next(ctx) { 452 | data := aggregateSyncData{} 453 | err := cursor.Decode(&data) 454 | if err != nil { 455 | return nil, err 456 | } 457 | if len(data.Total) != 0 { 458 | result.Total = data.Total[0].Count 459 | } 460 | if len(data.Synced) != 0 { 461 | result.Synced = data.Synced[0].Count 462 | } 463 | if len(data.Unsynced) != 0 { 464 | result.Unsynced = data.Unsynced[0].Count 465 | } 466 | } 467 | return result, nil 468 | } 469 | 470 | // New creates new instance of Entry Store based on MongoDB 471 | func New(cfg *config.Database) (peerstore.Provider, error) { 472 | timeout := time.Duration(cfg.Timeout) * time.Second 473 | opts := options.Client() 474 | 475 | opts.ApplyURI(cfg.URI) 476 | client, err := mongo.NewClient(opts) 477 | if err != nil { 478 | return nil, fmt.Errorf("connecton error [%s]: %w", opts.GetURI(), err) 479 | } 480 | 481 | // connect to the mongoDB cluster 482 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 483 | defer cancel() 484 | 485 | err = client.Connect(ctx) 486 | if err != nil { 487 | return nil, err 488 | } 489 | 490 | // test the connection 491 | err = client.Ping(ctx, readpref.Primary()) 492 | if err != nil { 493 | return nil, err 494 | } 495 | 496 | return &mongoStore{ 497 | client: client, 498 | coll: client.Database(cfg.Database).Collection(cfg.Collection), 499 | timeout: timeout, 500 | }, nil 501 | } 502 | -------------------------------------------------------------------------------- /store/peerstore/store.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package peerstore represents the data Store service 5 | package peerstore 6 | 7 | import ( 8 | "context" 9 | "time" 10 | 11 | "eth2-crawler/graph/model" 12 | "eth2-crawler/models" 13 | 14 | "github.com/libp2p/go-libp2p-core/peer" 15 | ) 16 | 17 | // Provider represents store provider interface that can be implemented by different DB engines 18 | type Provider interface { 19 | Create(ctx context.Context, peer *models.Peer) error 20 | Update(ctx context.Context, peer *models.Peer) error 21 | View(ctx context.Context, peerID peer.ID) (*models.Peer, error) 22 | Delete(ctx context.Context, peer *models.Peer) error 23 | // Todo: accept filter and find options to get limited information 24 | ViewAll(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.Peer, error) 25 | ListForJob(ctx context.Context, lastUpdated time.Duration, limit int) ([]*models.Peer, error) 26 | AggregateByAgentName(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) 27 | AggregateByOperatingSystem(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) 28 | AggregateByCountry(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) 29 | AggregateByNetworkType(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.AggregateData, error) 30 | AggregateBySyncStatus(ctx context.Context, peerFilter *model.PeerFilter) (*models.SyncAggregateData, error) 31 | AggregateByClientVersion(ctx context.Context, peerFilter *model.PeerFilter) ([]*models.ClientVersionAggregation, error) 32 | } 33 | -------------------------------------------------------------------------------- /store/record/mongo/mongo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package mongo implements all the store methods 5 | package mongo 6 | 7 | import ( 8 | "context" 9 | "encoding/hex" 10 | "eth2-crawler/graph/model" 11 | "eth2-crawler/models" 12 | "eth2-crawler/store/record" 13 | "eth2-crawler/utils/config" 14 | "fmt" 15 | "time" 16 | 17 | "go.mongodb.org/mongo-driver/bson" 18 | "go.mongodb.org/mongo-driver/bson/primitive" 19 | 20 | "go.mongodb.org/mongo-driver/mongo" 21 | "go.mongodb.org/mongo-driver/mongo/options" 22 | "go.mongodb.org/mongo-driver/mongo/readpref" 23 | ) 24 | 25 | type mongoStore struct { 26 | client *mongo.Client 27 | coll *mongo.Collection 28 | timeout time.Duration 29 | } 30 | 31 | // New creates new instance of History Store based on MongoDB 32 | func New(cfg *config.Database) (record.Provider, error) { 33 | timeout := time.Duration(cfg.Timeout) * time.Second 34 | opts := options.Client() 35 | 36 | opts.ApplyURI(cfg.URI) 37 | client, err := mongo.NewClient(opts) 38 | if err != nil { 39 | return nil, fmt.Errorf("connecton error [%s]: %w", opts.GetURI(), err) 40 | } 41 | 42 | // connect to the mongoDB cluster 43 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 44 | defer cancel() 45 | 46 | err = client.Connect(ctx) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | // test the connection 52 | err = client.Ping(ctx, readpref.Primary()) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | return &mongoStore{ 58 | client: client, 59 | coll: client.Database(cfg.Database).Collection(cfg.HistoryCollection), 60 | timeout: timeout, 61 | }, nil 62 | } 63 | 64 | func (s mongoStore) Create(ctx context.Context, history *models.History) error { 65 | _, err := s.coll.InsertOne(ctx, history, options.InsertOne()) 66 | return err 67 | } 68 | 69 | func (s mongoStore) GetHistory(ctx context.Context, start int64, end int64, peerFilter *model.PeerFilter) ([]*models.HistoryCount, error) { 70 | var filter primitive.D 71 | if peerFilter != nil && 72 | peerFilter.ForkDigest != nil { 73 | forkDigest := *(peerFilter.ForkDigest) 74 | if forkDigest[0:2] == "0x" { 75 | forkDigest = forkDigest[2:] 76 | } 77 | forkDigestBytes, err := hex.DecodeString(forkDigest) 78 | if err != nil { 79 | return nil, err 80 | } 81 | filter = bson.D{ 82 | { 83 | Key: "$match", Value: bson.D{{Key: "fork_digest", Value: forkDigestBytes}}, 84 | }, 85 | { 86 | Key: "$and", Value: bson.A{ 87 | bson.D{{Key: "time", Value: bson.D{{Key: "$gt", Value: start}}}}, 88 | bson.D{{Key: "time", Value: bson.D{{Key: "$lt", Value: end}}}}, 89 | }, 90 | }, 91 | } 92 | if err != nil { 93 | return nil, err 94 | } 95 | } else { 96 | filter = bson.D{ 97 | { 98 | Key: "$and", Value: bson.A{ 99 | bson.D{{Key: "time", Value: bson.D{{Key: "$gt", Value: start}}}}, 100 | bson.D{{Key: "time", Value: bson.D{{Key: "$lt", Value: end}}}}, 101 | }, 102 | }, 103 | } 104 | } 105 | 106 | cursor, err := s.coll.Find(ctx, filter) 107 | if err != nil { 108 | return nil, err 109 | } 110 | var result []*models.History 111 | for cursor.Next(ctx) { 112 | // create a value into which the single document can be decoded 113 | data := new(models.History) 114 | err := cursor.Decode(data) 115 | if err != nil { 116 | return nil, err 117 | } 118 | result = append(result, data) 119 | } 120 | 121 | count := make([]*models.HistoryCount, 0) 122 | for _, v := range result { 123 | count = append(count, &models.HistoryCount{ 124 | Time: v.Time, 125 | TotalNodes: v.Eth2Nodes, 126 | SyncedNodes: v.SyncNodes, 127 | }) 128 | } 129 | return count, nil 130 | } 131 | -------------------------------------------------------------------------------- /store/record/store.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package record implements db for historic data 5 | package record 6 | 7 | import ( 8 | "context" 9 | 10 | "eth2-crawler/graph/model" 11 | "eth2-crawler/models" 12 | ) 13 | 14 | // Provider represents store provider interface that can be implemented by different DB engines 15 | type Provider interface { 16 | Create(ctx context.Context, history *models.History) error 17 | GetHistory(ctx context.Context, start int64, end int64, peerFilter *model.PeerFilter) ([]*models.HistoryCount, error) 18 | } 19 | -------------------------------------------------------------------------------- /utils/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package config holds the config file parsing functionality 5 | package config 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "os" 11 | 12 | "gopkg.in/yaml.v2" 13 | ) 14 | 15 | // Configuration holds data necessary for configuring application 16 | type Configuration struct { 17 | Server *Server `yaml:"server,omitempty"` 18 | Database *Database `yaml:"database,omitempty"` 19 | Resolver *Resolver `yaml:"resolver,omitempty"` 20 | } 21 | 22 | // Server holds data necessary for server configuration 23 | type Server struct { 24 | Port string `yaml:"port,omitempty"` 25 | ReadTimeout int `yaml:"read_timeout_seconds,omitempty"` 26 | ReadHeaderTimeout int `yaml:"read_header_timeout_seconds,omitempty"` 27 | WriteTimeout int `yaml:"write_timeout_seconds,omitempty"` 28 | CORS []string `yaml:"cors,omitempty"` 29 | } 30 | 31 | // Database is a MongoDB config 32 | type Database struct { 33 | URI string `yaml:"-"` 34 | Timeout int `yaml:"request_timeout_sec"` 35 | Database string `yaml:"database"` 36 | Collection string `yaml:"collection"` 37 | HistoryCollection string `yaml:"history_collection"` 38 | } 39 | 40 | // Resolver provides config for resolver 41 | type Resolver struct { 42 | APIKey string `yaml:"-"` 43 | Timeout int `yaml:"request_timeout_sec"` 44 | } 45 | 46 | func loadDatabaseURI() (string, error) { 47 | mongoURI := os.Getenv("MONGODB_URI") 48 | if mongoURI == "" { 49 | return "", errors.New("MONGODB_URI is required") 50 | } 51 | return mongoURI, nil 52 | } 53 | 54 | func loadResolverAPIKey() (string, error) { 55 | resolverAPIKey := os.Getenv("RESOLVER_API_KEY") 56 | if resolverAPIKey == "" { 57 | return "", errors.New("RESOLVER_API_KEY is required") 58 | } 59 | return resolverAPIKey, nil 60 | } 61 | 62 | // Load returns Configuration struct 63 | func Load(path string) (*Configuration, error) { 64 | bytes, err := os.ReadFile(path) 65 | 66 | if err != nil { 67 | return nil, fmt.Errorf("error reading config file, %w", err) 68 | } 69 | 70 | var cfg = new(Configuration) 71 | 72 | if err = yaml.Unmarshal(bytes, cfg); err != nil { 73 | return nil, fmt.Errorf("unable to decode into struct, %w", err) 74 | } 75 | 76 | // load envs 77 | cfg.Database.URI, err = loadDatabaseURI() 78 | if err != nil { 79 | return nil, err 80 | } 81 | cfg.Resolver.APIKey, err = loadResolverAPIKey() 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | return cfg, nil 87 | } 88 | -------------------------------------------------------------------------------- /utils/server/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 ChainSafe Systems 2 | // SPDX-License-Identifier: LGPL-3.0-only 3 | 4 | // Package server configures the api server 5 | package server 6 | 7 | import ( 8 | "context" 9 | "eth2-crawler/utils/config" 10 | "net/http" 11 | "os" 12 | "os/signal" 13 | "time" 14 | 15 | "github.com/rs/cors" 16 | ) 17 | 18 | // Start starts the service 19 | func Start(ctx context.Context, cfg *config.Server, handler http.Handler) { 20 | cors := cors.New(cors.Options{ 21 | AllowedOrigins: cfg.CORS, 22 | AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD"}, 23 | ExposedHeaders: []string{"Content-Length", "Content-Type", "Content-Disposition"}, 24 | AllowCredentials: true, 25 | }) 26 | 27 | server := &http.Server{ 28 | Addr: ":" + cfg.Port, 29 | ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second, 30 | ReadHeaderTimeout: time.Duration(cfg.ReadHeaderTimeout) * time.Second, 31 | WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second, 32 | Handler: cors.Handler(handler), 33 | } 34 | 35 | go func() { 36 | if err := server.ListenAndServe(); err != nil { 37 | // log error 38 | } 39 | }() 40 | 41 | // Wait for interrupt signal to gracefully shutdown the server with a timeout of 10 seconds. 42 | quit := make(chan os.Signal, 1) 43 | signal.Notify(quit, os.Interrupt) 44 | <-quit 45 | 46 | ctx, ecancel := context.WithTimeout(ctx, 10*time.Second) 47 | defer ecancel() 48 | if err := server.Shutdown(ctx); err != nil { 49 | // log error 50 | } 51 | } 52 | --------------------------------------------------------------------------------