├── .env.example ├── .github └── workflows │ ├── check-formatting.yml │ ├── ci.yml │ ├── codeql-analysis.yml │ ├── indexer-agent-image.yml │ └── indexer-cli-image.yml ├── .gitignore ├── CLAUDE.md ├── Dockerfile.indexer-agent ├── Dockerfile.indexer-cli ├── LICENSE ├── README.md ├── SECURITY.md ├── docs ├── action-queue.md ├── auto-graft.md ├── errors.md ├── feature-support-matrix.md ├── images │ ├── query-fee-flow.png │ ├── synced-transfers.png │ └── vector-infrastructure-changes.png ├── networks.md ├── networks │ ├── arbitrum-one.md │ ├── arbitrum-sepolia.md │ └── config-examples │ │ ├── arbitrum-sepolia-indexer-agent-config.yaml │ │ └── sepolia-indexer-agent-config.yaml ├── subgraph-freshness.md └── testnet-setup.md ├── k8s ├── base │ ├── block-ingestor │ │ ├── service.yaml │ │ └── stateful_set.yaml │ ├── config │ │ └── graph-node.toml │ ├── grafana.yaml │ ├── index-node │ │ ├── service.yaml │ │ └── stateful_set.yaml │ ├── indexer-agent │ │ ├── service.yaml │ │ └── statefulset.yaml │ ├── ingress.yaml │ ├── issuer.yaml │ ├── kustomization.yaml │ ├── nfs.yaml │ ├── prometheus.yaml │ ├── query-node │ │ ├── backend_config.yaml │ │ ├── deployment.yaml │ │ ├── kustomization.yaml │ │ ├── proxy.yaml │ │ └── service.yaml │ └── shell.yaml └── overlays │ ├── block_ingestor.yaml │ ├── index_node.yaml │ ├── indexer_agent.yaml │ ├── indexer_service.yaml │ ├── kustomization.yaml │ ├── prometheus.yaml │ └── query_node.yaml ├── lerna.json ├── network-configs └── config.yaml ├── package.json ├── packages ├── indexer-agent │ ├── .eslintignore │ ├── .eslintrc.js │ ├── .prettierrc.json │ ├── LICENSE │ ├── README.md │ ├── bin │ │ └── graph-indexer-agent │ ├── config.yaml │ ├── jest.config.js │ ├── package.json │ ├── src │ │ ├── __tests__ │ │ │ ├── agent.ts │ │ │ └── indexer.ts │ │ ├── agent.ts │ │ ├── commands │ │ │ ├── common-options.ts │ │ │ ├── start-multi-network.ts │ │ │ └── start.ts │ │ ├── db │ │ │ ├── cli │ │ │ │ ├── migrator.js │ │ │ │ └── umzug.ts │ │ │ └── migrations │ │ │ │ ├── 00-cost-model-variables-jsonb.ts │ │ │ │ ├── 01-reset-state-channels.ts │ │ │ │ ├── 02-remove-state-channels.ts │ │ │ │ ├── 03-poi-disputes-add-subgraph-deployment-id.ts │ │ │ │ ├── 03-rename-receipts-table.ts │ │ │ │ ├── 04-rename-fee-columns.ts │ │ │ │ ├── 05-indexing-rules-add-subgraph-id.ts │ │ │ │ ├── 06-indexing-rules-add-max-lifetime.ts │ │ │ │ ├── 07-indexing-rules-add-require-supported.ts │ │ │ │ ├── 08-indexing-rules-add-auto-renewal.ts │ │ │ │ ├── 09-actions-expand-failure-reason-varchar.ts │ │ │ │ ├── 10-indexing-rules-add-safety.ts │ │ │ │ ├── 11-add-protocol-network-field.ts │ │ │ │ ├── 12-add-scalar-tap-table.ts │ │ │ │ ├── 13-add-scalar-tap-deny-list.ts │ │ │ │ ├── 14-cost-models-history.ts │ │ │ │ ├── 14-use-new-deployment-pause-mechanism.ts │ │ │ │ ├── 15-modify-scalar-tap-tables.ts │ │ │ │ ├── 16-modify-invalid-receipts-table.ts │ │ │ │ ├── 17-edit-cost-models-trigger.ts │ │ │ │ └── 18-actions-expand-action-status-add-deploying.ts │ │ ├── index.ts │ │ ├── syncing-server.ts │ │ └── types.ts │ └── tsconfig.json ├── indexer-cli │ ├── .eslintrc.js │ ├── .prettierrc.json │ ├── LICENSE │ ├── README.md │ ├── bin │ │ └── graph-indexer │ ├── jest.config.js │ ├── package.json │ ├── src │ │ ├── __tests__ │ │ │ ├── cli.test.ts │ │ │ ├── indexer │ │ │ │ ├── actions.test.ts │ │ │ │ ├── cost.test.ts │ │ │ │ └── rules.test.ts │ │ │ ├── references │ │ │ │ ├── basic.agora │ │ │ │ ├── help.stdout │ │ │ │ ├── indexer-actions-get-fields.stdout │ │ │ │ ├── indexer-actions-get-first-fields.stdout │ │ │ │ ├── indexer-actions-get-first.stdout │ │ │ │ ├── indexer-actions-get.stdout │ │ │ │ ├── indexer-actions.stdout │ │ │ │ ├── indexer-connect.stdout │ │ │ │ ├── indexer-cost-all.stdout │ │ │ │ ├── indexer-cost-command-identifier.stdout │ │ │ │ ├── indexer-cost-deployment-model-only.stdout │ │ │ │ ├── indexer-cost-fallback-global.stdout │ │ │ │ ├── indexer-cost-get-no-arg.stdout │ │ │ │ ├── indexer-cost-invalid-identifier-arg.stderr │ │ │ │ ├── indexer-cost-model-deployment-no-arg.stdout │ │ │ │ ├── indexer-cost-model-deployment.stdout │ │ │ │ ├── indexer-cost-set-invalid-arg.stdout │ │ │ │ ├── indexer-cost-set-no-arg.stdout │ │ │ │ ├── indexer-cost-variables-deployment.stdout │ │ │ │ ├── indexer-cost.stdout │ │ │ │ ├── indexer-help.stdout │ │ │ │ ├── indexer-not-connected.stderr │ │ │ │ ├── indexer-output-format.stdout │ │ │ │ ├── indexer-rule-deployment-always.stdout │ │ │ │ ├── indexer-rule-deployment-deleted-offchain-success.stdout │ │ │ │ ├── indexer-rule-deployment-deleted-success.stdout │ │ │ │ ├── indexer-rule-deployment-lifetime.stdout │ │ │ │ ├── indexer-rule-deployment-never.stdout │ │ │ │ ├── indexer-rule-deployment-offchain.stdout │ │ │ │ ├── indexer-rule-deployment-rules.stdout │ │ │ │ ├── indexer-rule-deployment-safety.stdout │ │ │ │ ├── indexer-rule-deployment-supported.stdout │ │ │ │ ├── indexer-rule-deployment-yaml.stdout │ │ │ │ ├── indexer-rule-global-rules.stdout │ │ │ │ ├── indexer-rule-subgraph-offchain.stdout │ │ │ │ ├── indexer-rule-subgraph-options.stdout │ │ │ │ ├── indexer-rule-subgraph-rules.stdout │ │ │ │ ├── indexer-rules-command-no-args.stdout │ │ │ │ ├── indexer-rules-invalid-identifier-arg.stderr │ │ │ │ ├── indexer-rules-invalid-set-arg.stdout │ │ │ │ ├── indexer-rules-no-identifier.stderr │ │ │ │ ├── indexer-rules-no-network.stderr │ │ │ │ ├── indexer-rules.stdout │ │ │ │ ├── indexer-status-not-connected.stderr │ │ │ │ └── indexer.stdout │ │ │ └── util.ts │ │ ├── actions.ts │ │ ├── allocations.ts │ │ ├── cli.ts │ │ ├── client.ts │ │ ├── command-helpers.ts │ │ ├── commands │ │ │ ├── indexer.ts │ │ │ └── indexer │ │ │ │ ├── actions.ts │ │ │ │ ├── actions │ │ │ │ ├── approve.ts │ │ │ │ ├── cancel.ts │ │ │ │ ├── delete.ts │ │ │ │ ├── execute.ts │ │ │ │ ├── get.ts │ │ │ │ ├── queue.ts │ │ │ │ └── update.ts │ │ │ │ ├── allocations.ts │ │ │ │ ├── allocations │ │ │ │ ├── close.ts │ │ │ │ ├── collect.ts │ │ │ │ ├── create.ts │ │ │ │ ├── get.ts │ │ │ │ └── reallocate.ts │ │ │ │ ├── connect.ts │ │ │ │ ├── cost.ts │ │ │ │ ├── cost │ │ │ │ ├── delete.ts │ │ │ │ ├── get.ts │ │ │ │ └── set │ │ │ │ │ └── model.ts │ │ │ │ ├── disputes.ts │ │ │ │ ├── disputes │ │ │ │ └── get.ts │ │ │ │ ├── rules.ts │ │ │ │ ├── rules │ │ │ │ ├── clear.ts │ │ │ │ ├── delete.ts │ │ │ │ ├── get.ts │ │ │ │ ├── maybe.ts │ │ │ │ ├── offchain.ts │ │ │ │ ├── set.ts │ │ │ │ ├── start.ts │ │ │ │ └── stop.ts │ │ │ │ └── status.ts │ │ ├── config.ts │ │ ├── cost.ts │ │ ├── disputes.ts │ │ └── rules.ts │ └── tsconfig.json └── indexer-common │ ├── .eslintignore │ ├── .eslintrc.js │ ├── .prettierrc.json │ ├── LICENSE │ ├── README.md │ ├── jest.config.js │ ├── package.json │ ├── src │ ├── __tests__ │ │ ├── ipfs.test.ts │ │ ├── network-specification-files │ │ │ ├── invalid-base58.yml │ │ │ ├── invalid-epoch-subgraph.yml │ │ │ ├── invalid-extra-field.yml │ │ │ ├── invalid-missing-field.yml │ │ │ ├── invalid-negative-max-block-distance.yml │ │ │ ├── invalid-network-identifier.yml │ │ │ ├── valid-missing.yml │ │ │ └── valid.yml │ │ ├── network-specification.test.ts │ │ └── subgraph.test.ts │ ├── actions.ts │ ├── allocations │ │ ├── __tests__ │ │ │ ├── escrow-accounts.test.ts │ │ │ ├── tap-pagination.test.ts │ │ │ ├── tap.test.ts │ │ │ └── validate-queries.test.ts │ │ ├── escrow-accounts.ts │ │ ├── index.ts │ │ ├── keys.ts │ │ ├── monitor.ts │ │ ├── query-fees.test.ts │ │ ├── query-fees.ts │ │ ├── tap-collector.ts │ │ └── types.ts │ ├── async-cache.ts │ ├── errors.ts │ ├── graph-node.ts │ ├── index.ts │ ├── indexer-management │ │ ├── __tests__ │ │ │ ├── allocations.test.ts │ │ │ ├── helpers.test.ts │ │ │ ├── resolvers │ │ │ │ ├── actions.test.ts │ │ │ │ ├── cost-models.test.ts │ │ │ │ ├── indexing-rules.test.ts │ │ │ │ └── poi-disputes.test.ts │ │ │ └── util.ts │ │ ├── actions.ts │ │ ├── allocations.ts │ │ ├── client.ts │ │ ├── index.ts │ │ ├── models │ │ │ ├── action.ts │ │ │ ├── cost-model.ts │ │ │ ├── index.ts │ │ │ ├── indexing-rule.ts │ │ │ └── poi-dispute.ts │ │ ├── monitor.ts │ │ ├── resolvers │ │ │ ├── actions.ts │ │ │ ├── allocations.ts │ │ │ ├── cost-models.ts │ │ │ ├── indexer-status.ts │ │ │ ├── indexing-rules.ts │ │ │ ├── poi-disputes.ts │ │ │ └── utils.ts │ │ ├── rules.ts │ │ ├── server.ts │ │ └── types.ts │ ├── multi-networks.ts │ ├── network-specification.ts │ ├── network.ts │ ├── operator.ts │ ├── parsers │ │ ├── __tests__ │ │ │ └── parsers.ts │ │ ├── basic-types.ts │ │ ├── error-handling.ts │ │ ├── index.ts │ │ ├── test-utils.ts │ │ └── validators.ts │ ├── query-fees │ │ ├── __test__ │ │ │ └── rav.test.ts │ │ ├── allocation-utils.ts │ │ ├── index.ts │ │ └── models.ts │ ├── rules.ts │ ├── sequential-timer.ts │ ├── subgraph-client.ts │ ├── subgraphs.ts │ ├── transactions.ts │ ├── types.ts │ └── utils.ts │ └── tsconfig.json ├── scripts ├── run-tests.sh └── update-dependency.sh ├── terraform ├── .gitignore ├── README.md ├── database.tf ├── main.tf ├── prometheus.tf ├── secrets.tf └── variables.tf ├── tsconfig.json └── yarn.lock /.env.example: -------------------------------------------------------------------------------- 1 | # Arbitrum Sepolia testnet JSON-RPC provider URL for testing (e.g., Infura, Alchemy, etc.) 2 | INDEXER_TEST_JRPC_PROVIDER_URL=https://arbitrum-sepolia.infura.io/v3/your-project-id 3 | 4 | # The Graph API key for querying subgraphs (from The Graph Studio) 5 | INDEXER_TEST_API_KEY=your-graph-api-key-here 6 | -------------------------------------------------------------------------------- /.github/workflows/check-formatting.yml: -------------------------------------------------------------------------------- 1 | name: "Check Formatting" 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: {} 7 | 8 | jobs: 9 | check: 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | - name: Set up Node.js v20 15 | uses: actions/setup-node@v2.1.5 16 | with: 17 | node-version: 20 18 | - name: Build and Format 19 | run: yarn 20 | - name: Check Formatting 21 | run: git diff --exit-code 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: CI 5 | 6 | on: 7 | push: 8 | branches: [main] 9 | pull_request: {} 10 | 11 | jobs: 12 | build: 13 | strategy: 14 | matrix: 15 | node-version: [20, 22] 16 | system: 17 | - os: ubuntu-22.04 18 | runs-on: ${{ matrix.system.os }} 19 | services: 20 | postgres: 21 | image: postgres:13 22 | env: 23 | POSTGRES_DB: indexer_tests 24 | POSTGRES_USER: testuser 25 | POSTGRES_PASSWORD: testpass 26 | ports: 27 | - 5432:5432 28 | # Set health checks to wait until postgres has started 29 | options: >- 30 | --health-cmd pg_isready 31 | --health-interval 10s 32 | --health-timeout 5s 33 | --health-retries 5 34 | steps: 35 | - uses: actions/checkout@v2 36 | - name: update OS 37 | run: | 38 | sudo apt-get update 39 | sudo apt install -y --no-install-recommends gcc g++ make build-essential 40 | - name: Use Node.js ${{ matrix.node-version }} 41 | uses: actions/setup-node@v1 42 | with: 43 | node-version: ${{ matrix.node-version }} 44 | registry-url: https://registry.npmjs.org/ 45 | - run: yarn install --frozen-lockfile 46 | env: 47 | NODE_AUTH_TOKEN: ${{secrets.npm_token}} 48 | - run: yarn test:ci 49 | env: 50 | POSTGRES_TEST_HOST: localhost 51 | POSTGRES_TEST_DATABASE: indexer_tests 52 | POSTGRES_TEST_USERNAME: testuser 53 | POSTGRES_TEST_PASSWORD: testpass 54 | NODE_OPTIONS: "--dns-result-order=ipv4first" 55 | INDEXER_TEST_JRPC_PROVIDER_URL: ${{ secrets.TESTS_RPC_PROVIDER }} 56 | INDEXER_TEST_API_KEY: ${{ secrets.TESTS_API_KEY }} 57 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '38 7 * * 4' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-22.04 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'javascript' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.github/workflows/indexer-agent-image.yml: -------------------------------------------------------------------------------- 1 | name: Indexer Agent Image 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - v*.*.* 9 | pull_request: 10 | branches: [main] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-22.04 15 | permissions: 16 | packages: write 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v2 21 | - name: Docker meta 22 | id: docker_meta 23 | uses: crazy-max/ghaction-docker-meta@v1 24 | with: 25 | images: ghcr.io/graphprotocol/indexer-agent 26 | tag-sha: true 27 | - name: Set up QEMU 28 | uses: docker/setup-qemu-action@v1 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v1 31 | - name: Login to GitHub Container Registry 32 | uses: docker/login-action@v2 33 | with: 34 | registry: ghcr.io 35 | username: ${{github.repository_owner}} 36 | password: ${{secrets.GITHUB_TOKEN}} 37 | - name: Setup python 38 | uses: actions/setup-python@v4 39 | with: 40 | python-version: '3.11' 41 | - name: Set up Node.js v20 42 | uses: actions/setup-node@v2.1.5 43 | with: 44 | node-version: 20 45 | - name: Build and push Indexer Agent image 46 | id: docker_build 47 | uses: docker/build-push-action@v2 48 | with: 49 | context: . 50 | file: Dockerfile.indexer-agent 51 | # Enabling the line below restricts Docker images to only be built for branches 52 | # push: ${{github.event_name != 'pull_request'}} 53 | push: true 54 | tags: ${{steps.docker_meta.outputs.tags}} 55 | labels: ${{steps.docker_meta.outputs.labels}} 56 | build-args: NPM_TOKEN=${{secrets.graphprotocol_npm_token}} 57 | -------------------------------------------------------------------------------- /.github/workflows/indexer-cli-image.yml: -------------------------------------------------------------------------------- 1 | name: Indexer CLI Image 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - v*.*.* 9 | pull_request: 10 | branches: [main] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-22.04 15 | permissions: 16 | packages: write 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v2 21 | - name: Docker meta 22 | id: docker_meta 23 | uses: crazy-max/ghaction-docker-meta@v1 24 | with: 25 | images: ghcr.io/graphprotocol/indexer-cli 26 | tag-sha: true 27 | - name: Set up QEMU 28 | uses: docker/setup-qemu-action@v1 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v1 31 | - name: Login to GitHub Container Registry 32 | uses: docker/login-action@v2 33 | with: 34 | registry: ghcr.io 35 | username: ${{github.repository_owner}} 36 | password: ${{secrets.GITHUB_TOKEN}} 37 | - name: Setup python 38 | uses: actions/setup-python@v4 39 | with: 40 | python-version: '3.11' 41 | - name: Set up Node.js v20 42 | uses: actions/setup-node@v2.1.5 43 | with: 44 | node-version: 20 45 | - name: Build and push Indexer CLI image 46 | id: docker_build 47 | uses: docker/build-push-action@v2 48 | with: 49 | context: . 50 | file: Dockerfile.indexer-cli 51 | # Enabling the line below restricts Docker images to only be built for branches 52 | # push: ${{github.event_name != 'pull_request'}} 53 | push: true 54 | tags: ${{steps.docker_meta.outputs.tags}} 55 | labels: ${{steps.docker_meta.outputs.labels}} 56 | build-args: NPM_TOKEN=${{secrets.graphprotocol_npm_token}} 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | .npmrc 106 | .yarnrc 107 | .envrc 108 | 109 | # Terraform state files 110 | *.tfstate* 111 | 112 | # Etherlime (added by server wallet) 113 | .etherlime-store 114 | 115 | # Yalc 116 | yalc.lock 117 | .yalc/ 118 | 119 | # IDE 120 | .idea/ 121 | .envrc 122 | .vscode 123 | -------------------------------------------------------------------------------- /Dockerfile.indexer-agent: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # Build image 3 | 4 | FROM node:20.11-bookworm-slim as build 5 | 6 | ENV NODE_ENV production 7 | 8 | RUN apt-get update && apt-get install -y python3 build-essential git curl 9 | 10 | WORKDIR /opt/indexer 11 | 12 | # Copy root files 13 | COPY package.json . 14 | COPY yarn.lock . 15 | COPY tsconfig.json . 16 | COPY lerna.json . 17 | 18 | # Copy shared and package files 19 | COPY packages/indexer-common/ ./packages/indexer-common 20 | COPY packages/indexer-agent/ ./packages/indexer-agent 21 | 22 | # Install dependencies; include dev dependencies 23 | RUN yarn --frozen-lockfile --non-interactive --production=false 24 | 25 | ######################################################################## 26 | # Runtime image 27 | 28 | FROM node:20.11-bookworm-slim 29 | 30 | ENV NODE_ENV production 31 | # When simulating large transactions, sometimes indexer-agent runs out of memory. 32 | # This flag seems force node into GC earlier, preventing the crash 33 | # 1536mb is 1.5GB, which is appropriate for an environment with 2GB RAM 34 | # todo: increase this temporarily to 4GB to see if it fixes the crash 35 | ENV NODE_OPTIONS="--max-old-space-size=4096" 36 | 37 | 38 | RUN apt-get update && apt-get install -y python3 build-essential git curl 39 | 40 | WORKDIR /opt/indexer 41 | 42 | # Copy root files 43 | COPY package.json . 44 | COPY yarn.lock . 45 | COPY tsconfig.json . 46 | COPY lerna.json . 47 | 48 | 49 | # Copy build output 50 | COPY --from=build /opt/indexer/packages/indexer-common/package.json /opt/indexer/packages/indexer-common/package.json 51 | COPY --from=build /opt/indexer/packages/indexer-common/dist /opt/indexer/packages/indexer-common/dist 52 | COPY --from=build /opt/indexer/packages/indexer-agent/package.json /opt/indexer/packages/indexer-agent/package.json 53 | COPY --from=build /opt/indexer/packages/indexer-agent/dist /opt/indexer/packages/indexer-agent/dist 54 | 55 | # Install dependencies; exclude dev dependencies 56 | RUN yarn --frozen-lockfile --non-interactive 57 | 58 | ENV ETHEREUM "" 59 | 60 | # Run the indexer-agent 61 | WORKDIR /opt/indexer/packages/indexer-agent 62 | ENTRYPOINT ["node", "dist/index.js", "start"] 63 | -------------------------------------------------------------------------------- /Dockerfile.indexer-cli: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # Build image 3 | 4 | FROM node:20.11-bookworm-slim as build 5 | 6 | ENV NODE_ENV production 7 | 8 | RUN apt-get update && apt-get install -y python3 build-essential git curl 9 | 10 | WORKDIR /opt/indexer 11 | 12 | # Copy root files 13 | COPY package.json . 14 | COPY yarn.lock . 15 | COPY tsconfig.json . 16 | COPY lerna.json . 17 | 18 | # Copy shared and package files 19 | COPY packages/indexer-common/ ./packages/indexer-common 20 | COPY packages/indexer-cli/ ./packages/indexer-cli 21 | 22 | # Install dependencies; include dev dependencies 23 | RUN yarn --frozen-lockfile --non-interactive --production=false 24 | 25 | ######################################################################## 26 | # Runtime image 27 | 28 | FROM node:20.11-bookworm-slim 29 | 30 | ENV NODE_ENV production 31 | 32 | RUN apt-get update && apt-get install -y python3 build-essential git curl 33 | 34 | # Install Rust 35 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh && sh /tmp/rustup.sh -y 36 | ENV PATH="/root/.cargo/bin:$PATH" 37 | 38 | WORKDIR /opt/indexer 39 | 40 | # Copy root files 41 | COPY package.json . 42 | COPY yarn.lock . 43 | COPY tsconfig.json . 44 | COPY lerna.json . 45 | 46 | # Copy build output 47 | COPY --from=build /opt/indexer/packages/indexer-common/package.json /opt/indexer/packages/indexer-common/package.json 48 | COPY --from=build /opt/indexer/packages/indexer-common/dist /opt/indexer/packages/indexer-common/dist 49 | COPY --from=build /opt/indexer/packages/indexer-cli/package.json /opt/indexer/packages/indexer-cli/package.json 50 | COPY --from=build /opt/indexer/packages/indexer-cli/dist /opt/indexer/packages/indexer-cli/dist 51 | COPY --from=build /opt/indexer/packages/indexer-cli/bin /opt/indexer/packages/indexer-cli/bin 52 | 53 | 54 | # Install dependencies; exclude dev dependencies 55 | RUN yarn --frozen-lockfile --non-interactive --production=true 56 | RUN ln -s /opt/indexer/packages/indexer-cli/bin/graph-indexer /usr/bin/graph 57 | 58 | 59 | ENV ETHEREUM "" 60 | 61 | ENV USER=graph 62 | ENV UID=10001 63 | 64 | RUN adduser --disabled-password --gecos "" --home "/home/graph" --shell "/bin/bash" --uid "${UID}" "${USER}" 65 | 66 | WORKDIR /home/${USER} 67 | USER ${USER} 68 | ENTRYPOINT ["cat"] 69 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020-2021 The Graph Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Please report vulnerabilities via https://thegraph.com/security/. 6 | -------------------------------------------------------------------------------- /docs/images/query-fee-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphprotocol/indexer/6d7a8cc1c4ceb70901ad5e4aa32fa4c9fbcf304f/docs/images/query-fee-flow.png -------------------------------------------------------------------------------- /docs/images/synced-transfers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphprotocol/indexer/6d7a8cc1c4ceb70901ad5e4aa32fa4c9fbcf304f/docs/images/synced-transfers.png -------------------------------------------------------------------------------- /docs/images/vector-infrastructure-changes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphprotocol/indexer/6d7a8cc1c4ceb70901ad5e4aa32fa4c9fbcf304f/docs/images/vector-infrastructure-changes.png -------------------------------------------------------------------------------- /docs/networks.md: -------------------------------------------------------------------------------- 1 | # Mainnet and Testnet Configuration 2 | 3 | [Arbitrum One configuration](./networks/arbitrum-one.md) 4 | 5 | [Arbitrum Sepolia configuration](./networks/arbitrum-sepolia.md) 6 | 7 | [Testnet setup instructions](./testnet-setup.md) 8 | 9 | [Feature Support Matrix (Ethereum & Arbitrum One)](./feature-support-matrix.md) 10 | -------------------------------------------------------------------------------- /docs/networks/config-examples/arbitrum-sepolia-indexer-agent-config.yaml: -------------------------------------------------------------------------------- 1 | networkIdentifier: 'arbitrum-sepolia' 2 | indexerOptions: 3 | address: '?' 4 | mnemonic: '?' 5 | url: '?' 6 | geoCoordinates: [118.2923, 36.5785] 7 | defaultAllocationAmount: 1000 8 | allocationManagementMode: 'oversight' 9 | restakeRewards: true 10 | poiDisputeMonitoring: true 11 | poiDisputableEpochs: 3 12 | voucherRedemptionThreshold: 100 13 | voucherRedemptionBatchThreshold: 250 14 | rebateClaimThreshold: 100 15 | rebateClaimBatchThreshold: 250 16 | gateway: 17 | url: 'https://gateway-testnet-arbitrum.network.thegraph.com' 18 | subgraphs: 19 | maxBlockDistance: 60 20 | freshnessSleepMilliseconds: 10000 21 | networkSubgraph: 22 | url: 'https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/3xQHhMudr1oh69ut36G2mbzpYmYxwqCeU6wwqyCDCnqV' 23 | deployment: 'QmT8UDGK7zKd2u2NQZwhLYHdA4KM55QsivkE3ouCuX6fEj' 24 | epochSubgraph: 25 | url: 'https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/BhnsdeZihU4SuokxZMLF4FQBVJ3jgtZf6v51gHvz3bSS' 26 | deployment: 'QmTpu2mVquoMpr4SWSM77nGkU3tcUS1Bhk1sVHpjDrAUAx' 27 | networkProvider: 28 | url: '?' 29 | transactionMonitoring: 30 | baseFeePerGasMax: 100 31 | gasIncreaseTimeout: 90 32 | gasIncreaseFactor: 1.5 33 | maxTransactionAttempts: 20 34 | -------------------------------------------------------------------------------- /docs/networks/config-examples/sepolia-indexer-agent-config.yaml: -------------------------------------------------------------------------------- 1 | networkIdentifier: 'sepolia' 2 | indexerOptions: 3 | address: '?' 4 | mnemonic: '?' 5 | url: '?' 6 | geoCoordinates: [118.2923, 36.5785] 7 | defaultAllocationAmount: 5000 8 | allocationManagementMode: 'oversight' 9 | restakeRewards: true 10 | poiDisputeMonitoring: true 11 | poiDisputableEpochs: 3 12 | voucherRedemptionThreshold: 100 13 | voucherRedemptionBatchThreshold: 250 14 | rebateClaimThreshold: 100 15 | rebateClaimBatchThreshold: 250 16 | gateway: 17 | url: 'https://gateway.testnet.thegraph.com' 18 | subgraphs: 19 | networkSubgraph: 20 | url: 'https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/8pVKDwHniAz87CHEQsiz2wgFXGZXrbMDkrxgauVVfMJC' 21 | deployment: 'QmdwyUJ2GNAkSxLc3dRgFULf5fN1mZWYeQLjXsdowQcfTg' 22 | epochSubgraph: 23 | url: 'https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/3nEnuQEQd1aP6wksKvRUnuwLQcQy1zD3HPFaHZ8cMVqM' 24 | deployment: 'QmUXAKvt84bYpkctHgXqWBjq1YY7DaTCKfmxEh8w9Xxb5X' 25 | networkProvider: 26 | url: '?' 27 | transactionMonitoring: 28 | baseFeePerGasMax: 100 29 | gasIncreaseTimeout: 90 30 | gasIncreaseFactor: 1.5 31 | maxTransactionAttempts: 20 32 | -------------------------------------------------------------------------------- /docs/subgraph-freshness.md: -------------------------------------------------------------------------------- 1 | # Subgraph Freshness Checker Feature 2 | 3 | ## Overview 4 | 5 | The `SubgraphFreshnessChecker` class is introduced to enhance the reliability and timeliness of subgraph queries in environments where the subgraph might lag significantly behind the most recent block on the blockchain. It primarily operates by validating subgraph freshness and issuing warnings if the subgraph is not sufficiently synchronized with the latest blockchain state. 6 | 7 | ## Key Concepts 8 | 9 | - **Subgraph Freshness:** A metric to determine how synchronized a subgraph is with the main blockchain. It's gauged by comparing the latest indexed block in the subgraph with the most recent block on the network. 10 | 11 | ## Feature Details 12 | 13 | ### 1. Continuous Retry Mechanism 14 | The `SubgraphFreshnessChecker` perpetually retries subgraph queries under circumstances where the subgraph is notably behind the most recent block on the blockchain. A warning, including the current block distance from the chain head, is issued if this condition is detected. 15 | 16 | ### 2. Configuration 17 | Configuration options have been expanded to allow control over the subgraph freshness checking mechanism via network specification files and Command Line Interface (CLI) parameters: 18 | 19 | - **maxBlockDistance:** An integer defining the acceptable distance (in blocks) between the latest indexed block in the subgraph and the most recent block on the network. If the distance exceeds this value, the subgraph is considered "stale," prompting a retry mechanism and possibly a warning. 20 | 21 | - **freshnessSleepMilliseconds:** An integer dictating the waiting duration (in milliseconds) before a query is retried when the subgraph is deemed stale. 22 | 23 | ### Example Configuration 24 | 25 | Here is a snippet of an Arbitrum network specification file with the suggested options for Arbitrum One and Arbitrum Sepolia: 26 | 27 | ```yaml 28 | subgraphs: 29 | maxBlockDistance: 5000 30 | freshnessSleepMilliseconds: 10000 31 | ``` 32 | 33 | ## Practical Implications 34 | 35 | The following default values have been established based on **Arbitrum-One** observations: 36 | 37 | - **maxBlockDistance:** 1000 blocks 38 | - **freshnessSleepMilliseconds:** 10000 (10 seconds) 39 | 40 | 41 | ### Potential Risk Warning 42 | 43 | Suppose the Agent or Service utilizes the default (Ethereum) settings on Arbitrum networks. In that case, a warning will inform users about the risk that queries may forever be considered non-fresh. 44 | 45 | Adjust the `maxBlockDistance` and `freshnessSleepMilliseconds` according to each network condition. 46 | 47 | ## Disabling this feature 48 | 49 | This feature can be virtually turned off by setting a very high value for the **maxBlockDistance** option, which will effectively cause the freshness check always to pass. 50 | -------------------------------------------------------------------------------- /docs/testnet-setup.md: -------------------------------------------------------------------------------- 1 | # Testnet indexer setup 2 | 3 | The Graph Network's testnet is on Arbitrum Sepolia, and network information can be found at https://testnet.thegraph.com/explorer/network. 4 | 5 | > See [networks/testnet.md](/networks/testnet.md) for the latest releases & configuration information. 6 | 7 | ### Registration / Funding (GRT) 8 | 9 | In order to participate in the testnet, you'll need Sepolia ETH and GRT. 10 | To be eligible for testnet GRT, you'll need to 11 | 12 | 1. join [The Graph Discord](https://thegraph.com/discord/), 13 | 2. get the `@testnetindexer` role in the `#roles` channel, 14 | 3. use the `#arbi-sepolia-faucet` channel to obtain testnet GRT. 15 | 16 | ### Approving And Staking 17 | 18 | The Graph Network testnet is open for everyone to participate in as an 19 | indexer. The only requirement is a minimum stake of 100k testnet GRT. 20 | 21 | #### Via Graph Explorer 22 | 23 | The Graph Explorer provides an easy way to approve and stake your GRT as an indexer via a web GUI. 24 | 25 | 1. Navigate to [the testnet explorer](https://testnet.thegraph.com/) 26 | 2. Login with Metamask and select the `Arbitrum Sepolia` network 27 | 3. Navigate to your profile (click your address/avatar at top right) 28 | 4. Select the `Indexing` tab and hit the `Stake` button 29 | 5. Follow the directions on the staking screen to stake the desired amount 30 | 31 | ### Via the Contracts CLI 32 | 33 | To approve your testnet GRT to be spent through the staking contract, first approve 34 | it in the GRT contract: 35 | 36 | ```bash 37 | git clone https://github.com/graphprotocol/contracts 38 | cd contracts 39 | 40 | # If you haven't done this before: 41 | npm install 42 | npm run compile 43 | 44 | ./cli/cli.ts -m -p \ 45 | contracts graphToken approve --account 0x35e3Cb6B317690d662160d5d02A5b364578F62c9 --amount 46 | ``` 47 | 48 | Afterwards, stake this amount: 49 | 50 | ```bash 51 | ./cli/cli.ts -m -p \ 52 | contracts staking stake --amount 53 | ``` 54 | 55 | ### Setting An Operator 56 | 57 | #### Via Graph Explorer 58 | 59 | 1. Navigate to [the testnet explorer](https://testnet.thegraph.com/) 60 | 2. Login with Metamask and select the `Arbitrum Sepolia` network 61 | 3. Navigate to your settings page (click profile dropdown at top right and select ⚙️ `Settings`) 62 | 4. Navigate to the `Operators` settings (click `Operators` button) 63 | 5. Click `+` to add your operator wallet address 64 | 6. Follow instructions to submit transaction 65 | 66 | ### Via the Contracts CLI 67 | 68 | ```bash 69 | ./cli/cli.ts -m -p \ 70 | contracts staking setOperator --operator --allowed true 71 | ``` 72 | 73 | You are now ready to set up your testnet indexer, visit [networks/arbitrum-sepolia.md](/networks/arbitrum-sepolia.md) for configuration information. 74 | -------------------------------------------------------------------------------- /k8s/base/block-ingestor/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: index-node-block-ingestor 5 | labels: 6 | app: block-ingestor 7 | network: all 8 | spec: 9 | selector: 10 | app: block-ingestor 11 | network: all 12 | ports: 13 | - name: json-rpc 14 | protocol: TCP 15 | port: 80 16 | targetPort: 8020 17 | -------------------------------------------------------------------------------- /k8s/base/config/graph-node.toml: -------------------------------------------------------------------------------- 1 | [general] 2 | query = "query_node_.*" 3 | 4 | [store] 5 | [store.primary] 6 | connection = "postgresql://$postgres_user:$postgres_pass@$postgres_host/$postgres_db" 7 | pool_size = [ 8 | { node = "query_node_.*", size = 60 }, 9 | { node = "index_node_.*", size = 35 }, 10 | { node = "index_node_block_ingestor_.*", size = 4 }, 11 | ] 12 | 13 | [deployment] 14 | [[deployment.rule]] 15 | store = "primary" 16 | indexers = [ "index_node_0", "index_node_1" ] 17 | 18 | [chains] 19 | ingestor = "index_node_block_ingestor_0" 20 | 21 | [chains.mainnet] 22 | shard = "primary" 23 | provider = [ 24 | { label = "mainnet-0", url = "https://eth-mainnet.node/", features = ["archive", "traces"] } 25 | ] 26 | -------------------------------------------------------------------------------- /k8s/base/index-node/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: index-node 5 | spec: 6 | selector: 7 | app: index-node 8 | ports: 9 | - name: json-rpc 10 | protocol: TCP 11 | port: 80 12 | targetPort: 8020 13 | -------------------------------------------------------------------------------- /k8s/base/index-node/stateful_set.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: index-node 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: index-node 9 | serviceName: index-node 10 | replicas: 1 11 | podManagementPolicy: Parallel 12 | updateStrategy: 13 | type: RollingUpdate 14 | template: 15 | metadata: 16 | labels: 17 | app: index-node 18 | annotations: 19 | prometheus.io/scrape: "true" 20 | prometheus.io/path: "/metrics" 21 | prometheus.io/port: "8040" 22 | spec: 23 | nodeSelector: 24 | index: "1" 25 | tolerations: 26 | - key: "index" 27 | operator: "Exists" 28 | effect: "NoSchedule" 29 | volumes: 30 | - name: nfs-shared 31 | persistentVolumeClaim: 32 | claimName: nfs-shared 33 | readOnly: false 34 | - name: graph-node-config 35 | configMap: 36 | name: graph-node-config 37 | containers: 38 | - name: graph-node 39 | image: graph-node-image 40 | resources: 41 | requests: 42 | cpu: 600m 43 | ports: 44 | - name: http 45 | containerPort: 8000 46 | - name: json-rpc 47 | containerPort: 8020 48 | - name: metrics 49 | containerPort: 8040 50 | volumeMounts: 51 | - name: nfs-shared 52 | mountPath: /var/lib/graph 53 | - name: graph-node-config 54 | mountPath: /etc/graph-node 55 | env: 56 | - name: node_id 57 | valueFrom: 58 | fieldRef: 59 | fieldPath: metadata.name 60 | - name: GRAPH_NODE_CONFIG 61 | value: /etc/graph-node/graph-node.toml 62 | - name: GRAPH_KILL_IF_UNRESPONSIVE 63 | value: "true" 64 | - name: postgres_host 65 | valueFrom: 66 | secretKeyRef: 67 | name: postgres-credentials 68 | key: host 69 | - name: postgres_user 70 | valueFrom: 71 | secretKeyRef: 72 | name: postgres-credentials 73 | key: user 74 | - name: postgres_pass 75 | valueFrom: 76 | secretKeyRef: 77 | name: postgres-credentials 78 | key: password 79 | - name: postgres_db 80 | valueFrom: 81 | secretKeyRef: 82 | name: postgres-credentials 83 | key: graph_db 84 | - name: ipfs 85 | value: https://ipfs.network.thegraph.com 86 | 87 | # Avoid some subgraphs fail with "Gas limit exceeded" error 88 | - name: GRAPH_MAX_GAS_PER_HANDLER 89 | value: "1_000_000_000_000_000" 90 | -------------------------------------------------------------------------------- /k8s/base/indexer-agent/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: indexer-agent 5 | spec: 6 | type: ClusterIP 7 | selector: 8 | app: indexer-agent 9 | ports: 10 | - name: syncing 11 | protocol: TCP 12 | port: 8002 13 | targetPort: 8000 14 | -------------------------------------------------------------------------------- /k8s/base/ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: indexer 5 | annotations: 6 | ingress.kubernetes.io/target-proxy: replace-me 7 | spec: 8 | rules: 9 | - http: 10 | paths: 11 | - path: /* 12 | backend: 13 | serviceName: indexer-service 14 | servicePort: 80 15 | - path: /prometheus/* 16 | backend: 17 | serviceName: prometheus 18 | servicePort: 80 19 | -------------------------------------------------------------------------------- /k8s/base/issuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Issuer 3 | metadata: 4 | name: self-signed-issuer 5 | namespace: default 6 | spec: 7 | selfSigned: {} 8 | -------------------------------------------------------------------------------- /k8s/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - https://github.com/jetstack/cert-manager/releases/download/v1.9.1/cert-manager.yaml 5 | - index-node/stateful_set.yaml 6 | - index-node/service.yaml 7 | - block-ingestor/stateful_set.yaml 8 | - block-ingestor/service.yaml 9 | - query-node/deployment.yaml 10 | - query-node/proxy.yaml 11 | - query-node/service.yaml 12 | - indexer-agent/service.yaml 13 | - indexer-agent/statefulset.yaml 14 | - shell.yaml 15 | - nfs.yaml 16 | - prometheus.yaml 17 | - grafana.yaml 18 | - ingress.yaml 19 | configMapGenerator: 20 | - name: graph-node-config 21 | files: 22 | - config/graph-node.toml 23 | namespace: default 24 | -------------------------------------------------------------------------------- /k8s/base/nfs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nfs-server 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | role: nfs-server 10 | template: 11 | metadata: 12 | labels: 13 | role: nfs-server 14 | spec: 15 | volumes: 16 | - name: nfs 17 | persistentVolumeClaim: 18 | claimName: nfs 19 | readOnly: false 20 | containers: 21 | - image: gists/nfs-server 22 | name: nfs-server 23 | imagePullPolicy: IfNotPresent 24 | securityContext: 25 | privileged: true 26 | volumeMounts: 27 | - name: nfs 28 | mountPath: /nfs-share 29 | env: 30 | - name: NFS_OPTION 31 | value: fsid=0,rw,sync,insecure,no_root_squash,no_subtree_check,nohide 32 | 33 | --- 34 | 35 | kind: Service 36 | apiVersion: v1 37 | metadata: 38 | name: nfs-service 39 | spec: 40 | selector: 41 | role: nfs-server 42 | ports: 43 | - name: tcp-2049 44 | port: 2049 45 | protocol: TCP 46 | 47 | --- 48 | 49 | apiVersion: v1 50 | kind: PersistentVolume 51 | metadata: 52 | name: nfs-shared 53 | spec: 54 | capacity: 55 | storage: 256Gi 56 | storageClassName: "standard" 57 | accessModes: 58 | - ReadWriteMany 59 | nfs: 60 | path: / 61 | server: nfs-service.default.svc.cluster.local 62 | 63 | --- 64 | 65 | apiVersion: v1 66 | kind: PersistentVolumeClaim 67 | metadata: 68 | name: nfs-shared 69 | spec: 70 | accessModes: 71 | - ReadWriteMany 72 | storageClassName: "standard" 73 | volumeName: nfs-shared 74 | resources: 75 | requests: 76 | storage: 256Gi 77 | -------------------------------------------------------------------------------- /k8s/base/query-node/backend_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cloud.google.com/v1beta1 2 | kind: BackendConfig 3 | metadata: 4 | name: query-node-backend-config 5 | spec: 6 | # Use a long timeout for WebSocket connections 7 | timeoutSec: 1800 8 | connectionDraining: 9 | drainingTimeoutSec: 0 10 | -------------------------------------------------------------------------------- /k8s/base/query-node/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - deployment.yaml 5 | - service.yaml 6 | - config_map.yaml 7 | - backend_config.yaml 8 | -------------------------------------------------------------------------------- /k8s/base/query-node/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: query-node 5 | spec: 6 | type: NodePort 7 | selector: 8 | app: query-node 9 | ports: 10 | - name: http-nginx 11 | protocol: TCP 12 | port: 80 13 | - name: http 14 | port: 8000 15 | - name: ws 16 | port: 8001 17 | - name: index-node 18 | port: 8030 19 | -------------------------------------------------------------------------------- /k8s/base/shell.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: shell 5 | namespace: default 6 | spec: 7 | volumes: 8 | - name: nfs-shared 9 | persistentVolumeClaim: 10 | claimName: nfs-shared 11 | readOnly: false 12 | containers: 13 | - image: graph-node-debug 14 | name: shell 15 | command: 16 | - /bin/sh 17 | - "-c" 18 | - "sleep infinity" 19 | imagePullPolicy: IfNotPresent 20 | volumeMounts: 21 | - name: nfs-shared 22 | mountPath: /var/lib/graph 23 | env: 24 | - name: ethereum 25 | valueFrom: 26 | secretKeyRef: 27 | name: ethereum-networks 28 | key: networks 29 | - name: PGHOST 30 | valueFrom: 31 | secretKeyRef: 32 | name: postgres-credentials 33 | key: host 34 | - name: PGUSER 35 | valueFrom: 36 | secretKeyRef: 37 | name: postgres-credentials 38 | key: user 39 | - name: PGPASSWORD 40 | valueFrom: 41 | secretKeyRef: 42 | name: postgres-credentials 43 | key: password 44 | - name: PGDATABASE 45 | value: graph 46 | - name: node_id 47 | valueFrom: 48 | fieldRef: 49 | fieldPath: metadata.name 50 | restartPolicy: Always 51 | -------------------------------------------------------------------------------- /k8s/overlays/block_ingestor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: index-node-block-ingestor 5 | spec: 6 | # Set this to the number of VM's in the index_pool node pool 7 | replicas: 2 8 | template: 9 | spec: 10 | containers: 11 | - name: graph-node 12 | resources: 13 | requests: 14 | # Set this to a little more than half the number of CPUs 15 | # that one VM in the node pool has 16 | cpu: 600m 17 | env: 18 | - name: GRAPH_LOG 19 | value: debug 20 | -------------------------------------------------------------------------------- /k8s/overlays/index_node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: index-node 5 | spec: 6 | # Set this to the number of VM's in the index_pool node pool 7 | replicas: 2 8 | template: 9 | spec: 10 | containers: 11 | - name: graph-node 12 | resources: 13 | requests: 14 | # Set this to a little more than half the number of CPUs 15 | # that one VM in the node pool has 16 | cpu: 600m 17 | env: 18 | - name: GRAPH_LOG 19 | value: debug 20 | - name: GRAPH_ALLOW_NON_DETERMINISTIC_IPFS 21 | value: "true" 22 | - name: GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH 23 | value: "true" 24 | -------------------------------------------------------------------------------- /k8s/overlays/indexer_agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: indexer-agent 5 | spec: 6 | template: 7 | spec: 8 | containers: 9 | - name: indexer-agent 10 | env: 11 | # Set this so it matches the index node pods 12 | - name: INDEXER_AGENT_INDEX_NODE_IDS 13 | value: index_node_0,index_node_1 14 | 15 | # Set this to your indexer's lat, long coordinates 16 | - name: INDEXER_AGENT_INDEXER_GEO_COORDINATES 17 | value: "0.00 0.00" 18 | 19 | # Set this to the public URL that your indexer serves from 20 | - name: INDEXER_AGENT_PUBLIC_INDEXER_URL 21 | value: http://not.localhost/ 22 | -------------------------------------------------------------------------------- /k8s/overlays/indexer_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: indexer-service 5 | spec: 6 | replicas: 2 7 | template: 8 | spec: 9 | containers: 10 | - name: indexer-service 11 | -------------------------------------------------------------------------------- /k8s/overlays/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | bases: 4 | - ../base 5 | images: 6 | - name: graph-node-image 7 | newName: graphprotocol/graph-node 8 | newTag: v0.27.0 9 | - name: graph-node-debug 10 | newName: graphprotocol/graph-node-debug 11 | newTag: v0.27.0 12 | - name: indexer-agent-image 13 | newName: ghcr.io/graphprotocol/indexer-agent 14 | newTag: v0.20.4 15 | - name: prometheus-image 16 | newName: prom/prometheus 17 | newTag: v2.32.1 18 | - name: busybox-image 19 | newName: busybox 20 | newTag: "1.30" 21 | - name: nginx-image 22 | newName: nginx 23 | newTag: latest 24 | - name: gists/nfs-server 25 | newTag: 2.5.4 26 | patchesStrategicMerge: 27 | - index_node.yaml 28 | - block_ingestor.yaml 29 | - query_node.yaml 30 | - indexer_agent.yaml 31 | - indexer_service.yaml 32 | - prometheus.yaml 33 | -------------------------------------------------------------------------------- /k8s/overlays/prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: prometheus-proxy-conf 5 | data: 6 | # Default password is 'total.apple.silk' 7 | .htpasswd: | 8 | admin:$apr1$BKQlWOsd$P5PO8l.j1hFDQZ2J.tVQj1 9 | --- 10 | 11 | # apiVersion: v1 12 | # kind: Service 13 | # metadata: 14 | # name: prometheus 15 | # annotations: 16 | # prometheus.io/scrape: 'true' 17 | # prometheus.io/port: '9099' 18 | # spec: 19 | # # For testing, we expose prometheus publicly 20 | # # Go to http:///prometheus to access it 21 | # type: LoadBalancer 22 | -------------------------------------------------------------------------------- /k8s/overlays/query_node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: query-node 5 | spec: 6 | # Set this to the number of VM's in the query_pool node pool 7 | replicas: 2 8 | template: 9 | spec: 10 | containers: 11 | - name: graph-node 12 | resources: 13 | requests: 14 | # Set this to a little more than half the number of CPUs 15 | # that one VM in the node pool has 16 | cpu: 600m 17 | env: 18 | - name: GRAPH_LOG 19 | value: debug 20 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": [ 3 | "packages/*" 4 | ], 5 | "npmClient": "yarn", 6 | "useWorkspaces": true, 7 | "version": "0.24.3" 8 | } 9 | -------------------------------------------------------------------------------- /network-configs/config.yaml: -------------------------------------------------------------------------------- 1 | networkIdentifier: "arbitrum-sepolia" 2 | gateway: 3 | url: "http://127.0.0.1:8030/" 4 | networkProvider: 5 | url: 6 | indexerOptions: 7 | address: "0xf56b5d582920E4527A818FBDd801C0D80A394CB8" 8 | mnemonic: "famous aspect index polar tornado zero wedding electric floor chalk tenant junk" 9 | url: "http://test-indexer.xyz" 10 | subgraphs: 11 | maxBlockDistance: 1000000 12 | networkSubgraph: 13 | url: "https://gateway-arbitrum.network.thegraph.com/api//subgraphs/id/3xQHhMudr1oh69ut36G2mbzpYmYxwqCeU6wwqyCDCnqV" 14 | epochSubgraph: 15 | url: "http://test-url.xyz" 16 | tapSubgraph: 17 | url: "https://gateway-arbitrum.network.thegraph.com/api//subgraphs/id/7ubx365MiqBH5iUz6XWXWT8PTof5BVAyEzdb8m17RvbD" 18 | transactionMonitoring: 19 | gasIncreaseTimeout: 240000 20 | gasIncreaseFactor: 1.2 21 | baseFeePerGasMax: 100000000000 22 | maxTransactionAttempts: 0 23 | tapAddressBook: 24 | "421614": 25 | TAPVerifier: "0xf56b5d582920E4527A818FBDd801C0D80A394CB8" 26 | AllocationIDTracker: "0xf56b5d582920E4527A818FBDd801C0D80A394CB8" 27 | Escrow: "0xf56b5d582920E4527A818FBDd801C0D80A394CB8" 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "root", 3 | "private": true, 4 | "workspaces": { 5 | "packages": [ 6 | "packages/indexer-common", 7 | "packages/indexer-cli", 8 | "packages/indexer-agent", 9 | "packages/indexer-native" 10 | ] 11 | }, 12 | "scripts": { 13 | "bootstrap": "lerna bootstrap", 14 | "prepare": "lerna run prepare", 15 | "release": "./scripts/release.sh", 16 | "test": "lerna --concurrency 1 run test --stream", 17 | "test:ci": "lerna --concurrency 1 run test:ci --stream", 18 | "clean": "lerna run clean && rm -rf ./node_modules", 19 | "compile": "lerna run compile" 20 | }, 21 | "devDependencies": { 22 | "@octokit/core": "3.2.0", 23 | "lerna": "6.1.0" 24 | }, 25 | "resolutions": { 26 | "ethers": "5.7.0", 27 | "sequelize": "6.33.0", 28 | "@ethersproject/bignumber": "5.7.0", 29 | "@ethersproject/providers": "5.7.0", 30 | "@urql/core": "3.1.0", 31 | "@urql/exchange-execute": "2.1.0", 32 | "punycode": "2.3.1", 33 | "uri-js": "4.2.2" 34 | }, 35 | "overrides": { 36 | "ethers": "5.7.0", 37 | "sequelize": "6.33.0", 38 | "@ethersproject/bignumber": "5.7.0", 39 | "@ethersproject/providers": "5.7.0", 40 | "@urql/core": "3.1.0", 41 | "@urql/exchange-execute": "2.1.0", 42 | "graphql": "16.8.0" 43 | }, 44 | "engines": { 45 | "node": ">=18.0.0" 46 | }, 47 | "jest": { 48 | "transformIgnorePatterns": [ 49 | "node_modules/(?!@thi.ng)" 50 | ] 51 | }, 52 | "packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610" 53 | } 54 | -------------------------------------------------------------------------------- /packages/indexer-agent/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage 4 | *.js 5 | -------------------------------------------------------------------------------- /packages/indexer-agent/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | parser: '@typescript-eslint/parser', 4 | plugins: ['@typescript-eslint'], 5 | extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'] 6 | } 7 | -------------------------------------------------------------------------------- /packages/indexer-agent/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "all", 3 | "tabWidth": 2, 4 | "semi": false, 5 | "singleQuote": true, 6 | "arrowParens": "avoid" 7 | } 8 | -------------------------------------------------------------------------------- /packages/indexer-agent/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 The Graph Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /packages/indexer-agent/bin/graph-indexer-agent: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('../dist/index') 4 | -------------------------------------------------------------------------------- /packages/indexer-agent/config.yaml: -------------------------------------------------------------------------------- 1 | log-level: "debug" 2 | -------------------------------------------------------------------------------- /packages/indexer-agent/jest.config.js: -------------------------------------------------------------------------------- 1 | const bail = s => { 2 | throw new Error(s) 3 | } 4 | 5 | // until we find a way to avoid `punycode` we suppress the warnings in tests 6 | process.env.NODE_NO_WARNINGS = '1' 7 | 8 | module.exports = { 9 | collectCoverage: true, 10 | forceExit: true, 11 | preset: 'ts-jest', 12 | testEnvironment: 'node', 13 | testPathIgnorePatterns: ['/node_modules/', '/dist/', '.yalc'], 14 | globals: { 15 | __DATABASE__: { 16 | host: 17 | process.env.POSTGRES_TEST_HOST || 18 | bail('POSTGRES_TEST_HOST is not defined'), 19 | port: parseInt(process.env.POSTGRES_TEST_PORT || '5432'), 20 | username: 21 | process.env.POSTGRES_TEST_USERNAME || 22 | bail('POSTGRES_TEST_USERNAME is not defined'), 23 | password: 24 | process.env.POSTGRES_TEST_PASSWORD || 25 | bail('POSTGRES_TEST_PASSWORD is not defined'), 26 | database: 27 | process.env.POSTGRES_TEST_DATABASE || 28 | bail('POSTGRES_TEST_DATABASE is not defined'), 29 | }, 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /packages/indexer-agent/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@graphprotocol/indexer-agent", 3 | "version": "0.24.3", 4 | "description": "Indexer agent", 5 | "main": "./dist/index.js", 6 | "types": "./dist/index.d.ts", 7 | "files": [ 8 | "/bin", 9 | "/dist" 10 | ], 11 | "repository": "https://github.com/graphprotocol/indexer", 12 | "author": "Graph Protocol", 13 | "private": false, 14 | "scripts": { 15 | "format": "prettier --write 'src/**/*.ts'", 16 | "lint": "eslint . --ext .ts,.tsx --fix", 17 | "compile": "tsc --build", 18 | "prepare": "yarn format && yarn lint && yarn compile", 19 | "start": "node ./dist/index.js start", 20 | "test": "jest --colors --verbose --detectOpenHandles", 21 | "test:ci": "jest --verbose --ci", 22 | "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo", 23 | "migrator:pending": "node src/db/cli/migrator pending", 24 | "migrator:executed": "node src/db/cli/migrator executed", 25 | "migrator:up": "node src/db/cli/migrator up", 26 | "migrator:down": "node src/db/cli/migrator down" 27 | }, 28 | "bin": { 29 | "graph-indexer-agent": "bin/graph-indexer-agent" 30 | }, 31 | "dependencies": { 32 | "@graphprotocol/common-ts": "2.0.11", 33 | "@graphprotocol/indexer-common": "^0.24.3", 34 | "@thi.ng/heaps": "^1.3.1", 35 | "axios": "0.26.1", 36 | "bs58": "5.0.0", 37 | "delay": "^5.0.0", 38 | "ethers": "5.7.0", 39 | "evt": "1.9.12", 40 | "global": "4.4.0", 41 | "graphql": "16.8.0", 42 | "graphql-tag": "2.12.6", 43 | "isomorphic-fetch": "3.0.0", 44 | "jayson": "3.6.6", 45 | "lodash.isequal": "4.5.0", 46 | "lodash.mapvalues": "^4.6.0", 47 | "lodash.zip": "^4.2.0", 48 | "ngeohash": "0.6.3", 49 | "p-filter": "2.1.0", 50 | "p-map": "4.0.0", 51 | "p-queue": "6.6.2", 52 | "p-reduce": "2.1.0", 53 | "p-retry": "4.6.1", 54 | "umzug": "3.0.0", 55 | "yaml": "^2.0.0-10", 56 | "yargs": "17.4.1", 57 | "zod": "^3.21.4", 58 | "zod-validation-error": "^1.3.0" 59 | }, 60 | "devDependencies": { 61 | "@types/bs58": "4.0.1", 62 | "@types/isomorphic-fetch": "0.0.36", 63 | "@types/jest": "29.5.4", 64 | "@types/lodash.countby": "^4.6.7", 65 | "@types/lodash.isequal": "4.5.6", 66 | "@types/lodash.mapvalues": "^4.6.7", 67 | "@types/lodash.zip": "^4.2.7", 68 | "@types/ngeohash": "0.6.4", 69 | "@types/node": "20.6.1", 70 | "@types/yargs": "17.0.10", 71 | "@typescript-eslint/eslint-plugin": "6.7.0", 72 | "@typescript-eslint/parser": "6.7.0", 73 | "eslint": "8.49.0", 74 | "eslint-config-prettier": "^8.5.0", 75 | "jest": "<30.0.0-0", 76 | "prettier": "3.0.3", 77 | "ts-jest": "^29.2.5", 78 | "ts-node": "10.7.0", 79 | "typechain": "8.0.0", 80 | "typescript": "5.2.2" 81 | }, 82 | "resolutions": { 83 | "ethers": "5.7.0", 84 | "sequelize": "6.33.0", 85 | "@ethersproject/bignumber": "5.7.0", 86 | "@ethersproject/providers": "5.7.0" 87 | }, 88 | "gitHead": "972ab96774007b2aee15b1da169d2ff4be9f9d27" 89 | } 90 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/commands/start-multi-network.ts: -------------------------------------------------------------------------------- 1 | import { Argv } from 'yargs' 2 | import { injectCommonStartupOptions } from './common-options' 3 | 4 | export const startMultiNetwork = { 5 | command: 'start', 6 | describe: 'Start the Agent in multiple Protocol Networks', 7 | builder: (args: Argv): Argv => { 8 | const updatedArgs = injectCommonStartupOptions(args) 9 | return updatedArgs.option('network-specifications-directory', { 10 | alias: 'dir', 11 | description: 'Path to a directory containing network specification files', 12 | type: 'string', 13 | required: true, 14 | }) 15 | }, 16 | // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any 17 | handler: (_argv: any) => {}, 18 | } 19 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/cli/migrator.js: -------------------------------------------------------------------------------- 1 | require('ts-node/register'); 2 | 3 | require('./umzug').migrator.runAsCLI(); 4 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/cli/umzug.ts: -------------------------------------------------------------------------------- 1 | import path from 'path' 2 | import { Umzug, SequelizeStorage } from 'umzug' 3 | import { Sequelize } from 'sequelize' 4 | import { createLogger } from '@graphprotocol/common-ts' 5 | 6 | const verbose_logging = process.env.VERBOSE === 'true' 7 | const log_level = verbose_logging ? 'trace' : 'info' 8 | const host = process.env.POSTGRES_HOST || 'localhost' 9 | const port = parseInt(process.env.POSTGRES_PORT || '5432') 10 | const database = process.env.POSTGRES_DATABASE 11 | const username = process.env.POSTGRES_USERNAME 12 | const password = process.env.POSTGRES_PASSWORD 13 | 14 | const logger = createLogger({ 15 | name: 'Migrations', 16 | async: false, 17 | level: log_level, 18 | }) 19 | 20 | if (!database) { 21 | throw Error( 22 | `Database name not defined, please set 'POSTGRES_DATABASE' ENV var`, 23 | ) 24 | } 25 | if (!username) { 26 | throw Error( 27 | `Database 'username' name not defined, please set 'POSTGRES_USERNAME' ENV var`, 28 | ) 29 | } 30 | 31 | logger.debug('Connect to database', { 32 | host, 33 | port, 34 | database, 35 | username, 36 | verbose_logging, 37 | }) 38 | 39 | const sequelize = new Sequelize({ 40 | dialect: 'postgres', 41 | host, 42 | port, 43 | username, 44 | password, 45 | database, 46 | pool: { 47 | max: 10, 48 | min: 0, 49 | }, 50 | logging: false, 51 | }) 52 | 53 | logger.debug('Successfully connected to DB', { name: database }) 54 | 55 | export const migrator = new Umzug({ 56 | migrations: { 57 | glob: path.join( 58 | __dirname, 59 | '..', 60 | '..', 61 | '..', 62 | 'dist', 63 | 'db', 64 | 'migrations', 65 | '*.js', 66 | ), 67 | }, 68 | context: { 69 | queryInterface: sequelize.getQueryInterface(), 70 | logger, 71 | }, 72 | storage: new SequelizeStorage({ 73 | sequelize, 74 | }), 75 | logger: console, 76 | }) 77 | 78 | export type Migration = typeof migrator._types.migration 79 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/01-reset-state-channels.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.info(`Reset state channels in the database`) 17 | const tablesToTruncate = [ 18 | 'signing_wallets', 19 | 'objectives', 20 | 'nonces', 21 | 'app_bytecode', 22 | ] 23 | 24 | const tables = await queryInterface.showAllTables() 25 | 26 | for (const table of tablesToTruncate) { 27 | if (!tables.includes(table)) { 28 | logger.info(`Skipping non-existent table ${table}`) 29 | } else { 30 | try { 31 | await queryInterface.sequelize.query(`TRUNCATE TABLE ${table} CASCADE`) 32 | } catch (err) { 33 | logger.warn(`Failed to reset table`, { table, err }) 34 | } 35 | } 36 | } 37 | } 38 | 39 | export async function down(): Promise { 40 | // Nothing to do here 41 | } 42 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/02-remove-state-channels.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.info(`Remove old state channels from the database`) 17 | 18 | const tables = [ 19 | 'app_bytecode', 20 | 'channels', 21 | 'funding', 22 | 'knex_migrations', 23 | 'knex_migrations_lock', 24 | 'ledger_requests', 25 | 'nonces', 26 | 'objectives', 27 | 'objectives_channels', 28 | 'signing_wallets', 29 | ] 30 | 31 | for (const table of tables) { 32 | try { 33 | await queryInterface.dropTable(table, { cascade: true }) 34 | } catch (err) { 35 | logger.warn( 36 | 'Failed to remove old state channels table from the database', 37 | { 38 | err, 39 | table, 40 | }, 41 | ) 42 | } 43 | } 44 | } 45 | 46 | export async function down(): Promise { 47 | // Nothing to do here 48 | } 49 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/03-poi-disputes-add-subgraph-deployment-id.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface, DataTypes } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.info(`Checking if POI disputes table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('POIDisputes')) { 19 | logger.info(`POI disputes table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.info(`Checking if POI disputes table needs to be migrated`) 24 | const table = await queryInterface.describeTable('POIDisputes') 25 | const subgraphDeploymentIDColumn = table.subgraphDeploymentID 26 | if (subgraphDeploymentIDColumn) { 27 | logger.info( 28 | `Subgraph deployment ID column already exists, migration not necessary`, 29 | ) 30 | return 31 | } 32 | 33 | logger.info(`Adding subgraphDeploymentID column to POIDisputes table`) 34 | await queryInterface.addColumn('POIDisputes', 'subgraphDeploymentID', { 35 | type: DataTypes.STRING, 36 | allowNull: false, 37 | defaultValue: 'notSet', 38 | }) 39 | } 40 | 41 | export async function down({ context }: Context): Promise { 42 | await context.queryInterface.removeColumn( 43 | 'POIDisputes', 44 | 'subgraphDeploymentID', 45 | ) 46 | } 47 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/03-rename-receipts-table.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.info(`Rename 'receipts' table to 'transfer_receipts'`) 17 | 18 | const tables = await queryInterface.showAllTables() 19 | 20 | // If we already have the transfer_receipts table, just drop the receipts 21 | if (tables.includes('transfer_receipts')) { 22 | await queryInterface.dropTable('receipts') 23 | } else { 24 | // Otherwise rename the table to transfer_receipts 25 | if (tables.includes('receipts')) { 26 | await queryInterface.renameTable('receipts', 'transfer_receipts') 27 | } 28 | } 29 | } 30 | 31 | export async function down({ context }: Context): Promise { 32 | const { queryInterface, logger } = context 33 | logger.info(`Rename 'transfer_receipts' table to 'receipts'`) 34 | const tables = await queryInterface.showAllTables() 35 | 36 | // If we already have a receipts table, just drop transfer_receipts 37 | if (tables.includes('receipts')) { 38 | await queryInterface.dropTable('transfer_receipts') 39 | } else { 40 | // Otherwise rename the table to receipts 41 | if (tables.includes('transfer_receipts')) { 42 | await queryInterface.renameTable('transfer_receipts', 'receipts') 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/05-indexing-rules-add-subgraph-id.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if indexing rules table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('IndexingRules')) { 19 | logger.info(`Indexing rules table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'IndexingRules' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('IndexingRules') 25 | const subgraphIdentifierTypeColumn = table.identifierType 26 | const subgraphIdentifierColumn = table.identifier 27 | if (subgraphIdentifierTypeColumn && subgraphIdentifierColumn) { 28 | logger.info( 29 | `'identifier' and 'identifierType' columns already exist, migration not necessary`, 30 | ) 31 | return 32 | } 33 | 34 | logger.info(`Add 'identifierType' column to 'IndexingRules' table`) 35 | await queryInterface.addColumn('IndexingRules', 'identifierType', { 36 | type: DataTypes.ENUM('deployment', 'subgraph', 'group'), 37 | primaryKey: true, 38 | defaultValue: 'deployment', 39 | }) 40 | 41 | logger.info(`Rename 'deployment' column to 'identifier'`) 42 | await queryInterface.renameColumn('IndexingRules', 'deployment', 'identifier') 43 | 44 | logger.info(`Update identifierType value for existing rules`) 45 | await queryInterface.sequelize.query( 46 | `update "IndexingRules" set "identifierType" = 'group' where "identifier" = 'global'`, 47 | ) 48 | } 49 | 50 | export async function down({ context }: Context): Promise { 51 | const { queryInterface, logger } = context 52 | 53 | return await queryInterface.sequelize.transaction({}, async transaction => { 54 | const tables = await queryInterface.showAllTables() 55 | 56 | if (tables.includes('IndexingRules')) { 57 | logger.info(`Remove 'identifierType' column`) 58 | await context.queryInterface.removeColumn( 59 | 'IndexingRules', 60 | 'identifierType', 61 | { transaction }, 62 | ) 63 | 64 | logger.info(`Rename 'identifier' column back to 'deployment'`) 65 | await queryInterface.renameColumn( 66 | 'IndexingRules', 67 | 'identifier', 68 | 'deployment', 69 | { transaction }, 70 | ) 71 | 72 | logger.info(`Remove 'enum_IndexingRules_identifierType' custom type`) 73 | await queryInterface.sequelize.query( 74 | `delete from pg_type where typname = 'enum_IndexingRules_identifierType'`, 75 | ) 76 | } 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/06-indexing-rules-add-max-lifetime.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if indexing rules table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('IndexingRules')) { 19 | logger.info(`Indexing rules table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'IndexingRules' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('IndexingRules') 25 | const allocationLifetime = table.allocationLifetime 26 | if (allocationLifetime) { 27 | logger.info( 28 | `'allocationLifetime' columns already exist, migration not necessary`, 29 | ) 30 | return 31 | } 32 | 33 | logger.info(`Add 'allocationLifetime' column to 'IndexingRules' table`) 34 | await queryInterface.addColumn('IndexingRules', 'allocationLifetime', { 35 | type: DataTypes.INTEGER, 36 | allowNull: true, 37 | defaultValue: null, 38 | }) 39 | } 40 | 41 | export async function down({ context }: Context): Promise { 42 | const { queryInterface, logger } = context 43 | 44 | return await queryInterface.sequelize.transaction({}, async transaction => { 45 | const tables = await queryInterface.showAllTables() 46 | 47 | if (tables.includes('IndexingRules')) { 48 | logger.info(`Remove 'allocationLifetime' column`) 49 | await context.queryInterface.removeColumn( 50 | 'IndexingRules', 51 | 'allocationLifetime', 52 | { transaction }, 53 | ) 54 | } 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/07-indexing-rules-add-require-supported.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if indexing rules table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('IndexingRules')) { 19 | logger.info(`Indexing rules table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'IndexingRules' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('IndexingRules') 25 | const requireSupportedColumn = table.requireSupported 26 | if (requireSupportedColumn) { 27 | logger.info( 28 | `'requireSupported' columns already exist, migration not necessary`, 29 | ) 30 | return 31 | } 32 | 33 | logger.info(`Add 'requireSupported' column to 'IndexingRules' table`) 34 | await queryInterface.addColumn('IndexingRules', 'requireSupported', { 35 | type: DataTypes.BOOLEAN, 36 | defaultValue: false, 37 | }) 38 | } 39 | 40 | export async function down({ context }: Context): Promise { 41 | const { queryInterface, logger } = context 42 | 43 | return await queryInterface.sequelize.transaction({}, async transaction => { 44 | const tables = await queryInterface.showAllTables() 45 | 46 | if (tables.includes('IndexingRules')) { 47 | logger.info(`Remove 'requireSupported' column`) 48 | await context.queryInterface.removeColumn( 49 | 'IndexingRules', 50 | 'requireSupported', 51 | { transaction }, 52 | ) 53 | } 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/08-indexing-rules-add-auto-renewal.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if indexing rules table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('IndexingRules')) { 19 | logger.info(`Indexing rules table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'IndexingRules' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('IndexingRules') 25 | const autoRenewal = table.autoRenewal 26 | if (autoRenewal) { 27 | logger.info(`'autoRenewal' columns already exist, migration not necessary`) 28 | return 29 | } 30 | 31 | logger.info(`Add 'autoRenewal' column to 'IndexingRules' table`) 32 | await queryInterface.addColumn('IndexingRules', 'autoRenewal', { 33 | type: DataTypes.BOOLEAN, 34 | allowNull: false, 35 | defaultValue: true, 36 | }) 37 | } 38 | 39 | export async function down({ context }: Context): Promise { 40 | const { queryInterface, logger } = context 41 | 42 | return await queryInterface.sequelize.transaction({}, async transaction => { 43 | const tables = await queryInterface.showAllTables() 44 | 45 | if (tables.includes('IndexingRules')) { 46 | logger.info(`Remove 'autoRenewal' column`) 47 | await context.queryInterface.removeColumn( 48 | 'IndexingRules', 49 | 'autoRenewal', 50 | { transaction }, 51 | ) 52 | } 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/09-actions-expand-failure-reason-varchar.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if 'Actions' table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('Actions')) { 19 | logger.info(`Actions table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'Actions' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('Actions') 25 | const failureReason = table.failureReason 26 | if (failureReason) { 27 | logger.debug( 28 | `'failureReason' columns exists with type = ${failureReason.type}`, 29 | ) 30 | logger.info( 31 | `Update 'failureReason' column to support large strings (up to length = 5000)`, 32 | ) 33 | await queryInterface.changeColumn( 34 | 'Actions', 35 | 'failureReason', 36 | DataTypes.STRING(5000), 37 | ) 38 | return 39 | } 40 | } 41 | 42 | export async function down({ context }: Context): Promise { 43 | const { logger } = context 44 | logger.info( 45 | `No 'down' migration needed since the 'up' migration simply expanded the 'failureReason' column size`, 46 | ) 47 | return 48 | } 49 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/10-indexing-rules-add-safety.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if indexing rules table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('IndexingRules')) { 19 | logger.info(`Indexing rules table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'IndexingRules' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('IndexingRules') 25 | const safetyColumn = table.safety 26 | if (safetyColumn) { 27 | logger.info(`'safety' column already exist, migration not necessary`) 28 | return 29 | } 30 | 31 | logger.info(`Add 'safety' column to 'IndexingRules' table`) 32 | await queryInterface.addColumn('IndexingRules', 'safety', { 33 | type: DataTypes.BOOLEAN, 34 | defaultValue: true, 35 | }) 36 | } 37 | 38 | export async function down({ context }: Context): Promise { 39 | const { queryInterface, logger } = context 40 | 41 | return await queryInterface.sequelize.transaction({}, async transaction => { 42 | const tables = await queryInterface.showAllTables() 43 | 44 | if (tables.includes('IndexingRules')) { 45 | logger.info(`Remove 'safety' column`) 46 | await context.queryInterface.removeColumn('IndexingRules', 'safety', { 47 | transaction, 48 | }) 49 | } 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/13-add-scalar-tap-deny-list.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | 3 | import { QueryInterface, DataTypes } from 'sequelize' 4 | 5 | interface MigrationContext { 6 | queryInterface: QueryInterface 7 | logger: Logger 8 | } 9 | 10 | interface Context { 11 | context: MigrationContext 12 | } 13 | 14 | export async function up({ context }: Context): Promise { 15 | const { queryInterface, logger } = context 16 | 17 | const tables = await queryInterface.showAllTables() 18 | logger.debug(`Checking if scalar_tap_denylist table exists`, { tables }) 19 | 20 | if (tables.includes('scalar_tap_denylist')) { 21 | logger.debug(`scalar_tap_denylist already exist, migration not necessary`) 22 | } else { 23 | logger.info(`Create scalar_tap_denylist`) 24 | await queryInterface.createTable('scalar_tap_denylist', { 25 | id: { 26 | type: DataTypes.BIGINT, 27 | primaryKey: true, 28 | autoIncrement: true, 29 | }, 30 | sender_address: { 31 | type: DataTypes.CHAR(40), 32 | allowNull: false, 33 | }, 34 | }) 35 | } 36 | const functionSQL = ` 37 | CREATE FUNCTION scalar_tap_deny_notify() 38 | RETURNS trigger AS 39 | $$ 40 | BEGIN 41 | IF TG_OP = 'DELETE' THEN 42 | PERFORM pg_notify('scalar_tap_deny_notification', format('{"tg_op": "DELETE", "sender_address": "%s"}', OLD.sender_address)); 43 | RETURN OLD; 44 | ELSIF TG_OP = 'INSERT' THEN 45 | PERFORM pg_notify('scalar_tap_deny_notification', format('{"tg_op": "INSERT", "sender_address": "%s"}', NEW.sender_address)); 46 | RETURN NEW; 47 | ELSE -- UPDATE OR TRUNCATE, should never happen 48 | PERFORM pg_notify('scalar_tap_deny_notification', format('{"tg_op": "%s", "sender_address": null}', TG_OP, NEW.sender_address)); 49 | RETURN NEW; 50 | END IF; 51 | END; 52 | $$ LANGUAGE 'plpgsql'; 53 | ` 54 | const triggerSQL = ` 55 | CREATE TRIGGER deny_update AFTER INSERT OR UPDATE OR DELETE 56 | ON scalar_tap_denylist 57 | FOR EACH ROW EXECUTE PROCEDURE scalar_tap_deny_notify(); 58 | ` 59 | 60 | await queryInterface.sequelize.query(functionSQL) 61 | await queryInterface.sequelize.query(triggerSQL) 62 | } 63 | 64 | export async function down({ context }: Context): Promise { 65 | const { queryInterface, logger } = context 66 | 67 | logger.info(`Drop scalar_tap_denylist`) 68 | await queryInterface.sequelize.query( 69 | 'DROP TRIGGER IF EXISTS deny_update ON scalar_tap_denylist', 70 | ) 71 | logger.info(`Drop function scalar_tap_deny_notify`) 72 | await queryInterface.sequelize.query( 73 | 'DROP FUNCTION IF EXISTS scalar_tap_deny_notify', 74 | ) 75 | logger.info(`Drop table scalar_tap_denylist`) 76 | await queryInterface.dropTable('scalar_tap_denylist') 77 | } 78 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/15-modify-scalar-tap-tables.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface, DataTypes } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | const tables = await queryInterface.showAllTables() 17 | logger.debug( 18 | `Modifying tables scalar_Tap_Ravs, scalar_tap_receipts and scalar_tap_receipts_invalid with correct value types`, 19 | ) 20 | 21 | if (tables.includes('scalar_tap_ravs')) { 22 | await queryInterface.changeColumn('scalar_tap_ravs', 'value_aggregate', { 23 | type: DataTypes.DECIMAL(39), 24 | allowNull: false, 25 | }) 26 | } 27 | if (tables.includes('scalar_tap_receipts')) { 28 | await queryInterface.changeColumn('scalar_tap_receipts', 'nonce', { 29 | type: DataTypes.DECIMAL(20), 30 | allowNull: false, 31 | }) 32 | } 33 | 34 | if (tables.includes('scalar_tap_receipts_invalid')) { 35 | await queryInterface.changeColumn('scalar_tap_receipts_invalid', 'nonce', { 36 | type: DataTypes.DECIMAL(20), 37 | allowNull: false, 38 | }) 39 | } 40 | } 41 | 42 | export async function down({ context }: Context): Promise { 43 | const { queryInterface, logger } = context 44 | // Drop the scalar_tap_ravs table 45 | logger.info(`Drop table`) 46 | await queryInterface.dropTable('scalar_tap_ravs') 47 | 48 | logger.info(`Drop function, trigger, indices, and table`) 49 | await queryInterface.sequelize.query( 50 | 'DROP TRIGGER IF EXISTS receipt_update ON scalar_tap_receipts', 51 | ) 52 | await queryInterface.sequelize.query( 53 | 'DROP FUNCTION IF EXISTS scalar_tap_receipt_notify', 54 | ) 55 | await queryInterface.removeIndex( 56 | 'scalar_tap_receipts', 57 | 'scalar_tap_receipts_allocation_id_idx', 58 | ) 59 | await queryInterface.removeIndex( 60 | 'scalar_tap_receipts', 61 | 'scalar_tap_receipts_timestamp_ns_idx', 62 | ) 63 | await queryInterface.dropTable('scalar_tap_receipts') 64 | } 65 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/16-modify-invalid-receipts-table.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface, DataTypes } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | const tables = await queryInterface.showAllTables() 17 | logger.debug( 18 | `Modifying tables scalar_tap_receipts_invalid to add extra column`, 19 | ) 20 | 21 | if (tables.includes('scalar_tap_receipts_invalid')) { 22 | await queryInterface.addColumn('scalar_tap_receipts_invalid', 'error_log', { 23 | type: DataTypes.TEXT, 24 | allowNull: false, 25 | defaultValue: '', 26 | }) 27 | } 28 | } 29 | 30 | export async function down({ context }: Context): Promise { 31 | const { queryInterface, logger } = context 32 | logger.info(`Drop function, trigger, indices, and table`) 33 | queryInterface.removeColumn('scalar_tap_receipts_invalid', 'error_log') 34 | } 35 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/17-edit-cost-models-trigger.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.info( 17 | 'Deleting old function/trigger for cost models to add the model field', 18 | ) 19 | 20 | const dropFunctionSQL = ` 21 | DROP FUNCTION IF EXISTS cost_models_update_notify() CASCADE; 22 | ` 23 | await queryInterface.sequelize.query(dropFunctionSQL) 24 | 25 | const functionSQL = ` 26 | CREATE FUNCTION cost_models_update_notify() 27 | RETURNS trigger AS 28 | $$ 29 | BEGIN 30 | IF TG_OP = 'DELETE' THEN 31 | PERFORM pg_notify('cost_models_update_notification', format('{"tg_op": "DELETE", "deployment": "%s"}', OLD.deployment)); 32 | RETURN OLD; 33 | ELSIF TG_OP = 'INSERT' THEN 34 | PERFORM pg_notify('cost_models_update_notification', format('{"tg_op": "INSERT", "deployment": "%s", "model": "%s", "variables": "%s"}', NEW.deployment, NEW.model, NEW.variables)); 35 | RETURN NEW; 36 | ELSE 37 | PERFORM pg_notify('cost_models_update_notification', format('{"tg_op": "%s", "deployment": "%s", "model": "%s", "variables": "%s" }', NEW.deployment, NEW.model, NEW.variables)); 38 | RETURN NEW; 39 | END IF; 40 | END; 41 | $$ LANGUAGE 'plpgsql'; 42 | ` 43 | const triggerSQL = ` 44 | CREATE TRIGGER cost_models_update AFTER INSERT OR UPDATE OR DELETE 45 | ON "CostModelsHistory" 46 | FOR EACH ROW EXECUTE PROCEDURE cost_models_update_notify(); 47 | ` 48 | await queryInterface.sequelize.query(functionSQL) 49 | await queryInterface.sequelize.query(triggerSQL) 50 | } 51 | 52 | export async function down({ context }: Context): Promise { 53 | const { queryInterface, logger } = context 54 | logger.info(`Drop function, trigger, indices, and table`) 55 | await queryInterface.sequelize.query( 56 | 'DROP TRIGGER IF EXISTS cost_models_update ON "CostModelsHistory" CASCADE;', 57 | ) 58 | 59 | await queryInterface.sequelize.query( 60 | 'DROP FUNCTION IF EXISTS cost_models_update_notify() CASCADE;', 61 | ) 62 | } 63 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/db/migrations/18-actions-expand-action-status-add-deploying.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { DataTypes, QueryInterface } from 'sequelize' 3 | 4 | interface MigrationContext { 5 | queryInterface: QueryInterface 6 | logger: Logger 7 | } 8 | 9 | interface Context { 10 | context: MigrationContext 11 | } 12 | 13 | export async function up({ context }: Context): Promise { 14 | const { queryInterface, logger } = context 15 | 16 | logger.debug(`Checking if 'Actions' table exists`) 17 | const tables = await queryInterface.showAllTables() 18 | if (!tables.includes('Actions')) { 19 | logger.info(`Actions table does not exist, migration not necessary`) 20 | return 21 | } 22 | 23 | logger.debug(`Checking if 'Actions' table needs to be migrated`) 24 | const table = await queryInterface.describeTable('Actions') 25 | const statusColumn = table.status 26 | if (statusColumn) { 27 | logger.debug(`'status' column exists with type = ${statusColumn.type}`) 28 | logger.info(`Update 'status' column to support variant 'deploying' status`) 29 | await queryInterface.changeColumn('Actions', 'status', { 30 | type: DataTypes.ENUM( 31 | 'queued', 32 | 'approved', 33 | 'deploying', 34 | 'pending', 35 | 'success', 36 | 'failed', 37 | 'canceled', 38 | ), 39 | allowNull: false, 40 | }) 41 | return 42 | } 43 | } 44 | 45 | export async function down({ context }: Context): Promise { 46 | const { logger } = context 47 | logger.info( 48 | `No 'down' migration needed since the 'up' migration simply added a new status 'deploying'`, 49 | ) 50 | return 51 | } 52 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/index.ts: -------------------------------------------------------------------------------- 1 | import { createLogger } from '@graphprotocol/common-ts' 2 | import * as yargs from 'yargs' 3 | import { 4 | start, 5 | createNetworkSpecification, 6 | reviewArgumentsForWarnings, 7 | AgentOptions, 8 | run, 9 | } from './commands/start' 10 | import { startMultiNetwork } from './commands/start-multi-network' 11 | import { parseNetworkSpecifications } from '@graphprotocol/indexer-common' 12 | 13 | const MULTINETWORK_MODE: boolean = 14 | !!process.env.INDEXER_AGENT_MULTINETWORK_MODE && 15 | process.env.INDEXER_AGENT_MULTINETWORK_MODE.toLowerCase() !== 'false' 16 | 17 | function parseArguments(): AgentOptions { 18 | let builder = yargs.scriptName('indexer-agent').env('INDEXER_AGENT') 19 | 20 | // Dynamic argument parser construction based on network mode 21 | if (MULTINETWORK_MODE) { 22 | console.log('Starting the Indexer Agent in multi-network mode') 23 | builder = builder.command(startMultiNetwork) 24 | } else { 25 | console.log('Starting the Indexer Agent in single-network mode') 26 | builder = builder.command(start) 27 | } 28 | 29 | return ( 30 | builder 31 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 32 | .fail(function (msg, err, _yargs) { 33 | console.error('The Indexer Agent command has failed.') 34 | if (err) { 35 | console.error(err) 36 | } else { 37 | console.error(msg) 38 | } 39 | process.exit(1) 40 | }) 41 | .demandCommand( 42 | 1, 43 | 'You need at least one command before continuing.' + 44 | " See 'indexer-agent --help' for usage instructions.", 45 | ) 46 | .help().argv 47 | ) 48 | } 49 | 50 | async function processArgumentsAndRun(args: AgentOptions): Promise { 51 | const logger = createLogger({ 52 | name: 'IndexerAgent', 53 | async: false, 54 | level: args.logLevel, 55 | }) 56 | if (MULTINETWORK_MODE) { 57 | const specifications = parseNetworkSpecifications(args, logger) 58 | await run(args, specifications, logger) 59 | } else { 60 | reviewArgumentsForWarnings(args, logger) 61 | const specification = await createNetworkSpecification(args, logger) 62 | await run(args, [specification], logger) 63 | } 64 | } 65 | 66 | async function main(): Promise { 67 | const args = parseArguments() 68 | await processArgumentsAndRun(args) 69 | } 70 | 71 | void main() 72 | -------------------------------------------------------------------------------- /packages/indexer-agent/src/types.ts: -------------------------------------------------------------------------------- 1 | import { Logger, Metrics, SubgraphDeploymentID } from '@graphprotocol/common-ts' 2 | import { 3 | Network, 4 | GraphNode, 5 | DeploymentManagementMode, 6 | IndexerManagementClient, 7 | Operator, 8 | } from '@graphprotocol/indexer-common' 9 | 10 | // Represents a pair of Network and Operator instances belonging to the same protocol 11 | // network. Used when mapping over multiple protocol networks. 12 | export type NetworkAndOperator = { 13 | network: Network 14 | operator: Operator 15 | } 16 | 17 | export interface AgentConfigs { 18 | logger: Logger 19 | metrics: Metrics 20 | graphNode: GraphNode 21 | operators: Operator[] 22 | indexerManagement: IndexerManagementClient 23 | networks: Network[] 24 | deploymentManagement: DeploymentManagementMode 25 | autoMigrationSupport: boolean 26 | offchainSubgraphs: SubgraphDeploymentID[] 27 | pollingInterval: number 28 | } 29 | -------------------------------------------------------------------------------- /packages/indexer-agent/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "rootDir": "src", 5 | "outDir": "dist", 6 | "target": "es2020", 7 | "module": "commonjs", 8 | "declaration": true, 9 | "sourceMap": true, 10 | "esModuleInterop": true, 11 | "strict": true, 12 | "composite": true, 13 | "lib": ["es2015", "es6", "esnext.asynciterable", "dom", "ES2020.Promise"] 14 | }, 15 | "include": ["src/**/*.ts", "src/**/*.d.ts"], 16 | "exclude": ["src/**/__tests__/*.ts"], 17 | "references": [{ "path": "../indexer-common" }] 18 | } 19 | -------------------------------------------------------------------------------- /packages/indexer-cli/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: false, 3 | parser: '@typescript-eslint/parser', 4 | plugins: ['@typescript-eslint'], 5 | extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'] 6 | } 7 | -------------------------------------------------------------------------------- /packages/indexer-cli/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "trailingComma": "all", 4 | "printWidth": 90, 5 | "singleQuote": true, 6 | "arrowParens": "avoid" 7 | } 8 | -------------------------------------------------------------------------------- /packages/indexer-cli/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 The Graph Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /packages/indexer-cli/bin/graph-indexer: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // run the CLI with the current process arguments 4 | require('../dist/cli').run(process.argv) 5 | -------------------------------------------------------------------------------- /packages/indexer-cli/jest.config.js: -------------------------------------------------------------------------------- 1 | const bail = s => { 2 | throw new Error(s) 3 | } 4 | 5 | // until we find a way to avoid `punycode` we suppress the warnings in tests 6 | process.env.NODE_NO_WARNINGS = '1' 7 | 8 | module.exports = { 9 | collectCoverage: false, 10 | preset: 'ts-jest', 11 | forceExit: true, 12 | testEnvironment: 'node', 13 | // The glob patterns Jest uses to detect test files 14 | testMatch: ['**/?(*.)+(spec|test).ts?(x)'], 15 | testPathIgnorePatterns: ['/node_modules/', '/dist/', '.yalc'], 16 | testTimeout: 60000, 17 | globals: { 18 | __DATABASE__: { 19 | host: process.env.POSTGRES_TEST_HOST || bail('POSTGRES_TEST_HOST is not defined'), 20 | port: parseInt(process.env.POSTGRES_TEST_PORT || '5432'), 21 | username: 22 | process.env.POSTGRES_TEST_USERNAME || 23 | bail('POSTGRES_TEST_USERNAME is not defined'), 24 | password: 25 | process.env.POSTGRES_TEST_PASSWORD || 26 | bail('POSTGRES_TEST_PASSWORD is not defined'), 27 | database: 28 | process.env.POSTGRES_TEST_DATABASE || 29 | bail('POSTGRES_TEST_DATABASE is not defined'), 30 | }, 31 | __LOG_LEVEL__: process.env.LOG_LEVEL || 'info', 32 | }, 33 | } 34 | -------------------------------------------------------------------------------- /packages/indexer-cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@graphprotocol/indexer-cli", 3 | "version": "0.24.3", 4 | "description": "Indexer CLI for The Graph Network", 5 | "main": "./dist/cli.js", 6 | "files": [ 7 | "/bin", 8 | "/dist" 9 | ], 10 | "repository": "https://github.com/graphprotocol/cli", 11 | "author": "The Graph Foundation", 12 | "license": "MIT", 13 | "bin": { 14 | "graph-indexer": "bin/graph-indexer" 15 | }, 16 | "scripts": { 17 | "format": "prettier --write 'src/**/*.ts'", 18 | "lint": "eslint . --ext .ts,.tsx --fix", 19 | "compile": "tsc --build", 20 | "prepare": "yarn format && yarn lint && yarn compile", 21 | "disputes": "yarn prepare && ./dist/cli.js indexer disputes get", 22 | "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo", 23 | "test": "jest --colors --verbose --runInBand --detectOpenHandles", 24 | "test:ci": "LOG_LEVEL=error jest --verbose --runInBand --ci", 25 | "test:debug": "LOG_LEVEL=debug jest --runInBand --detectOpenHandles --verbose", 26 | "test:watch": "jest --watch --detectOpenHandles --verbose" 27 | }, 28 | "dependencies": { 29 | "@graphprotocol/common-ts": "2.0.11", 30 | "@graphprotocol/indexer-common": "^0.24.3", 31 | "@iarna/toml": "2.2.5", 32 | "@thi.ng/iterators": "5.1.74", 33 | "@urql/core": "3.1.0", 34 | "chalk": "4.1.2", 35 | "env-paths": "2.2.1", 36 | "ethers": "5.7.0", 37 | "gluegun": "4.7.0", 38 | "graphql-tag": "2.12.6", 39 | "isomorphic-fetch": "3.0.0", 40 | "table": "6.7.5", 41 | "yaml": "1.10.2" 42 | }, 43 | "devDependencies": { 44 | "@types/isomorphic-fetch": "0.0.36", 45 | "@types/lodash.clonedeep": "^4.5.7", 46 | "@typescript-eslint/eslint-plugin": "6.7.0", 47 | "@typescript-eslint/parser": "6.7.0", 48 | "eslint": "8.49.0", 49 | "eslint-config-prettier": "8.5.0", 50 | "lodash.clonedeep": "^4.5.0", 51 | "prettier": "3.0.3", 52 | "ts-jest": "^29.2.5", 53 | "typescript": "5.2.2" 54 | }, 55 | "resolutions": { 56 | "ethers": "5.7.0", 57 | "sequelize": "6.33.0" 58 | }, 59 | "gitHead": "972ab96774007b2aee15b1da169d2ff4be9f9d27" 60 | } 61 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/cli.test.ts: -------------------------------------------------------------------------------- 1 | import { cliTest, setupMultiNetworks, teardown } from './util' 2 | import path from 'path' 3 | 4 | const baseDir = path.join(__dirname) 5 | 6 | describe('Indexer cli tests', () => { 7 | beforeEach(setupMultiNetworks) 8 | afterEach(teardown) 9 | 10 | describe('General', () => { 11 | cliTest('Indexer help', ['indexer', '--help'], 'references/indexer-help', { 12 | expectedExitCode: 255, 13 | cwd: baseDir, 14 | timeout: 10000, 15 | }) 16 | 17 | cliTest('Indexer (no args)', ['indexer'], 'references/indexer-help', { 18 | expectedExitCode: 255, 19 | cwd: baseDir, 20 | timeout: 10000, 21 | }) 22 | cliTest( 23 | 'Indexer connect - success', 24 | ['indexer', 'connect', 'http://127.0.0.1:18000'], 25 | 'references/indexer-connect', 26 | { 27 | expectedExitCode: 0, 28 | cwd: baseDir, 29 | timeout: 10000, 30 | }, 31 | ) 32 | // TODO: Test that connect should fail with helpful error message if incorrect port is provided or server isn't live 33 | }) 34 | }) 35 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/indexer/actions.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | cliTest, 3 | deleteFromAllTables, 4 | seedActions, 5 | setupMultiNetworks, 6 | teardown, 7 | } from '../util' 8 | import path from 'path' 9 | 10 | const baseDir = path.join(__dirname, '..') 11 | describe('Indexer actions tests', () => { 12 | describe('With indexer management server', () => { 13 | beforeAll(setupMultiNetworks) 14 | afterAll(teardown) 15 | beforeEach(seedActions) 16 | afterEach(deleteFromAllTables) 17 | describe('Actions help', () => { 18 | cliTest('Indexer actions', ['indexer', 'actions'], 'references/indexer-actions', { 19 | expectedExitCode: 255, 20 | cwd: baseDir, 21 | timeout: 10000, 22 | }) 23 | cliTest( 24 | 'Indexer actions help', 25 | ['indexer', 'actions', '--help'], 26 | 'references/indexer-actions', 27 | { 28 | expectedExitCode: 255, 29 | cwd: baseDir, 30 | timeout: 10000, 31 | }, 32 | ) 33 | cliTest( 34 | 'Indexer actions get', 35 | ['indexer', 'actions', 'get', 'all'], 36 | 'references/indexer-actions-get', 37 | { 38 | expectedExitCode: 0, 39 | cwd: baseDir, 40 | timeout: 10000, 41 | }, 42 | ) 43 | }) 44 | describe('Actions queue', () => { 45 | cliTest( 46 | 'Indexer actions get', 47 | ['indexer', 'actions', 'get', 'all'], 48 | 'references/indexer-actions-get', 49 | { 50 | expectedExitCode: 0, 51 | cwd: baseDir, 52 | timeout: 10000, 53 | }, 54 | ) 55 | cliTest( 56 | 'Indexer actions get - fields', 57 | ['indexer', 'actions', 'get', 'all', '--fields', 'id,type,deploymentID,status'], 58 | 'references/indexer-actions-get-fields', 59 | { 60 | expectedExitCode: 0, 61 | cwd: baseDir, 62 | timeout: 10000, 63 | }, 64 | ) 65 | cliTest( 66 | 'Indexer actions get - first', 67 | ['indexer', 'actions', 'get', '--first', '1'], 68 | 'references/indexer-actions-get-first', 69 | { 70 | expectedExitCode: 0, 71 | cwd: baseDir, 72 | timeout: 10000, 73 | }, 74 | ) 75 | cliTest( 76 | 'Indexer actions get - first + fields', 77 | [ 78 | 'indexer', 79 | 'actions', 80 | 'get', 81 | '--first', 82 | '1', 83 | '--fields', 84 | 'id,type,deploymentID,status', 85 | ], 86 | 'references/indexer-actions-get-first-fields', 87 | { 88 | expectedExitCode: 0, 89 | cwd: baseDir, 90 | timeout: 10000, 91 | }, 92 | ) 93 | }) 94 | }) 95 | }) 96 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/basic.agora: -------------------------------------------------------------------------------- 1 | default => 0.00025; 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/help.stdout: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graphprotocol/indexer/6d7a8cc1c4ceb70901ad5e4aa32fa4c9fbcf304f/packages/indexer-cli/src/__tests__/references/help.stdout -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-actions-get-fields.stdout: -------------------------------------------------------------------------------- 1 | ┌────┬────────────┬────────────────────────────────────────────────┬─────────┐ 2 | │ id │ type │ deploymentID │ status │ 3 | ├────┼────────────┼────────────────────────────────────────────────┼─────────┤ 4 | │ 2 │ unallocate │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ failed │ 5 | ├────┼────────────┼────────────────────────────────────────────────┼─────────┤ 6 | │ 1 │ allocate │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ success │ 7 | └────┴────────────┴────────────────────────────────────────────────┴─────────┘ 8 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-actions-get-first-fields.stdout: -------------------------------------------------------------------------------- 1 | ┌────┬────────────┬────────────────────────────────────────────────┬────────┐ 2 | │ id │ type │ deploymentID │ status │ 3 | ├────┼────────────┼────────────────────────────────────────────────┼────────┤ 4 | │ 2 │ unallocate │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ failed │ 5 | └────┴────────────┴────────────────────────────────────────────────┴────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-actions-get-first.stdout: -------------------------------------------------------------------------------- 1 | ┌────┬──────────────────┬────────────┬────────────────────────────────────────────────┬──────────────┬────────┬──────┬───────┬──────────┬────────┬────────┬───────────────┬─────────────┬────────┐ 2 | │ id │ protocolNetwork │ type │ deploymentID │ allocationID │ amount │ poi │ force │ priority │ status │ source │ failureReason │ transaction │ reason │ 3 | ├────┼──────────────────┼────────────┼────────────────────────────────────────────────┼──────────────┼────────┼──────┼───────┼──────────┼────────┼────────┼───────────────┼─────────────┼────────┤ 4 | │ 2 │ arbitrum-sepolia │ unallocate │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ null │ null │ null │ null │ 0 │ failed │ test │ null │ null │ test │ 5 | └────┴──────────────────┴────────────┴────────────────────────────────────────────────┴──────────────┴────────┴──────┴───────┴──────────┴────────┴────────┴───────────────┴─────────────┴────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-actions-get.stdout: -------------------------------------------------------------------------------- 1 | ┌────┬──────────────────┬────────────┬────────────────────────────────────────────────┬──────────────┬────────┬──────┬───────┬──────────┬─────────┬────────┬───────────────┬─────────────┬────────┐ 2 | │ id │ protocolNetwork │ type │ deploymentID │ allocationID │ amount │ poi │ force │ priority │ status │ source │ failureReason │ transaction │ reason │ 3 | ├────┼──────────────────┼────────────┼────────────────────────────────────────────────┼──────────────┼────────┼──────┼───────┼──────────┼─────────┼────────┼───────────────┼─────────────┼────────┤ 4 | │ 2 │ arbitrum-sepolia │ unallocate │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ null │ null │ null │ null │ 0 │ failed │ test │ null │ null │ test │ 5 | ├────┼──────────────────┼────────────┼────────────────────────────────────────────────┼──────────────┼────────┼──────┼───────┼──────────┼─────────┼────────┼───────────────┼─────────────┼────────┤ 6 | │ 1 │ arbitrum-sepolia │ allocate │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ null │ null │ null │ null │ 0 │ success │ test │ null │ null │ test │ 7 | └────┴──────────────────┴────────────┴────────────────────────────────────────────────┴──────────────┴────────┴──────┴───────┴──────────┴─────────┴────────┴───────────────┴─────────────┴────────┘ 8 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-actions.stdout: -------------------------------------------------------------------------------- 1 | Manage indexer actions 2 | 3 | indexer actions update Update one or more actions 4 | indexer actions queue Queue an action item 5 | indexer actions get List one or more actions 6 | indexer actions execute Execute approved items in the action queue 7 | indexer actions delete Delete one or many actions in the queue 8 | indexer actions cancel Cancel an item in the queue 9 | indexer actions approve Approve an action item 10 | indexer actions Manage indexer actions 11 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-connect.stdout: -------------------------------------------------------------------------------- 1 | Indexer management API URL configured as "http://127.0.0.1:18000" 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-all.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬─────────────────────┐ 2 | │ deployment │ model │ 3 | ├────────────────────────────────────────────────┼─────────────────────┤ 4 | │ QmZfeJYR86UARzp9HiXbURWunYgC9ywvPvoePNbuaATrEK │ default => 0.00025; │ 5 | ├────────────────────────────────────────────────┼─────────────────────┤ 6 | │ global │ default => 0.00025; │ 7 | └────────────────────────────────────────────────┴─────────────────────┘ 8 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-command-identifier.stdout: -------------------------------------------------------------------------------- 1 | Error: Invalid deployment ID "0x0000000000000000000000000000000000000000-0". Cost models currently must use deployment identifiers, please provide a valid deployment ID. 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-deployment-model-only.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬─────────────────────┐ 2 | │ deployment │ model │ 3 | ├────────────────────────────────────────────────┼─────────────────────┤ 4 | │ QmQ44hgrWWt3Qf2X9XEX2fPyTbmQbChxwNm5c1t4mhKpGt │ default => 0.00025; │ 5 | └────────────────────────────────────────────────┴─────────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-fallback-global.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬─────────────────────┐ 2 | │ deployment │ model │ 3 | ├────────────────────────────────────────────────┼─────────────────────┤ 4 | │ QmVqMeQUwvQ3XjzCYiMhRvQjRiQLGpVt8C3oHgvDi3agJ2 │ default => 0.00025; │ 5 | └────────────────────────────────────────────────┴─────────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-get-no-arg.stdout: -------------------------------------------------------------------------------- 1 | Error: No deployment ID provided. Must be a valid deployment ID 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-invalid-identifier-arg.stderr: -------------------------------------------------------------------------------- 1 | Error: Invalid subgraph identifier "Qmemememememe". Subgraph identifier should match 1 type of [deployment ID, subgraph ID, group identifier]. 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-model-deployment-no-arg.stdout: -------------------------------------------------------------------------------- 1 | Failed to load cost model from "undefined": The "path" argument must be of type string or an instance of Buffer or URL. Received undefined 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-model-deployment.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬─────────────────────┐ 2 | │ deployment │ model │ 3 | ├────────────────────────────────────────────────┼─────────────────────┤ 4 | │ QmXRpJW3qBuYaiBYHdhv8DF4bHDZhXBmh91MtrnhJfQ5Lk │ default => 0.00025; │ 5 | └────────────────────────────────────────────────┴─────────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-set-invalid-arg.stdout: -------------------------------------------------------------------------------- 1 | Indexing rule attribute 'allocatoinAmount' doesn't exist, please check spelling 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-set-no-arg.stdout: -------------------------------------------------------------------------------- 1 | Failed to load cost model from "undefined": The "path" argument must be of type string or an instance of Buffer or URL. Received undefined 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost-variables-deployment.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬─────────────────────┐ 2 | │ deployment │ model │ 3 | ├────────────────────────────────────────────────┼─────────────────────┤ 4 | │ QmQ44hgrWWt3Qf2X9XEX2fPyTbmQbChxwNm5c1t4mhKpGt │ default => 0.00025; │ 5 | └────────────────────────────────────────────────┴─────────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-cost.stdout: -------------------------------------------------------------------------------- 1 | Manage costing for subgraphs 2 | 3 | indexer cost set model Update a cost model 4 | indexer cost get Get cost models for one or all subgraphs 5 | indexer cost delete Remove one or many cost models 6 | indexer cost Manage costing for subgraphs 7 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-not-connected.stderr: -------------------------------------------------------------------------------- 1 | [Network] request to http://127.0.0.1:18000/ failed, reason: connect ECONNREFUSED 127.0.0.1:18000 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-output-format.stdout: -------------------------------------------------------------------------------- 1 | Invalid output format "josn" 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-always.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ always │ true │ true │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-deleted-offchain-success.stdout: -------------------------------------------------------------------------------- 1 | Deleted indexing rules for "QmZfeJYR86UARzp9HiXbURWunYgC9ywvPvoePNbuaATrEK" (deployment) on network: 'arbitrum-sepolia' 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-deleted-success.stdout: -------------------------------------------------------------------------------- 1 | Deleted indexing rules for "QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q" (deployment) on network: 'arbitrum-sepolia' 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-lifetime.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmZfeJYR86UARzp9HiXbURWunYgC9ywvPvoePNbuaATrEK │ deployment │ null │ 21 │ false │ null │ null │ null │ null │ null │ null │ null │ offchain │ true │ true │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-never.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ never │ true │ true │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-offchain.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmZfeJYR86UARzp9HiXbURWunYgC9ywvPvoePNbuaATrEK │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ offchain │ true │ true │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-rules.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ rules │ true │ true │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-safety.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmVEV7RA2U6BJT9Ssjxcfyrk4YQUnVqSRNX4TvYagjzh9h │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ rules │ true │ false │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-supported.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ QmVEV7RA2U6BJT9Ssjxcfyrk4YQUnVqSRNX4TvYagjzh9h │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ rules │ false │ true │ arbitrum-sepolia │ 5 | └────────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-deployment-yaml.stdout: -------------------------------------------------------------------------------- 1 | identifier: QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q 2 | identifierType: deployment 3 | allocationAmount: null 4 | allocationLifetime: null 5 | autoRenewal: true 6 | parallelAllocations: null 7 | maxAllocationPercentage: null 8 | minSignal: null 9 | maxSignal: null 10 | minStake: null 11 | minAverageQueryFees: null 12 | custom: null 13 | decisionBasis: rules 14 | requireSupported: true 15 | safety: true 16 | protocolNetwork: arbitrum-sepolia 17 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-global-rules.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ global │ group │ 0.01 │ null │ true │ null │ null │ 500.0 │ null │ null │ null │ null │ rules │ true │ true │ arbitrum-sepolia │ 5 | └────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-subgraph-offchain.stdout: -------------------------------------------------------------------------------- 1 | ┌──────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├──────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ 0x0000000000000000000000000000000000000000-1 │ subgraph │ 1,000.0 │ 12 │ true │ null │ null │ null │ null │ null │ null │ null │ offchain │ true │ true │ arbitrum-sepolia │ 5 | └──────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-subgraph-options.stdout: -------------------------------------------------------------------------------- 1 | ┌──────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├──────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ 0x0000000000000000000000000000000000000000-2 │ subgraph │ 1,000.0 │ 12 │ true │ null │ null │ null │ null │ null │ null │ null │ offchain │ true │ true │ arbitrum-sepolia │ 5 | └──────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rule-subgraph-rules.stdout: -------------------------------------------------------------------------------- 1 | ┌──────────────────────────────────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┬──────────────────┐ 2 | │ identifier │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ protocolNetwork │ 3 | ├──────────────────────────────────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┼──────────────────┤ 4 | │ 0x0000000000000000000000000000000000000000-0 │ subgraph │ 1,000.0 │ null │ true │ null │ null │ null │ null │ null │ null │ null │ rules │ true │ true │ arbitrum-sepolia │ 5 | └──────────────────────────────────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┴──────────────────┘ 6 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rules-command-no-args.stdout: -------------------------------------------------------------------------------- 1 | ┌────────────────────────────────────────────────┬──────────────────┬────────────────┬──────────────────┬────────────────────┬─────────────┬─────────────────────┬─────────────────────────┬───────────┬───────────┬──────────┬─────────────────────┬────────┬───────────────┬──────────────────┬────────┐ 2 | │ identifier │ protocolNetwork │ identifierType │ allocationAmount │ allocationLifetime │ autoRenewal │ parallelAllocations │ maxAllocationPercentage │ minSignal │ maxSignal │ minStake │ minAverageQueryFees │ custom │ decisionBasis │ requireSupported │ safety │ 3 | ├────────────────────────────────────────────────┼──────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┤ 4 | │ global │ arbitrum-sepolia │ group │ 0.01 │ null │ true │ null │ null │ 500.0 │ null │ null │ null │ null │ rules │ true │ true │ 5 | ├────────────────────────────────────────────────┼──────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┤ 6 | │ 0x0000000000000000000000000000000000000000-0 │ arbitrum-sepolia │ subgraph │ 1,000.0 │ null │ true │ null │ null │ null │ null │ null │ null │ null │ rules │ true │ true │ 7 | ├────────────────────────────────────────────────┼──────────────────┼────────────────┼──────────────────┼────────────────────┼─────────────┼─────────────────────┼─────────────────────────┼───────────┼───────────┼──────────┼─────────────────────┼────────┼───────────────┼──────────────────┼────────┤ 8 | │ QmSrf6VVPyg9NGdS1xhLmoosk3qZQaWhfoSTHE2H7sht6Q │ arbitrum-sepolia │ deployment │ null │ null │ true │ null │ null │ null │ null │ null │ null │ null │ rules │ true │ true │ 9 | └────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────────┴────────────────────┴─────────────┴─────────────────────┴─────────────────────────┴───────────┴───────────┴──────────┴─────────────────────┴────────┴───────────────┴──────────────────┴────────┘ 10 | Offchain sync list 11 | 0x0000000000000000000000000000000000000000-1,0x0000000000000000000000000000000000000000-2,QmZfeJYR86UARzp9HiXbURWunYgC9ywvPvoePNbuaATrEK 12 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rules-invalid-identifier-arg.stderr: -------------------------------------------------------------------------------- 1 | Error: Invalid subgraph identifier "Qmemememememe". Subgraph identifier should match 1 type of [deployment ID, subgraph ID, group identifier]. 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rules-invalid-set-arg.stdout: -------------------------------------------------------------------------------- 1 | Error: Indexing rule attribute 'allocationAmoewt' not supported, did you mean? 2 | - allocationAmount 3 | - allocationLifetime 4 | - maxAllocationPercentage 5 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rules-no-identifier.stderr: -------------------------------------------------------------------------------- 1 | Error: Invalid subgraph identifier "undefined". Subgraph identifier should match 1 type of [deployment ID, subgraph ID, group identifier]. 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rules-no-network.stderr: -------------------------------------------------------------------------------- 1 | Error: The option '--network' is required 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-rules.stdout: -------------------------------------------------------------------------------- 1 | Configure indexing rules 2 | 3 | indexer rules stop (never) Never index a deployment (and stop indexing it if necessary) 4 | indexer rules start (always) Always index a deployment (and start indexing it if necessary) 5 | indexer rules set Set one or more indexing rules 6 | indexer rules prepare (offchain) Offchain index a deployment (and start indexing it if necessary) 7 | indexer rules maybe Index a deployment based on rules 8 | indexer rules get Get one or more indexing rules 9 | indexer rules delete Remove one or many indexing rules 10 | indexer rules clear (reset) Clear one or more indexing rules 11 | indexer rules Configure indexing rules 12 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer-status-not-connected.stderr: -------------------------------------------------------------------------------- 1 | Error fetching status information: [Network] request to http://localhost:18000/ failed, reason: connect ECONNREFUSED 127.0.0.1:18000 2 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/__tests__/references/indexer.stdout: -------------------------------------------------------------------------------- 1 | Manage indexer configuration 2 | 3 | indexer status Check the status of an indexer 4 | indexer rules stop (never) Never index a deployment (and stop indexing it if necessary) 5 | indexer rules start (always) Always index a deployment (and start indexing it if necessary) 6 | indexer rules set Set one or more indexing rules 7 | indexer rules prepare (offchain) Offchain index a deployment (and start indexing it if necessary) 8 | indexer rules maybe Index a deployment based on rules 9 | indexer rules get Get one or more indexing rules 10 | indexer rules delete Remove one or many indexing rules 11 | indexer rules clear (reset) Clear one or more indexing rules 12 | indexer rules Configure indexing rules 13 | indexer disputes get Cross-check POIs submitted in the network 14 | indexer disputes Configure allocation POI monitoring 15 | indexer cost set model Update a cost model 16 | indexer cost get Get cost models and/or variables for one or all subgraphs 17 | indexer cost Manage costing for subgraphs 18 | indexer cost delete Remove one or many cost models 19 | indexer connect Connect to indexer management API 20 | indexer allocations reallocate Reallocate to subgraph deployment 21 | indexer allocations get List one or more allocations 22 | indexer allocations create Create an allocation 23 | indexer allocations close Close an allocation 24 | indexer allocations Manage indexer allocations 25 | indexer Manage indexer configuration 26 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/cli.ts: -------------------------------------------------------------------------------- 1 | import { build } from 'gluegun' 2 | 3 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 4 | const run = async (argv: any) => { 5 | const cli = build() 6 | .brand('graph-indexer') 7 | .help() 8 | .version() 9 | .src(__dirname) 10 | .defaultCommand() 11 | .create() 12 | 13 | return await cli.run(argv) 14 | } 15 | 16 | module.exports = { run } 17 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/client.ts: -------------------------------------------------------------------------------- 1 | import { createClient } from '@urql/core' 2 | import fetch from 'isomorphic-fetch' 3 | import { IndexerManagementClient } from '@graphprotocol/indexer-common' 4 | 5 | export const createIndexerManagementClient = async ({ 6 | url, 7 | }: { 8 | url: string 9 | }): Promise => { 10 | return createClient({ url, fetch }) as unknown as IndexerManagementClient 11 | } 12 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | 3 | module.exports = { 4 | name: 'indexer', 5 | alias: [], 6 | description: 'Manage indexer configuration', 7 | hidden: false, 8 | dashed: false, 9 | run: async (toolbox: GluegunToolbox) => { 10 | const { print } = toolbox 11 | print.info(toolbox.command?.description) 12 | print.printCommands(toolbox, ['indexer']) 13 | process.exitCode = -1 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/actions.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | 3 | module.exports = { 4 | name: 'actions', 5 | alias: [], 6 | description: 'Manage indexer actions', 7 | hidden: false, 8 | dashed: false, 9 | run: async (toolbox: GluegunToolbox) => { 10 | const { print } = toolbox 11 | print.info(toolbox.command?.description) 12 | print.printCommands(toolbox, ['indexer', 'actions']) 13 | process.exitCode = -1 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/actions/execute.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | import { resolveChainAlias } from '@graphprotocol/indexer-common' 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { fixParameters, printObjectOrArray } from '../../../command-helpers' 7 | import { executeApprovedActions } from '../../../actions' 8 | 9 | const HELP = ` 10 | ${chalk.bold('graph indexer actions execute approved')} [options] 11 | 12 | ${chalk.dim('Options:')} 13 | 14 | -h, --help Show usage information 15 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 16 | ` 17 | 18 | module.exports = { 19 | name: 'execute', 20 | alias: [], 21 | description: 'Execute approved items in the action queue', 22 | run: async (toolbox: GluegunToolbox) => { 23 | const { print, parameters } = toolbox 24 | 25 | const spinner = toolbox.print.spin('Execute approved actions') 26 | 27 | const { h, help, o, output } = parameters.options 28 | 29 | const [type] = fixParameters(parameters, { h, help }) || [] 30 | 31 | const outputFormat = o || output || 'table' 32 | const toHelp = help || h || undefined 33 | 34 | if (toHelp) { 35 | spinner.stopAndPersist({ symbol: '💁', text: HELP }) 36 | process.exitCode = 1 37 | return 38 | } 39 | 40 | if (!['approved'].includes(type)) { 41 | spinner.fail(`Invalid action type '${type}', must be one of ['approved']`) 42 | process.exitCode = 1 43 | return 44 | } 45 | 46 | if (!['json', 'yaml', 'table'].includes(outputFormat)) { 47 | spinner.fail( 48 | `Invalid output format "${outputFormat}", must be one of ['json', 'yaml', 'table']`, 49 | ) 50 | print.info(HELP) 51 | process.exitCode = 1 52 | return 53 | } 54 | 55 | try { 56 | const config = loadValidatedConfig() 57 | const client = await createIndexerManagementClient({ url: config.api }) 58 | 59 | const executedActions = await executeApprovedActions(client) 60 | 61 | spinner.succeed(`Executed approved actions`) 62 | 63 | // Format Actions 'protocolNetwork' field to display human-friendly chain aliases instead of CAIP2-IDs 64 | executedActions.forEach( 65 | action => (action.protocolNetwork = resolveChainAlias(action.protocolNetwork)), 66 | ) 67 | 68 | printObjectOrArray(print, outputFormat, executedActions, [ 69 | 'id', 70 | 'protocolNetwork', 71 | 'status', 72 | 'type', 73 | 'deploymentID', 74 | 'allocationID', 75 | 'amount', 76 | 'poi', 77 | 'force', 78 | 'priority', 79 | 'transaction', 80 | 'failureReason', 81 | 'source', 82 | 'reason', 83 | ]) 84 | } catch (error) { 85 | spinner.fail(error.toString()) 86 | process.exitCode = 1 87 | return 88 | } 89 | }, 90 | } 91 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/allocations.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | 3 | module.exports = { 4 | name: 'allocations', 5 | alias: [], 6 | description: 'Manage indexer allocations', 7 | hidden: false, 8 | dashed: false, 9 | run: async (toolbox: GluegunToolbox) => { 10 | const { print } = toolbox 11 | print.info(toolbox.command?.description) 12 | print.printCommands(toolbox, ['indexer', 'allocations']) 13 | process.exitCode = 1 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/allocations/collect.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { submitCollectReceiptsJob } from '../../../allocations' 7 | import { validateNetworkIdentifier } from '@graphprotocol/indexer-common' 8 | 9 | const HELP = ` 10 | ${chalk.bold('graph indexer allocations collect')} [options] 11 | 12 | ${chalk.dim('Options:')} 13 | 14 | -h, --help Show usage information 15 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 16 | 17 | ${chalk.dim('Networks:')} 18 | mainnet, arbitrum-one, sepolia or arbitrum sepolia 19 | ` 20 | 21 | module.exports = { 22 | name: 'collect', 23 | alias: [], 24 | description: 'Collect receipts for an allocation', 25 | run: async (toolbox: GluegunToolbox) => { 26 | const { print, parameters } = toolbox 27 | 28 | const spinner = toolbox.print.spin('Processing inputs') 29 | 30 | const { h, help, o, output } = parameters.options 31 | 32 | const outputFormat = o || output || 'table' 33 | const toHelp = help || h || undefined 34 | 35 | if (toHelp) { 36 | spinner.stopAndPersist({ symbol: '💁', text: HELP }) 37 | return 38 | } 39 | 40 | if (!['json', 'yaml', 'table'].includes(outputFormat)) { 41 | spinner.fail(`Invalid output format "${outputFormat}"`) 42 | process.exitCode = 1 43 | return 44 | } 45 | 46 | const [network, id] = parameters.array || [] 47 | 48 | if (id === undefined) { 49 | spinner.fail(`Missing required argument: 'id'`) 50 | print.info(HELP) 51 | process.exitCode = 1 52 | return 53 | } 54 | 55 | let protocolNetwork: string 56 | if (!network) { 57 | spinner.fail(`Missing required argument: 'network'`) 58 | print.info(HELP) 59 | process.exitCode = 1 60 | return 61 | } else { 62 | try { 63 | protocolNetwork = validateNetworkIdentifier(network) 64 | } catch (error) { 65 | spinner.fail(`Invalid value for argument 'network': '${network}' `) 66 | process.exitCode = 1 67 | return 68 | } 69 | } 70 | 71 | spinner.text = `Collecting receipts for allocation '${id}` 72 | try { 73 | const config = loadValidatedConfig() 74 | const client = await createIndexerManagementClient({ url: config.api }) 75 | await submitCollectReceiptsJob(client, id, protocolNetwork) 76 | 77 | spinner.succeed('Submitted collect receipts job') 78 | } catch (error) { 79 | spinner.fail(error.toString()) 80 | process.exitCode = 1 81 | return 82 | } 83 | }, 84 | } 85 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/connect.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | import { loadConfig, writeConfig } from '../../config' 4 | 5 | const HELP = ` 6 | ${chalk.bold('graph indexer connect')} 7 | 8 | ${chalk.dim('Options:')} 9 | 10 | -h, --help Show usage information 11 | ` 12 | 13 | module.exports = { 14 | name: 'connect', 15 | alias: [], 16 | description: 'Connect to indexer management API', 17 | run: async (toolbox: GluegunToolbox) => { 18 | const { print, parameters } = toolbox 19 | 20 | const { h, help } = toolbox.parameters.options 21 | const urlString = parameters.first 22 | 23 | if (help || h) { 24 | print.info(HELP) 25 | return 26 | } 27 | 28 | if (!urlString) { 29 | print.error('No indexer management API URL provided') 30 | print.info(HELP) 31 | process.exitCode = 1 32 | return 33 | } 34 | 35 | let url: URL 36 | try { 37 | url = new URL(urlString) 38 | } catch (e) { 39 | print.error(`Indexer management API URL "${urlString}" is invalid: ${e.message}`) 40 | process.exitCode = 1 41 | return 42 | } 43 | 44 | const config = loadConfig() 45 | config.api = url.toString() 46 | writeConfig(config) 47 | print.success(`Indexer management API URL configured as "${urlString}"`) 48 | }, 49 | } 50 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/cost.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | 3 | module.exports = { 4 | name: 'cost', 5 | alias: [], 6 | description: 'Manage costing for subgraphs', 7 | hidden: false, 8 | dashed: false, 9 | run: async (toolbox: GluegunToolbox) => { 10 | const { print } = toolbox 11 | print.info(toolbox.command?.description) 12 | print.printCommands(toolbox, ['indexer', 'cost']) 13 | process.exitCode = -1 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/cost/delete.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { fixParameters } from '../../../command-helpers' 7 | import { costModels, deleteCostModels, parseDeploymentID } from '../../../cost' 8 | import { CostModelAttributes } from '@graphprotocol/indexer-common' 9 | 10 | const HELP = ` 11 | ${chalk.bold('graph indexer cost delete')} [options] all 12 | ${chalk.bold('graph indexer cost delete')} [options] global 13 | ${chalk.bold('graph indexer cost delete')} [options] 14 | 15 | ${chalk.dim('Options:')} 16 | 17 | -h, --help Show usage information 18 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 19 | ` 20 | 21 | module.exports = { 22 | name: 'delete', 23 | alias: [], 24 | description: 'Remove one or many cost models', 25 | run: async (toolbox: GluegunToolbox) => { 26 | const { print, parameters } = toolbox 27 | 28 | const { h, help, o, output } = parameters.options 29 | const [rawDeployment] = fixParameters(parameters, { h, help }) || [] 30 | const outputFormat = o || output || 'table' 31 | 32 | if (help || h) { 33 | print.info(HELP) 34 | return 35 | } 36 | 37 | if (!['json', 'yaml', 'table'].includes(outputFormat)) { 38 | print.error(`Invalid output format "${outputFormat}"`) 39 | process.exitCode = 1 40 | return 41 | } 42 | 43 | try { 44 | const config = loadValidatedConfig() 45 | // Create indexer API client 46 | const client = await createIndexerManagementClient({ url: config.api }) 47 | const deployment = parseDeploymentID(rawDeployment) 48 | switch (deployment) { 49 | case 'all': { 50 | const deployments: string[] = (await costModels(client)) 51 | .filter((model): model is CostModelAttributes => !!model.deployment) 52 | .map(model => model.deployment.toString()) 53 | 54 | const numberDeleted = await deleteCostModels(client, deployments) 55 | print.success( 56 | `Deleted ${numberDeleted} cost model(s) for: \n${deployments.join('\n')}`, 57 | ) 58 | break 59 | } 60 | case 'global': { 61 | await deleteCostModels(client, [deployment]) 62 | print.success(`Deleted cost model for ${deployment}`) 63 | break 64 | } 65 | default: { 66 | await deleteCostModels(client, [deployment.bytes32]) 67 | print.success(`Deleted cost model for ${rawDeployment}`) 68 | } 69 | } 70 | } catch (error) { 71 | print.error(error.toString()) 72 | process.exitCode = 1 73 | return 74 | } 75 | }, 76 | } 77 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/cost/get.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { fixParameters } from '../../../command-helpers' 7 | import { costModel, costModels, parseDeploymentID, printCostModels } from '../../../cost' 8 | import { validateDeploymentID } from '@graphprotocol/indexer-common' 9 | 10 | const HELP = ` 11 | ${chalk.bold('graph indexer cost get')} [options] all 12 | ${chalk.bold('graph indexer cost get')} [options] 13 | 14 | ${chalk.bold('graph indexer cost get')} [options] model all 15 | ${chalk.bold('graph indexer cost get')} [options] model 16 | 17 | ${chalk.dim('Options:')} 18 | 19 | -h, --help Show usage information 20 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 21 | ` 22 | 23 | module.exports = { 24 | name: 'get', 25 | alias: [], 26 | description: 'Get cost models for one or all subgraphs', 27 | run: async (toolbox: GluegunToolbox) => { 28 | const { print, parameters } = toolbox 29 | 30 | const { h, help, merged, o, output } = parameters.options 31 | const [first, second] = fixParameters(parameters, { h, help, merged }) || [] 32 | const outputFormat = o || output || 'table' 33 | 34 | if (help || h) { 35 | print.info(HELP) 36 | return 37 | } 38 | 39 | if (!['json', 'yaml', 'table'].includes(outputFormat)) { 40 | print.error(`Invalid output format "${outputFormat}"`) 41 | process.exitCode = 1 42 | return 43 | } 44 | 45 | const fields = ['model'].includes(first) 46 | ? ['deployment', first] 47 | : ['deployment', 'model'] 48 | const rawDeployment = ['model'].includes(first) ? second : first 49 | 50 | try { 51 | if (rawDeployment !== 'all' && rawDeployment !== 'global') { 52 | await validateDeploymentID(rawDeployment) 53 | } 54 | } catch (error) { 55 | print.error(error.toString()) 56 | process.exitCode = 1 57 | return 58 | } 59 | 60 | const config = loadValidatedConfig() 61 | const deployment = parseDeploymentID(rawDeployment) 62 | 63 | // Create indexer API client 64 | const client = await createIndexerManagementClient({ url: config.api }) 65 | try { 66 | const costModelOrModels = 67 | deployment === 'all' 68 | ? await costModels(client) 69 | : await costModel(client, deployment) 70 | 71 | printCostModels(print, outputFormat, deployment, costModelOrModels, fields) 72 | } catch (error) { 73 | print.error(error.toString()) 74 | process.exitCode = 1 75 | } 76 | }, 77 | } 78 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/cost/set/model.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | import fs from 'fs' 4 | 5 | import { loadValidatedConfig } from '../../../../config' 6 | import { createIndexerManagementClient } from '../../../../client' 7 | import { fixParameters } from '../../../../command-helpers' 8 | import { 9 | parseCostModel, 10 | parseDeploymentID, 11 | printCostModels, 12 | setCostModel, 13 | } from '../../../../cost' 14 | import { validateDeploymentID } from '@graphprotocol/indexer-common' 15 | 16 | const HELP = ` 17 | ${chalk.bold('graph indexer cost set model')} [options] 18 | ${chalk.bold('graph indexer cost set model')} [options] global 19 | 20 | ${chalk.dim('Options:')} 21 | 22 | -h, --help Show usage information 23 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 24 | ` 25 | 26 | module.exports = { 27 | name: 'model', 28 | alias: [], 29 | description: 'Update a cost model', 30 | run: async (toolbox: GluegunToolbox) => { 31 | const { print, parameters } = toolbox 32 | 33 | const { h, help, merged, o, output } = parameters.options 34 | const [deployment, filename] = fixParameters(parameters, { h, help, merged }) || [] 35 | const outputFormat = o || output || 'table' 36 | 37 | if (help || h) { 38 | print.info(HELP) 39 | return 40 | } 41 | 42 | if (!['json', 'yaml', 'table'].includes(outputFormat)) { 43 | print.error(`Invalid output format "${outputFormat}"`) 44 | process.exitCode = 1 45 | return 46 | } 47 | 48 | if (deployment != 'global') { 49 | try { 50 | await validateDeploymentID(deployment) 51 | } catch (error) { 52 | print.error(error.toString()) 53 | process.exitCode = 1 54 | return 55 | } 56 | } 57 | 58 | let model = null 59 | try { 60 | model = fs.readFileSync(filename, 'utf8').trim() 61 | } catch (error) { 62 | print.error(`Failed to load cost model from "${filename}": ${error.message}`) 63 | process.exitCode = 1 64 | return 65 | } 66 | 67 | const config = loadValidatedConfig() 68 | let costModel = parseCostModel({ 69 | deployment, 70 | model, 71 | }) 72 | 73 | try { 74 | const client = await createIndexerManagementClient({ url: config.api }) 75 | costModel = await setCostModel(client, costModel) 76 | printCostModels(print, outputFormat, parseDeploymentID(deployment), costModel, []) 77 | } catch (error) { 78 | print.error(error.toString()) 79 | process.exitCode = 1 80 | } 81 | }, 82 | } 83 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/disputes.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | 3 | module.exports = { 4 | name: 'disputes', 5 | alias: [], 6 | description: 'Configure allocation POI monitoring', 7 | hidden: false, 8 | dashed: false, 9 | run: async (toolbox: GluegunToolbox) => { 10 | const { print } = toolbox 11 | print.info(toolbox.command?.description) 12 | print.printCommands(toolbox, ['indexer', 'disputes']) 13 | process.exitCode = -1 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/disputes/get.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { disputes, printDisputes } from '../../../disputes' 7 | import { parseOutputFormat, extractProtocolNetworkOption } from '../../../command-helpers' 8 | 9 | const HELP = ` 10 | ${chalk.bold( 11 | 'graph indexer disputes get', 12 | )} [options] potential 13 | 14 | ${chalk.dim('Options:')} 15 | 16 | -n, --network Filter by protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) 17 | -h, --help Show usage information 18 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 19 | ` 20 | 21 | module.exports = { 22 | name: 'get', 23 | alias: [], 24 | description: `Cross-check POIs submitted in the network`, 25 | run: async (toolbox: GluegunToolbox) => { 26 | const { print, parameters } = toolbox 27 | 28 | const { h, help, o, output } = parameters.options 29 | const [status, minAllocationClosedEpoch] = parameters.array || [] 30 | const outputFormat = parseOutputFormat(print, o || output || 'table') 31 | 32 | if (help || h) { 33 | print.info(HELP) 34 | return 35 | } 36 | if (!outputFormat) { 37 | process.exitCode = 1 38 | return 39 | } 40 | 41 | if (!status || status !== 'potential') { 42 | print.error(`Must provide a dispute status filter, options: 'potential'`) 43 | process.exitCode = 1 44 | return 45 | } 46 | 47 | if (minAllocationClosedEpoch === null || minAllocationClosedEpoch === undefined) { 48 | print.error(`No minimum epoch for closed allocations provided`) 49 | process.exitCode = 1 50 | return 51 | } 52 | 53 | const config = loadValidatedConfig() 54 | 55 | // Create indexer API client 56 | const client = await createIndexerManagementClient({ url: config.api }) 57 | try { 58 | const protocolNetwork = extractProtocolNetworkOption(parameters.options) 59 | const storedDisputes = await disputes( 60 | client, 61 | status, 62 | +minAllocationClosedEpoch, 63 | protocolNetwork, 64 | ) 65 | printDisputes(print, outputFormat, storedDisputes) 66 | } catch (error) { 67 | print.error(error.toString()) 68 | process.exitCode = 1 69 | } 70 | }, 71 | } 72 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/rules.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | 3 | module.exports = { 4 | name: 'rules', 5 | alias: [], 6 | description: 'Configure indexing rules', 7 | hidden: false, 8 | dashed: false, 9 | run: async (toolbox: GluegunToolbox) => { 10 | const { print } = toolbox 11 | print.info(toolbox.command?.description) 12 | print.printCommands(toolbox, ['indexer', 'rules']) 13 | process.exitCode = -1 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/rules/maybe.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { 7 | requireProtocolNetworkOption, 8 | fixParameters, 9 | parseOutputFormat, 10 | } from '../../../command-helpers' 11 | import { IndexingDecisionBasis, processIdentifier } from '@graphprotocol/indexer-common' 12 | import { setIndexingRule, displayRules, parseIndexingRule } from '../../../rules' 13 | 14 | const HELP = ` 15 | ${chalk.bold('graph indexer rules maybe')} [options] global 16 | ${chalk.bold('graph indexer rules maybe')} [options] 17 | 18 | ${chalk.dim('Options:')} 19 | 20 | -h, --help Show usage information 21 | -n, --network [Required] the rule's protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) 22 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 23 | ` 24 | 25 | module.exports = { 26 | name: 'maybe', 27 | alias: [], 28 | description: 'Index a deployment based on rules', 29 | run: async (toolbox: GluegunToolbox) => { 30 | const { print, parameters } = toolbox 31 | 32 | const { h, help, o, output } = parameters.options 33 | const [id] = fixParameters(parameters, { h, help }) || [] 34 | const outputFormat = parseOutputFormat(print, o || output || 'table') 35 | 36 | if (help || h) { 37 | print.info(HELP) 38 | return 39 | } 40 | if (!outputFormat) { 41 | process.exitCode = 1 42 | return 43 | } 44 | 45 | const config = loadValidatedConfig() 46 | 47 | try { 48 | const protocolNetwork = requireProtocolNetworkOption(parameters.options) 49 | const [identifier, identifierType] = await processIdentifier(id, { 50 | all: false, 51 | global: true, 52 | }) 53 | 54 | const inputRule = parseIndexingRule({ 55 | identifier, 56 | identifierType, 57 | decisionBasis: IndexingDecisionBasis.RULES, 58 | protocolNetwork, 59 | }) 60 | 61 | const client = await createIndexerManagementClient({ url: config.api }) 62 | const rule = await setIndexingRule(client, inputRule) 63 | print.info(displayRules(outputFormat, identifier, rule, [])) 64 | } catch (error) { 65 | print.error(error.toString()) 66 | process.exitCode = 1 67 | } 68 | }, 69 | } 70 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/rules/offchain.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { 7 | requireProtocolNetworkOption, 8 | fixParameters, 9 | parseOutputFormat, 10 | } from '../../../command-helpers' 11 | import { IndexingDecisionBasis, processIdentifier } from '@graphprotocol/indexer-common' 12 | import { setIndexingRule, displayRules, parseIndexingRule } from '../../../rules' 13 | 14 | const HELP = ` 15 | ${chalk.bold('graph indexer rules offchain')} [options] global 16 | ${chalk.bold('graph indexer rules offchain')} [options] 17 | ${chalk.bold('graph indexer rules prepare')} [options] global 18 | ${chalk.bold('graph indexer rules prepare')} [options] 19 | 20 | ${chalk.dim('Options:')} 21 | 22 | , -h, --help Show usage information 23 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 24 | ` 25 | 26 | module.exports = { 27 | name: 'prepare', 28 | alias: ['offchain'], 29 | description: 'Offchain index a deployment (and start indexing it if necessary)', 30 | run: async (toolbox: GluegunToolbox) => { 31 | const { print, parameters } = toolbox 32 | 33 | const { h, help, o, output } = parameters.options 34 | const [id] = fixParameters(parameters, { h, help }) || [] 35 | const outputFormat = parseOutputFormat(print, o || output || 'table') 36 | 37 | if (help || h) { 38 | print.info(HELP) 39 | return 40 | } 41 | if (!outputFormat) { 42 | process.exitCode = 1 43 | return 44 | } 45 | 46 | const config = loadValidatedConfig() 47 | 48 | try { 49 | const protocolNetwork = requireProtocolNetworkOption(parameters.options) 50 | const [identifier, identifierType] = await processIdentifier(id, { 51 | all: false, 52 | global: true, 53 | }) 54 | 55 | const inputRule = parseIndexingRule({ 56 | identifier, 57 | identifierType, 58 | protocolNetwork, 59 | decisionBasis: IndexingDecisionBasis.OFFCHAIN, 60 | }) 61 | 62 | // Update the indexing rule according to the key/value pairs 63 | const client = await createIndexerManagementClient({ url: config.api }) 64 | const rule = await setIndexingRule(client, inputRule) 65 | print.info(displayRules(outputFormat, identifier, rule, [])) 66 | } catch (error) { 67 | print.error(error.toString()) 68 | process.exitCode = 1 69 | } 70 | }, 71 | } 72 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/rules/start.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { 7 | requireProtocolNetworkOption, 8 | fixParameters, 9 | parseOutputFormat, 10 | } from '../../../command-helpers' 11 | import { IndexingDecisionBasis, processIdentifier } from '@graphprotocol/indexer-common' 12 | import { setIndexingRule, displayRules, parseIndexingRule } from '../../../rules' 13 | 14 | const HELP = ` 15 | ${chalk.bold('graph indexer rules start')} [options] global 16 | ${chalk.bold('graph indexer rules start')} [options] 17 | ${chalk.bold('graph indexer rules always')} [options] global 18 | ${chalk.bold('graph indexer rules always')} [options] 19 | 20 | ${chalk.dim('Options:')} 21 | 22 | -h, --help Show usage information 23 | -n, --network [Required] the rule's protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) 24 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 25 | ` 26 | 27 | module.exports = { 28 | name: 'start', 29 | alias: ['always'], 30 | description: 'Always index a deployment (and start indexing it if necessary)', 31 | run: async (toolbox: GluegunToolbox) => { 32 | const { print, parameters } = toolbox 33 | 34 | const { h, help, o, output } = parameters.options 35 | const [id] = fixParameters(parameters, { h, help }) || [] 36 | const outputFormat = parseOutputFormat(print, o || output || 'table') 37 | 38 | if (help || h) { 39 | print.info(HELP) 40 | return 41 | } 42 | if (!outputFormat) { 43 | process.exitCode = 1 44 | return 45 | } 46 | 47 | const config = loadValidatedConfig() 48 | 49 | try { 50 | const protocolNetwork = requireProtocolNetworkOption(parameters.options) 51 | const [identifier, identifierType] = await processIdentifier(id, { 52 | all: false, 53 | global: true, 54 | }) 55 | 56 | const inputRule = parseIndexingRule({ 57 | identifier, 58 | identifierType, 59 | decisionBasis: IndexingDecisionBasis.ALWAYS, 60 | protocolNetwork, 61 | }) 62 | 63 | // Update the indexing rule according to the key/value pairs 64 | const client = await createIndexerManagementClient({ url: config.api }) 65 | const rule = await setIndexingRule(client, inputRule) 66 | print.info(displayRules(outputFormat, identifier, rule, [])) 67 | } catch (error) { 68 | print.error(error.toString()) 69 | process.exitCode = 1 70 | } 71 | }, 72 | } 73 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/commands/indexer/rules/stop.ts: -------------------------------------------------------------------------------- 1 | import { GluegunToolbox } from 'gluegun' 2 | import chalk from 'chalk' 3 | 4 | import { loadValidatedConfig } from '../../../config' 5 | import { createIndexerManagementClient } from '../../../client' 6 | import { 7 | requireProtocolNetworkOption, 8 | fixParameters, 9 | parseOutputFormat, 10 | } from '../../../command-helpers' 11 | import { IndexingDecisionBasis, processIdentifier } from '@graphprotocol/indexer-common' 12 | import { setIndexingRule, displayRules, parseIndexingRule } from '../../../rules' 13 | 14 | const HELP = ` 15 | ${chalk.bold('graph indexer rules stop')} [options] global 16 | ${chalk.bold('graph indexer rules stop')} [options] 17 | ${chalk.bold('graph indexer rules never')} [options] global 18 | ${chalk.bold('graph indexer rules never')} [options] 19 | 20 | ${chalk.dim('Options:')} 21 | 22 | -h, --help Show usage information 23 | -n, --network [Required] the rule's protocol network (mainnet, arbitrum-one, sepolia, arbitrum-sepolia) 24 | -o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML 25 | ` 26 | 27 | module.exports = { 28 | name: 'stop', 29 | alias: ['never'], 30 | description: 'Never index a deployment (and stop indexing it if necessary)', 31 | run: async (toolbox: GluegunToolbox) => { 32 | const { print, parameters } = toolbox 33 | 34 | const { h, help, o, output } = parameters.options 35 | const [id] = fixParameters(parameters, { h, help }) || [] 36 | const outputFormat = parseOutputFormat(print, o || output || 'table') 37 | 38 | if (help || h) { 39 | print.info(HELP) 40 | return 41 | } 42 | if (!outputFormat) { 43 | process.exitCode = 1 44 | return 45 | } 46 | 47 | const config = loadValidatedConfig() 48 | 49 | try { 50 | const protocolNetwork = requireProtocolNetworkOption(parameters.options) 51 | const [identifier, identifierType] = await processIdentifier(id, { 52 | all: false, 53 | global: true, 54 | }) 55 | 56 | const inputRule = parseIndexingRule({ 57 | identifier, 58 | identifierType, 59 | decisionBasis: IndexingDecisionBasis.NEVER, 60 | protocolNetwork, 61 | }) 62 | 63 | const client = await createIndexerManagementClient({ url: config.api }) 64 | const rule = await setIndexingRule(client, inputRule) 65 | print.info(displayRules(outputFormat, identifier, rule, [])) 66 | } catch (error) { 67 | print.error(error.toString()) 68 | process.exitCode = 1 69 | } 70 | }, 71 | } 72 | -------------------------------------------------------------------------------- /packages/indexer-cli/src/config.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import path from 'path' 3 | import envPaths from 'env-paths' 4 | import toml, { JsonMap } from '@iarna/toml' 5 | import { print } from 'gluegun' 6 | 7 | export interface UnvalidatedIndexingConfig { 8 | api?: string 9 | } 10 | 11 | export interface IndexingConfig { 12 | api: string 13 | } 14 | 15 | const DEFAULT_CONFIG = {} 16 | 17 | const CONFIG_FILE = path.join( 18 | envPaths('graph-cli', { suffix: '' }).config, 19 | 'indexing.toml', 20 | ) 21 | 22 | export const loadConfig = (): UnvalidatedIndexingConfig => { 23 | if (fs.existsSync(CONFIG_FILE)) { 24 | return toml.parse(fs.readFileSync(CONFIG_FILE, 'utf-8')) 25 | } else { 26 | return DEFAULT_CONFIG 27 | } 28 | } 29 | 30 | export const loadValidatedConfig = (): IndexingConfig => { 31 | const config = loadConfig() 32 | const errors = [] 33 | if (!config.api) { 34 | errors.push(`- 'api' is not set. Please run 'graph indexer connect ' first`) 35 | } 36 | 37 | if (errors.length > 0) { 38 | print.error(`Failed to load indexer CLI configuration:`) 39 | print.error(errors.join('\n')) 40 | process.exit(1) 41 | } 42 | 43 | return config as IndexingConfig 44 | } 45 | 46 | export const writeConfig = (config: UnvalidatedIndexingConfig): void => { 47 | if (!fs.existsSync(path.dirname(CONFIG_FILE))) { 48 | fs.mkdirSync(path.dirname(CONFIG_FILE), { recursive: true }) 49 | } 50 | fs.writeFileSync(CONFIG_FILE, toml.stringify(config as JsonMap), { encoding: 'utf-8' }) 51 | } 52 | -------------------------------------------------------------------------------- /packages/indexer-cli/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "rootDir": "src", 5 | "outDir": "dist", 6 | "target": "es2020", 7 | "module": "commonjs", 8 | "composite": false, 9 | "declaration": false, 10 | "sourceMap": false, 11 | "esModuleInterop": true, 12 | "strict": true, 13 | "lib": ["esnext", "es2015", "es6", "esnext.asynciterable", "dom", "ES2020.Promise"] 14 | }, 15 | "include": [ 16 | "src/*.ts", 17 | "src/commands/*.ts", 18 | "src/commands/**/*.ts", 19 | "src/commands/**/**/*.ts" 20 | ], 21 | "exclude": ["src/commands/__tests__/*.ts"], 22 | "references": [{ "path": "../indexer-common" }] 23 | } 24 | -------------------------------------------------------------------------------- /packages/indexer-common/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage -------------------------------------------------------------------------------- /packages/indexer-common/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: false, 3 | parser: '@typescript-eslint/parser', 4 | plugins: ['@typescript-eslint'], 5 | extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'] 6 | } 7 | -------------------------------------------------------------------------------- /packages/indexer-common/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "trailingComma": "all", 4 | "printWidth": 90, 5 | "singleQuote": true 6 | } 7 | -------------------------------------------------------------------------------- /packages/indexer-common/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 The Graph Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /packages/indexer-common/README.md: -------------------------------------------------------------------------------- 1 | # Common Indexer Library for The Graph Network 2 | 3 | This library includes code that is shared between the different indexer packages 4 | in this repository. 5 | 6 | # Copyright 7 | 8 | Copyright © 2020 The Graph Foundation 9 | 10 | Licensed under the [MIT license](LICENSE). 11 | -------------------------------------------------------------------------------- /packages/indexer-common/jest.config.js: -------------------------------------------------------------------------------- 1 | const bail = (s) => { 2 | throw new Error(s) 3 | } 4 | 5 | // until we find a way to avoid `punycode` we suppress the warnings in tests 6 | process.env.NODE_NO_WARNINGS = '1' 7 | 8 | module.exports = { 9 | collectCoverage: true, 10 | preset: 'ts-jest', 11 | forceExit: true, 12 | testEnvironment: 'node', 13 | testPathIgnorePatterns: ['/node_modules/', '/dist/', '/.yalc', 'util.ts'], 14 | transformIgnorePatterns: ['!node_modules/'], 15 | globals: { 16 | __DATABASE__: { 17 | host: process.env.POSTGRES_TEST_HOST || bail('POSTGRES_TEST_HOST is not defined'), 18 | port: parseInt(process.env.POSTGRES_TEST_PORT || '5432'), 19 | username: 20 | process.env.POSTGRES_TEST_USERNAME || 21 | bail('POSTGRES_TEST_USERNAME is not defined'), 22 | password: 23 | process.env.POSTGRES_TEST_PASSWORD || 24 | bail('POSTGRES_TEST_PASSWORD is not defined'), 25 | database: 26 | process.env.POSTGRES_TEST_DATABASE || 27 | bail('POSTGRES_TEST_DATABASE is not defined'), 28 | }, 29 | __LOG_LEVEL__: process.env.LOG_LEVEL || 'info', 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/invalid-base58.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: mainnet 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | transactionMonitoring: 22 | gasIncreaseTimeout: 10 23 | gasIncreaseFactor: 10 24 | baseFeePerGasMax: 10 25 | maxTransactionAttempts: 10 26 | subgraphs: 27 | networkSubgraph: 28 | deployment: abcdefg # <-- invalid base58 field 29 | epochSubgraph: 30 | url: http://subgraph 31 | tapSubgraph: 32 | url: http://subgraph 33 | networkProvider: 34 | url: http://provider 35 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/invalid-epoch-subgraph.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: mainnet 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | transactionMonitoring: 22 | gasIncreaseTimeout: 10 23 | gasIncreaseFactor: 10 24 | baseFeePerGasMax: 10 25 | maxTransactionAttempts: 10 26 | subgraphs: 27 | networkSubgraph: 28 | url: http://subgraph 29 | epochSubgraph: 30 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 31 | # <-- Missing `url` field 32 | tapSubgraph: 33 | url: http://subgraph 34 | networkProvider: 35 | url: http://provider 36 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/invalid-extra-field.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: mainnet 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | invalidExtraField: abcd # <-- invalid extra field 22 | transactionMonitoring: 23 | gasIncreaseTimeout: 10 24 | gasIncreaseFactor: 10 25 | baseFeePerGasMax: 10 26 | maxTransactionAttempts: 10 27 | subgraphs: 28 | networkSubgraph: 29 | url: http://subgraph 30 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 31 | epochSubgraph: 32 | url: http://subgraph 33 | tapSubgraph: 34 | url: http://subgraph 35 | networkProvider: 36 | url: http://provider 37 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/invalid-missing-field.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: mainnet 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | # missing indexer address field 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | transactionMonitoring: 22 | gasIncreaseTimeout: 10 23 | gasIncreaseFactor: 10 24 | baseFeePerGasMax: 10 25 | maxTransactionAttempts: 10 26 | subgraphs: 27 | networkSubgraph: 28 | url: http://subgraph 29 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 30 | epochSubgraph: 31 | url: http://subgraph 32 | tapSubgraph: 33 | url: http://subgraph 34 | networkProvider: 35 | url: http://provider 36 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/invalid-negative-max-block-distance.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: mainnet 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | transactionMonitoring: 22 | gasIncreaseTimeout: 10 23 | gasIncreaseFactor: 10 24 | baseFeePerGasMax: 10 25 | maxTransactionAttempts: 10 26 | subgraphs: 27 | maxBlockDistance: -10 28 | networkSubgraph: 29 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 30 | epochSubgraph: 31 | url: http://subgraph 32 | tapSubgraph: 33 | url: http://subgraph 34 | networkProvider: 35 | url: http://provider 36 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/invalid-network-identifier.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: invalid # <-- invalid network identifier 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | transactionMonitoring: 22 | gasIncreaseTimeout: 10 23 | gasIncreaseFactor: 10 24 | baseFeePerGasMax: 10 25 | maxTransactionAttempts: 10 26 | subgraphs: 27 | networkSubgraph: 28 | url: http://subgraph 29 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 30 | epochSubgraph: 31 | url: http://subgraph 32 | tapSubgraph: 33 | url: http://subgraph 34 | networkProvider: 35 | url: http://provider 36 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/valid-missing.yml: -------------------------------------------------------------------------------- 1 | # this file is missing the whole `transactionMonitoring` entry, and is still valid 2 | networkIdentifier: mainnet 3 | gateway: 4 | url: http://gateway 5 | indexerOptions: 6 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 7 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 8 | url: http://indexer 9 | geoCoordinates: [25.1, -71.2] 10 | restakeRewards: true 11 | rebateClaimThreshold: 400 12 | rebateClaimBatchThreshold: 5000 13 | rebateClaimMaxBatchSize: 10 14 | poiDisputeMonitoring: false 15 | poiDisputableEpochs: 5 16 | defaultAllocationAmount: 0.05 17 | voucherRedemptionThreshold: 2 18 | voucherRedemptionBatchThreshold: 2000 19 | voucherRedemptionMaxBatchSize: 15 20 | allocationManagementMode: "auto" 21 | autoAllocationMinBatchSize: 20 22 | subgraphs: 23 | networkSubgraph: 24 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 25 | epochSubgraph: 26 | url: http://subgraph 27 | tapSubgraph: 28 | url: http://subgraph 29 | networkProvider: 30 | url: http://provider 31 | -------------------------------------------------------------------------------- /packages/indexer-common/src/__tests__/network-specification-files/valid.yml: -------------------------------------------------------------------------------- 1 | networkIdentifier: mainnet 2 | gateway: 3 | url: http://gateway 4 | indexerOptions: 5 | address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448" 6 | mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport 7 | url: http://indexer 8 | geoCoordinates: [25.1, -71.2] 9 | restakeRewards: true 10 | rebateClaimThreshold: 400 11 | rebateClaimBatchThreshold: 5000 12 | rebateClaimMaxBatchSize: 10 13 | poiDisputeMonitoring: false 14 | poiDisputableEpochs: 5 15 | defaultAllocationAmount: 0.05 16 | voucherRedemptionThreshold: 2 17 | voucherRedemptionBatchThreshold: 2000 18 | voucherRedemptionMaxBatchSize: 15 19 | allocationManagementMode: "auto" 20 | autoAllocationMinBatchSize: 20 21 | transactionMonitoring: 22 | gasIncreaseTimeout: 10 23 | gasIncreaseFactor: 10 24 | baseFeePerGasMax: 10 25 | maxTransactionAttempts: 10 26 | subgraphs: 27 | networkSubgraph: 28 | deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB 29 | epochSubgraph: 30 | url: http://subgraph 31 | tapSubgraph: 32 | url: http://subgraph 33 | networkProvider: 34 | url: http://provider 35 | -------------------------------------------------------------------------------- /packages/indexer-common/src/allocations/__tests__/escrow-accounts.test.ts: -------------------------------------------------------------------------------- 1 | import { Address, toAddress } from '@graphprotocol/common-ts' 2 | import { EscrowAccountResponse, EscrowAccounts } from '../escrow-accounts' 3 | 4 | const timeout = 30000 5 | 6 | const SENDER_ADDRESS_1 = toAddress('ffcf8fdee72ac11b5c542428b35eef5769c409f0') 7 | const SENDER_ADDRESS_2 = toAddress('dead47df40c29949a75a6693c77834c00b8ad624') 8 | const SENDER_ADDRESS_3 = toAddress('6aea8894b5ab5a36cdc2d8be9290046801dd5fed') 9 | 10 | describe('EscrowAccounts', () => { 11 | test( 12 | 'fromResponse should create correctly EscrowAccount', 13 | () => { 14 | const response: EscrowAccountResponse = { 15 | escrowAccounts: [ 16 | { 17 | sender: { 18 | id: SENDER_ADDRESS_1, 19 | }, 20 | balance: '1000', 21 | }, 22 | { 23 | sender: { 24 | id: SENDER_ADDRESS_2, 25 | }, 26 | balance: '2000', 27 | }, 28 | ], 29 | } 30 | 31 | const escrowAccounts = EscrowAccounts.fromResponse(response) 32 | 33 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_1)).toEqual(1000n) 34 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_2)).toEqual(2000n) 35 | expect(() => escrowAccounts.getBalanceForSender(SENDER_ADDRESS_3)).toThrowError() 36 | }, 37 | timeout, 38 | ) 39 | test('test subtractSenderBalance', () => { 40 | const balances = new Map() 41 | balances.set(SENDER_ADDRESS_1, 1000n) 42 | balances.set(SENDER_ADDRESS_2, 1000n) 43 | balances.set(SENDER_ADDRESS_3, 1000n) 44 | const escrowAccounts = new EscrowAccounts(balances) 45 | 46 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_1)).toEqual(1000n) 47 | 48 | escrowAccounts.subtractSenderBalance(SENDER_ADDRESS_1, 100n) 49 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_1)).toEqual(900n) 50 | 51 | escrowAccounts.subtractSenderBalance(SENDER_ADDRESS_1, 100n) 52 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_1)).toEqual(800n) 53 | 54 | escrowAccounts.subtractSenderBalance(SENDER_ADDRESS_1, 600n) 55 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_1)).toEqual(200n) 56 | 57 | expect(() => 58 | escrowAccounts.subtractSenderBalance(SENDER_ADDRESS_1, 400n), 59 | ).toThrowError() 60 | 61 | escrowAccounts.subtractSenderBalance(SENDER_ADDRESS_1, 200n) 62 | 63 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_1)).toEqual(0n) 64 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_2)).toEqual(1000n) 65 | expect(escrowAccounts.getBalanceForSender(SENDER_ADDRESS_3)).toEqual(1000n) 66 | }) 67 | }) 68 | -------------------------------------------------------------------------------- /packages/indexer-common/src/allocations/escrow-accounts.ts: -------------------------------------------------------------------------------- 1 | import { Address, toAddress } from '@graphprotocol/common-ts' 2 | import { SubgraphClient } from '../subgraph-client' 3 | import gql from 'graphql-tag' 4 | 5 | type U256 = bigint 6 | 7 | export type EscrowAccountResponse = { 8 | escrowAccounts: { 9 | balance: string 10 | sender: { 11 | id: string 12 | } 13 | }[] 14 | } 15 | 16 | export class EscrowAccounts { 17 | constructor(private sendersBalances: Map) {} 18 | 19 | getBalanceForSender(sender: Address): U256 { 20 | const balance = this.sendersBalances.get(sender) 21 | if (balance === undefined) { 22 | throw new Error(`No balance found for sender: ${sender}`) 23 | } 24 | return balance 25 | } 26 | 27 | subtractSenderBalance(sender: Address, ravValue: U256) { 28 | const balance = this.getBalanceForSender(sender) 29 | if (balance < ravValue) { 30 | throw new Error(`Negative balances are not allowed`) 31 | } 32 | const newBalance = balance - ravValue 33 | this.sendersBalances.set(sender, newBalance) 34 | } 35 | 36 | static fromResponse(response: EscrowAccountResponse): EscrowAccounts { 37 | const sendersBalances = new Map() 38 | response.escrowAccounts.forEach((account) => { 39 | sendersBalances.set(toAddress(account.sender.id), BigInt(account.balance)) 40 | }) 41 | 42 | return new EscrowAccounts(sendersBalances) 43 | } 44 | } 45 | 46 | export const getEscrowAccounts = async ( 47 | tapSubgraph: SubgraphClient, 48 | indexer: Address, 49 | ): Promise => { 50 | const result = await tapSubgraph.query( 51 | gql` 52 | query EscrowAccountQuery($indexer: ID!) { 53 | escrowAccounts(where: { receiver_: { id: $indexer } }) { 54 | balance 55 | sender { 56 | id 57 | } 58 | } 59 | } 60 | `, 61 | { indexer }, 62 | ) 63 | if (!result.data) { 64 | throw `There was an error while querying Tap Subgraph. Errors: ${result.error}` 65 | } 66 | return EscrowAccounts.fromResponse(result.data) 67 | } 68 | -------------------------------------------------------------------------------- /packages/indexer-common/src/allocations/index.ts: -------------------------------------------------------------------------------- 1 | export * from './escrow-accounts' 2 | export * from './keys' 3 | export * from './query-fees' 4 | export * from './tap-collector' 5 | export * from './monitor' 6 | export * from './types' 7 | -------------------------------------------------------------------------------- /packages/indexer-common/src/allocations/types.ts: -------------------------------------------------------------------------------- 1 | import { BigNumber } from 'ethers' 2 | import { SubgraphClient, SubgraphDeployment } from '@graphprotocol/indexer-common' 3 | 4 | import { Logger, Address } from '@graphprotocol/common-ts' 5 | 6 | export interface Allocation { 7 | id: Address 8 | status: AllocationStatus 9 | subgraphDeployment: SubgraphDeployment 10 | indexer: Address 11 | allocatedTokens: BigNumber 12 | createdAtEpoch: number 13 | createdAtBlockHash: string 14 | closedAtEpoch: number 15 | closedAtEpochStartBlockHash: string | undefined 16 | previousEpochStartBlockHash: string | undefined 17 | closedAtBlockHash: string 18 | poi: string | undefined 19 | queryFeeRebates: BigNumber | undefined 20 | queryFeesCollected: BigNumber | undefined 21 | } 22 | 23 | export enum AllocationStatus { 24 | NULL = 'Null', 25 | ACTIVE = 'Active', 26 | CLOSED = 'Closed', 27 | FINALIZED = 'Finalized', 28 | CLAIMED = 'Claimed', 29 | } 30 | 31 | export interface MonitorEligibleAllocationsOptions { 32 | indexer: Address 33 | logger: Logger 34 | networkSubgraph: SubgraphClient 35 | interval: number 36 | protocolNetwork: string 37 | } 38 | -------------------------------------------------------------------------------- /packages/indexer-common/src/async-cache.ts: -------------------------------------------------------------------------------- 1 | // Cache which avoids concurrently getting the same thing more than once. 2 | export class AsyncCache { 3 | private readonly _attempts: Map> = new Map() 4 | private readonly _fn: (k: K) => Promise 5 | 6 | constructor(fn: (k: K) => Promise) { 7 | this._fn = fn 8 | } 9 | 10 | get(k: K): Promise { 11 | const cached = this._attempts.get(k) 12 | if (cached) { 13 | return cached 14 | } 15 | 16 | // This can throw, even though we do not 17 | // await the promise. Because of this, 18 | // we need to construct p outside of 19 | // attempt. If p was constructed in the attempt 20 | // there is a subtle timing bug. 21 | // 1. The 'attempt' promise starts to be constructed. 22 | // 2. Within the attempt promise, calling this._fn(k) fails 23 | // before the await. This means that the catch is called 24 | // synchronously. 25 | // 3. Within the catch, we delete this._attempts[k]. 26 | // 4. 'attempt' is constructed successfully as a failed promise. 27 | // 5. Finally, this._attempts is set - but in a perpetually 28 | // failed state. 29 | // By constructing the promise here, if the construction 30 | // fails then it never ends up in _attempts and this throws 31 | // right away. 32 | const p = this._fn(k) 33 | 34 | // This shares concurrent attempts, but still retries on failure. 35 | const attempt = (async () => { 36 | try { 37 | return await p 38 | } catch (e) { 39 | // By removing the cached attempt we ensure this is retried 40 | this._attempts.delete(k) 41 | throw e 42 | } 43 | })() 44 | this._attempts.set(k, attempt) 45 | return attempt 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /packages/indexer-common/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './actions' 2 | export * from './allocations' 3 | export * from './async-cache' 4 | export * from './errors' 5 | export * from './indexer-management' 6 | export * from './graph-node' 7 | export * from './operator' 8 | export * from './network' 9 | export * from './query-fees' 10 | export * from './rules' 11 | export * from './subgraphs' 12 | export * from './subgraph-client' 13 | export * from './transactions' 14 | export * from './types' 15 | export * from './utils' 16 | export * from './parsers' 17 | export * as specification from './network-specification' 18 | export * from './multi-networks' 19 | export * from './sequential-timer' 20 | -------------------------------------------------------------------------------- /packages/indexer-common/src/indexer-management/index.ts: -------------------------------------------------------------------------------- 1 | export * from './actions' 2 | export * from './allocations' 3 | export * from './client' 4 | export * from './models' 5 | export * from './monitor' 6 | export * from './server' 7 | export * from './rules' 8 | export * from './types' 9 | -------------------------------------------------------------------------------- /packages/indexer-common/src/indexer-management/models/index.ts: -------------------------------------------------------------------------------- 1 | import { Sequelize } from 'sequelize' 2 | 3 | import { IndexingRuleModels, defineIndexingRuleModels } from './indexing-rule' 4 | import { CostModelModels, defineCostModelModels } from './cost-model' 5 | import { POIDisputeModels, definePOIDisputeModels } from './poi-dispute' 6 | import { ActionModels, defineActionModels } from './action' 7 | 8 | export * from './cost-model' 9 | export * from './indexing-rule' 10 | export * from './poi-dispute' 11 | export * from './action' 12 | 13 | export type IndexerManagementModels = IndexingRuleModels & 14 | CostModelModels & 15 | POIDisputeModels & 16 | ActionModels 17 | 18 | export const defineIndexerManagementModels = ( 19 | sequelize: Sequelize, 20 | ): IndexerManagementModels => 21 | Object.assign( 22 | {}, 23 | defineCostModelModels(sequelize), 24 | defineIndexingRuleModels(sequelize), 25 | definePOIDisputeModels(sequelize), 26 | defineActionModels(sequelize), 27 | ) 28 | -------------------------------------------------------------------------------- /packages/indexer-common/src/indexer-management/resolvers/utils.ts: -------------------------------------------------------------------------------- 1 | import { 2 | MultiNetworks, 3 | Network, 4 | validateNetworkIdentifier, 5 | } from '@graphprotocol/indexer-common' 6 | 7 | export function extractNetwork( 8 | unvalidatedNetworkIdentifier: string, 9 | multiNetworks: MultiNetworks, 10 | ): Network { 11 | let networkIdentifier: string 12 | try { 13 | networkIdentifier = validateNetworkIdentifier(unvalidatedNetworkIdentifier) 14 | } catch (parseError) { 15 | throw new Error( 16 | `Invalid protocol network identifier: '${unvalidatedNetworkIdentifier}'. Error: ${parseError}`, 17 | ) 18 | } 19 | const network = multiNetworks.inner[networkIdentifier] 20 | if (!network) { 21 | throw new Error( 22 | `Could not find a configured protocol network named ${networkIdentifier}`, 23 | ) 24 | } 25 | return network 26 | } 27 | -------------------------------------------------------------------------------- /packages/indexer-common/src/indexer-management/rules.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@graphprotocol/common-ts' 2 | import { 3 | IndexerManagementModels, 4 | INDEXING_RULE_GLOBAL, 5 | IndexingRule, 6 | IndexingRuleAttributes, 7 | } from '@graphprotocol/indexer-common' 8 | import { parseIndexingRule } from '../rules' 9 | import groupBy from 'lodash.groupby' 10 | 11 | export const fetchIndexingRules = async ( 12 | models: IndexerManagementModels, 13 | merged: boolean, 14 | protocolNetwork?: string, 15 | ): Promise => { 16 | // If unspecified, select indexing rules from all protocol networks 17 | const whereClause = protocolNetwork ? { protocolNetwork } : {} 18 | const rules = await models.IndexingRule.findAll({ 19 | where: whereClause, 20 | order: [ 21 | ['identifierType', 'DESC'], 22 | ['identifier', 'ASC'], 23 | ], 24 | }) 25 | if (merged) { 26 | // Merge rules by protocol network 27 | return Object.entries(groupBy(rules, (rule) => rule.protocolNetwork)) 28 | .map(([protocolNetwork, rules]) => { 29 | const global = rules.find((rule) => rule.identifier === INDEXING_RULE_GLOBAL) 30 | if (!global) { 31 | throw Error(`Could not find global rule for network '${protocolNetwork}'`) 32 | } 33 | return rules.map((rule) => rule.mergeGlobal(global)) 34 | }) 35 | .flat() 36 | } else { 37 | return rules 38 | } 39 | } 40 | 41 | export const upsertIndexingRule = async ( 42 | logger: Logger, 43 | models: IndexerManagementModels, 44 | newRule: Partial, 45 | ): Promise => { 46 | const indexingRule = parseIndexingRule(newRule) 47 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 48 | const [updatedRule, _created] = await models.IndexingRule.upsert(indexingRule) 49 | 50 | logger.debug( 51 | `DecisionBasis.${indexingRule.decisionBasis} rule merged into indexing rules`, 52 | { 53 | rule: updatedRule, 54 | }, 55 | ) 56 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 57 | return updatedRule! 58 | } 59 | -------------------------------------------------------------------------------- /packages/indexer-common/src/indexer-management/server.ts: -------------------------------------------------------------------------------- 1 | import { Stream } from 'stream' 2 | import express from 'express' 3 | import cors from 'cors' 4 | import bodyParser from 'body-parser' 5 | import morgan from 'morgan' 6 | import { Logger } from '@graphprotocol/common-ts' 7 | 8 | import { IndexerManagementClient } from './client' 9 | import http from 'http' 10 | 11 | export interface CreateIndexerManagementServerOptions { 12 | logger: Logger 13 | client: IndexerManagementClient 14 | port: number 15 | } 16 | 17 | export const createIndexerManagementServer = async ({ 18 | logger, 19 | client, 20 | port, 21 | }: CreateIndexerManagementServerOptions): Promise => { 22 | logger = logger.child({ component: 'IndexerManagementServer' }) 23 | 24 | const loggerStream = new Stream.Writable() 25 | loggerStream._write = (chunk, _, next) => { 26 | logger.debug(chunk.toString().trim()) 27 | next() 28 | } 29 | 30 | const app = express() 31 | 32 | // Log requests to the logger stream 33 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 34 | app.use(morgan('tiny', { stream: loggerStream }) as any) 35 | app.use(cors()) 36 | 37 | // Endpoint for health checks 38 | app.get('/', (_, res) => { 39 | res.status(200).send('Ready to roll!') 40 | }) 41 | 42 | // GraphQL endpoint 43 | app.post('/', bodyParser.json(), async (req, res) => { 44 | const { query, variables } = req.body 45 | 46 | const result = query.startsWith('mutation') 47 | ? await client.mutation(query, variables).toPromise() 48 | : await client.query(query, variables).toPromise() 49 | 50 | res.status(200).send({ 51 | data: result.data, 52 | errors: result.error ? result.error.graphQLErrors : null, 53 | extensions: result.extensions, 54 | }) 55 | }) 56 | 57 | const server = app.listen(port, () => { 58 | logger.debug(`Listening on port ${port}`) 59 | }) 60 | 61 | return server 62 | } 63 | -------------------------------------------------------------------------------- /packages/indexer-common/src/parsers/__tests__/parsers.ts: -------------------------------------------------------------------------------- 1 | import { validateNetworkIdentifier } from '../validators' 2 | 3 | describe('validateNetworkIdentifier tests', () => { 4 | it('should parse valid network identifiers', () => { 5 | expect(validateNetworkIdentifier('sepolia')).toBe('eip155:11155111') 6 | expect(validateNetworkIdentifier('mainnet')).toBe('eip155:1') 7 | expect(validateNetworkIdentifier('eip155:1')).toBe('eip155:1') 8 | expect(validateNetworkIdentifier('eip155:421614')).toBe('eip155:421614') 9 | }) 10 | }) 11 | -------------------------------------------------------------------------------- /packages/indexer-common/src/parsers/basic-types.ts: -------------------------------------------------------------------------------- 1 | // Parser combinators for basic types 2 | 3 | import P from 'parsimmon' 4 | import { resolveChainId } from '../indexer-management/types' 5 | 6 | // Checks if the provided network identifier is supported by the Indexer Agent. 7 | function validateNetworkIdentifier(n: string): P.Parser { 8 | try { 9 | const valid = resolveChainId(n) 10 | return P.succeed(valid) 11 | } catch (e) { 12 | return P.fail('a supported network identifier') 13 | } 14 | } 15 | 16 | // A basic URL parser. 17 | export const url = P.regex(/^https?:.*/) 18 | .map((x) => new URL(x)) 19 | .desc('a valid URL') 20 | 21 | // Source: https://github.com/ChainAgnostic/CAIPs/blob/master/CAIPs/caip-2.md 22 | export const caip2IdRegex = /^[-a-z0-9]{3,8}:[-_a-zA-Z0-9]{1,32}$/ 23 | const caip2Id = P.regex(caip2IdRegex).chain(validateNetworkIdentifier) 24 | 25 | // A valid human friendly network name / alias. 26 | const networkAlias = P.regex(/[a-z-]+/).chain(validateNetworkIdentifier) 27 | 28 | // Either a CAIP-2 or an alias. 29 | export const networkIdentifier = P.alt(caip2Id, networkAlias) 30 | 31 | // A basic `base58btc` parser for CIDv0 (IPFS Hashes) 32 | export const base58 = P.regex(/^Qm[1-9A-HJ-NP-Za-km-z]{44,}$/).desc( 33 | 'An IPFS Content Identifer (Qm...)', 34 | ) 35 | -------------------------------------------------------------------------------- /packages/indexer-common/src/parsers/error-handling.ts: -------------------------------------------------------------------------------- 1 | import { ZodError } from 'zod' 2 | import { fromZodError, FromZodErrorOptions, ValidationError } from 'zod-validation-error' 3 | 4 | type ErrorFormatOptions = Pick< 5 | Required, 6 | 'issueSeparator' | 'prefix' | 'prefixSeparator' 7 | > 8 | 9 | const errorFormatOptions: ErrorFormatOptions = { 10 | // Arbitrary character sequence that is unlikely to appear as part of a validation 11 | // message. It is used for splitting the concateneted error message into individual 12 | // issues. 13 | issueSeparator: '@#validation-error#@', 14 | prefixSeparator: ':', 15 | prefix: 'Indexer Agent Configuration Error(s)', 16 | } 17 | 18 | // Converts a ValidationError into human-friendly error messages. It utilizes 19 | // 'zod-validation-error' to produce these messages from Zod errors, then re-formats the 20 | // concatenated messages into a list with one issue per line, optionally including the 21 | // original file path in the error message. 22 | function formatError( 23 | error: ValidationError, 24 | errorFormatOptions: ErrorFormatOptions, 25 | filePath?: string, 26 | ) { 27 | const prefix = errorFormatOptions.prefix + errorFormatOptions.prefixSeparator 28 | const issues = error 29 | .toString() 30 | .substring(prefix.length) 31 | .split(errorFormatOptions.issueSeparator) 32 | .map((issue) => issue.trim()) 33 | .map((issue) => `- ${issue}`) 34 | .join('\n') 35 | const file = filePath ? ` [ file: ${filePath} ]` : '' 36 | return `${prefix}${file}\n${issues}` 37 | } 38 | 39 | // Helper funciton that processeses a ZodError and displays validation issues in the 40 | // terminal using a human-friendly format 41 | export function displayZodParsingError(error: ZodError, filePath?: string) { 42 | const validationError = fromZodError(error, errorFormatOptions) 43 | const formattedError = formatError(validationError, errorFormatOptions, filePath) 44 | console.error(formattedError) 45 | } 46 | -------------------------------------------------------------------------------- /packages/indexer-common/src/parsers/index.ts: -------------------------------------------------------------------------------- 1 | export * from './validators' 2 | export * from './test-utils' 3 | export * from './error-handling' 4 | -------------------------------------------------------------------------------- /packages/indexer-common/src/parsers/validators.ts: -------------------------------------------------------------------------------- 1 | // Final parsers (validators) that use parser combinators defined in the 'parsers' module but don't 2 | // expose their internal parsing interface. 3 | 4 | import P from 'parsimmon' 5 | 6 | import { networkIdentifier, base58 } from './basic-types' 7 | export { caip2IdRegex } from './basic-types' 8 | 9 | // Generic function that takes a parser of type T and attempts to parse it from a string. If it 10 | // fails, then it will throw an error with an explanation of what was expected, as well as the 11 | // portion of the input that was parsed and what's remaining to parse. 12 | function parse(parser: P.Parser, input: string): T { 13 | const parseResult = parser.parse(input) 14 | if (parseResult.status) { 15 | return parseResult.value 16 | } 17 | const expected = parseResult.expected[0] 18 | const parsed = input.slice(0, parseResult.index.offset) 19 | const remaining = input.slice(parseResult.index.offset) 20 | throw new Error( 21 | `Failed to parse "${input}". Expected: ${expected}. Parsed up to: "${parsed}". Remaining: "${remaining}"`, 22 | ) 23 | } 24 | export function validateNetworkIdentifier(input: string): string { 25 | return parse(networkIdentifier, input) 26 | } 27 | 28 | export function validateIpfsHash(input: string): string { 29 | return parse(base58, input) 30 | } 31 | -------------------------------------------------------------------------------- /packages/indexer-common/src/query-fees/__test__/rav.test.ts: -------------------------------------------------------------------------------- 1 | // test the conversion of a database rav to a RavData 2 | 3 | import { connectDatabase, toAddress } from '@graphprotocol/common-ts' 4 | import { defineQueryFeeModels } from '../models' 5 | 6 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 7 | declare const __DATABASE__: any 8 | describe('RavData', () => { 9 | test('load ravData into database', async () => { 10 | let sequelize = await connectDatabase(__DATABASE__) 11 | const queryFeeModels = defineQueryFeeModels(sequelize) 12 | sequelize = await sequelize.sync({ force: true }) 13 | const value = { 14 | allocationId: toAddress('0xabababababababababababababababababababab'), 15 | final: false, 16 | last: true, 17 | senderAddress: toAddress('0xabababababababababababababababababababab'), 18 | timestampNs: 1709067401177959664n, 19 | valueAggregate: 20000000000000n, 20 | signature: Buffer.from( 21 | '0x56f8e2b7fecee908ff0d28ed172a61bee8ffe87b498c23da7eaca74cd7c5b5ce4ead667e3eb4f1d8c4a20e6f0e5dbe73f9f63e9d0d23e606b3239332c5f213371b', 22 | 'hex', 23 | ), 24 | redeemedAt: null, 25 | createdAt: new Date(), 26 | updatedAt: new Date(), 27 | } 28 | 29 | await queryFeeModels.receiptAggregateVouchers.create(value) 30 | 31 | const result = await queryFeeModels.receiptAggregateVouchers.findAll() 32 | // expect([value]).toEqual(result) 33 | expect(result).toEqual([ 34 | expect.objectContaining({ 35 | allocationId: value.allocationId, 36 | final: value.final, 37 | last: value.last, 38 | senderAddress: value.senderAddress, 39 | signature: value.signature, 40 | timestampNs: value.timestampNs, 41 | valueAggregate: value.valueAggregate, 42 | }), 43 | ]) 44 | }) 45 | }) 46 | -------------------------------------------------------------------------------- /packages/indexer-common/src/query-fees/allocation-utils.ts: -------------------------------------------------------------------------------- 1 | import { Address } from '@graphprotocol/common-ts' 2 | import { Transaction } from 'sequelize' 3 | import { AllocationSummary, QueryFeeModels } from './models' 4 | 5 | export const ensureAllocationSummary = async ( 6 | models: QueryFeeModels, 7 | allocation: Address, 8 | transaction: Transaction, 9 | protocolNetwork: string, 10 | ): Promise<[AllocationSummary, boolean]> => { 11 | const [summary, isNew] = await models.allocationSummaries.findOrBuild({ 12 | where: { allocation }, 13 | defaults: { 14 | allocation, 15 | closedAt: null, 16 | createdTransfers: 0, 17 | resolvedTransfers: 0, 18 | failedTransfers: 0, 19 | openTransfers: 0, 20 | collectedFees: '0', 21 | withdrawnFees: '0', 22 | protocolNetwork, 23 | }, 24 | transaction, 25 | }) 26 | return [summary, isNew] 27 | } 28 | -------------------------------------------------------------------------------- /packages/indexer-common/src/query-fees/index.ts: -------------------------------------------------------------------------------- 1 | export * from './models' 2 | export * from './allocation-utils' 3 | -------------------------------------------------------------------------------- /packages/indexer-common/src/rules.ts: -------------------------------------------------------------------------------- 1 | import { IndexingDecisionBasis, IndexingRuleAttributes } from './indexer-management' 2 | import { nullPassThrough, parseBoolean } from './utils' 3 | import { parseGRT } from '@graphprotocol/common-ts' 4 | import { validateNetworkIdentifier } from './parsers' 5 | 6 | export const parseDecisionBasis = (s: string): IndexingDecisionBasis => { 7 | if (!['always', 'never', 'rules', 'offchain'].includes(s)) { 8 | throw new Error( 9 | `Unknown decision basis "${s}". Supported: always, never, rules, offchain`, 10 | ) 11 | } else { 12 | return s as IndexingDecisionBasis 13 | } 14 | } 15 | 16 | // TODO: Merge with parsers in indexer-cli/src/rules.ts 17 | const INDEXING_RULE_READABLE_TO_MODEL_PARSERS: Record< 18 | keyof IndexingRuleAttributes, 19 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 20 | (x: never) => any 21 | > = { 22 | id: (x) => x, 23 | identifier: (x) => x, 24 | identifierType: (x) => x, 25 | allocationAmount: (x: string) => parseGRT(x).toString(), 26 | allocationLifetime: nullPassThrough(parseInt), 27 | autoRenewal: nullPassThrough(parseBoolean), 28 | parallelAllocations: nullPassThrough(parseInt), 29 | minSignal: nullPassThrough((x: string) => parseGRT(x).toString()), 30 | maxSignal: nullPassThrough((x: string) => parseGRT(x).toString()), 31 | minStake: nullPassThrough((x: string) => parseGRT(x).toString()), 32 | maxAllocationPercentage: nullPassThrough(parseFloat), 33 | minAverageQueryFees: nullPassThrough((x: string) => parseGRT(x).toString()), 34 | decisionBasis: nullPassThrough(parseDecisionBasis), 35 | custom: nullPassThrough(JSON.parse), 36 | requireSupported: (x) => parseBoolean(x), 37 | safety: (x) => parseBoolean(x), 38 | protocolNetwork: (x: string) => validateNetworkIdentifier(x), 39 | } 40 | 41 | export const parseIndexingRule = ( 42 | rule: Partial, 43 | ): Partial => { 44 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 45 | const obj = {} as any 46 | for (const [key, value] of Object.entries(rule)) { 47 | try { 48 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 49 | obj[key] = (INDEXING_RULE_READABLE_TO_MODEL_PARSERS as any)[key](value) 50 | } catch { 51 | throw new Error(key) 52 | } 53 | } 54 | return obj as Partial 55 | } 56 | -------------------------------------------------------------------------------- /packages/indexer-common/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "rootDir": "src", 5 | "outDir": "dist", 6 | "target": "es2020", 7 | "module": "nodenext", 8 | "moduleResolution": "nodenext", 9 | "allowSyntheticDefaultImports": true, 10 | "declaration": true, 11 | "sourceMap": true, 12 | "esModuleInterop": true, 13 | "strict": true, 14 | "noImplicitAny": false, 15 | "composite": true, 16 | "lib": [ 17 | "esnext.asynciterable", 18 | "dom", 19 | "es2020.promise", 20 | "es2015.iterable", 21 | "es2015.promise" 22 | ], 23 | "resolveJsonModule": true 24 | }, 25 | "include": [ 26 | "src/**/*.ts" 27 | ], 28 | "exclude": [], 29 | "references": [] 30 | } 31 | -------------------------------------------------------------------------------- /scripts/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Cleanup function to stop and remove the container 3 | cleanup() { 4 | echo "Cleaning up..." 5 | docker stop indexer-test-db >/dev/null 2>&1 6 | docker rm indexer-test-db >/dev/null 2>&1 7 | } 8 | 9 | # Register the cleanup function to run on script exit 10 | trap cleanup EXIT 11 | 12 | # Start PostgreSQL container with the same configuration as CI 13 | docker run -d \ 14 | --name indexer-test-db \ 15 | -e POSTGRES_DB=indexer_tests \ 16 | -e POSTGRES_USER=testuser \ 17 | -e POSTGRES_PASSWORD=testpass \ 18 | -p 5432:5432 \ 19 | postgres:13 20 | 21 | # Wait for PostgreSQL to be ready 22 | echo "Waiting for PostgreSQL to be ready..." 23 | until docker exec indexer-test-db pg_isready > /dev/null 2>&1; do 24 | sleep 1 25 | done 26 | echo "PostgreSQL is ready!" 27 | 28 | # Load environment variables from .env file 29 | if [ -f .env ]; then 30 | echo "Loading .env file..." 31 | source .env 32 | else 33 | echo "Warning: .env file not found" 34 | fi 35 | 36 | # Run the tests 37 | echo "Running tests..." 38 | POSTGRES_TEST_HOST=localhost \ 39 | POSTGRES_TEST_DATABASE=indexer_tests \ 40 | POSTGRES_TEST_USERNAME=testuser \ 41 | POSTGRES_TEST_PASSWORD=testpass \ 42 | NODE_OPTIONS="--dns-result-order=ipv4first" \ 43 | yarn test:ci 44 | -------------------------------------------------------------------------------- /scripts/update-dependency.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PACKAGE=$1 4 | VERSION=$2 5 | 6 | if [[ -z "$PACKAGE" ]] || [[ -z "$VERSION" ]]; then 7 | echo "Usage: $0 " 8 | exit 1 9 | fi 10 | 11 | for pkg in $(ls -1 packages); do 12 | pushd packages/$pkg 13 | yarn add $PACKAGE@$VERSION 14 | popd 15 | done 16 | -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- 1 | .tf-indexer-* 2 | .terraform 3 | .gcloud-credentials.json 4 | terraform.tfvars 5 | terraform.tfstate* -------------------------------------------------------------------------------- /terraform/database.tf: -------------------------------------------------------------------------------- 1 | # 2 | # CloudSQL Postgres Database 3 | # 4 | 5 | # CloudSQL requires that a fairly long, unspecified amount of time 6 | # passes before a database name can be reused. To ease testing where 7 | # we create and destroy databases a lot, we append 4 random digits to 8 | # the database name. 9 | resource "random_integer" "dbname" { 10 | min = 1000 11 | max = 9999 12 | keepers = { 13 | indexer = "${var.indexer}" 14 | } 15 | } 16 | 17 | resource "google_sql_database_instance" "graph" { 18 | database_version = "POSTGRES_12" 19 | name = "${var.indexer}-${random_integer.dbname.result}" 20 | settings { 21 | activation_policy = "ALWAYS" 22 | availability_type = "ZONAL" 23 | disk_autoresize = true 24 | disk_size = 100 25 | disk_type = "PD_SSD" 26 | tier = var.database_tier 27 | ip_configuration { 28 | ipv4_enabled = false 29 | private_network = "projects/${var.project}/global/networks/default" 30 | } 31 | backup_configuration { 32 | binary_log_enabled = false 33 | enabled = true 34 | start_time = "02:00" 35 | } 36 | database_flags { 37 | name = "log_temp_files" 38 | value = "-1" 39 | } 40 | database_flags { 41 | name = "log_lock_waits" 42 | value = "on" 43 | } 44 | } 45 | } 46 | 47 | resource "google_sql_database" "graph" { 48 | name = "graph" 49 | instance = google_sql_database_instance.graph.name 50 | } 51 | 52 | resource "google_sql_database" "indexer-service" { 53 | name = "indexer-service" 54 | instance = google_sql_database_instance.graph.name 55 | } 56 | 57 | resource "google_sql_database" "vector" { 58 | name = "vector" 59 | instance = google_sql_database_instance.graph.name 60 | } 61 | 62 | resource "google_sql_user" "graph" { 63 | name = "graph" 64 | instance = google_sql_database_instance.graph.name 65 | password = var.database_password 66 | } 67 | -------------------------------------------------------------------------------- /terraform/prometheus.tf: -------------------------------------------------------------------------------- 1 | # 2 | # Prometheus 3 | # 4 | 5 | resource "google_compute_disk" "prometheus" { 6 | name = "${var.indexer}-prometheus" 7 | type = "pd-standard" 8 | size = var.prometheus_disk_size 9 | } 10 | 11 | resource "kubernetes_persistent_volume" "prometheus" { 12 | metadata { 13 | name = "prometheus" 14 | } 15 | spec { 16 | capacity = { 17 | storage = "${var.prometheus_disk_size}Gi" 18 | } 19 | access_modes = ["ReadWriteOnce"] 20 | persistent_volume_source { 21 | gce_persistent_disk { 22 | pd_name = google_compute_disk.prometheus.name 23 | fs_type = "ext4" 24 | } 25 | } 26 | storage_class_name = "standard" 27 | } 28 | } 29 | 30 | resource "kubernetes_persistent_volume_claim" "prometheus" { 31 | metadata { 32 | name = "prometheus" 33 | } 34 | spec { 35 | access_modes = ["ReadWriteOnce"] 36 | resources { 37 | requests = { 38 | storage = "${var.prometheus_disk_size}Gi" 39 | } 40 | } 41 | selector { 42 | match_labels = { 43 | "name" = kubernetes_persistent_volume.prometheus.metadata.0.name 44 | } 45 | } 46 | storage_class_name = "standard" 47 | volume_name = kubernetes_persistent_volume.prometheus.metadata.0.name 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /terraform/secrets.tf: -------------------------------------------------------------------------------- 1 | # 2 | # Databases and credentials 3 | # 4 | resource "kubernetes_secret" "postgres-credentials" { 5 | metadata { 6 | name = "postgres-credentials" 7 | } 8 | data = { 9 | host = google_sql_database_instance.graph.first_ip_address 10 | user = google_sql_user.graph.name 11 | password = var.database_password 12 | graph_db = "graph" 13 | indexer_db = "indexer-service" 14 | vector_db = "vector" 15 | } 16 | } 17 | 18 | # 19 | # Operator mnemonic, indexer address and more. 20 | # 21 | resource "kubernetes_secret" "indexer" { 22 | metadata { 23 | name = "indexer" 24 | } 25 | data = { 26 | mnemonic = var.indexer_mnemonic 27 | indexer_address = var.indexer_address 28 | free_query_auth_token = var.free_query_auth_token 29 | } 30 | } 31 | 32 | # 33 | # Network subgraph configuration 34 | # 35 | resource "kubernetes_secret" "network-subgraph" { 36 | metadata { 37 | name = "network-subgraph" 38 | } 39 | data = { 40 | endpoint = var.network_subgraph_endpoint 41 | deployment = var.network_subgraph_deployment 42 | } 43 | } 44 | 45 | # 46 | # Ethereum provider URL 47 | # 48 | resource "kubernetes_secret" "ethereum" { 49 | metadata { 50 | name = "ethereum" 51 | } 52 | data = { 53 | network_name = var.ethereum_chain_name 54 | chain_id = var.ethereum_chain_id 55 | url = var.ethereum_provider 56 | } 57 | } 58 | 59 | # 60 | # Scalar configuration 61 | # 62 | resource "kubernetes_secret" "scalar" { 63 | metadata { 64 | name = "scalar" 65 | } 66 | data = { 67 | collect_receipts_endpoint = var.scalar_collect_receipts_endpoint 68 | client_signer_address = var.scalar_client_signer_address 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": "./packages", 4 | "paths": { 5 | "@graphprotocol/*": ["./*/src"] 6 | }, 7 | "skipLibCheck": true, 8 | "useUnknownInCatchVariables": false 9 | } 10 | } 11 | --------------------------------------------------------------------------------