├── .dockerignore ├── .env ├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── .gitignore ├── .npmrc ├── .squidignore ├── LICENSE ├── README.md ├── assets └── README.MD ├── commands.json ├── db └── migrations │ └── 1659350771447-Data.js ├── docker-compose.yml ├── package.json ├── schema.graphql ├── scripts ├── docker-run.sh └── sub-client.js ├── squid.yaml ├── src ├── main.ts ├── model │ ├── generated │ │ ├── account.model.ts │ │ ├── index.ts │ │ ├── marshal.ts │ │ └── transfer.model.ts │ └── index.ts ├── processor.ts └── types │ ├── balances │ └── events.ts │ ├── events.ts │ ├── index.ts │ ├── support.ts │ ├── v1020.ts │ ├── v1050.ts │ └── v9130.ts ├── tsconfig.json └── typegen.json /.dockerignore: -------------------------------------------------------------------------------- 1 | /.git 2 | /node_modules 3 | /lib 4 | /*Versions.jsonl 5 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | DB_NAME=squid 2 | DB_PORT=23798 3 | GQL_PORT=4350 4 | 5 | # JSON-RPC node endpoints (wss or https) 6 | # Use private endpoints in production! If deploying to Cloud: 7 | # - use our RPC proxy service: https://docs.subsquid.io/cloud/resources/rpc-proxy/, OR 8 | # - save your private url as a secret: https://docs.subsquid.io/cloud/resources/env-variables/#secrets 9 | RPC_KUSAMA_HTTP=https://kusama-rpc.polkadot.io 10 | 11 | # Uncommenting this line enables the debug mode 12 | # More info at https://docs.subsquid.io/basics/logging/ 13 | #SQD_DEBUG=* 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment (please complete the following information):** 27 | - specify the version of the core packages (@subsquid/cli, @subsquid/substrate-processor, ...) 28 | - node.js version 29 | - npm version 30 | - OS version 31 | - Reproducible example or a repo link 32 | 33 | **Applicable to decoding issues:** 34 | - chain name 35 | - typesBundle (if it's not built-in) 36 | optional: 37 | - block 38 | - extrinsic 39 | - call 40 | - event 41 | - spec version 42 | - endpoint 43 | 44 | **Additional context** 45 | Add any other context about the problem here. 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /lib 3 | /builds 4 | 5 | /**Versions.json 6 | /**Versions.jsonl 7 | 8 | # IDE files 9 | /.idea 10 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | engine-strict=true -------------------------------------------------------------------------------- /.squidignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | builds 3 | lib 4 | Dockerfile 5 | .git 6 | .github 7 | .idea 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Subsquid Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Squid template project 2 | 3 | A starter [Squid](https://subsquid.io) project to demonstrate its structure and conventions. 4 | It accumulates [kusama](https://kusama.network) account transfers and serves them via GraphQL API. 5 | 6 | ## Summary 7 | 8 | - [Quickstart](#quickly-running-the-sample) 9 | - [Public archives for Parachains](#public-archives-for-parachains) 10 | - [Self-hosted archive](#self-hosted-archive) 11 | - [Development flow](#dev-flow) 12 | - [Database Schema](#1-define-database-schema) 13 | - [Entity classes](#2-generate-typeorm-classes) 14 | - [DB migrations](#3-generate-database-migration) 15 | - [Typegen for Events, Extrinsics and Storage Calls](#4-generate-typescript-definitions-for-substrate-events-calls-and-storage) 16 | - [Deploy the Squid](#deploy-the-squid) 17 | - [Conventions](#project-conventions) 18 | - [Type Bundles](#types-bundle) 19 | 20 | ## Prerequisites 21 | 22 | * node 16.x 23 | * docker 24 | * npm -- note that `yarn` package manager is not supported 25 | 26 | ## Quickly running the sample 27 | 28 | Example commands below use [sqd](https://docs.subsquid.io/squid-cli/). 29 | Please [install](https://docs.subsquid.io/squid-cli/installation/) it before proceeding. 30 | 31 | ```bash 32 | # 1. Install dependencies 33 | npm i 34 | 35 | # 2. Start target Postgres database and detach 36 | sqd up 37 | 38 | # 3. Build the project 39 | sqd build 40 | 41 | # 4. Start both the squid processor and the GraphQL server 42 | sqd run . 43 | ``` 44 | A GraphiQL playground will be available at [localhost:4350/graphql](http://localhost:4350/graphql). 45 | 46 | ## Public archives for Parachains 47 | 48 | Subsquid provides archive data sources [for most parachains](https://docs.subsquid.io/substrate-indexing/supported-networks/). Use `lookupArchive(, )` from `@subsquid/archive-registry` to look up the archive endpoint by the network name, e.g. 49 | 50 | ```typescript 51 | processor.setDataSource({ 52 | archive: lookupArchive("kusama", { release: "ArrowSquid" }) 53 | //... 54 | }); 55 | ``` 56 | 57 | To make sure you're indexing the right chain one can additionally filter by the genesis block hash: 58 | 59 | ```typescript 60 | processor.setDataSource({ 61 | archive: lookupArchive("kusama", { 62 | release: "ArrowSquid", 63 | genesis: "0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe" 64 | }), 65 | //... 66 | }); 67 | ``` 68 | 69 | If the chain is not yet supported, you can still index it using [RPC ingestion](https://docs.subsquid.io/substrate-indexing/setup/general/#set-data-source). If you take this route, use [metadata exporer](https://github.com/subsquid/squid-sdk/tree/master/substrate/substrate-metadata-explorer) with [Substrate typegen](https://docs.subsquid.io/substrate-indexing/squid-substrate-typegen/) for help with decoding. 70 | 71 | You can also fill out this [form](https://forms.gle/Vhr3exPs4HrF4Zt36) to submit a request for an Archive/Subsquid Network dataset. 72 | 73 | ## Self-hosted archive 74 | 75 | Self-hosted Archives are deprecated by the ArrowSquid release. Keep an eye on updates on [Subsquid Network](https://docs.subsquid.io/subsquid-network/) and use it instead once it is released. 76 | 77 | ## Dev flow 78 | 79 | ### 1. Define database schema 80 | 81 | Start development by defining the schema of the target database via `schema.graphql`. 82 | Schema definition consists of regular graphql type declarations annotated with custom directives. 83 | Full description of `schema.graphql` dialect is available [here](https://docs.subsquid.io/store/postgres/schema-file/). 84 | 85 | ### 2. Generate TypeORM classes 86 | 87 | Mapping developers use [TypeORM](https://typeorm.io) entities 88 | to interact with the target database during data processing. All necessary entity classes are 89 | [generated](https://docs.subsquid.io/store/postgres/schema-file/intro/) by the squid framework from `schema.graphql`. This is done by running `npx squid-typeorm-codegen` 90 | or (equivalently) `sqd codegen` command. 91 | 92 | ### 3. Generate database migration 93 | 94 | All database changes are applied through migration files located at `db/migrations`. 95 | `squid-typeorm-migration(1)` tool provides several commands to drive the process. 96 | It is all [TypeORM](https://typeorm.io/#/migrations) under the hood. 97 | 98 | ```bash 99 | # Connect to database, analyze its state and generate migration to match the target schema. 100 | # The target schema is derived from entity classes generated earlier. 101 | # Don't forget to compile your entity classes beforehand! 102 | npx squid-typeorm-migration generate 103 | 104 | # Create template file for custom database changes 105 | npx squid-typeorm-migration create 106 | 107 | # Apply database migrations from `db/migrations` 108 | npx squid-typeorm-migration apply 109 | 110 | # Revert the last performed migration 111 | npx squid-typeorm-migration revert 112 | ``` 113 | Available `sqd` shortcuts: 114 | ```bash 115 | # Build the project, remove any old migrations, then run `npx squid-typeorm-migration generate` 116 | sqd migration:generate 117 | 118 | # Run npx squid-typeorm-migration apply 119 | sqd migration:apply 120 | ``` 121 | 122 | ### 4. Generate TypeScript definitions for substrate events, calls and storage 123 | 124 | This is an optional part, but it is very advisable. 125 | 126 | Event, call and runtime storage data come to mapping handlers as raw untyped json. 127 | While it is possible to work with raw untyped json data, 128 | it's extremely error-prone and the json structure may change over time due to runtime upgrades. 129 | 130 | Squid framework provides a tool for generating type-safe wrappers around events, calls and runtime storage items for 131 | each historical change in the spec version. See the [Substrate typegen](https://docs.subsquid.io/substrate-indexing/squid-substrate-typegen/) documentation page. 132 | 133 | ## Deploy the Squid 134 | 135 | After a local run, obtain a deployment key by signing into [Subsquid Cloud](https://app.subsquid.io) and run 136 | 137 | ```sh 138 | npx sqd auth -k YOUR_DEPLOYMENT_KEY 139 | ``` 140 | 141 | Next, inspect the Squid CLI help to deploy and manage your squid: 142 | 143 | ```sh 144 | npx sqd squid --help 145 | ``` 146 | 147 | For more information, consult the [Deployment Guide](https://docs.subsquid.io/deploy-squid/). 148 | 149 | ## Project conventions 150 | 151 | Squid tools assume a certain project layout. 152 | 153 | * All compiled js files must reside in `lib` and all TypeScript sources in `src`. 154 | The layout of `lib` must reflect `src`. 155 | * All TypeORM classes must be exported by `src/model/index.ts` (`lib/model` module). 156 | * Database schema must be defined in `schema.graphql`. 157 | * Database migrations must reside in `db/migrations` and must be plain js files. 158 | * `squid-*(1)` executables consult `.env` file for a number of environment variables. 159 | 160 | See the [full desription](https://docs.subsquid.io/basics/squid-structure/) in the documentation. 161 | 162 | ## Types bundle 163 | 164 | Substrate chains that have blocks with metadata versions below 14 don't provide enough 165 | information to decode their data. For those chains, external [type](https://polkadot.js.org/docs/api/start/types.extend) [definitions](https://polkadot.js.org/docs/api/start/types.extend) are required. 166 | 167 | Subsquid tools include definitions for many chains, however sometimes external 168 | definitions are still required. 169 | 170 | You can pass them as a special json file (types bundle) of the following structure: 171 | 172 | ```json5 173 | { 174 | "types": { 175 | "AccountId": "[u8; 32]" 176 | }, 177 | "typesAlias": { 178 | "assets": { 179 | "Balance": "u64" 180 | } 181 | }, 182 | "versions": [ 183 | { 184 | "minmax": [0, 1000], // spec version range with inclusive boundaries 185 | "types": { 186 | "AccountId": "[u8; 16]" 187 | }, 188 | "typesAlias": { 189 | "assets": { 190 | "Balance": "u32" 191 | } 192 | } 193 | } 194 | ] 195 | } 196 | ``` 197 | 198 | * `.types` - scale type definitions similar to [polkadot.js types](https://polkadot.js.org/docs/api/start/types.extend#extension) 199 | * `.typesAlias` - similar to [polkadot.js type aliases](https://polkadot.js.org/docs/api/start/types.extend#type-clashes) 200 | * `.versions` - per-block range overrides/patches for above fields. 201 | 202 | All fields in the type bundle are optional and applied on top of a fixed set of well-known frame types. 203 | 204 | Note, that although the structure of subsquid types bundle is very similar to the one from polkadot.js, 205 | those two are not fully compatible. 206 | 207 | ## Differences from polkadot.js 208 | 209 | Polkadot.js provides lots of [specialized classes](https://polkadot.js.org/docs/api/start/types.basics) for various types of data. 210 | Even primitives like `u32` are exposed through special classes. 211 | In contrast, the squid framework works only with plain js primitives and objects. 212 | For instance, account data is passed to the handler context as a plain byte array. To convert it into a standard human-readable format one should explicitly use a utility lib `@subsquid/ss58`: 213 | 214 | ```typescript 215 | // ... 216 | from: ss58.codec('kusama').encode(rec.from), 217 | to: ss58.codec('kusama').encode(rec.to), 218 | ``` 219 | 220 | ## Graphql server extensions 221 | 222 | It is possible to extend `squid-graphql-server(1)` with custom 223 | [type-graphql](https://typegraphql.com) resolvers and to add request validation. 224 | For more details, consult [docs](https://docs.subsquid.io/graphql-api/). 225 | -------------------------------------------------------------------------------- /assets/README.MD: -------------------------------------------------------------------------------- 1 | # Assets 2 | 3 | `assets` is the designated folder for any additional files to be used by the squid, for example a static data file. The folder is added by default to `Dockerfile` and is kept when the squid is deployed to the Aquairum. -------------------------------------------------------------------------------- /commands.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://cdn.subsquid.io/schemas/commands.json", 3 | "commands": { 4 | "clean": { 5 | "description": "delete all build artifacts", 6 | "cmd": ["npx", "--yes", "rimraf", "lib"] 7 | }, 8 | "build": { 9 | "description": "Build the squid project", 10 | "deps": ["clean"], 11 | "cmd": ["tsc"] 12 | }, 13 | "up": { 14 | "description": "Start a PG database", 15 | "cmd": ["docker", "compose", "up", "-d"] 16 | }, 17 | "down": { 18 | "description": "Drop a PG database", 19 | "cmd": ["docker", "compose", "down"] 20 | }, 21 | "migration:apply": { 22 | "description": "Apply the DB migrations", 23 | "cmd": ["squid-typeorm-migration", "apply"] 24 | }, 25 | "migration:generate": { 26 | "description": "Generate a DB migration matching the TypeORM entities", 27 | "deps": ["build", "migration:clean"], 28 | "cmd": ["squid-typeorm-migration", "generate"] 29 | }, 30 | "migration:clean": { 31 | "description": "Clean the migrations folder", 32 | "cmd": ["npx", "--yes", "rimraf", "./db/migrations"] 33 | }, 34 | "migration": { 35 | "deps": ["build"], 36 | "cmd": ["squid-typeorm-migration", "generate"], 37 | "hidden": true 38 | }, 39 | "codegen": { 40 | "description": "Generate TypeORM entities from the schema file", 41 | "cmd": ["squid-typeorm-codegen"] 42 | }, 43 | "typegen": { 44 | "description": "Generate data access classes for an substrate metadata", 45 | "cmd": ["squid-substrate-typegen", "./typegen.json"] 46 | }, 47 | "process": { 48 | "description": "Load .env and start the squid processor", 49 | "deps": ["build", "migration:apply"], 50 | "cmd": ["node", "--require=dotenv/config", "lib/main.js"] 51 | }, 52 | "process:prod": { 53 | "description": "Start the squid processor", 54 | "deps": ["migration:apply"], 55 | "cmd": ["node", "lib/main.js"], 56 | "hidden": true 57 | }, 58 | "serve": { 59 | "description": "Start the GraphQL API server", 60 | "cmd": ["squid-graphql-server"] 61 | }, 62 | "serve:prod": { 63 | "description": "Start the GraphQL API server with caching and limits", 64 | "cmd": ["squid-graphql-server", 65 | "--dumb-cache", "in-memory", 66 | "--dumb-cache-ttl", "1000", 67 | "--dumb-cache-size", "100", 68 | "--dumb-cache-max-age", "1000" ] 69 | }, 70 | "check-updates": { 71 | "cmd": ["npx", "--yes", "npm-check-updates", "--filter=/subsquid/", "--upgrade"], 72 | "hidden": true 73 | }, 74 | "bump": { 75 | "description": "Bump @subsquid packages to the latest versions", 76 | "deps": ["check-updates"], 77 | "cmd": ["npm", "i", "-f"] 78 | }, 79 | "open": { 80 | "description": "Open a local browser window", 81 | "cmd": ["npx", "--yes", "opener"] 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /db/migrations/1659350771447-Data.js: -------------------------------------------------------------------------------- 1 | module.exports = class Data1659350771447 { 2 | name = 'Data1659350771447' 3 | 4 | async up(db) { 5 | await db.query(`CREATE TABLE "transfer" ("id" character varying NOT NULL, "block_number" integer NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "extrinsic_hash" text, "amount" numeric NOT NULL, "fee" numeric, "from_id" character varying, "to_id" character varying, CONSTRAINT "PK_fd9ddbdd49a17afcbe014401295" PRIMARY KEY ("id"))`) 6 | await db.query(`CREATE INDEX "IDX_d6624eacc30144ea97915fe846" ON "transfer" ("block_number") `) 7 | await db.query(`CREATE INDEX "IDX_70ff8b624c3118ac3a4862d22c" ON "transfer" ("timestamp") `) 8 | await db.query(`CREATE INDEX "IDX_070c555a86b0b41a534a55a659" ON "transfer" ("extrinsic_hash") `) 9 | await db.query(`CREATE INDEX "IDX_76bdfed1a7eb27c6d8ecbb7349" ON "transfer" ("from_id") `) 10 | await db.query(`CREATE INDEX "IDX_0751309c66e97eac9ef1149362" ON "transfer" ("to_id") `) 11 | await db.query(`CREATE INDEX "IDX_f4007436c1b546ede08a4fd7ab" ON "transfer" ("amount") `) 12 | await db.query(`CREATE TABLE "account" ("id" character varying NOT NULL, CONSTRAINT "PK_54115ee388cdb6d86bb4bf5b2ea" PRIMARY KEY ("id"))`) 13 | await db.query(`ALTER TABLE "transfer" ADD CONSTRAINT "FK_76bdfed1a7eb27c6d8ecbb73496" FOREIGN KEY ("from_id") REFERENCES "account"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) 14 | await db.query(`ALTER TABLE "transfer" ADD CONSTRAINT "FK_0751309c66e97eac9ef11493623" FOREIGN KEY ("to_id") REFERENCES "account"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) 15 | } 16 | 17 | async down(db) { 18 | await db.query(`DROP TABLE "transfer"`) 19 | await db.query(`DROP INDEX "public"."IDX_d6624eacc30144ea97915fe846"`) 20 | await db.query(`DROP INDEX "public"."IDX_70ff8b624c3118ac3a4862d22c"`) 21 | await db.query(`DROP INDEX "public"."IDX_070c555a86b0b41a534a55a659"`) 22 | await db.query(`DROP INDEX "public"."IDX_76bdfed1a7eb27c6d8ecbb7349"`) 23 | await db.query(`DROP INDEX "public"."IDX_0751309c66e97eac9ef1149362"`) 24 | await db.query(`DROP INDEX "public"."IDX_f4007436c1b546ede08a4fd7ab"`) 25 | await db.query(`DROP TABLE "account"`) 26 | await db.query(`ALTER TABLE "transfer" DROP CONSTRAINT "FK_76bdfed1a7eb27c6d8ecbb73496"`) 27 | await db.query(`ALTER TABLE "transfer" DROP CONSTRAINT "FK_0751309c66e97eac9ef11493623"`) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: postgres:15 4 | environment: 5 | POSTGRES_DB: squid 6 | POSTGRES_PASSWORD: postgres 7 | shm_size: 1gb 8 | ports: 9 | - "${DB_PORT}:5432" 10 | # command: ["postgres", "-c", "log_statement=all"] 11 | # volumes: 12 | # - ./data/db:/var/lib/postgresql/data 13 | 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "squid", 3 | "private": true, 4 | "engines": { 5 | "node": ">=16" 6 | }, 7 | "scripts": { 8 | "build": "rm -rf lib && tsc" 9 | }, 10 | "dependencies": { 11 | "@subsquid/graphql-server": "^4.6.0", 12 | "@subsquid/ss58": "^2.0.2", 13 | "@subsquid/substrate-processor": "^8.4.1", 14 | "@subsquid/typeorm-migration": "^1.3.0", 15 | "@subsquid/typeorm-store": "^1.5.1", 16 | "dotenv": "^16.4.5", 17 | "pg": "8.12.0", 18 | "typeorm": "^0.3.20" 19 | }, 20 | "devDependencies": { 21 | "@subsquid/substrate-metadata-explorer": "^3.1.2", 22 | "@subsquid/substrate-typegen": "^8.1.0", 23 | "@subsquid/typeorm-codegen": "^2.0.1", 24 | "@types/node": "^20.14.8", 25 | "typescript": "^5.5.2" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /schema.graphql: -------------------------------------------------------------------------------- 1 | type Account @entity { 2 | "Account address" 3 | id: ID! 4 | transfersTo: [Transfer!] @derivedFrom(field: "to") 5 | transfersFrom: [Transfer!] @derivedFrom(field: "from") 6 | } 7 | 8 | type Transfer @entity { 9 | id: ID! 10 | blockNumber: Int! @index 11 | timestamp: DateTime! @index 12 | extrinsicHash: String @index 13 | from: Account! 14 | to: Account! 15 | amount: BigInt! @index 16 | fee: BigInt! # fee is calculated at the best effort and may be zero for some old extrinsics 17 | } 18 | -------------------------------------------------------------------------------- /scripts/docker-run.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | docker build . --target processor -t squid-processor 3 | # make sure the port matches .env. 4 | # For Linux, add --add-host=host.docker.internal:host-gateway 5 | docker run --rm -e DB_HOST=host.docker.internal --env-file=.env squid-processor -------------------------------------------------------------------------------- /scripts/sub-client.js: -------------------------------------------------------------------------------- 1 | const WebSocket = require('ws') 2 | const { createClient } = require('graphql-ws'); 3 | 4 | const port = process.env.GQL_PORT || 4350 5 | const host = process.env.GQL_HOST || 'localhost' 6 | const proto = process.env.GQL_PROTO || 'ws' 7 | 8 | 9 | const client = createClient({ 10 | webSocketImpl: WebSocket, 11 | url: `${proto}://${host}:${port}/graphql`, 12 | }); 13 | 14 | client.subscribe( 15 | { 16 | query: ` 17 | subscription { 18 | transfers(limit: 5, orderBy: timestamp_DESC) { 19 | amount 20 | blockNumber 21 | from { 22 | id 23 | } 24 | to { 25 | id 26 | } 27 | } 28 | } 29 | `, 30 | }, 31 | { 32 | next: (data) => { 33 | console.log(`New transfers: ${JSON.stringify(data)}`); 34 | }, 35 | error: (error) => { 36 | console.error('error', error); 37 | }, 38 | complete: () => { 39 | console.log('done!'); 40 | }, 41 | } 42 | ); -------------------------------------------------------------------------------- /squid.yaml: -------------------------------------------------------------------------------- 1 | manifestVersion: subsquid.io/v0.1 2 | name: substrate-template 3 | version: 1 4 | description: |- 5 | Substrate squid template 6 | build: 7 | 8 | deploy: 9 | addons: 10 | postgres: 11 | rpc: 12 | - kusama.http # RPC proxy URL will be supplied via the RPC_KUSAMA_HTTP env variable 13 | processor: 14 | cmd: [ "sqd", "process:prod" ] 15 | api: 16 | cmd: [ "sqd", "serve:prod" ] 17 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import {TypeormDatabase, Store} from '@subsquid/typeorm-store' 2 | import {In} from 'typeorm' 3 | import * as ss58 from '@subsquid/ss58' 4 | import assert from 'assert' 5 | 6 | import {processor, ProcessorContext} from './processor' 7 | import {Account, Transfer} from './model' 8 | import {events} from './types' 9 | 10 | processor.run(new TypeormDatabase({supportHotBlocks: true}), async (ctx) => { 11 | let transferEvents: TransferEvent[] = getTransferEvents(ctx) 12 | 13 | let accounts: Map = await createAccounts(ctx, transferEvents) 14 | let transfers: Transfer[] = createTransfers(transferEvents, accounts) 15 | 16 | await ctx.store.upsert([...accounts.values()]) 17 | await ctx.store.insert(transfers) 18 | }) 19 | 20 | interface TransferEvent { 21 | id: string 22 | blockNumber: number 23 | timestamp: Date 24 | extrinsicHash?: string 25 | from: string 26 | to: string 27 | amount: bigint 28 | fee?: bigint 29 | } 30 | 31 | function getTransferEvents(ctx: ProcessorContext): TransferEvent[] { 32 | // Filters and decodes the arriving events 33 | let transfers: TransferEvent[] = [] 34 | for (let block of ctx.blocks) { 35 | for (let event of block.events) { 36 | if (event.name == events.balances.transfer.name) { 37 | let rec: {from: string; to: string; amount: bigint} 38 | if (events.balances.transfer.v1020.is(event)) { 39 | let [from, to, amount] = events.balances.transfer.v1020.decode(event) 40 | rec = {from, to, amount} 41 | } 42 | else if (events.balances.transfer.v1050.is(event)) { 43 | let [from, to, amount] = events.balances.transfer.v1050.decode(event) 44 | rec = {from, to, amount} 45 | } 46 | else if (events.balances.transfer.v9130.is(event)) { 47 | rec = events.balances.transfer.v9130.decode(event) 48 | } 49 | else { 50 | throw new Error('Unsupported spec') 51 | } 52 | 53 | assert(block.header.timestamp, `Got an undefined timestamp at block ${block.header.height}`) 54 | 55 | transfers.push({ 56 | id: event.id, 57 | blockNumber: block.header.height, 58 | timestamp: new Date(block.header.timestamp), 59 | extrinsicHash: event.extrinsic?.hash, 60 | from: ss58.codec('kusama').encode(rec.from), 61 | to: ss58.codec('kusama').encode(rec.to), 62 | amount: rec.amount, 63 | fee: event.extrinsic?.fee || 0n, 64 | }) 65 | } 66 | } 67 | } 68 | return transfers 69 | } 70 | 71 | async function createAccounts(ctx: ProcessorContext, transferEvents: TransferEvent[]): Promise> { 72 | const accountIds = new Set() 73 | for (let t of transferEvents) { 74 | accountIds.add(t.from) 75 | accountIds.add(t.to) 76 | } 77 | 78 | const accounts = await ctx.store.findBy(Account, {id: In([...accountIds])}).then((accounts) => { 79 | return new Map(accounts.map((a) => [a.id, a])) 80 | }) 81 | 82 | for (let t of transferEvents) { 83 | updateAccounts(t.from) 84 | updateAccounts(t.to) 85 | } 86 | 87 | function updateAccounts(id: string): void { 88 | const acc = accounts.get(id) 89 | if (acc == null) { 90 | accounts.set(id, new Account({id})) 91 | } 92 | } 93 | 94 | return accounts 95 | } 96 | 97 | function createTransfers(transferEvents: TransferEvent[], accounts: Map): Transfer[] { 98 | let transfers: Transfer[] = [] 99 | for (let t of transferEvents) { 100 | let {id, blockNumber, timestamp, extrinsicHash, amount, fee} = t 101 | let from = accounts.get(t.from) 102 | let to = accounts.get(t.to) 103 | transfers.push(new Transfer({ 104 | id, 105 | blockNumber, 106 | timestamp, 107 | extrinsicHash, 108 | from, 109 | to, 110 | amount, 111 | fee, 112 | })) 113 | } 114 | return transfers 115 | } 116 | -------------------------------------------------------------------------------- /src/model/generated/account.model.ts: -------------------------------------------------------------------------------- 1 | import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, OneToMany as OneToMany_} from "typeorm" 2 | import {Transfer} from "./transfer.model" 3 | 4 | @Entity_() 5 | export class Account { 6 | constructor(props?: Partial) { 7 | Object.assign(this, props) 8 | } 9 | 10 | /** 11 | * Account address 12 | */ 13 | @PrimaryColumn_() 14 | id!: string 15 | 16 | @OneToMany_(() => Transfer, e => e.to) 17 | transfersTo!: Transfer[] 18 | 19 | @OneToMany_(() => Transfer, e => e.from) 20 | transfersFrom!: Transfer[] 21 | } 22 | -------------------------------------------------------------------------------- /src/model/generated/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./account.model" 2 | export * from "./transfer.model" 3 | -------------------------------------------------------------------------------- /src/model/generated/marshal.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | 3 | 4 | export interface Marshal { 5 | fromJSON(value: unknown): T 6 | toJSON(value: T): S 7 | } 8 | 9 | 10 | export const string: Marshal = { 11 | fromJSON(value: unknown): string { 12 | assert(typeof value === 'string', 'invalid String') 13 | return value 14 | }, 15 | toJSON(value) { 16 | return value 17 | }, 18 | } 19 | 20 | 21 | export const id = string 22 | 23 | 24 | export const int: Marshal = { 25 | fromJSON(value: unknown): number { 26 | assert(Number.isInteger(value), 'invalid Int') 27 | return value as number 28 | }, 29 | toJSON(value) { 30 | return value 31 | }, 32 | } 33 | 34 | 35 | export const float: Marshal = { 36 | fromJSON(value: unknown): number { 37 | assert(typeof value === 'number', 'invalid Float') 38 | return value as number 39 | }, 40 | toJSON(value) { 41 | return value 42 | }, 43 | } 44 | 45 | 46 | export const boolean: Marshal = { 47 | fromJSON(value: unknown): boolean { 48 | assert(typeof value === 'boolean', 'invalid Boolean') 49 | return value 50 | }, 51 | toJSON(value: boolean): boolean { 52 | return value 53 | }, 54 | } 55 | 56 | 57 | export const bigint: Marshal = { 58 | fromJSON(value: unknown): bigint { 59 | assert(typeof value === 'string', 'invalid BigInt') 60 | return BigInt(value) 61 | }, 62 | toJSON(value: bigint): string { 63 | return value.toString() 64 | }, 65 | } 66 | 67 | 68 | // credit - https://github.com/Urigo/graphql-scalars/blob/91b4ea8df891be8af7904cf84751930cc0c6613d/src/scalars/iso-date/validator.ts#L122 69 | const RFC_3339_REGEX = 70 | /^(\d{4}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60))(\.\d{1,})?([Z])$/ 71 | 72 | 73 | function isIsoDateTimeString(s: string): boolean { 74 | return RFC_3339_REGEX.test(s) 75 | } 76 | 77 | 78 | export const datetime: Marshal = { 79 | fromJSON(value: unknown): Date { 80 | assert(typeof value === 'string', 'invalid DateTime') 81 | assert(isIsoDateTimeString(value), 'invalid DateTime') 82 | return new Date(value) 83 | }, 84 | toJSON(value: Date): string { 85 | return value.toISOString() 86 | }, 87 | } 88 | 89 | 90 | export const bytes: Marshal = { 91 | fromJSON(value: unknown): Buffer { 92 | assert(typeof value === 'string', 'invalid Bytes') 93 | assert(value.length % 2 === 0, 'invalid Bytes') 94 | assert(/^0x[0-9a-f]+$/i.test(value), 'invalid Bytes') 95 | return Buffer.from(value.slice(2), 'hex') 96 | }, 97 | toJSON(value: Uint8Array): string { 98 | if (Buffer.isBuffer(value)) { 99 | return '0x' + value.toString('hex') 100 | } else { 101 | return '0x' + Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString('hex') 102 | } 103 | }, 104 | } 105 | 106 | 107 | export function fromList(list: unknown, f: (val: unknown) => T): T[] { 108 | assert(Array.isArray(list)) 109 | return list.map((val) => f(val)) 110 | } 111 | 112 | 113 | export function nonNull(val: T | undefined | null): T { 114 | assert(val != null, 'non-nullable value is null') 115 | return val 116 | } 117 | 118 | 119 | export const bigintTransformer = { 120 | to(x?: bigint) { 121 | return x?.toString() 122 | }, 123 | from(s?: string): bigint | undefined { 124 | return s == null ? undefined : BigInt(s) 125 | } 126 | } 127 | 128 | 129 | export function enumFromJson(json: unknown, enumObject: E): E[keyof E] { 130 | assert(typeof json == 'string', 'invalid enum value') 131 | let val = (enumObject as any)[json] 132 | assert(typeof val == 'string', `invalid enum value`) 133 | return val as any 134 | } 135 | -------------------------------------------------------------------------------- /src/model/generated/transfer.model.ts: -------------------------------------------------------------------------------- 1 | import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_, ManyToOne as ManyToOne_} from "typeorm" 2 | import * as marshal from "./marshal" 3 | import {Account} from "./account.model" 4 | 5 | @Entity_() 6 | export class Transfer { 7 | constructor(props?: Partial) { 8 | Object.assign(this, props) 9 | } 10 | 11 | @PrimaryColumn_() 12 | id!: string 13 | 14 | @Index_() 15 | @Column_("int4", {nullable: false}) 16 | blockNumber!: number 17 | 18 | @Index_() 19 | @Column_("timestamp with time zone", {nullable: false}) 20 | timestamp!: Date 21 | 22 | @Index_() 23 | @Column_("text", {nullable: true}) 24 | extrinsicHash!: string | undefined | null 25 | 26 | @Index_() 27 | @ManyToOne_(() => Account, {nullable: true}) 28 | from!: Account 29 | 30 | @Index_() 31 | @ManyToOne_(() => Account, {nullable: true}) 32 | to!: Account 33 | 34 | @Index_() 35 | @Column_("numeric", {transformer: marshal.bigintTransformer, nullable: false}) 36 | amount!: bigint 37 | 38 | @Column_("numeric", {transformer: marshal.bigintTransformer, nullable: false}) 39 | fee!: bigint 40 | } 41 | -------------------------------------------------------------------------------- /src/model/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./generated" 2 | -------------------------------------------------------------------------------- /src/processor.ts: -------------------------------------------------------------------------------- 1 | import {assertNotNull} from '@subsquid/util-internal' 2 | import { 3 | BlockHeader, 4 | DataHandlerContext, 5 | SubstrateBatchProcessor, 6 | SubstrateBatchProcessorFields, 7 | Event as _Event, 8 | Call as _Call, 9 | Extrinsic as _Extrinsic 10 | } from '@subsquid/substrate-processor' 11 | 12 | import {events} from './types' 13 | 14 | export const processor = new SubstrateBatchProcessor() 15 | // Lookup archive by the network name in Subsquid registry 16 | // See https://docs.subsquid.io/substrate-indexing/supported-networks/ 17 | .setGateway('https://v2.archive.subsquid.io/network/kusama') 18 | // Chain RPC endpoint is required on Substrate for metadata and real-time updates 19 | .setRpcEndpoint({ 20 | // Set via .env for local runs or via secrets when deploying to Subsquid Cloud 21 | // https://docs.subsquid.io/deploy-squid/env-variables/ 22 | url: assertNotNull(process.env.RPC_KUSAMA_HTTP, 'No RPC endpoint supplied'), 23 | // More RPC connection options at https://docs.subsquid.io/substrate-indexing/setup/general/#set-data-source 24 | rateLimit: 10 25 | }) 26 | .addEvent({ 27 | name: [events.balances.transfer.name], 28 | extrinsic: true 29 | }) 30 | .setFields({ 31 | event: { 32 | args: true 33 | }, 34 | extrinsic: { 35 | hash: true, 36 | fee: true 37 | }, 38 | block: { 39 | timestamp: true 40 | } 41 | }) 42 | // Uncomment to disable RPC ingestion and drastically reduce no of RPC calls 43 | //.useArchiveOnly() 44 | 45 | export type Fields = SubstrateBatchProcessorFields 46 | export type Block = BlockHeader 47 | export type Event = _Event 48 | export type Call = _Call 49 | export type Extrinsic = _Extrinsic 50 | export type ProcessorContext = DataHandlerContext 51 | -------------------------------------------------------------------------------- /src/types/balances/events.ts: -------------------------------------------------------------------------------- 1 | import {sts, Block, Bytes, Option, Result, EventType, RuntimeCtx} from '../support' 2 | import * as v1020 from '../v1020' 3 | import * as v1050 from '../v1050' 4 | import * as v9130 from '../v9130' 5 | 6 | export const transfer = { 7 | name: 'Balances.Transfer', 8 | /** 9 | * Transfer succeeded (from, to, value, fees). 10 | */ 11 | v1020: new EventType( 12 | 'Balances.Transfer', 13 | sts.tuple([v1020.AccountId, v1020.AccountId, v1020.Balance, v1020.Balance]) 14 | ), 15 | /** 16 | * Transfer succeeded (from, to, value). 17 | */ 18 | v1050: new EventType( 19 | 'Balances.Transfer', 20 | sts.tuple([v1050.AccountId, v1050.AccountId, v1050.Balance]) 21 | ), 22 | /** 23 | * Transfer succeeded. 24 | */ 25 | v9130: new EventType( 26 | 'Balances.Transfer', 27 | sts.struct({ 28 | from: v9130.AccountId32, 29 | to: v9130.AccountId32, 30 | amount: sts.bigint(), 31 | }) 32 | ), 33 | } 34 | -------------------------------------------------------------------------------- /src/types/events.ts: -------------------------------------------------------------------------------- 1 | export * as balances from './balances/events' 2 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | export * as v1020 from './v1020' 2 | export * as v1050 from './v1050' 3 | export * as v9130 from './v9130' 4 | export * as events from './events' 5 | -------------------------------------------------------------------------------- /src/types/support.ts: -------------------------------------------------------------------------------- 1 | import type {BitSequence, Bytes, QualifiedName, Runtime} from '@subsquid/substrate-runtime' 2 | import * as sts from '@subsquid/substrate-runtime/lib/sts' 3 | import {Option, Result} from '@subsquid/substrate-runtime/lib/sts' 4 | import assert from 'assert' 5 | 6 | 7 | export {sts, Bytes, BitSequence, Option, Result} 8 | 9 | 10 | export interface RuntimeCtx { 11 | _runtime: Runtime 12 | } 13 | 14 | 15 | export interface Block extends RuntimeCtx { 16 | hash: Bytes 17 | height: number 18 | } 19 | 20 | 21 | interface Event { 22 | block: RuntimeCtx 23 | name: QualifiedName 24 | args: unknown 25 | } 26 | 27 | 28 | interface Call { 29 | block: RuntimeCtx 30 | name: QualifiedName 31 | args: unknown 32 | } 33 | 34 | 35 | export class EventType { 36 | constructor(public readonly name: QualifiedName, private type: T) {} 37 | 38 | matches(block: RuntimeCtx): boolean { 39 | return block._runtime.events.checkType(this.name, this.type) 40 | } 41 | 42 | is(event: Event): boolean { 43 | return this.name == event.name && this.matches(event.block) 44 | } 45 | 46 | decode(event: Event): sts.GetType { 47 | assert(this.is(event)) 48 | return event.block._runtime.decodeJsonEventRecordArguments(event) 49 | } 50 | } 51 | 52 | 53 | export class CallType { 54 | constructor(public readonly name: QualifiedName, private type: T) {} 55 | 56 | matches(block: RuntimeCtx): boolean { 57 | return block._runtime.calls.checkType(this.name, this.type) 58 | } 59 | 60 | is(call: Call): boolean { 61 | return this.name == call.name && this.matches(call.block) 62 | } 63 | 64 | decode(call: Call): sts.GetType { 65 | assert(this.is(call)) 66 | return call.block._runtime.decodeJsonCallRecordArguments(call) 67 | } 68 | } 69 | 70 | 71 | export class ConstantType { 72 | constructor(private name: QualifiedName, private type: T) {} 73 | 74 | is(block: RuntimeCtx): boolean { 75 | return block._runtime.checkConstantType(this.name, this.type) 76 | } 77 | 78 | get(block: RuntimeCtx): sts.GetType { 79 | assert(this.is(block)) 80 | return block._runtime.getConstant(this.name) 81 | } 82 | } 83 | 84 | 85 | export class StorageType { 86 | constructor( 87 | private name: QualifiedName, 88 | private modifier: 'Required' | 'Optional' | 'Default', 89 | private key: sts.Type[], 90 | private value: sts.Type 91 | ) {} 92 | 93 | is(block: RuntimeCtx): boolean { 94 | return block._runtime.checkStorageType(this.name, this.modifier, this.key, this.value) 95 | } 96 | 97 | async get(block: Block, ...key: any[]): Promise { 98 | assert(this.is(block)) 99 | return block._runtime.getStorage(block.hash, this.name, ...key) 100 | } 101 | 102 | async getAll(block: Block): Promise { 103 | assert(this.is(block)) 104 | return block._runtime.queryStorage(block.hash, this.name) 105 | } 106 | 107 | async getMany(block: Block, keys: any[]): Promise { 108 | assert(this.is(block)) 109 | return block._runtime.queryStorage(block.hash, this.name, keys) 110 | } 111 | 112 | async getKeys(block: Block, ...args: any[]): Promise { 113 | assert(this.is(block)) 114 | return block._runtime.getStorageKeys(block.hash, this.name, ...args) 115 | } 116 | 117 | async getRawKeys(block: Block, ...args: any[]): Promise { 118 | assert(this.is(block)) 119 | return block._runtime.getStorageRawKeys(block.hash, this.name, ...args) 120 | } 121 | 122 | getKeysPaged(pageSize: number, block: Block, ...args: any[]): AsyncIterable { 123 | assert(this.is(block)) 124 | return block._runtime.getStorageKeysPaged(pageSize, block.hash, this.name, ...args) 125 | } 126 | 127 | async getPairs(block: Block, ...args: any[]): Promise<[key: any, value: any][]> { 128 | assert(this.is(block)) 129 | return block._runtime.getStoragePairs(block.hash, this.name, ...args) 130 | } 131 | 132 | getPairsPaged(pageSize: number, block: Block, ...args: any[]): AsyncIterable<[key: any, value: any][]> { 133 | assert(this.is(block)) 134 | return block._runtime.getStoragePairsPaged(pageSize, block.hash, this.name, ...args) 135 | } 136 | 137 | getDefault(block: Block): any { 138 | assert(this.modifier == 'Default') 139 | assert(this.is(block)) 140 | return block._runtime.getStorageFallback(this.name) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/types/v1020.ts: -------------------------------------------------------------------------------- 1 | import {sts, Result, Option, Bytes, BitSequence} from './support' 2 | 3 | export const Balance = sts.bigint() 4 | 5 | export const AccountId = sts.bytes() 6 | -------------------------------------------------------------------------------- /src/types/v1050.ts: -------------------------------------------------------------------------------- 1 | import {sts, Result, Option, Bytes, BitSequence} from './support' 2 | 3 | export const Balance = sts.bigint() 4 | 5 | export const AccountId = sts.bytes() 6 | -------------------------------------------------------------------------------- /src/types/v9130.ts: -------------------------------------------------------------------------------- 1 | import {sts, Result, Option, Bytes, BitSequence} from './support' 2 | 3 | export const AccountId32 = sts.bytes() 4 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es2020", 5 | "outDir": "lib", 6 | "rootDir": "src", 7 | "strict": true, 8 | "declaration": false, 9 | "sourceMap": true, 10 | "esModuleInterop": true, 11 | "experimentalDecorators": true, 12 | "emitDecoratorMetadata": true, 13 | "skipLibCheck": true 14 | }, 15 | "include": ["src"], 16 | "exclude": [ 17 | "node_modules" 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /typegen.json: -------------------------------------------------------------------------------- 1 | { 2 | "outDir": "src/types", 3 | "specVersions": "https://v2.archive.subsquid.io/metadata/kusama", 4 | "pallets": { 5 | "Balances": { 6 | "events": [ 7 | "Transfer" 8 | ], 9 | "calls": [], 10 | "storage": [], 11 | "constants": [] 12 | } 13 | } 14 | } 15 | --------------------------------------------------------------------------------