├── scripts ├── tmp │ └── .gitkeep └── kubo-init.js ├── config ├── prometheus │ ├── rules │ │ └── recording_rules.yml │ └── prometheus.yml ├── grafana │ ├── dashboard.yaml │ ├── datasources │ │ └── prometheus.yml │ └── dashboards │ │ └── lodestar-libp2p.json ├── docker-compose.yml └── README.md ├── bin ├── helia-gateway.js └── helia-gateway-health-check.js ├── src ├── types.d.ts ├── healthcheck.ts ├── ipns-address-utils.ts ├── content-type-parser.ts ├── dns-link-labels.ts ├── helia-server.ts ├── get-custom-helia.ts ├── constants.ts ├── helia-rpc-api.ts ├── get-custom-libp2p.ts ├── helia-http-gateway.ts └── index.ts ├── CODE_OF_CONDUCT.md ├── tsconfig.json ├── docker-compose.yml ├── .github ├── workflows │ ├── semantic-pull-request.yml │ ├── stale.yml │ ├── generated-pr.yml │ ├── playwright.yml │ ├── docker.yml │ └── gateway-conformance.yml ├── dependabot.yml └── pull_request_template.md ├── .gitignore ├── e2e-tests ├── gc.spec.ts ├── version-response.spec.ts └── smoketest.spec.ts ├── .aegir.js ├── .env-trustless-only ├── .env-all ├── .env-delegated-routing ├── patches └── @fastify+cors+8.5.0.patch ├── LICENSE-MIT ├── .env-gwc ├── Dockerfile ├── playwright.config.ts ├── debugging ├── README.md ├── until-death.sh ├── test-gateways.sh └── time-permutations.sh ├── .dockerignore ├── DEVELOPER-NOTES.md ├── package.json ├── LICENSE-APACHE └── README.md /scripts/tmp/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /config/prometheus/rules/recording_rules.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | -------------------------------------------------------------------------------- /bin/helia-gateway.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import '../dist/src/index.js' 3 | -------------------------------------------------------------------------------- /bin/helia-gateway-health-check.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import '../dist/src/health-check.js' 3 | -------------------------------------------------------------------------------- /src/types.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*/package.json' { 2 | export const name: string 3 | export const version: string 4 | } 5 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test", 9 | "e2e-tests" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | name: helia 3 | 4 | services: 5 | http-gateway: 6 | build: . 7 | restart: always 8 | ports: 9 | - "${PORT:-8080}:8080" 10 | environment: 11 | - DEBUG="${DEBUG:-helia-http-gateway*}" 12 | -------------------------------------------------------------------------------- /.github/workflows/semantic-pull-request.yml: -------------------------------------------------------------------------------- 1 | name: Semantic PR 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | main: 12 | uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | dist 4 | .docs 5 | .coverage 6 | node_modules 7 | package-lock.json 8 | yarn.lock 9 | .vscode 10 | datastore 11 | debugging/*.log 12 | scripts/tmp 13 | test 14 | test-results 15 | *-report.json 16 | *.log 17 | playwright-report 18 | .tmp-compiled-docs 19 | tsconfig-doc-check.aegir.json 20 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close Stale Issues 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 15 | -------------------------------------------------------------------------------- /.github/workflows/generated-pr.yml: -------------------------------------------------------------------------------- 1 | name: Close Generated PRs 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 15 | -------------------------------------------------------------------------------- /config/grafana/dashboard.yaml: -------------------------------------------------------------------------------- 1 | # from https://stackoverflow.com/a/74995091/592760 2 | apiVersion: 1 3 | 4 | providers: 5 | - name: "Dashboard provider" 6 | orgId: 1 7 | type: file 8 | disableDeletion: false 9 | updateIntervalSeconds: 10 10 | allowUiUpdates: true 11 | options: 12 | path: /var/lib/grafana/dashboards 13 | foldersFromFilesStructure: true 14 | -------------------------------------------------------------------------------- /src/healthcheck.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { HOST, RPC_PORT } from './constants.js' 3 | 4 | /** 5 | * This healthcheck script is used to check if the server is running and healthy 6 | */ 7 | const rootReq = await fetch(`http://${HOST}:${RPC_PORT}/api/v0/http-gateway-healthcheck`, { 8 | method: 'GET' 9 | }) 10 | const status = rootReq.status 11 | 12 | process.exit(status === 200 ? 0 : 1) 13 | -------------------------------------------------------------------------------- /e2e-tests/gc.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from '@playwright/test' 2 | import { RPC_PORT } from '../src/constants.js' 3 | 4 | test('POST /api/v0/repo/gc', async ({ page }) => { 5 | const result = await page.request.post(`http://localhost:${RPC_PORT}/api/v0/repo/gc`) 6 | expect(result?.status()).toBe(200) 7 | 8 | const maybeContent = await result?.text() 9 | expect(maybeContent).toEqual('OK') 10 | }) 11 | -------------------------------------------------------------------------------- /.aegir.js: -------------------------------------------------------------------------------- 1 | /** @type {import('aegir').PartialOptions} */ 2 | export default { 3 | dependencyCheck: { 4 | ignore: [ 5 | 'dotenv', 6 | 'typescript', 7 | 'wait-on', 8 | 'pino-pretty' // implicit dependency for fastify logging 9 | ], 10 | productionIgnorePatterns: [ 11 | '.aegir.js', 12 | 'playwright.config.ts', 13 | 'scripts/**', 14 | 'e2e-tests/**' 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /config/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s # How frequently to scrape targets by default. 3 | evaluation_interval: 15s # How frequently to evaluate rules. 4 | 5 | scrape_configs: 6 | - job_name: 'helia-http-gateway' 7 | metrics_path: '/metrics' 8 | static_configs: 9 | - targets: ['host.docker.internal:8080'] 10 | 11 | rule_files: 12 | - "rules/*.yml" # This line tells Prometheus where to find the rule files. 13 | -------------------------------------------------------------------------------- /config/grafana/datasources/prometheus.yml: -------------------------------------------------------------------------------- 1 | datasources: 2 | - name: prometheus 3 | orgId: 1 4 | type: prometheus 5 | typeName: Prometheus 6 | typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg 7 | access: proxy 8 | url: http://host.docker.internal:9090 9 | user: "" 10 | database: "" 11 | basicAuth: false 12 | isDefault: true 13 | jsonData: 14 | httpMethod: POST 15 | readOnly: false 16 | -------------------------------------------------------------------------------- /config/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | prometheus: 4 | image: prom/prometheus 5 | ports: 6 | - 9090:9090 7 | volumes: 8 | - ./prometheus:/etc/prometheus 9 | command: 10 | - '--config.file=/etc/prometheus/prometheus.yml' 11 | restart: unless-stopped 12 | 13 | grafana: 14 | image: grafana/grafana 15 | ports: 16 | - 9191:3000 17 | volumes: 18 | - ./grafana/datasources:/etc/grafana/provisioning/datasources 19 | - ./grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml 20 | - ./grafana:/var/lib/grafana 21 | restart: unless-stopped 22 | -------------------------------------------------------------------------------- /.env-trustless-only: -------------------------------------------------------------------------------- 1 | # ENV vars recommended for running with trustless gateways only 2 | USE_LIBP2P=false 3 | USE_BITSWAP=false 4 | DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' 5 | USE_TRUSTLESS_GATEWAYS=true 6 | USE_DELEGATED_ROUTING=false 7 | 8 | # Uncomment the two below to save blockstore and datastore to disk 9 | # FILE_DATASTORE_PATH=./data/datastore 10 | # FILE_BLOCKSTORE_PATH=./data/blockstore 11 | 12 | # If you want to run the gateway with a local trustless gateway, uncomment the below 13 | # PORT=8090 # or whatever port you want to run helia-http-gateway on 14 | # TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 15 | -------------------------------------------------------------------------------- /.env-all: -------------------------------------------------------------------------------- 1 | # ENV vars recommended for running with everything enabled: 2 | # * trustless gateways 3 | # * delegated routing 4 | # * libp2p 5 | # * bitswap 6 | USE_LIBP2P=true 7 | USE_BITSWAP=true 8 | DEBUG='helia-http-gateway*,*helia-fetch*' 9 | USE_TRUSTLESS_GATEWAYS=true 10 | USE_DELEGATED_ROUTING=true 11 | 12 | # Uncomment the two below to save blockstore and datastore to disk 13 | # FILE_DATASTORE_PATH=./data/datastore 14 | # FILE_BLOCKSTORE_PATH=./data/blockstore 15 | 16 | # If you want to run the gateway with a local trustless gateway, uncomment the below 17 | # PORT=8090 # or whatever port you want to run helia-http-gateway on 18 | # TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 19 | -------------------------------------------------------------------------------- /.env-delegated-routing: -------------------------------------------------------------------------------- 1 | # ENV vars recommended for running with only delegated-routing-v1-http 2 | USE_LIBP2P=false 3 | # needed to request blocks from peers we discover with delegated routing 4 | USE_BITSWAP=true 5 | # TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 6 | DEBUG='helia-http-gateway*,*helia-fetch*,*delegated-routing-v1-http-api-client*' 7 | USE_TRUSTLESS_GATEWAYS=false 8 | USE_DELEGATED_ROUTING=true 9 | 10 | # IF you're delegating to kubo running locally you should uncomment the two below: 11 | # PORT=8090 12 | # DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 13 | 14 | # Uncomment the two below to save blockstore and datastore to disk 15 | # FILE_DATASTORE_PATH=./data/datastore 16 | # FILE_BLOCKSTORE_PATH=./data/blockstore 17 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directories: 5 | - "/" 6 | schedule: 7 | interval: daily 8 | time: "10:00" 9 | open-pull-requests-limit: 20 10 | commit-message: 11 | prefix: "deps" 12 | prefix-development: "chore" 13 | groups: 14 | helia-deps: # group all deps that should be updated when Helia deps need updated 15 | patterns: 16 | - "*helia*" 17 | - "*libp2p*" 18 | - "*multiformats*" 19 | - "*blockstore*" 20 | - "*datastore*" 21 | kubo-deps: # group kubo, kubo-rpc-client, and ipfsd-ctl updates 22 | patterns: 23 | - "*kubo*" 24 | - "ipfsd-ctl" 25 | - package-ecosystem: "github-actions" 26 | directory: "/" 27 | schedule: 28 | interval: "weekly" 29 | commit-message: 30 | prefix: chore 31 | -------------------------------------------------------------------------------- /patches/@fastify+cors+8.5.0.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/@fastify/cors/index.js b/node_modules/@fastify/cors/index.js 2 | index 28dfc9a..64c5081 100644 3 | --- a/node_modules/@fastify/cors/index.js 4 | +++ b/node_modules/@fastify/cors/index.js 5 | @@ -175,7 +175,9 @@ function addCorsHeadersHandler (fastify, options, req, reply, next) { 6 | 7 | addCorsHeaders(req, reply, resolvedOriginOption, options) 8 | 9 | - if (req.raw.method === 'OPTIONS' && options.preflight === true) { 10 | + // gateway conformance tests require preflight headers even for non OPTIONS requests 11 | + // if (req.raw.method === 'OPTIONS' && options.preflight === true) { 12 | + if (options.preflight === true) { 13 | // Strict mode enforces the required headers for preflight 14 | if (options.strictPreflight === true && (!req.headers.origin || !req.headers['access-control-request-method'])) { 15 | reply.status(400).type('text/plain').send('Invalid Preflight Request') 16 | -------------------------------------------------------------------------------- /e2e-tests/version-response.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from '@playwright/test' 2 | import { RPC_PORT } from '../src/constants.js' 3 | 4 | function validateResponse (content: string): void { 5 | expect(() => JSON.parse(content)).not.toThrow() 6 | const versionObj = JSON.parse(content) 7 | 8 | expect(versionObj).toHaveProperty('Version') 9 | expect(versionObj).toHaveProperty('Commit') 10 | } 11 | 12 | test('GET /api/v0/version', async ({ page }) => { 13 | const result = await page.goto(`http://localhost:${RPC_PORT}/api/v0/version`) 14 | expect(result?.status()).toBe(200) 15 | 16 | const maybeContent = await result?.text() 17 | expect(maybeContent).not.toBe(undefined) 18 | validateResponse(maybeContent as string) 19 | }) 20 | 21 | test('POST /api/v0/version', async ({ page }) => { 22 | const result = await page.request.post(`http://localhost:${RPC_PORT}/api/v0/version`) 23 | expect(result?.status()).toBe(200) 24 | 25 | const maybeContent = await result?.text() 26 | expect(maybeContent).not.toBe(undefined) 27 | validateResponse(maybeContent) 28 | }) 29 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /config/README.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | This folder is intended to contain various configuration files for the project. 4 | 5 | ## Metrics 6 | 7 | ### Using our provided grafana dashboard with prometheus and grafana 8 | 9 | From inside this directory, you can run the following command to start up a grafana and prometheus instance with some default dashboards and datasources set up. 10 | 11 | ```sh 12 | docker compose -f docker-compose.yml up -d 13 | ``` 14 | 15 | Then visit and login with the default credentials (admin:admin). The prometheus datasource and the dashboard should be automatically set up. 16 | 17 | If you want to generate some metrics quickly, you can run `npm run debug:until-death` and you should start seeing metrics in the dashboard for the results of querying the gateway for the websites listed by 18 | 19 | If you need to reset the grafana database for whatever reason, you can try this command: `cd config && docker compose down && rm grafana/grafana.db && docker compose rm -fsv && docker compose up -d` 20 | -------------------------------------------------------------------------------- /src/ipns-address-utils.ts: -------------------------------------------------------------------------------- 1 | import { peerIdFromString } from '@libp2p/peer-id' 2 | import { CID } from 'multiformats/cid' 3 | import type { PeerId } from '@libp2p/interface' 4 | 5 | const HAS_UPPERCASE_REGEX = /[A-Z]/ 6 | 7 | interface IpnsAddressDetails { 8 | peerId: PeerId | null 9 | cid: CID | null 10 | } 11 | 12 | /** 13 | * This method should be called with the key/address value of an IPNS route. 14 | * 15 | * It will return return an object with some useful properties about the address 16 | * 17 | * @example 18 | * 19 | * http://.ipns./* 20 | * http:///ipns//* 21 | */ 22 | export function getIpnsAddressDetails (address: string): IpnsAddressDetails { 23 | let cid: CID | null = null 24 | let peerId: PeerId | null = null 25 | if (HAS_UPPERCASE_REGEX.test(address)) { 26 | try { 27 | // could be CIDv0 or PeerId at this point. 28 | cid = CID.parse(address).toV1() 29 | } catch { 30 | // ignore 31 | } 32 | 33 | try { 34 | peerId = peerIdFromString(address) 35 | } catch { 36 | // ignore error 37 | } 38 | } 39 | 40 | return { 41 | peerId, 42 | cid 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Title 2 | 3 | 9 | 10 | ## Description 11 | 12 | 18 | 19 | ## Notes & open questions 20 | 21 | 24 | 25 | ## Change checklist 26 | 27 | - [ ] I have performed a self-review of my own code 28 | - [ ] I have made corresponding changes to the documentation if necessary (this includes comments as well) 29 | - [ ] I have added tests that prove my fix is effective or that my feature works 30 | -------------------------------------------------------------------------------- /.github/workflows/playwright.yml: -------------------------------------------------------------------------------- 1 | name: Playwright Tests 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | workflow_dispatch: 8 | 9 | jobs: 10 | test: 11 | timeout-minutes: 60 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: actions/setup-node@v3 16 | with: 17 | node-version: 20 18 | - name: Install dependencies 19 | run: npm ci 20 | - name: Install Playwright Browsers 21 | run: npx playwright install --with-deps 22 | 23 | # Cache playwright binaries 24 | - uses: actions/cache@v3 25 | id: playwright-cache 26 | with: 27 | path: | 28 | ~/.cache/ms-playwright 29 | key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} 30 | - run: npx playwright install --with-deps 31 | if: steps.playwright-cache.outputs.cache-hit != 'true' 32 | 33 | # Cache datastores 34 | - uses: actions/cache@v3 35 | id: e2e-datastore-and-blockstore 36 | with: 37 | path: | 38 | ./test/fixtures/e2e 39 | key: ${{ runner.os }}-e2e-stores-${{ hashFiles('**/package-lock.json') }} 40 | 41 | - name: Run Playwright tests 42 | run: npm run test:e2e 43 | env: 44 | METRICS: false 45 | - uses: actions/upload-artifact@v4 46 | if: always() 47 | with: 48 | name: playwright-report 49 | path: playwright-report/ 50 | retention-days: 30 51 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Publish docker images 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | workflow_dispatch: 9 | inputs: 10 | push: 11 | description: 'Push to registry' 12 | required: false 13 | default: 'false' 14 | 15 | jobs: 16 | publish: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - uses: docker/login-action@v3 22 | with: 23 | registry: ghcr.io 24 | username: ${{ github.actor }} 25 | password: ${{ github.token }} 26 | 27 | - name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@v3 29 | 30 | - name: Docker meta 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: ghcr.io/${{ github.repository }} 35 | # https://github.com/docker/metadata-action/tree/v5/#latest-tag 36 | tags: | 37 | type=raw,value=latest,enable={{is_default_branch}} 38 | type=ref,event=branch 39 | 40 | - name: Build and push 41 | uses: docker/build-push-action@v5 42 | with: 43 | context: . 44 | file: ./Dockerfile 45 | provenance: false 46 | push: ${{ github.event_name == 'push' || github.event.inputs.push == 'true' }} 47 | tags: ${{ steps.meta.outputs.tags }} 48 | labels: ${{ steps.meta.outputs.labels }} 49 | platforms: linux/arm64, linux/amd64 50 | -------------------------------------------------------------------------------- /.env-gwc: -------------------------------------------------------------------------------- 1 | # ENV vars recommended for running gateway-conformance tests 2 | export USE_LIBP2P=true 3 | export USE_BITSWAP=true 4 | export USE_SUBDOMAINS=false 5 | export PORT="8080" # helia-http-gateway should be running here 6 | export KUBO_PORT="8081" # Kubo should be running here 7 | export TRUSTLESS_GATEWAYS="http://127.0.0.1:8081" # Kubo should be running here 8 | export DELEGATED_ROUTING_V1_HOST="http://127.0.0.1:8081" # Kubo should be running here 9 | # DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' 10 | export DEBUG='helia*,helia*:trace' 11 | export USE_TRUSTLESS_GATEWAYS=true 12 | export USE_DELEGATED_ROUTING=true 13 | 14 | # Uncomment the two below to save blockstore and datastore to disk 15 | # FILE_DATASTORE_PATH=./data/datastore 16 | # FILE_BLOCKSTORE_PATH=./data/blockstore 17 | 18 | # Uncomment the below to see request & response headers in the logs 19 | # ECHO_HEADERS=true 20 | 21 | export GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 22 | 23 | # skip most of the tests 24 | export GWC_SKIP="^.*(TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" 25 | export GWC_GATEWAY_URL="http://helia-http-gateway.localhost" 26 | # GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" 27 | # GWC_GATEWAY_URL="http://127.0.0.1:8080" 28 | export GWC_GATEWAY_URL="http://host.docker.internal:8080" 29 | export GWC_SUBDOMAIN_URL="http://host.docker.internal:8080" 30 | -------------------------------------------------------------------------------- /src/content-type-parser.ts: -------------------------------------------------------------------------------- 1 | import { fileTypeFromBuffer } from '@sgtpooki/file-type' 2 | 3 | // default from verified-fetch is application/octect-stream, which forces a download. This is not what we want for MANY file types. 4 | const defaultMimeType = 'text/html; charset=utf-8' 5 | function checkForSvg (bytes: Uint8Array): string { 6 | return /^(<\?xml[^>]+>)?[^<^\w]+ { 13 | const detectedType = (await fileTypeFromBuffer(bytes))?.mime 14 | if (detectedType != null) { 15 | return detectedType 16 | } 17 | 18 | if (fileName == null) { 19 | // no other way to determine file-type. 20 | return checkForSvg(bytes) 21 | } 22 | 23 | // no need to include file-types listed at https://github.com/SgtPooki/file-type#supported-file-types 24 | switch (fileName.split('.').pop()) { 25 | case 'css': 26 | return 'text/css' 27 | case 'html': 28 | return 'text/html; charset=utf-8' 29 | case 'js': 30 | return 'application/javascript' 31 | case 'json': 32 | return 'application/json' 33 | case 'txt': 34 | return 'text/plain' 35 | case 'woff2': 36 | return 'font/woff2' 37 | // see bottom of https://github.com/SgtPooki/file-type#supported-file-types 38 | case 'svg': 39 | return 'image/svg+xml' 40 | case 'csv': 41 | return 'text/csv' 42 | case 'doc': 43 | return 'application/msword' 44 | case 'xls': 45 | return 'application/vnd.ms-excel' 46 | case 'ppt': 47 | return 'application/vnd.ms-powerpoint' 48 | case 'msi': 49 | return 'application/x-msdownload' 50 | default: 51 | return defaultMimeType 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Application Build Stage 2 | FROM --platform=$BUILDPLATFORM node:20-slim as builder 3 | 4 | # Install dependencies required for building the app 5 | RUN apt-get update && \ 6 | apt-get install -y build-essential wget tini && \ 7 | apt-get clean && \ 8 | rm -rf /var/lib/apt/lists/* 9 | 10 | WORKDIR /app 11 | 12 | COPY package*.json ./ 13 | RUN npm ci --quiet 14 | 15 | COPY . . 16 | RUN npm run build 17 | RUN npm prune --omit=dev 18 | 19 | # Final Stage 20 | FROM --platform=$BUILDPLATFORM node:20-slim as app 21 | 22 | ENV NODE_ENV production 23 | WORKDIR /app 24 | 25 | # copy built application from the builder stage 26 | COPY --from=builder /app ./ 27 | 28 | # copy tini from the builder stage 29 | COPY --from=builder /usr/bin/tini /usr/bin/tini 30 | 31 | # port for RPC API 32 | EXPOSE 5001 33 | 34 | # port for HTTP Gateway 35 | EXPOSE 8080 36 | 37 | HEALTHCHECK --interval=12s --timeout=12s --start-period=10s CMD node dist/src/healthcheck.js 38 | 39 | # use level datastore by default 40 | ENV FILE_DATASTORE_PATH=/data/ipfs/datastore 41 | 42 | # use filesystem blockstore by default 43 | ENV FILE_BLOCKSTORE_PATH=/data/ipfs/blockstore 44 | 45 | # enable metrics by default 46 | ENV METRICS=true 47 | 48 | # redirect ipfs/ipns paths to subdomains to prevent cookie theft by websites 49 | # loaded via the gateway 50 | ENV SUB_DOMAINS=true 51 | 52 | # Use tini to handle signals properly, see https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#handling-kernel-signals 53 | ENTRYPOINT ["/usr/bin/tini", "-p", "SIGKILL", "--"] 54 | 55 | CMD [ "node", "dist/src/index.js" ] 56 | 57 | # for best practices, see: 58 | # * https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/ 59 | # * https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md 60 | # * https://nodejs.org/en/docs/guides/nodejs-docker-webapp 61 | -------------------------------------------------------------------------------- /e2e-tests/smoketest.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from '@playwright/test' 2 | import { HTTP_PORT } from '../src/constants.js' 3 | 4 | // test all the same pages listed at https://probelab.io/websites/ 5 | const pages = [ 6 | 'blog.ipfs.tech', 7 | 'blog.libp2p.io', 8 | 'consensuslab.world', 9 | 'docs.ipfs.tech', 10 | 'docs.libp2p.io', 11 | // 'drand.love', // no dnsaddr or dnslink TXT record, only "x-ipfs-path" header (supported only by ipfs-companion and brave) 12 | 'fil.org', 13 | 'filecoin.io', 14 | 'green.filecoin.io', 15 | 'ipfs.tech', 16 | 'ipld.io', 17 | 'libp2p.io', 18 | 'n0.computer', 19 | 'probelab.io', 20 | 'protocol.ai', 21 | 'research.protocol.ai', 22 | // 'singularity.storage', // broken as of 09-07-2024 23 | 'specs.ipfs.tech', 24 | // 'strn.network', // redirects to saturn.tech 25 | 'saturn.tech', 26 | 'web3.storage' 27 | ] 28 | 29 | // increase default test timeout to 2 minutes 30 | test.setTimeout(120000) 31 | 32 | // now for each page, make sure we can request the website, the content is not empty, and status code is 200 33 | test.beforeEach(async ({ context }) => { 34 | // Block any asset requests for tests in this file. 35 | await context.route(/.(css|js|svg|png|jpg|woff2|otf|webp|ttf|json)(?:\?.*)?$/, async route => route.abort()) 36 | }) 37 | 38 | pages.forEach((pagePath) => { 39 | const url = `http://${pagePath}.ipns.localhost:${HTTP_PORT}` 40 | test(`helia-http-gateway can load path '${url}'`, async ({ page }) => { 41 | // only wait for 'commit' because we don't want to wait for all the assets to load, we just want to make sure that they *would* load (e.g. the html is valid) 42 | const heliaGatewayResponse = await page.goto(`${url}`, { waitUntil: 'commit' }) 43 | expect(heliaGatewayResponse?.status()).toBe(200) 44 | // await page.waitForSelector('body') 45 | expect(await heliaGatewayResponse?.text()).not.toEqual('') 46 | const headers = heliaGatewayResponse?.headers() 47 | expect(headers).not.toBeNull() 48 | expect(headers?.['content-type']).toContain('text/') 49 | }) 50 | }) 51 | -------------------------------------------------------------------------------- /src/dns-link-labels.ts: -------------------------------------------------------------------------------- 1 | import { CID } from 'multiformats/cid' 2 | 3 | /** 4 | * For dnslinks see https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header 5 | * DNSLink names include . which means they must be inlined into a single DNS label to provide unique origin and work with wildcard TLS certificates. 6 | */ 7 | 8 | // DNS label can have up to 63 characters, consisting of alphanumeric 9 | // characters or hyphens -, but it must not start or end with a hyphen. 10 | const dnsLabelRegex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/ 11 | 12 | /** 13 | * We can receive either IPNS Name string or DNSLink label string here. 14 | * IPNS Names do not have dots or dashes. 15 | */ 16 | export function isValidDnsLabel (label: string): boolean { 17 | // If string is not a valid IPNS Name (CID) 18 | // then we assume it may be a valid DNSLabel. 19 | try { 20 | CID.parse(label) 21 | return false 22 | } catch { 23 | return dnsLabelRegex.test(label) 24 | } 25 | } 26 | 27 | /** 28 | * Checks if label looks like inlined DNSLink. 29 | * (https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header) 30 | */ 31 | export function isInlinedDnsLink (label: string): boolean { 32 | return isValidDnsLabel(label) && label.includes('-') && !label.includes('.') 33 | } 34 | 35 | /** 36 | * DNSLink label decoding 37 | * Every standalone - is replaced with . 38 | * Every remaining -- is replaced with - 39 | * 40 | * @example en-wikipedia--on--ipfs-org.ipns.example.net -> example.net/ipns/en.wikipedia-on-ipfs.org 41 | */ 42 | export function dnsLinkLabelDecoder (linkLabel: string): string { 43 | return linkLabel.replace(/--/g, '%').replace(/-/g, '.').replace(/%/g, '-') 44 | } 45 | 46 | /** 47 | * DNSLink label encoding: 48 | * Every - is replaced with -- 49 | * Every . is replaced with - 50 | * 51 | * @example example.net/ipns/en.wikipedia-on-ipfs.org → Host: en-wikipedia--on--ipfs-org.ipns.example.net 52 | */ 53 | export function dnsLinkLabelEncoder (linkLabel: string): string { 54 | return linkLabel.replace(/-/g, '--').replace(/\./g, '-') 55 | } 56 | -------------------------------------------------------------------------------- /src/helia-server.ts: -------------------------------------------------------------------------------- 1 | import { setMaxListeners } from 'node:events' 2 | import type { Logger } from '@libp2p/interface' 3 | import type { FastifyRequest } from 'fastify' 4 | 5 | export function getFullUrlFromFastifyRequest (request: FastifyRequest, log: Logger): string { 6 | let query = '' 7 | if (request.query != null) { 8 | log('request.query:', request.query) 9 | const pairs: string[] = [] 10 | Object.keys(request.query).forEach((key: string) => { 11 | const value = (request.query as Record)[key] 12 | pairs.push(`${key}=${value}`) 13 | }) 14 | if (pairs.length > 0) { 15 | query += '?' + pairs.join('&') 16 | } 17 | } 18 | 19 | return `${request.protocol}://${request.hostname}${request.url}${query}` 20 | } 21 | 22 | export interface GetRequestAwareSignalOpts { 23 | timeout?: number 24 | url?: string 25 | } 26 | 27 | export function getRequestAwareSignal (request: FastifyRequest, log: Logger, options: GetRequestAwareSignalOpts = {}): AbortSignal { 28 | const url = options.url ?? getFullUrlFromFastifyRequest(request, log) 29 | 30 | const opController = new AbortController() 31 | setMaxListeners(Infinity, opController.signal) 32 | const cleanupFn = (): void => { 33 | if (request.raw.readableAborted) { 34 | log.trace('request aborted by client for url "%s"', url) 35 | } else if (request.raw.destroyed) { 36 | log.trace('request destroyed for url "%s"', url) 37 | } else if (request.raw.complete) { 38 | log.trace('request closed or ended in completed state for url "%s"', url) 39 | } else { 40 | log.trace('request closed or ended gracefully for url "%s"', url) 41 | } 42 | 43 | // we want to stop all further processing because the request is closed 44 | opController.abort() 45 | } 46 | 47 | /** 48 | * The 'close' event is emitted when the stream and any of its underlying resources (a file descriptor, for example) have been closed. The event indicates that no more events will be emitted, and no further computation will occur. 49 | * A Readable stream will always emit the 'close' event if it is created with the emitClose option. 50 | * 51 | * @see https://nodejs.org/api/stream.html#event-close_1 52 | */ 53 | request.raw.on('close', cleanupFn) 54 | 55 | if (options.timeout != null) { 56 | setTimeout(() => { 57 | log.trace('request timed out for url "%s"', url) 58 | opController.abort() 59 | }, options.timeout) 60 | } 61 | 62 | return opController.signal 63 | } 64 | -------------------------------------------------------------------------------- /src/get-custom-helia.ts: -------------------------------------------------------------------------------- 1 | import { bitswap, trustlessGateway } from '@helia/block-brokers' 2 | import { createHeliaHTTP } from '@helia/http' 3 | import { delegatedHTTPRouting, httpGatewayRouting } from '@helia/routers' 4 | import { FsBlockstore } from 'blockstore-fs' 5 | import { LevelDatastore } from 'datastore-level' 6 | import { createHelia } from 'helia' 7 | import { DELEGATED_ROUTING_V1_HOST, FILE_BLOCKSTORE_PATH, FILE_DATASTORE_PATH, TRUSTLESS_GATEWAYS, USE_BITSWAP, USE_DELEGATED_ROUTING, USE_LIBP2P, USE_TRUSTLESS_GATEWAYS } from './constants.js' 8 | import { getCustomLibp2p } from './get-custom-libp2p.js' 9 | import type { Helia } from '@helia/interface' 10 | import type { HeliaInit } from 'helia' 11 | 12 | export async function getCustomHelia (): Promise { 13 | const datastore = await configureDatastore() 14 | 15 | if (USE_LIBP2P || USE_BITSWAP) { 16 | return createHelia({ 17 | libp2p: await getCustomLibp2p({ datastore }), 18 | blockstore: await configureBlockstore(), 19 | datastore, 20 | blockBrokers: configureBlockBrokers(), 21 | routers: configureRouters() 22 | }) 23 | } 24 | 25 | return createHeliaHTTP({ 26 | blockstore: await configureBlockstore(), 27 | datastore, 28 | blockBrokers: configureBlockBrokers(), 29 | routers: configureRouters() 30 | }) 31 | } 32 | 33 | async function configureBlockstore (): Promise { 34 | if (FILE_BLOCKSTORE_PATH != null && FILE_BLOCKSTORE_PATH !== '') { 35 | const fs = new FsBlockstore(FILE_BLOCKSTORE_PATH) 36 | await fs.open() 37 | return fs 38 | } 39 | } 40 | 41 | async function configureDatastore (): Promise { 42 | if (FILE_DATASTORE_PATH != null && FILE_DATASTORE_PATH !== '') { 43 | const db = new LevelDatastore(FILE_DATASTORE_PATH) 44 | await db.open() 45 | 46 | return db 47 | } 48 | } 49 | 50 | function configureBlockBrokers (): HeliaInit['blockBrokers'] { 51 | const blockBrokers: HeliaInit['blockBrokers'] = [] 52 | 53 | if (USE_BITSWAP) { 54 | blockBrokers.push(bitswap()) 55 | } 56 | 57 | if (USE_TRUSTLESS_GATEWAYS) { 58 | blockBrokers.push(trustlessGateway()) 59 | } 60 | 61 | return blockBrokers 62 | } 63 | 64 | function configureRouters (): HeliaInit['routers'] { 65 | const routers: HeliaInit['routers'] = [] 66 | 67 | if (TRUSTLESS_GATEWAYS != null) { 68 | routers.push(httpGatewayRouting({ 69 | gateways: TRUSTLESS_GATEWAYS 70 | })) 71 | } 72 | 73 | if (USE_DELEGATED_ROUTING) { 74 | routers.push(delegatedHTTPRouting(DELEGATED_ROUTING_V1_HOST)) 75 | } 76 | 77 | return routers 78 | } 79 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Where we listen for gateway requests 3 | */ 4 | export const HTTP_PORT = Number(process.env.HTTP_PORT ?? 8080) 5 | 6 | /** 7 | * Where we listen for RPC API requests 8 | */ 9 | export const RPC_PORT = Number(process.env.RPC_PORT ?? 5001) 10 | 11 | export const HOST = process.env.HOST ?? '0.0.0.0' 12 | 13 | export const DEBUG = process.env.DEBUG ?? '' 14 | export const FASTIFY_DEBUG = process.env.FASTIFY_DEBUG ?? '' 15 | 16 | export const USE_SUBDOMAINS = process.env.USE_SUBDOMAINS !== 'false' 17 | 18 | export const USE_SESSIONS = process.env.USE_SESSIONS !== 'false' 19 | 20 | export const ECHO_HEADERS = process.env.ECHO_HEADERS === 'true' 21 | 22 | /** 23 | * If set to any value other than 'true', we will disable prometheus metrics. 24 | * 25 | * @default 'true' 26 | */ 27 | export const METRICS = process.env.METRICS ?? 'true' 28 | 29 | /** 30 | * If not set, we will enable bitswap by default. 31 | */ 32 | export const USE_BITSWAP = process.env.USE_BITSWAP !== 'false' 33 | 34 | /** 35 | * If not set, we will use the default gateways that come from https://github.com/ipfs/helia/blob/43932a54036dafdf1265b034b30b12784fd22d82/packages/helia/src/block-brokers/trustless-gateway/index.ts 36 | */ 37 | export const TRUSTLESS_GATEWAYS = process.env.TRUSTLESS_GATEWAYS?.split(',') ?? undefined 38 | 39 | /** 40 | * If not set, we will use trustless gateways by default. 41 | */ 42 | export const USE_TRUSTLESS_GATEWAYS = process.env.USE_TRUSTLESS_GATEWAYS !== 'false' 43 | 44 | /** 45 | * If not set, we will enable libp2p by default. 46 | */ 47 | export const USE_LIBP2P = process.env.USE_LIBP2P !== 'false' 48 | 49 | /** 50 | * If not set, we will use a memory datastore by default. 51 | */ 52 | export const FILE_DATASTORE_PATH = process.env.FILE_DATASTORE_PATH ?? undefined 53 | 54 | /** 55 | * If not set, we will use a memory blockstore by default. 56 | */ 57 | export const FILE_BLOCKSTORE_PATH = process.env.FILE_BLOCKSTORE_PATH ?? undefined 58 | 59 | /** 60 | * Whether to use the delegated routing v1 API. Defaults to true. 61 | */ 62 | export const USE_DELEGATED_ROUTING = process.env.USE_DELEGATED_ROUTING !== 'false' 63 | 64 | /** 65 | * Whether to use the DHT for routing 66 | * 67 | * @default true 68 | */ 69 | export const USE_DHT_ROUTING = process.env.USE_DHT_ROUTING !== 'false' 70 | 71 | /** 72 | * If not set, we will default delegated routing to `https://delegated-ipfs.dev` 73 | */ 74 | export const DELEGATED_ROUTING_V1_HOST = process.env.DELEGATED_ROUTING_V1_HOST ?? 'https://delegated-ipfs.dev' 75 | 76 | /** 77 | * How long to wait for GC to complete 78 | */ 79 | export const GC_TIMEOUT_MS = 20000 80 | 81 | /** 82 | * How long to wait for the healthcheck retrieval of an identity CID to complete 83 | */ 84 | export const HEALTHCHECK_TIMEOUT_MS = 1000 85 | -------------------------------------------------------------------------------- /playwright.config.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'node:path' 2 | import { defineConfig, devices } from '@playwright/test' 3 | import { HTTP_PORT } from './src/constants.js' 4 | 5 | /** 6 | * Run one of the variants of `npm run start` by setting the PLAYWRIGHT_START_CMD_MOD environment variable. 7 | * 8 | * For example, to run `npm run start:dev-doctor`: `PLAYWRIGHT_START_CMD_MOD=:dev-doctor npm run test:e2e` 9 | * For example, to run `npm run start:env-to`: `PLAYWRIGHT_START_CMD_MOD=:env-to npm run test:e2e` 10 | */ 11 | const PLAYWRIGHT_START_CMD_MOD = process.env.PLAYWRIGHT_START_CMD_MOD ?? '' 12 | 13 | /** 14 | * See https://playwright.dev/docs/test-configuration. 15 | */ 16 | export default defineConfig({ 17 | testDir: './e2e-tests', 18 | /* Run tests in files in parallel */ 19 | fullyParallel: true, 20 | /* Fail the build on CI if you accidentally left test.only in the source code. */ 21 | forbidOnly: Boolean(process.env.CI), 22 | /* Retry on CI only */ 23 | retries: (process.env.CI != null) ? 2 : 0, 24 | /** 25 | * Opt out of parallel tests by setting workers to 1. 26 | * We don't want to bombard Helia gateway with parallel requests, it's not ready for that yet. 27 | */ 28 | workers: 1, 29 | /* Reporter to use. See https://playwright.dev/docs/test-reporters */ 30 | reporter: 'html', 31 | /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ 32 | use: { 33 | /* Base URL to use in actions like `await page.goto('/')`. */ 34 | // baseURL: 'http://127.0.0.1:3000', 35 | 36 | /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ 37 | trace: 'on-first-retry' 38 | }, 39 | maxFailures: 5, 40 | 41 | /* Configure projects for major browsers */ 42 | projects: [ 43 | { 44 | name: 'chromium', 45 | use: { ...devices['Desktop Chrome'] } 46 | } 47 | 48 | // { 49 | // name: 'firefox', 50 | // use: { ...devices['Desktop Firefox'] }, 51 | // }, 52 | 53 | // { 54 | // name: 'webkit', 55 | // use: { ...devices['Desktop Safari'] }, 56 | // }, 57 | 58 | ], 59 | 60 | /* Run your local dev server before starting the tests */ 61 | webServer: { 62 | command: `npm run build && npm run start${PLAYWRIGHT_START_CMD_MOD}`, 63 | port: HTTP_PORT, 64 | // Tiros does not re-use the existing server. 65 | reuseExistingServer: process.env.CI == null, 66 | env: { 67 | METRICS: process.env.METRICS ?? 'false', 68 | DEBUG: process.env.DEBUG ?? ' ', 69 | // we save to the filesystem so github CI can cache the data. 70 | FILE_BLOCKSTORE_PATH: process.env.FILE_BLOCKSTORE_PATH ?? join(process.cwd(), 'test', 'fixtures', 'e2e', 'blockstore'), 71 | FILE_DATASTORE_PATH: process.env.FILE_DATASTORE_PATH ?? (process.cwd(), 'test', 'fixtures', 'e2e', 'datastore') 72 | } 73 | } 74 | }) 75 | -------------------------------------------------------------------------------- /debugging/README.md: -------------------------------------------------------------------------------- 1 | This file documents some methods used for debugging and testing of helia-http-gateway. 2 | 3 | Any files in this directory should be considered temporary and may be deleted at any time. 4 | 5 | You should also run any of these scripts from the repository root. 6 | 7 | # Scripts 8 | 9 | ## test-gateways.sh 10 | This script is used to test the gateways. It assumes you have booted up the `helia-http-gateway` via docker or otherwise and will query the gateway for the same websites listed at https://probelab.io/websites/#time-to-first-byte-using-kubo, outputting HTTP status codes and response times. 11 | 12 | *Example* 13 | ```sh 14 | ./debugging/test-gateways.sh 15 | ``` 16 | 17 | ## until-death.sh 18 | This script will start up the gateway and run the `test-gateway.sh` script until the gateway dies. This is useful for load-testing helia-http-gateway in a similar manner to how it will be used by https://github.com/plprobelab/tiros 19 | 20 | *Example* 21 | ```sh 22 | ./debugging/until-death.sh 23 | ``` 24 | 25 | # Debugging the docker container 26 | 27 | ## Setup 28 | 29 | First, build the docker container. This will tag the container as `helia-http-gateway:local-` (or `helia-http-gateway:local-arm64` on M1 macOS) and will be used in the following examples. 30 | 31 | ```sh 32 | USE_SUBDOMAINS=true USE_REDIRECTS=false docker build . --platform linux/$(arch) --tag helia-http-gateway:local-$(arch) 33 | `````` 34 | 35 | Then we need to start the container 36 | 37 | ```sh 38 | docker run -it -p 5001:5001 -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway:local-$(arch) 39 | # or 40 | docker run -it -p 5001:5001 -p 8080:8080 -e DEBUG="helia-http-gateway*" -e USE_REDIRECTS="false" -e USE_SUBDOMAINS="true" helia-http-gateway:local-$(arch) 41 | ``` 42 | 43 | ## Running tests against the container 44 | 45 | Then you just need to execute one of the debugging scripts: 46 | 47 | ```sh 48 | npm run debug:until-death # continuous testing until the container dies (hopefully it doesn't) 49 | npm run debug:test-gateways # one round of testing all the websites tiros will test. 50 | ``` 51 | 52 | # Profiling in chrome/chromium devtools 53 | 54 | ## Setup 55 | 56 | 1. Start the process 57 | 58 | ```sh 59 | # in terminal 1 60 | npm run start:inspect 61 | ``` 62 | 63 | 2. Open `chrome://inspect` and click the 'inspect' link for the process you just started 64 | * it should say something like `dist/src/index.js file:///Users/sgtpooki/code/work/protocol.ai/ipfs/helia-http-gateway/dist/src/index.js` with an 'inspect' link below it. 65 | 66 | 3. In the inspector, click the `performance` or `memory` tab and start recording. 67 | 68 | ## Execute the operations that will be profiled: 69 | 70 | 1. In another terminal, run 71 | 72 | ```sh 73 | # in terminal 2 74 | npm run debug:test-gateways # or npm run debug:until-death 75 | ``` 76 | 77 | 2. Stop recording in the inspector and analyze the results. 78 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | build 94 | 95 | # Gatsby files 96 | .cache/ 97 | # Comment in the public line in if your project uses Gatsby and not Next.js 98 | # https://nextjs.org/blog/next-9-1#public-directory-support 99 | # public 100 | 101 | # vuepress build output 102 | .vuepress/dist 103 | 104 | # vuepress v2.x temp and cache directory 105 | .temp 106 | .cache 107 | 108 | # Docusaurus cache and generated files 109 | .docusaurus 110 | 111 | # Serverless directories 112 | .serverless/ 113 | 114 | # FuseBox cache 115 | .fusebox/ 116 | 117 | # DynamoDB Local files 118 | .dynamodb/ 119 | 120 | # TernJS port file 121 | .tern-port 122 | 123 | # Stores VSCode versions used for testing VSCode extensions 124 | .vscode-test 125 | 126 | # yarn v2 127 | .yarn/cache 128 | .yarn/unplugged 129 | .yarn/build-state.yml 130 | .yarn/install-state.gz 131 | .pnp.* 132 | 133 | # other dotfiles 134 | .git 135 | .gitignore 136 | .dockerignore 137 | .github 138 | .tool-versions 139 | .envrc 140 | 141 | # playwright stuff 142 | e2e-tests 143 | playwright.config.ts 144 | playwright-report 145 | screenshots 146 | 147 | # Other misc files that don't make sense in the docker container 148 | README.md 149 | -------------------------------------------------------------------------------- /debugging/until-death.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # If this is not executed from the root of the helia-http-gateway repository, then exit with non-zero error code 4 | if [ ! -f "package.json" ]; then 5 | echo "This script must be executed from the root of the helia-http-gateway repository" 6 | exit 1 7 | fi 8 | 9 | # You have to pass `DEBUG=" " to disable debugging when using this script` 10 | export DEBUG=${DEBUG:-"helia-http-gateway,helia-http-gateway:server,helia-http-gateway:*:helia-fetch"} 11 | export HTTP_PORT=${HTTP_PORT:-8080} 12 | export RPC_PORT=${RPC_PORT:-5001} 13 | export FILE_DATASTORE_PATH=datastore 14 | export FILE_BLOCKSTORE_PATH=blockstore 15 | EXIT_CODE=0 16 | 17 | cleanup_until_death_called=false 18 | cleanup_until_death() { 19 | if [ "$cleanup_until_death_called" = true ]; then 20 | echo "cleanup_until_death_called already called" 21 | return 22 | fi 23 | echo "cleanup_until_death called" 24 | cleanup_until_death_called=true 25 | if [ "$gateway_already_running" != true ]; then 26 | lsof -i TCP:$HTTP_PORT | grep LISTEN | awk '{print $2}' | xargs --no-run-if-empty kill -9 27 | 28 | echo "waiting for the gateway to exit" 29 | npx wait-on "tcp:$HTTP_PORT" -t 10000 -r # wait for the port to be released 30 | fi 31 | 32 | exit $EXIT_CODE 33 | } 34 | 35 | trap cleanup_until_death EXIT 36 | 37 | # Before starting, output all env vars that helia-http-gateway uses 38 | echo "DEBUG=$DEBUG" 39 | echo "FASTIFY_DEBUG=$FASTIFY_DEBUG" 40 | echo "HTTP_PORT=$HTTP_PORT" 41 | echo "RPC_PORT=$RPC_PORT" 42 | echo "HOST=$HOST" 43 | echo "USE_SUBDOMAINS=$USE_SUBDOMAINS" 44 | echo "METRICS=$METRICS" 45 | echo "USE_BITSWAP=$USE_BITSWAP" 46 | echo "USE_TRUSTLESS_GATEWAYS=$USE_TRUSTLESS_GATEWAYS" 47 | echo "TRUSTLESS_GATEWAYS=$TRUSTLESS_GATEWAYS" 48 | echo "USE_LIBP2P=$USE_LIBP2P" 49 | echo "ECHO_HEADERS=$ECHO_HEADERS" 50 | echo "USE_DELEGATED_ROUTING=$USE_DELEGATED_ROUTING" 51 | echo "DELEGATED_ROUTING_V1_HOST=$DELEGATED_ROUTING_V1_HOST" 52 | echo "FILE_DATASTORE_PATH=$FILE_DATASTORE_PATH" 53 | echo "FILE_BLOCKSTORE_PATH=$FILE_BLOCKSTORE_PATH" 54 | echo "ALLOW_UNHANDLED_ERROR_RECOVERY=$ALLOW_UNHANDLED_ERROR_RECOVERY" 55 | 56 | gateway_already_running=false 57 | if nc -z localhost $HTTP_PORT; then 58 | echo "gateway is already running" 59 | gateway_already_running=true 60 | fi 61 | 62 | start_gateway() { 63 | if [ "$gateway_already_running" = true ]; then 64 | echo "gateway is already running" 65 | return 66 | fi 67 | # if DEBUG_NO_BUILD is set, then we assume the gateway is already built 68 | if [ "$DEBUG_NO_BUILD" != true ]; then 69 | npm run build 70 | fi 71 | echo "starting gateway..." 72 | # npx clinic doctor --open=false -- node dist/src/index.js & 73 | (node --trace-warnings --inspect dist/src/index.js) & 74 | process_id=$! 75 | # echo "process id: $!" 76 | npx wait-on "tcp:$HTTP_PORT" -t 10000 || { 77 | EXIT_CODE=1 78 | cleanup_until_death 79 | } 80 | } 81 | start_gateway 82 | 83 | ensure_gateway_running() { 84 | npx wait-on "tcp:$HTTP_PORT" -t 5000 || { 85 | EXIT_CODE=1 86 | cleanup_until_death 87 | } 88 | } 89 | 90 | max_timeout=${1:-15} 91 | while [ $? -ne 1 ]; do 92 | ensure_gateway_running 93 | ./debugging/test-gateways.sh $max_timeout # 2>&1 | tee -a debugging/test-gateways.log 94 | done 95 | 96 | cleanup_until_death 97 | -------------------------------------------------------------------------------- /debugging/test-gateways.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | EXIT_CODE=0 4 | cleanup_gateway_test_called=false 5 | cleanup_gateway_test() { 6 | if [ "$cleanup_gateway_test_called" = true ]; then 7 | echo "cleanup_gateway_test already called" 8 | return 9 | fi 10 | echo "cleanup_gateway_test called" 11 | cleanup_gateway_test_called=true 12 | 13 | exit $EXIT_CODE 14 | } 15 | 16 | trap cleanup_gateway_test EXIT 17 | 18 | # Query all endpoints until failure 19 | # This script is intended to be run from the root of the helia-http-gateway repository 20 | 21 | HTTP_PORT=${HTTP_PORT:-8080} 22 | RPC_PORT=${RPC_PORT:-5001} 23 | # If localhost:$HTTP_PORT is not listening, then exit with non-zero error code 24 | if ! nc -z localhost $HTTP_PORT; then 25 | echo "localhost:$HTTP_PORT is not listening" 26 | exit 1 27 | fi 28 | 29 | ensure_gateway_running() { 30 | npx wait-on "tcp:$HTTP_PORT" -t 1000 || { 31 | EXIT_CODE=1 32 | cleanup_gateway_test 33 | } 34 | } 35 | 36 | # Use the first argument to this script (if any) as the maximum timeout for curl 37 | max_timeout=${1:-60} 38 | test_website() { 39 | ensure_gateway_running 40 | local website=$1 41 | echo "Requesting $website" 42 | curl -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds (TTFB: %{time_starttransfer}, redirect: %{time_redirect})\n" -L $website 43 | echo "running GC" 44 | curl -X POST -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds\n" http://localhost:$RPC_PORT/api/v0/repo/gc 45 | } 46 | 47 | #test_website http://localhost:$HTTP_PORT/ipns/blog.ipfs.tech 48 | 49 | #test_website http://localhost:$HTTP_PORT/ipns/blog.libp2p.io 50 | 51 | #test_website http://localhost:$HTTP_PORT/ipns/consensuslab.world 52 | 53 | #test_website http://localhost:$HTTP_PORT/ipns/docs.ipfs.tech 54 | 55 | #test_website http://localhost:$HTTP_PORT/ipns/docs.libp2p.io 56 | 57 | # test_website http://localhost:$HTTP_PORT/ipns/drand.love #drand.love is not publishing dnslink records 58 | 59 | #test_website http://localhost:$HTTP_PORT/ipns/fil.org 60 | 61 | #test_website http://localhost:$HTTP_PORT/ipns/filecoin.io 62 | 63 | test_website http://localhost:$HTTP_PORT/ipns/green.filecoin.io 64 | 65 | #test_website http://localhost:$HTTP_PORT/ipns/ipfs.tech 66 | 67 | #test_website http://localhost:$HTTP_PORT/ipns/ipld.io 68 | 69 | #test_website http://localhost:$HTTP_PORT/ipns/libp2p.io 70 | 71 | #test_website http://localhost:$HTTP_PORT/ipns/n0.computer 72 | 73 | #test_website http://localhost:$HTTP_PORT/ipns/probelab.io 74 | 75 | #test_website http://localhost:$HTTP_PORT/ipns/protocol.ai 76 | 77 | #test_website http://localhost:$HTTP_PORT/ipns/research.protocol.ai 78 | 79 | #test_website http://localhost:$HTTP_PORT/ipns/singularity.storage 80 | 81 | #test_website http://localhost:$HTTP_PORT/ipns/specs.ipfs.tech 82 | 83 | # test_website http://localhost:$HTTP_PORT/ipns/strn.network 84 | #test_website http://localhost:$HTTP_PORT/ipns/saturn.tech 85 | 86 | test_website http://localhost:$HTTP_PORT/ipns/web3.storage 87 | 88 | #test_website http://localhost:$HTTP_PORT/ipfs/bafkreiezuss4xkt5gu256vjccx7vocoksxk77vwmdrpwoumfbbxcy2zowq # stock images 3 sec skateboarder video 89 | 90 | #test_website http://localhost:$HTTP_PORT/ipfs/bafybeidsp6fva53dexzjycntiucts57ftecajcn5omzfgjx57pqfy3kwbq # big buck bunny 91 | 92 | #test_website http://localhost:$HTTP_PORT/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze # wikipedia 93 | 94 | #test_website http://localhost:$HTTP_PORT/ipfs/bafybeifaiclxh6pc3bdtrrkpbvvqqxq6hz5r6htdzxaga4fikfpu2u56qi # uniswap interface 95 | 96 | #test_website http://localhost:$HTTP_PORT/ipfs/bafybeiae366charqmeewxags5b2jxtkhfmqyyagvqhrr5l7l7xfpp5ikpa # cid.ipfs.tech 97 | 98 | #test_website http://localhost:$HTTP_PORT/ipfs/bafybeiedlhslivmuj2iinnpd24ulx3fyd7cjenddbkeoxbf3snjiz3npda # docs.ipfs.tech 99 | -------------------------------------------------------------------------------- /src/helia-rpc-api.ts: -------------------------------------------------------------------------------- 1 | import { GC_TIMEOUT_MS, HEALTHCHECK_TIMEOUT_MS } from './constants.js' 2 | import { getRequestAwareSignal } from './helia-server.js' 3 | import type { VerifiedFetch } from '@helia/verified-fetch' 4 | import type { FastifyReply, FastifyRequest, RouteOptions } from 'fastify' 5 | import type { Helia } from 'helia' 6 | 7 | const HELIA_RELEASE_INFO_API = (version: string): string => `https://api.github.com/repos/ipfs/helia/git/ref/tags/helia-v${version}` 8 | 9 | export interface HeliaRPCAPIOptions { 10 | helia: Helia 11 | fetch: VerifiedFetch 12 | } 13 | 14 | export function rpcApi (opts: HeliaRPCAPIOptions): RouteOptions[] { 15 | const log = opts.helia.logger.forComponent('rpc-api') 16 | let heliaVersionInfo: { Version: string, Commit: string } | undefined 17 | 18 | /** 19 | * Get the helia version 20 | */ 21 | async function heliaVersion (request: FastifyRequest, reply: FastifyReply): Promise { 22 | try { 23 | if (heliaVersionInfo === undefined) { 24 | log('fetching Helia version info') 25 | const { default: packageJson } = await import('../../node_modules/helia/package.json', { 26 | assert: { type: 'json' } 27 | }) 28 | const { version: heliaVersionString } = packageJson 29 | log('helia version string:', heliaVersionString) 30 | 31 | // handling the next versioning strategy 32 | const [heliaNextVersion, heliaNextCommit] = heliaVersionString.split('-') 33 | if (heliaNextCommit != null) { 34 | heliaVersionInfo = { 35 | Version: heliaNextVersion, 36 | Commit: heliaNextCommit 37 | } 38 | } else { 39 | // if this is not a next version, we will fetch the commit from github. 40 | const ghResp = await (await fetch(HELIA_RELEASE_INFO_API(heliaVersionString))).json() 41 | heliaVersionInfo = { 42 | Version: heliaVersionString, 43 | Commit: ghResp.object.sha.slice(0, 7) 44 | } 45 | } 46 | } 47 | 48 | log('helia version info:', heliaVersionInfo) 49 | await reply.code(200).header('Content-Type', 'application/json; charset=utf-8').send(heliaVersionInfo) 50 | } catch (err) { 51 | log.error('could not send version', err) 52 | await reply.code(500).send(err) 53 | } 54 | } 55 | 56 | /** 57 | * GC the node 58 | */ 59 | async function gc (request: FastifyRequest, reply: FastifyReply): Promise { 60 | log('running `gc` on Helia node') 61 | const signal = getRequestAwareSignal(request, log, { 62 | timeout: GC_TIMEOUT_MS 63 | }) 64 | await opts.helia.gc({ signal }) 65 | await reply.code(200).send('OK') 66 | } 67 | 68 | async function healthCheck (request: FastifyRequest, reply: FastifyReply): Promise { 69 | const signal = getRequestAwareSignal(request, log, { 70 | timeout: HEALTHCHECK_TIMEOUT_MS 71 | }) 72 | try { 73 | // echo "hello world" | npx kubo add --cid-version 1 -Q --inline 74 | // inline CID is bafkqaddimvwgy3zao5xxe3debi 75 | await opts.fetch('ipfs://bafkqaddimvwgy3zao5xxe3debi', { signal, redirect: 'follow' }) 76 | await reply.code(200).send('OK') 77 | } catch (err) { 78 | log.error('could not run healthcheck', err) 79 | await reply.code(500).send(err) 80 | } 81 | } 82 | 83 | return [{ 84 | url: '/api/v0/version', 85 | method: ['POST', 'GET'], 86 | handler: async (request, reply): Promise => heliaVersion(request, reply) 87 | }, { 88 | url: '/api/v0/repo/gc', 89 | method: ['POST', 'GET'], 90 | handler: async (request, reply): Promise => gc(request, reply) 91 | }, { 92 | url: '/api/v0/http-gateway-healthcheck', 93 | method: 'GET', 94 | handler: async (request, reply): Promise => healthCheck(request, reply) 95 | }, { 96 | url: '/*', 97 | method: 'GET', 98 | handler: async (request, reply): Promise => { 99 | await reply.code(400).send('API + Method not supported') 100 | } 101 | }] 102 | } 103 | -------------------------------------------------------------------------------- /src/get-custom-libp2p.ts: -------------------------------------------------------------------------------- 1 | import { noise } from '@chainsafe/libp2p-noise' 2 | import { yamux } from '@chainsafe/libp2p-yamux' 3 | import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' 4 | import { bootstrap } from '@libp2p/bootstrap' 5 | import { circuitRelayTransport } from '@libp2p/circuit-relay-v2' 6 | import { identify } from '@libp2p/identify' 7 | import { kadDHT, removePrivateAddressesMapper } from '@libp2p/kad-dht' 8 | import { mplex } from '@libp2p/mplex' 9 | import { prometheusMetrics } from '@libp2p/prometheus-metrics' 10 | import { tcp } from '@libp2p/tcp' 11 | import { tls } from '@libp2p/tls' 12 | import { webRTC, webRTCDirect } from '@libp2p/webrtc' 13 | import { webSockets } from '@libp2p/websockets' 14 | import { ipnsSelector } from 'ipns/selector' 15 | import { ipnsValidator } from 'ipns/validator' 16 | import { createLibp2p as create } from 'libp2p' 17 | import isPrivate from 'private-ip' 18 | import { ping } from '@libp2p/ping' 19 | import { DELEGATED_ROUTING_V1_HOST, METRICS, USE_DELEGATED_ROUTING, USE_DHT_ROUTING } from './constants.js' 20 | import type { Identify } from '@libp2p/identify' 21 | import type { Libp2p, ServiceMap } from '@libp2p/interface' 22 | import type { KadDHT } from '@libp2p/kad-dht' 23 | import type { Multiaddr } from '@multiformats/multiaddr' 24 | import type { HeliaInit } from 'helia' 25 | import type { Libp2pOptions, ServiceFactoryMap } from 'libp2p' 26 | import type { Ping } from '@libp2p/ping' 27 | 28 | interface HeliaGatewayLibp2pServices extends ServiceMap { 29 | dht?: KadDHT 30 | delegatedRouting?: unknown 31 | identify: Identify 32 | ping: Ping 33 | } 34 | 35 | interface HeliaGatewayLibp2pOptions extends Partial> { 36 | 37 | } 38 | 39 | const IP4 = 4 40 | const IP6 = 41 41 | 42 | export async function getCustomLibp2p ({ datastore }: HeliaGatewayLibp2pOptions): Promise> { 43 | const libp2pServices: ServiceFactoryMap = { 44 | identify: identify(), 45 | ping: ping() 46 | } 47 | 48 | if (USE_DELEGATED_ROUTING) { 49 | libp2pServices.delegatedRouting = () => createDelegatedRoutingV1HttpApiClient(DELEGATED_ROUTING_V1_HOST) 50 | } 51 | 52 | if (USE_DHT_ROUTING) { 53 | libp2pServices.dht = kadDHT({ 54 | protocol: '/ipfs/kad/1.0.0', 55 | peerInfoMapper: removePrivateAddressesMapper, 56 | // don't do DHT server work. 57 | clientMode: true, 58 | validators: { 59 | ipns: ipnsValidator 60 | }, 61 | selectors: { 62 | ipns: ipnsSelector 63 | } 64 | }) 65 | } 66 | 67 | const options: Libp2pOptions = { 68 | datastore, 69 | addresses: { 70 | listen: [ 71 | // helia-http-gateway is not dialable, we're only retrieving data from IPFS network, and then providing that data via a web2 http interface. 72 | ] 73 | }, 74 | transports: [ 75 | circuitRelayTransport(), 76 | tcp(), 77 | webRTC(), 78 | webRTCDirect(), 79 | webSockets() 80 | ], 81 | connectionEncrypters: [ 82 | noise(), 83 | tls() 84 | ], 85 | streamMuxers: [ 86 | yamux(), 87 | mplex() 88 | ], 89 | peerDiscovery: [ 90 | bootstrap({ 91 | list: [ 92 | '/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN', 93 | '/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa', 94 | '/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb', 95 | '/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt', 96 | '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ' 97 | ] 98 | }) 99 | ], 100 | services: libp2pServices, 101 | connectionGater: { 102 | denyDialMultiaddr: async (multiaddr: Multiaddr) => { 103 | const [[proto, address]] = multiaddr.stringTuples() 104 | 105 | // deny private ip4/ip6 addresses 106 | if (proto === IP4 || proto === IP6) { 107 | return Boolean(isPrivate(`${address}`)) 108 | } 109 | 110 | // all other addresses are ok 111 | return false 112 | } 113 | } 114 | } 115 | 116 | if (METRICS === 'true') { 117 | options.metrics = prometheusMetrics({ 118 | // this is done by fastify-metrics 119 | collectDefaultMetrics: false, 120 | // do not remove metrics configured by fastify-metrics 121 | preserveExistingMetrics: true 122 | }) 123 | } 124 | 125 | return create(options) 126 | } 127 | -------------------------------------------------------------------------------- /DEVELOPER-NOTES.md: -------------------------------------------------------------------------------- 1 | # Developer notes 2 | 3 | 4 | 5 | ## Gateway Conformance testing 6 | 7 | We have some code enabled that makes running gateway-conformance testing against helia-http-gateway easy. Follow the instructions in this section to run gateway-conformance tests locally 8 | 9 | ### Prerequisites 10 | 11 | 1. [Install docker](https://docs.docker.com/get-docker/) 12 | 2. [Install nodejs](https://nodejs.org/en/download/) 13 | 14 | ### Run gateway-conformance tests locally (once) 15 | 16 | ```sh 17 | $ npm run test:gwc 18 | ``` 19 | 20 | ### Continuously develop while running gateway-conformance tests 21 | 22 | ```sh 23 | # terminal 1 24 | $ npm run test:gwc-kubo 25 | # You can also set up the kubo backing node with the instructions at https://github.com/ipfs/gateway-conformance/blob/main/docs/development.md#developing-against-kubo 26 | 27 | # terminal 2 28 | # you will need to stop and start this one in-between code changes. It's not watching for changes 29 | $ DEBUG="helia-http-gateway*" FILE_DATASTORE_PATH=./tmp/datastore npm run test:gwc-helia 30 | 31 | # terminal 3 32 | $ npm run test:gwc-execute 33 | # To skip some tests, run something like: 34 | # npm run test:gwc-execute -- -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|TestGatewaySubdomainAndIPNS).*$' 35 | 36 | # OR from the gateway-conformance repo directly with something like: 37 | go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m 38 | 39 | ``` 40 | 41 | 42 | 43 | ### Some callouts 44 | 45 | 1. You may want to set up an nginx proxy from `http://helia-http-gateway.localhost` to `http://localhost:8090`to help with the gateway-conformance tests. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185) 46 | 1. You may want to run the gateway-conformance tests directly from the repo if you're on a macOS M1 due to some issues with docker and the proxying that the gateway-conformance testing tool uses. If you do this, you'll need to run `make gateway-conformance` in the `gateway-conformance` repo root, and then run the tests with something like `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m`. 47 | - If you want to run a specific test, you can pass the `-run` gotest flag. e.g. `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --json gwc-report.json -- -timeout 30m -run 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28HTTP_proxy_tunneling_via_CONNECT%29#01'` 48 | 1. The file `./scripts/kubo-init.js` executes kubo using `execa` instead of `ipfsd-ctl` so there may be some gotchas, but it should be as cross-platform and stable as the `execa` library. 49 | 1. The IPFS_PATH used is a temporary directory. Your OS should handle removing it when vital, but you can also remove it manually. The path to this directory is printed out when the tests start, and saved in a file at `./scripts/tmp/kubo-path.txt`. 50 | 1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures. 51 | 1. The results of the gateway-conformance tests are saved to `./gwc-report.json`. This file is overwritten every time the tests are run. 52 | 1. The gateway-conformance tests are flaky and commands & documentation are not up to date. Running commands with CLI flags is supposed to work, and env vars aren't documented, but ENV_VARs are the only way to get the tests to run, but not when ran with docker. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-1839801366) 53 | 54 | 55 | ## Tiros info 56 | 57 | ### Deploying to Tiros 58 | 59 | Go to https://github.com/plprobelab/probelab-infra/blob/main/aws/tf/tiros.tf 60 | 61 | update helia stuff 62 | 63 | run terraform apply (with AWS Config) 64 | 65 | ### Kick off helia-http-gateway task manually: 66 | 67 | 1. https://us-west-1.console.aws.amazon.com/ecs/v2/clusters/prod-usw1-ecs-cluster/run-task?region=us-west-1 68 | 1. select launch type FARGATE 69 | 1. select deployment configuration application type to be task 70 | 1. select task-helia family 71 | 1. select vpc (non-default) 72 | 1. select security group for tiros 73 | 1. Click create 74 | 75 | ### Todo 76 | 77 | - [ ] Fix helia_health_cmd (https://github.com/plprobelab/probelab-infra/blob/7ec47f16e84113545cdb06b07f865a3bc5787a0b/aws/tf/tiros.tf#L5C4-L5C4) to use the new helia-health command 78 | - [ ] fix node max memory for helia-http-gateway docker container to support at least 4GB 79 | 80 | ### Links 81 | 82 | graphs for runs are at https://probelab.grafana.net/d/GpwxraxVk/tiros-ops?orgId=1&from=now-7d&to=now&inspect=8&inspectTab=error 83 | AWS logs for runs are at https://us-west-1.console.aws.amazon.com/cloudwatch/home?region=us-west-1#logsV2:log-groups/log-group/prod-usw1-cmi-tiros-ipfs (you need to be logged in to AWS) 84 | 85 | -------------------------------------------------------------------------------- /.github/workflows/gateway-conformance.yml: -------------------------------------------------------------------------------- 1 | name: Gateway Conformance 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | gateway-conformance: 11 | runs-on: ubuntu-latest 12 | steps: 13 | # 1. Start the Kubo gateway 14 | - name: Setup Go 15 | uses: actions/setup-go@v4 16 | with: 17 | go-version: 1.21.x 18 | 19 | - name: Install Kubo gateway from source 20 | #uses: ipfs/download-ipfs-distribution-action@v1 21 | run: | 22 | go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0 23 | - name: Setup kubo config 24 | run: | 25 | ipfs init --profile=test 26 | ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" 27 | 28 | # 2. Download the gateway-conformance fixtures 29 | - name: Download gateway-conformance fixtures 30 | uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.4.1 31 | with: 32 | output: fixtures 33 | 34 | - name: Start Kubo gateway 35 | uses: ipfs/start-ipfs-daemon-action@v1 36 | 37 | # 3. Populate the Kubo gateway with the gateway-conformance fixtures 38 | - name: Import fixtures 39 | run: | 40 | # Import car files 41 | find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false --offline {} \; 42 | 43 | # Import ipns records 44 | records=$(find ./fixtures -name '*.ipns-record') 45 | for record in $records 46 | do 47 | key=$(basename -s .ipns-record "$record" | cut -d'_' -f1) 48 | ipfs routing put --allow-offline "/ipns/$key" "$record" 49 | done 50 | 51 | # Import dnslink records 52 | # the IPFS_NS_MAP env will be used by the daemon 53 | export IPFS_NS_MAP=$(cat "./fixtures/dnslinks.json" | jq -r '.subdomains | to_entries | map("\(.key).example.com:\(.value)") | join(",")') 54 | export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" 55 | echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV 56 | 57 | # 4. Build helia-http-gateway 58 | - name: Setup Node 59 | uses: actions/setup-node@v3 60 | with: 61 | node-version: 20 62 | 63 | - name: Checkout helia-http-gateway 64 | uses: actions/checkout@v3 65 | with: 66 | path: helia-http-gateway 67 | 68 | - name: Install dependencies 69 | run: npm ci 70 | working-directory: helia-http-gateway 71 | 72 | - name: Build helia-http-gateway 73 | run: npm run build 74 | working-directory: helia-http-gateway 75 | 76 | # 5. Start helia-http-gateway 77 | - name: Start helia-http-gateway 78 | env: 79 | GATEWAY_CONFORMANCE_TEST: true 80 | TRUSTLESS_GATEWAYS: "http://127.0.0.1:8080" 81 | USE_LIBP2P: false 82 | PORT: 8090 83 | run: | 84 | # run gw 85 | node dist/src/index.js & 86 | working-directory: helia-http-gateway 87 | 88 | # 6. Run the gateway-conformance tests 89 | - name: Run gateway-conformance tests 90 | uses: ipfs/gateway-conformance/.github/actions/test@v0.4.1 91 | with: 92 | gateway-url: http://127.0.0.1:8090 93 | subdomain-url: http://127.0.0.1:8090 94 | json: output.json 95 | xml: output.xml 96 | html: output.html 97 | markdown: output.md 98 | # specs: subdomain-ipfs-gateway,subdomain-ipns-gateway 99 | # use below to skip specific test if needed 100 | # args: -skip 'TestFooBr/GET_response_for_something' 101 | # 102 | # only-if-cached: helia-ht does not guarantee local cache, we will 103 | # adjust upstream test (which was Kubo-specific) 104 | # for now disabling these test cases 105 | args: -skip '^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$' 106 | 107 | # 7. Upload the results 108 | - name: Upload MD summary 109 | if: failure() || success() 110 | run: cat output.md >> $GITHUB_STEP_SUMMARY 111 | - name: Upload HTML report 112 | if: failure() || success() 113 | uses: actions/upload-artifact@v4 114 | with: 115 | name: gateway-conformance.html 116 | path: output.html 117 | - name: Upload JSON report 118 | if: failure() || success() 119 | uses: actions/upload-artifact@v4 120 | with: 121 | name: gateway-conformance.json 122 | path: output.json 123 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@helia/http-gateway", 3 | "version": "0.0.1", 4 | "description": "HTTP IPFS Gateway implemented using Helia", 5 | "license": "Apache-2.0 OR MIT", 6 | "homepage": "https://github.com/ipfs/helia-http-gateway#readme", 7 | "repository": { 8 | "type": "git", 9 | "url": "git+https://github.com/ipfs/helia-http-gateway.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/ipfs/helia-http-gateway/issues" 13 | }, 14 | "publishConfig": { 15 | "access": "public", 16 | "provenance": true 17 | }, 18 | "keywords": [ 19 | "docker", 20 | "helia", 21 | "p2p", 22 | "retrievals" 23 | ], 24 | "type": "module", 25 | "types": "./dist/src/index.d.ts", 26 | "files": [ 27 | "src", 28 | "dist", 29 | "!dist/test", 30 | "!**/*.tsbuildinfo" 31 | ], 32 | "exports": { 33 | ".": { 34 | "types": "./dist/src/index.d.ts", 35 | "import": "./dist/src/index.js" 36 | } 37 | }, 38 | "scripts": { 39 | "clean": "aegir clean dist playwright-report test", 40 | "lint": "aegir lint", 41 | "build": "aegir build --bundle false", 42 | "docs": "aegir docs", 43 | "dep-check": "aegir dep-check", 44 | "doc-check": "aegir doc-check", 45 | "start": "node --trace-warnings dist/src/index.js", 46 | "start:env-dr": "node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-delegated-routing", 47 | "start:env-to": "node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-trustless-only", 48 | "start:dev": "npm run build && node dist/src/index.js", 49 | "start:dev-trace": "npm run build && node --trace-warnings dist/src/index.js", 50 | "start:dev-doctor": "npm run build && npx clinic doctor --name playwright -- node dist/src/index.js", 51 | "start:dev-flame": "npm run build && npx clinic flame --name playwright -- node dist/src/index.js", 52 | "start:inspect": "npm run build && node --inspect dist/src/index.js", 53 | "test": "echo \"Error: no test specified\" && exit 1", 54 | "test:e2e": "playwright test", 55 | "test:http-e2e": "cross-env USE_BITSWAP=false USE_LIBP2P=false playwright test", 56 | "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$HTTP_PORT' && npm run test:e2e\"", 57 | "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$HTTP_PORT' && npm run test:e2e\"", 58 | "test:gwc-kubo": "node ./scripts/kubo-init.js", 59 | "test:gwc-helia": "wait-on \"tcp:$KUBO_PORT\" && npm run start:dev-trace", 60 | "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\"", 61 | "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json $GWC_SPECS -- -test.skip=\"$GWC_SKIP\" -timeout 30m", 62 | "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\" \"wait-on 'tcp:$PORT' && npm run test:gwc-execute\"", 63 | "healthcheck": "node dist/src/healthcheck.js", 64 | "debug:until-death": "./debugging/until-death.sh", 65 | "debug:test-gateways": "./debugging/test-gateways.sh", 66 | "postinstall": "patch-package" 67 | }, 68 | "dependencies": { 69 | "@chainsafe/libp2p-noise": "^16.1.0", 70 | "@chainsafe/libp2p-yamux": "^7.0.1", 71 | "@fastify/compress": "^8.0.1", 72 | "@fastify/cors": "^11.0.1", 73 | "@helia/block-brokers": "^4.1.0", 74 | "@helia/delegated-routing-v1-http-api-client": "^4.2.2", 75 | "@helia/http": "^2.0.5", 76 | "@helia/interface": "^5.2.1", 77 | "@helia/routers": "^3.0.1", 78 | "@helia/verified-fetch": "^2.6.4", 79 | "@libp2p/bootstrap": "^11.0.32", 80 | "@libp2p/circuit-relay-v2": "^3.2.8", 81 | "@libp2p/identify": "^3.0.27", 82 | "@libp2p/interface": "^2.7.0", 83 | "@libp2p/kad-dht": "^15.1.0", 84 | "@libp2p/mplex": "^11.0.32", 85 | "@libp2p/peer-id": "^5.1.0", 86 | "@libp2p/ping": "^2.0.31", 87 | "@libp2p/prometheus-metrics": "^4.3.15", 88 | "@libp2p/tcp": "^10.1.8", 89 | "@libp2p/tls": "^2.1.1", 90 | "@libp2p/webrtc": "^5.2.9", 91 | "@libp2p/websockets": "^9.2.8", 92 | "@multiformats/multiaddr": "^12.2.1", 93 | "@sgtpooki/file-type": "^1.0.1", 94 | "blockstore-fs": "^2.0.2", 95 | "datastore-level": "^11.0.1", 96 | "fastify": "^5.2.2", 97 | "fastify-metrics": "^12.1.0", 98 | "helia": "^5.3.0", 99 | "ipns": "^10.0.2", 100 | "libp2p": "^2.8.2", 101 | "multiformats": "^13.1.0", 102 | "pino-pretty": "^13.0.0", 103 | "private-ip": "^3.0.2", 104 | "race-signal": "^1.0.2" 105 | }, 106 | "devDependencies": { 107 | "@playwright/test": "^1.43.0", 108 | "aegir": "^47.0.10", 109 | "clinic": "^13.0.0", 110 | "concurrently": "^9.1.2", 111 | "cross-env": "^7.0.3", 112 | "debug": "^4.3.4", 113 | "dotenv": "^16.4.5", 114 | "execa": "^9.3.0", 115 | "glob": "^11.0.0", 116 | "kubo": "^0.34.1", 117 | "patch-package": "^8.0.0", 118 | "typescript": "5.x", 119 | "wait-on": "^8.0.3" 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /scripts/kubo-init.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This is part of the gateway conformance testing of helia-http-gateway. See ../DEVELOPER-NOTES.md for more details. 3 | */ 4 | 5 | import { writeFile, readFile } from 'node:fs/promises' 6 | import { tmpdir } from 'node:os' 7 | import { dirname, basename, relative } from 'node:path' 8 | import debug from 'debug' 9 | import { $ } from 'execa' 10 | import { glob } from 'glob' 11 | 12 | const log = debug('kubo-init') 13 | const error = log.extend('error') 14 | debug.enable('kubo-init*') 15 | 16 | const kuboFilePath = './scripts/tmp/kubo-path.txt' 17 | const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures` 18 | const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE ?? 'ghcr.io/ipfs/gateway-conformance:v0.5.0' 19 | 20 | async function main () { 21 | await $`mkdir -p ${dirname(kuboFilePath)}` 22 | 23 | const tmpDir = await writeKuboMetaData() 24 | 25 | await attemptKuboInit(tmpDir) 26 | 27 | await configureKubo(tmpDir) 28 | 29 | const ipfsNsMap = await loadFixtures(tmpDir) 30 | // execute the daemon 31 | const execaOptions = getExecaOptions({ tmpDir, ipfsNsMap }) 32 | log('Starting Kubo daemon...') 33 | await $(execaOptions)`npx kubo daemon --offline` 34 | } 35 | 36 | /** 37 | * 38 | * @param {Record} param0 39 | * @param {string} param0.tmpDir 40 | * @param {string} [param0.cwd] 41 | * @param {string} [param0.ipfsNsMap] 42 | * 43 | * @returns {import('execa').Options} 44 | */ 45 | function getExecaOptions ({ cwd, ipfsNsMap, tmpDir }) { 46 | return { 47 | cwd, 48 | env: { 49 | IPFS_PATH: tmpDir, 50 | IPFS_NS_MAP: ipfsNsMap 51 | } 52 | } 53 | } 54 | 55 | async function attemptKuboInit (tmpDir) { 56 | const execaOptions = getExecaOptions({ tmpDir }) 57 | try { 58 | await $(execaOptions)`npx -y kubo init --profile test` 59 | log('Kubo initialized at %s', tmpDir) 60 | } catch (e) { 61 | if (!e.stderr.includes('already exists!')) { 62 | throw e 63 | } 64 | log('Kubo was already initialized at %s', tmpDir) 65 | } 66 | } 67 | 68 | async function writeKuboMetaData () { 69 | let tmpDir 70 | try { 71 | const currentIpfsPath = await readFile('./scripts/tmp/kubo-path.txt', 'utf-8') 72 | log('Existing kubo path found at %s', currentIpfsPath) 73 | tmpDir = currentIpfsPath 74 | } catch (e) { 75 | error('Failed to read Kubo path from %s', kuboFilePath, e) 76 | tmpDir = tmpdir() + '/kubo-tmp' 77 | log('Using temporary Kubo path at %s', tmpDir) 78 | } 79 | try { 80 | await writeFile(kuboFilePath, tmpDir) 81 | } catch (e) { 82 | error('Failed to save Kubo path to %s', kuboFilePath, e) 83 | } 84 | return tmpDir 85 | } 86 | 87 | async function configureKubo (tmpDir) { 88 | const execaOptions = getExecaOptions({ tmpDir }) 89 | try { 90 | await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/${process.env.KUBO_PORT ?? 8081}` 91 | await $(execaOptions)`npx -y kubo config --json Bootstrap ${JSON.stringify([])}` 92 | await $(execaOptions)`npx -y kubo config --json Swarm.DisableNatPortMap true` 93 | await $(execaOptions)`npx -y kubo config --json Discovery.MDNS.Enabled false` 94 | await $(execaOptions)`npx -y kubo config --json Gateway.NoFetch true` 95 | await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true` 96 | await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin ${JSON.stringify(['*'])}` 97 | await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods ${JSON.stringify(['GET', 'POST', 'PUT', 'OPTIONS'])}` 98 | log('Kubo configured') 99 | } catch (e) { 100 | error('Failed to configure Kubo', e) 101 | } 102 | } 103 | 104 | async function downloadFixtures () { 105 | log('Downloading fixtures to %s using %s', GWC_FIXTURES_PATH, GWC_DOCKER_IMAGE) 106 | try { 107 | await $`docker run -v ${process.cwd()}:/workspace -w /workspace ${GWC_DOCKER_IMAGE} extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` 108 | } catch (e) { 109 | error('Error downloading fixtures, assuming current or previous success', e) 110 | } 111 | } 112 | 113 | async function loadFixtures (tmpDir) { 114 | await downloadFixtures() 115 | const execaOptions = getExecaOptions({ tmpDir }) 116 | 117 | for (const carFile of await glob([`${GWC_FIXTURES_PATH}/**/*.car`])) { 118 | log('Loading *.car fixture %s', carFile) 119 | const { stdout } = await $(execaOptions)`npx kubo dag import --pin-roots=false --offline ${carFile}` 120 | stdout.split('\n').forEach(log) 121 | } 122 | 123 | for (const ipnsRecord of await glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) { 124 | const key = basename(ipnsRecord, '.ipns-record') 125 | const relativePath = relative(GWC_FIXTURES_PATH, ipnsRecord) 126 | log('Loading *.ipns-record fixture %s', relativePath) 127 | const { stdout } = await $(({ ...execaOptions }))`cd ${GWC_FIXTURES_PATH} && npx kubo routing put --allow-offline "/ipns/${key}" "${relativePath}"` 128 | stdout.split('\n').forEach(log) 129 | } 130 | 131 | const json = await readFile(`${GWC_FIXTURES_PATH}/dnslinks.json`, 'utf-8') 132 | const { subdomains, domains } = JSON.parse(json) 133 | const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.example.com:${value}`).join(',') 134 | const domainDnsLinks = Object.entries(domains).map(([key, value]) => `${key}:${value}`).join(',') 135 | const ipfsNsMap = `${domainDnsLinks},${subdomainDnsLinks}` 136 | 137 | return ipfsNsMap 138 | } 139 | 140 | main().catch(e => { 141 | error(e) 142 | process.exit(1) 143 | }) 144 | -------------------------------------------------------------------------------- /debugging/time-permutations.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ### 4 | # This script is intended to be run from the root of the helia-http-gateway repository 5 | # It will run the gateway with different configurations and measure the time it takes to run 6 | # The results will be written to a CSV file, and runs that fail prior to the timeout are considered failed runs. 7 | # 8 | # Use like `./debugging/time-permutations.sh 30s 100` to execute 100 iterations of all permutations, where each permutation is run for a maximum of 30 seconds 9 | # This command can be run to ensure that until-death is properly cleaning up after itself (starting/stopping the gateway) 10 | # 11 | # Realistically, you should be running something like `./debugging/time-permutations.sh 15m 100` to get some logs of failure cases like those investigated in https://github.com/ipfs/helia-http-gateway/issues/18 12 | # 13 | 14 | # globals.. same for all configurations 15 | # export DEBUG="helia*,helia*:trace,libp2p*,libp2p*:trace" 16 | export DEBUG="*,*:trace" 17 | unset FASTIFY_DEBUG 18 | export HTTP_PORT=8080 19 | export RPC_PORT=5001 20 | export HOST="0.0.0.0" 21 | export ECHO_HEADERS=false 22 | export METRICS=true 23 | export USE_TRUSTLESS_GATEWAYS=false # always set to false since helia-dr and helia-all are the failing cases. 24 | export USE_BITSWAP=true # needs to be true to be able to fetch content without USE_TRUSTLESS_GATEWAYS 25 | export ALLOW_UNHANDLED_ERROR_RECOVERY=false 26 | unset DELEGATED_ROUTING_V1_HOST 27 | unset TRUSTLESS_GATEWAYS 28 | 29 | unset_all() { 30 | unset USE_SUBDOMAINS 31 | # unset USE_BITSWAP 32 | # unset USE_TRUSTLESS_GATEWAYS 33 | unset USE_LIBP2P 34 | unset USE_DELEGATED_ROUTING 35 | unset FILE_DATASTORE_PATH 36 | unset FILE_BLOCKSTORE_PATH 37 | } 38 | 39 | max_time=${1:-30m} 40 | max_iterations=${2:-10} 41 | 42 | mkdir -p permutation-logs 43 | rm -rf permutation-logs/* 44 | 45 | # Results file 46 | results_file="results.csv" 47 | echo "USE_SUBDOMAINS,USE_BITSWAP,USE_TRUSTLESS_GATEWAYS,USE_LIBP2P,USE_DELEGATED_ROUTING,time(max=${max_time}),successful_run" > $results_file 48 | 49 | run_test() { 50 | 51 | npx wait-on "tcp:$HTTP_PORT" -t 10000 -r # wait for the port to be released 52 | 53 | config_id="USE_SUBDOMAINS=$USE_SUBDOMAINS,USE_BITSWAP=$USE_BITSWAP,USE_TRUSTLESS_GATEWAYS=$USE_TRUSTLESS_GATEWAYS,USE_LIBP2P=$USE_LIBP2P,USE_DELEGATED_ROUTING=$USE_DELEGATED_ROUTING" 54 | # if we cannot get any data, we should skip this run.. we need at least USE_BITSWAP enabled, plus either USE_LIBP2P or USE_DELEGATED_ROUTING 55 | if [ "$USE_BITSWAP" = false ]; then 56 | echo "Skipping test for configuration: $config_id" 57 | return 58 | fi 59 | # TODO: we should also allow USE_TRUSTLESS_GATEWAYS=true, but we need to fix the issue with helia-dr and helia-all first 60 | if [ "$USE_LIBP2P" = false ] && [ "$USE_DELEGATED_ROUTING" = false ]; then 61 | echo "Skipping test for configuration: $config_id" 62 | return 63 | fi 64 | echo "Running test for configuration: $config_id" 65 | 66 | rm -f time_info_pipe 67 | mkfifo time_info_pipe 68 | 69 | # log file with config_id and timestamp 70 | run_log_file="permutation-logs/${USE_SUBDOMAINS}-${USE_BITSWAP}-${USE_TRUSTLESS_GATEWAYS}-${USE_LIBP2P}-${USE_DELEGATED_ROUTING}+$(date +%Y-%m-%d%H:%M:%S.%3N).log" 71 | 72 | # This is complicated, but we need to run the command in a subshell to be able to kill it if it takes too long, and also to get the timing information 73 | (timeout --signal=SIGTERM ${max_time} bash -c "time (./debugging/until-death.sh 2 &>${run_log_file})" 2>&1) &> time_info_pipe & 74 | subshell_pid=$! 75 | 76 | # Wait for the process to complete and get the timing information 77 | time_output=$(cat time_info_pipe) 78 | wait $subshell_pid 79 | exit_status=$? # get the exit status of the subshell 80 | 81 | # remove the fifo 82 | rm time_info_pipe 83 | was_successful=false 84 | if [ $exit_status -eq 124 ]; then 85 | echo "timeout occurred... (SUCCESSFUL RUN)" 86 | was_successful=true 87 | real_time="${max_time}" 88 | # remove the log file because the test didn't fail before the timeout 89 | rm $run_log_file 90 | else 91 | echo "no timeout occurred...(FAILED RUN)" 92 | was_successful=false 93 | 94 | real_time=$(echo "$time_output" | grep real | awk '{print $2}') 95 | fi 96 | 97 | # Write to file 98 | echo "$USE_SUBDOMAINS,$USE_BITSWAP,$USE_TRUSTLESS_GATEWAYS,$USE_LIBP2P,$USE_DELEGATED_ROUTING,$real_time,$was_successful" >> $results_file 99 | } 100 | 101 | main() { 102 | # Iterate over boolean values for a subset of environment variables 103 | for USE_SUBDOMAINS_VAL in true false; do 104 | # for USE_BITSWAP_VAL in true false; do 105 | # for USE_TRUSTLESS_GATEWAYS_VAL in true false; do 106 | for USE_LIBP2P_VAL in true false; do 107 | for USE_DELEGATED_ROUTING_VAL in true false; do 108 | unset_all 109 | 110 | # Export each variable 111 | export USE_SUBDOMAINS=$USE_SUBDOMAINS_VAL 112 | # export USE_BITSWAP=$USE_BITSWAP_VAL 113 | # export USE_TRUSTLESS_GATEWAYS=$USE_TRUSTLESS_GATEWAYS_VAL 114 | export USE_LIBP2P=$USE_LIBP2P_VAL 115 | export USE_DELEGATED_ROUTING=$USE_DELEGATED_ROUTING_VAL 116 | run_test 117 | done 118 | done 119 | # done 120 | # done 121 | done 122 | } 123 | 124 | cleanup_permutations_called=false 125 | cleanup_permutations() { 126 | if [ "$cleanup_permutations_called" = true ]; then 127 | echo "cleanup_permutations already called" 128 | return 129 | fi 130 | echo "cleanup_permutations called" 131 | cleanup_permutations_called=true 132 | 133 | kill -s TERM $subshell_pid 134 | echo "sent TERM signal to subshell" 135 | wait $subshell_pid # wait for the process to exit 136 | 137 | npx wait-on "tcp:$HTTP_PORT" -t 10000 -r # wait for the port to be released 138 | 139 | exit 1 140 | } 141 | 142 | trap cleanup_permutations SIGINT 143 | trap cleanup_permutations SIGTERM 144 | 145 | npm run build 146 | # Tell until-death.sh not build the gateway 147 | export DEBUG_NO_BUILD=true 148 | 149 | for ((i = 1; i <= $max_iterations; i++)) 150 | do 151 | echo "Iteration $i" 152 | main 153 | done 154 | 155 | -------------------------------------------------------------------------------- /src/helia-http-gateway.ts: -------------------------------------------------------------------------------- 1 | import { raceSignal } from 'race-signal' 2 | import { USE_SUBDOMAINS, USE_SESSIONS } from './constants.js' 3 | import { dnsLinkLabelEncoder, isInlinedDnsLink } from './dns-link-labels.js' 4 | import { getFullUrlFromFastifyRequest, getRequestAwareSignal } from './helia-server.js' 5 | import { getIpnsAddressDetails } from './ipns-address-utils.js' 6 | import type { VerifiedFetch } from '@helia/verified-fetch' 7 | import type { AbortOptions } from '@multiformats/multiaddr' 8 | import type { FastifyReply, FastifyRequest, RouteOptions } from 'fastify' 9 | import type { Helia } from 'helia' 10 | 11 | interface EntryParams { 12 | ns: string 13 | address: string 14 | '*': string 15 | } 16 | 17 | export interface HeliaHTTPGatewayOptions { 18 | helia: Helia 19 | fetch: VerifiedFetch 20 | } 21 | 22 | export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { 23 | const log = opts.helia.logger.forComponent('http-gateway') 24 | 25 | /** 26 | * Redirects to the subdomain gateway. 27 | */ 28 | async function handleEntry (request: FastifyRequest, reply: FastifyReply): Promise { 29 | const { params } = request 30 | log('fetch request %s', request.url) 31 | const { ns: namespace, '*': relativePath, address } = params as EntryParams 32 | 33 | log('handling entry: ', { address, namespace, relativePath }) 34 | 35 | if (!USE_SUBDOMAINS) { 36 | log('subdomains are disabled, fetching without subdomain') 37 | return fetch(request, reply) 38 | } else { 39 | log('subdomains are enabled, redirecting to subdomain') 40 | } 41 | 42 | const { peerId, cid } = getIpnsAddressDetails(address) 43 | 44 | if (peerId != null) { 45 | return fetch(request, reply) 46 | } 47 | 48 | const cidv1Address = cid?.toString() 49 | 50 | const query = request.query as Record 51 | log.trace('query: ', query) 52 | // eslint-disable-next-line no-warning-comments 53 | // TODO: enable support for query params 54 | if (query != null) { 55 | // http://localhost:8090/ipfs/bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly/dir?format=raw 56 | // eslint-disable-next-line no-warning-comments 57 | // TODO: temporary ipfs gateway spec? 58 | // if (query.uri != null) { 59 | // // Test = http://localhost:8080/ipns/?uri=ipns%3A%2F%2Fdnslink-subdomain-gw-test.example.org 60 | // log('got URI query parameter: ', query.uri) 61 | // const url = new URL(query.uri) 62 | // address = url.hostname 63 | // } 64 | // finalUrl += encodeURIComponent(`?${new URLSearchParams(request.query).toString()}`) 65 | } 66 | let encodedDnsLink = address 67 | 68 | if (!isInlinedDnsLink(address)) { 69 | encodedDnsLink = dnsLinkLabelEncoder(address) 70 | } 71 | 72 | const finalUrl = `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` 73 | log('redirecting to final URL:', finalUrl) 74 | await reply 75 | .headers({ 76 | Location: finalUrl 77 | }) 78 | .code(301) 79 | .send() 80 | } 81 | 82 | /** 83 | * Fetches a content for a subdomain, which basically queries delegated 84 | * routing API and then fetches the path from helia. 85 | */ 86 | async function fetch (request: FastifyRequest, reply: FastifyReply): Promise { 87 | const url = getFullUrlFromFastifyRequest(request, log) 88 | log('fetching url "%s" with @helia/verified-fetch', url) 89 | 90 | const signal = getRequestAwareSignal(request, log, { 91 | url 92 | }) 93 | 94 | // if subdomains are disabled, have @helia/verified-fetch follow redirects 95 | // internally, otherwise let the client making the request do it 96 | const resp = await opts.fetch(url, { 97 | signal, 98 | redirect: USE_SUBDOMAINS ? 'manual' : 'follow', 99 | session: USE_SESSIONS 100 | }) 101 | 102 | await convertVerifiedFetchResponseToFastifyReply(url, resp, reply, { 103 | signal 104 | }) 105 | } 106 | 107 | async function convertVerifiedFetchResponseToFastifyReply (url: string, verifiedFetchResponse: Response, reply: FastifyReply, options: AbortOptions): Promise { 108 | if (!verifiedFetchResponse.ok) { 109 | log('verified-fetch response for %s not ok: ', url, verifiedFetchResponse.status) 110 | await reply.code(verifiedFetchResponse.status).send(verifiedFetchResponse.statusText) 111 | return 112 | } 113 | 114 | const contentType = verifiedFetchResponse.headers.get('content-type') 115 | 116 | if (contentType == null) { 117 | log('verified-fetch response for %s has no content-type', url) 118 | await reply.code(200).send(verifiedFetchResponse.body) 119 | return 120 | } 121 | 122 | if (verifiedFetchResponse.body == null) { 123 | // this should never happen 124 | log('verified-fetch response for %s has no body', url) 125 | await reply.code(501).send('empty') 126 | return 127 | } 128 | 129 | const headers: Record = {} 130 | 131 | for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) { 132 | headers[headerName] = headerValue 133 | } 134 | 135 | // Fastify really does not like streams despite what the documentation and 136 | // github issues say 137 | const reader = verifiedFetchResponse.body.getReader() 138 | reply.raw.writeHead(verifiedFetchResponse.status, headers) 139 | 140 | try { 141 | let done = false 142 | let value 143 | 144 | while (!done) { 145 | ({ done, value } = await raceSignal(reader.read(), options.signal)) 146 | 147 | if (value != null) { 148 | reply.raw.write(Buffer.from(value)) 149 | } 150 | } 151 | } catch (err) { 152 | log.error('error reading response for %s', url, err) 153 | await reader.cancel(err) 154 | } finally { 155 | log.error('reading response for %s ended', url) 156 | reply.raw.end() 157 | } 158 | } 159 | 160 | return [{ 161 | // without this non-wildcard postfixed path, the '/*' route will match first. 162 | url: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid 163 | method: 'GET', 164 | handler: async (request, reply): Promise => handleEntry(request, reply) 165 | }, { 166 | url: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath 167 | method: 'GET', 168 | handler: async (request, reply): Promise => handleEntry(request, reply) 169 | }, { 170 | url: '/*', 171 | method: 'GET', 172 | handler: async (request, reply): Promise => { 173 | try { 174 | await fetch(request, reply) 175 | } catch { 176 | await reply.code(200).send('try /ipfs/ or /ipns/') 177 | } 178 | } 179 | }, { 180 | url: '/', 181 | method: 'GET', 182 | handler: async (request, reply): Promise => { 183 | if (USE_SUBDOMAINS && request.hostname.split('.').length > 1) { 184 | return fetch(request, reply) 185 | } 186 | await reply.code(200).send('try /ipfs/ or /ipns/') 187 | } 188 | }] 189 | } 190 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # helia-http-gateway 2 | 3 | # @helia/http-gateway 4 | 5 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 6 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 7 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-http-gateway.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-http-gateway) 8 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-http-gateway/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-http-gateway/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 9 | 10 | > HTTP IPFS Gateway implemented using Helia 11 | 12 | # About 13 | 14 | 28 | 29 | A Dockerized application that implements the [HTTP IPFS-gateway API](https://docs.ipfs.tech/concepts/ipfs-gateway/#gateway-types) spec and responds to the incoming requests using [Helia](https://github.com/ipfs/helia) to fetch the content from IPFS. 30 | 31 | ## Run from the github container registry 32 | 33 | ```sh 34 | $ docker run -it -p 8080:8080 ghcr.io/ipfs/helia-http-gateway:latest 35 | ``` 36 | 37 | See for more information. 38 | 39 | ## Run Using Docker Compose 40 | 41 | ```sh 42 | $ docker-compose up 43 | ``` 44 | 45 | ## Run Using Docker 46 | 47 | ### Build 48 | 49 | ```sh 50 | $ docker build . --tag helia-http-gateway:local 51 | ``` 52 | 53 | Pass the explicit platform when building on a Mac. 54 | 55 | ```sh 56 | $ docker build . --platform linux/arm64 --tag helia-http-gateway:local-arm64 57 | ``` 58 | 59 | ### Running 60 | 61 | ```sh 62 | $ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway:local # or helia-http-gateway:local-arm64 63 | ``` 64 | 65 | ## Run without Docker 66 | 67 | \### Build 68 | 69 | ```sh 70 | $ npm run build 71 | ``` 72 | 73 | ### Running 74 | 75 | ```sh 76 | $ npm start 77 | ``` 78 | 79 | ## Supported Environment Variables 80 | 81 | | Variable | Description | Default | 82 | | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | 83 | | `DEBUG` | Debug level | `''` | 84 | | `FASTIFY_DEBUG` | Debug level for fastify's logger | `''` | 85 | | `PORT` | Port to listen on | `8080` | 86 | | `HOST` | Host to listen on | `0.0.0.0` | 87 | | `USE_SUBDOMAINS` | Whether to use [origin isolation](https://docs.ipfs.tech/how-to/gateway-best-practices/#use-subdomain-gateway-resolution-for-origin-isolation) | `true` | 88 | | `METRICS` | Whether to enable prometheus metrics. Any value other than 'true' will disable metrics. | `true` | 89 | | `USE_BITSWAP` | Use bitswap to fetch content from IPFS | `true` | 90 | | `USE_TRUSTLESS_GATEWAYS` | Whether to fetch content from trustless-gateways or not | `true` | 91 | | `TRUSTLESS_GATEWAYS` | Comma separated list of trusted gateways to fetch content from | [Defined in Helia](https://github.com/ipfs/helia/blob/main/packages/helia/src/block-brokers/trustless-gateway/index.ts) | 92 | | `USE_LIBP2P` | Whether to use libp2p networking | `true` | 93 | | `ECHO_HEADERS` | A debug flag to indicate whether you want to output request and response headers | `false` | 94 | | `USE_DELEGATED_ROUTING` | Whether to use the delegated routing v1 API | `true` | 95 | | `DELEGATED_ROUTING_V1_HOST` | Hostname to use for delegated routing v1 | `https://delegated-ipfs.dev` | 96 | | `USE_DHT_ROUTING` | Whether to use @libp2p/kad-dht for routing when libp2p is enabled | `true` | 97 | | `USE_SESSIONS` | If true, use a blockstore session per IPFS/IPNS path | `true` | 98 | 99 | 104 | 105 | See the source of truth for all `process.env.` environment variables at [src/constants.ts](src/constants.ts). 106 | 107 | You can also see some recommended environment variable configurations at: 108 | 109 | - [./.env-all](./.env-all) 110 | - [./.env-delegated-routing](./.env-delegated-routing) 111 | - [./.env-gwc](./.env-gwc) 112 | - [./.env-trustless-only](./.env-trustless-only) 113 | 114 | ### Running with custom configurations 115 | 116 | Note that any of the following calls to docker can be replaced with something like `MY_ENV_VAR="MY_VALUE" npm run start` 117 | 118 | #### Disable libp2p 119 | 120 | ```sh 121 | $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia 122 | ``` 123 | 124 | #### Disable bitswap 125 | 126 | ```sh 127 | $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia 128 | ``` 129 | 130 | #### Disable trustless gateways 131 | 132 | ```sh 133 | $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia 134 | ``` 135 | 136 | #### Customize trustless gateways 137 | 138 | ```sh 139 | $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia 140 | ``` 141 | 142 | 153 | 154 | ## E2E Testing 155 | 156 | We have some tests enabled that simulate running inside of [ProbeLab's Tiros](https://github.com/plprobelab/tiros), via playwright. These tests request the same paths from ipfs.io and ensure that the resulting text matches. This is not a direct replacement for [gateway conformance testing](https://github.com/ipfs/gateway-conformance), but helps us ensure the helia-http-gateway is working as expected. 157 | 158 | By default, these tests: 159 | 160 | 1. Run in serial 161 | 2. Allow for up to 5 failures before failing the whole suite run. 162 | 3. Have an individual test timeout of two minutes. 163 | 164 | ### Run e2e tests locally 165 | 166 | ```sh 167 | $ npm run test:e2e # run all tests 168 | $ npm run test:e2e -- ${PLAYWRIGHT_OPTIONS} # run tests with custom playwright options. 169 | 170 | ``` 171 | 172 | ### Get clinicjs flamecharts and doctor reports from e2e tests 173 | 174 | ```sh 175 | $ npm run test:e2e-doctor # Run the dev server with clinicjs doctor, execute e2e tests, and generate a report. 176 | $ npm run test:e2e-flame # Run the dev server with clinicjs flame, execute e2e tests, and generate a report. 177 | ``` 178 | 179 | ## Metrics 180 | 181 | Running with `METRICS=true` will enable collecting Fastify/libp2p metrics and 182 | will expose a prometheus collection endpoint at 183 | 184 | js-libp2p metrics are collected by default, but can be disabled by setting `USE_LIBP2P_METRICS=false`. Find out more details at 185 | 186 | ### Viewing metrics 187 | 188 | See [Metrics config](./config/README.md#metrics) for more information on how to view the generated metrics. 189 | 190 | # Install 191 | 192 | ```console 193 | $ npm i @helia/http-gateway 194 | ``` 195 | 196 | # API Docs 197 | 198 | - 199 | 200 | # License 201 | 202 | Licensed under either of 203 | 204 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/helia-http-gateway/LICENSE-APACHE) / ) 205 | - MIT ([LICENSE-MIT](https://github.com/ipfs/helia-http-gateway/LICENSE-MIT) / ) 206 | 207 | # Contribute 208 | 209 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-http-gateway/issues). 210 | 211 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 212 | 213 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 214 | 215 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 216 | 217 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 218 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @packageDocumentation 3 | * 4 | * A Dockerized application that implements the [HTTP IPFS-gateway API](https://docs.ipfs.tech/concepts/ipfs-gateway/#gateway-types) spec and responds to the incoming requests using [Helia](https://github.com/ipfs/helia) to fetch the content from IPFS. 5 | * 6 | * ## Run from the github container registry 7 | * 8 | * ```sh 9 | * $ docker run -it -p 8080:8080 ghcr.io/ipfs/helia-http-gateway:latest 10 | * ``` 11 | * 12 | * See for more information. 13 | * 14 | * ## Run Using Docker Compose 15 | * 16 | * ```sh 17 | * $ docker-compose up 18 | * ``` 19 | * 20 | * ## Run Using Docker 21 | * 22 | * ### Build 23 | * 24 | * ```sh 25 | * $ docker build . --tag helia-http-gateway:local 26 | * ``` 27 | * 28 | * Pass the explicit platform when building on a Mac. 29 | * 30 | * ```sh 31 | * $ docker build . --platform linux/arm64 --tag helia-http-gateway:local-arm64 32 | * ``` 33 | * 34 | * ### Running 35 | * 36 | * ```sh 37 | * $ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway:local # or helia-http-gateway:local-arm64 38 | * ``` 39 | * 40 | * ## Run without Docker 41 | * 42 | * ### Build 43 | * 44 | * ```sh 45 | * $ npm run build 46 | * ``` 47 | * 48 | * ### Running 49 | * 50 | * ```sh 51 | * $ npm start 52 | * ``` 53 | * 54 | * ## Supported Environment Variables 55 | * 56 | * | Variable | Description | Default | 57 | * | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | 58 | * | `DEBUG` | Debug level | `''` | 59 | * | `FASTIFY_DEBUG` | Debug level for fastify's logger | `''` | 60 | * | `PORT` | Port to listen on | `8080` | 61 | * | `HOST` | Host to listen on | `0.0.0.0` | 62 | * | `USE_SUBDOMAINS` | Whether to use [origin isolation](https://docs.ipfs.tech/how-to/gateway-best-practices/#use-subdomain-gateway-resolution-for-origin-isolation) | `true` | 63 | * | `METRICS` | Whether to enable prometheus metrics. Any value other than 'true' will disable metrics. | `true` | 64 | * | `USE_BITSWAP` | Use bitswap to fetch content from IPFS | `true` | 65 | * | `USE_TRUSTLESS_GATEWAYS` | Whether to fetch content from trustless-gateways or not | `true` | 66 | * | `TRUSTLESS_GATEWAYS` | Comma separated list of trusted gateways to fetch content from | [Defined in Helia](https://github.com/ipfs/helia/blob/main/packages/helia/src/block-brokers/trustless-gateway/index.ts) | 67 | * | `USE_LIBP2P` | Whether to use libp2p networking | `true` | 68 | * | `ECHO_HEADERS` | A debug flag to indicate whether you want to output request and response headers | `false` | 69 | * | `USE_DELEGATED_ROUTING` | Whether to use the delegated routing v1 API | `true` | 70 | * | `DELEGATED_ROUTING_V1_HOST` | Hostname to use for delegated routing v1 | `https://delegated-ipfs.dev` | 71 | * 72 | * 77 | * 78 | * See the source of truth for all `process.env.` environment variables at [src/constants.ts](src/constants.ts). 79 | * 80 | * You can also see some recommended environment variable configurations at: 81 | * 82 | * - [./.env-all](./.env-all) 83 | * - [./.env-delegated-routing](./.env-delegated-routing) 84 | * - [./.env-gwc](./.env-gwc) 85 | * - [./.env-trustless-only](./.env-trustless-only) 86 | * 87 | * ### Running with custom configurations 88 | * 89 | * Note that any of the following calls to docker can be replaced with something like `MY_ENV_VAR="MY_VALUE" npm run start` 90 | * 91 | * #### Disable libp2p 92 | * 93 | * ```sh 94 | * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia 95 | * ``` 96 | * 97 | * #### Disable bitswap 98 | * 99 | * ```sh 100 | * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia 101 | * ``` 102 | * 103 | * #### Disable trustless gateways 104 | * 105 | * ```sh 106 | * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia 107 | * ``` 108 | * 109 | * #### Customize trustless gateways 110 | * 111 | * ```sh 112 | * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia 113 | * ``` 114 | * 115 | * 126 | * 127 | * ## E2E Testing 128 | * 129 | * We have some tests enabled that simulate running inside of [ProbeLab's Tiros](https://github.com/plprobelab/tiros), via playwright. These tests request the same paths from ipfs.io and ensure that the resulting text matches. This is not a direct replacement for [gateway conformance testing](https://github.com/ipfs/gateway-conformance), but helps us ensure the helia-http-gateway is working as expected. 130 | * 131 | * By default, these tests: 132 | * 133 | * 1. Run in serial 134 | * 2. Allow for up to 5 failures before failing the whole suite run. 135 | * 3. Have an individual test timeout of two minutes. 136 | * 137 | * ### Run e2e tests locally 138 | * 139 | * ```sh 140 | * $ npm run test:e2e # run all tests 141 | * $ npm run test:e2e -- ${PLAYWRIGHT_OPTIONS} # run tests with custom playwright options. 142 | * 143 | * ``` 144 | * 145 | * ### Get clinicjs flamecharts and doctor reports from e2e tests 146 | * 147 | * ```sh 148 | * $ npm run test:e2e-doctor # Run the dev server with clinicjs doctor, execute e2e tests, and generate a report. 149 | * $ npm run test:e2e-flame # Run the dev server with clinicjs flame, execute e2e tests, and generate a report. 150 | * ``` 151 | * 152 | * ## Metrics 153 | * 154 | * Running with `METRICS=true` will enable collecting Fastify/libp2p metrics and 155 | * will expose a prometheus collection endpoint at 156 | */ 157 | 158 | import compress from '@fastify/compress' 159 | import cors from '@fastify/cors' 160 | import { createVerifiedFetch } from '@helia/verified-fetch' 161 | import fastify from 'fastify' 162 | import metricsPlugin from 'fastify-metrics' 163 | import { HOST, HTTP_PORT, RPC_PORT, METRICS, ECHO_HEADERS, FASTIFY_DEBUG } from './constants.js' 164 | import { contentTypeParser } from './content-type-parser.js' 165 | import { getCustomHelia } from './get-custom-helia.js' 166 | import { httpGateway } from './helia-http-gateway.js' 167 | import { rpcApi } from './helia-rpc-api.js' 168 | import type { FastifyInstance, RouteOptions } from 'fastify' 169 | 170 | const helia = await getCustomHelia() 171 | const fetch = await createVerifiedFetch(helia, { contentTypeParser }) 172 | const log = helia.logger.forComponent('index') 173 | 174 | const [rpcApiServer, httpGatewayServer] = await Promise.all([ 175 | createServer('rpc-api', rpcApi({ 176 | helia, 177 | fetch 178 | }), { 179 | metrics: false 180 | }), 181 | createServer('http-gateway', httpGateway({ 182 | helia, 183 | fetch 184 | }), { 185 | metrics: METRICS === 'true' 186 | }) 187 | ]) 188 | 189 | await Promise.all([ 190 | rpcApiServer.listen({ port: RPC_PORT, host: HOST }), 191 | httpGatewayServer.listen({ port: HTTP_PORT, host: HOST }) 192 | ]) 193 | 194 | console.info(`API server listening on /ip4/${HOST}/tcp/${RPC_PORT}`) // eslint-disable-line no-console 195 | console.info(`Gateway (readonly) server listening on /ip4/${HOST}/tcp/${HTTP_PORT}`) // eslint-disable-line no-console 196 | 197 | const stopWebServer = async (): Promise => { 198 | try { 199 | await rpcApiServer.close() 200 | await httpGatewayServer.close() 201 | } catch (error) { 202 | log.error(error) 203 | process.exit(1) 204 | } 205 | log('Closed out remaining webServer connections.') 206 | } 207 | 208 | let shutdownRequested = false 209 | async function closeGracefully (signal: number | string): Promise { 210 | log(`Received signal to terminate: ${signal}`) 211 | if (shutdownRequested) { 212 | log('closeGracefully: shutdown already requested, exiting callback.') 213 | return 214 | } 215 | shutdownRequested = true 216 | 217 | await stopWebServer() 218 | await fetch.stop() 219 | await helia.stop() 220 | 221 | log('Stopped Helia.') 222 | 223 | process.kill(process.pid, signal) 224 | } 225 | 226 | ['SIGHUP', 'SIGINT', 'SIGTERM', 'beforeExit'].forEach((signal: string) => { 227 | process.once(signal, closeGracefully) 228 | }) 229 | 230 | interface ServerOptions { 231 | metrics: boolean 232 | } 233 | 234 | async function createServer (name: string, routes: RouteOptions[], options: ServerOptions): Promise { 235 | const app = fastify({ 236 | logger: { 237 | enabled: FASTIFY_DEBUG !== '', 238 | msgPrefix: `helia-http-gateway:fastify:${name} `, 239 | level: 'info', 240 | transport: { 241 | target: 'pino-pretty' 242 | } 243 | } 244 | }) 245 | 246 | if (options.metrics) { 247 | // Add the prometheus middleware 248 | await app.register(metricsPlugin.default, { endpoint: '/metrics' }) 249 | } 250 | 251 | await app.register(cors, { 252 | /** 253 | * @see https://github.com/ipfs/gateway-conformance/issues/186 254 | * @see https://github.com/ipfs/gateway-conformance/blob/d855ec4fb9dac4a5aaecf3776037b005cc74c566/tests/path_gateway_cors_test.go#L16-L56 255 | */ 256 | allowedHeaders: ['Content-Type', 'Range', 'User-Agent', 'X-Requested-With'], 257 | origin: '*', 258 | exposedHeaders: [ 259 | 'Content-Range', 260 | 'Content-Length', 261 | 'X-Ipfs-Path', 262 | 'X-Ipfs-Roots', 263 | 'X-Chunked-Output', 264 | 'X-Stream-Output' 265 | ], 266 | methods: ['GET', 'HEAD', 'OPTIONS'], 267 | strictPreflight: false, 268 | preflightContinue: true, 269 | preflight: true 270 | }) 271 | 272 | // enable compression 273 | await app.register(compress, { 274 | global: true 275 | }) 276 | 277 | // set up routes 278 | routes.forEach(route => { 279 | app.route(route) 280 | }) 281 | 282 | if ([ECHO_HEADERS].includes(true)) { 283 | app.addHook('onRequest', async (request, reply) => { 284 | if (ECHO_HEADERS) { 285 | log('fastify hook onRequest: echoing headers:') 286 | Object.keys(request.headers).forEach((headerName) => { 287 | log('\t %s: %s', headerName, request.headers[headerName]) 288 | }) 289 | } 290 | }) 291 | 292 | app.addHook('onSend', async (request, reply, payload) => { 293 | if (ECHO_HEADERS) { 294 | log('fastify hook onSend: echoing headers:') 295 | const responseHeaders = reply.getHeaders() 296 | Object.keys(responseHeaders).forEach((headerName) => { 297 | log('\t %s: %s', headerName, responseHeaders[headerName]) 298 | }) 299 | } 300 | return payload 301 | }) 302 | } 303 | 304 | return app 305 | } 306 | -------------------------------------------------------------------------------- /config/grafana/dashboards/lodestar-libp2p.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_PROMETHEUS", 5 | "label": "prometheus", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__elements": {}, 13 | "__requires": [ 14 | { 15 | "type": "grafana", 16 | "id": "grafana", 17 | "name": "Grafana", 18 | "version": "10.4.1" 19 | }, 20 | { 21 | "type": "datasource", 22 | "id": "prometheus", 23 | "name": "Prometheus", 24 | "version": "1.0.0" 25 | }, 26 | { 27 | "type": "panel", 28 | "id": "timeseries", 29 | "name": "Time series", 30 | "version": "" 31 | } 32 | ], 33 | "annotations": { 34 | "list": [ 35 | { 36 | "builtIn": 1, 37 | "datasource": { 38 | "type": "datasource", 39 | "uid": "grafana" 40 | }, 41 | "enable": true, 42 | "hide": true, 43 | "iconColor": "rgba(0, 211, 255, 1)", 44 | "name": "Annotations & Alerts", 45 | "target": { 46 | "limit": 100, 47 | "matchAny": false, 48 | "tags": [], 49 | "type": "dashboard" 50 | }, 51 | "type": "dashboard" 52 | } 53 | ] 54 | }, 55 | "editable": true, 56 | "fiscalYearStartMonth": 0, 57 | "graphTooltip": 1, 58 | "id": null, 59 | "links": [ 60 | { 61 | "asDropdown": true, 62 | "icon": "external link", 63 | "includeVars": true, 64 | "keepTime": true, 65 | "tags": [ 66 | "lodestar" 67 | ], 68 | "targetBlank": false, 69 | "title": "Lodestar dashboards", 70 | "tooltip": "", 71 | "type": "dashboards", 72 | "url": "" 73 | } 74 | ], 75 | "liveNow": false, 76 | "panels": [ 77 | { 78 | "collapsed": false, 79 | "datasource": { 80 | "type": "prometheus", 81 | "uid": "${DS_PROMETHEUS}" 82 | }, 83 | "gridPos": { 84 | "h": 1, 85 | "w": 24, 86 | "x": 0, 87 | "y": 0 88 | }, 89 | "id": 12, 90 | "panels": [], 91 | "targets": [ 92 | { 93 | "datasource": { 94 | "type": "prometheus", 95 | "uid": "${DS_PROMETHEUS}" 96 | }, 97 | "refId": "A" 98 | } 99 | ], 100 | "title": "General", 101 | "type": "row" 102 | }, 103 | { 104 | "datasource": { 105 | "type": "prometheus", 106 | "uid": "${DS_PROMETHEUS}" 107 | }, 108 | "fieldConfig": { 109 | "defaults": { 110 | "color": { 111 | "mode": "palette-classic" 112 | }, 113 | "custom": { 114 | "axisBorderShow": false, 115 | "axisCenteredZero": false, 116 | "axisColorMode": "text", 117 | "axisLabel": "", 118 | "axisPlacement": "auto", 119 | "barAlignment": 0, 120 | "drawStyle": "line", 121 | "fillOpacity": 0, 122 | "gradientMode": "none", 123 | "hideFrom": { 124 | "legend": false, 125 | "tooltip": false, 126 | "viz": false 127 | }, 128 | "insertNulls": false, 129 | "lineInterpolation": "linear", 130 | "lineWidth": 1, 131 | "pointSize": 5, 132 | "scaleDistribution": { 133 | "type": "linear" 134 | }, 135 | "showPoints": "auto", 136 | "spanNulls": false, 137 | "stacking": { 138 | "group": "A", 139 | "mode": "none" 140 | }, 141 | "thresholdsStyle": { 142 | "mode": "off" 143 | } 144 | }, 145 | "mappings": [], 146 | "thresholds": { 147 | "mode": "absolute", 148 | "steps": [ 149 | { 150 | "color": "green", 151 | "value": null 152 | }, 153 | { 154 | "color": "red", 155 | "value": 80 156 | } 157 | ] 158 | } 159 | }, 160 | "overrides": [] 161 | }, 162 | "gridPos": { 163 | "h": 8, 164 | "w": 12, 165 | "x": 0, 166 | "y": 1 167 | }, 168 | "id": 27, 169 | "options": { 170 | "legend": { 171 | "calcs": [], 172 | "displayMode": "list", 173 | "placement": "bottom", 174 | "showLegend": true 175 | }, 176 | "tooltip": { 177 | "mode": "single", 178 | "sort": "none" 179 | } 180 | }, 181 | "targets": [ 182 | { 183 | "datasource": { 184 | "type": "prometheus", 185 | "uid": "${DS_PROMETHEUS}" 186 | }, 187 | "editorMode": "code", 188 | "exemplar": false, 189 | "expr": "libp2p_connection_manager_connections", 190 | "interval": "", 191 | "legendFormat": "{{libp2p_connection_manager_connections}}", 192 | "range": true, 193 | "refId": "A" 194 | } 195 | ], 196 | "title": "Total Connections", 197 | "type": "timeseries" 198 | }, 199 | { 200 | "datasource": { 201 | "type": "prometheus", 202 | "uid": "${DS_PROMETHEUS}" 203 | }, 204 | "fieldConfig": { 205 | "defaults": { 206 | "color": { 207 | "mode": "palette-classic" 208 | }, 209 | "custom": { 210 | "axisBorderShow": false, 211 | "axisCenteredZero": false, 212 | "axisColorMode": "text", 213 | "axisLabel": "", 214 | "axisPlacement": "auto", 215 | "barAlignment": 0, 216 | "drawStyle": "line", 217 | "fillOpacity": 0, 218 | "gradientMode": "none", 219 | "hideFrom": { 220 | "legend": false, 221 | "tooltip": false, 222 | "viz": false 223 | }, 224 | "insertNulls": false, 225 | "lineInterpolation": "linear", 226 | "lineWidth": 1, 227 | "pointSize": 5, 228 | "scaleDistribution": { 229 | "type": "linear" 230 | }, 231 | "showPoints": "auto", 232 | "spanNulls": false, 233 | "stacking": { 234 | "group": "A", 235 | "mode": "none" 236 | }, 237 | "thresholdsStyle": { 238 | "mode": "off" 239 | } 240 | }, 241 | "mappings": [], 242 | "thresholds": { 243 | "mode": "absolute", 244 | "steps": [ 245 | { 246 | "color": "green", 247 | "value": null 248 | }, 249 | { 250 | "color": "red", 251 | "value": 80 252 | } 253 | ] 254 | } 255 | }, 256 | "overrides": [] 257 | }, 258 | "gridPos": { 259 | "h": 8, 260 | "w": 12, 261 | "x": 12, 262 | "y": 1 263 | }, 264 | "id": 28, 265 | "options": { 266 | "legend": { 267 | "calcs": [], 268 | "displayMode": "list", 269 | "placement": "bottom", 270 | "showLegend": true 271 | }, 272 | "tooltip": { 273 | "mode": "single", 274 | "sort": "none" 275 | } 276 | }, 277 | "targets": [ 278 | { 279 | "datasource": { 280 | "type": "prometheus", 281 | "uid": "${DS_PROMETHEUS}" 282 | }, 283 | "exemplar": false, 284 | "expr": "libp2p_connection_manager_protocol_streams_per_connection_90th_percentile", 285 | "interval": "", 286 | "legendFormat": "{{protocol}}", 287 | "refId": "A" 288 | } 289 | ], 290 | "title": "Streams Per Connection (90th percentile)", 291 | "type": "timeseries" 292 | }, 293 | { 294 | "datasource": { 295 | "type": "prometheus", 296 | "uid": "${DS_PROMETHEUS}" 297 | }, 298 | "fieldConfig": { 299 | "defaults": { 300 | "color": { 301 | "mode": "palette-classic" 302 | }, 303 | "custom": { 304 | "axisBorderShow": false, 305 | "axisCenteredZero": false, 306 | "axisColorMode": "text", 307 | "axisLabel": "", 308 | "axisPlacement": "auto", 309 | "barAlignment": 0, 310 | "drawStyle": "line", 311 | "fillOpacity": 0, 312 | "gradientMode": "none", 313 | "hideFrom": { 314 | "legend": false, 315 | "tooltip": false, 316 | "viz": false 317 | }, 318 | "insertNulls": false, 319 | "lineInterpolation": "linear", 320 | "lineWidth": 1, 321 | "pointSize": 5, 322 | "scaleDistribution": { 323 | "type": "linear" 324 | }, 325 | "showPoints": "auto", 326 | "spanNulls": false, 327 | "stacking": { 328 | "group": "A", 329 | "mode": "none" 330 | }, 331 | "thresholdsStyle": { 332 | "mode": "off" 333 | } 334 | }, 335 | "mappings": [], 336 | "thresholds": { 337 | "mode": "absolute", 338 | "steps": [ 339 | { 340 | "color": "green", 341 | "value": null 342 | }, 343 | { 344 | "color": "red", 345 | "value": 80 346 | } 347 | ] 348 | } 349 | }, 350 | "overrides": [] 351 | }, 352 | "gridPos": { 353 | "h": 8, 354 | "w": 12, 355 | "x": 0, 356 | "y": 9 357 | }, 358 | "id": 29, 359 | "options": { 360 | "legend": { 361 | "calcs": [], 362 | "displayMode": "list", 363 | "placement": "bottom", 364 | "showLegend": true 365 | }, 366 | "tooltip": { 367 | "mode": "single", 368 | "sort": "none" 369 | } 370 | }, 371 | "targets": [ 372 | { 373 | "datasource": { 374 | "type": "prometheus", 375 | "uid": "${DS_PROMETHEUS}" 376 | }, 377 | "editorMode": "code", 378 | "expr": "libp2p_dial_queue", 379 | "instant": false, 380 | "legendFormat": "{{libp2p_dial_queue}}", 381 | "range": true, 382 | "refId": "A" 383 | } 384 | ], 385 | "title": "Dial Queue", 386 | "type": "timeseries" 387 | }, 388 | { 389 | "datasource": { 390 | "type": "prometheus", 391 | "uid": "${DS_PROMETHEUS}" 392 | }, 393 | "fieldConfig": { 394 | "defaults": { 395 | "color": { 396 | "mode": "palette-classic" 397 | }, 398 | "custom": { 399 | "axisBorderShow": false, 400 | "axisCenteredZero": false, 401 | "axisColorMode": "text", 402 | "axisLabel": "", 403 | "axisPlacement": "auto", 404 | "barAlignment": 0, 405 | "drawStyle": "line", 406 | "fillOpacity": 0, 407 | "gradientMode": "none", 408 | "hideFrom": { 409 | "legend": false, 410 | "tooltip": false, 411 | "viz": false 412 | }, 413 | "insertNulls": false, 414 | "lineInterpolation": "linear", 415 | "lineWidth": 1, 416 | "pointSize": 5, 417 | "scaleDistribution": { 418 | "type": "linear" 419 | }, 420 | "showPoints": "auto", 421 | "spanNulls": false, 422 | "stacking": { 423 | "group": "A", 424 | "mode": "none" 425 | }, 426 | "thresholdsStyle": { 427 | "mode": "off" 428 | } 429 | }, 430 | "mappings": [], 431 | "thresholds": { 432 | "mode": "absolute", 433 | "steps": [ 434 | { 435 | "color": "green", 436 | "value": null 437 | }, 438 | { 439 | "color": "red", 440 | "value": 80 441 | } 442 | ] 443 | }, 444 | "unit": "none" 445 | }, 446 | "overrides": [] 447 | }, 448 | "gridPos": { 449 | "h": 8, 450 | "w": 12, 451 | "x": 12, 452 | "y": 9 453 | }, 454 | "id": 30, 455 | "options": { 456 | "legend": { 457 | "calcs": [], 458 | "displayMode": "list", 459 | "placement": "bottom", 460 | "showLegend": true 461 | }, 462 | "tooltip": { 463 | "mode": "single", 464 | "sort": "none" 465 | } 466 | }, 467 | "targets": [ 468 | { 469 | "datasource": { 470 | "type": "prometheus", 471 | "uid": "${DS_PROMETHEUS}" 472 | }, 473 | "editorMode": "code", 474 | "exemplar": false, 475 | "expr": "libp2p_protocol_streams_total", 476 | "interval": "", 477 | "legendFormat": "{{protocol}}", 478 | "range": true, 479 | "refId": "A" 480 | } 481 | ], 482 | "title": "Total Streams", 483 | "type": "timeseries" 484 | }, 485 | { 486 | "datasource": { 487 | "type": "prometheus", 488 | "uid": "${DS_PROMETHEUS}" 489 | }, 490 | "fieldConfig": { 491 | "defaults": { 492 | "color": { 493 | "mode": "palette-classic" 494 | }, 495 | "custom": { 496 | "axisBorderShow": false, 497 | "axisCenteredZero": false, 498 | "axisColorMode": "text", 499 | "axisLabel": "", 500 | "axisPlacement": "auto", 501 | "barAlignment": 0, 502 | "drawStyle": "line", 503 | "fillOpacity": 0, 504 | "gradientMode": "none", 505 | "hideFrom": { 506 | "legend": false, 507 | "tooltip": false, 508 | "viz": false 509 | }, 510 | "insertNulls": false, 511 | "lineInterpolation": "linear", 512 | "lineWidth": 1, 513 | "pointSize": 5, 514 | "scaleDistribution": { 515 | "type": "linear" 516 | }, 517 | "showPoints": "auto", 518 | "spanNulls": false, 519 | "stacking": { 520 | "group": "A", 521 | "mode": "none" 522 | }, 523 | "thresholdsStyle": { 524 | "mode": "off" 525 | } 526 | }, 527 | "mappings": [], 528 | "thresholds": { 529 | "mode": "absolute", 530 | "steps": [ 531 | { 532 | "color": "green", 533 | "value": null 534 | }, 535 | { 536 | "color": "red", 537 | "value": 80 538 | } 539 | ] 540 | }, 541 | "unit": "Bps" 542 | }, 543 | "overrides": [] 544 | }, 545 | "gridPos": { 546 | "h": 9, 547 | "w": 12, 548 | "x": 0, 549 | "y": 17 550 | }, 551 | "id": 31, 552 | "options": { 553 | "legend": { 554 | "calcs": [], 555 | "displayMode": "list", 556 | "placement": "bottom", 557 | "showLegend": true 558 | }, 559 | "tooltip": { 560 | "mode": "single", 561 | "sort": "none" 562 | } 563 | }, 564 | "targets": [ 565 | { 566 | "datasource": { 567 | "type": "prometheus", 568 | "uid": "${DS_PROMETHEUS}" 569 | }, 570 | "editorMode": "code", 571 | "exemplar": false, 572 | "expr": "rate(libp2p_data_transfer_bytes_total[$rate_interval])", 573 | "interval": "", 574 | "legendFormat": "{{protocol}}", 575 | "range": true, 576 | "refId": "A" 577 | } 578 | ], 579 | "title": "Total Bandwidth", 580 | "type": "timeseries" 581 | }, 582 | { 583 | "collapsed": false, 584 | "datasource": { 585 | "type": "prometheus", 586 | "uid": "${DS_PROMETHEUS}" 587 | }, 588 | "gridPos": { 589 | "h": 1, 590 | "w": 24, 591 | "x": 0, 592 | "y": 26 593 | }, 594 | "id": 24, 595 | "panels": [], 596 | "targets": [ 597 | { 598 | "datasource": { 599 | "type": "prometheus", 600 | "uid": "${DS_PROMETHEUS}" 601 | }, 602 | "refId": "A" 603 | } 604 | ], 605 | "title": "TCP", 606 | "type": "row" 607 | }, 608 | { 609 | "datasource": { 610 | "type": "prometheus", 611 | "uid": "${DS_PROMETHEUS}" 612 | }, 613 | "fieldConfig": { 614 | "defaults": { 615 | "color": { 616 | "mode": "palette-classic" 617 | }, 618 | "custom": { 619 | "axisBorderShow": false, 620 | "axisCenteredZero": false, 621 | "axisColorMode": "text", 622 | "axisLabel": "", 623 | "axisPlacement": "auto", 624 | "barAlignment": 0, 625 | "drawStyle": "line", 626 | "fillOpacity": 0, 627 | "gradientMode": "none", 628 | "hideFrom": { 629 | "legend": false, 630 | "tooltip": false, 631 | "viz": false 632 | }, 633 | "insertNulls": false, 634 | "lineInterpolation": "linear", 635 | "lineWidth": 1, 636 | "pointSize": 5, 637 | "scaleDistribution": { 638 | "type": "linear" 639 | }, 640 | "showPoints": "auto", 641 | "spanNulls": false, 642 | "stacking": { 643 | "group": "A", 644 | "mode": "none" 645 | }, 646 | "thresholdsStyle": { 647 | "mode": "off" 648 | } 649 | }, 650 | "mappings": [], 651 | "thresholds": { 652 | "mode": "absolute", 653 | "steps": [ 654 | { 655 | "color": "green", 656 | "value": null 657 | }, 658 | { 659 | "color": "red", 660 | "value": 80 661 | } 662 | ] 663 | }, 664 | "unit": "none" 665 | }, 666 | "overrides": [] 667 | }, 668 | "gridPos": { 669 | "h": 8, 670 | "w": 12, 671 | "x": 0, 672 | "y": 27 673 | }, 674 | "id": 32, 675 | "options": { 676 | "legend": { 677 | "calcs": [], 678 | "displayMode": "list", 679 | "placement": "bottom", 680 | "showLegend": true 681 | }, 682 | "tooltip": { 683 | "mode": "single", 684 | "sort": "none" 685 | } 686 | }, 687 | "targets": [ 688 | { 689 | "datasource": { 690 | "type": "prometheus", 691 | "uid": "${DS_PROMETHEUS}" 692 | }, 693 | "editorMode": "code", 694 | "exemplar": false, 695 | "expr": "rate(libp2p_tcp_dialer_events_total[$rate_interval])", 696 | "interval": "", 697 | "legendFormat": "{{event}}", 698 | "range": true, 699 | "refId": "A" 700 | } 701 | ], 702 | "title": "Dialer Event Rate", 703 | "type": "timeseries" 704 | }, 705 | { 706 | "datasource": { 707 | "type": "prometheus", 708 | "uid": "${DS_PROMETHEUS}" 709 | }, 710 | "fieldConfig": { 711 | "defaults": { 712 | "color": { 713 | "mode": "palette-classic" 714 | }, 715 | "custom": { 716 | "axisBorderShow": false, 717 | "axisCenteredZero": false, 718 | "axisColorMode": "text", 719 | "axisLabel": "", 720 | "axisPlacement": "auto", 721 | "barAlignment": 0, 722 | "drawStyle": "line", 723 | "fillOpacity": 0, 724 | "gradientMode": "none", 725 | "hideFrom": { 726 | "legend": false, 727 | "tooltip": false, 728 | "viz": false 729 | }, 730 | "insertNulls": false, 731 | "lineInterpolation": "linear", 732 | "lineWidth": 1, 733 | "pointSize": 5, 734 | "scaleDistribution": { 735 | "type": "linear" 736 | }, 737 | "showPoints": "auto", 738 | "spanNulls": false, 739 | "stacking": { 740 | "group": "A", 741 | "mode": "none" 742 | }, 743 | "thresholdsStyle": { 744 | "mode": "off" 745 | } 746 | }, 747 | "mappings": [], 748 | "thresholds": { 749 | "mode": "absolute", 750 | "steps": [ 751 | { 752 | "color": "green", 753 | "value": null 754 | }, 755 | { 756 | "color": "red", 757 | "value": 80 758 | } 759 | ] 760 | } 761 | }, 762 | "overrides": [] 763 | }, 764 | "gridPos": { 765 | "h": 8, 766 | "w": 12, 767 | "x": 12, 768 | "y": 27 769 | }, 770 | "id": 20, 771 | "options": { 772 | "legend": { 773 | "calcs": [], 774 | "displayMode": "list", 775 | "placement": "bottom", 776 | "showLegend": true 777 | }, 778 | "tooltip": { 779 | "mode": "single", 780 | "sort": "none" 781 | } 782 | }, 783 | "targets": [ 784 | { 785 | "datasource": { 786 | "type": "prometheus", 787 | "uid": "${DS_PROMETHEUS}" 788 | }, 789 | "editorMode": "code", 790 | "expr": "rate(libp2p_tcp_listener_events_total[$rate_interval])", 791 | "legendFormat": "{{address}}", 792 | "range": true, 793 | "refId": "A" 794 | } 795 | ], 796 | "title": "Listener Event Rate", 797 | "type": "timeseries" 798 | }, 799 | { 800 | "datasource": { 801 | "type": "prometheus", 802 | "uid": "${DS_PROMETHEUS}" 803 | }, 804 | "fieldConfig": { 805 | "defaults": { 806 | "color": { 807 | "mode": "palette-classic" 808 | }, 809 | "custom": { 810 | "axisBorderShow": false, 811 | "axisCenteredZero": false, 812 | "axisColorMode": "text", 813 | "axisLabel": "", 814 | "axisPlacement": "auto", 815 | "barAlignment": 0, 816 | "drawStyle": "line", 817 | "fillOpacity": 0, 818 | "gradientMode": "none", 819 | "hideFrom": { 820 | "legend": false, 821 | "tooltip": false, 822 | "viz": false 823 | }, 824 | "insertNulls": false, 825 | "lineInterpolation": "linear", 826 | "lineWidth": 1, 827 | "pointSize": 5, 828 | "scaleDistribution": { 829 | "type": "linear" 830 | }, 831 | "showPoints": "auto", 832 | "spanNulls": false, 833 | "stacking": { 834 | "group": "A", 835 | "mode": "none" 836 | }, 837 | "thresholdsStyle": { 838 | "mode": "off" 839 | } 840 | }, 841 | "decimals": 0, 842 | "mappings": [ 843 | { 844 | "options": { 845 | "0": { 846 | "index": 0, 847 | "text": "Down" 848 | }, 849 | "1": { 850 | "index": 1, 851 | "text": "Up" 852 | } 853 | }, 854 | "type": "value" 855 | } 856 | ], 857 | "max": 2, 858 | "min": -1, 859 | "thresholds": { 860 | "mode": "absolute", 861 | "steps": [ 862 | { 863 | "color": "green", 864 | "value": null 865 | }, 866 | { 867 | "color": "red", 868 | "value": 80 869 | } 870 | ] 871 | }, 872 | "unit": "short" 873 | }, 874 | "overrides": [] 875 | }, 876 | "gridPos": { 877 | "h": 8, 878 | "w": 12, 879 | "x": 0, 880 | "y": 35 881 | }, 882 | "id": 22, 883 | "options": { 884 | "legend": { 885 | "calcs": [], 886 | "displayMode": "list", 887 | "placement": "bottom", 888 | "showLegend": true 889 | }, 890 | "tooltip": { 891 | "mode": "single", 892 | "sort": "none" 893 | } 894 | }, 895 | "targets": [ 896 | { 897 | "datasource": { 898 | "type": "prometheus", 899 | "uid": "${DS_PROMETHEUS}" 900 | }, 901 | "editorMode": "code", 902 | "expr": "libp2p_tcp_listener_status_info", 903 | "legendFormat": "{{address}}", 904 | "range": true, 905 | "refId": "A" 906 | } 907 | ], 908 | "title": "Listener Status", 909 | "type": "timeseries" 910 | }, 911 | { 912 | "datasource": { 913 | "type": "prometheus", 914 | "uid": "${DS_PROMETHEUS}" 915 | }, 916 | "fieldConfig": { 917 | "defaults": { 918 | "color": { 919 | "mode": "palette-classic" 920 | }, 921 | "custom": { 922 | "axisBorderShow": false, 923 | "axisCenteredZero": false, 924 | "axisColorMode": "text", 925 | "axisLabel": "", 926 | "axisPlacement": "auto", 927 | "barAlignment": 0, 928 | "drawStyle": "line", 929 | "fillOpacity": 0, 930 | "gradientMode": "none", 931 | "hideFrom": { 932 | "legend": false, 933 | "tooltip": false, 934 | "viz": false 935 | }, 936 | "insertNulls": false, 937 | "lineInterpolation": "linear", 938 | "lineWidth": 1, 939 | "pointSize": 5, 940 | "scaleDistribution": { 941 | "type": "linear" 942 | }, 943 | "showPoints": "auto", 944 | "spanNulls": false, 945 | "stacking": { 946 | "group": "A", 947 | "mode": "none" 948 | }, 949 | "thresholdsStyle": { 950 | "mode": "off" 951 | } 952 | }, 953 | "mappings": [], 954 | "thresholds": { 955 | "mode": "absolute", 956 | "steps": [ 957 | { 958 | "color": "green", 959 | "value": null 960 | }, 961 | { 962 | "color": "red", 963 | "value": 80 964 | } 965 | ] 966 | } 967 | }, 968 | "overrides": [] 969 | }, 970 | "gridPos": { 971 | "h": 8, 972 | "w": 12, 973 | "x": 12, 974 | "y": 35 975 | }, 976 | "id": 26, 977 | "options": { 978 | "legend": { 979 | "calcs": [], 980 | "displayMode": "list", 981 | "placement": "bottom", 982 | "showLegend": true 983 | }, 984 | "tooltip": { 985 | "mode": "single", 986 | "sort": "none" 987 | } 988 | }, 989 | "targets": [ 990 | { 991 | "datasource": { 992 | "type": "prometheus", 993 | "uid": "${DS_PROMETHEUS}" 994 | }, 995 | "editorMode": "code", 996 | "expr": "rate(libp2p_tcp_listener_errors_total[$rate_interval])", 997 | "legendFormat": "{{address}}", 998 | "range": true, 999 | "refId": "A" 1000 | } 1001 | ], 1002 | "title": "Listener Error Rate", 1003 | "type": "timeseries" 1004 | }, 1005 | { 1006 | "collapsed": false, 1007 | "datasource": { 1008 | "type": "prometheus", 1009 | "uid": "${DS_PROMETHEUS}" 1010 | }, 1011 | "gridPos": { 1012 | "h": 1, 1013 | "w": 24, 1014 | "x": 0, 1015 | "y": 43 1016 | }, 1017 | "id": 14, 1018 | "panels": [], 1019 | "targets": [ 1020 | { 1021 | "datasource": { 1022 | "type": "prometheus", 1023 | "uid": "${DS_PROMETHEUS}" 1024 | }, 1025 | "refId": "A" 1026 | } 1027 | ], 1028 | "title": "Noise", 1029 | "type": "row" 1030 | }, 1031 | { 1032 | "datasource": { 1033 | "type": "prometheus", 1034 | "uid": "${DS_PROMETHEUS}" 1035 | }, 1036 | "fieldConfig": { 1037 | "defaults": { 1038 | "color": { 1039 | "mode": "palette-classic" 1040 | }, 1041 | "custom": { 1042 | "axisBorderShow": false, 1043 | "axisCenteredZero": false, 1044 | "axisColorMode": "text", 1045 | "axisLabel": "", 1046 | "axisPlacement": "auto", 1047 | "barAlignment": 0, 1048 | "drawStyle": "line", 1049 | "fillOpacity": 0, 1050 | "gradientMode": "none", 1051 | "hideFrom": { 1052 | "legend": false, 1053 | "tooltip": false, 1054 | "viz": false 1055 | }, 1056 | "insertNulls": false, 1057 | "lineInterpolation": "linear", 1058 | "lineWidth": 1, 1059 | "pointSize": 5, 1060 | "scaleDistribution": { 1061 | "type": "linear" 1062 | }, 1063 | "showPoints": "auto", 1064 | "spanNulls": false, 1065 | "stacking": { 1066 | "group": "A", 1067 | "mode": "none" 1068 | }, 1069 | "thresholdsStyle": { 1070 | "mode": "off" 1071 | } 1072 | }, 1073 | "mappings": [], 1074 | "thresholds": { 1075 | "mode": "absolute", 1076 | "steps": [ 1077 | { 1078 | "color": "green", 1079 | "value": null 1080 | }, 1081 | { 1082 | "color": "red", 1083 | "value": 80 1084 | } 1085 | ] 1086 | }, 1087 | "unit": "outcomes/interval" 1088 | }, 1089 | "overrides": [] 1090 | }, 1091 | "gridPos": { 1092 | "h": 8, 1093 | "w": 12, 1094 | "x": 0, 1095 | "y": 44 1096 | }, 1097 | "id": 33, 1098 | "options": { 1099 | "legend": { 1100 | "calcs": [], 1101 | "displayMode": "list", 1102 | "placement": "bottom", 1103 | "showLegend": true 1104 | }, 1105 | "tooltip": { 1106 | "mode": "single", 1107 | "sort": "none" 1108 | } 1109 | }, 1110 | "targets": [ 1111 | { 1112 | "datasource": { 1113 | "type": "prometheus", 1114 | "uid": "${DS_PROMETHEUS}" 1115 | }, 1116 | "editorMode": "code", 1117 | "expr": "rate(libp2p_noise_xxhandshake_successes_total[$rate_interval])", 1118 | "instant": false, 1119 | "legendFormat": "success", 1120 | "range": true, 1121 | "refId": "A" 1122 | }, 1123 | { 1124 | "datasource": { 1125 | "type": "prometheus", 1126 | "uid": "${DS_PROMETHEUS}" 1127 | }, 1128 | "editorMode": "code", 1129 | "expr": "rate(libp2p_noise_xxhandshake_error_total[$rate_interval])", 1130 | "hide": false, 1131 | "instant": false, 1132 | "legendFormat": "failure", 1133 | "range": true, 1134 | "refId": "B" 1135 | } 1136 | ], 1137 | "title": "Handshake Outcome Rate", 1138 | "type": "timeseries" 1139 | }, 1140 | { 1141 | "datasource": { 1142 | "type": "prometheus", 1143 | "uid": "${DS_PROMETHEUS}" 1144 | }, 1145 | "fieldConfig": { 1146 | "defaults": { 1147 | "color": { 1148 | "mode": "palette-classic" 1149 | }, 1150 | "custom": { 1151 | "axisBorderShow": false, 1152 | "axisCenteredZero": false, 1153 | "axisColorMode": "text", 1154 | "axisLabel": "", 1155 | "axisPlacement": "auto", 1156 | "barAlignment": 0, 1157 | "drawStyle": "line", 1158 | "fillOpacity": 0, 1159 | "gradientMode": "none", 1160 | "hideFrom": { 1161 | "legend": false, 1162 | "tooltip": false, 1163 | "viz": false 1164 | }, 1165 | "insertNulls": false, 1166 | "lineInterpolation": "linear", 1167 | "lineWidth": 1, 1168 | "pointSize": 5, 1169 | "scaleDistribution": { 1170 | "type": "linear" 1171 | }, 1172 | "showPoints": "auto", 1173 | "spanNulls": false, 1174 | "stacking": { 1175 | "group": "A", 1176 | "mode": "none" 1177 | }, 1178 | "thresholdsStyle": { 1179 | "mode": "off" 1180 | } 1181 | }, 1182 | "mappings": [], 1183 | "thresholds": { 1184 | "mode": "absolute", 1185 | "steps": [ 1186 | { 1187 | "color": "green", 1188 | "value": null 1189 | }, 1190 | { 1191 | "color": "red", 1192 | "value": 80 1193 | } 1194 | ] 1195 | }, 1196 | "unit": "packets/interval" 1197 | }, 1198 | "overrides": [] 1199 | }, 1200 | "gridPos": { 1201 | "h": 8, 1202 | "w": 12, 1203 | "x": 12, 1204 | "y": 44 1205 | }, 1206 | "id": 34, 1207 | "options": { 1208 | "legend": { 1209 | "calcs": [], 1210 | "displayMode": "list", 1211 | "placement": "bottom", 1212 | "showLegend": true 1213 | }, 1214 | "tooltip": { 1215 | "mode": "single", 1216 | "sort": "none" 1217 | } 1218 | }, 1219 | "targets": [ 1220 | { 1221 | "datasource": { 1222 | "type": "prometheus", 1223 | "uid": "${DS_PROMETHEUS}" 1224 | }, 1225 | "editorMode": "code", 1226 | "expr": "rate(libp2p_noise_encrypted_packets_total[$rate_interval])", 1227 | "instant": false, 1228 | "legendFormat": "encrypted", 1229 | "range": true, 1230 | "refId": "A" 1231 | }, 1232 | { 1233 | "datasource": { 1234 | "type": "prometheus", 1235 | "uid": "${DS_PROMETHEUS}" 1236 | }, 1237 | "editorMode": "code", 1238 | "expr": "rate(libp2p_noise_decrypted_packets_total[$rate_interval])", 1239 | "hide": false, 1240 | "instant": false, 1241 | "legendFormat": "decrypted", 1242 | "range": true, 1243 | "refId": "B" 1244 | } 1245 | ], 1246 | "title": "Packet Rate", 1247 | "type": "timeseries" 1248 | }, 1249 | { 1250 | "datasource": { 1251 | "type": "prometheus", 1252 | "uid": "${DS_PROMETHEUS}" 1253 | }, 1254 | "fieldConfig": { 1255 | "defaults": { 1256 | "color": { 1257 | "mode": "palette-classic" 1258 | }, 1259 | "custom": { 1260 | "axisBorderShow": false, 1261 | "axisCenteredZero": false, 1262 | "axisColorMode": "text", 1263 | "axisLabel": "", 1264 | "axisPlacement": "auto", 1265 | "barAlignment": 0, 1266 | "drawStyle": "line", 1267 | "fillOpacity": 0, 1268 | "gradientMode": "none", 1269 | "hideFrom": { 1270 | "legend": false, 1271 | "tooltip": false, 1272 | "viz": false 1273 | }, 1274 | "insertNulls": false, 1275 | "lineInterpolation": "linear", 1276 | "lineWidth": 1, 1277 | "pointSize": 5, 1278 | "scaleDistribution": { 1279 | "type": "linear" 1280 | }, 1281 | "showPoints": "auto", 1282 | "spanNulls": false, 1283 | "stacking": { 1284 | "group": "A", 1285 | "mode": "none" 1286 | }, 1287 | "thresholdsStyle": { 1288 | "mode": "off" 1289 | } 1290 | }, 1291 | "mappings": [], 1292 | "thresholds": { 1293 | "mode": "absolute", 1294 | "steps": [ 1295 | { 1296 | "color": "green", 1297 | "value": null 1298 | }, 1299 | { 1300 | "color": "red", 1301 | "value": 80 1302 | } 1303 | ] 1304 | }, 1305 | "unit": "errors/interval" 1306 | }, 1307 | "overrides": [] 1308 | }, 1309 | "gridPos": { 1310 | "h": 8, 1311 | "w": 12, 1312 | "x": 0, 1313 | "y": 52 1314 | }, 1315 | "id": 35, 1316 | "options": { 1317 | "legend": { 1318 | "calcs": [], 1319 | "displayMode": "list", 1320 | "placement": "bottom", 1321 | "showLegend": false 1322 | }, 1323 | "tooltip": { 1324 | "mode": "single", 1325 | "sort": "none" 1326 | } 1327 | }, 1328 | "targets": [ 1329 | { 1330 | "datasource": { 1331 | "type": "prometheus", 1332 | "uid": "${DS_PROMETHEUS}" 1333 | }, 1334 | "editorMode": "code", 1335 | "expr": "rate(libp2p_noise_decrypt_errors_total[$rate_interval])", 1336 | "instant": false, 1337 | "legendFormat": "Decryption Errors", 1338 | "range": true, 1339 | "refId": "A" 1340 | } 1341 | ], 1342 | "title": "Packet Decryption Error Rate", 1343 | "type": "timeseries" 1344 | }, 1345 | { 1346 | "datasource": { 1347 | "type": "prometheus", 1348 | "uid": "${DS_PROMETHEUS}" 1349 | }, 1350 | "fieldConfig": { 1351 | "defaults": { 1352 | "color": { 1353 | "mode": "palette-classic" 1354 | }, 1355 | "custom": { 1356 | "axisBorderShow": false, 1357 | "axisCenteredZero": false, 1358 | "axisColorMode": "text", 1359 | "axisLabel": "", 1360 | "axisPlacement": "auto", 1361 | "barAlignment": 0, 1362 | "drawStyle": "line", 1363 | "fillOpacity": 0, 1364 | "gradientMode": "none", 1365 | "hideFrom": { 1366 | "legend": false, 1367 | "tooltip": false, 1368 | "viz": false 1369 | }, 1370 | "insertNulls": false, 1371 | "lineInterpolation": "linear", 1372 | "lineWidth": 1, 1373 | "pointSize": 5, 1374 | "scaleDistribution": { 1375 | "type": "linear" 1376 | }, 1377 | "showPoints": "auto", 1378 | "spanNulls": false, 1379 | "stacking": { 1380 | "group": "A", 1381 | "mode": "none" 1382 | }, 1383 | "thresholdsStyle": { 1384 | "mode": "off" 1385 | } 1386 | }, 1387 | "mappings": [], 1388 | "thresholds": { 1389 | "mode": "absolute", 1390 | "steps": [ 1391 | { 1392 | "color": "green", 1393 | "value": null 1394 | }, 1395 | { 1396 | "color": "red", 1397 | "value": 80 1398 | } 1399 | ] 1400 | }, 1401 | "unit": "percentunit" 1402 | }, 1403 | "overrides": [] 1404 | }, 1405 | "gridPos": { 1406 | "h": 8, 1407 | "w": 12, 1408 | "x": 12, 1409 | "y": 52 1410 | }, 1411 | "id": 36, 1412 | "options": { 1413 | "legend": { 1414 | "calcs": [], 1415 | "displayMode": "list", 1416 | "placement": "bottom", 1417 | "showLegend": true 1418 | }, 1419 | "tooltip": { 1420 | "mode": "single", 1421 | "sort": "none" 1422 | } 1423 | }, 1424 | "targets": [ 1425 | { 1426 | "datasource": { 1427 | "type": "prometheus", 1428 | "uid": "${DS_PROMETHEUS}" 1429 | }, 1430 | "editorMode": "code", 1431 | "expr": "rate(libp2p_noise_decrypt_errors_total[$rate_interval]) / rate(libp2p_noise_decrypted_packets_total[$rate_interval])", 1432 | "instant": false, 1433 | "legendFormat": "Decryption Error Percentage", 1434 | "range": true, 1435 | "refId": "A" 1436 | } 1437 | ], 1438 | "title": "Packet Decryption Error Percentage", 1439 | "type": "timeseries" 1440 | } 1441 | ], 1442 | "refresh": "10s", 1443 | "schemaVersion": 39, 1444 | "tags": [ 1445 | "lodestar" 1446 | ], 1447 | "templating": { 1448 | "list": [ 1449 | { 1450 | "current": { 1451 | "selected": false, 1452 | "text": "default", 1453 | "value": "default" 1454 | }, 1455 | "hide": 0, 1456 | "includeAll": false, 1457 | "label": "datasource", 1458 | "multi": false, 1459 | "name": "DS_PROMETHEUS", 1460 | "options": [], 1461 | "query": "prometheus", 1462 | "queryValue": "", 1463 | "refresh": 1, 1464 | "regex": "", 1465 | "skipUrlSync": false, 1466 | "type": "datasource" 1467 | }, 1468 | { 1469 | "auto": true, 1470 | "auto_count": 30, 1471 | "auto_min": "10s", 1472 | "current": { 1473 | "selected": false, 1474 | "text": "1h", 1475 | "value": "1h" 1476 | }, 1477 | "hide": 0, 1478 | "label": "rate() interval", 1479 | "name": "rate_interval", 1480 | "options": [ 1481 | { 1482 | "selected": false, 1483 | "text": "auto", 1484 | "value": "$__auto_interval_rate_interval" 1485 | }, 1486 | { 1487 | "selected": false, 1488 | "text": "1m", 1489 | "value": "1m" 1490 | }, 1491 | { 1492 | "selected": false, 1493 | "text": "10m", 1494 | "value": "10m" 1495 | }, 1496 | { 1497 | "selected": false, 1498 | "text": "30m", 1499 | "value": "30m" 1500 | }, 1501 | { 1502 | "selected": true, 1503 | "text": "1h", 1504 | "value": "1h" 1505 | }, 1506 | { 1507 | "selected": false, 1508 | "text": "6h", 1509 | "value": "6h" 1510 | }, 1511 | { 1512 | "selected": false, 1513 | "text": "12h", 1514 | "value": "12h" 1515 | }, 1516 | { 1517 | "selected": false, 1518 | "text": "1d", 1519 | "value": "1d" 1520 | }, 1521 | { 1522 | "selected": false, 1523 | "text": "7d", 1524 | "value": "7d" 1525 | }, 1526 | { 1527 | "selected": false, 1528 | "text": "14d", 1529 | "value": "14d" 1530 | }, 1531 | { 1532 | "selected": false, 1533 | "text": "30d", 1534 | "value": "30d" 1535 | } 1536 | ], 1537 | "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", 1538 | "queryValue": "", 1539 | "refresh": 2, 1540 | "skipUrlSync": false, 1541 | "type": "interval" 1542 | } 1543 | ] 1544 | }, 1545 | "time": { 1546 | "from": "now-15m", 1547 | "to": "now" 1548 | }, 1549 | "timepicker": { 1550 | "refresh_intervals": [ 1551 | "5s", 1552 | "10s", 1553 | "30s", 1554 | "1m", 1555 | "5m", 1556 | "15m", 1557 | "30m", 1558 | "1h", 1559 | "2h", 1560 | "1d" 1561 | ] 1562 | }, 1563 | "timezone": "utc", 1564 | "title": "Lodestar - libp2p", 1565 | "uid": "lodestar_libp2p", 1566 | "version": 2, 1567 | "weekStart": "monday" 1568 | } 1569 | --------------------------------------------------------------------------------