├── .yarnrc ├── .tool-versions ├── .npmignore ├── .github ├── CODEOWNERS ├── actions │ ├── setup │ │ └── action.yaml │ ├── set-git-credentials │ │ └── action.yaml │ └── validate-version-labels │ │ └── action.yaml └── workflows │ ├── pinned-dependencies.yaml │ ├── publish.yaml │ ├── add-or-validate-labels.yaml │ └── main.yaml ├── scripts ├── generator-adapter │ ├── generators │ │ └── app │ │ │ └── templates │ │ │ ├── CHANGELOG.md │ │ │ ├── src │ │ │ ├── config │ │ │ │ ├── overrides.json │ │ │ │ └── index.ts.ejs │ │ │ ├── endpoint │ │ │ │ ├── index.ts.ejs │ │ │ │ ├── base.ts.ejs │ │ │ │ ├── endpoint.ts.ejs │ │ │ │ └── endpoint-router.ts.ejs │ │ │ ├── index.ts.ejs │ │ │ └── transport │ │ │ │ ├── customfg.ts.ejs │ │ │ │ ├── ws.ts.ejs │ │ │ │ ├── http.ts.ejs │ │ │ │ └── custombg.ts.ejs │ │ │ ├── jest.config.js │ │ │ ├── test-payload.json │ │ │ ├── babel.config.js │ │ │ ├── tsconfig.test.json │ │ │ ├── README.md │ │ │ ├── tsconfig.json │ │ │ ├── package.json │ │ │ ├── test │ │ │ ├── fixtures.ts.ejs │ │ │ ├── adapter.test.ts.ejs │ │ │ └── adapter-ws.test.ts.ejs │ │ │ └── tsconfig.base.json │ ├── tsconfig.json │ └── package.json ├── adapter-generator.ts ├── metrics-table.ts └── ea-settings-table.ts ├── .c8rc.json ├── test ├── tsconfig.json ├── _force-exit.mjs ├── helper.ts ├── custom-logger.test.ts ├── metrics │ ├── feed-id.test.ts │ ├── por-metrics.test.ts │ ├── polling-metrics.test.ts │ ├── redis-metrics.test.ts │ ├── helper.ts │ ├── labels.test.ts │ └── warmer-metrics.test.ts ├── por.test.ts ├── utils.test.ts ├── adapter.test.ts ├── subscription-set │ └── subscription-set-factory.test.ts ├── adapter │ └── market-status.test.ts ├── util │ └── testing-utils.test.ts ├── correlation.test.ts ├── logger.test.ts ├── cache │ ├── local.test.ts │ └── helper.ts ├── background-executor.test.ts └── debug-endpoints.test.ts ├── .gitignore ├── typedoc.json ├── src ├── adapter │ ├── index.ts │ ├── stock.ts │ ├── market-status.ts │ └── lwba.ts ├── util │ ├── censor │ │ └── censor-list.ts │ ├── settings.ts │ ├── subscription-set │ │ ├── redis-sorted-set.ts │ │ ├── subscription-set.ts │ │ └── expiring-sorted-set.ts │ └── group-runner.ts ├── metrics │ ├── util.ts │ └── constants.ts ├── rate-limiting │ ├── factory.ts │ ├── fixed-interval.ts │ └── burst.ts ├── cache │ ├── factory.ts │ └── metrics.ts ├── validation │ ├── preset-tokens.json │ ├── market-status.ts │ ├── error.ts │ └── utils.ts ├── debug │ ├── router.ts │ └── settings-page.ts ├── transports │ ├── metrics.ts │ └── abstract │ │ ├── subscription.ts │ │ └── streaming.ts ├── status │ └── router.ts └── background-executor.ts ├── renovate.json ├── .prettierignore ├── tsconfig.json ├── docker-compose.yaml ├── docs ├── components │ ├── transports.md │ ├── transport-types │ │ ├── streaming-transport.md │ │ └── subscription-transport.md │ └── tests.md └── guides │ └── creating-a-new-v3-ea.md ├── README.md ├── package.json └── eslint.config.mjs /.yarnrc: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | nodejs 24.12.0 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | dist/examples 2 | examples 3 | ./dist/examples -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @smartcontractkit/data-feeds-engineers 2 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.c8rc.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporter": ["lcov", "text", "json-summary"] 3 | } 4 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/config/overrides.json: -------------------------------------------------------------------------------- 1 | { 2 | "<%= adapterName %>": {} 3 | } 4 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | modulePathIgnorePatterns: ['./dist'], 3 | } -------------------------------------------------------------------------------- /test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "include": ["./**/*.ts"], 4 | "exclude": ["../src/**/*.ts"] 5 | } 6 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/test-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "requests": [{ 3 | "from": "BTC", 4 | "to": "USD" 5 | }] 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | srcdocs/ 4 | .DS_Store 5 | coverage/ 6 | .clinic/ 7 | .idea/ 8 | yarn-error.log 9 | package-lock.json 10 | .vscode/ 11 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://typedoc.org/schema.json", 3 | "entryPoints": ["./src/index.ts"], 4 | "out": "srcdocs", 5 | "includeVersion": true 6 | } 7 | -------------------------------------------------------------------------------- /test/_force-exit.mjs: -------------------------------------------------------------------------------- 1 | import process from 'node:process'; 2 | import { registerCompletionHandler } from 'ava'; 3 | 4 | registerCompletionHandler(() => { 5 | process.exit(); 6 | }); 7 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'], 3 | } -------------------------------------------------------------------------------- /src/adapter/index.ts: -------------------------------------------------------------------------------- 1 | export * from './basic' 2 | export * from './endpoint' 3 | export * from './lwba' 4 | export * from './market-status' 5 | export * from './price' 6 | export * from './types' 7 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/endpoint/index.ts.ejs: -------------------------------------------------------------------------------- 1 | <% for(let i=0; iexport { endpoint as <%- endpoints[i].normalizedEndpointName %> } from './<%= endpoints[i].inputEndpointName %>' <%- '\n' %><%}%> -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended", 5 | "preview:dockerCompose", 6 | "preview:dockerVersions", 7 | "schedule:weekends" 8 | ], 9 | "prConcurrentLimit": 5 10 | } 11 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "<%- standalone ? "./tsconfig.base.json" : "../../tsconfig.base.json" %>", 3 | "include": ["src/**/*", "**/test", "src/**/*.json"], 4 | "compilerOptions": { 5 | "noEmit": true 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/README.md: -------------------------------------------------------------------------------- 1 | # Chainlink External Adapter for <%= adapterName %> 2 | 3 | This README will be generated automatically when code is merged to `main`. If you would like to generate a preview of the README, please run `yarn generate:readme <%= adapterName %>`. 4 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "<%- standalone ? "./tsconfig.base.json" : "../../tsconfig.base.json" %>", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "rootDir": "src" 6 | }, 7 | "include": ["src/**/*", "src/**/*.json"], 8 | "exclude": ["dist", "**/*.spec.ts", "**/*.test.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /src/util/censor/censor-list.ts: -------------------------------------------------------------------------------- 1 | export default class CensorList { 2 | static censorList: CensorKeyValue[] = [] 3 | static getAll(): CensorKeyValue[] { 4 | return this.censorList 5 | } 6 | static set(censorList: CensorKeyValue[]) { 7 | this.censorList = censorList 8 | } 9 | } 10 | 11 | export interface CensorKeyValue { 12 | key: string 13 | value: RegExp 14 | } 15 | -------------------------------------------------------------------------------- /test/helper.ts: -------------------------------------------------------------------------------- 1 | import FakeTimers from '@sinonjs/fake-timers' 2 | 3 | export const installTimers = () => { 4 | return FakeTimers.install({ 5 | toFake: [ 6 | 'setTimeout', 7 | 'clearTimeout', 8 | 'setImmediate', 9 | 'clearImmediate', 10 | 'setInterval', 11 | 'clearInterval', 12 | 'Date', 13 | 'hrtime', 14 | ], 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /scripts/adapter-generator.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { resolve } from 'path' 4 | import { execSync } from 'child_process' 5 | 6 | const pathArg = process.argv[2] || '' 7 | 8 | const generatorPath = resolve(__dirname, './generator-adapter/generators/app/index.js') 9 | const generatorCommand = `yo ${generatorPath} ${pathArg}` 10 | 11 | execSync(generatorCommand, { stdio: 'inherit' }) 12 | -------------------------------------------------------------------------------- /scripts/generator-adapter/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "rootDir": "./../", 4 | "outDir": "../../dist/src", 5 | "target": "es2022", 6 | "downlevelIteration": true, 7 | "esModuleInterop": true, 8 | "moduleResolution": "nodenext", 9 | "module": "nodenext", 10 | "resolveJsonModule": true 11 | }, 12 | "files": [ 13 | "./generators/app/index.ts" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # don't ever lint node_modules 2 | node_modules 3 | # don't lint build output (make sure it's set to your correct build folder name) 4 | dist 5 | # don't lint nyc coverage output 6 | coverage 7 | test-payload.json 8 | .yarn 9 | .vscode 10 | .pnp.cjs 11 | .pnp.loader.mjs 12 | # don't prettify ea-generator folder since it contains invalid syntax 13 | scripts/generator-adapter 14 | 15 | packages/k6/k8s/templates 16 | -------------------------------------------------------------------------------- /.github/actions/setup/action.yaml: -------------------------------------------------------------------------------- 1 | name: 'Setup' 2 | description: 'Sets up the project, installs dependencies, caches results' 3 | runs: 4 | using: 'composite' 5 | steps: 6 | - uses: actions/setup-node@v6 7 | with: 8 | node-version: 24.12 9 | registry-url: https://registry.npmjs.org 10 | always-auth: true 11 | cache: yarn 12 | - run: yarn install --frozen-lockfile 13 | shell: bash 14 | -------------------------------------------------------------------------------- /scripts/generator-adapter/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "generator-adapter", 3 | "version": "0.0.1", 4 | "files": [ 5 | "generators" 6 | ], 7 | "type": "module", 8 | "main": "generators/app/index.js", 9 | "keywords": [ 10 | "yeoman-generator", 11 | "extnerl-adapter-generator" 12 | ], 13 | "devDependencies": { 14 | "yeoman-generator": "7.5.1", 15 | "@yeoman/types": "1.9.1", 16 | "mem-fs": "4.1.2", 17 | "@yeoman/adapter": "4.0.1", 18 | "@types/node": "24.10.4" 19 | } 20 | } -------------------------------------------------------------------------------- /src/metrics/util.ts: -------------------------------------------------------------------------------- 1 | import { EndpointGenerics } from '../adapter' 2 | import { calculateFeedId } from '../cache' 3 | import { AdapterMetricsMeta, AdapterRequestData } from '../util' 4 | import { InputParameters } from '../validation' 5 | 6 | export const getMetricsMeta = ( 7 | args: { 8 | inputParameters: InputParameters 9 | adapterSettings: T['Settings'] 10 | }, 11 | data: AdapterRequestData, 12 | ): AdapterMetricsMeta => { 13 | const feedId = calculateFeedId(args, data) 14 | return { feedId } 15 | } 16 | -------------------------------------------------------------------------------- /.github/actions/set-git-credentials/action.yaml: -------------------------------------------------------------------------------- 1 | name: 'Set git credentials' 2 | description: 'Sets config for git, used when we need to run raw git commands' 3 | runs: 4 | using: 'composite' 5 | steps: 6 | - run: | 7 | # Below are credentials for the built-in GitHub Actions bot: https://github.com/orgs/community/discussions/26560 8 | git config --global user.name "github-actions[bot]" 9 | git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" 10 | git config --global push.followTags true 11 | shell: bash 12 | -------------------------------------------------------------------------------- /src/rate-limiting/factory.ts: -------------------------------------------------------------------------------- 1 | import { BurstRateLimiter, RateLimiter } from '.' 2 | import { FixedIntervalRateLimiter } from './fixed-interval' 3 | 4 | export enum RateLimitingStrategy { 5 | BURST = 'burst', 6 | FIXED_INTERVAL = 'fixed-interval', 7 | } 8 | 9 | export class RateLimiterFactory { 10 | static buildRateLimiter(strategy: RateLimitingStrategy): RateLimiter { 11 | switch (strategy) { 12 | case RateLimitingStrategy.BURST: 13 | return new BurstRateLimiter() 14 | case RateLimitingStrategy.FIXED_INTERVAL: 15 | return new FixedIntervalRateLimiter() 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /scripts/metrics-table.ts: -------------------------------------------------------------------------------- 1 | import { metrics } from '../src/metrics/index' 2 | 3 | metrics.initialize() 4 | 5 | const metricDefs = metrics.getMetricsDefinition() as unknown as { 6 | [key: string]: { 7 | help: string 8 | labelNames: string[] 9 | } 10 | } 11 | 12 | const sortedMetrics = Object.entries(metricDefs).sort(([metricName1], [metricName2]) => 13 | metricName1.localeCompare(metricName2), 14 | ) 15 | 16 | let output = `# Metrics\n\n|Name|Type|Help|Labels|\n|---|---|---|---|\n` 17 | 18 | for (const [name, metric] of sortedMetrics) { 19 | const type = metric.constructor.name 20 | const help = metric.help 21 | const labels = metric.labelNames.map((l) => `- ${l}`).join('
') 22 | output += `|${name}|${type}|${help}|${labels}|\n` 23 | } 24 | 25 | // eslint-disable-next-line no-console 26 | console.log(output) 27 | -------------------------------------------------------------------------------- /src/cache/factory.ts: -------------------------------------------------------------------------------- 1 | import Redis from 'ioredis' 2 | import { makeLogger } from '../util' 3 | import { LocalCache } from './local' 4 | import { RedisCache } from './redis' 5 | 6 | const logger = makeLogger('CacheFactory') 7 | export class CacheFactory { 8 | static buildCache( 9 | { cacheType, maxSizeForLocalCache }: { cacheType: string; maxSizeForLocalCache: number }, 10 | redisClient?: Redis, 11 | ) { 12 | logger.info(`Using "${cacheType}" cache.`) 13 | switch (cacheType) { 14 | case 'local': 15 | return new LocalCache(maxSizeForLocalCache) 16 | case 'redis': { 17 | if (!redisClient) { 18 | throw new Error('Redis client undefined. Cannot create Redis cache') 19 | } 20 | return new RedisCache(redisClient, maxSizeForLocalCache) 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "dist", 4 | "rootDir": ".", 5 | "target": "es2022", 6 | "esModuleInterop": true, 7 | "moduleResolution": "node", 8 | "module": "CommonJS", 9 | "resolveJsonModule": true, 10 | "declaration": true, // Generate .d.ts files 11 | "sourceMap": true, // Keep map of ts contents behind transpiled js 12 | 13 | // Type checking 14 | "allowUnreachableCode": false, 15 | "allowUnusedLabels": false, 16 | "alwaysStrict": true, 17 | "noImplicitAny": true, 18 | "noImplicitOverride": true, 19 | "noImplicitReturns": false, 20 | "noImplicitThis": true, 21 | "noPropertyAccessFromIndexSignature": true, 22 | "noUnusedLocals": true, 23 | "strict": true 24 | }, 25 | "exclude": ["src/**/test/**/*"], 26 | "include": ["./src/**/*"] 27 | } 28 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Very basic stack to demonstrate horizontal scaling 2 | services: 3 | redis: 4 | image: redis 5 | container_name: redis 6 | 7 | coingecko-reader: 8 | image: node:alpine 9 | ports: 10 | - 8080 11 | deploy: 12 | mode: replicated 13 | replicas: 2 14 | environment: 15 | - DISABLE_BACKGROUND_EXECUTOR=true 16 | volumes: 17 | - ./:/home/node/app 18 | command: node /home/node/app/dist/test.js 19 | 20 | coingecko-writer: 21 | image: node:alpine 22 | volumes: 23 | - ./:/home/node/app 24 | environment: 25 | - DISABLE_REST_API=true 26 | command: node /home/node/app/dist/test.js 27 | 28 | lb: 29 | image: dockercloud/haproxy 30 | links: 31 | - coingecko-reader 32 | ports: 33 | - '80:80' 34 | volumes: 35 | - /var/run/docker.sock:/var/run/docker.sock 36 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@chainlink/<%= adapterName %>-adapter", 3 | "version": "0.0.0", 4 | "description": "Chainlink <%= adapterName %> adapter.", 5 | "keywords": [ 6 | "Chainlink", 7 | "LINK", 8 | "blockchain", 9 | "oracle", 10 | "<%= adapterName %>" 11 | ], 12 | "main": "dist/index.js", 13 | "types": "dist/index.d.ts", 14 | "files": [ 15 | "dist" 16 | ], 17 | "repository": { 18 | "url": "https://github.com/smartcontractkit/external-adapters-js", 19 | "type": "git" 20 | }, 21 | "license": "MIT", 22 | "scripts": { 23 | "clean": "rm -rf dist && rm -f tsconfig.tsbuildinfo", 24 | "prepack": "yarn build", 25 | "build": "tsc -b", 26 | "server": "node -e 'require(\"./index.js\").server()'", 27 | "server:dist": "node -e 'require(\"./dist/index.js\").server()'", 28 | "start": "yarn server:dist" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/metrics/constants.ts: -------------------------------------------------------------------------------- 1 | export enum HttpRequestType { 2 | CACHE_HIT = 'cacheHit', 3 | DATA_PROVIDER_HIT = 'dataProviderHit', 4 | ADAPTER_ERROR = 'adapterError', 5 | INPUT_ERROR = 'inputError', 6 | RATE_LIMIT_ERROR = 'rateLimitError', 7 | // BURST_LIMIT_ERROR = 'burstLimitError', 8 | // BACKOFF_ERROR = 'backoffError', 9 | DP_ERROR = 'dataProviderError', 10 | TIMEOUT_ERROR = 'timeoutError', 11 | CONNECTION_ERROR = 'connectionError', 12 | // RES_EMPTY_ERROR = 'responseEmptyError', 13 | // RES_INVALID_ERROR = 'responseInvalidError', 14 | CUSTOM_ERROR = 'customError', 15 | LWBA_ERROR = 'lwbaError', 16 | } 17 | 18 | /** 19 | * Maxiumum number of characters that a feedId can contain. 20 | */ 21 | export const MAX_FEED_ID_LENGTH = 300 22 | 23 | // We should tune these as we collect data, this is the default bucket distribution that prom comes with 24 | export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] 25 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/config/index.ts.ejs: -------------------------------------------------------------------------------- 1 | import { AdapterConfig } from '@chainlink/external-adapter-framework/config' 2 | 3 | export const config = new AdapterConfig( 4 | { 5 | API_KEY: { 6 | description: 7 | 'An API key for Data Provider', 8 | type: 'string', 9 | required: true, 10 | sensitive: true, 11 | }, 12 | API_ENDPOINT: { 13 | description: 14 | 'An API endpoint for Data Provider', 15 | type: 'string', 16 | default: 'https://dataproviderapi.com', 17 | }, 18 | WS_API_ENDPOINT: { 19 | description: 20 | 'WS endpoint for Data Provider', 21 | type: 'string', 22 | default: 'ws://localhost:9090', 23 | }, 24 | <% if (setBgExecuteMsEnv) { %> 25 | BACKGROUND_EXECUTE_MS: { 26 | description: 27 | 'The amount of time the background execute should sleep before performing the next request', 28 | type: 'number', 29 | default: 10_000, 30 | },<% } %> 31 | }, 32 | ) 33 | -------------------------------------------------------------------------------- /src/util/settings.ts: -------------------------------------------------------------------------------- 1 | import { Adapter } from '../adapter' 2 | import { SettingDefinitionDetails } from '../config' 3 | import { censor } from './index' 4 | import CensorList from './censor/censor-list' 5 | 6 | export type DebugPageSetting = SettingDefinitionDetails & { name: string; value: unknown } 7 | 8 | /** 9 | * Builds a list of adapter settings with sensitive values censored 10 | * Used by both debug settings page and status endpoint 11 | */ 12 | export const buildSettingsList = (adapter: Adapter): DebugPageSetting[] => { 13 | // Censor EA settings 14 | const settings = adapter.config.settings 15 | const censoredValues = CensorList.getAll() 16 | const censoredSettings: Array = [] 17 | 18 | for (const [key, value] of Object.entries(settings)) { 19 | const definitionDetails = adapter.config.getSettingDebugDetails(key) 20 | censoredSettings.push({ 21 | name: key, 22 | ...definitionDetails, 23 | value: censor(value, censoredValues), 24 | }) 25 | } 26 | 27 | return censoredSettings.sort((a, b) => a.name.localeCompare(b.name)) 28 | } 29 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/endpoint/base.ts.ejs: -------------------------------------------------------------------------------- 1 | <% if (includeComments) { %>// Input parameters define the structure of the request expected by the endpoint. The second parameter defines example input data that will be used in EA readme<% } %> 2 | export const inputParameters = new InputParameters({ 3 | base: { 4 | aliases: ['from', 'coin', 'symbol', 'market'], 5 | required: true, 6 | type: 'string', 7 | description: 'The symbol of symbols of the currency to query', 8 | }, 9 | quote: { 10 | aliases: ['to', 'convert'], 11 | required: true, 12 | type: 'string', 13 | description: 'The symbol of the currency to convert to', 14 | }, 15 | }, [ 16 | { 17 | base: 'BTC', 18 | quote: 'USD' 19 | } 20 | ]) 21 | <% if (includeComments) { %>// Endpoints contain a type parameter that allows specifying relevant types of an endpoint, for example, request payload type, Adapter response type and Adapter configuration (environment variables) type<% } %> 22 | export type BaseEndpointTypes = { 23 | Parameters: typeof inputParameters.definition 24 | Response: SingleNumberResultResponse 25 | Settings: typeof config.settings 26 | } -------------------------------------------------------------------------------- /src/validation/preset-tokens.json: -------------------------------------------------------------------------------- 1 | { 2 | "ethereum": { 3 | "LINK": "0x514910771af9ca656af840dff83e8264ecf986ca", 4 | "WETH": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", 5 | "ETH": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", 6 | "stETH": "0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84", 7 | "DIGG": "0x798d1be841a82a273720ce31c822c61a67a601c3", 8 | "WBTC": "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", 9 | "RAI": "0x03ab458634910aad20ef5f1c8ee96f1d6ac54919", 10 | "RGT": "0xD291E7a03283640FDc51b121aC401383A46cC623", 11 | "RARI": "0xFca59Cd816aB1eaD66534D82bc21E7515cE441CF", 12 | "SFI": "0xb753428af26e81097e7fd17f40c88aaa3e04902c", 13 | "LDO": "0x5a98fcbea516cf06857215779fd812ca3bef1b32", 14 | "VSP": "0x1b40183EFB4Dd766f11bDa7A7c3AD8982e998421", 15 | "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", 16 | "USDT": "0xdAC17F958D2ee523a2206206994597C13D831ec7", 17 | "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F", 18 | "FRAX": "0x853d955aCEf822Db058eb8505911ED77F175b99e", 19 | "BOND": "0x0391d2021f89dc339f60fff84546ea23e337750f", 20 | "FEI": "0x956f47f50a910163d8bf957cf5846d573e7f87ca", 21 | "TRIBE": "0xc7283b66Eb1EB5FB86327f08e1B5816b0720212B" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /scripts/ea-settings-table.ts: -------------------------------------------------------------------------------- 1 | import { BaseSettingsDefinition, SettingDefinition } from '../src/config/index' 2 | 3 | const spacer = Array(30).fill(' ').join('') 4 | 5 | let output = `# EA Settings\n\n|Name|Type|Default|${spacer}Description${spacer}|${spacer}Validation${spacer}|Min|Max\n|---|---|---|---|---|---|---|\n` 6 | 7 | const sortedSettings: Array<[string, SettingDefinition]> = Object.entries( 8 | BaseSettingsDefinition, 9 | ).sort(([settingName1], [settingName2]) => settingName1.localeCompare(settingName2)) 10 | 11 | for (const [name, setting] of sortedSettings) { 12 | let validation = '' 13 | let min: number | string = '' 14 | let max: number | string = '' 15 | 16 | if (setting.validate) { 17 | if (setting.validate.meta.details) { 18 | validation = `- ${setting.validate.meta.details.split(', ').join('
- ')}` 19 | } 20 | 21 | if (typeof setting.validate.meta.min === 'number') { 22 | min = setting.validate.meta.min 23 | } 24 | if (typeof setting.validate.meta.max === 'number') { 25 | max = setting.validate.meta.max 26 | } 27 | } 28 | 29 | output += `|${name}|${setting.type}|${setting.default}|${setting.description}|${validation}|${min}|${max}\n` 30 | } 31 | 32 | // eslint-disable-next-line no-console 33 | console.log(output) 34 | -------------------------------------------------------------------------------- /test/custom-logger.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import { start } from '../src' 3 | import { Adapter, AdapterEndpoint } from '../src/adapter' 4 | import { LoggerFactory } from '../src/util/logger' 5 | import { NopTransport } from '../src/util/testing-utils' 6 | 7 | test('custom logger instance is properly injected', async (t) => { 8 | let timesCalled = 0 9 | 10 | const loggerFactory: LoggerFactory = { 11 | child: () => { 12 | return { 13 | fatal: () => { 14 | timesCalled++ 15 | }, 16 | error: () => { 17 | timesCalled++ 18 | }, 19 | warn: () => { 20 | timesCalled++ 21 | }, 22 | info: () => { 23 | timesCalled++ 24 | }, 25 | debug: () => { 26 | timesCalled++ 27 | }, 28 | trace: () => { 29 | timesCalled++ 30 | }, 31 | } 32 | }, 33 | } as unknown as LoggerFactory 34 | 35 | const adapter = new Adapter({ 36 | name: 'TEST', 37 | endpoints: [ 38 | new AdapterEndpoint({ 39 | name: 'test', 40 | transport: new NopTransport(), 41 | }), 42 | ], 43 | }) 44 | 45 | t.is(timesCalled, 0) 46 | 47 | await start(adapter, { 48 | loggerFactory, 49 | }) 50 | 51 | t.true(timesCalled > 0) 52 | }) 53 | -------------------------------------------------------------------------------- /docs/components/transports.md: -------------------------------------------------------------------------------- 1 | # Transports 2 | 3 | To learn more about what transports are and what they do, please refer to the [EA Basics Doc](../basics.md) 4 | 5 | Define transport file in a seperate folder called `transport`. The name of the file is the same as its associated endpoint. In case endpoint supports multiple transports, transport file names can be suffixed with transport types. 6 | 7 | ## Choosing Transports 8 | 9 | ### Basic Transports 10 | 11 | The v3 framework provides transports to fetch data from a Provider using the common protocols they might use. Please refer to the guides listed below for the relevant transports your adapter endpoints need. 12 | 13 | - [HTTP Transport](./transport-types/http-transport.md) 14 | - [Websocket Transport](./transport-types/websocket-transport.md) 15 | - [SSE Transport](./transport-types/sse-transport.md) 16 | - [Custom Transport](./transport-types/custom-transport.md) 17 | 18 | ### Abstract Transports 19 | 20 | If you find that the built-in features of a transport do not meet your endpoint's requirements, you can define a custom transport extending one of the abstract transports or existing basic ones to include the custom functionality yourself. 21 | 22 | - [Subscription Transport](./transport-types/subscription-transport.md) 23 | - [Streaming Transport](./transport-types/streaming-transport.md) 24 | -------------------------------------------------------------------------------- /src/debug/router.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from 'fastify' 2 | import { join } from 'path' 3 | import { Adapter } from '../adapter' 4 | import settingsPage from './settings-page' 5 | import { buildSettingsList } from '../util/settings' 6 | 7 | /** 8 | * This function registers the debug endpoints for the adapter. 9 | * These endpoints are intended to be used for debugging purposes only, and should not be publicly accessible. 10 | * 11 | * @param app - the fastify instance that has been created 12 | * @param adapter - the adapter for which to create the debug endpoints 13 | */ 14 | export default function registerDebugEndpoints(app: FastifyInstance, adapter: Adapter) { 15 | // Debug endpoint to return the current settings in raw JSON (censoring sensitive values) 16 | app.get(join(adapter.config.settings.BASE_URL, '/debug/settings/raw'), async () => { 17 | const censoredSettings = buildSettingsList(adapter) 18 | return JSON.stringify( 19 | censoredSettings.sort((a, b) => a.name.localeCompare(b.name)), 20 | null, 21 | 2, 22 | ) 23 | }) 24 | 25 | // Helpful UI to visualize current settings 26 | app.get(join(adapter.config.settings.BASE_URL, '/debug/settings'), (req, reply) => { 27 | const censoredSettings = buildSettingsList(adapter) 28 | const censoredSettingsPage = settingsPage(censoredSettings) 29 | reply.headers({ 'content-type': 'text/html' }).send(censoredSettingsPage) 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /test/metrics/feed-id.test.ts: -------------------------------------------------------------------------------- 1 | import untypedTest, { TestFn } from 'ava' 2 | import { calculateFeedId } from '../../src/cache' 3 | import { AdapterSettings, buildAdapterSettings } from '../../src/config' 4 | import { LoggerFactoryProvider } from '../../src/util' 5 | import { InputParameters } from '../../src/validation' 6 | import { InputParametersDefinition } from '../../src/validation/input-params' 7 | 8 | const test = untypedTest as TestFn<{ 9 | inputParameters: InputParameters 10 | endpointName: string 11 | adapterSettings: AdapterSettings 12 | }> 13 | 14 | test.before(() => { 15 | LoggerFactoryProvider.set() 16 | }) 17 | 18 | test.beforeEach(async (t) => { 19 | t.context.endpointName = 'TEST' 20 | t.context.inputParameters = new InputParameters({}) 21 | t.context.adapterSettings = buildAdapterSettings({}) 22 | }) 23 | 24 | test.serial('no parameters returns N/A', async (t) => { 25 | t.is(calculateFeedId(t.context, {}), 'N/A') 26 | }) 27 | 28 | test.serial('builds feed ID correctly from input params', async (t) => { 29 | t.context.inputParameters = new InputParameters({ 30 | base: { 31 | type: 'string', 32 | description: 'base', 33 | required: true, 34 | }, 35 | quote: { 36 | type: 'string', 37 | description: 'quote', 38 | required: true, 39 | }, 40 | }) 41 | const data = { base: 'ETH', quote: 'BTC' } 42 | t.is(calculateFeedId(t.context, data), '{"base":"eth","quote":"btc"}') 43 | }) 44 | -------------------------------------------------------------------------------- /test/por.test.ts: -------------------------------------------------------------------------------- 1 | import { NopTransport, TestAdapter } from '../src/util/testing-utils' 2 | import untypedTest, { TestFn } from 'ava' 3 | import { PoRAdapter, PoRBalanceEndpoint } from '../src/adapter/por' 4 | import { AdapterConfig } from '../src/config' 5 | 6 | type TestContext = { 7 | testAdapter: TestAdapter 8 | } 9 | const test = untypedTest as TestFn 10 | 11 | test('PoRAdapter has BACKGROUND_EXECUTE_TIMEOUT setting set to highest value as default', async (t) => { 12 | const adapter = new PoRAdapter({ 13 | name: 'TEST', 14 | config: new AdapterConfig({ 15 | test: { description: 'test', type: 'string' }, 16 | }), 17 | endpoints: [ 18 | new PoRBalanceEndpoint({ 19 | name: 'test', 20 | transport: new NopTransport(), 21 | }), 22 | ], 23 | }) 24 | t.is(adapter.config.settings.BACKGROUND_EXECUTE_TIMEOUT, 180_000) 25 | }) 26 | 27 | test('PoRAdapter uses BACKGROUND_EXECUTE_TIMEOUT value if provided', async (t) => { 28 | const adapter = new PoRAdapter({ 29 | name: 'TEST', 30 | config: new AdapterConfig( 31 | { 32 | test: { description: 'test', type: 'string' }, 33 | }, 34 | { envDefaultOverrides: { BACKGROUND_EXECUTE_TIMEOUT: 100 } }, 35 | ), 36 | endpoints: [ 37 | new PoRBalanceEndpoint({ 38 | name: 'test', 39 | transport: new NopTransport(), 40 | }), 41 | ], 42 | }) 43 | t.is(adapter.config.settings.BACKGROUND_EXECUTE_TIMEOUT, 100) 44 | }) 45 | -------------------------------------------------------------------------------- /.github/workflows/pinned-dependencies.yaml: -------------------------------------------------------------------------------- 1 | name: 'Pinned dependencies' 2 | on: 3 | push: 4 | pull_request: 5 | 6 | jobs: 7 | check-dependencies: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v6 11 | - name: Extract all versions from package.json 12 | uses: sergeysova/jq-action@a3f0d4ff59cc1dddf023fc0b325dd75b10deec58 # v2.3.0 13 | id: versions 14 | with: 15 | cmd: 'jq ''[getpath(["dependencies"],["devDependencies"],["optionalDependencies"])] | del(..|nulls) | map(.[]) | join(",")'' package.json -r' 16 | multiline: true 17 | - name: Check for un-pinned versions 18 | run: | 19 | versions="${{ steps.versions.outputs.value }}" 20 | IFS="," read -a versionsList <<< $versions 21 | # Simple regex that does what we need it for 22 | regex="^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-\w+)?(\+\w+)?(\.(0|[1-9][0-9]*))?$" 23 | exitCode=0 24 | for version in ${versionsList[@]}; do 25 | if echo "$version" | grep -Ev "$regex" > /dev/null; then 26 | lineNumber=$(grep -F -n -m 1 "$version" "package.json" | sed 's/\([0-9]*\).*/\1/') 27 | MESSAGE="Dependency version is not pinned: $version" 28 | echo "::error file=package.json,line=$lineNumber,endLine=$lineNumber,title=Dependency::$MESSAGE" 29 | exitCode=1 30 | fi 31 | done 32 | exit "$exitCode" 33 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/test/fixtures.ts.ejs: -------------------------------------------------------------------------------- 1 | <% if (includeHttpFixtures) { %>import nock from 'nock'<% } %> 2 | <% if (includeWsFixtures) { %>import { MockWebsocketServer } from '@chainlink/external-adapter-framework/util/testing-utils'<% } %> 3 | <% if (includeHttpFixtures) { %> 4 | export const mockResponseSuccess = (): nock.Scope => 5 | nock('https://dataproviderapi.com', { 6 | encodedQueryParams: true, 7 | }) 8 | .get('/cryptocurrency/price') 9 | .query({ 10 | symbol: 'ETH', 11 | convert: 'USD', 12 | }) 13 | .reply(200, () => ({ ETH: { price: 10000 } }), [ 14 | 'Content-Type', 15 | 'application/json', 16 | 'Connection', 17 | 'close', 18 | 'Vary', 19 | 'Accept-Encoding', 20 | 'Vary', 21 | 'Origin', 22 | ]) 23 | .persist() 24 | <% } %> 25 | <% if (includeWsFixtures) { %> 26 | export const mockWebsocketServer = (URL: string): MockWebsocketServer => { 27 | const mockWsServer = new MockWebsocketServer(URL, { mock: false }) 28 | mockWsServer.on('connection', (socket) => { 29 | socket.on('message', (message) => { 30 | return socket.send( 31 | JSON.stringify({ 32 | success: true, 33 | price: 1000, 34 | base: 'ETH', 35 | quote: 'USD', 36 | time: '1999999' 37 | }), 38 | ) 39 | }) 40 | }) 41 | 42 | return mockWsServer 43 | } 44 | <% } %> -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/endpoint/endpoint.ts.ejs: -------------------------------------------------------------------------------- 1 | import { AdapterEndpoint } from '@chainlink/external-adapter-framework/adapter' 2 | import { InputParameters } from '@chainlink/external-adapter-framework/validation' 3 | import { SingleNumberResultResponse } from '@chainlink/external-adapter-framework/util' 4 | import { config } from '../config' 5 | import overrides from '../config/overrides.json' 6 | import { <%= inputTransports[0].name %> } from '../transport/<%= inputEndpointName %>' 7 | 8 | <%- include ('base.ts.ejs') %> 9 | 10 | export const endpoint = new AdapterEndpoint({ 11 | <% if (includeComments) { -%> 12 | // Endpoint name 13 | <% } -%><%= ' ' %> name: '<%= inputEndpointName %>', 14 | <% if (includeComments) { -%> 15 | // Alternative endpoint names for this endpoint 16 | <% } -%><%= ' ' %> aliases: <%- endpointAliases.length ? JSON.stringify(endpointAliases) : JSON.stringify([]) -%>, 17 | <% if (includeComments) { -%> 18 | // Transport handles incoming requests, data processing and communication for this endpoint 19 | <% } -%><%= ' ' %> transport: <%= inputTransports[0].name %>, 20 | <% if (includeComments) { -%> 21 | // Supported input parameters for this endpoint 22 | <% } -%><%= ' ' %> inputParameters, 23 | <% if (includeComments) { -%> 24 | // Overrides are defined in the `/config/overrides.json` file. They allow input parameters to be overriden from a generic symbol to something more specific for the data provider such as an ID. 25 | <% } -%><%= ' ' %> overrides: overrides['<%= adapterName %>'] 26 | }) 27 | -------------------------------------------------------------------------------- /src/util/subscription-set/redis-sorted-set.ts: -------------------------------------------------------------------------------- 1 | import Redis from 'ioredis' 2 | import { CMD_SENT_STATUS, recordRedisCommandMetric } from '../../metrics' 3 | import { SubscriptionSet } from './subscription-set' 4 | 5 | export class RedisSubscriptionSet implements SubscriptionSet { 6 | private redisClient: Redis 7 | // Key for Redis sorted set containing all subscriptions 8 | private subscriptionSetKey: string 9 | 10 | constructor(redisClient: Redis, subscriptionSetKey: string) { 11 | this.redisClient = redisClient 12 | this.subscriptionSetKey = subscriptionSetKey 13 | } 14 | 15 | async add(value: T, ttl: number): Promise { 16 | const storedValue = JSON.stringify(value) 17 | await this.redisClient.zadd(this.subscriptionSetKey, Date.now() + ttl, storedValue) 18 | recordRedisCommandMetric(CMD_SENT_STATUS.SUCCESS, 'zadd') 19 | return 20 | } 21 | 22 | async getAll(): Promise { 23 | // Remove expired keys from sorted set 24 | await this.redisClient.zremrangebyscore(this.subscriptionSetKey, '-inf', Date.now()) 25 | recordRedisCommandMetric(CMD_SENT_STATUS.SUCCESS, 'zremrangebyscore') 26 | const parsedRequests: T[] = [] 27 | const validEntries = await this.redisClient.zrange(this.subscriptionSetKey, 0, -1) 28 | recordRedisCommandMetric(CMD_SENT_STATUS.SUCCESS, 'zrange') 29 | validEntries.forEach((entry) => { 30 | // Separate request and cache key prior to populating results array 31 | parsedRequests.push(JSON.parse(entry)) 32 | }) 33 | return parsedRequests 34 | } 35 | 36 | get(): T | undefined { 37 | return undefined 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/cache/metrics.ts: -------------------------------------------------------------------------------- 1 | import { metrics } from '../metrics' 2 | 3 | export interface CacheMetricsLabels { 4 | participant_id: string 5 | feed_id: string 6 | cache_type: string 7 | } 8 | 9 | export const cacheGet = ( 10 | label: CacheMetricsLabels, 11 | value: unknown, 12 | staleness: { 13 | cache: number 14 | total: number | null 15 | }, 16 | ) => { 17 | if (typeof value === 'number' || typeof value === 'string') { 18 | const parsedValue = Number(value) 19 | if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) { 20 | metrics.get('cacheDataGetValues').labels(label).set(parsedValue) 21 | } 22 | } 23 | metrics.get('cacheDataGetCount').labels(label).inc() 24 | metrics.get('cacheDataStalenessSeconds').labels(label).set(staleness.cache) 25 | if (staleness.total) { 26 | metrics.get('totalDataStalenessSeconds').labels(label).set(staleness.total) 27 | } 28 | } 29 | 30 | export const cacheSet = ( 31 | label: CacheMetricsLabels, 32 | maxAge: number, 33 | timeDelta: number | undefined, 34 | ) => { 35 | metrics.get('cacheDataSetCount').labels(label).inc() 36 | metrics.get('cacheDataMaxAge').labels(label).set(maxAge) 37 | metrics.get('cacheDataStalenessSeconds').labels(label).set(0) 38 | if (timeDelta) { 39 | metrics.get('providerTimeDelta').labels({ feed_id: label.feed_id }).set(timeDelta) 40 | } 41 | } 42 | 43 | export const cacheMetricsLabel = (cacheKey: string, feedId: string, cacheType: string) => ({ 44 | participant_id: cacheKey, 45 | feed_id: feedId, 46 | cache_type: cacheType, 47 | }) 48 | 49 | export enum CacheTypes { 50 | Redis = 'redis', 51 | Local = 'local', 52 | } 53 | -------------------------------------------------------------------------------- /src/adapter/stock.ts: -------------------------------------------------------------------------------- 1 | import { InputParametersDefinition } from '../validation/input-params' 2 | import { TransportGenerics } from '../transports' 3 | import { SingleNumberResultResponse } from '../util' 4 | import { AdapterEndpoint } from './endpoint' 5 | 6 | /** 7 | * Type for the base input parameter config that any [[StockEndpoint]] must extend 8 | */ 9 | export type StockEndpointInputParametersDefinition = InputParametersDefinition & { 10 | base: { 11 | aliases: readonly ['from', 'symbol', 'asset', 'coin', 'ticker', ...string[]] 12 | type: 'string' 13 | description: 'The stock ticker to query' 14 | required: boolean 15 | } 16 | } 17 | 18 | /** 19 | * Base input parameter config that any [[StockEndpoint]] must extend 20 | */ 21 | export const stockEndpointInputParametersDefinition = { 22 | base: { 23 | aliases: ['from', 'symbol', 'asset', 'coin', 'ticker'], 24 | type: 'string', 25 | description: 'The stock ticker to query', 26 | required: true, 27 | }, 28 | } as const satisfies StockEndpointInputParametersDefinition 29 | 30 | /** 31 | * Helper type structure that contains the different types passed to the generic parameters of a StockEndpoint 32 | */ 33 | export type StockEndpointGenerics = TransportGenerics & { 34 | Parameters: StockEndpointInputParametersDefinition 35 | Response: SingleNumberResultResponse 36 | } 37 | 38 | /** 39 | * A StockEndpoint is a specific type of AdapterEndpoint. Meant to comply with standard practices for 40 | * Data Feeds, its InputParameters must extend the basic ones (base). 41 | */ 42 | export class StockEndpoint extends AdapterEndpoint {} 43 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/index.ts.ejs: -------------------------------------------------------------------------------- 1 | import { expose, ServerInstance } from '@chainlink/external-adapter-framework' 2 | import { Adapter } from '@chainlink/external-adapter-framework/adapter' 3 | import { config } from './config' 4 | import { <%= endpointNames %> } from './endpoint' 5 | 6 | export const adapter = new Adapter({ 7 | <% if (includeComments) { -%> 8 | //Requests will direct to this endpoint if the `endpoint` input parameter is not specified. 9 | <% } -%><%= ' ' %> defaultEndpoint: <%= defaultEndpoint.normalizedEndpointName %>.name, 10 | <% if (includeComments) { -%> 11 | // Adapter name 12 | <% } -%><%= ' ' %> name: '<%= adapterName.toUpperCase().replace('-', '_') %>', 13 | <% if (includeComments) { -%> 14 | // Adapter configuration (environment variables) 15 | <% } -%><%= ' ' %> config, 16 | <% if (includeComments) { -%> 17 | // List of supported endpoints 18 | <% } -%><%= ' ' %> endpoints: [<%= endpointNames %>], 19 | <% if (Object.values(endpoints).some(endpoint => endpoint.inputTransports.some(({ type }) => type === 'http'))) { -%> 20 | <% if (includeComments) { -%> 21 | // Rate limit otherwise we send requests as fast as possible without delay. 22 | // You should adjust this based on what the data provider can tolerate and 23 | // how fresh you need your data to be. The rate limit is global for all 24 | // requests going through transport.dependencies.requester, which includes 25 | // the requests prepared in HttpTransport.prepareRequests. 26 | <% } -%><%= ' ' %> rateLimiting: { 27 | tiers: { 28 | default: { 29 | rateLimit1m: 6, 30 | }, 31 | }, 32 | }, 33 | <% } -%> 34 | }) 35 | 36 | export const server = (): Promise => expose(adapter) 37 | -------------------------------------------------------------------------------- /docs/components/transport-types/streaming-transport.md: -------------------------------------------------------------------------------- 1 | # Streaming Transport 2 | 3 | The `StreamingTransport` is an **abstract transport** (class) that extends the [SubscriptionTransport](./subscription-transport.md) and provides a foundation for implementing streaming-based transports. It handles incoming requests, manages subscriptions, and defines an abstract `streamHandler` method to process subscription deltas. This class is intended to be extended by specific transport implementations. 4 | 5 | All incoming requests to the adapter for an endpoint that uses stream-based transport are stored in a cached set (`SubscriptionSet`). Periodically, the background execute loop of the adapter will read the entire subscription set and call the `backgroundHandler` method of the transport.`backgroundHandler` method is already implemented in `StreamingTransport`. It calculates subscription deltas (new subscriptions and subscriptions to unsubscribe) based on the all subscriptions and the current local subscriptions. The deltas are then passed to the `streamHandler` method for further processing. 6 | 7 | When extending `StreamingTransport` there are two abstract methods that should be implemented by subclasses. 8 | 9 | 1. `streamHandler` receives endpoint context as first argument and object containing details for the desired, new, and stale subscriptions as second argument and is responsible for handling the streaming connection, sending messages to the streaming source, and processing subscription deltas. 10 | 2. `getSubscriptionTtlFromConfig` is an abstract method from `SubscriptionTransport`. It receives adapter settings and should return time-to-live (TTL) value for subscription set. 11 | 12 | An example of `StreamingTransport` is built-in [Websocket Transport](./websocket-transport.md) and [SSE Transport](./sse-transport.md) 13 | -------------------------------------------------------------------------------- /test/metrics/por-metrics.test.ts: -------------------------------------------------------------------------------- 1 | import { InstalledClock } from '@sinonjs/fake-timers' 2 | import { installTimers } from '../helper' 3 | import untypedTest, { TestFn } from 'ava' 4 | import { NopTransport, TestAdapter } from '../../src/util/testing-utils' 5 | import { InputParameters } from '../../src/validation' 6 | import { 7 | PoRAdapter, 8 | PoRBalanceEndpoint, 9 | porBalanceEndpointInputParametersDefinition, 10 | } from '../../src/adapter/por' 11 | 12 | const test = untypedTest as TestFn<{ 13 | testAdapter: TestAdapter 14 | clock: InstalledClock 15 | }> 16 | 17 | const inputParameters = new InputParameters(porBalanceEndpointInputParametersDefinition) 18 | 19 | test.before(async (t) => { 20 | process.env['METRICS_ENABLED'] = 'true' 21 | 22 | const adapter = new PoRAdapter({ 23 | name: 'TEST', 24 | defaultEndpoint: 'test', 25 | endpoints: [ 26 | new PoRBalanceEndpoint({ 27 | name: 'test', 28 | inputParameters, 29 | transport: new NopTransport(), 30 | }), 31 | ], 32 | }) 33 | 34 | // Start the adapter 35 | t.context.clock = installTimers() 36 | t.context.testAdapter = await TestAdapter.startWithMockedCache(adapter, t.context) 37 | }) 38 | 39 | test.after((t) => { 40 | t.context.clock.uninstall() 41 | }) 42 | 43 | test.serial('PoR adapter addresses metric', async (t) => { 44 | const error = await t.context.testAdapter.request({ 45 | addresses: [ 46 | { 47 | address: 'address1', 48 | }, 49 | { 50 | address: 'address2', 51 | }, 52 | ], 53 | }) 54 | t.is(error.statusCode, 504) 55 | 56 | const metrics = await t.context.testAdapter.getMetrics() 57 | metrics.assert(t, { 58 | name: 'por_balance_address_length', 59 | labels: { feed_id: "{'addresses':[{'address':'address1'},{'address':'address2'}]}" }, 60 | expectedValue: 2, 61 | }) 62 | }) 63 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/test/adapter.test.ts.ejs: -------------------------------------------------------------------------------- 1 | import { 2 | TestAdapter, 3 | setEnvVariables, 4 | } from '@chainlink/external-adapter-framework/util/testing-utils' 5 | import * as nock from 'nock' 6 | import { mockResponseSuccess } from './fixtures' 7 | 8 | describe('execute', () => { 9 | let spy: jest.SpyInstance 10 | let testAdapter: TestAdapter 11 | let oldEnv: NodeJS.ProcessEnv 12 | 13 | beforeAll(async () => { 14 | oldEnv = JSON.parse(JSON.stringify(process.env)) 15 | process.env.API_KEY = process.env.API_KEY ?? 'fake-api-key' 16 | <% if (setBgExecuteMsEnv) { %>process.env.BACKGROUND_EXECUTE_MS = process.env.BACKGROUND_EXECUTE_MS ?? '0'<% } %> 17 | const mockDate = new Date('2001-01-01T11:11:11.111Z') 18 | spy = jest.spyOn(Date, 'now').mockReturnValue(mockDate.getTime()) 19 | 20 | const adapter = (await import('./../../src')).adapter 21 | adapter.rateLimiting = undefined 22 | testAdapter = await TestAdapter.startWithMockedCache(adapter, { 23 | testAdapter: {} as TestAdapter, 24 | }) 25 | }) 26 | 27 | afterAll(async () => { 28 | setEnvVariables(oldEnv) 29 | await testAdapter.api.close() 30 | nock.restore() 31 | nock.cleanAll() 32 | spy.mockRestore() 33 | }) 34 | 35 | <% for(var i=0; i 36 | describe('<%= endpoints[i].inputEndpointName %> endpoint', () => { 37 | it('should return success', async () => { 38 | const data = { 39 | base: 'ETH', 40 | quote: 'USD', 41 | endpoint: '<%= endpoints[i].inputEndpointName %>', 42 | transport: '<%= transportName %>' 43 | } 44 | mockResponseSuccess() 45 | const response = await testAdapter.request(data) 46 | expect(response.statusCode).toBe(200) 47 | expect(response.json()).toMatchSnapshot() 48 | }) 49 | }) 50 | <% } %> 51 | }) 52 | -------------------------------------------------------------------------------- /test/metrics/polling-metrics.test.ts: -------------------------------------------------------------------------------- 1 | import { InstalledClock } from '@sinonjs/fake-timers' 2 | import { installTimers } from '../helper' 3 | import untypedTest, { TestFn } from 'ava' 4 | import { TestAdapter } from '../../src/util/testing-utils' 5 | import { buildHttpAdapter } from './helper' 6 | import MockAdapter from 'axios-mock-adapter' 7 | import axios from 'axios' 8 | 9 | const test = untypedTest as TestFn<{ 10 | testAdapter: TestAdapter 11 | clock: InstalledClock 12 | }> 13 | 14 | const URL = 'http://test-url.com' 15 | const endpoint = '/price' 16 | const axiosMock = new MockAdapter(axios) 17 | 18 | test.before(async (t) => { 19 | process.env['METRICS_ENABLED'] = 'true' 20 | // Set higher retries for polling metrics testing 21 | process.env['CACHE_POLLING_MAX_RETRIES'] = '5' 22 | 23 | const adapter = buildHttpAdapter() 24 | 25 | // Start the adapter 26 | t.context.clock = installTimers() 27 | t.context.testAdapter = await TestAdapter.startWithMockedCache(adapter, t.context) 28 | }) 29 | 30 | test.after((t) => { 31 | axiosMock.reset() 32 | t.context.clock.uninstall() 33 | }) 34 | 35 | const from = 'ETH' 36 | const to = 'USD' 37 | const price = 1234 38 | 39 | axiosMock 40 | .onPost(URL + endpoint, { 41 | pairs: [ 42 | { 43 | base: from, 44 | quote: to, 45 | }, 46 | ], 47 | }) 48 | .reply(200, { 49 | prices: [ 50 | { 51 | pair: `${from}/${to}`, 52 | price, 53 | }, 54 | ], 55 | }) 56 | 57 | test.serial('Test cache warmer active metric', async (t) => { 58 | const error = await t.context.testAdapter.request({ from, to }) 59 | t.is(error.statusCode, 504) 60 | 61 | const metrics = await t.context.testAdapter.getMetrics() 62 | metrics.assert(t, { 63 | name: 'transport_polling_failure_count', 64 | labels: { adapter_endpoint: 'test' }, 65 | expectedValue: 1, 66 | }) 67 | metrics.assertPositiveNumber(t, { 68 | name: 'transport_polling_duration_seconds', 69 | labels: { 70 | adapter_endpoint: 'test', 71 | succeeded: 'false', 72 | }, 73 | }) 74 | }) 75 | -------------------------------------------------------------------------------- /src/util/subscription-set/subscription-set.ts: -------------------------------------------------------------------------------- 1 | import Redis from 'ioredis' 2 | import { PromiseOrValue } from '..' 3 | import { AdapterSettings } from '../../config' 4 | import { ExpiringSortedSet } from './expiring-sorted-set' 5 | import { RedisSubscriptionSet } from './redis-sorted-set' 6 | 7 | /** 8 | * Set to hold items to subscribe to from a provider (regardless of protocol) 9 | */ 10 | export interface SubscriptionSet { 11 | /** Add a new subscription to the set */ 12 | add(value: T, ttl: number, key?: string): PromiseOrValue 13 | 14 | /** Get all subscriptions from the set as a list */ 15 | getAll(): PromiseOrValue 16 | 17 | get: (key: string) => T | undefined 18 | } 19 | 20 | export class SubscriptionSetFactory { 21 | private cacheType: AdapterSettings['CACHE_TYPE'] 22 | private redisClient?: Redis 23 | private adapterName?: string 24 | private capacity: AdapterSettings['SUBSCRIPTION_SET_MAX_ITEMS'] 25 | private cachePrefix: AdapterSettings['CACHE_PREFIX'] 26 | 27 | constructor(adapterSettings: AdapterSettings, adapterName: string, redisClient?: Redis) { 28 | this.cacheType = adapterSettings.CACHE_TYPE 29 | this.redisClient = redisClient 30 | this.adapterName = adapterName 31 | this.capacity = adapterSettings.SUBSCRIPTION_SET_MAX_ITEMS 32 | this.cachePrefix = adapterSettings.CACHE_PREFIX 33 | } 34 | 35 | buildSet(endpointName: string, transportName: string): SubscriptionSet { 36 | switch (this.cacheType) { 37 | case 'local': 38 | return new ExpiringSortedSet(this.capacity) 39 | case 'redis': { 40 | if (!this.redisClient) { 41 | throw new Error('Redis client undefined. Cannot create Redis subscription set') 42 | } 43 | // Identifier key used for the subscription set in redis 44 | const cachePrefix = this.cachePrefix ? `${this.cachePrefix}-` : '' 45 | const subscriptionSetKey = `${cachePrefix}${this.adapterName}-${endpointName}-${transportName}-subscriptionSet` 46 | return new RedisSubscriptionSet(this.redisClient, subscriptionSetKey) 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/endpoint/endpoint-router.ts.ejs: -------------------------------------------------------------------------------- 1 | import { AdapterEndpoint } from '@chainlink/external-adapter-framework/adapter' 2 | import { InputParameters } from '@chainlink/external-adapter-framework/validation' 3 | import { SingleNumberResultResponse } from '@chainlink/external-adapter-framework/util' 4 | import { TransportRoutes } from '@chainlink/external-adapter-framework/transports' 5 | import { config } from '../config' 6 | import overrides from '../config/overrides.json' 7 | <% for(let i=0; i 8 | import { <%= inputTransports[i].name %> } from '../transport/<%= inputEndpointName %>-<%= inputTransports[i].type %>' <% } 9 | %> 10 | 11 | <%- include ('base.ts.ejs') %> 12 | 13 | export const endpoint = new AdapterEndpoint({ 14 | <% if (includeComments) { -%> 15 | // Endpoint name 16 | <% } -%><%= ' ' %> name: '<%= inputEndpointName %>', 17 | <% if (includeComments) { -%> 18 | // Alternative endpoint names for this endpoint 19 | <% } -%><%= ' ' %> aliases: <%- endpointAliases.length ? JSON.stringify(endpointAliases) : JSON.stringify([]) -%>, 20 | <% if (includeComments) { -%> 21 | // Transport handles incoming requests, data processing and communication for this endpoint. 22 | // In case endpoint supports multiple transports (i.e. http and websocket) TransportRoutes is used to register all supported transports. 23 | // To use specific transport, provide `transport: [transportName]` in the request 24 | <% } -%><%= ' ' %> transportRoutes: new TransportRoutes() 25 | <% for(let i=0; i 26 | .register('<%- inputTransports[i].type === "http" ? `rest` : inputTransports[i].type %>', <%- inputTransports[i].name %>)<%}%>, 27 | <% if (includeComments) { -%> 28 | // Supported input parameters for this endpoint 29 | <% } -%><%= ' ' %> inputParameters, 30 | <% if (includeComments) { -%> 31 | // Overrides are defined in the `/config/overrides.json` file. They allow input parameters to be overriden from a generic symbol to something more specific for the data provider such as an ID. 32 | <% } -%><%= ' ' %> overrides: overrides['<%= adapterName %>'] 33 | }) 34 | -------------------------------------------------------------------------------- /src/validation/market-status.ts: -------------------------------------------------------------------------------- 1 | import { AdapterInputError } from '../validation/error' 2 | import { TZDate } from '@date-fns/tz' 3 | 4 | export const parseWeekendString = (weekend?: string) => { 5 | const dayHour = /[0-6](0\d|1\d|2[0-3])/ 6 | const timezonePattern = /[^\s]+/ 7 | const regex = new RegExp(`^(${dayHour.source})-(${dayHour.source}):(${timezonePattern.source})$`) 8 | 9 | const match = weekend?.match(regex) 10 | if (!match) { 11 | throw new AdapterInputError({ 12 | statusCode: 400, 13 | message: '[Param: weekend] does not match format of DHH-DHH:TZ', 14 | }) 15 | } 16 | 17 | const result = { 18 | start: match[1], 19 | end: match[3], 20 | tz: match[5], 21 | } 22 | 23 | try { 24 | // eslint-disable-next-line new-cap 25 | Intl.DateTimeFormat(undefined, { timeZone: result.tz }) 26 | } catch (error) { 27 | throw new AdapterInputError({ 28 | statusCode: 400, 29 | message: `timezone ${result.tz} in [Param: weekend] is not valid: ${error}`, 30 | }) 31 | } 32 | 33 | return result 34 | } 35 | 36 | export const isWeekendNow = (weekend?: string) => { 37 | const parsed = parseWeekendString(weekend) 38 | 39 | const startDay = Number(parsed.start[0]) 40 | const startHour = Number(parsed.start.slice(1)) 41 | const endDay = Number(parsed.end[0]) 42 | const endHour = Number(parsed.end.slice(1)) 43 | 44 | const nowDay = TZDate.tz(parsed.tz).getDay() 45 | const nowHour = TZDate.tz(parsed.tz).getHours() 46 | 47 | // Case 1: weekend does NOT wrap around the week 48 | if (startDay < endDay || (startDay === endDay && startHour < endHour)) { 49 | if (nowDay < startDay || nowDay > endDay) { 50 | return false 51 | } else if (nowDay === startDay && nowHour < startHour) { 52 | return false 53 | } else if (nowDay === endDay && nowHour >= endHour) { 54 | return false 55 | } 56 | return true 57 | } 58 | 59 | // Case 2: weekend wraps around (e.g. Fri → Sun) 60 | if (nowDay > startDay || nowDay < endDay) { 61 | return true 62 | } else if (nowDay === startDay && nowHour >= startHour) { 63 | return true 64 | } else if (nowDay === endDay && nowHour < endHour) { 65 | return true 66 | } 67 | return false 68 | } 69 | -------------------------------------------------------------------------------- /test/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | groupArrayByKey, 3 | splitArrayIntoChunks, 4 | getCanonicalAdapterName, 5 | canonicalizeAdapterNameKeys, 6 | } from '../src/util' 7 | import test from 'ava' 8 | 9 | test('Test splitArrayIntoChunks function', async (t) => { 10 | const array = [1, 2, 3, 4, 5] 11 | let chunks = splitArrayIntoChunks(array, 2) 12 | t.deepEqual(chunks, [[1, 2], [3, 4], [5]]) 13 | 14 | // Size greater than array length returns whole array 15 | chunks = splitArrayIntoChunks(array, 10) 16 | t.deepEqual(chunks, [[1, 2, 3, 4, 5]]) 17 | 18 | chunks = splitArrayIntoChunks([], 10) 19 | t.deepEqual(chunks, [[]]) 20 | }) 21 | 22 | test('Test groupArrayByKey function', async (t) => { 23 | const array = [ 24 | { base: 'BTC', quote: 'ETH' }, 25 | { base: 'BTC', quote: 'USD' }, 26 | { base: 'LTC', quote: 'DASH' }, 27 | ] 28 | let grouped = groupArrayByKey(array, 'base') 29 | t.deepEqual(grouped, { 30 | BTC: [ 31 | { base: 'BTC', quote: 'ETH' }, 32 | { base: 'BTC', quote: 'USD' }, 33 | ], 34 | LTC: [{ base: 'LTC', quote: 'DASH' }], 35 | }) 36 | 37 | grouped = groupArrayByKey(array, 'quote') 38 | 39 | t.deepEqual(grouped, { 40 | ETH: [{ base: 'BTC', quote: 'ETH' }], 41 | USD: [{ base: 'BTC', quote: 'USD' }], 42 | DASH: [{ base: 'LTC', quote: 'DASH' }], 43 | }) 44 | }) 45 | 46 | test('Test getCanonicalAdapterName', async (t) => { 47 | t.is(getCanonicalAdapterName(undefined), undefined) 48 | t.is(getCanonicalAdapterName('test'), 'test') 49 | t.is(getCanonicalAdapterName('TEST'), 'test') 50 | t.is(getCanonicalAdapterName('TEST-adapter'), 'test_adapter') 51 | t.is(getCanonicalAdapterName('TEST_ADAPTER'), 'test_adapter') 52 | t.is(getCanonicalAdapterName('A-B-C-D-e-f'), 'a_b_c_d_e_f') 53 | }) 54 | 55 | test('Test canonicalizeAdapterNameKeys', async (t) => { 56 | t.deepEqual(canonicalizeAdapterNameKeys(undefined), undefined) 57 | t.deepEqual( 58 | canonicalizeAdapterNameKeys({ 59 | test: 1, 60 | TEST2: 2, 61 | 'TEST-adapter': 3, 62 | TEST_ADAPTER2: 4, 63 | 'A-B-C-D-e-f': 5, 64 | }), 65 | { 66 | test: 1, 67 | test2: 2, 68 | test_adapter: 3, 69 | test_adapter2: 4, 70 | a_b_c_d_e_f: 5, 71 | }, 72 | ) 73 | }) 74 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: 'Publish NPM Package' 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | dry-run: 7 | description: 'If true, skips commit on npm version and passes the --dry-run flag to npm publish. Useful for testing.' 8 | required: false 9 | default: 'false' 10 | push: 11 | branches: 12 | - main 13 | paths: 14 | - 'package.json' 15 | 16 | jobs: 17 | publish: 18 | permissions: 19 | id-token: write # Required for OIDC 20 | runs-on: ubuntu-latest 21 | environment: main 22 | outputs: 23 | VERSION_TAG: ${{ steps.get-version-tag.outputs.VERSION_TAG }} 24 | steps: 25 | - uses: actions/checkout@v6 26 | - id: version-check 27 | run: | 28 | PACKAGE_VERSION=$(cat package.json \ 29 | | grep version \ 30 | | head -1 \ 31 | | awk -F: '{ print $2 }' \ 32 | | sed 's/[", ]//g') 33 | PUBLISHED_VERSION=$(npm show . version) 34 | 35 | # Check that the versions are different 36 | if [[ $PACKAGE_VERSION != $PUBLISHED_VERSION ]]; then 37 | echo "Current version ($PACKAGE_VERSION) is different than the published one ($PUBLISHED_VERSION), will publish" 38 | echo "SHOULD_PUBLISH=true" >> $GITHUB_OUTPUT 39 | fi 40 | - if: steps.version-check.outputs.SHOULD_PUBLISH == 'true' 41 | uses: ./.github/actions/setup 42 | - name: Install npm version needed for trusted publishing 43 | if: steps.version-check.outputs.SHOULD_PUBLISH == 'true' 44 | run: npm install -g npm@11 45 | - if: steps.version-check.outputs.SHOULD_PUBLISH == 'true' 46 | run: yarn build 47 | - if: steps.version-check.outputs.SHOULD_PUBLISH == 'true' 48 | run: | 49 | params=(--access public) 50 | if [[ ${{ inputs.dry-run || 'false' }} == "true" ]]; then 51 | params+=(--dry-run) 52 | fi 53 | if ! npm publish "${params[@]}" ; then # scoped packages are restricted by default, but this is set because not all branches currently have a scoped package name in package.json 54 | echo "Failed to publish package" 55 | exit 1 56 | fi 57 | working-directory: dist/src 58 | -------------------------------------------------------------------------------- /test/metrics/redis-metrics.test.ts: -------------------------------------------------------------------------------- 1 | import untypedTest, { TestFn } from 'ava' 2 | import Redis from 'ioredis' 3 | import { Adapter, AdapterDependencies, AdapterEndpoint, EndpointGenerics } from '../../src/adapter' 4 | import { Cache, RedisCache } from '../../src/cache' 5 | import { AdapterConfig } from '../../src/config' 6 | import { BasicCacheSetterTransport, cacheTestInputParameters } from '../cache/helper' 7 | import { NopTransport, RedisMock, TestAdapter } from '../../src/util/testing-utils' 8 | 9 | export const test = untypedTest as TestFn<{ 10 | testAdapter: TestAdapter 11 | cache: Cache 12 | adapterEndpoint: AdapterEndpoint 13 | }> 14 | 15 | test.before(async (t) => { 16 | process.env['METRICS_ENABLED'] = 'true' 17 | // Set unique port between metrics tests to avoid conflicts in metrics servers 18 | process.env['METRICS_PORT'] = '9091' 19 | const config = new AdapterConfig( 20 | {}, 21 | { 22 | envDefaultOverrides: { 23 | CACHE_POLLING_SLEEP_MS: 10, 24 | CACHE_POLLING_MAX_RETRIES: 3, 25 | }, 26 | }, 27 | ) 28 | const adapter = new Adapter({ 29 | name: 'TEST', 30 | defaultEndpoint: 'test', 31 | config, 32 | endpoints: [ 33 | new AdapterEndpoint({ 34 | name: 'test', 35 | inputParameters: cacheTestInputParameters, 36 | transport: new BasicCacheSetterTransport(), 37 | }), 38 | new AdapterEndpoint({ 39 | name: 'nowork', 40 | transport: new NopTransport(), 41 | }), 42 | ], 43 | }) 44 | 45 | const cache = new RedisCache(new RedisMock() as unknown as Redis, 10000) // Fake redis 46 | const dependencies: Partial = { 47 | cache, 48 | } 49 | 50 | t.context.cache = cache 51 | t.context.testAdapter = await TestAdapter.start(adapter, t.context, dependencies) 52 | }) 53 | 54 | test.serial('Test redis sent command metric', async (t) => { 55 | const data = { 56 | base: 'eth', 57 | factor: 123, 58 | } 59 | 60 | await t.context.testAdapter.request(data) 61 | 62 | const metrics = await t.context.testAdapter.getMetrics() 63 | metrics.assert(t, { 64 | name: 'redis_commands_sent_count', 65 | labels: { 66 | status: 'SUCCESS', 67 | function_name: 'exec', 68 | }, 69 | expectedValue: 1, 70 | }) 71 | }) 72 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/test/adapter-ws.test.ts.ejs: -------------------------------------------------------------------------------- 1 | import { WebSocketClassProvider } from '@chainlink/external-adapter-framework/transports' 2 | import { 3 | TestAdapter, 4 | setEnvVariables, 5 | mockWebSocketProvider, 6 | MockWebsocketServer, 7 | } from '@chainlink/external-adapter-framework/util/testing-utils' 8 | import FakeTimers from '@sinonjs/fake-timers' 9 | import { mockWebsocketServer } from './fixtures' 10 | 11 | 12 | describe('websocket', () => { 13 | let mockWsServer: MockWebsocketServer | undefined 14 | let testAdapter: TestAdapter 15 | const wsEndpoint = 'ws://localhost:9090' 16 | let oldEnv: NodeJS.ProcessEnv 17 | <% for(let i=0; i 18 | const data<%- endpoints[i].normalizedEndpointNameCap %> = { 19 | base: 'ETH', 20 | quote: 'USD', 21 | endpoint: '<%- endpoints[i].inputEndpointName %>', 22 | transport: 'ws' 23 | } 24 | <% } %> 25 | beforeAll(async () => { 26 | oldEnv = JSON.parse(JSON.stringify(process.env)) 27 | process.env['WS_API_ENDPOINT'] = wsEndpoint 28 | process.env['API_KEY'] = 'fake-api-key' 29 | mockWebSocketProvider(WebSocketClassProvider) 30 | mockWsServer = mockWebsocketServer(wsEndpoint) 31 | 32 | const adapter = (await import('./../../src')).adapter 33 | testAdapter = await TestAdapter.startWithMockedCache(adapter, { 34 | clock: FakeTimers.install(), 35 | testAdapter: {} as TestAdapter, 36 | }) 37 | 38 | // Send initial request to start background execute and wait for cache to be filled with results 39 | <% for(var i=0; i 40 | await testAdapter.request(data<%- endpoints[i].normalizedEndpointNameCap %>) <% } %> 41 | await testAdapter.waitForCache(<%- endpoints.length %>) 42 | }) 43 | 44 | afterAll(async () => { 45 | setEnvVariables(oldEnv) 46 | mockWsServer?.close() 47 | testAdapter.clock?.uninstall() 48 | await testAdapter.api.close() 49 | }) 50 | <% for(var i=0; i 51 | describe('<%= endpoints[i].inputEndpointName %> endpoint', () => { 52 | it('should return success', async () => { 53 | const response = await testAdapter.request(data<%- endpoints[i].normalizedEndpointNameCap %>) 54 | expect(response.json()).toMatchSnapshot() 55 | }) 56 | }) 57 | <% } %> 58 | }) 59 | -------------------------------------------------------------------------------- /.github/actions/validate-version-labels/action.yaml: -------------------------------------------------------------------------------- 1 | name: 'Validate version labels' 2 | description: 'Checks that a PR has a valid version bump label' 3 | inputs: 4 | LABELS: 5 | description: 'The labels from the PR, or provided by the user in a manual run. Only the version bump label is required, but if you choose to provide others, the format is "LABEL1,LABEL2,LABEL3""' 6 | required: true 7 | outputs: 8 | VERSION_INSTRUCTION: 9 | description: "Which of 'patch', 'minor', or 'major' to pass to npm version, or if we should run npm version at all ('none')." 10 | value: ${{ steps.get-version-instruction.outputs.VERSION_INSTRUCTION }} 11 | MESSAGE: 12 | description: 'The message to pass to the PR comment if this job fails. If this job succeeds, this output is empty.' 13 | value: ${{ steps.get-version-instruction.outputs.MESSAGE }} 14 | JOB_ID: 15 | description: 'The ID of the job calling this action. Used to print a link in the PR comment.' 16 | value: ${{ steps.get-version-instruction.outputs.JOB_ID }} 17 | 18 | runs: 19 | using: 'composite' 20 | steps: 21 | - id: get-version-instruction 22 | run: | 23 | shopt -s nocasematch 24 | echo "JOB_ID=${{ github.job }}" >> $GITHUB_OUTPUT 25 | LABELS=${{ inputs.LABELS }} 26 | 27 | # Do a regex search for version bump labels. This loads the BASH_REMATCH array with the matches so we can ensure there is only one. 28 | echo "Searching labels for version bump: $LABELS" 29 | if ! [[ $LABELS =~ (major|minor|patch|none|version-bump) ]]; then 30 | MESSAGE="No labels present on the PR, expected version bump label (one of 'major', 'minor', 'patch', or 'none')" 31 | echo "MESSAGE=$MESSAGE" >> $GITHUB_OUTPUT 32 | exit 1 33 | elif [[ ${#BASH_REMATCH[@]} > 2 ]]; then #BASH_REMATCH[0] is the full match, BASH_REMATCH[1] is the first capture group, so >2 means there were more than one capture groups 34 | MESSAGE="Multiple version bump labels present on the PR, expected only one of 'major', 'minor', 'patch', or 'none'" 35 | echo "MESSAGE=$MESSAGE" >> $GITHUB_OUTPUT 36 | exit 1 37 | fi 38 | 39 | LABEL=${BASH_REMATCH[1]} 40 | echo "Version bump label found: $LABEL" 41 | echo "VERSION_INSTRUCTION=$LABEL" >> $GITHUB_OUTPUT 42 | echo "JOB_ID=${{ github.job}}" >> $GITHUB_OUTPUT 43 | shell: bash 44 | -------------------------------------------------------------------------------- /test/adapter.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import { expose, start } from '../src' 3 | import { Adapter, AdapterEndpoint } from '../src/adapter' 4 | import { NopTransport, TestAdapter } from '../src/util/testing-utils' 5 | 6 | test('duplicate endpoint names throw error on startup', async (t) => { 7 | const adapter = new Adapter({ 8 | name: 'TEST', 9 | endpoints: [ 10 | new AdapterEndpoint({ 11 | name: 'test', 12 | transport: new NopTransport(), 13 | }), 14 | new AdapterEndpoint({ 15 | name: 'another', 16 | aliases: ['test'], 17 | transport: new NopTransport(), 18 | }), 19 | ], 20 | }) 21 | 22 | await t.throwsAsync(async () => expose(adapter), { 23 | message: 'Duplicate endpoint / alias: "test"', 24 | }) 25 | }) 26 | 27 | test('lowercase adapter name throws error on startup', async (t) => { 28 | const adapter = new Adapter({ 29 | // @ts-expect-error - tests that lowercase names throw errors in runtime 30 | name: 'test', 31 | endpoints: [ 32 | new AdapterEndpoint({ 33 | name: 'test', 34 | transport: new NopTransport(), 35 | }), 36 | ], 37 | }) 38 | 39 | await t.throwsAsync(async () => expose(adapter), { 40 | message: 'Adapter name must be uppercase', 41 | }) 42 | }) 43 | 44 | test('Bootstrap function runs if provided', async (t) => { 45 | const adapter = new Adapter({ 46 | name: 'TEST', 47 | endpoints: [ 48 | new AdapterEndpoint({ 49 | name: 'test', 50 | transport: new NopTransport(), 51 | }), 52 | ], 53 | bootstrap: async (ea) => { 54 | ea.name = 'BOOTSTRAPPED' 55 | }, 56 | }) 57 | await start(adapter) 58 | t.is(adapter.name, 'BOOTSTRAPPED') 59 | }) 60 | test.serial('Throws when transport.registerRequest errors', async (t) => { 61 | const adapter = new Adapter({ 62 | name: 'TEST', 63 | endpoints: [ 64 | new AdapterEndpoint({ 65 | name: 'test', 66 | transport: new (class extends NopTransport { 67 | async registerRequest() { 68 | throw new Error('Error from registerRequest') 69 | } 70 | })(), 71 | }), 72 | ], 73 | }) 74 | 75 | const testAdapter = await TestAdapter.start(adapter, { testAdapter: {} as TestAdapter }) 76 | const error = await testAdapter.request({ 77 | endpoint: 'test', 78 | }) 79 | 80 | t.is(error.statusCode, 500) 81 | t.is(error.body, 'Error from registerRequest') 82 | }) 83 | -------------------------------------------------------------------------------- /test/metrics/helper.ts: -------------------------------------------------------------------------------- 1 | import { AxiosResponse } from 'axios' 2 | import { Adapter, AdapterEndpoint } from '../../src/adapter' 3 | import { EmptyCustomSettings } from '../../src/config' 4 | import { HttpTransport } from '../../src/transports' 5 | import { SingleNumberResultResponse } from '../../src/util' 6 | import { InputParameters } from '../../src/validation' 7 | 8 | export const buildHttpAdapter = (): Adapter => { 9 | return new Adapter({ 10 | name: 'TEST', 11 | defaultEndpoint: 'test', 12 | endpoints: [ 13 | new AdapterEndpoint({ 14 | name: 'test', 15 | inputParameters, 16 | transport: new MockHttpTransport(), 17 | }), 18 | ], 19 | }) 20 | } 21 | 22 | const URL = 'http://test-url.com' 23 | 24 | interface ProviderRequestBody { 25 | pairs: Array<{ 26 | base: string 27 | quote: string 28 | }> 29 | } 30 | 31 | interface ProviderResponseBody { 32 | prices: Array<{ 33 | pair: string 34 | price: number 35 | }> 36 | } 37 | 38 | type HttpEndpointTypes = { 39 | Parameters: typeof inputParameters.definition 40 | Response: SingleNumberResultResponse 41 | Settings: EmptyCustomSettings 42 | Provider: { 43 | RequestBody: ProviderRequestBody 44 | ResponseBody: ProviderResponseBody 45 | } 46 | } 47 | 48 | class MockHttpTransport extends HttpTransport { 49 | constructor() { 50 | super({ 51 | prepareRequests: (params) => { 52 | return { 53 | params, 54 | request: { 55 | baseURL: URL, 56 | url: '/price', 57 | method: 'POST', 58 | data: { 59 | pairs: params.map((p) => ({ base: p.from, quote: p.to })), 60 | }, 61 | }, 62 | } 63 | }, 64 | parseResponse: (params, res: AxiosResponse) => { 65 | return res.data.prices.map((p) => { 66 | const [from, to] = p.pair.split('/') 67 | return { 68 | params: { from, to }, 69 | response: { 70 | data: { 71 | result: p.price, 72 | }, 73 | result: p.price, 74 | }, 75 | } 76 | }) 77 | }, 78 | }) 79 | } 80 | } 81 | 82 | const inputParameters = new InputParameters({ 83 | from: { 84 | type: 'string', 85 | description: 'from', 86 | required: true, 87 | }, 88 | to: { 89 | type: 'string', 90 | description: 'to', 91 | required: true, 92 | }, 93 | }) 94 | -------------------------------------------------------------------------------- /src/rate-limiting/fixed-interval.ts: -------------------------------------------------------------------------------- 1 | import { AdapterRateLimitTier, RateLimiter } from '.' 2 | import { AdapterEndpoint, EndpointGenerics } from '../adapter' 3 | import { makeLogger, sleep } from '../util' 4 | 5 | const logger = makeLogger('FixedIntervalRateLimiter') 6 | 7 | /** 8 | * The simplest version of a rate limit. This will not take any bursts into accoung, 9 | * and always rely on a fixed request per second rate. The only "complex" mechanism it has 10 | * is checking when the last request was made to this rate limiter, to account for a period 11 | * of time with no requests and avoiding the wait of the initial request. 12 | */ 13 | export class FixedIntervalRateLimiter implements RateLimiter { 14 | period!: number 15 | lastRequestAt: number | null = null 16 | 17 | initialize( 18 | endpoints: AdapterEndpoint[], 19 | limits?: AdapterRateLimitTier, 20 | ) { 21 | // Translate the hourly and minute limits into reqs per minute 22 | const perHourLimitInRPS = (limits?.rateLimit1h || Infinity) / 60 / 60 23 | const perMinuteLimitInRPS = (limits?.rateLimit1m || Infinity) / 60 24 | const perSecondLimitInRPS = limits?.rateLimit1s || Infinity 25 | this.period = (1 / Math.min(perHourLimitInRPS, perMinuteLimitInRPS, perSecondLimitInRPS)) * 1000 26 | logger.debug(`Using fixed interval rate limiting settings: period = ${this.period}`) 27 | 28 | return this 29 | } 30 | 31 | msUntilNextExecution(): number { 32 | const now = Date.now() 33 | 34 | if (!this.lastRequestAt) { 35 | logger.trace( 36 | `First request for the rate limiter, sending immediately. All subsequent requests will wait ${this.period}ms.`, 37 | ) 38 | return 0 39 | } 40 | 41 | const timeSinceLastRequest = now - this.lastRequestAt // Positive int 42 | const remainingTime = Math.max(0, this.period - timeSinceLastRequest) 43 | logger.trace(`Rate limiting details: 44 | now: ${now} 45 | timeSinceLastRequest: ${timeSinceLastRequest} 46 | period: ${this.period} 47 | remainingTime: ${remainingTime} 48 | `) 49 | return remainingTime 50 | } 51 | 52 | async waitForRateLimit(): Promise { 53 | const timeToWait = this.msUntilNextExecution() 54 | if (timeToWait > 0) { 55 | logger.debug(`Sleeping for ${timeToWait}ms to wait for rate limiting interval to pass`) 56 | await sleep(timeToWait) 57 | } else { 58 | logger.debug(`Enough time has passed since last request, no need to sleep`) 59 | } 60 | this.lastRequestAt = Date.now() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/transports/metrics.ts: -------------------------------------------------------------------------------- 1 | import { TransportGenerics } from '.' 2 | import { EndpointContext } from '../adapter' 3 | import { calculateFeedId } from '../cache' 4 | import { metrics } from '../metrics' 5 | import { InputParameters } from '../validation' 6 | import { TypeFromDefinition } from '../validation/input-params' 7 | 8 | // Websocket Metrics 9 | export const connectionErrorLabels = (message: string) => ({ 10 | // Key, 11 | message, 12 | }) 13 | 14 | export type MessageDirection = 'sent' | 'received' 15 | 16 | export const messageSubsLabels = ( 17 | context: { 18 | inputParameters: InputParameters 19 | endpointName: string 20 | adapterSettings: T['Settings'] 21 | }, 22 | params: TypeFromDefinition, 23 | ) => { 24 | const feedId = calculateFeedId(context, params) 25 | 26 | return { 27 | feed_id: feedId, 28 | subscription_key: `${context.endpointName}-${feedId}`, 29 | } 30 | } 31 | 32 | export const recordWsMessageSentMetrics = ( 33 | context: EndpointContext, 34 | subscribes: TypeFromDefinition[], 35 | unsubscribes: TypeFromDefinition[], 36 | ) => { 37 | for (const params of [...subscribes, ...unsubscribes]) { 38 | const baseLabels = messageSubsLabels(context, params) 39 | 40 | // Record total number of ws messages sent 41 | metrics 42 | .get('wsMessageTotal') 43 | .labels({ ...baseLabels, direction: 'sent' }) 44 | .inc() 45 | } 46 | } 47 | 48 | // Record WS message and subscription metrics 49 | // Recalculate cacheKey and feedId for metrics 50 | // since avoiding storing extra info in expiring sorted set 51 | export const recordWsMessageSubMetrics = ( 52 | context: EndpointContext, 53 | subscribes: TypeFromDefinition[], 54 | unsubscribes: TypeFromDefinition[], 55 | ): void => { 56 | const recordMetrics = (params: TypeFromDefinition, type: 'sub' | 'unsub') => { 57 | const baseLabels = messageSubsLabels(context, params) 58 | 59 | // Record total number of subscriptions made 60 | if (type === 'sub') { 61 | metrics.get('wsSubscriptionTotal').labels(baseLabels).inc() 62 | metrics.get('wsSubscriptionActive').labels(baseLabels).inc() 63 | } else { 64 | metrics.get('wsSubscriptionActive').labels(baseLabels).dec() 65 | } 66 | } 67 | 68 | subscribes.forEach((params) => { 69 | recordMetrics(params, 'sub') 70 | }) 71 | unsubscribes.forEach((params) => { 72 | recordMetrics(params, 'unsub') 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /test/subscription-set/subscription-set-factory.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import Redis from 'ioredis' 3 | import { buildAdapterSettings } from '../../src/config' 4 | import { LoggerFactoryProvider, SubscriptionSetFactory } from '../../src/util' 5 | import { ExpiringSortedSet } from '../../src/util/subscription-set/expiring-sorted-set' 6 | import { RedisSubscriptionSet } from '../../src/util/subscription-set/redis-sorted-set' 7 | import { RedisMock } from '../../src/util/testing-utils' 8 | 9 | test.before(() => { 10 | LoggerFactoryProvider.set() 11 | }) 12 | 13 | test('subscription set factory (local cache)', async (t) => { 14 | process.env['CACHE_TYPE'] = 'local' 15 | const config = buildAdapterSettings({}) 16 | const factory = new SubscriptionSetFactory(config, 'test') 17 | const subscriptionSet = factory.buildSet('test', 'test') 18 | t.is(subscriptionSet instanceof ExpiringSortedSet, true) 19 | }) 20 | 21 | test('subscription set factory (redis cache)', async (t) => { 22 | process.env['CACHE_TYPE'] = 'redis' 23 | const config = buildAdapterSettings({}) 24 | const factory = new SubscriptionSetFactory(config, 'test', new RedisMock() as unknown as Redis) 25 | const subscriptionSet = factory.buildSet('test', 'test') 26 | t.is(subscriptionSet instanceof RedisSubscriptionSet, true) 27 | const value = subscriptionSet.get('testKey') 28 | t.is(value, undefined) 29 | }) 30 | 31 | test('subscription set factory (redis cache missing client)', async (t) => { 32 | process.env['CACHE_TYPE'] = 'redis' 33 | const config = buildAdapterSettings({}) 34 | const factory = new SubscriptionSetFactory(config, 'test') 35 | try { 36 | factory.buildSet('test', 'test') 37 | t.fail() 38 | } catch (_) { 39 | t.pass() 40 | } 41 | }) 42 | 43 | test('subscription set factory (local cache) max capacity', async (t) => { 44 | process.env['CACHE_TYPE'] = 'local' 45 | process.env['SUBSCRIPTION_SET_MAX_ITEMS'] = '3' 46 | const config = buildAdapterSettings({}) 47 | const factory = new SubscriptionSetFactory(config, 'test') 48 | const subscriptionSet = factory.buildSet('test', 'test') 49 | 50 | await subscriptionSet.add(1, 10000, '1') 51 | await subscriptionSet.add(2, 10000, '2') 52 | await subscriptionSet.add(3, 10000, '3') 53 | await subscriptionSet.add(4, 10000, '4') 54 | 55 | const value1 = await subscriptionSet.get('1') 56 | const value2 = await subscriptionSet.get('2') 57 | const value3 = await subscriptionSet.get('3') 58 | const value4 = await subscriptionSet.get('4') 59 | 60 | t.is(value1, undefined) 61 | t.is(value2, 2) 62 | t.is(value3, 3) 63 | t.is(value4, 4) 64 | 65 | const allValues = await subscriptionSet.getAll() 66 | t.deepEqual(allValues, [2, 3, 4]) 67 | }) 68 | -------------------------------------------------------------------------------- /src/adapter/market-status.ts: -------------------------------------------------------------------------------- 1 | import { TransportGenerics } from '../transports' 2 | import { AdapterEndpoint } from './endpoint' 3 | import { AdapterEndpointParams } from './types' 4 | import { parseWeekendString } from '../validation/market-status' 5 | import { AdapterInputError } from '../validation/error' 6 | 7 | /** 8 | * Base input parameter config that any [[MarketStatusEndpoint]] must extend 9 | */ 10 | export const marketStatusEndpointInputParametersDefinition = { 11 | market: { 12 | aliases: [], 13 | type: 'string', 14 | description: 'The name of the market', 15 | required: true, 16 | }, 17 | type: { 18 | type: 'string', 19 | description: 'Type of the market status', 20 | options: ['regular', '24/5'], 21 | default: 'regular', 22 | }, 23 | weekend: { 24 | type: 'string', 25 | description: 26 | 'DHH-DHH:TZ, 520-020:America/New_York means Fri 20:00 to Sun 20:00 Eastern Time Zone', 27 | }, 28 | } as const 29 | 30 | export enum MarketStatus { 31 | UNKNOWN = 0, 32 | CLOSED = 1, 33 | OPEN = 2, 34 | } 35 | 36 | export enum TwentyfourFiveMarketStatus { 37 | UNKNOWN = 0, 38 | PRE_MARKET = 1, 39 | REGULAR = 2, 40 | POST_MARKET = 3, 41 | OVERNIGHT = 4, 42 | WEEKEND = 5, 43 | } 44 | 45 | type AggregatedMarketStatus = MarketStatus | TwentyfourFiveMarketStatus 46 | 47 | export type MarketStatusResultResponse = { 48 | Result: AggregatedMarketStatus 49 | Data: { 50 | result: AggregatedMarketStatus 51 | statusString: string 52 | } 53 | } 54 | 55 | /** 56 | * Helper type structure that contains the different types passed to the generic parameters of a PriceEndpoint 57 | */ 58 | export type MarketStatusEndpointGenerics = TransportGenerics & { 59 | Parameters: typeof marketStatusEndpointInputParametersDefinition 60 | Response: MarketStatusResultResponse 61 | } 62 | 63 | /** 64 | * A MarketStatusEndpoint is a specific type of AdapterEndpoint. Meant to comply with standard practices for 65 | * Data Feeds, its InputParameters must extend the basic ones (base). 66 | */ 67 | export class MarketStatusEndpoint< 68 | T extends MarketStatusEndpointGenerics, 69 | > extends AdapterEndpoint { 70 | constructor(params: AdapterEndpointParams) { 71 | params.customInputValidation = (req, _adapterSettings) => { 72 | const data = req.requestContext.data as Record 73 | if (data['type'] === '24/5') { 74 | parseWeekendString(data['weekend']) 75 | } 76 | if (data['type'] === 'regular' && data['weekend']) { 77 | throw new AdapterInputError({ 78 | statusCode: 400, 79 | message: '[Param: weekend] must be empty when [Param: type] is regular', 80 | }) 81 | } 82 | return undefined 83 | } 84 | super(params) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "incremental": true /* Enable incremental compilation */, 5 | "target": "ES2020" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */, 6 | "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */, 7 | "composite": true /* Enable project compilation */, 8 | "declaration": true /* Generates corresponding '.d.ts' file. */, 9 | "declarationMap": true /* Generates a sourcemap for each corresponding '.d.ts' file. */, 10 | "noEmit": false /* Do not emit outputs. */, 11 | "noErrorTruncation": true /* Do not truncate error messages */, 12 | "skipLibCheck": true /* Skip type checking of declaration files. Requires TypeScript version 2.0 or later. */, 13 | "importHelpers": true /* Import emit helpers from 'tslib'. */, 14 | 15 | /* Strict Type-Checking Options */ 16 | "strict": true /* Enable all strict type-checking options. */, 17 | 18 | /* Additional Checks */ 19 | "noUnusedLocals": true /* Report errors on unused locals. */, 20 | "noUnusedParameters": true /* Report errors on unused parameters. */, 21 | "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, 22 | "noFallthroughCasesInSwitch": true /* Report errors for fallthrough cases in switch statement. */, 23 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */, 24 | 25 | /* Module Resolution Options */ 26 | "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, 27 | "allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */, 28 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 29 | 30 | /* Source Map Options */ 31 | "inlineSourceMap": true /* Emit a single file with source maps instead of having a separate file. */, 32 | "inlineSources": true /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */, 33 | 34 | /* Experimental Options */ 35 | "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, 36 | "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, 37 | 38 | "resolveJsonModule": true /* Allows importing modules with a ‘.json’ extension */ 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /.github/workflows/add-or-validate-labels.yaml: -------------------------------------------------------------------------------- 1 | # Adds labels to pull requests based on the branch name. Labels are required by the "publish" workflow to determine 2 | name: 'Validate PR labels' 3 | 4 | on: 5 | workflow_dispatch: # Lets you see what labels would be added, but doesn't actually add them because no PR triggered it 6 | pull_request: 7 | types: 8 | - opened 9 | - labeled 10 | - unlabeled 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | validate-labels: 18 | name: Validate PR labels 19 | runs-on: ubuntu-latest 20 | outputs: 21 | VERSION_INSTRUCTION: ${{ steps.validate.outputs.VERSION_INSTRUCTION }} 22 | steps: 23 | - uses: actions/checkout@v6 24 | - name: Validate that version bump labels are on the PR 25 | id: validate 26 | uses: ./.github/actions/validate-version-labels 27 | with: 28 | LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} 29 | 30 | upsert-pr-comment: 31 | name: Upsert PR labels comment 32 | if: always() 33 | needs: validate-labels 34 | runs-on: ubuntu-latest 35 | steps: 36 | - name: Build comment contents 37 | id: bc 38 | run: | 39 | VERSION_INSTRUCTION=${{ needs.validate-labels.outputs.VERSION_INSTRUCTION }} 40 | 41 | if [[ -z $VERSION_INSTRUCTION ]]; then 42 | MESSAGE=":stop_sign: This PR needs labels to indicate how to increase the current package version in the automated workflows. Please add one of the following labels: \`none\`, \`patch\`, \`minor\`, or \`major\`." 43 | elif [[ $VERSION_INSTRUCTION == none ]]; then 44 | MESSAGE=":large_blue_circle: This PR has the \`none\` label set and it will not cause a version bump." 45 | else 46 | MESSAGE=":green_circle: This PR has valid version labels and will cause a \`$VERSION_INSTRUCTION\` bump." 47 | fi 48 | 49 | echo "MESSAGE=$MESSAGE" >> $GITHUB_OUTPUT 50 | 51 | - name: Find previous comment 52 | uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 53 | id: fc 54 | with: 55 | issue-number: ${{ github.event.pull_request.number }} 56 | comment-author: 'github-actions[bot]' 57 | body-includes: NPM Publishing labels 58 | 59 | - name: Create or update comment 60 | uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 61 | with: 62 | comment-id: ${{ steps.fc.outputs.comment-id }} 63 | issue-number: ${{ github.event.pull_request.number }} 64 | body: | 65 | ### NPM Publishing labels :label: 66 | ${{ steps.bc.outputs.MESSAGE }} 67 | edit-mode: replace 68 | -------------------------------------------------------------------------------- /src/util/group-runner.ts: -------------------------------------------------------------------------------- 1 | type Callback = () => Promise 2 | type Resolve = (arg: T | Promise) => void 3 | 4 | // Runs tasks in groups of a fixed size. 5 | // The next group won't be started until the previous group is finished. 6 | // 7 | // Example usage: 8 | // 9 | // const fetchBalances(addresses: string[]): Promise { 10 | // // addresses can contain thousands of addresses 11 | // const groupRunner = new GroupRunner(10) 12 | // const getBalance = groupRunner.wrapFunction(fetchBalance) 13 | // const balancePromises: Promise[] = [] 14 | // for (const address of addresses) { 15 | // // There will be at most 10 concurrent calls to fetchBalance. 16 | // // fetchBalance might do an RPC and we don't want to get rate limited. 17 | // balancePromises.push(getBalance(address))) 18 | // } 19 | // return Promise.all(balancePromises) 20 | // } 21 | // 22 | // 23 | // Implementation note: 24 | // Once the size has been reached, we wait for all previous tasks to finish 25 | // before running the new task. 26 | // Alternatively, we could run more tasks as soon as some (rather than all) 27 | // tasks have finished, to make progress sooner, but the former is what's 28 | // currently used in multiple places in the external-adapters-js repo so we 29 | // chose that behavior. 30 | export class GroupRunner { 31 | private currentGroup: Promise[] = [] 32 | private previousStartRunning: Promise = Promise.resolve() 33 | 34 | constructor(private groupSize: number) {} 35 | 36 | // Calls the given callback eventually but makes sure any previous group of 37 | // groupSize size has settled before calling and subsequent callbacks. 38 | run(callback: Callback): Promise { 39 | return new Promise((resolve) => { 40 | // This creates an implicit queue which guarantees that there are no 41 | // concurrent calls into startRunning. This is necessary to avoid having 42 | // currentGroup being cleared concurrently. 43 | this.previousStartRunning = this.previousStartRunning.then(() => { 44 | return this.startRunning(callback, resolve) 45 | }) 46 | }) 47 | } 48 | 49 | // Waits for a previous group to finish, if necessary, and then runs the 50 | // given callback. When this method resolves, the callback has been called 51 | // but not necessarily resolved. 52 | async startRunning(callback: Callback, resolve: Resolve) { 53 | if (this.currentGroup.length >= this.groupSize) { 54 | await Promise.allSettled(this.currentGroup) 55 | this.currentGroup = [] 56 | } 57 | const promise = callback() 58 | this.currentGroup.push(promise) 59 | resolve(promise) 60 | } 61 | 62 | wrapFunction( 63 | func: (...args: Args) => Promise, 64 | ): (...args: Args) => Promise { 65 | return (...args: Args) => { 66 | return this.run(() => func(...args)) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/status/router.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from 'fastify' 2 | import { join } from 'path' 3 | import { hostname } from 'os' 4 | import { Adapter } from '../adapter' 5 | import { buildSettingsList } from '../util/settings' 6 | 7 | export interface StatusResponse { 8 | adapter: { 9 | name: string 10 | version: string 11 | uptimeSeconds: number 12 | } 13 | endpoints: { 14 | name: string 15 | aliases: string[] 16 | transports: string[] 17 | }[] 18 | defaultEndpoint: string 19 | configuration: { 20 | name: string 21 | value: unknown 22 | type: string 23 | description: string 24 | required: boolean 25 | default: unknown 26 | customSetting: boolean 27 | envDefaultOverride: unknown 28 | }[] 29 | runtime: { 30 | nodeVersion: string 31 | platform: string 32 | architecture: string 33 | hostname: string 34 | } 35 | metrics: { 36 | enabled: boolean 37 | port?: number 38 | endpoint?: string 39 | } 40 | } 41 | 42 | /** 43 | * This function registers the status endpoint for the adapter. 44 | * This endpoint provides comprehensive information about the adapter including: 45 | * - Adapter metadata (name, version, commit) 46 | * - Configuration (obfuscated sensitive values) 47 | * - Runtime information 48 | * - Dependencies status 49 | * - Endpoints and transports 50 | * 51 | * @param app - the fastify instance that has been created 52 | * @param adapter - the adapter for which to create the status endpoint 53 | */ 54 | export default function registerStatusEndpoint(app: FastifyInstance, adapter: Adapter) { 55 | // Status endpoint that returns comprehensive adapter information 56 | app.get(join(adapter.config.settings.BASE_URL, '/status'), async () => { 57 | const metricsEndpoint = adapter.config.settings.METRICS_USE_BASE_URL 58 | ? join(adapter.config.settings.BASE_URL, 'metrics') 59 | : '/metrics' 60 | 61 | const statusResponse: StatusResponse = { 62 | adapter: { 63 | name: adapter.name, 64 | version: process.env['npm_package_version'] || 'unknown', 65 | uptimeSeconds: process.uptime(), 66 | }, 67 | endpoints: adapter.endpoints.map((endpoint) => ({ 68 | name: endpoint.name, 69 | aliases: endpoint.aliases || [], 70 | transports: endpoint.transportRoutes.routeNames(), 71 | })), 72 | defaultEndpoint: adapter.defaultEndpoint || '', 73 | configuration: buildSettingsList(adapter), 74 | runtime: { 75 | nodeVersion: process.version, 76 | platform: process.platform, 77 | architecture: process.arch, 78 | hostname: hostname(), 79 | }, 80 | metrics: { 81 | enabled: adapter.config.settings.METRICS_ENABLED, 82 | port: adapter.config.settings.METRICS_ENABLED 83 | ? adapter.config.settings.METRICS_PORT 84 | : undefined, 85 | endpoint: adapter.config.settings.METRICS_ENABLED ? metricsEndpoint : undefined, 86 | }, 87 | } 88 | 89 | return statusResponse 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /test/adapter/market-status.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import '../../src/adapter' 3 | import { 4 | MarketStatusEndpoint, 5 | marketStatusEndpointInputParametersDefinition, 6 | MarketStatusEndpointGenerics, 7 | } from '../../src/adapter/market-status' 8 | import { InputParameters } from '../../src/validation' 9 | import { TestAdapter } from '../../src/util/testing-utils' 10 | import { Adapter } from '../../src/adapter/basic' 11 | 12 | import { Transport } from '../../src/transports' 13 | import { ResponseCache } from '../../src/cache/response' 14 | 15 | test('MarketStatusEndpoint - validates weekend', async (t) => { 16 | class MarketStatusTestTransport implements Transport { 17 | name!: string 18 | responseCache!: ResponseCache 19 | 20 | async initialize() {} 21 | 22 | async foregroundExecute() { 23 | return { 24 | data: { 25 | result: 2, 26 | statusString: 'OPEN', 27 | }, 28 | result: 2, 29 | statusCode: 200, 30 | timestamps: { 31 | providerDataRequestedUnixMs: 0, 32 | providerDataReceivedUnixMs: 0, 33 | providerIndicatedTimeUnixMs: 0, 34 | }, 35 | } 36 | } 37 | } 38 | 39 | const adapter = new Adapter({ 40 | name: 'TEST', 41 | endpoints: [ 42 | new MarketStatusEndpoint({ 43 | name: 'test', 44 | inputParameters: new InputParameters(marketStatusEndpointInputParametersDefinition), 45 | transport: new MarketStatusTestTransport(), 46 | }), 47 | ], 48 | }) 49 | 50 | const testAdapter = await TestAdapter.start( 51 | adapter, 52 | {} as { 53 | testAdapter: TestAdapter 54 | }, 55 | ) 56 | 57 | const response1 = await testAdapter.request({ 58 | market: 'BTC', 59 | type: 'regular', 60 | endpoint: 'test', 61 | }) 62 | t.is(response1.statusCode, 200, 'Should succeed with empty weekend when type is regular') 63 | 64 | const response2 = await testAdapter.request({ 65 | market: 'BTC', 66 | type: 'regular', 67 | weekend: '520-020', 68 | endpoint: 'test', 69 | }) 70 | t.is(response2.statusCode, 400, 'Should fail with weekend when type is regular') 71 | t.true( 72 | response2 73 | .json() 74 | .error.message.includes('[Param: weekend] must be empty when [Param: type] is regular'), 75 | ) 76 | 77 | const response3 = await testAdapter.request({ 78 | market: 'BTC', 79 | type: '24/5', 80 | weekend: '520-020', 81 | endpoint: 'test', 82 | }) 83 | t.is(response3.statusCode, 400, 'Should fail with invalid weekend format when type is 24/5') 84 | t.true(response3.json().error.message.includes('[Param: weekend] does not match format')) 85 | 86 | const response4 = await testAdapter.request({ 87 | market: 'BTC', 88 | type: '24/5', 89 | weekend: '520-020:America/New_York', 90 | endpoint: 'test', 91 | }) 92 | t.is(response4.statusCode, 200, 'Should succeed with valid weekend when type is 24/5') 93 | 94 | await testAdapter.api.close() 95 | }) 96 | -------------------------------------------------------------------------------- /test/util/testing-utils.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import { allowedUndefinedStubProps, makeStub } from '../../src/util/testing-utils' 3 | 4 | test('make a stub', async (t) => { 5 | const stub = makeStub('stub', { 6 | name: 'stub-name', 7 | count: 5, 8 | }) 9 | 10 | t.is(stub.name, 'stub-name') 11 | t.is(stub.count, 5) 12 | }) 13 | 14 | test('make a stub with nested fields', async (t) => { 15 | const stub = makeStub('stub', { 16 | name: 'stub-name', 17 | nested: { 18 | count: 5, 19 | }, 20 | }) 21 | 22 | t.is(stub.name, 'stub-name') 23 | t.is(stub.nested.count, 5) 24 | }) 25 | 26 | test('accessing an absent field should throw an error', async (t) => { 27 | const stub = makeStub('stub', { 28 | name: 'stub-name', 29 | nested: { 30 | count: 5, 31 | }, 32 | }) 33 | 34 | t.throws( 35 | () => { 36 | // @ts-expect-error intended 37 | t.is(stub.count, undefined) 38 | }, 39 | { 40 | message: "Property 'stub.count' does not exist", 41 | }, 42 | ) 43 | }) 44 | 45 | test('accessing a nested absent field should throw an error', async (t) => { 46 | const stub = makeStub('stub', { 47 | name: 'stub-name', 48 | nested: { 49 | count: 5, 50 | }, 51 | }) 52 | 53 | t.throws( 54 | () => { 55 | // @ts-expect-error intended 56 | t.is(stub.nested.name, undefined) 57 | }, 58 | { 59 | message: "Property 'stub.nested.name' does not exist", 60 | }, 61 | ) 62 | }) 63 | 64 | test('fields used by jest are allowed to be undefined', async (t) => { 65 | const stub = makeStub('stub', { 66 | name: 'stub-name', 67 | count: 5, 68 | }) 69 | 70 | // @ts-expect-error intended 71 | t.is(stub.nodeType, undefined) 72 | // @ts-expect-error intended 73 | t.is(stub.tagName, undefined) 74 | }) 75 | 76 | test('Symbol props are allowed to be undefined', async (t) => { 77 | const stub = makeStub('stub', { 78 | name: 'stub-name', 79 | count: 5, 80 | }) 81 | 82 | // @ts-expect-error intended 83 | t.is(stub[Symbol('my symbol')], undefined) 84 | }) 85 | 86 | test('allowedUndefinedStubProps can be extended and restored', async (t) => { 87 | const customProp = 'myCustomProp' 88 | 89 | const stub = makeStub('stub', { 90 | name: 'stub-name', 91 | count: 5, 92 | }) 93 | 94 | t.throws( 95 | () => { 96 | // @ts-expect-error intended 97 | t.is(stub[customProp], undefined) 98 | }, 99 | { 100 | message: "Property 'stub.myCustomProp' does not exist", 101 | }, 102 | ) 103 | 104 | allowedUndefinedStubProps.push('myCustomProp') 105 | 106 | // @ts-expect-error intended 107 | t.is(stub[customProp], undefined) 108 | 109 | allowedUndefinedStubProps.pop() 110 | 111 | t.throws( 112 | () => { 113 | // @ts-expect-error intended 114 | t.is(stub[customProp], undefined) 115 | }, 116 | { 117 | message: "Property 'stub.myCustomProp' does not exist", 118 | }, 119 | ) 120 | }) 121 | -------------------------------------------------------------------------------- /src/util/subscription-set/expiring-sorted-set.ts: -------------------------------------------------------------------------------- 1 | import { SubscriptionSet } from './subscription-set' 2 | import { DoubleLinkedList, LinkedListNode, makeLogger, PromiseOrValue } from '..' 3 | 4 | const logger = makeLogger('ExpiringSortedSet') 5 | 6 | /** 7 | * An object describing an entry in the expiring sorted set. 8 | * @typeParam T - the type of the entry's value 9 | */ 10 | interface ExpiringSortedSetEntry { 11 | value: T 12 | expirationTimestamp: number 13 | } 14 | 15 | /** 16 | * This class implements a set of unique items, each of which has an expiration timestamp. 17 | * On reads, items that have expired will be deleted from the set and not returned. 18 | * 19 | * @typeParam T - the type of the set entries' values 20 | */ 21 | export class ExpiringSortedSet implements SubscriptionSet { 22 | capacity: number 23 | map: Map>> 24 | list: DoubleLinkedList 25 | 26 | constructor(capacity: number) { 27 | this.capacity = capacity 28 | this.map = new Map>>() 29 | this.list = new DoubleLinkedList() 30 | } 31 | 32 | add(value: T, ttl: number, key: string) { 33 | let node = this.map.get(key) 34 | if (node) { 35 | node.data = { 36 | value, 37 | expirationTimestamp: Date.now() + ttl, 38 | } 39 | this.moveToTail(node) 40 | } else { 41 | this.evictIfNeeded() 42 | const data = { 43 | value, 44 | expirationTimestamp: Date.now() + ttl, 45 | } 46 | node = new LinkedListNode(key, data) 47 | this.list.insertAtTail(node) 48 | this.map.set(key, node) 49 | } 50 | } 51 | 52 | getAll(): PromiseOrValue { 53 | const results: T[] = [] 54 | for (const [key] of this.map.entries()) { 55 | const value = this.get(key) as T 56 | if (value) { 57 | results.push(value) 58 | } 59 | } 60 | return results 61 | } 62 | 63 | get(key: string): T | undefined { 64 | const node = this.map.get(key) 65 | if (node) { 66 | const expired = node.data.expirationTimestamp <= Date.now() 67 | if (expired) { 68 | this.delete(key) 69 | return undefined 70 | } else { 71 | return node.data.value 72 | } 73 | } else { 74 | return undefined 75 | } 76 | } 77 | 78 | delete(key: string): void { 79 | const node = this.map.get(key) 80 | if (node) { 81 | this.list.remove(node) 82 | this.map.delete(key) 83 | } 84 | } 85 | 86 | private moveToTail(node: LinkedListNode) { 87 | this.list.remove(node) 88 | this.list.insertAtTail(node) 89 | } 90 | 91 | private evictIfNeeded() { 92 | if (this.list.size >= this.capacity) { 93 | const node = this.list.removeHead() 94 | if (node) { 95 | logger.warn( 96 | `List reached maximum capacity, evicting least recently updated entry. The subscription with key ${node.key} was removed.`, 97 | ) 98 | this.map.delete(node.key) 99 | } 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /test/correlation.test.ts: -------------------------------------------------------------------------------- 1 | import untypedTest, { ExecutionContext, TestFn } from 'ava' 2 | import { Adapter, AdapterEndpoint } from '../src/adapter' 3 | import { AdapterResponse, sleep } from '../src/util' 4 | import { Store, asyncLocalStorage } from '../src/util/logger' 5 | import { NopTransport, NopTransportTypes, TestAdapter } from '../src/util/testing-utils' 6 | 7 | type TestContext = { 8 | testAdapter: TestAdapter 9 | adapterEndpoint: AdapterEndpoint 10 | } 11 | const test = untypedTest as TestFn 12 | 13 | const startAdapter = async ( 14 | enabled: boolean, 15 | context: ExecutionContext['context'], 16 | ) => { 17 | process.env['CORRELATION_ID_ENABLED'] = enabled.toString() 18 | 19 | const adapter = new Adapter({ 20 | name: 'TEST', 21 | defaultEndpoint: 'test', 22 | endpoints: [ 23 | new AdapterEndpoint({ 24 | name: 'test', 25 | transport: new (class extends NopTransport { 26 | override async foregroundExecute() { 27 | const store = asyncLocalStorage.getStore() as Store 28 | if (store !== undefined && store['correlationId'] === '1') { 29 | await sleep(100) 30 | } 31 | return { 32 | data: null, 33 | statusCode: 200, 34 | result: store as unknown as null, 35 | } as AdapterResponse<{ 36 | Data: null 37 | Result: null 38 | }> 39 | } 40 | })(), 41 | }), 42 | ], 43 | }) 44 | 45 | context.testAdapter = await TestAdapter.start(adapter, context) 46 | return context.testAdapter 47 | } 48 | 49 | test.serial('uses the correct correlation id when it is passed in a header', async (t) => { 50 | const testId = 'test' 51 | const testAdapter = await startAdapter(true, t.context) 52 | const response = await testAdapter.request({ base: 'asd' }, { 'x-correlation-id': testId }) 53 | t.is(response.json().result.correlationId, testId) 54 | }) 55 | 56 | test.serial('sets a correlation id when it is enabled as an env var', async (t) => { 57 | const testAdapter = await startAdapter(true, t.context) 58 | const response = await testAdapter.request({ base: 'asd' }) 59 | t.is(typeof response.json().result.correlationId, 'string') 60 | }) 61 | 62 | test.serial('correlation Id is not set when enabled is set to false', async (t) => { 63 | const testAdapter = await startAdapter(false, t.context) 64 | const response = await testAdapter.request({ base: 'asd' }) 65 | t.is(response.json().result, undefined) 66 | }) 67 | 68 | test.serial('preserves concurrency through subsequent calls', async (t) => { 69 | const testAdapter = await startAdapter(true, t.context) 70 | const request1 = testAdapter.request({ base: 'asd' }, { 'x-correlation-id': '1' }) 71 | const request2 = testAdapter.request({ base: 'asd' }, { 'x-correlation-id': '2' }) 72 | 73 | // Check that each call has the correct correlation Id 74 | const response2 = await request2 75 | const response1 = await request1 76 | t.is(response1.json().result.correlationId === '1', true) 77 | t.is(response2.json().result.correlationId === '2', true) 78 | }) 79 | -------------------------------------------------------------------------------- /src/adapter/lwba.ts: -------------------------------------------------------------------------------- 1 | import { TransportGenerics } from '../transports' 2 | import { AdapterLWBAError } from '../validation/error' 3 | import { AdapterEndpoint } from './endpoint' 4 | import { AdapterEndpointParams, PriceEndpointInputParametersDefinition } from './index' 5 | 6 | /** 7 | * Type for the base input parameter config that any [[LwbaEndpoint]] must extend 8 | */ 9 | export type LwbaEndpointInputParametersDefinition = PriceEndpointInputParametersDefinition 10 | 11 | /** 12 | * Base input parameter config that any [[LwbaEndpoint]] must extend 13 | */ 14 | export const lwbaEndpointInputParametersDefinition = { 15 | base: { 16 | aliases: ['from', 'coin'], 17 | type: 'string', 18 | description: 'The symbol of symbols of the currency to query', 19 | required: true, 20 | }, 21 | quote: { 22 | aliases: ['to', 'market'], 23 | type: 'string', 24 | description: 'The symbol of the currency to convert to', 25 | required: true, 26 | }, 27 | } as const satisfies LwbaEndpointInputParametersDefinition 28 | 29 | export type LwbaResponseDataFields = { 30 | Result: null 31 | Data: { 32 | mid: number 33 | bid: number 34 | ask: number 35 | } 36 | } 37 | 38 | /** 39 | * Helper type structure that contains the different types passed to the generic parameters of a PriceEndpoint 40 | */ 41 | export type LwbaEndpointGenerics = TransportGenerics & { 42 | Parameters: LwbaEndpointInputParametersDefinition 43 | Response: LwbaResponseDataFields 44 | } 45 | 46 | export const DEFAULT_LWBA_ALIASES = ['crypto-lwba', 'crypto_lwba', 'cryptolwba'] 47 | 48 | export const validateLwbaResponse = (bid?: number, mid?: number, ask?: number): string => { 49 | if (!mid || !bid || !ask) { 50 | return `Invariant violation. LWBA response must contain mid, bid and ask prices. Got: (bid: ${bid}, mid: ${mid}, ask: ${ask})` 51 | } 52 | if (mid < bid || mid > ask) { 53 | return `Invariant violation. Mid price must be between bid and ask prices. Got: (bid: ${bid}, mid: ${mid}, ask: ${ask})` 54 | } 55 | return '' 56 | } 57 | 58 | /** 59 | * An LwbaEndpoint is a specific type of AdapterEndpoint. Meant to comply with standard practices for 60 | * LWBA (lightweight bid/ask) Data Feeds, its InputParameters must extend the basic ones (base/quote). 61 | */ 62 | export class LwbaEndpoint extends AdapterEndpoint { 63 | constructor(params: AdapterEndpointParams) { 64 | if (!params.aliases) { 65 | params.aliases = [] 66 | } 67 | for (const alias of DEFAULT_LWBA_ALIASES) { 68 | if (params.name !== alias && !params.aliases.includes(alias)) { 69 | params.aliases.push(alias) 70 | } 71 | } 72 | 73 | // All LWBA requests must have a mid, bid, and ask 74 | // Response validation ensures that we meet the invariant: bid <= mid <= ask 75 | params.customOutputValidation = (output) => { 76 | const data = output.data as LwbaResponseDataFields['Data'] 77 | const error = validateLwbaResponse(data.bid, data.mid, data.ask) 78 | 79 | if (error) { 80 | throw new AdapterLWBAError({ statusCode: 500, message: error }) 81 | } 82 | 83 | return undefined 84 | } 85 | super(params) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/logger.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import { start } from '../src' 3 | import { Adapter, AdapterEndpoint } from '../src/adapter' 4 | import { AdapterConfig, SettingsDefinitionMap } from '../src/config' 5 | import CensorList from '../src/util/censor/censor-list' 6 | import { COLORS, censor, colorFactory } from '../src/util/logger' 7 | import { NopTransport } from '../src/util/testing-utils' 8 | 9 | test.before(async () => { 10 | const customSettings = { 11 | API_KEY: { 12 | description: 'Test custom env var', 13 | type: 'string', 14 | sensitive: true, 15 | }, 16 | } satisfies SettingsDefinitionMap 17 | process.env['API_KEY'] = 'mock-api-key' 18 | const config = new AdapterConfig(customSettings) 19 | const adapter = new Adapter({ 20 | name: 'TEST', 21 | config, 22 | endpoints: [ 23 | new AdapterEndpoint({ 24 | name: 'test', 25 | transport: new NopTransport(), 26 | }), 27 | ], 28 | }) 29 | await start(adapter) 30 | }) 31 | 32 | test('properly builds censor list', async (t) => { 33 | const censorList = CensorList.getAll() 34 | // eslint-disable-next-line prefer-regex-literals 35 | t.deepEqual(censorList[0], { key: 'API_KEY', value: RegExp('mock\\-api\\-key', 'gi') }) 36 | }) 37 | 38 | test('properly redacts API_KEY (string)', async (t) => { 39 | const redacted = censor('mock-api-key', CensorList.getAll()) 40 | t.is(redacted, '[API_KEY REDACTED]') 41 | }) 42 | 43 | test('properly redacts API_KEY (string with added text)', async (t) => { 44 | const redacted = censor('Bearer mock-api-key', CensorList.getAll()) 45 | t.is(redacted, 'Bearer [API_KEY REDACTED]') 46 | }) 47 | 48 | test('properly redacts API_KEY (object)', async (t) => { 49 | const redacted = censor({ apiKey: 'mock-api-key' }, CensorList.getAll()) 50 | t.deepEqual(redacted, { apiKey: '[API_KEY REDACTED]' }) 51 | }) 52 | 53 | test('properly redacts API_KEY (object with added text)', async (t) => { 54 | const redacted = censor({ apiKey: 'Bearer mock-api-key' }, CensorList.getAll()) 55 | t.deepEqual(redacted, { apiKey: 'Bearer [API_KEY REDACTED]' }) 56 | }) 57 | 58 | test('properly handles undefined property', async (t) => { 59 | const redacted = censor({ apiKey: undefined }, CensorList.getAll()) 60 | t.deepEqual(redacted, {}) 61 | }) 62 | 63 | test('properly handles undefined', async (t) => { 64 | const redacted = censor(undefined, CensorList.getAll()) 65 | t.deepEqual(redacted, undefined) 66 | }) 67 | 68 | test('properly redacts API_KEY (multiple nested values)', async (t) => { 69 | const redacted = censor( 70 | { apiKey: 'mock-api-key', config: { headers: { auth: 'mock-api-key' } } }, 71 | CensorList.getAll(), 72 | ) 73 | t.deepEqual(redacted, { 74 | apiKey: '[API_KEY REDACTED]', 75 | config: { headers: { auth: '[API_KEY REDACTED]' } }, 76 | }) 77 | }) 78 | 79 | test('Test color factory', async (t) => { 80 | const nextColor = colorFactory(COLORS) 81 | for (let i = 0; i < COLORS.length; i++) { 82 | t.is(nextColor(), COLORS[i]) 83 | } 84 | // Test that the colors cycle back 85 | t.is(nextColor(), COLORS[0]) 86 | }) 87 | 88 | test('properly handle circular references', async (t) => { 89 | const a = { 90 | b: {}, 91 | } 92 | a.b = a 93 | const log = censor(a, CensorList.getAll()) 94 | t.is(log, '[Unknown]') 95 | }) 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EA Framework v3 2 | 3 | [![NPM version](https://img.shields.io/npm/v/@chainlink/external-adapter-framework.svg?style=flat)](https://www.npmjs.com/package/@chainlink/external-adapter-framework) 4 | ![Coverage](https://img.shields.io/badge/coverage-99.1%25-green) 5 | 6 | Framework to create External adapters, microservices that serve as middleware to facilitate connections between Chainlink Nodes and Data Providers (DP). 7 | 8 | ## Requirements 9 | 10 | - Node.js 22+ 11 | - Yarn 12 | 13 | ### Optional development tools 14 | 15 | If available, consider setting up your development environment with: 16 | 17 | - ESLint 18 | - Prettier 19 | 20 | Note that both of the above are not necessary, but PRs submitted to the repo will be blocked from merging unless they comply with the linting and formatting rules. 21 | 22 | ## Setup 23 | 24 | ```sh 25 | yarn # Install yarn dependencies 26 | ``` 27 | 28 | ## Guides & Docs 29 | 30 | - [Basics](./docs/basics.md) 31 | - Components 32 | - [Adapter](./docs/components/adapter.md) 33 | - [Endpoints](./docs/components/endpoints.md) 34 | - [Tests](./docs/components/tests.md) 35 | - [Transports](./docs/components/transports.md) 36 | - Transport Types 37 | - [HTTP](./docs/components/transport-types/http-transport.md) 38 | - [WebSocket](./docs/components/transport-types/websocket-transport.md) 39 | - [SSE](./docs/components/transport-types/sse-transport.md) 40 | - [Subscription](./docs/components/transport-types/subscription-transport.md) 41 | - [Streaming](./docs/components/transport-types/streaming-transport.md) 42 | - [Custom](./docs/components/transport-types/custom-transport.md) 43 | - Guides 44 | - [Porting a v2 EA to v3](./docs/guides/porting-a-v2-ea-to-v3.md) 45 | - [Creating a new v3 EA](./docs/guides/creating-a-new-v3-ea.md) 46 | - Reference Tables 47 | - [EA Settings](./docs//reference-tables/ea-settings.md) 48 | - [Metrics](./docs/reference-tables/metrics.md) 49 | 50 | ## Testing 51 | 52 | The EA framework is tested by a suite of integration tests located [here](./test). 53 | 54 | ``` 55 | yarn test 56 | ``` 57 | 58 | ## Publishing releases 59 | 60 | ### Automatic 61 | 62 | The normal flow for publishing a release is through a series o GitHub actions that are triggered when a PR is closed by merging with the base branch. Full details about our workflows can be found in [docs](./docs/github_workflows.md). A summary of our publish workflow follows: 63 | 64 | 1. Close a PR containing your changes 65 | 2. If the PR was merged and if it contains a version label instruction (patch, minor, major, none), a new PR will be created that contains the result of running `npm version LABEL` on main with the original PR author assigned as a reviewer. 66 | 3. A link to the newly created version bump PR will be added to your original PR. Click on that link, and approve it to be merged. 67 | 4. Close the version bump PR. If merged, the package will be published to npm. 68 | 5. When the publish workflow finishes, a comment will be added to the version bump PR that tells you if it ran successfully 69 | 70 | This adds an extra step (approving a version bump PR) that has to be taken every time a PR is merged. This is annoying, but it is an effective workaround for permissions issues when running against protected branches, and eliminates the need for the PR author to manually update their branch's version by referring to main. 71 | -------------------------------------------------------------------------------- /src/debug/settings-page.ts: -------------------------------------------------------------------------------- 1 | import { DebugPageSetting } from '../util/settings' 2 | 3 | // To enable syntax highlighting in VSCode, you can download the "Comment tagged templates" extension: 4 | // https://marketplace.visualstudio.com/items?itemName=bierner.comment-tagged-templates 5 | const settingsPage = (settings: DebugPageSetting[]) => /* HTML */ ` 6 | 7 | 8 | 9 | 10 | EA Settings 11 | 41 | 42 | 50 | 51 | 52 | 53 |

EA Settings

54 |

55 | This page shows the current settings for the EA. It is intended to be used for debugging 56 | purposes, and should not be publicly accessible. 57 |

58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | ${settings 74 | .map( 75 | (setting) => /* HTML */ ` 76 | 77 | 78 | 79 | 80 | 83 | 84 | 85 | 86 | 87 | 88 | `, 89 | ) 90 | .join('')} 91 | 92 |
NameTypeDescriptionValueRequiredDefaultCustom SettingEnv Default Override
${setting.name}${setting.type}${setting.description} 81 | ${setting.value || ''} 82 | ${setting.required ? '✅' : ''}${setting.default || ''}${setting.customSetting ? '✅' : ''}${setting.envDefaultOverride || ''}
93 | 94 | 95 | ` 96 | export default settingsPage 97 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/transport/customfg.ts.ejs: -------------------------------------------------------------------------------- 1 | import { Transport, TransportDependencies } from '@chainlink/external-adapter-framework/transports' 2 | import { ResponseCache } from '@chainlink/external-adapter-framework/cache/response' 3 | import { Requester } from '@chainlink/external-adapter-framework/util/requester' 4 | import { 5 | AdapterRequest, 6 | AdapterResponse, 7 | } from '@chainlink/external-adapter-framework/util' 8 | import { BaseEndpointTypes, inputParameters } from '../endpoint/<%= inputEndpointName %>' 9 | 10 | <% if (includeComments) { -%> 11 | // CustomTransport extends base types from endpoint and adds additional, Provider-specific types (if needed). 12 | <% } -%> 13 | export type CustomTransportTypes = BaseEndpointTypes & { 14 | Provider: { 15 | RequestBody: never 16 | ResponseBody: any 17 | } 18 | } 19 | <% if (includeComments) { -%> 20 | // CustomTransport is used to perform custom data fetching and processing from a Provider. The framework provides built-in transports to 21 | // fetch data from a Provider using several protocols, including `http`, `websocket`, and `sse`. Use CustomTransport when the Provider uses 22 | // different protocol, or you need custom functionality that built-in transports don't support. For example, custom, multistep authentication 23 | // for requests, paginated requests, on-chain data retrieval using third party libraries, and so on. 24 | <% } -%> 25 | export class CustomTransport implements Transport { 26 | <% if (includeComments) { -%> 27 | // name of the transport, used for logging 28 | <% } -%> 29 | name!: string 30 | <% if (includeComments) { -%> 31 | // cache instance for caching responses from provider 32 | <% } -%> 33 | responseCache!: ResponseCache 34 | <% if (includeComments) { -%> 35 | // instance of Requester to be used for data fetching. Use this instance to perform http calls 36 | <% } -%> 37 | requester!: Requester 38 | 39 | <% if (includeComments) { -%> 40 | // REQUIRED. Transport will be automatically initialized by the framework using this method. It will be called with transport 41 | // dependencies, adapter settings, endpoint name, and transport name as arguments. Use this method to initialize transport state 42 | <% } -%> 43 | async initialize(dependencies: TransportDependencies, _adapterSettings: CustomTransportTypes['Settings'], _endpointName: string, transportName: string): Promise { 44 | this.responseCache = dependencies.responseCache 45 | this.requester = dependencies.requester 46 | this.name = transportName 47 | } 48 | <% if (includeComments) { -%> 49 | // 'foregroundExecute' performs synchronous fetch/processing of information within the lifecycle of an incoming request. It takes 50 | // request object (adapter request, which is wrapper around fastify request) and adapter settings. Use this method to handle the incoming 51 | // request, process it,save it in the cache and return to user. 52 | <% } -%> 53 | async foregroundExecute( 54 | _: AdapterRequest 55 | ): Promise> { 56 | 57 | // Custom transport logic 58 | 59 | const response = { 60 | data: { 61 | result: 100, 62 | }, 63 | statusCode: 200, 64 | result: 100, 65 | timestamps: { 66 | providerDataRequestedUnixMs: Date.now(), 67 | providerDataReceivedUnixMs: Date.now(), 68 | providerIndicatedTimeUnixMs: undefined, 69 | }, 70 | } 71 | 72 | return response 73 | } 74 | } 75 | 76 | export const customTransport = new CustomTransport() -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@chainlink/external-adapter-framework", 3 | "version": "2.11.4", 4 | "main": "dist/index.js", 5 | "license": "MIT", 6 | "repository": "git://github.com/smartcontractkit/ea-framework-js.git", 7 | "dependencies": { 8 | "ajv": "8.17.1", 9 | "axios": "1.13.2", 10 | "eventsource": "4.1.0", 11 | "fastify": "5.6.2", 12 | "ioredis": "5.8.2", 13 | "mock-socket": "9.3.1", 14 | "pino": "10.1.0", 15 | "pino-pretty": "13.1.3", 16 | "prom-client": "15.1.3", 17 | "redlock": "5.0.0-beta.2", 18 | "ws": "8.18.3", 19 | "@date-fns/tz": "1.4.1" 20 | }, 21 | "scripts": { 22 | "build": "rm -rf dist/src && mkdir -p ./dist/src && cp package.json dist/src && cp README.md dist/src && tsc && yarn pre-build-generator", 23 | "pre-build-generator": "cd scripts/generator-adapter && yarn && cd .. && cd .. && yarn build-generator", 24 | "build-generator": "mkdir -p ./dist/src/generator-adapter/generators/app/templates && cp -R scripts/generator-adapter/generators/app/templates dist/src/generator-adapter/generators/app && cp scripts/generator-adapter/package.json dist/src/generator-adapter && cp -R scripts/generator-adapter/node_modules dist/src/generator-adapter && tsc --project scripts/generator-adapter/tsconfig.json && tsc scripts/adapter-generator.ts --outDir dist/src", 25 | "generate-docs": "typedoc src/**/*.ts", 26 | "generate-ref-tables": "ts-node scripts/metrics-table.ts > docs/reference-tables/metrics.md && ts-node scripts/ea-settings-table.ts > docs/reference-tables/ea-settings.md && yarn prettier --write docs/reference-tables", 27 | "lint-fix": "eslint --max-warnings=0 --fix . && prettier --write '**/*.{ts,md,json,yaml}'", 28 | "lint": "eslint --max-warnings=0 . && prettier --check '**/*.{ts,md,json,yaml}'", 29 | "portal-path": "echo \"portal:$(readlink -f ./dist/src)\"", 30 | "test-debug": "EA_HOST=localhost LOG_LEVEL=trace DEBUG=true EA_PORT=0 c8 ava --verbose", 31 | "test": "tsc -p test/tsconfig.json --noEmit && yarn ava", 32 | "ava": "EA_HOST=localhost LOG_LEVEL=error EA_PORT=0 c8 ava", 33 | "verify": "yarn lint && yarn build && yarn build -p ./test/tsconfig.json && yarn test && yarn code-coverage", 34 | "code-coverage": "c8 check-coverage --statements 95 --lines 95 --functions 95 --branches 90" 35 | }, 36 | "bin": { 37 | "create-external-adapter": "adapter-generator.js" 38 | }, 39 | "devDependencies": { 40 | "@sinonjs/fake-timers": "15.0.0", 41 | "@types/node": "24.10.4", 42 | "@types/sinonjs__fake-timers": "15.0.1", 43 | "@types/ws": "8.18.1", 44 | "@typescript-eslint/eslint-plugin": "8.49.0", 45 | "@typescript-eslint/parser": "8.49.0", 46 | "ava": "6.4.1", 47 | "axios-mock-adapter": "2.1.0", 48 | "c8": "10.1.3", 49 | "eslint": "9.39.2", 50 | "eslint-config-prettier": "10.1.8", 51 | "eslint-plugin-tsdoc": "0.5.0", 52 | "mocksse": "1.0.4", 53 | "prettier": "3.7.4", 54 | "ts-node": "10.9.2", 55 | "ts-node-dev": "2.0.0", 56 | "typedoc": "0.28.15", 57 | "typescript": "5.9.3" 58 | }, 59 | "prettier": { 60 | "semi": false, 61 | "singleQuote": true, 62 | "printWidth": 100, 63 | "endOfLine": "auto", 64 | "trailingComma": "all", 65 | "arrowParens": "always" 66 | }, 67 | "ava": { 68 | "files": [ 69 | "test/**/*.test.ts" 70 | ], 71 | "extensions": { 72 | "mjs": true, 73 | "ts": "commonjs" 74 | }, 75 | "require": [ 76 | "ts-node/register", 77 | "./test/_force-exit.mjs" 78 | ], 79 | "workerThreads": false, 80 | "environmentVariables": { 81 | "METRICS_ENABLED": "false" 82 | }, 83 | "timeout": "20s" 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /test/metrics/labels.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import { priceEndpointInputParametersDefinition } from '../../src/adapter' 3 | import { cacheMetricsLabel } from '../../src/cache/metrics' 4 | import { AdapterSettings } from '../../src/config' 5 | import { buildHttpRequestMetricsLabel } from '../../src/metrics' 6 | import { HttpRequestType } from '../../src/metrics/constants' 7 | import { connectionErrorLabels, messageSubsLabels } from '../../src/transports/metrics' 8 | import { InputParameters } from '../../src/validation' 9 | import { AdapterError } from '../../src/validation/error' 10 | 11 | test('Generate cache label test', (t) => { 12 | const result = { 13 | participant_id: 'test-{"base":"eth","quote":"btc"}', 14 | feed_id: '{"base":"eth","quote":"btc"}', 15 | cache_type: 'local', 16 | } 17 | t.deepEqual( 18 | cacheMetricsLabel('test-{"base":"eth","quote":"btc"}', '{"base":"eth","quote":"btc"}', 'local'), 19 | result, 20 | ) 21 | }) 22 | 23 | test('Generate http request metrics label test (response status code)', (t) => { 24 | const label = buildHttpRequestMetricsLabel( 25 | 'test-{"base":"eth","quote":"btc"}', 26 | undefined, 27 | false, 28 | 400, 29 | ) 30 | const result = { 31 | feed_id: 'test-{"base":"eth","quote":"btc"}', 32 | method: 'POST', 33 | status_code: 400, 34 | provider_status_code: 200, 35 | type: HttpRequestType.DATA_PROVIDER_HIT, 36 | } 37 | t.deepEqual(label, result) 38 | }) 39 | 40 | test('Generate http request metrics label test (adapter error)', (t) => { 41 | const label = buildHttpRequestMetricsLabel( 42 | 'test-{"base":"eth","quote":"btc"}', 43 | new AdapterError({ 44 | metricsLabel: HttpRequestType.DP_ERROR, 45 | providerStatusCode: 500, 46 | statusCode: 200, 47 | }), 48 | ) 49 | const result = { 50 | feed_id: 'test-{"base":"eth","quote":"btc"}', 51 | method: 'POST', 52 | status_code: 200, 53 | type: HttpRequestType.DP_ERROR, 54 | provider_status_code: 500, 55 | } 56 | t.deepEqual(label, result) 57 | }) 58 | 59 | test('Generate http request metrics label test (generic error)', (t) => { 60 | const label = buildHttpRequestMetricsLabel('test-{"base":"eth","quote":"btc"}', new Error('Test')) 61 | const result = { 62 | feed_id: 'test-{"base":"eth","quote":"btc"}', 63 | method: 'POST', 64 | status_code: 500, 65 | type: HttpRequestType.ADAPTER_ERROR, 66 | } 67 | t.deepEqual(label, result) 68 | }) 69 | 70 | test('Generate data provider metrics label test', (t) => { 71 | const result = { 72 | participant_id: 'test-{"base":"eth","quote":"btc"}', 73 | feed_id: '{"base":"eth","quote":"btc"}', 74 | cache_type: 'local', 75 | } 76 | t.deepEqual( 77 | cacheMetricsLabel('test-{"base":"eth","quote":"btc"}', '{"base":"eth","quote":"btc"}', 'local'), 78 | result, 79 | ) 80 | }) 81 | 82 | test('Generate WS connection error label test', (t) => { 83 | const result = { 84 | message: 'error', 85 | } 86 | t.deepEqual(connectionErrorLabels('error'), result) 87 | }) 88 | 89 | test('Generate WS message and subscription label test', (t) => { 90 | const result = { 91 | feed_id: '{"base":"eth","quote":"btc"}', 92 | subscription_key: 'test-{"base":"eth","quote":"btc"}', 93 | } 94 | t.deepEqual( 95 | messageSubsLabels( 96 | { 97 | adapterSettings: {} as AdapterSettings, 98 | inputParameters: new InputParameters(priceEndpointInputParametersDefinition), 99 | endpointName: 'test', 100 | }, 101 | { 102 | base: 'ETH', 103 | quote: 'BTC', 104 | }, 105 | ), 106 | result, 107 | ) 108 | }) 109 | -------------------------------------------------------------------------------- /src/transports/abstract/subscription.ts: -------------------------------------------------------------------------------- 1 | import { Transport, TransportDependencies, TransportGenerics } from '..' 2 | import { EndpointContext } from '../../adapter' 3 | import { ResponseCache } from '../../cache/response' 4 | import { metrics } from '../../metrics' 5 | import { SubscriptionSet, censorLogs, makeLogger } from '../../util' 6 | import { AdapterRequest } from '../../util/types' 7 | import { TypeFromDefinition } from '../../validation/input-params' 8 | 9 | const logger = makeLogger('SubscriptionTransport') 10 | 11 | /** 12 | * Abstract Transport that will take incoming requests and add them to a subscription set as part 13 | * of the registration. Then it will provide those entries to the (abstract) backgroundHandler method. 14 | * 15 | * @typeParam T - all types related to the [[Transport]] 16 | */ 17 | export abstract class SubscriptionTransport implements Transport { 18 | responseCache!: ResponseCache 19 | subscriptionSet!: SubscriptionSet> 20 | subscriptionTtl!: number 21 | name!: string 22 | 23 | async initialize( 24 | dependencies: TransportDependencies, 25 | adapterSettings: T['Settings'], 26 | endpointName: string, 27 | name: string, 28 | ): Promise { 29 | this.responseCache = dependencies.responseCache 30 | this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet(endpointName, name) 31 | this.subscriptionTtl = this.getSubscriptionTtlFromConfig(adapterSettings) // Will be implemented by subclasses 32 | this.name = name 33 | } 34 | 35 | async registerRequest( 36 | req: AdapterRequest>, 37 | _: T['Settings'], 38 | ): Promise { 39 | censorLogs(() => 40 | logger.debug( 41 | `Adding entry to subscription set (ttl ${this.subscriptionTtl}): [${req.requestContext.cacheKey}] = ${req.requestContext.data}`, 42 | ), 43 | ) 44 | 45 | // This might need coalescing to avoid too frequent ttl updates 46 | await this.subscriptionSet.add( 47 | req.requestContext.data, 48 | this.subscriptionTtl, 49 | req.requestContext.cacheKey, 50 | ) 51 | } 52 | 53 | async backgroundExecute(context: EndpointContext): Promise { 54 | logger.debug('Starting background execute') 55 | 56 | const entries = await this.subscriptionSet.getAll() 57 | 58 | // Keep track of active subscriptions for background execute 59 | // Note: for those coming from reasonable OOP languages, don't fret; this is JS: 60 | // this.constructor.name will resolve to the instance name, not the class one (i.e., will use the implementing class' name) 61 | metrics 62 | .get('bgExecuteSubscriptionSetCount') 63 | .labels({ 64 | adapter_endpoint: context.endpointName, 65 | transport_type: this.constructor.name, 66 | transport: this.name, 67 | }) 68 | .set(entries.length) 69 | 70 | await this.backgroundHandler(context, entries) 71 | } 72 | 73 | /** 74 | * Handler specific to the subscription transport, that is provided all entries in the subscription set. 75 | * 76 | * @param context - background context for the execution of this handler 77 | * @param entries - all the entries in the subscription set 78 | */ 79 | abstract backgroundHandler( 80 | context: EndpointContext, 81 | entries: TypeFromDefinition[], 82 | ): Promise 83 | 84 | /** 85 | * Helper method to be defined in subclasses, for each of them to carry their own TTL definition in the EA config. 86 | * 87 | * @param adapterSettings - the config for this adapter 88 | */ 89 | abstract getSubscriptionTtlFromConfig(adapterSettings: T['Settings']): number 90 | } 91 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | # This is the entry point for CI. It will setup the application, then run lint, test, and eventually publish if not the main branch 2 | name: 'Main' 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - main 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | concurrency: build-${{ github.ref}} 14 | if: ${{ !startsWith(github.head_ref, 'version-bump') }} 15 | steps: 16 | - uses: actions/checkout@v6 17 | - uses: ./.github/actions/setup 18 | - run: yarn build 19 | lint: 20 | runs-on: ubuntu-latest 21 | concurrency: lint-${{ github.ref }} 22 | if: ${{ !startsWith(github.head_ref, 'version-bump') }} 23 | steps: 24 | - uses: actions/checkout@v6 25 | - uses: ./.github/actions/setup 26 | - run: yarn lint 27 | test: 28 | concurrency: test-${{ github.ref }} 29 | runs-on: ubuntu-latest 30 | if: ${{ !startsWith(github.head_ref, 'version-bump') }} 31 | steps: 32 | - uses: actions/checkout@v6 33 | - uses: ./.github/actions/setup 34 | - run: | 35 | # Check for test.only() in test files 36 | set +e 37 | files=$(find ${{ github.workspace }}/test -name "*.test.ts" | xargs grep -l 'test.only') 38 | set -e 39 | 40 | if [[ -n $files ]]; then 41 | echo "Error: Found /test.only/ in following test files:" 42 | echo "$files" 43 | exit 1 44 | fi 45 | - run: yarn test 46 | - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 47 | with: 48 | name: coverage-reports 49 | path: | 50 | ./**/coverage/tmp/*.json 51 | ./**/coverage/coverage-summary.json 52 | # Only run if tests have run and completed successfully 53 | code-coverage: 54 | needs: test 55 | runs-on: ubuntu-latest 56 | if: ${{ !startsWith(github.head_ref, 'version-bump') }} 57 | steps: 58 | - uses: actions/checkout@v6 59 | - uses: ./.github/actions/setup 60 | - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 61 | with: 62 | name: coverage-reports 63 | - run: yarn code-coverage 64 | - uses: ./.github/actions/set-git-credentials 65 | - name: Save coverage summary 66 | if: github.event.pull_request.number # check if the context is pull request 67 | run: | 68 | # save coverage summary in a branch 'coverage-pr-PR_NUMBER' so we can have access to it in a different workflow 69 | BRANCH="coverage-pr-${{ github.event.pull_request.number }}" 70 | git checkout -b $BRANCH 71 | git add -f coverage/coverage-summary.json 72 | git commit -m "coverage summary report" 73 | git push --set-upstream origin $BRANCH --force 74 | env: 75 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 76 | 77 | generate-example-adapter: 78 | runs-on: ubuntu-latest 79 | if: ${{ !startsWith(github.head_ref, 'version-bump') }} 80 | steps: 81 | - uses: actions/checkout@v6 82 | - uses: ./.github/actions/setup 83 | - name: Generate new adapter, build, run the tests 84 | run: | 85 | yarn build 86 | path_to_generator="$GITHUB_WORKSPACE/dist/src/generator-adapter/generators/app/index.js" 87 | npm install -g yo@5.0.0 88 | # The command bellow will generate new EA with default name 'example-adapter' in specified directory 'examples' 89 | DEBUG=* yo "$path_to_generator" examples 90 | cd examples/example-adapter 91 | yarn build 92 | yarn test 93 | env: 94 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 95 | EXTERNAL_ADAPTER_GENERATOR_NO_INTERACTIVE: 'true' 96 | EXTERNAL_ADAPTER_GENERATOR_STANDALONE: 'true' 97 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/transport/ws.ts.ejs: -------------------------------------------------------------------------------- 1 | import { WebSocketTransport } from '@chainlink/external-adapter-framework/transports' 2 | import { BaseEndpointTypes } from '../endpoint/<%= inputEndpointName %>' 3 | 4 | export interface WSResponse { 5 | success: boolean 6 | price: number 7 | base: string 8 | quote: string 9 | time: number 10 | } 11 | 12 | <% if (includeComments) { -%> 13 | // WsTransport extends base types from endpoint and adds additional, Provider-specific types like 'WsMessage', which is the type of 14 | // websocket received message 15 | <% } -%> 16 | export type WsTransportTypes = BaseEndpointTypes & { 17 | Provider: { 18 | WsMessage: WSResponse 19 | } 20 | } 21 | <% if (includeComments) { -%> 22 | // WebSocketTransport is used to fetch and process data from a Provider using Websocket protocol. 23 | <% } -%> 24 | export const wsTransport = new WebSocketTransport({ 25 | <% if (includeComments) { -%> 26 | // use `url` method to provide connection url. It accepts adapter context, so you have access to adapter config(environment variables) and 27 | // request payload if needed 28 | <% } -%> 29 | url: (context) => context.adapterSettings.WS_API_ENDPOINT, 30 | <% if (includeComments) { -%> 31 | // 'handler' contains two helpful methods. one of them is `message`. This method is called when there is a new websocket message. 32 | // The other one is 'open' method. It is called when the websocket connection is successfully opened. Use this method to execute some logic 33 | // when the connection is established (custom authentication, logging, ...) 34 | <% } -%> 35 | handlers: { 36 | <% if (includeComments) { -%> 37 | // 'message' handler receives a raw websocket message as first argument and adapter context as second and should return an array of 38 | // response objects. Use this method to construct a list of response objects, and the framework will save them in cache and return to user 39 | <% } -%> 40 | message(message) { 41 | <% if (includeComments) { -%> 42 | // in cases when error or unknown message is received, use 'return' to skip the iteration. 43 | <% } -%> 44 | if (message.success === false) { 45 | return 46 | } 47 | 48 | <% if (includeComments) { -%> 49 | // Response objects, whether successful or errors (if not skipped), contain two properties, 'params' and 'response'. 'response' is what 50 | // will be stored in the cache and returned as adapter response and 'params' determines the identifier so that the next request with 51 | // same 'params' will immediately return the response from the cache 52 | <% } -%> 53 | return [ 54 | { 55 | params: { base: message.base, quote: message.quote }, 56 | response: { 57 | result: message.price, 58 | data: { 59 | result: message.price 60 | }, 61 | timestamps: { 62 | providerIndicatedTimeUnixMs: message.time, 63 | }, 64 | }, 65 | }, 66 | ] 67 | }, 68 | }, 69 | <% if (includeComments) { -%> 70 | // `builders` are builder methods, that will be used to prepare specific WS messages to be sent to Data Provider 71 | <% } -%> 72 | builders: { 73 | <% if (includeComments) { -%> 74 | // `subscribeMessage` accepts request parameters and should construct and return a payload that will be sent to Data Provider 75 | // Use this method to subscribe to live feeds 76 | <% } -%> 77 | subscribeMessage: (params) => { 78 | return { 79 | type: 'subscribe', 80 | symbols: `${params.base}/${params.quote}`.toUpperCase() 81 | } 82 | }, 83 | <% if (includeComments) { -%> 84 | // `unsubscribeMessage` accepts request parameters and should construct and return a payload that will be sent to Data Provider 85 | // Use this method to unsubscribe from live feeds 86 | <% } -%> 87 | unsubscribeMessage: (params) => { 88 | return { 89 | type: 'unsubscribe', 90 | symbols: `${params.base}/${params.quote}`.toUpperCase() 91 | } 92 | }, 93 | }, 94 | }) 95 | -------------------------------------------------------------------------------- /src/validation/error.ts: -------------------------------------------------------------------------------- 1 | import { HttpRequestType } from '../metrics/constants' 2 | import { ResponseTimestamps } from '../util' 3 | 4 | type ErrorBasic = { 5 | name: string 6 | message: string 7 | } 8 | type ErrorFull = ErrorBasic & { 9 | stack: string 10 | cause: string 11 | } 12 | 13 | export type AdapterErrorResponse = { 14 | status: string 15 | statusCode: number 16 | providerStatusCode?: number 17 | error: ErrorBasic | ErrorFull 18 | } 19 | 20 | export class AdapterError extends Error { 21 | status: string 22 | statusCode: number 23 | override cause: unknown 24 | url?: string 25 | errorResponse: unknown 26 | feedID?: string 27 | providerStatusCode?: number 28 | metricsLabel?: HttpRequestType 29 | 30 | override name: string 31 | override message: string 32 | 33 | constructor({ 34 | status = 'errored', 35 | statusCode = 500, 36 | name = 'AdapterError', 37 | message = 'There was an unexpected error in the adapter.', 38 | cause, 39 | url, 40 | feedID, 41 | errorResponse, 42 | providerStatusCode, 43 | metricsLabel = HttpRequestType.ADAPTER_ERROR, 44 | }: Partial) { 45 | super(message) 46 | 47 | this.status = status 48 | this.statusCode = statusCode 49 | this.name = name 50 | this.message = message 51 | this.cause = cause 52 | if (url) { 53 | this.url = url 54 | } 55 | if (feedID) { 56 | this.feedID = feedID 57 | } 58 | this.errorResponse = errorResponse 59 | this.providerStatusCode = providerStatusCode 60 | this.metricsLabel = metricsLabel 61 | } 62 | 63 | toJSONResponse(): AdapterErrorResponse { 64 | const showDebugInfo = process.env['DEBUG'] === 'true' 65 | const errorBasic = { 66 | name: this.name, 67 | message: this.message, 68 | url: this.url, 69 | errorResponse: this.errorResponse, 70 | feedID: this.feedID, 71 | } 72 | const errorFull = { ...errorBasic, stack: this.stack, cause: this.cause } 73 | return { 74 | status: this.status, 75 | statusCode: this.statusCode, 76 | providerStatusCode: this.providerStatusCode, 77 | error: showDebugInfo ? errorFull : errorBasic, 78 | } 79 | } 80 | } 81 | 82 | export class AdapterInputError extends AdapterError { 83 | constructor(input: Partial) { 84 | super({ ...input, metricsLabel: HttpRequestType.INPUT_ERROR }) 85 | } 86 | } 87 | export class AdapterRateLimitError extends AdapterError { 88 | msUntilNextExecution: number 89 | 90 | constructor( 91 | input: Partial & { 92 | msUntilNextExecution?: number 93 | }, 94 | ) { 95 | super({ ...input, metricsLabel: HttpRequestType.RATE_LIMIT_ERROR }) 96 | this.msUntilNextExecution = input.msUntilNextExecution ?? 0 97 | } 98 | } 99 | export class AdapterTimeoutError extends AdapterError { 100 | constructor(input: Partial) { 101 | super({ ...input, metricsLabel: HttpRequestType.TIMEOUT_ERROR }) 102 | } 103 | } 104 | export class AdapterDataProviderError extends AdapterError { 105 | constructor( 106 | input: Partial, 107 | public timestamps: ResponseTimestamps, 108 | ) { 109 | super({ ...input, metricsLabel: HttpRequestType.DP_ERROR }) 110 | } 111 | } 112 | export class AdapterConnectionError extends AdapterError { 113 | constructor( 114 | input: Partial, 115 | public timestamps: ResponseTimestamps, 116 | ) { 117 | super({ ...input, metricsLabel: HttpRequestType.CONNECTION_ERROR }) 118 | } 119 | } 120 | export class AdapterCustomError extends AdapterError { 121 | constructor(input: Partial) { 122 | super({ ...input, metricsLabel: HttpRequestType.CUSTOM_ERROR }) 123 | } 124 | } 125 | 126 | export class AdapterLWBAError extends AdapterError { 127 | constructor(input: Partial) { 128 | super({ 129 | ...input, 130 | name: 'AdapterLWBAError', 131 | metricsLabel: HttpRequestType.LWBA_ERROR, 132 | }) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/transport/http.ts.ejs: -------------------------------------------------------------------------------- 1 | import { HttpTransport } from '@chainlink/external-adapter-framework/transports' 2 | import { BaseEndpointTypes } from '../endpoint/<%= inputEndpointName %>' 3 | 4 | export interface ResponseSchema { 5 | [key: string]: { 6 | price: number 7 | errorMessage?: string 8 | } 9 | } 10 | 11 | <% if (includeComments) { -%> 12 | // HttpTransport extends base types from endpoint and adds additional, Provider-specific types like 'RequestBody', which is the type of 13 | // request body (not the request to adapter, but the request that adapter sends to Data Provider), and 'ResponseBody' which is 14 | // the type of raw response from Data Provider 15 | <% } -%> 16 | export type HttpTransportTypes = BaseEndpointTypes & { 17 | Provider: { 18 | RequestBody: never 19 | ResponseBody: ResponseSchema 20 | } 21 | } 22 | <% if (includeComments) { -%> 23 | // HttpTransport is used to fetch and process data from a Provider using HTTP(S) protocol. It usually needs two methods 24 | // `prepareRequests` and `parseResponse` 25 | <% } -%> 26 | export const httpTransport = new HttpTransport({ 27 | <% if (includeComments) { -%> 28 | // `prepareRequests` method receives request payloads sent to associated endpoint alongside adapter config(environment variables) 29 | // and should return 'request information' to the Data Provider. Use this method to construct one or many requests, and the framework 30 | // will send them to Data Provider 31 | <% } -%> 32 | prepareRequests: (params, config) => { 33 | return params.map((param) => { 34 | return { 35 | <% if (includeComments) { -%> 36 | // `params` are parameters associated to this single request and will also be available in the 'parseResponse' method. 37 | <% } -%> 38 | params: [param], 39 | <% if (includeComments) { -%> 40 | // `request` contains any valid axios request configuration 41 | <% } -%> 42 | request: { 43 | baseURL: config.API_ENDPOINT, 44 | url: '/cryptocurrency/price', 45 | headers: { 46 | 'X_API_KEY': config.API_KEY, 47 | }, 48 | params: { 49 | symbol: param.base.toUpperCase(), 50 | convert: param.quote.toUpperCase(), 51 | }, 52 | }, 53 | } 54 | }) 55 | }, 56 | <% if (includeComments) { -%> 57 | // `parseResponse` takes the 'params' specified in the `prepareRequests` and the 'response' from Data Provider and should return 58 | // an array of response objects to be stored in cache. Use this method to construct a list of response objects for every parameter in 'params' 59 | // and the framework will save them in cache and return to user 60 | <% } -%> 61 | parseResponse: (params, response) => { 62 | <% if (includeComments) { -%> 63 | // In case error was received, it's a good practice to return meaningful information to user 64 | <% } -%> 65 | if (!response.data) { 66 | return params.map((param) => { 67 | return { 68 | params: param, 69 | response: { 70 | errorMessage: `The data provider didn't return any value for ${param.base}/${param.quote}`, 71 | statusCode: 502, 72 | }, 73 | } 74 | }) 75 | } 76 | 77 | <% if (includeComments) { -%> 78 | // For successful responses for each 'param' a new response object is created and returned as an array 79 | <% } -%> 80 | return params.map((param) => { 81 | const result = response.data[param.base.toUpperCase()].price 82 | <% if (includeComments) { -%> 83 | // Response objects, whether successful or errors, contain two properties, 'params' and 'response'. 'response' is what will be 84 | // stored in the cache and returned as adapter response and 'params' determines the identifier so that the next request with same 'params' 85 | // will immediately return the response from the cache 86 | <% } -%> 87 | return { 88 | params: param, 89 | response: { 90 | result, 91 | data: { 92 | result 93 | } 94 | }, 95 | } 96 | }) 97 | }, 98 | }) 99 | -------------------------------------------------------------------------------- /test/metrics/warmer-metrics.test.ts: -------------------------------------------------------------------------------- 1 | import { InstalledClock } from '@sinonjs/fake-timers' 2 | import { installTimers } from '../helper' 3 | import untypedTest, { TestFn } from 'ava' 4 | import axios from 'axios' 5 | import MockAdapter from 'axios-mock-adapter' 6 | import { runAllUntilTime, TestAdapter } from '../../src/util/testing-utils' 7 | import { buildHttpAdapter } from './helper' 8 | 9 | const test = untypedTest as TestFn<{ 10 | testAdapter: TestAdapter 11 | clock: InstalledClock 12 | }> 13 | 14 | const URL = 'http://test-url.com' 15 | const endpoint = '/price' 16 | const axiosMock = new MockAdapter(axios) 17 | 18 | test.before(async (t) => { 19 | process.env['METRICS_ENABLED'] = 'true' 20 | // Disable retries to make the testing flow easier 21 | process.env['CACHE_POLLING_MAX_RETRIES'] = '0' 22 | // So that we don't have to wait that long in the test for the subscription to expire 23 | process.env['WARMUP_SUBSCRIPTION_TTL'] = '5000' 24 | // So that we don't see errors from the mocked clock running until axios' http timeout timer 25 | process.env['API_TIMEOUT'] = '0' 26 | process.env['RATE_LIMIT_CAPACITY_SECOND'] = '1' 27 | process.env['CACHE_MAX_AGE'] = '2000' 28 | 29 | const adapter = buildHttpAdapter() 30 | 31 | // Start the adapter 32 | t.context.clock = installTimers() 33 | t.context.testAdapter = await TestAdapter.startWithMockedCache(adapter, t.context) 34 | }) 35 | 36 | test.after((t) => { 37 | axiosMock.reset() 38 | t.context.clock.uninstall() 39 | }) 40 | 41 | const from = 'ETH' 42 | const to = 'USD' 43 | const price = 1234 44 | 45 | test.serial('Test cache warmer active metric', async (t) => { 46 | axiosMock 47 | .onPost(URL + endpoint, { 48 | pairs: [ 49 | { 50 | base: from, 51 | quote: to, 52 | }, 53 | ], 54 | }) 55 | .reply(() => { 56 | t.context.clock.tick(1) 57 | return [ 58 | 200, 59 | { 60 | prices: [ 61 | { 62 | pair: `${from}/${to}`, 63 | price, 64 | }, 65 | ], 66 | }, 67 | ] 68 | }) 69 | 70 | await t.context.testAdapter.startBackgroundExecuteThenGetResponse(t, { 71 | requestData: { 72 | from, 73 | to, 74 | }, 75 | }) 76 | 77 | let metrics = await t.context.testAdapter.getMetrics() 78 | metrics.assert(t, { 79 | name: 'cache_warmer_get_count', 80 | labels: { isBatched: 'true' }, 81 | expectedValue: 1, 82 | }) 83 | metrics.assert(t, { 84 | name: 'bg_execute_total', 85 | labels: { adapter_endpoint: 'test', transport: 'default_single_transport' }, 86 | expectedValue: 2, 87 | }) 88 | metrics.assert(t, { 89 | name: 'bg_execute_subscription_set_count', 90 | labels: { 91 | adapter_endpoint: 'test', 92 | transport_type: 'MockHttpTransport', 93 | transport: 'default_single_transport', 94 | }, 95 | expectedValue: 1, 96 | }) 97 | metrics.assertPositiveNumber(t, { 98 | name: 'bg_execute_duration_seconds', 99 | labels: { adapter_endpoint: 'test', transport: 'default_single_transport' }, 100 | }) 101 | 102 | // Wait until the cache expires, and the subscription is out 103 | await runAllUntilTime(t.context.clock, 10000) // The provider response is slower 104 | 105 | // Now that the cache is out and the subscription no longer there, this should time out 106 | const error2 = await t.context.testAdapter.request({ 107 | from, 108 | to, 109 | }) 110 | t.is(error2?.statusCode, 504) 111 | 112 | metrics = await t.context.testAdapter.getMetrics() 113 | metrics.assert(t, { 114 | name: 'cache_warmer_get_count', 115 | labels: { isBatched: 'true' }, 116 | expectedValue: 0, 117 | }) 118 | metrics.assert(t, { 119 | name: 'bg_execute_total', 120 | labels: { adapter_endpoint: 'test', transport: 'default_single_transport' }, 121 | expectedValue: 12, 122 | }) 123 | metrics.assert(t, { 124 | name: 'bg_execute_subscription_set_count', 125 | labels: { 126 | adapter_endpoint: 'test', 127 | transport_type: 'MockHttpTransport', 128 | transport: 'default_single_transport', 129 | }, 130 | expectedValue: 0, 131 | }) 132 | }) 133 | -------------------------------------------------------------------------------- /src/background-executor.ts: -------------------------------------------------------------------------------- 1 | import { Adapter, AdapterEndpoint, EndpointContext, EndpointGenerics } from './adapter' 2 | import { metrics } from './metrics' 3 | import { Transport, TransportGenerics } from './transports' 4 | import { asyncLocalStorage, censorLogs, makeLogger, timeoutPromise } from './util' 5 | 6 | const logger = makeLogger('BackgroundExecutor') 7 | 8 | /** 9 | * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports. 10 | * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution. 11 | * 12 | * @param adapter - an initialized External Adapter 13 | * @param server - the http server to attach an on close listener to 14 | */ 15 | export async function callBackgroundExecutes(adapter: Adapter, apiShutdownPromise?: Promise) { 16 | // Set up variable to check later on to see if we need to stop this background "thread" 17 | // If no server is provided, the listener won't be set and serverClosed will always be false 18 | let serverClosed = false 19 | 20 | const timeoutsMap: { 21 | [endpointName: string]: NodeJS.Timeout 22 | } = {} 23 | 24 | apiShutdownPromise?.then(() => { 25 | serverClosed = true 26 | for (const endpointName in timeoutsMap) { 27 | logger.debug(`Clearing timeout for endpoint "${endpointName}"`) 28 | timeoutsMap[endpointName].unref() 29 | clearTimeout(timeoutsMap[endpointName]) 30 | } 31 | }) 32 | 33 | // Checks if an individual transport has a backgroundExecute function, and executes it if it does 34 | const callBackgroundExecute = ( 35 | endpoint: AdapterEndpoint, 36 | transport: Transport, 37 | transportName?: string, 38 | ) => { 39 | const backgroundExecute = transport.backgroundExecute?.bind(transport) 40 | if (!backgroundExecute) { 41 | logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`) 42 | return 43 | } 44 | 45 | const context: EndpointContext = { 46 | endpointName: endpoint.name, 47 | inputParameters: endpoint.inputParameters, 48 | adapterSettings: adapter.config.settings, 49 | } 50 | 51 | const handler = async () => { 52 | if (serverClosed) { 53 | logger.info('Server closed, stopping recursive backgroundExecute handler chain') 54 | return 55 | } 56 | 57 | // Count number of background executions per endpoint 58 | metrics 59 | .get('bgExecuteTotal') 60 | .labels({ adapter_endpoint: endpoint.name, transport: transportName }) 61 | .inc() 62 | 63 | // Time the duration of the background execute process excluding sleep time 64 | const metricsTimer = metrics 65 | .get('bgExecuteDurationSeconds') 66 | .labels({ adapter_endpoint: endpoint.name, transport: transportName }) 67 | .startTimer() 68 | 69 | logger.debug(`Calling background execute for endpoint "${endpoint.name}"`) 70 | 71 | try { 72 | await timeoutPromise( 73 | 'Background Execute', 74 | asyncLocalStorage.run( 75 | { 76 | correlationId: `Endpoint: ${endpoint.name} - Transport: ${transport.constructor.name}`, 77 | }, 78 | () => { 79 | return backgroundExecute(context) 80 | }, 81 | ), 82 | adapter.config.settings.BACKGROUND_EXECUTE_TIMEOUT, 83 | ) 84 | } catch (error) { 85 | censorLogs(() => logger.error(error, (error as Error).stack)) 86 | metrics 87 | .get('bgExecuteErrors') 88 | .labels({ adapter_endpoint: endpoint.name, transport: transportName }) 89 | .inc() 90 | } 91 | 92 | // This background execute loop is no longer the one to determine the sleep between bg execute calls. 93 | // That is now instead responsibility of each transport, to allow for custom ones to implement their own timings. 94 | logger.trace( 95 | `Finished background execute for endpoint "${endpoint.name}", calling it again in 10ms...`, 96 | ) 97 | metricsTimer() 98 | timeoutsMap[endpoint.name] = setTimeout(handler, 10) // 10ms 99 | } 100 | 101 | // Start recursive async calls 102 | handler() 103 | } 104 | 105 | for (const endpoint of adapter.endpoints) { 106 | for (const [transportName, transport] of endpoint.transportRoutes.entries()) { 107 | callBackgroundExecute(endpoint, transport, transportName) 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/transports/abstract/streaming.ts: -------------------------------------------------------------------------------- 1 | import { TransportGenerics } from '..' 2 | import { EndpointContext } from '../../adapter' 3 | import { censorLogs, makeLogger } from '../../util' 4 | import { TypeFromDefinition } from '../../validation/input-params' 5 | import { SubscriptionTransport } from './subscription' 6 | import { metrics } from '../../metrics' 7 | 8 | const logger = makeLogger('StreamingTransport') 9 | 10 | /** 11 | * Object to carry details about the current subscriptions for a StreamingTransport. 12 | */ 13 | export type SubscriptionDeltas = { 14 | /** All the subscriptions that are valid at this point in time */ 15 | desired: T[] 16 | 17 | /** The subscriptions that have not been processed yet (also included in the desired property) */ 18 | new: T[] 19 | 20 | /** Subscriptions that have expired from the subscription set */ 21 | stale: T[] 22 | } 23 | 24 | /** 25 | * Abstract Transport that will take incoming requests and add them to a subscription set as part 26 | * of the registration. It also defines an abstract stream handler method, that will be called by the 27 | * background execute and provided with calculated subscription deltas. 28 | * 29 | * @typeParam T - all types related to the [[Transport]] 30 | */ 31 | export abstract class StreamingTransport< 32 | const T extends TransportGenerics, 33 | > extends SubscriptionTransport { 34 | // The double sets serve to create a simple polling mechanism instead of needing a subscription 35 | // This one would not; this is always local state 36 | localSubscriptions: TypeFromDefinition[] = [] 37 | 38 | // This only tracks when the connection was established, not when the subscription was requested 39 | providerDataStreamEstablished = 0 40 | 41 | retryCount = 0 42 | retryTime = 0 43 | 44 | override async backgroundHandler( 45 | context: EndpointContext, 46 | desiredSubs: TypeFromDefinition[], 47 | ): Promise { 48 | if (this.retryTime > Date.now()) { 49 | return 50 | } 51 | 52 | logger.debug('Generating delta (subscribes & unsubscribes)') 53 | 54 | const desiredSubsSet = new Set(desiredSubs.map((s) => JSON.stringify(s))) 55 | const localSubscriptionsSet = new Set(this.localSubscriptions.map((s) => JSON.stringify(s))) 56 | 57 | const subscriptions = { 58 | desired: desiredSubs, 59 | new: desiredSubs.filter((s) => !localSubscriptionsSet.has(JSON.stringify(s))), 60 | stale: this.localSubscriptions.filter((s) => !desiredSubsSet.has(JSON.stringify(s))), 61 | } 62 | 63 | logger.debug( 64 | `${subscriptions.new.length} new subscriptions; ${subscriptions.stale.length} to unsubscribe`, 65 | ) 66 | if (subscriptions.new.length) { 67 | censorLogs(() => logger.trace(`Will subscribe to: ${JSON.stringify(subscriptions.new)}`)) 68 | } 69 | if (subscriptions.stale.length) { 70 | censorLogs(() => logger.trace(`Will unsubscribe to: ${JSON.stringify(subscriptions.stale)}`)) 71 | } 72 | 73 | try { 74 | await this.streamHandler(context, subscriptions) 75 | this.retryCount = 0 76 | } catch (error) { 77 | censorLogs(() => logger.error(error, (error as Error).stack)) 78 | metrics 79 | .get('streamHandlerErrors') 80 | .labels({ adapter_endpoint: context.endpointName, transport: this.name }) 81 | .inc() 82 | const timeout = Math.min( 83 | context.adapterSettings.STREAM_HANDLER_RETRY_MIN_MS * 84 | context.adapterSettings.STREAM_HANDLER_RETRY_EXP_FACTOR ** this.retryCount, 85 | context.adapterSettings.STREAM_HANDLER_RETRY_MAX_MS, 86 | ) 87 | this.retryTime = Date.now() + timeout 88 | this.retryCount += 1 89 | logger.info(`Waiting ${timeout}ms before backgroundHandler retry #${this.retryCount}`) 90 | return 91 | } 92 | 93 | logger.debug('Setting local state to subscription set value') 94 | this.localSubscriptions = desiredSubs 95 | 96 | return 97 | } 98 | 99 | /** 100 | * Abstract method that will be provided with context and subscription details, and should take care of 101 | * handling the connection and messages sent to whatever streaming source is used. 102 | * 103 | * @param context - the context related to this background execution 104 | * @param subscriptions - object containing details for the desired, new, and stale subscriptions 105 | */ 106 | abstract streamHandler( 107 | context: EndpointContext, 108 | subscriptions: SubscriptionDeltas>, 109 | ): Promise 110 | } 111 | -------------------------------------------------------------------------------- /src/rate-limiting/burst.ts: -------------------------------------------------------------------------------- 1 | import { AdapterRateLimitTier, RateLimiter } from '.' 2 | import { AdapterEndpoint, EndpointGenerics } from '../adapter' 3 | import { makeLogger, sleep } from '../util' 4 | 5 | const logger = makeLogger('BurstRateLimiter') 6 | 7 | /** 8 | * This rate limiter is the simplest stateful option. 9 | * It will keep track of used API credits (by default, one request equals one credit) and 10 | * upon reaching the limits in a fixed time window, will block the requests until API credits are available again. 11 | * On startup, it'll compare the different thresholds for each tier, calculate them all 12 | * in the finest window we'll use (seconds), and use the most restrictive one. 13 | * This is so if the EA were to restart, we don't need to worry about persisting state 14 | * for things like daily quotas. The downside is that this does not work well for bursty 15 | * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low. 16 | */ 17 | export class BurstRateLimiter implements RateLimiter { 18 | latestSecondInterval = 0 19 | creditsThisSecond = 0 20 | latestMinuteInterval = 0 21 | creditsThisMinute = 0 22 | perSecondLimit!: number 23 | perMinuteLimit!: number 24 | 25 | initialize( 26 | endpoints: AdapterEndpoint[], 27 | limits?: AdapterRateLimitTier, 28 | ) { 29 | // Translate the hourly limit into reqs per minute 30 | const perHourLimit = (limits?.rateLimit1h || Infinity) / 60 31 | this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit) 32 | this.perSecondLimit = limits?.rateLimit1s || Infinity 33 | logger.debug( 34 | `Using API credit limit settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`, 35 | ) 36 | return this 37 | } 38 | 39 | private updateIntervals() { 40 | const now = Date.now() 41 | const nearestSecondInterval = Math.floor(now / 1000) 42 | const nearestMinuteInterval = Math.floor(now / (1000 * 60)) 43 | const nextSecondInterval = (nearestSecondInterval + 1) * 1000 44 | const nextMinuteInterval = (nearestMinuteInterval + 1) * 1000 * 60 45 | 46 | // This should always run to completion, even if it doesn't look atomic; therefore the 47 | // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness. 48 | if (nearestSecondInterval !== this.latestSecondInterval) { 49 | logger.trace( 50 | `Clearing latest second interval, # of credits logged was: ${this.creditsThisSecond} `, 51 | ) 52 | this.latestSecondInterval = nearestSecondInterval 53 | this.creditsThisSecond = 0 54 | } 55 | 56 | if (nearestMinuteInterval !== this.latestMinuteInterval) { 57 | logger.trace( 58 | `Clearing latest second minute, # of credits logged was: ${this.creditsThisMinute} `, 59 | ) 60 | this.latestMinuteInterval = nearestMinuteInterval 61 | this.creditsThisMinute = 0 62 | } 63 | 64 | return { 65 | now, 66 | nextSecondInterval, 67 | nextMinuteInterval, 68 | } 69 | } 70 | 71 | msUntilNextExecution(): number { 72 | // If the limit is set to infinity, there was no tier limit specified 73 | if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) { 74 | return 0 75 | } 76 | 77 | const { now, nextSecondInterval, nextMinuteInterval } = this.updateIntervals() 78 | 79 | const timeToWaitForNextSecond = 80 | this.creditsThisSecond < this.perSecondLimit ? 0 : nextSecondInterval - now 81 | const timeToWaitForNextMinute = 82 | this.creditsThisMinute < this.perMinuteLimit ? 0 : nextMinuteInterval - now 83 | const timeToWait = Math.max(timeToWaitForNextSecond, timeToWaitForNextMinute) 84 | 85 | return timeToWait 86 | } 87 | 88 | async waitForRateLimit(creditCost = 1): Promise { 89 | const timeToWait = this.msUntilNextExecution() 90 | 91 | if (timeToWait === 0) { 92 | logger.trace( 93 | `API credits under limits, current count: (S = ${this.creditsThisSecond} | M = ${this.creditsThisMinute}, | C = ${creditCost})`, 94 | ) 95 | } else { 96 | logger.trace( 97 | `Capacity for provider API credits has been reached this interval (S = ${this.creditsThisSecond} | M = ${this.creditsThisMinute} | C = ${creditCost}), need to wait ${timeToWait}ms`, 98 | ) 99 | await sleep(timeToWait) 100 | this.updateIntervals() 101 | } 102 | this.creditsThisSecond += creditCost 103 | this.creditsThisMinute += creditCost 104 | 105 | logger.trace( 106 | `Request is now ready to go, updated count: (S = ${this.creditsThisSecond} | M = ${this.creditsThisMinute})`, 107 | ) 108 | 109 | return 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /test/cache/local.test.ts: -------------------------------------------------------------------------------- 1 | import { InstalledClock } from '@sinonjs/fake-timers' 2 | import { installTimers } from '../helper' 3 | import untypedTest, { TestFn } from 'ava' 4 | import { Adapter, AdapterDependencies, AdapterEndpoint } from '../../src/adapter' 5 | import { Cache, CacheFactory, LocalCache } from '../../src/cache' 6 | import { AdapterConfig } from '../../src/config' 7 | import { PartialAdapterResponse } from '../../src/util' 8 | import { NopTransport, TestAdapter, runAllUntilTime } from '../../src/util/testing-utils' 9 | import { BasicCacheSetterTransport, cacheTestInputParameters } from './helper' 10 | 11 | const test = untypedTest as TestFn<{ 12 | clock: InstalledClock 13 | testAdapter: TestAdapter 14 | cache: Cache 15 | }> 16 | 17 | test.before(async (t) => { 18 | t.context.clock = installTimers() 19 | }) 20 | 21 | test.beforeEach(async (t) => { 22 | const config = new AdapterConfig( 23 | {}, 24 | { 25 | envDefaultOverrides: { 26 | CACHE_POLLING_SLEEP_MS: 10, 27 | CACHE_POLLING_MAX_RETRIES: 3, 28 | }, 29 | }, 30 | ) 31 | const adapter = new Adapter({ 32 | name: 'TEST', 33 | defaultEndpoint: 'test', 34 | config, 35 | endpoints: [ 36 | new AdapterEndpoint({ 37 | name: 'test', 38 | inputParameters: cacheTestInputParameters, 39 | transport: new BasicCacheSetterTransport(), 40 | }), 41 | new AdapterEndpoint({ 42 | name: 'nowork', 43 | transport: new NopTransport(), 44 | }), 45 | ], 46 | }) 47 | 48 | const cache = new LocalCache(adapter.config.settings.CACHE_MAX_ITEMS) 49 | const dependencies: Partial = { 50 | cache, 51 | } 52 | 53 | t.context.cache = cache 54 | t.context.testAdapter = await TestAdapter.start(adapter, t.context, dependencies) 55 | }) 56 | 57 | test.afterEach((t) => { 58 | t.context.clock.reset() 59 | }) 60 | 61 | test.serial('Test cache factory success (redis)', async (t) => { 62 | try { 63 | CacheFactory.buildCache({ cacheType: 'local', maxSizeForLocalCache: 10000 }) 64 | t.pass() 65 | } catch (_) { 66 | t.fail() 67 | } 68 | }) 69 | 70 | test.serial('Test local cache max size', async (t) => { 71 | const cache = CacheFactory.buildCache({ 72 | cacheType: 'local', 73 | maxSizeForLocalCache: 3, 74 | }) as LocalCache 75 | await cache.set('1', 1, 10000) 76 | await cache.set('2', 2, 10000) 77 | await cache.set('3', 3, 10000) 78 | await cache.set('4', 4, 10000) 79 | 80 | const value1 = await cache.get('1') 81 | t.is(value1, undefined) 82 | const value2 = await cache.get('2') 83 | t.is(value2, 2) 84 | const value3 = await cache.get('3') 85 | t.is(value3, 3) 86 | const value4 = await cache.get('4') 87 | t.is(value4, 4) 88 | }) 89 | 90 | test.serial('error responses are not overwriting successful cache entries', async (t) => { 91 | const cache = CacheFactory.buildCache({ 92 | cacheType: 'local', 93 | maxSizeForLocalCache: 10000, 94 | }) as LocalCache 95 | const cacheKey = 'KEY' 96 | const successResponse = { result: 1, data: { result: 1 } } 97 | const errorResponse = { errorMessage: 'Error', statusCode: 500 } 98 | 99 | await cache.set(cacheKey, successResponse, 10000) 100 | const value1 = await cache.get(cacheKey) 101 | t.is(value1, successResponse) 102 | 103 | await cache.set(cacheKey, errorResponse, 10000) 104 | const value2 = await cache.get(cacheKey) 105 | t.is(value2, successResponse) 106 | 107 | await runAllUntilTime(t.context.clock, 11000) 108 | 109 | await cache.set(cacheKey, errorResponse, 10000) 110 | const value3 = await cache.get(cacheKey) 111 | t.is(value3, errorResponse) 112 | 113 | await runAllUntilTime(t.context.clock, 11000) 114 | 115 | const errorResponse2 = { errorMessage: 'Error2', statusCode: 500 } 116 | await cache.set(cacheKey, errorResponse, 10000) 117 | await cache.set(cacheKey, errorResponse2, 10000) 118 | const value4 = await cache.get(cacheKey) 119 | t.is(value4, errorResponse2) 120 | }) 121 | 122 | test.serial('Test setting ttl for a cached value', async (t) => { 123 | const cache = CacheFactory.buildCache({ 124 | cacheType: 'local', 125 | maxSizeForLocalCache: 10000, 126 | }) as LocalCache 127 | const cacheKey = 'KEY' 128 | const response = { result: 1, data: { result: 1 } } 129 | 130 | await cache.set(cacheKey, response, 10000) 131 | await cache.setTTL(cacheKey, 20000) 132 | await runAllUntilTime(t.context.clock, 15000) 133 | let value = await cache.get(cacheKey) 134 | t.is(value, response) 135 | // Advance the clock an additional 6000 ms so that cache expires 136 | await runAllUntilTime(t.context.clock, 6000) 137 | value = await cache.get(cacheKey) 138 | t.is(value, undefined) 139 | }) 140 | -------------------------------------------------------------------------------- /docs/components/tests.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | Define tests in the `/test` folder. Separate folder can be created for `integration`, `unit`, and `e2e` tests within. 4 | 5 | ### Integration Tests 6 | 7 | Integration tests can be setup with the following structure. 8 | 9 | ``` 10 | test 11 | ├─ integration 12 | │ ├─ __snapshots__ 13 | | | ├─ adapter.test.ts.snap // Contains snapshot for all test responses 14 | │ ├─ adapter.test.ts // Contains the integration tests 15 | | └─ fixture.ts // Contains the nocks for DP APIs and mock WS server for WS tests 16 | ``` 17 | 18 | Use nock for DP API mocks, and run tests with Jest where you compare outputs with snapshots. 19 | 20 | You should be running integration tests without metrics, and the tests should support the EA running on any arbitrary port. 21 | 22 | #### HTTP 23 | 24 | The following is an example of HTTP transport integration tests (adapter.test.ts) 25 | 26 | ```typescript 27 | import { 28 | TestAdapter, 29 | setEnvVariables, 30 | } from '@chainlink/external-adapter-framework/util/testing-utils' 31 | import * as nock from 'nock' 32 | import { mockResponseSuccess } from './fixtures' 33 | 34 | describe('execute', () => { 35 | let spy: jest.SpyInstance 36 | let testAdapter: TestAdapter 37 | let oldEnv: NodeJS.ProcessEnv 38 | 39 | beforeAll(async () => { 40 | oldEnv = JSON.parse(JSON.stringify(process.env)) 41 | const mockDate = new Date('2022-01-01T11:11:11.111Z') 42 | spy = jest.spyOn(Date, 'now').mockReturnValue(mockDate.getTime()) 43 | 44 | const adapter = (await import('./../../src')).adapter 45 | adapter.rateLimiting = undefined 46 | testAdapter = await TestAdapter.startWithMockedCache(adapter, { 47 | testAdapter: {} as TestAdapter, 48 | }) 49 | }) 50 | 51 | afterAll(async () => { 52 | setEnvVariables(oldEnv) 53 | await testAdapter.api.close() 54 | nock.restore() 55 | nock.cleanAll() 56 | spy.mockRestore() 57 | }) 58 | 59 | describe('test endpont', () => { 60 | it('should return success', async () => { 61 | const data = { 62 | base: 'ETH', 63 | quote: 'USD', 64 | } 65 | mockResponseSuccess() 66 | const response = await testAdapter.request(data) 67 | expect(response.statusCode).toBe(200) 68 | expect(response.json()).toMatchSnapshot() 69 | }) 70 | }) 71 | }) 72 | ``` 73 | 74 | #### Websocket 75 | 76 | The following is an example of Websocket transport integration tests (adapter-ws.test.ts) 77 | 78 | ```typescript 79 | import { WebSocketClassProvider } from '@chainlink/external-adapter-framework/transports' 80 | import { 81 | TestAdapter, 82 | setEnvVariables, 83 | mockWebSocketProvider, 84 | MockWebsocketServer, 85 | } from '@chainlink/external-adapter-framework/util/testing-utils' 86 | import FakeTimers from '@sinonjs/fake-timers' 87 | import { Adapter } from '@chainlink/external-adapter-framework/adapter' 88 | 89 | // Usually this function is inside fixtures.ts 90 | const mockCryptoWebSocketServer = (URL: string): MockWebsocketServer => { 91 | const mockWsServer = new MockWebsocketServer(URL, { mock: false }) 92 | mockWsServer.on('connection', (socket) => { 93 | socket.on('message', () => { 94 | socket.send(JSON.stringify(mockCryptoResponse)) 95 | }) 96 | }) 97 | return mockWsServer 98 | } 99 | 100 | describe('websocket', () => { 101 | let mockWsServer: MockWebsocketServer | undefined 102 | let testAdapter: TestAdapter 103 | const wsEndpoint = 'ws://localhost:9090' 104 | let oldEnv: NodeJS.ProcessEnv 105 | const data = { 106 | // Adapter request data, i.e. {base: 'ETH', quote: 'USD'} 107 | } 108 | 109 | beforeAll(async () => { 110 | oldEnv = JSON.parse(JSON.stringify(process.env)) 111 | process.env['METRICS_ENABLED'] = 'false' 112 | process.env['WS_API_ENDPOINT'] = wsEndpoint 113 | 114 | mockWebSocketProvider(WebSocketClassProvider) 115 | mockWsServer = mockCryptoWebSocketServer(wsEndpoint) 116 | 117 | const adapter = (await import('./../../src')).adapter 118 | testAdapter = await TestAdapter.startWithMockedCache(adapter, { 119 | clock: FakeTimers.install(), 120 | testAdapter: {} as TestAdapter, 121 | }) 122 | 123 | // Send initial request to start background execute and wait for cache to be filled with results 124 | await testAdapter.request(data) 125 | await testAdapter.waitForCache(1) 126 | }) 127 | 128 | afterAll(async () => { 129 | setEnvVariables(oldEnv) 130 | mockWsServer?.close() 131 | testAdapter.clock?.uninstall() 132 | await testAdapter.api.close() 133 | }) 134 | 135 | describe('crypto endpoint', () => { 136 | it('should return success', async () => { 137 | const response = await testAdapter.request(data) 138 | expect(response.json()).toMatchSnapshot() 139 | }) 140 | }) 141 | }) 142 | ``` 143 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import typescriptEslint from "@typescript-eslint/eslint-plugin"; 2 | import tsdoc from "eslint-plugin-tsdoc"; 3 | import tsParser from "@typescript-eslint/parser"; 4 | import path from "node:path"; 5 | import { fileURLToPath } from "node:url"; 6 | import js from "@eslint/js"; 7 | import { FlatCompat } from "@eslint/eslintrc"; 8 | 9 | const __filename = fileURLToPath(import.meta.url); 10 | const __dirname = path.dirname(__filename); 11 | const compat = new FlatCompat({ 12 | baseDirectory: __dirname, 13 | recommendedConfig: js.configs.recommended, 14 | allConfig: js.configs.all 15 | }); 16 | 17 | export default [{ 18 | ignores: [ 19 | "**/node_modules", 20 | "**/dist", 21 | "**/coverage", 22 | "scripts/generator-adapter", 23 | "**/.yarn", 24 | "**/.vscode", 25 | ], 26 | }, ...compat.extends("eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier"), { 27 | plugins: { 28 | "@typescript-eslint": typescriptEslint, 29 | tsdoc, 30 | }, 31 | 32 | languageOptions: { 33 | parser: tsParser, 34 | }, 35 | 36 | rules: { 37 | "tsdoc/syntax": "warn", 38 | 39 | "array-callback-return": ["error", { 40 | allowImplicit: false, 41 | checkForEach: true, 42 | }], 43 | 44 | "no-constant-binary-expression": "error", 45 | "no-constructor-return": "error", 46 | "no-duplicate-imports": "error", 47 | "no-promise-executor-return": "error", 48 | "no-self-compare": "error", 49 | "no-template-curly-in-string": "error", 50 | "no-unmodified-loop-condition": "error", 51 | "no-unreachable-loop": "error", 52 | "no-unused-private-class-members": "error", 53 | "require-atomic-updates": "error", 54 | 55 | "@typescript-eslint/no-unused-vars": ["warn", { 56 | argsIgnorePattern: "^_", 57 | caughtErrorsIgnorePattern: "^_" 58 | }], 59 | 60 | "capitalized-comments": ["error", "always", { 61 | ignoreConsecutiveComments: true, 62 | }], 63 | 64 | complexity: ["error", 25], 65 | curly: "error", 66 | "default-case-last": "error", 67 | "default-param-last": "error", 68 | eqeqeq: ["error", "smart"], 69 | "func-names": "error", 70 | 71 | "func-style": ["error", "declaration", { 72 | allowArrowFunctions: true, 73 | }], 74 | 75 | "grouped-accessor-pairs": ["error", "getBeforeSet"], 76 | "max-depth": ["error", 4], 77 | "max-nested-callbacks": ["error", 3], 78 | "max-params": ["error", 4], 79 | 80 | "new-cap": ["error", { 81 | newIsCapExceptions: ["ctor"], 82 | }], 83 | 84 | "no-caller": "error", 85 | 86 | "no-confusing-arrow": ["error", { 87 | allowParens: true, 88 | }], 89 | 90 | "no-console": ["warn"], 91 | "no-div-regex": "error", 92 | "no-eval": "error", 93 | "no-extend-native": "error", 94 | "no-extra-bind": "error", 95 | "no-extra-label": "error", 96 | "no-extra-semi": "error", 97 | "no-floating-decimal": "error", 98 | "no-implied-eval": "error", 99 | "no-invalid-this": "error", 100 | "no-labels": "error", 101 | "no-lonely-if": "error", 102 | "no-multi-assign": "error", 103 | "no-multi-str": "error", 104 | "no-nested-ternary": "error", 105 | "no-new": "error", 106 | "no-new-func": "error", 107 | "no-new-object": "error", 108 | "no-new-wrappers": "error", 109 | "no-param-reassign": "error", 110 | "no-proto": "error", 111 | "no-return-assign": "error", 112 | "no-return-await": "error", 113 | "no-sequences": "error", 114 | "no-shadow": "off", 115 | "@typescript-eslint/no-shadow": "error", 116 | "no-unneeded-ternary": "error", 117 | "no-useless-call": "error", 118 | "no-useless-computed-key": "error", 119 | "no-useless-concat": "error", 120 | "no-useless-rename": "error", 121 | "no-var": "error", 122 | "operator-assignment": ["error", "always"], 123 | "prefer-arrow-callback": "error", 124 | "prefer-const": "error", 125 | "prefer-exponentiation-operator": "error", 126 | "prefer-object-spread": "error", 127 | "prefer-promise-reject-errors": "error", 128 | "prefer-regex-literals": "error", 129 | "prefer-rest-params": "error", 130 | "prefer-spread": "error", 131 | "prefer-template": "error", 132 | "spaced-comment": ["error", "always"], 133 | "symbol-description": "error", 134 | yoda: "error", 135 | }, 136 | }, { 137 | files: ["test/**/*.ts"], 138 | 139 | rules: { 140 | "require-atomic-updates": "off", 141 | }, 142 | }]; -------------------------------------------------------------------------------- /docs/components/transport-types/subscription-transport.md: -------------------------------------------------------------------------------- 1 | # Subscription Transport 2 | 3 | The `SubscriptionTransport` is an **abstract transport** (class) that serves as the foundation for implementing subscription-based transports. It handles incoming requests, adds them to a subscription set, and provides those entries to a background handler method. This class is intended to be extended by specific transport implementations. 4 | 5 | All incoming requests to the adapter for an endpoint that uses subscription-based transport are stored in a cached set (`SubscriptionSet`). 6 | Periodically, the background execute loop of the adapter will read the entire subscription set and call the `backgroundHandler` method of the transport. 7 | 8 | `SubscriptionTransport` has two abstract methods that should be implemented by subclasses. 9 | 10 | 1. `backgroundHandler` is called on each background execution iteration. It receives endpoint context as first argument and an array of all the entries in the subscription set as second argument. Sub-transport logic should be defined in this method. 11 | 2. `getSubscriptionTtlFromConfig` receives adapter settings and should return time-to-live (TTL) value for subscription set. 12 | 13 | ## Example implementation of SubscriptionTransport 14 | 15 | ```typescript 16 | // `AddressTransport` is a custom subscription-based transport that extends `SubscriptionTransport` 17 | // It uses `ethers` library to fetch data from a contract 18 | export class AddressTransport extends SubscriptionTransport { 19 | // JsonRpcProvider provider instance to be used for contract calls in this example 20 | provider!: ethers.providers.JsonRpcProvider 21 | 22 | // Initialize the transport with necessary dependencies, adapter settings, endpoint name, and a transport name. 23 | // You can initialize additional properties here as well, like in this case `this.provider` 24 | async initialize( 25 | dependencies: TransportDependencies, 26 | adapterSettings: AddressTransportTypes['Settings'], 27 | endpointName: string, 28 | transportName: string, 29 | ): Promise { 30 | // when initializing additional properties don't forget to call super.initialize() 31 | await super.initialize(dependencies, adapterSettings, endpointName, transportName) 32 | 33 | this.provider = new ethers.providers.JsonRpcProvider( 34 | adapterSettings.RPC_URL, 35 | adapterSettings.CHAIN_ID, 36 | ) 37 | } 38 | 39 | // backgroundHandler receives endpoint context and entries in subscription set and should implement the transport logic 40 | async backgroundHandler( 41 | context: EndpointContext, 42 | entries: RequestParams[], 43 | ) { 44 | // Processes each entry in subscription set 45 | await Promise.all(entries.map(async (param) => this.handleRequest(param))) 46 | // Sleeps for BACKGROUND_EXECUTE_MS miliseconds after processing all entries in subscription set 47 | await sleep(context.adapterSettings.BACKGROUND_EXECUTE_MS) 48 | } 49 | 50 | // helper method that takes params in subscription set, cocnstructs and saves a response object into a cache. 51 | private async handleRequest(param: RequestParams) { 52 | let response: AdapterResponse 53 | try { 54 | response = await this._handleRequest(param) 55 | } catch (e) { 56 | const errorMessage = e instanceof Error ? e.message : 'Unknown error occurred' 57 | response = { 58 | statusCode: 502, 59 | errorMessage, 60 | timestamps: { 61 | providerDataRequestedUnixMs: 0, 62 | providerDataReceivedUnixMs: 0, 63 | providerIndicatedTimeUnixMs: undefined, 64 | }, 65 | } 66 | } 67 | // save response to cache 68 | await this.responseCache.write(this.name, [{ params: param, response }]) 69 | } 70 | 71 | // helper method that gets the data from a contract and returns as AdapterResponse object 72 | private async _handleRequest( 73 | param: RequestParams, 74 | ): Promise> { 75 | const { contractAddress } = param 76 | const contract = new ethers.Contract(contractAddress, ABI, this.provider) 77 | 78 | const providerDataRequestedUnixMs = Date.now() 79 | const addressList = await contract.getAddressList() 80 | 81 | return { 82 | data: { 83 | result: addressList, 84 | }, 85 | statusCode: 200, 86 | result: null, 87 | timestamps: { 88 | providerDataRequestedUnixMs, 89 | providerDataReceivedUnixMs: Date.now(), 90 | providerIndicatedTimeUnixMs: undefined, 91 | }, 92 | } 93 | } 94 | 95 | // getSubscriptionTtlFromConfig method should return TTL number for subscription sets in this transport 96 | getSubscriptionTtlFromConfig(adapterSettings: BaseEndpointTypes['Settings']): number { 97 | return adapterSettings.WARMUP_SUBSCRIPTION_TTL 98 | } 99 | } 100 | ``` 101 | 102 | Another example of `SubscriptionTransport` is built-in [HTTP Transport](./http-transport.md). 103 | -------------------------------------------------------------------------------- /docs/guides/creating-a-new-v3-ea.md: -------------------------------------------------------------------------------- 1 | # Creating a new v3 EA 2 | 3 | This guide outlines the steps needed to setup a basic adapter. 4 | 5 | This guide carries stylistic biases, mainly in the context of the collection of EAs found in the [External Adapters Monorepo](https://github.com/smartcontractkit/external-adapters-js) under `packages/sources/`. That being said, this framework can be used outside of that to build standalone adapters, in which case its organization does not need to follow the structure laid out in this guide. 6 | 7 | ## Creating A New Adapter 8 | 9 | The framework provides an interactive EA generator script to help create new adapters. 10 | To create a new adapter in the [External Adapters Monorepo](https://github.com/smartcontractkit/external-adapters-js) run `yarn new`. It will ask several questions regarding adapter and endpoints and will generate the file structure with all the boilerplate code. 11 | 12 | ## Steps to create a new Adapter manually 13 | 14 | 1. [Create the Folder Structure](#create-the-folder-structure) 15 | 2. [Create a Transport](#create-a-transport) 16 | 3. [Create an Endpoint](#create-an-endpoint) 17 | 4. [Create the Adapter](#create-the-adapter) 18 | 19 | ## Create the Folder Structure 20 | 21 | An adapter built with the v3 framework has the following folder structure. 22 | 23 | This structure is a standard we want to keep for consistency in the monorepo but it is not strictly required as it was in v2. 24 | 25 | ``` 26 | adapter // Name after the adapter 27 | ├─ config 28 | │ ├─ index.ts // Custom settings. Optional but very common 29 | │ ├─ overrides.json // Overrides file. Optional 30 | | └─ includes.json // Includes file (e.x. inverses). Optional 31 | ├─ endpoint 32 | │ └─ endpoint.ts // Endpoint is defined here. Name after API endpoint 33 | ├─ transport 34 | │ └─ transport.ts // Transport is defined here. Name as associated endpoint name 35 | ├─ index.ts // Adapter defined here 36 | ├─ test 37 | ├─ package.json 38 | ├─ CHANGELOG.md // Content auto-updated through changesets 39 | ├─ README.md // Content auto-generated by CI scripts 40 | ├─ test-payload.json 41 | ├─ tsconfig.json 42 | └─ tsconfig.test.json 43 | ``` 44 | 45 | ## Create a Transport 46 | 47 | Once the folder structure has been set up, a transport can be defined in its respective transport file in `transport` folder. The v3 framework provides different types of transports to allow data retrieval through different protocols, such as HTTP, Websocket, and SSE. 48 | 49 | To learn more about transports and their specifications, please refer to the [Transports Guide](../components/transports.md). 50 | 51 | For the purpose of this guide, an example HTTP transport is shown below. 52 | 53 | ```typescript 54 | const transport = new HttpTransport({ 55 | // Return list of ProviderRequestConfigs 56 | prepareRequests: (params, config) => { 57 | return params.map((param) => { 58 | const symbol = param.symbol.toLowerCase() 59 | const url = `/price/${symbol}` 60 | 61 | return { 62 | params: param, 63 | request: { 64 | baseURL: config.API_ENDPOINT, 65 | url, 66 | }, 67 | } 68 | }) 69 | }, 70 | // Parse response into individual ProviderResults for each set of params 71 | parseResponse: (params, res) => { 72 | return res.data.map((result) => { 73 | return { 74 | params: { symbol: result.symbol }, 75 | response: { 76 | data: { 77 | result: result.price, 78 | }, 79 | result: result.price, 80 | }, 81 | } 82 | }) 83 | }, 84 | }) 85 | ``` 86 | 87 | ## Create an Endpoint 88 | 89 | The endpoint can be defined referring to the transport created in the previous step as shown in the example below. 90 | 91 | To learn more about `AdapterEndpoint` and its parameters, please refer to the [Endpoints Guide](../components/endpoints.md). 92 | 93 | ```typescript 94 | import { AdapterEndpoint } from '@chainlink/external-adapter-framework/adapter' 95 | import { transport } from '../transport/endpoint' 96 | 97 | export const endpoint = new AdapterEndpoint({ 98 | name: 'endpoint', // The name of this endpoint 99 | transport: transport, // The transport this endpoint is retrieves data through 100 | inputParameters, // Input parameters sent in requests to this endpoint 101 | }) 102 | ``` 103 | 104 | ## Create the Adapter 105 | 106 | Finally, create the adapter in the root-level `index.ts` file. Reference the endpoint created in the previous step as shown in the example below. 107 | 108 | To learn more about `Adapter` and its parameters, please refer to the [Adapter Guide](../components/adapter.md). 109 | 110 | ```typescript 111 | import { expose } from '@chainlink/external-adapter-framework' 112 | import { endpoint } from './endpoint' 113 | 114 | export const adapter = new Adapter({ 115 | name: 'ADAPTER_NAME', // The EA name, in uppercase without any spaces 116 | endpoints: [endpoint], // An array of all endpoints available. Defined in the endpoints folder 117 | }) 118 | 119 | // Expose the server to start the EA 120 | export const server = () => expose(adapter) 121 | ``` 122 | -------------------------------------------------------------------------------- /scripts/generator-adapter/generators/app/templates/src/transport/custombg.ts.ejs: -------------------------------------------------------------------------------- 1 | import { TransportDependencies } from '@chainlink/external-adapter-framework/transports' 2 | import { ResponseCache } from '@chainlink/external-adapter-framework/cache/response' 3 | import { Requester } from '@chainlink/external-adapter-framework/util/requester' 4 | import { 5 | AdapterResponse, sleep, makeLogger 6 | } from '@chainlink/external-adapter-framework/util' 7 | import { SubscriptionTransport } from '@chainlink/external-adapter-framework/transports/abstract/subscription' 8 | import { EndpointContext } from '@chainlink/external-adapter-framework/adapter' 9 | import { BaseEndpointTypes, inputParameters } from '../endpoint/<%= inputEndpointName %>' 10 | import { AdapterError } from '@chainlink/external-adapter-framework/validation/error' 11 | 12 | const logger = makeLogger('CustomTransport') 13 | 14 | type RequestParams = typeof inputParameters.validated 15 | 16 | <% if (includeComments) { -%> 17 | // CustomTransport extends base types from endpoint and adds additional, Provider-specific types (if needed). 18 | <% } -%> 19 | export type CustomTransportTypes = BaseEndpointTypes & { 20 | Provider: { 21 | RequestBody: never 22 | ResponseBody: any 23 | } 24 | } 25 | <% if (includeComments) { -%> 26 | // CustomTransport is used to perform custom data fetching and processing from a Provider. The framework provides built-in transports to 27 | // fetch data from a Provider using several protocols, including `http`, `websocket`, and `sse`. Use CustomTransport when the Provider uses 28 | // different protocol, or you need custom functionality that built-in transports don't support. For example, custom, multistep authentication 29 | // for requests, paginated requests, on-chain data retrieval using third party libraries, and so on. 30 | <% } -%> 31 | export class CustomTransport extends SubscriptionTransport { 32 | <% if (includeComments) { -%> 33 | // name of the transport, used for logging 34 | <% } -%> 35 | name!: string 36 | <% if (includeComments) { -%> 37 | // cache instance for caching responses from provider 38 | <% } -%> 39 | responseCache!: ResponseCache 40 | <% if (includeComments) { -%> 41 | // instance of Requester to be used for data fetching. Use this instance to perform http calls 42 | <% } -%> 43 | requester!: Requester 44 | 45 | <% if (includeComments) { -%> 46 | // REQUIRED. Transport will be automatically initialized by the framework using this method. It will be called with transport 47 | // dependencies, adapter settings, endpoint name, and transport name as arguments. Use this method to initialize transport state 48 | <% } -%> 49 | async initialize(dependencies: TransportDependencies, adapterSettings: CustomTransportTypes['Settings'], endpointName: string, transportName: string): Promise { 50 | await super.initialize(dependencies, adapterSettings, endpointName, transportName) 51 | this.requester = dependencies.requester 52 | } 53 | <% if (includeComments) { -%> 54 | // 'backgroundHandler' is called on each background execution iteration. It receives endpoint context as first argument 55 | // and an array of all the entries in the subscription set as second argument. Use this method to handle the incoming 56 | // request, process it and save it in the cache. 57 | <% } -%> 58 | async backgroundHandler(context: EndpointContext, entries: RequestParams[]) { 59 | await Promise.all(entries.map(async (param) => this.handleRequest(param))) 60 | await sleep(context.adapterSettings.BACKGROUND_EXECUTE_MS) 61 | } 62 | 63 | async handleRequest(param: RequestParams) { 64 | let response: AdapterResponse 65 | try { 66 | response = await this._handleRequest(param) 67 | } catch (e) { 68 | const errorMessage = e instanceof Error ? e.message : 'Unknown error occurred' 69 | logger.error(e, errorMessage) 70 | response = { 71 | statusCode: (e as AdapterError)?.statusCode || 502, 72 | errorMessage, 73 | timestamps: { 74 | providerDataRequestedUnixMs: 0, 75 | providerDataReceivedUnixMs: 0, 76 | providerIndicatedTimeUnixMs: undefined, 77 | }, 78 | } 79 | } 80 | await this.responseCache.write(this.name, [{ params: param, response }]) 81 | } 82 | 83 | async _handleRequest( 84 | _: RequestParams, 85 | ): Promise> { 86 | 87 | const providerDataRequestedUnixMs = Date.now() 88 | 89 | // custom transport logic 90 | 91 | return { 92 | data: { 93 | result: 2000, 94 | }, 95 | statusCode: 200, 96 | result: 2000, 97 | timestamps: { 98 | providerDataRequestedUnixMs, 99 | providerDataReceivedUnixMs: Date.now(), 100 | providerIndicatedTimeUnixMs: undefined, 101 | }, 102 | } 103 | } 104 | 105 | getSubscriptionTtlFromConfig(adapterSettings: CustomTransportTypes['Settings']): number { 106 | return adapterSettings.WARMUP_SUBSCRIPTION_TTL 107 | } 108 | } 109 | 110 | export const customSubscriptionTransport = new CustomTransport() -------------------------------------------------------------------------------- /src/validation/utils.ts: -------------------------------------------------------------------------------- 1 | import { isIP } from 'net' 2 | import { ValidationErrorMessage } from '../config' 3 | 4 | export type ValidatorMeta = { 5 | min?: number 6 | max?: number 7 | details?: string 8 | } 9 | 10 | export type Validator = { 11 | meta: ValidatorMeta 12 | fn: (value?: V) => ValidationErrorMessage 13 | } 14 | export type ValidatorWithParams = (param: P, customError?: string) => Validator 15 | 16 | // Composes complex validator function that runs each validator in order and returns first occurred error description and skips the rest of validation 17 | const compose: (f: Validator[]) => Validator = ( 18 | validatorFunctions: Validator[], 19 | ) => { 20 | const meta: ValidatorMeta = {} 21 | const details: string[] = [] 22 | 23 | for (const validator of validatorFunctions) { 24 | if (validator.meta.min !== undefined) { 25 | meta.min = validator.meta.min 26 | } 27 | if (validator.meta.max !== undefined) { 28 | meta.max = validator.meta.max 29 | } 30 | if (validator.meta.details) { 31 | details.push(validator.meta.details) 32 | } 33 | } 34 | 35 | meta.details = details.join(', ') 36 | 37 | return { 38 | meta: meta, 39 | fn: (value) => { 40 | for (const validator of validatorFunctions) { 41 | const errorText = validator.fn(value) 42 | if (errorText?.length) { 43 | return errorText 44 | } 45 | } 46 | return 47 | }, 48 | } 49 | } 50 | 51 | const _integer: () => Validator = () => ({ 52 | meta: { 53 | details: 'Value must be an integer', 54 | }, 55 | fn: (value) => { 56 | if (!Number.isInteger(value)) { 57 | return `Value must be an integer (no floating point)., Received ${typeof value} ${value}` 58 | } 59 | return 60 | }, 61 | }) 62 | 63 | const positive: () => Validator = () => ({ 64 | meta: { 65 | details: 'Value must be a positive number', 66 | }, 67 | fn: (value) => { 68 | if (value !== undefined && Number(value) < 0) { 69 | return `Value must be positive number, Received ${value}` 70 | } 71 | return 72 | }, 73 | }) 74 | 75 | const minNumber: ValidatorWithParams = (param) => ({ 76 | meta: { 77 | details: 'Value must be above the minimum', 78 | min: param, 79 | }, 80 | fn: (value) => { 81 | if (value !== undefined && Number(value) < param) { 82 | return `Minimum allowed value is ${param}. Received ${value}` 83 | } 84 | return 85 | }, 86 | }) 87 | 88 | const maxNumber: ValidatorWithParams = (param) => ({ 89 | meta: { 90 | details: 'Value must be below the maximum', 91 | max: param, 92 | }, 93 | fn: (value) => { 94 | if (value !== undefined && Number(value) > param) { 95 | return `Maximum allowed value is ${param}. Received ${value}` 96 | } 97 | return 98 | }, 99 | }) 100 | 101 | const url: () => Validator = () => ({ 102 | meta: { 103 | details: 'Value must be a valid URL', 104 | }, 105 | fn: (value) => { 106 | try { 107 | /* eslint-disable-next-line @typescript-eslint/no-unused-expressions */ 108 | value && new URL(value) 109 | } catch (e) { 110 | return `Value must be valid URL. Received ${value}, error ${e}` 111 | } 112 | return 113 | }, 114 | }) 115 | 116 | const host: () => Validator = () => ({ 117 | meta: { 118 | details: 'Value must be a valid IP address', 119 | }, 120 | fn: (value) => { 121 | const result = isIP(value || '') 122 | if (result === 0 && value !== 'localhost') { 123 | return `Value is not valid IP address. Received ${value}` 124 | } 125 | return 126 | }, 127 | }) 128 | 129 | const positiveInteger = () => compose([_integer(), positive()]) 130 | 131 | const integer = (params?: { min?: number; max?: number }) => { 132 | const validators = [_integer()] 133 | if (params?.min !== undefined) { 134 | validators.push(minNumber(params.min)) 135 | } 136 | if (params?.max !== undefined) { 137 | validators.push(maxNumber(params.max)) 138 | } 139 | return compose(validators) 140 | } 141 | 142 | const port = () => integer({ min: 1, max: 65535 }) 143 | 144 | // Validates that value is a valid timestamp from 2018-01-01 to now + 50ms to account for clock drift 145 | const responseTimestamp = () => integer({ min: 1514764861000, max: new Date().getTime() + 50 }) 146 | 147 | const base64: () => Validator = () => ({ 148 | meta: { 149 | details: 'Value must be a valid base64 string', 150 | }, 151 | fn: (value) => { 152 | const errorMessage = `Value is not valid base64 string.` 153 | if (!value) { 154 | return errorMessage 155 | } 156 | try { 157 | const decoded = Buffer.from(value, 'base64').toString('utf-8') 158 | const encodedAgain = Buffer.from(decoded, 'utf-8').toString('base64') 159 | return value !== encodedAgain ? errorMessage : undefined 160 | } catch (err) { 161 | return `Value is not valid base64 string. ${err}` 162 | } 163 | }, 164 | }) 165 | 166 | export const validator = { 167 | integer, 168 | positiveInteger, 169 | port, 170 | url, 171 | host, 172 | responseTimestamp, 173 | base64, 174 | compose, 175 | } 176 | -------------------------------------------------------------------------------- /test/cache/helper.ts: -------------------------------------------------------------------------------- 1 | import untypedTest, { TestFn } from 'ava' 2 | import { Adapter, AdapterEndpoint } from '../../src/adapter' 3 | import { Cache } from '../../src/cache' 4 | import { EmptyCustomSettings } from '../../src/config' 5 | import { AdapterRequest } from '../../src/util' 6 | import { InputParameters } from '../../src/validation' 7 | import { TypeFromDefinition } from '../../src/validation/input-params' 8 | import { NopTransport, NopTransportTypes, TestAdapter } from '../../src/util/testing-utils' 9 | 10 | export const test = untypedTest as TestFn<{ 11 | cache: Cache 12 | adapterEndpoint: AdapterEndpoint 13 | testAdapter: TestAdapter 14 | }> 15 | 16 | export const cacheTestInputParameters = new InputParameters({ 17 | base: { 18 | type: 'string', 19 | description: 'base', 20 | required: true, 21 | }, 22 | factor: { 23 | type: 'number', 24 | description: 'factor', 25 | required: true, 26 | }, 27 | }) 28 | 29 | export type CacheTestTransportTypes = { 30 | Parameters: typeof cacheTestInputParameters.definition 31 | Response: { 32 | Data: null 33 | Result: number 34 | } 35 | Settings: EmptyCustomSettings 36 | } 37 | 38 | export class BasicCacheSetterTransport extends NopTransport { 39 | override async foregroundExecute( 40 | req: AdapterRequest>, 41 | ): Promise { 42 | await this.responseCache.write(this.name, [ 43 | { 44 | params: req.requestContext.data, 45 | response: { 46 | data: null, 47 | result: req.requestContext.data.factor, 48 | timestamps: { 49 | providerDataRequestedUnixMs: 0, 50 | providerDataReceivedUnixMs: 0, 51 | providerIndicatedTimeUnixMs: undefined, 52 | }, 53 | }, 54 | }, 55 | ]) 56 | } 57 | } 58 | 59 | export class DifferentResultTransport extends NopTransport { 60 | override async foregroundExecute( 61 | req: AdapterRequest>, 62 | ): Promise { 63 | await this.responseCache.write(this.name, [ 64 | { 65 | params: req.requestContext.data, 66 | response: { 67 | data: null, 68 | result: Date.now(), 69 | timestamps: { 70 | providerDataRequestedUnixMs: 0, 71 | providerDataReceivedUnixMs: 0, 72 | providerIndicatedTimeUnixMs: undefined, 73 | }, 74 | }, 75 | }, 76 | ]) 77 | } 78 | } 79 | 80 | export function buildDiffResultAdapter(name: Uppercase) { 81 | return new Adapter({ 82 | name, 83 | defaultEndpoint: 'test', 84 | endpoints: [ 85 | new AdapterEndpoint({ 86 | name: 'test', 87 | inputParameters: cacheTestInputParameters, 88 | transport: new DifferentResultTransport(), 89 | }), 90 | ], 91 | }) 92 | } 93 | 94 | export const cacheTests = () => { 95 | test('returns value set in cache from setup', async (t) => { 96 | const data = { 97 | base: 'eth', 98 | factor: 123, 99 | } 100 | 101 | const response = await t.context.testAdapter.request(data) 102 | t.is(response.json().result, 123) 103 | }) 104 | 105 | test('returns value already found in cache', async (t) => { 106 | const data = { 107 | base: 'qweqwe', 108 | factor: 111, 109 | } 110 | 111 | const cacheKey = 'TEST-test-default_single_transport-{"base":"qweqwe","factor":111}' 112 | 113 | // Inject values directly into the cache 114 | const injectedEntry = { 115 | data: null, 116 | statusCode: 200, 117 | result: 'injected', 118 | } 119 | 120 | t.context.cache.set(cacheKey, injectedEntry, 10000) 121 | 122 | const response = await t.context.testAdapter.request(data) 123 | t.is(response.json().result, 'injected') 124 | }) 125 | 126 | test('deletes the value in cache', async (t) => { 127 | const data = { 128 | base: 'qweqwe', 129 | factor: 111, 130 | } 131 | 132 | const cacheKey = 'TEST-test-default_single_transport-{"base":"qweqwe","factor":111}' 133 | 134 | // Inject values directly into the cache 135 | const injectedEntry = { 136 | data: null, 137 | statusCode: 200, 138 | result: 'injected', 139 | } 140 | 141 | t.context.cache.set(cacheKey, injectedEntry, 10000) 142 | 143 | const response = await t.context.testAdapter.request(data) 144 | t.is(response.json().result, 'injected') 145 | await t.context.cache.delete(cacheKey) 146 | const secondResponse = await t.context.testAdapter.request(data) 147 | t.is(secondResponse.json().result, 111) 148 | }) 149 | 150 | test('skips expired cache entry and returns set up value', async (t) => { 151 | const data = { 152 | base: 'sdfghj', 153 | factor: 24637, 154 | } 155 | 156 | const cacheKey = 'TEST-test-default_single_transport-{"base":"sdfghj","factor":24637}' 157 | 158 | // Inject values directly into the cache 159 | const injectedEntry = { 160 | data: null, 161 | statusCode: 200, 162 | result: 'injected', 163 | } 164 | 165 | t.context.cache.set(cacheKey, injectedEntry, -10) 166 | 167 | const response = await t.context.testAdapter.request(data) 168 | t.is(response.json().result, 24637) 169 | }) 170 | 171 | test('polls forever and returns timeout', async (t) => { 172 | const error = await t.context.testAdapter.request({ 173 | endpoint: 'nowork', 174 | }) 175 | t.is(error.statusCode, 504) 176 | }) 177 | } 178 | -------------------------------------------------------------------------------- /test/background-executor.test.ts: -------------------------------------------------------------------------------- 1 | import { InstalledClock } from '@sinonjs/fake-timers' 2 | import { installTimers } from './helper' 3 | import untypedTest, { TestFn } from 'ava' 4 | import { start } from '../src' 5 | import { Adapter, AdapterEndpoint, EndpointContext } from '../src/adapter' 6 | import { metrics as eaMetrics } from '../src/metrics' 7 | import { deferredPromise, sleep } from '../src/util' 8 | import { NopTransport, NopTransportTypes, TestAdapter } from '../src/util/testing-utils' 9 | 10 | const test = untypedTest as TestFn<{ 11 | testAdapter: TestAdapter 12 | clock: InstalledClock 13 | }> 14 | 15 | test.serial('background executor calls transport function with background context', async (t) => { 16 | const [promise, resolve] = deferredPromise>() 17 | 18 | const transport = new (class extends NopTransport { 19 | async backgroundExecute(context: EndpointContext): Promise { 20 | resolve(context) 21 | } 22 | })() 23 | 24 | const adapter = new Adapter({ 25 | name: 'TEST', 26 | endpoints: [ 27 | new AdapterEndpoint({ 28 | name: 'test', 29 | transport, 30 | }), 31 | new AdapterEndpoint({ 32 | name: 'skipped', 33 | transport: new NopTransport(), // Also add coverage for skipped executors 34 | }), 35 | ], 36 | }) 37 | 38 | const instance = await start(adapter) 39 | const context = await promise 40 | t.is(context.endpointName, 'test') 41 | await instance.api?.close() 42 | }) 43 | 44 | test.serial('background executor ends recursive chain on server close', async (t) => { 45 | const clock = installTimers() 46 | let timesCalled = 0 47 | 48 | const transport = new (class extends NopTransport { 49 | async backgroundExecute(): Promise { 50 | timesCalled++ 51 | } 52 | })() 53 | 54 | const adapter = new Adapter({ 55 | name: 'TEST', 56 | endpoints: [ 57 | new AdapterEndpoint({ 58 | name: 'test', 59 | transport, 60 | }), 61 | ], 62 | }) 63 | 64 | const server = await start(adapter) 65 | t.is(timesCalled, 1) 66 | await server.api?.close() 67 | t.is(timesCalled, 1) // The background process closed, so this was never called again 68 | 69 | clock.uninstall() 70 | }) 71 | 72 | test.serial('background executor error does not stop the loop', async (t) => { 73 | let iteration = 0 74 | const [promise, resolve] = deferredPromise>() 75 | 76 | const transport = new (class extends NopTransport { 77 | async backgroundExecute(context: EndpointContext): Promise { 78 | if (iteration === 0) { 79 | iteration++ 80 | throw new Error('Forced bg execute error') 81 | } 82 | resolve(context) 83 | } 84 | })() 85 | 86 | process.env['METRICS_ENABLED'] = 'true' 87 | const adapter = new Adapter({ 88 | name: 'TEST', 89 | endpoints: [ 90 | new AdapterEndpoint({ 91 | name: 'test', 92 | transport, 93 | }), 94 | new AdapterEndpoint({ 95 | name: 'skipped', 96 | transport: new NopTransport(), // Also add coverage for skipped executors 97 | }), 98 | ], 99 | }) 100 | 101 | const testAdapter = await TestAdapter.start(adapter, t.context) 102 | const context = await promise 103 | t.is(context.endpointName, 'test') 104 | const metrics = await testAdapter.getMetrics() 105 | 106 | metrics.assert(t, { 107 | name: 'bg_execute_errors', 108 | labels: { 109 | adapter_endpoint: 'test', 110 | transport: 'default_single_transport', 111 | }, 112 | expectedValue: 1, 113 | }) 114 | metrics.assert(t, { 115 | name: 'bg_execute_total', 116 | labels: { 117 | adapter_endpoint: 'test', 118 | transport: 'default_single_transport', 119 | }, 120 | expectedValue: 4, 121 | }) 122 | 123 | await testAdapter.api.close() 124 | }) 125 | 126 | test.serial('background executor timeout does not stop the loop', async (t) => { 127 | eaMetrics.clear() 128 | const clock = installTimers() 129 | const [promise, resolve] = deferredPromise>() 130 | let iteration = 0 131 | 132 | const transport = new (class extends NopTransport { 133 | async backgroundExecute(context: EndpointContext): Promise { 134 | if (iteration === 0) { 135 | iteration++ 136 | await sleep(100_000) 137 | } else { 138 | resolve(context) 139 | await sleep(10_000) 140 | } 141 | } 142 | })() 143 | 144 | process.env['METRICS_ENABLED'] = 'true' 145 | const adapter = new Adapter({ 146 | name: 'TEST', 147 | endpoints: [ 148 | new AdapterEndpoint({ 149 | name: 'test', 150 | transport, 151 | }), 152 | new AdapterEndpoint({ 153 | name: 'skipped', 154 | transport: new NopTransport(), // Also add coverage for skipped executors 155 | }), 156 | ], 157 | }) 158 | 159 | const testAdapter = await TestAdapter.start(adapter, t.context) 160 | await clock.tickAsync(120_000) 161 | await promise 162 | const metrics = await testAdapter.getMetrics() 163 | 164 | metrics.assert(t, { 165 | name: 'bg_execute_errors', 166 | labels: { 167 | adapter_endpoint: 'test', 168 | transport: 'default_single_transport', 169 | }, 170 | expectedValue: 1, 171 | }) 172 | metrics.assert(t, { 173 | name: 'bg_execute_total', 174 | labels: { 175 | adapter_endpoint: 'test', 176 | transport: 'default_single_transport', 177 | }, 178 | expectedValue: 4, 179 | }) 180 | 181 | clock.uninstall() 182 | await testAdapter.api.close() 183 | }) 184 | -------------------------------------------------------------------------------- /test/debug-endpoints.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | import { start } from '../src' 3 | import { Adapter } from '../src/adapter' 4 | import { AdapterConfig } from '../src/config' 5 | import { DebugPageSetting } from '../src/util/settings' 6 | 7 | test.serial('debug endpoints return 404 if env var is omitted', async (t) => { 8 | process.env['DEBUG_ENDPOINTS'] = undefined 9 | 10 | const adapter = new Adapter({ 11 | name: 'TEST', 12 | endpoints: [], 13 | }) 14 | 15 | const { api } = await start(adapter) 16 | 17 | const expect404ForPath = async (path: string) => { 18 | const error = await api?.inject({ path }) 19 | t.is(error?.statusCode, 404) 20 | } 21 | 22 | await expect404ForPath('/debug/settings') 23 | await expect404ForPath('/debug/settings/raw') 24 | }) 25 | 26 | test.serial('debug endpoints return 404 if env var is false', async (t) => { 27 | process.env['DEBUG_ENDPOINTS'] = 'false' 28 | 29 | const adapter = new Adapter({ 30 | name: 'TEST', 31 | endpoints: [], 32 | }) 33 | 34 | const { api } = await start(adapter) 35 | 36 | const expect404ForPath = async (path: string) => { 37 | const error = await api?.inject({ path }) 38 | t.is(error?.statusCode, 404) 39 | } 40 | 41 | await expect404ForPath('/debug/settings') 42 | await expect404ForPath('/debug/settings/raw') 43 | }) 44 | 45 | test.serial('/debug/settings/raw endpoint returns expected values', async (t) => { 46 | process.env['DEBUG_ENDPOINTS'] = 'true' 47 | process.env['API_KEY'] = '12312341234' 48 | 49 | const config = new AdapterConfig( 50 | { 51 | API_KEY: { 52 | description: 'Api key', 53 | type: 'string', 54 | sensitive: true, 55 | required: true, 56 | }, 57 | }, 58 | { 59 | envDefaultOverrides: { 60 | REQUESTER_SLEEP_BEFORE_REQUEUEING_MS: 9999, 61 | }, 62 | }, 63 | ) 64 | 65 | const adapter = new Adapter({ 66 | name: 'TEST', 67 | endpoints: [], 68 | config, 69 | }) 70 | 71 | const { api } = await start(adapter) 72 | const settingsResponse = await api?.inject({ path: '/debug/settings/raw' }) 73 | if (!settingsResponse?.body) { 74 | t.fail() 75 | return 76 | } 77 | const parsedResponse = settingsResponse?.json() as DebugPageSetting[] 78 | 79 | // Test that framework setting is correctly set 80 | t.deepEqual( 81 | parsedResponse.find((s) => s.name === 'DEBUG_ENDPOINTS'), 82 | { 83 | type: 'boolean', 84 | description: 85 | 'Whether to enable debug enpoints (/debug/*) for this adapter. Enabling them might consume more resources.', 86 | name: 'DEBUG_ENDPOINTS', 87 | required: false, 88 | default: false, 89 | customSetting: false, 90 | value: true, 91 | }, 92 | ) 93 | // Test that env override is correctly accounted for 94 | t.deepEqual( 95 | parsedResponse.find((s) => s.name === 'REQUESTER_SLEEP_BEFORE_REQUEUEING_MS'), 96 | { 97 | type: 'number', 98 | description: 99 | 'Time to sleep after a failed HTTP request before re-queueing the request (in ms)', 100 | name: 'REQUESTER_SLEEP_BEFORE_REQUEUEING_MS', 101 | required: false, 102 | default: 0, 103 | customSetting: false, 104 | envDefaultOverride: 9999, 105 | value: 9999, 106 | }, 107 | ) 108 | // Test that custom adapter setting is correctly set and censored 109 | t.deepEqual( 110 | parsedResponse.find((s) => s.name === 'API_KEY'), 111 | { 112 | type: 'string', 113 | description: 'Api key', 114 | name: 'API_KEY', 115 | required: true, 116 | sensitive: true, 117 | customSetting: true, 118 | value: '[API_KEY REDACTED]', 119 | }, 120 | ) 121 | // Test that setting with default is assigned that value if not set in env vars 122 | t.deepEqual( 123 | parsedResponse.find((s) => s.name === 'API_TIMEOUT'), 124 | { 125 | type: 'number', 126 | default: 30000, 127 | description: 128 | 'The number of milliseconds a request can be pending before returning a timeout error for data provider request', 129 | name: 'API_TIMEOUT', 130 | required: false, 131 | customSetting: false, 132 | value: 30000, 133 | }, 134 | ) 135 | // Test that setting with no default is correctly processed 136 | t.deepEqual( 137 | parsedResponse.find((s) => s.name === 'RATE_LIMIT_API_TIER'), 138 | { 139 | type: 'string', 140 | description: 141 | 'Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.', 142 | name: 'RATE_LIMIT_API_TIER', 143 | required: false, 144 | customSetting: false, 145 | }, 146 | ) 147 | }) 148 | 149 | test.serial('/debug/settings returns html response', async (t) => { 150 | process.env['DEBUG_ENDPOINTS'] = 'true' 151 | process.env['API_KEY'] = '12312341234' 152 | 153 | const config = new AdapterConfig( 154 | { 155 | API_KEY: { 156 | description: 'Api key', 157 | type: 'string', 158 | sensitive: true, 159 | required: true, 160 | }, 161 | }, 162 | { 163 | envDefaultOverrides: { 164 | REQUESTER_SLEEP_BEFORE_REQUEUEING_MS: 9999, 165 | }, 166 | }, 167 | ) 168 | 169 | const adapter = new Adapter({ 170 | name: 'TEST', 171 | endpoints: [], 172 | config, 173 | }) 174 | 175 | const { api } = await start(adapter) 176 | const settingsResponse = await api?.inject({ path: '/debug/settings' }) 177 | const text = settingsResponse?.payload.trim() 178 | t.is(settingsResponse?.headers['content-type'], 'text/html') 179 | t.true(text?.startsWith('') && text?.endsWith('')) 180 | }) 181 | --------------------------------------------------------------------------------