├── src ├── index.ts ├── types │ ├── Tag.ts │ ├── ChainId.ts │ ├── index.ts │ ├── TokenList.ts │ ├── Token.ts │ └── TokenSet.ts ├── providers │ ├── index.ts │ ├── Provider.ts │ ├── ProviderJupiterTokenList.ts │ ├── ProviderIgnore.ts │ ├── ProviderTrusted.ts │ ├── ProviderCoinGecko.ts │ └── ProviderLegacyToken.ts ├── utils │ └── rpc.ts └── generator.ts ├── .prettierrc ├── tsconfig.cjs.json ├── .gitignore ├── .eslintrc ├── tsconfig.json ├── package.json └── README.md /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './generator' 2 | export * from './providers' 3 | export * from './types' 4 | -------------------------------------------------------------------------------- /src/types/Tag.ts: -------------------------------------------------------------------------------- 1 | export enum Tag { 2 | LP_TOKEN = 'lp-token', 3 | JUPITER = 'jupiter', 4 | } 5 | -------------------------------------------------------------------------------- /src/types/ChainId.ts: -------------------------------------------------------------------------------- 1 | export enum ChainId { 2 | MAINNET = 101, 3 | TESTNET = 102, 4 | DEVNET = 103, 5 | } 6 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 4, 4 | "semi": false, 5 | "singleQuote": true 6 | } 7 | -------------------------------------------------------------------------------- /tsconfig.cjs.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./lib/cjs/", 5 | "module": "CommonJS" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './ChainId' 2 | export * from './Tag' 3 | export * from './Token' 4 | export * from './TokenList' 5 | export * from './TokenSet' 6 | -------------------------------------------------------------------------------- /src/providers/index.ts: -------------------------------------------------------------------------------- 1 | export * from './Provider' 2 | export * from './ProviderCoinGecko' 3 | export * from './ProviderLegacyToken' 4 | export * from './ProviderTrusted' 5 | export * from './ProviderIgnore' 6 | export * from './ProviderJupiterTokenList' 7 | -------------------------------------------------------------------------------- /src/types/TokenList.ts: -------------------------------------------------------------------------------- 1 | import { Token } from './Token' 2 | 3 | export interface TokenList { 4 | name: string 5 | logoURI: string 6 | keywords: string[] 7 | tags: object 8 | timestamp: string 9 | tokens: Token | { tags: string[] }[] 10 | } 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | lib 3 | .idea 4 | *.iml 5 | 6 | .DS_Store 7 | 8 | .env.local 9 | .env.development.local 10 | .env.test.local 11 | .env.production.local 12 | .env 13 | 14 | npm-debug.log* 15 | yarn-debug.log* 16 | yarn-error.log* 17 | 18 | .token*.json 19 | -------------------------------------------------------------------------------- /src/types/Token.ts: -------------------------------------------------------------------------------- 1 | import { ChainId, Tag } from './index' 2 | 3 | export interface Token { 4 | address: string 5 | chainId: ChainId 6 | name: string 7 | symbol: string 8 | logoURI: string | null 9 | verified: boolean 10 | tags: Set 11 | decimals: number | null 12 | holders: number | null 13 | extensions?: { 14 | coingeckoId: string 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es6": true, 4 | "node": true 5 | }, 6 | "parser": "@typescript-eslint/parser", 7 | "plugins": ["import", "@typescript-eslint"], 8 | "extends": [ 9 | "eslint:recommended", 10 | "plugin:@typescript-eslint/eslint-recommended", 11 | "plugin:@typescript-eslint/recommended" 12 | ], 13 | "rules": { 14 | "import/order": [ 15 | "error", 16 | { 17 | "newlines-between": "always", 18 | "groups": [["builtin", "external"], "parent", "sibling", "index"], 19 | "warnOnUnassignedImports": true, 20 | "alphabetize": { 21 | "order": "asc" 22 | } 23 | } 24 | ] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": "./src", 4 | "outDir": "./lib/esm", 5 | "target": "ES5", 6 | "module": "ES6", 7 | "lib": [ 8 | "esnext" 9 | ], 10 | "declaration": true, 11 | "moduleResolution": "node", 12 | "noUnusedLocals": true, 13 | "noUnusedParameters": true, 14 | "esModuleInterop": true, 15 | "noImplicitReturns": true, 16 | "noImplicitThis": true, 17 | "noImplicitAny": false, 18 | "strictNullChecks": true, 19 | "suppressImplicitAnyIndexErrors": true, 20 | "allowSyntheticDefaultImports": true, 21 | "downlevelIteration": true, 22 | "skipLibCheck": true, 23 | "types": [ 24 | "node" 25 | ] 26 | }, 27 | "include": [ 28 | "src" 29 | ], 30 | "exclude": [ 31 | "node_modules", 32 | "lib", 33 | "example" 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@solflare-wallet/utl-aggregator", 3 | "version": "0.0.17", 4 | "main": "lib/cjs/index.js", 5 | "module": "lib/esm/index.js", 6 | "author": "Solflare Developers ", 7 | "license": "ISC", 8 | "files": [ 9 | "lib/" 10 | ], 11 | "scripts": { 12 | "start:esm": "tsc --watch", 13 | "start:cjs": "tsc --project tsconfig.cjs.json --watch", 14 | "start": "npm-run-all -p start:esm start:cjs", 15 | "build:esm": "tsc", 16 | "build:cjs": "tsc --project tsconfig.cjs.json", 17 | "build": "npm run build:esm && npm run build:cjs", 18 | "deploy": "npm run build && npm publish --access public" 19 | }, 20 | "peerDependencies": { 21 | "@solana/web3.js": "^1.95.0" 22 | }, 23 | "dependencies": { 24 | "@metaplex-foundation/umi": "^0.9.2", 25 | "@metaplex-foundation/umi-bundle-defaults": "^0.9.2", 26 | "axios": "^0.27.2", 27 | "axios-retry": "^3.2.5", 28 | "lodash": "^4.17.21", 29 | "temp-dir": "^2.0.0", 30 | "ts-node": "^10.9.2" 31 | }, 32 | "devDependencies": { 33 | "@metaplex-foundation/mpl-token-metadata": "^3.2.1", 34 | "@types/axios": "^0.14.0", 35 | "@types/lodash": "^4.14.182", 36 | "@types/node": "^18.11.18", 37 | "@typescript-eslint/eslint-plugin": "^5.25.0", 38 | "@typescript-eslint/parser": "^5.25.0", 39 | "eslint": "^8.15.0", 40 | "eslint-plugin-import": "^2.26.0", 41 | "eslint-plugin-node": "^11.1.0", 42 | "npm-run-all": "^4.1.5", 43 | "prettier": "^2.6.2", 44 | "typescript": "^4.6.4" 45 | }, 46 | "engines": { 47 | "node": ">=16" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/providers/Provider.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import tempDir from 'temp-dir' 3 | import { TokenSet } from 'types' 4 | 5 | export abstract class Provider { 6 | protected static removeCachedJSON(path: string) { 7 | if (!path.length) { 8 | throw new Error('Cache path cant be empty') 9 | } 10 | 11 | try { 12 | const fullPath = `${tempDir}/${path}.json` 13 | fs.unlinkSync(fullPath) 14 | } catch (err) { 15 | throw new Error(`Failed to remove cache in ${path} path: ${err}`) 16 | } 17 | } 18 | 19 | protected static saveCachedJSON(path: string, content: string) { 20 | if (!path.length) { 21 | throw new Error('Cache path cant be empty') 22 | } 23 | 24 | try { 25 | const fullPath = `${tempDir}/${path}.json` 26 | fs.writeFileSync(fullPath, content, 'utf8') 27 | } catch (err) { 28 | throw new Error(`Failed to save cache in ${path} path: ${err}`) 29 | } 30 | } 31 | 32 | protected static readCachedJSON(path: string): string { 33 | if (!path.length) { 34 | throw new Error('Cache path cant be empty') 35 | } 36 | 37 | const fullPath = `${tempDir}/${path}.json` 38 | 39 | if (!fs.existsSync(fullPath)) { 40 | throw new Error( 41 | `Failed to read cache in ${path} path: No such file` 42 | ) 43 | } 44 | 45 | try { 46 | return fs.readFileSync(fullPath).toString() 47 | } catch (err) { 48 | throw new Error(`Failed to read cache in ${path} path: ${err}`) 49 | } 50 | } 51 | 52 | abstract getTokens(): Promise 53 | } 54 | -------------------------------------------------------------------------------- /src/types/TokenSet.ts: -------------------------------------------------------------------------------- 1 | import { Token } from './index' 2 | 3 | export class TokenSet { 4 | constructor( 5 | private source: string, 6 | private map = new Map() 7 | ) {} 8 | 9 | protected static tokenKey(mint: string, chainId: number) { 10 | return `${mint}:${chainId}` 11 | } 12 | 13 | protected static keyToMint(key: string) { 14 | return key.split(':')[0] 15 | } 16 | 17 | sourceName(): string { 18 | return this.source 19 | } 20 | 21 | mints(): string[] { 22 | return Array.from(this.map.keys()).map((key) => TokenSet.keyToMint(key)) 23 | } 24 | 25 | tokens(): Token[] { 26 | return Array.from(this.map.values()) 27 | } 28 | 29 | set(token: Token): this { 30 | this.map.set(TokenSet.tokenKey(token.address, token.chainId), token) 31 | return this 32 | } 33 | 34 | hasByMint(mint: string, chainId: number): boolean { 35 | return this.map.has(TokenSet.tokenKey(mint, chainId)) 36 | } 37 | 38 | hasByToken(token: Token): boolean { 39 | return this.map.has(TokenSet.tokenKey(token.address, token.chainId)) 40 | } 41 | 42 | getByMint(mint: string, chainId: number): Token | undefined { 43 | return this.map.get(TokenSet.tokenKey(mint, chainId)) 44 | } 45 | 46 | getByToken(token: Token): Token | undefined { 47 | return this.map.get(TokenSet.tokenKey(token.address, token.chainId)) 48 | } 49 | 50 | deleteByMint(mint: string, chainId: number): boolean { 51 | return this.map.delete(TokenSet.tokenKey(mint, chainId)) 52 | } 53 | 54 | deleteByToken(token: Token): boolean { 55 | return this.map.delete(TokenSet.tokenKey(token.address, token.chainId)) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/providers/ProviderJupiterTokenList.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import console from 'console' 3 | 4 | import { ChainId, Tag, Token, TokenSet } from '../types' 5 | 6 | import { Provider } from './index' 7 | 8 | type JupiterToken = { 9 | address: string 10 | chainId: ChainId 11 | name: string 12 | symbol: string 13 | logoURI: string 14 | tags: string[] 15 | decimals: number 16 | extensions?: { 17 | coingeckoId: string 18 | } 19 | } 20 | export class ProviderJupiterTokenList extends Provider { 21 | constructor(private readonly listUrl: string) { 22 | super() 23 | } 24 | 25 | async getTokens(): Promise { 26 | const tokenMap = new TokenSet('JupiterProvider') 27 | console.log(`[JUP] fetch list`) 28 | const tokens = await axios.get(this.listUrl) 29 | console.log(`[JUP] fetched list`) 30 | for (let i = 0; i < tokens.data.length; i++) { 31 | const token = tokens.data[i] 32 | 33 | const t: Token = { 34 | chainId: ChainId.MAINNET, 35 | name: token.name, 36 | symbol: token.symbol.toUpperCase(), 37 | address: token.address, 38 | decimals: token.decimals, 39 | logoURI: token.logoURI, 40 | tags: new Set([...(token.tags as Tag[]), Tag.JUPITER]), 41 | verified: 42 | token.tags.includes('verified') || 43 | token.tags.includes('strict'), 44 | holders: null, 45 | extensions: token.extensions?.coingeckoId 46 | ? { 47 | coingeckoId: token.extensions.coingeckoId, 48 | } 49 | : undefined, 50 | } 51 | 52 | tokenMap.set(t) 53 | } 54 | console.log(`[JUP] imported list`) 55 | return tokenMap 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/providers/ProviderIgnore.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import console from 'console' 3 | import _ from 'lodash' 4 | 5 | import { Tag, TokenSet } from '../types' 6 | 7 | import { Provider } from './Provider' 8 | 9 | interface IgnoreListToken { 10 | chainId: number 11 | name: string 12 | symbol: string 13 | decimals: number 14 | logoURI: string 15 | tags: Tag[] 16 | address: string 17 | } 18 | 19 | interface IgnoreList { 20 | tokens: IgnoreListToken[] 21 | } 22 | 23 | export class ProviderIgnore extends Provider { 24 | constructor( 25 | private readonly url: string, 26 | private readonly skipTags: Tag[], // Filter out specific tags 27 | private readonly chainId: number = 101 // Filter by chain id 28 | ) { 29 | super() 30 | } 31 | 32 | async getTokens(): Promise { 33 | const tokenMap = new TokenSet('IgnoreProvider') 34 | 35 | const tokens = await axios.get(this.url) 36 | for (let i = 0; i < tokens.data.tokens.length; i++) { 37 | const token: IgnoreListToken = tokens.data.tokens[i] 38 | 39 | // Get only tokens for mainnet and devnet 40 | if ( 41 | this.chainId === token.chainId && 42 | !_.intersection(token.tags, this.skipTags).length 43 | ) { 44 | tokenMap.set({ 45 | chainId: token.chainId, 46 | name: token.name, 47 | symbol: token.symbol, 48 | address: token.address, 49 | decimals: token.decimals, 50 | logoURI: token.logoURI, 51 | tags: new Set(token.tags), 52 | verified: true, 53 | holders: null, 54 | }) 55 | } 56 | } 57 | 58 | console.log(`[IL] Loaded tokens from ${this.url} - ${this.chainId}`) 59 | return tokenMap 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/providers/ProviderTrusted.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import console from 'console' 3 | import _ from 'lodash' 4 | 5 | import { Tag, TokenSet } from '../types' 6 | 7 | import { Provider } from './Provider' 8 | 9 | interface TrustedListToken { 10 | chainId: number 11 | name: string 12 | symbol: string 13 | decimals: number 14 | logoURI: string 15 | tags: Tag[] 16 | address: string 17 | } 18 | 19 | interface TrustedList { 20 | tokens: TrustedListToken[] 21 | } 22 | 23 | export class ProviderTrusted extends Provider { 24 | constructor( 25 | private readonly url: string, 26 | private readonly skipTags: Tag[], // Filter out specific tags 27 | private readonly chainId: number = 101 // Filter by chain id 28 | ) { 29 | super() 30 | } 31 | 32 | async getTokens(): Promise { 33 | const tokenMap = new TokenSet('TrustedProvider') 34 | 35 | const tokens = await axios.get(this.url) 36 | for (let i = 0; i < tokens.data.tokens.length; i++) { 37 | const token: TrustedListToken = tokens.data.tokens[i] 38 | 39 | // Get only tokens for mainnet and devnet 40 | if ( 41 | this.chainId === token.chainId && 42 | !_.intersection(token.tags, this.skipTags).length 43 | ) { 44 | tokenMap.set({ 45 | chainId: token.chainId, 46 | name: token.name, 47 | symbol: token.symbol, 48 | address: token.address, 49 | decimals: token.decimals, 50 | logoURI: token.logoURI, 51 | tags: new Set(token.tags), 52 | verified: true, 53 | holders: null, 54 | }) 55 | } 56 | } 57 | 58 | console.log(`[TL] Loaded tokens from ${this.url} - ${this.chainId}`) 59 | return tokenMap 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/utils/rpc.ts: -------------------------------------------------------------------------------- 1 | export function RpcRequestSignature(mint: string) { 2 | return { 3 | jsonrpc: '2.0', 4 | id: mint, 5 | method: 'getSignaturesForAddress', 6 | params: [ 7 | `${mint}`, 8 | { 9 | limit: 1, 10 | }, 11 | ], 12 | } 13 | } 14 | 15 | export interface RpcResponseSignature { 16 | id: string 17 | jsonrpc: string 18 | result: { 19 | blockTime: number 20 | confirmationStatus: number 21 | }[] 22 | } 23 | 24 | export function RpcRequestAccountInfo(mint: string) { 25 | return { 26 | jsonrpc: '2.0', 27 | id: mint, 28 | method: 'getAccountInfo', 29 | params: [ 30 | `${mint}`, 31 | { 32 | encoding: 'jsonParsed', 33 | }, 34 | ], 35 | } 36 | } 37 | 38 | export interface RpcResponseAccountInfo { 39 | id: string 40 | error: object | undefined 41 | jsonrpc: string 42 | result: { 43 | value: { 44 | data: { 45 | parsed: { 46 | info: { 47 | decimals: number 48 | } 49 | type: string // "mint" 50 | } 51 | program: string // "spl-token" 52 | } 53 | } 54 | } 55 | } 56 | 57 | export function RpcRequestHolders(mint: string) { 58 | return { 59 | jsonrpc: '2.0', 60 | id: mint, 61 | method: 'getProgramAccounts', 62 | params: [ 63 | 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA', // TOKEN_PROGRAM_ID 64 | { 65 | encoding: 'base64', 66 | dataSlice: { 67 | offset: 0, 68 | length: 0, 69 | }, 70 | filters: [ 71 | { 72 | dataSize: 165, 73 | }, 74 | { 75 | memcmp: { 76 | offset: 0, 77 | bytes: mint, 78 | }, 79 | }, 80 | ], 81 | }, 82 | ], 83 | } 84 | } 85 | 86 | export interface RpcResponseHolders { 87 | id: string 88 | jsonrpc: string 89 | result: { 90 | pubKey: string 91 | }[] 92 | } 93 | -------------------------------------------------------------------------------- /src/generator.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import axiosRetry from 'axios-retry' 3 | 4 | import { Provider } from './providers' 5 | import { Tag, TokenSet, TokenList } from './types' 6 | 7 | export class Generator { 8 | constructor( 9 | private readonly standardSources: Provider[], 10 | private readonly ignoreSources: Provider[] 11 | ) {} 12 | 13 | private static upsertTokenMints( 14 | tokenMints: TokenSet, 15 | newTokenMints: TokenSet 16 | ) { 17 | for (const token of newTokenMints.tokens()) { 18 | token.logoURI = Generator.sanitizeUrl(token.logoURI) 19 | 20 | const currentToken = tokenMints.getByToken(token) 21 | if (currentToken) { 22 | if (!currentToken.decimals && token.decimals) { 23 | currentToken.decimals = token.decimals 24 | } 25 | if (!currentToken.logoURI && token.logoURI) { 26 | currentToken.logoURI = token.logoURI 27 | } 28 | if (!currentToken.tags && token.tags) { 29 | currentToken.tags = new Set([ 30 | ...currentToken.tags, 31 | ...token.tags, 32 | ]) 33 | } 34 | if (!currentToken.holders && token.holders) { 35 | currentToken.holders = token.holders 36 | } 37 | if (token.extensions !== undefined) { 38 | currentToken.extensions = { 39 | ...currentToken.extensions, 40 | ...token.extensions, 41 | } 42 | } 43 | tokenMints.set(currentToken) 44 | } else { 45 | tokenMints.set(token) 46 | } 47 | } 48 | } 49 | 50 | private static removeTokenMints( 51 | tokenMints: TokenSet, 52 | newTokenMints: TokenSet 53 | ) { 54 | for (const token of newTokenMints.tokens()) { 55 | tokenMints.deleteByToken(token) 56 | } 57 | } 58 | 59 | async generateTokens() { 60 | axiosRetry(axios, { 61 | retries: 3, 62 | retryDelay: (retryCount) => { 63 | return retryCount * 1000 64 | }, 65 | }) 66 | 67 | const tokenMap = new TokenSet('List') 68 | 69 | let min = 0 70 | 71 | const id = setInterval(() => { 72 | console.log(`====> Minute: ${++min}`) 73 | }, 60 * 1000) 74 | 75 | try { 76 | // Add tokens from standard sources 77 | const results = await Promise.allSettled( 78 | this.standardSources.map((source) => source.getTokens()) 79 | ) 80 | for (const result of results) { 81 | if (result.status === 'fulfilled') { 82 | const lastCount = tokenMap.tokens().length 83 | Generator.upsertTokenMints(tokenMap, result.value) 84 | console.log( 85 | result.value.sourceName(), 86 | 'upsert count:', 87 | tokenMap.tokens().length - lastCount 88 | ) 89 | } else { 90 | console.log(JSON.stringify(result, null, 2)) 91 | console.log(`Generate failed ${result.reason}`) 92 | throw new Error(`Generate standard failed ${result.reason}`) 93 | } 94 | } 95 | 96 | // Remove tokens from ignore sources 97 | const resultsIgnore = await Promise.allSettled( 98 | this.ignoreSources.map((source) => source.getTokens()) 99 | ) 100 | for (const result of resultsIgnore) { 101 | if (result.status === 'fulfilled') { 102 | Generator.removeTokenMints(tokenMap, result.value) 103 | } else { 104 | console.log(`Generate failed ${result.reason}`) 105 | throw new Error(`Generate ignore failed ${result.reason}`) 106 | } 107 | } 108 | } catch (e) { 109 | clearInterval(id) 110 | throw e 111 | } 112 | 113 | clearInterval(id) 114 | return tokenMap 115 | } 116 | 117 | async generateTokenList(): Promise { 118 | const tokensMap = await this.generateTokens() 119 | 120 | const tags: object = {} 121 | const tagNames = Object.values(Tag) 122 | 123 | for (const tag of tagNames) { 124 | tags[tag] = { 125 | name: tag, 126 | description: '', 127 | } 128 | } 129 | 130 | return { 131 | name: 'Solana Token List', 132 | logoURI: '', 133 | keywords: ['solana', 'spl'], 134 | tags, 135 | timestamp: new Date().toISOString(), 136 | tokens: tokensMap.tokens().map((token) => { 137 | return { ...token, tags: [...token.tags] } 138 | }), 139 | } 140 | } 141 | 142 | private static sanitizeUrl(string) { 143 | let url 144 | 145 | try { 146 | url = new URL(string) 147 | } catch (_) { 148 | return null 149 | } 150 | 151 | return url.protocol === 'http:' || url.protocol === 'https:' 152 | ? url 153 | : null 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/providers/ProviderCoinGecko.ts: -------------------------------------------------------------------------------- 1 | import axios, { AxiosPromise } from 'axios' 2 | import console from 'console' 3 | import _ from 'lodash' 4 | 5 | import { ChainId, Tag, Token, TokenSet } from '../types' 6 | import { RpcRequestAccountInfo, RpcResponseAccountInfo } from '../utils/rpc' 7 | 8 | import { Provider } from './index' 9 | 10 | interface SimpleCoin { 11 | id: string 12 | symbol: string 13 | name: string 14 | platforms: { 15 | solana?: string 16 | } 17 | } 18 | 19 | interface ThrottleOptions { 20 | throttle: number 21 | throttleCoinGecko: number 22 | batchAccountsInfo: number 23 | batchCoinGecko: number 24 | } 25 | 26 | export class ProviderCoinGecko extends Provider { 27 | private readonly apiUrl = 'https://api.coingecko.com/api/v3' 28 | private readonly apiProUrl = 'https://pro-api.coingecko.com/api/v3' 29 | private readonly chainId = ChainId.MAINNET 30 | 31 | constructor( 32 | private readonly apiKey: string | null, 33 | private readonly rpcUrl: string, 34 | private readonly throttleOpts: ThrottleOptions = { 35 | throttle: 0, // Add sleep after batch RPC request to avoid rate limits 36 | throttleCoinGecko: 60 * 1000, // Add sleep after batch HTTP calls for CoinGecko 37 | batchAccountsInfo: 250, // Batch RPC calls in single RPC request 38 | batchCoinGecko: 50, // Batch CoinGecko token HTTP call 39 | } 40 | ) { 41 | super() 42 | } 43 | 44 | async getTokens(): Promise { 45 | const tokenMap = new TokenSet('CoinGeckoProvider') 46 | 47 | const tokens = await axios.get( 48 | this.coinGeckoApiUrl(`/coins/list?include_platform=true`) 49 | ) 50 | 51 | for (let i = 0; i < tokens.data.length; i++) { 52 | const token = tokens.data[i] 53 | if ( 54 | token.platforms.solana !== undefined && 55 | token.platforms.solana.length 56 | ) { 57 | const t: Token = { 58 | chainId: ChainId.MAINNET, 59 | name: token.name, 60 | symbol: token.symbol.toUpperCase(), 61 | address: token.platforms.solana, 62 | decimals: null, 63 | logoURI: null, 64 | tags: new Set(), 65 | verified: true, 66 | holders: null, 67 | extensions: { 68 | coingeckoId: token.id, 69 | }, 70 | } 71 | 72 | tokenMap.set(t) 73 | } 74 | } 75 | 76 | await this.filterByOnChain(tokenMap) 77 | await this.fetchDetails(tokenMap) 78 | return tokenMap 79 | } 80 | 81 | /** 82 | * Filter by account info 83 | * check if mint and get decimals 84 | * @param tokenMap 85 | */ 86 | private async filterByOnChain(tokenMap: TokenSet) { 87 | // Batch RPC calls 88 | const rpcCalls: object[] = [] 89 | for (const mint of tokenMap.mints()) { 90 | rpcCalls.push(RpcRequestAccountInfo(mint)) 91 | } 92 | 93 | // Chunk batched requests 94 | let progress = 0 95 | const chunks = _.chunk(rpcCalls, this.throttleOpts.batchAccountsInfo) 96 | for (const dataChunk of chunks) { 97 | console.log( 98 | `[CG] filter by account info ${++progress}/${chunks.length}` 99 | ) 100 | 101 | const response = await axios.post( 102 | this.rpcUrl, 103 | dataChunk 104 | ) 105 | 106 | for (const mintResponse of response.data) { 107 | // Remove token if not a mint 108 | if ( 109 | mintResponse.error || 110 | !mintResponse.result.value || 111 | !mintResponse.result.value.data['parsed'] || 112 | !mintResponse.result.value.data['program'] || 113 | (mintResponse.result.value.data.program !== 'spl-token' && 114 | mintResponse.result.value.data.program !== 115 | 'spl-token-2022') || 116 | mintResponse.result.value.data.parsed.type !== 'mint' 117 | ) { 118 | tokenMap.deleteByMint(mintResponse.id, this.chainId) 119 | continue 120 | } 121 | 122 | // Update decimals for token 123 | const token = tokenMap.getByMint(mintResponse.id, this.chainId) 124 | if (token) { 125 | token.decimals = 126 | mintResponse.result.value.data.parsed.info.decimals 127 | tokenMap.set(token) 128 | } 129 | } 130 | 131 | if (this.throttleOpts.throttle > 0) { 132 | await new Promise((f) => 133 | setTimeout(f, this.throttleOpts.throttle) 134 | ) 135 | } 136 | } 137 | } 138 | 139 | /** 140 | * Fetch details such as logo 141 | * @param tokenMap 142 | */ 143 | private async fetchDetails(tokenMap: TokenSet) { 144 | const batches = _.chunk( 145 | tokenMap.mints(), 146 | this.throttleOpts.batchCoinGecko 147 | ) 148 | 149 | let progress = 0 150 | for (const batch of batches) { 151 | console.log(`[CG] get logo ${++progress}/${batches.length}`) 152 | 153 | const requests: AxiosPromise[] = [] 154 | for (const mint of batch) { 155 | requests.push( 156 | axios.get( 157 | this.coinGeckoApiUrl(`/coins/solana/contract/${mint}`) 158 | ) 159 | ) 160 | } 161 | 162 | // Wait for batch of axios request to finish 163 | const responses = await Promise.allSettled(requests) 164 | for (const response of responses) { 165 | if (response.status === 'fulfilled') { 166 | // CoinGecko returns mint address in all uppercase 167 | // so mint address cannot be taken from response 168 | const mintAddress = response.value.config.url 169 | ?.split('/contract/')[1] 170 | .substring(0, 44) 171 | .split('?')[0] 172 | 173 | if (!mintAddress) { 174 | throw new Error( 175 | `Failed to fetch token info: No mint address` 176 | ) 177 | } 178 | 179 | const token = tokenMap.getByMint(mintAddress, this.chainId) 180 | 181 | if (token) { 182 | token.logoURI = response.value.data.image.large 183 | tokenMap.set(token) 184 | } 185 | } else { 186 | throw new Error( 187 | `Failed to fetch token info: ${response.reason}` 188 | ) 189 | } 190 | } 191 | 192 | if (this.throttleOpts.throttleCoinGecko) { 193 | await new Promise((f) => 194 | setTimeout(f, this.throttleOpts.throttleCoinGecko) 195 | ) 196 | } 197 | } 198 | } 199 | 200 | private coinGeckoApiUrl(path: string): string { 201 | if (!this.apiKey) { 202 | return `${this.apiUrl}${path}` 203 | } 204 | 205 | return `${this.apiProUrl}${path}${ 206 | path.includes('?') ? '&' : '?' 207 | }x_cg_pro_api_key=${this.apiKey}` 208 | } 209 | } 210 | 211 | interface ApiCoinContractResponse { 212 | id: string 213 | symbol: string 214 | name: string 215 | contract_address: string 216 | image: { 217 | large: string 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Unified Token List Aggregator 2 | 3 | The Unified Token List Aggregator (`UTL`) module generates Solana token list JSON based on user specified list of `provider` sources. 4 | 5 | By changing the provider source list in the aggregator config one can fine tune the output (explained below), and choose which providers are trusted, and filter out tokens (for example exclude Liquidity Pool (`LP`)-tokens which could be consumed from other sources). 6 | 7 | Running a script to call this module periodically will ensure that generated UTL is up-to-date. 8 | 9 | Generated JSON can be hosted on CDN or imported in DB to be exposed through API. 10 | 11 | The UTL generated through the aggregation process should be considered as a common source of truth for verified tokens across wallets and dApps. 12 | 13 | ## Our Goal 14 | 15 | We want to provide every community member a same base source of truth generated by Token List Aggregator - by soing do, we'll provide the community with a base verified token list. Anyone can use this module without any infrastructure or cost. 16 | 17 | Everything after that is only building on top of that, so Token List API is extension, and Token List SDK is extension on top of that. Every step is making things more efficient and optimised. 18 | 19 | Everyone can choose what they want to use, host and consume depending on their needs and requirements. 20 | 21 | ## Related repos 22 | 23 | - [Token List API](https://github.com/solflare-wallet/utl-api) 24 | - [Token List SDK](https://github.com/solflare-wallet/utl-sdk) 25 | - [Solflare Token List](https://github.com/solflare-wallet/token-list) 26 | 27 | 28 | ## Installation 29 | ```shell 30 | npm i @solflare-wallet/utl-aggregator 31 | ``` 32 | 33 | ## Usage 34 | 35 | Example usage can be found in [Solfare's Token List repo](https://github.com/solflare-wallet/token-list). 36 | 37 | 38 | Simple usage: 39 | 40 | ```ts 41 | import { 42 | Generator, 43 | ProviderCoinGecko, 44 | ProviderLegacyToken, 45 | ProviderTrusted, 46 | ProviderIgnore, 47 | ChainId, 48 | Tag, 49 | } from "@solflare-wallet/utl-aggregator"; 50 | import { clusterApiUrl } from '@solana/web3.js' 51 | import { writeFile } from 'fs/promises' 52 | 53 | const SECOND = 1000; 54 | const SECONDS = SECOND; 55 | const MINUTE = 60 * SECOND; 56 | const MINUTES = MINUTE; 57 | 58 | // Your Solana RPC URL - may be an open provider like 59 | // clusterApiUrl("mainnet-beta") 60 | // Or your own RPC instance like QuickNode etc. 61 | const SOLANA_RPC_URL = clusterApiUrl("mainnet-beta") 62 | 63 | async function main() { 64 | // Optionally clear the cache for each provider: 65 | // ProviderLegacyToken.clearCache(ChainId.MAINNET) 66 | // ProviderLegacyToken.clearCache(ChainId.DEVNET) 67 | 68 | const generator = new Generator([ 69 | // Providers are listen in order of preference 70 | new ProviderCoinGecko(null, SOLANA_RPC_URL, { 71 | // Add sleep after batch RPC request to avoid rate limits 72 | throttle: 1 * SECOND, 73 | // Add sleep after batch HTTP calls for CoinGecko 74 | throttleCoinGecko: 65 * SECONDS, 75 | // Batch RPC calls in single RPC request 76 | batchAccountsInfo: 100, 77 | // Batch CoinGecko token HTTP call 78 | batchCoinGecko: 25, 79 | }), 80 | new ProviderLegacyToken( 81 | 'https://cdn.jsdelivr.net/gh/solana-labs/token-list@main/src/tokens/solana.tokenlist.json', 82 | SOLANA_RPC_URL, 83 | { 84 | // Add sleep after batch RPC request to avoid rate limits 85 | throttle: 1000, 86 | // Batch RPC calls in single RPC request 87 | batchSignatures: 100, 88 | batchAccountsInfo: 100, 89 | // Batch parallel RPC requests 90 | batchTokenHolders: 1, 91 | }, 92 | // Filter out by tags, eg. remove Liquidity Pool (LP) tokens 93 | [Tag.LP_TOKEN], 94 | // Make sure ChainId is for RPC endpoint above 95 | ChainId.MAINNET, 96 | // Signature date filter, keep tokens with latest signature in last 30 days 97 | 30, 98 | // Keep tokens with more than 100 holders 99 | 100 100 | ), 101 | new ProviderTrusted( 102 | 'https://raw.githubusercontent.com/solflare-wallet/token-list/master/trusted-tokenlist.json', 103 | // Filter out by tags, eg. remove Liquidity Pool (LP) tokens 104 | [Tag.LP_TOKEN], 105 | // Filter by chainId 106 | ChainId.MAINNET, 107 | ), 108 | ], 109 | [ 110 | new ProviderIgnore( 111 | 'https://raw.githubusercontent.com/solflare-wallet/token-list/master/ignore-tokenlist.json', 112 | // Filter out by tags 113 | [], 114 | // Filter by chainId 115 | ChainId.MAINNET, 116 | ), 117 | ] 118 | ) 119 | 120 | 121 | const tokenList = await generator.generateTokenList() 122 | 123 | await writeFile('./solana-tokenlist.json', JSON.stringify(tokenList), 'utf8') 124 | 125 | console.log('UTL Completed, the file was saved!') 126 | } 127 | 128 | main() 129 | 130 | ``` 131 | 132 | 133 | ## Token List Providers 134 | Providers are listed in an aggregator. If for example mint/token A is in both CoinGecko and Orca list, only one instance/data will be kept for the final token list, and this is determined based on whether CoinGecko or Orca is positioned higher in the list. If Orca is above CoinGecko, mint A from Orca will be kept, and CoinGecko's mint A will be ignored. 135 | 136 | _**Built-in provider sources**_ will be the Pruned Legacy Token List (`LTL`) and CoinGecko (`CG`). 137 | CoinGecko has high barrier of entry for tokens, and is generally excellent when it comes to maintaining token list (since it's their job and business to do so). 138 | Legacy token list will be pruned (remove invalid mints, filtering by holders, last activity, LP tokens, scam tokens; this processed was described in Telegram chat) and transformed into the new standardized format. 139 | 140 | [To-Do] _**External Provider sources**_ (Orca, Raydium, Saber, etc..) can host and maintain their own list of verified tokens, that aggregator can use when generating unified token list. 141 | Each external provider will have to expose endpoint with a list of tokens they view as verified. This list will be in standardize format (which will include if token is LP-token, etc). 142 | 143 | [To-Do] Base external provider repo so any project (Orca, Raydium, Saber..) can host and expose their own verified token list with little developer effort. This allows them to serve as trusted providers for other. 144 | 145 | ### CoinGecko Provider 146 | Uses CoinGecko API to fetch all tokens with valid Solana mint address. 147 | Token' logoURI is fetched from CoinGecko also, while decimal is fetched from chain. 148 | That is why this provider also requires Solana RPC mainnet endpoint. 149 | 150 | **Throttle notes:** 151 | 152 | CoinGecko Free API usually has 25-50 calls/min limit, to avoid `HTTP 429 Too Many Requests` use `batchCoinGecko: 25` 153 | and `throttleCoinGecko: 65 * 1000` 154 | 155 | With CoinGecko Pro API Key, you can increase request sizes eg. `batchCoinGecko: 400` 156 | 157 | ```ts 158 | new ProviderCoinGecko( 159 | COINGECKO_API_KEY, 160 | RPC_URL, 161 | { // ThrottleOptions 162 | throttle: 1 * SECOND, // Add sleep after batch RPC request to avoid rate limits 163 | throttleCoinGecko: 65 * SECONDS, // Add sleep after batch HTTP calls for CoinGecko 164 | batchAccountsInfo: 100, // Batch RPC calls in single RPC request 165 | batchCoinGecko: 25, // Batch CoinGecko token HTTP call 166 | } 167 | ) 168 | 169 | ``` 170 | 171 | 172 | ### Legacy Token List Provider 173 | This provider uses existing token list and pulls active and relevant tokens from it. 174 | 175 | This is done in following steps: 176 | - Filter by chainId and tags 177 | - Remove by token content (remove already labeled scam and phishing) 178 | - Check if account is a mint (using getAccountInfo) 179 | - Remove by latest signature date 180 | - Remove by holders count 181 | 182 | **Caching:** 183 | 184 | Since RPC endpoints calls can fail or take long time on larger requests, 185 | this provider caches few result sets to increase speed for subsequent runs. 186 | 187 | Latest signatures are cached and tokens with holder count larger than 1000 are cached. 188 | This means that after first run, every other run will be faster. 189 | 190 | To clear cache you can use: 191 | ```javascript 192 | ProviderLegacyToken.clearCache(ChainId.MAINNET) 193 | ProviderLegacyToken.clearCache(ChainId.DEVNET) 194 | ``` 195 | 196 | 197 | **Throttle notes:** 198 | 199 | Different RPC endpoints have very different limits, to avoid `HTTP 429 Too Many Requests` try to thinker with `ThrottleOptions`. 200 | 201 | 202 | ```ts 203 | new ProviderLegacyToken( 204 | CDN_URL, 205 | RPC_URL, // Make sure RPC Endpoint is for ChainId specified below 206 | { // ThrottleOptions 207 | throttle: 1 * SECOND, // Add sleep after batch RPC request to avoid rate limits 208 | batchSignatures: 100, // Batch RPC calls in single RPC request 209 | batchAccountsInfo: 100, // Batch RPC calls in single RPC request 210 | batchTokenHolders: 1, // Batch parallel RPC requests 211 | }, 212 | [Tag.LP_TOKEN], // Filter out by tags, eg. remove LP tokens 213 | ChainId.MAINNET, // Keep only chainId 101 tokens 214 | 30, // Signature date filter, keep tokens with latest signature in last 30 days 215 | 100, // Keep tokens with more than 100 holders 216 | ) 217 | 218 | ``` 219 | -------------------------------------------------------------------------------- /src/providers/ProviderLegacyToken.ts: -------------------------------------------------------------------------------- 1 | import axios, { AxiosPromise } from 'axios' 2 | import * as console from 'console' 3 | import _ from 'lodash' 4 | 5 | import { Tag, TokenSet } from '../types' 6 | import { 7 | RpcRequestAccountInfo, 8 | RpcRequestHolders, 9 | RpcRequestSignature, 10 | RpcResponseAccountInfo, 11 | RpcResponseHolders, 12 | RpcResponseSignature, 13 | } from '../utils/rpc' 14 | 15 | import { Provider } from './Provider' 16 | 17 | interface LegacyListToken { 18 | chainId: number 19 | name: string 20 | symbol: string 21 | logoURI: string 22 | tags: Tag[] 23 | address: string 24 | } 25 | 26 | interface LegacyList { 27 | tokens: LegacyListToken[] 28 | } 29 | 30 | const LARGEST_MINTS = [ 31 | 'EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v', // USDC 32 | 'kinXdEcpDQeHPEuQnqmUgtYykqKGVFq6CeVX5iAHJq6', // KIN 33 | 'XzR7CUMqhDBzbAm4aUNvwhVCxjWGn1KEvqTp3Y8fFCD', // SCAM 34 | 'AFbX8oGjGpmVFywbVouvhQSRmiW2aR1mohfahi4Y2AdB', // GST 35 | 'CKaKtYvz6dKPyMvYq9Rh3UBrnNqYZAyd7iF4hJtjUvks', // GARI 36 | 'xxxxa1sKNGwFtw2kFn8XauW9xq8hBZ5kVtcSesTT9fW', // SLIM 37 | 'Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB', // USDT 38 | '7i5KKsX2weiTkry7jA4ZwSuXGhs5eJBEjY8vVxR4pfRx', // GMT 39 | 'foodQJAztMzX1DKpLaiounNe2BDMds5RNuPC6jsNrDG', // FOOOOOOD, 40 | '4k3Dyjzvzp8eMZWUXbBCjEvwSkkk59S5iCNLY3QrkX6R', // RAY, 41 | 'So11111111111111111111111111111111111111112', // SOL, 42 | '9LzCMqDgTKYz9Drzqnpgee3SGa89up3a247ypMj2xrqM', // AUDIO 43 | ] 44 | 45 | interface ThrottleOptions { 46 | throttle: number 47 | batchSignatures: number 48 | batchAccountsInfo: number 49 | batchTokenHolders: number 50 | } 51 | 52 | export class ProviderLegacyToken extends Provider { 53 | constructor( 54 | private readonly cdnUrl: string, 55 | private readonly rpcUrl: string, 56 | private readonly throttleOpts: ThrottleOptions = { 57 | throttle: 0, // Add sleep after batch RPC request to avoid rate limits 58 | batchSignatures: 100, // Batch RPC calls in single RPC request 59 | batchAccountsInfo: 250, // Batch RPC calls in single RPC request 60 | batchTokenHolders: 5, // Batch parallel RPC requests 61 | }, 62 | private readonly skipTags: Tag[], // Filter out specific tags 63 | private readonly chainId: number = 101, // Filter by chain id 64 | private readonly signatureDays = 30, // Filter tokens by last signature date 65 | private readonly minHolders = 100 // Filter tokens by number holders 66 | ) { 67 | super() 68 | } 69 | 70 | async getTokens(): Promise { 71 | const tokenMap = new TokenSet('LegacyProvider') 72 | 73 | const tokens = await axios.get(this.cdnUrl) 74 | for (let i = 0; i < tokens.data.tokens.length; i++) { 75 | const token: LegacyListToken = tokens.data.tokens[i] 76 | 77 | // Get only tokens for mainnet and devnet 78 | if ( 79 | this.chainId === token.chainId && 80 | !_.intersection(token.tags, this.skipTags).length 81 | ) { 82 | tokenMap.set({ 83 | chainId: token.chainId, 84 | name: token.name, 85 | symbol: token.symbol, 86 | address: token.address, 87 | decimals: null, 88 | logoURI: token.logoURI, 89 | tags: new Set(token.tags), 90 | verified: true, 91 | holders: null, 92 | }) 93 | } 94 | } 95 | 96 | await this.removeByContent(tokenMap, [ 97 | 'scam', 98 | 'phishing', 99 | 'please ignore', 100 | ]) 101 | await this.filterAccountInfo(tokenMap) 102 | await this.filterLatestSignature(tokenMap) 103 | await this.filterHolders(tokenMap) 104 | return tokenMap 105 | } 106 | 107 | /** 108 | * Remove tokens by their content 109 | * @param tokenMap 110 | * @param contentArray 111 | */ 112 | removeByContent(tokenMap: TokenSet, contentArray: string[]) { 113 | for (const token of tokenMap.tokens()) { 114 | for (const content of contentArray) { 115 | if (token.name.toLowerCase().includes(content.toLowerCase())) { 116 | tokenMap.deleteByToken(token) 117 | } 118 | if ( 119 | token.symbol.toLowerCase().includes(content.toLowerCase()) 120 | ) { 121 | tokenMap.deleteByToken(token) 122 | } 123 | } 124 | } 125 | } 126 | 127 | /** 128 | * Filter by account info 129 | * and fetch decimals 130 | * @param tokenMap 131 | */ 132 | async filterAccountInfo(tokenMap: TokenSet) { 133 | // Batch RPC calls 134 | let rpcCalls: object[] = [] 135 | for (const mint of tokenMap.mints()) { 136 | rpcCalls.push(RpcRequestAccountInfo(mint)) 137 | } 138 | 139 | // Chunk batched requests 140 | while (rpcCalls.length > 0) { 141 | let progress = 0 142 | const chunks = _.chunk( 143 | rpcCalls, 144 | this.throttleOpts.batchAccountsInfo 145 | ) 146 | rpcCalls = [] 147 | for (const dataChunk of chunks) { 148 | console.log( 149 | `[LTL] filter by account info ${++progress}/${ 150 | chunks.length 151 | }` 152 | ) 153 | 154 | const response = await axios.post( 155 | this.rpcUrl, 156 | dataChunk 157 | ) 158 | 159 | console.log( 160 | `[LTL] filter by account info done ${progress}/${chunks.length}` 161 | ) 162 | 163 | for (const mintResponse of response.data) { 164 | // Remove token if not a mint 165 | if ( 166 | !mintResponse.result || 167 | !mintResponse.result.value || 168 | !mintResponse.result.value.data['parsed'] || 169 | !mintResponse.result.value.data['program'] || 170 | (mintResponse.result.value.data.program !== 171 | 'spl-token' && 172 | mintResponse.result.value.data.program !== 173 | 'spl-token-2022') || 174 | mintResponse.result.value.data.parsed.type !== 'mint' 175 | ) { 176 | if (!mintResponse.result) { 177 | console.log( 178 | `[LTL] filter by account mint ${mintResponse.id} no result (chainId: ${this.chainId})` 179 | ) 180 | rpcCalls.push( 181 | RpcRequestAccountInfo(mintResponse.id) 182 | ) 183 | } else { 184 | tokenMap.deleteByMint(mintResponse.id, this.chainId) 185 | } 186 | continue 187 | } 188 | 189 | // Update decimals for token 190 | const token = tokenMap.getByMint( 191 | mintResponse.id, 192 | this.chainId 193 | ) 194 | if (token) { 195 | token.decimals = 196 | mintResponse.result.value.data.parsed.info.decimals 197 | tokenMap.set(token) 198 | } 199 | } 200 | 201 | if (this.throttleOpts.throttle > 0) { 202 | await new Promise((f) => 203 | setTimeout(f, this.throttleOpts.throttle) 204 | ) 205 | } 206 | } 207 | if (rpcCalls.length > 0) { 208 | console.log( 209 | `[LTL] filter by account mint, retry ${rpcCalls.length} failed requests (chainId: ${this.chainId})` 210 | ) 211 | } 212 | } 213 | } 214 | 215 | /** 216 | * Remove inactive tokens by their 217 | * latest signature from RPC 218 | * @param tokenMap 219 | */ 220 | async filterLatestSignature(tokenMap: TokenSet) { 221 | // Get cached recent signatures, so we can skip RPC requests for them. 222 | let cachedRecentSignatures = new Map() 223 | try { 224 | cachedRecentSignatures = new Map( 225 | Object.entries( 226 | JSON.parse( 227 | Provider.readCachedJSON( 228 | ProviderLegacyToken.cacheKeyRecentSignatures( 229 | this.chainId 230 | ) 231 | ) 232 | ) 233 | ) 234 | ) 235 | console.log('[LTL] Use cache for recent signatures') 236 | } catch (e) { 237 | console.log('[LTL] No cache for recent signatures') 238 | } 239 | 240 | // Calculate minimum unix timestamp 241 | const date = 242 | Math.ceil(Date.now() / 1000) - this.signatureDays * 24 * 60 * 60 243 | 244 | // Batch latest signature calls 245 | let rpcCalls: object[] = [] 246 | for (const mint of tokenMap.mints()) { 247 | const cached = cachedRecentSignatures.get(mint) 248 | if (cached && cached > date) { 249 | continue 250 | } 251 | 252 | rpcCalls.push(RpcRequestSignature(mint)) 253 | } 254 | 255 | while (rpcCalls.length > 0) { 256 | // Chunk batches 257 | let progress = 0 258 | const chunks = _.chunk(rpcCalls, this.throttleOpts.batchSignatures) 259 | rpcCalls = [] 260 | for (const dataChunk of chunks) { 261 | console.log( 262 | `[LTL] filter by signature ${++progress}/${ 263 | chunks.length 264 | } (chainId: ${this.chainId})` 265 | ) 266 | 267 | const response = await axios.post( 268 | this.rpcUrl, 269 | dataChunk 270 | ) 271 | 272 | for (const mintResponse of response.data) { 273 | if ( 274 | !mintResponse.result || 275 | !mintResponse.result.length || 276 | mintResponse.result[0].blockTime < date 277 | ) { 278 | if (!mintResponse.result) { 279 | console.log( 280 | `[LTL] filter by signature mint ${mintResponse.id} no result, retry (chainId: ${this.chainId})` 281 | ) 282 | rpcCalls.push(RpcRequestSignature(mintResponse.id)) 283 | } else { 284 | tokenMap.deleteByMint(mintResponse.id, this.chainId) 285 | } 286 | } else { 287 | cachedRecentSignatures.set( 288 | mintResponse.id, 289 | mintResponse.result[0].blockTime 290 | ) 291 | } 292 | } 293 | 294 | if (this.throttleOpts.throttle > 0) { 295 | await new Promise((f) => 296 | setTimeout(f, this.throttleOpts.throttle) 297 | ) 298 | } 299 | } 300 | if (rpcCalls.length > 0) { 301 | console.log( 302 | `[LTL] filter by signature, retry ${rpcCalls.length} failed requests (chainId: ${this.chainId})` 303 | ) 304 | } 305 | } 306 | 307 | Provider.saveCachedJSON( 308 | ProviderLegacyToken.cacheKeyRecentSignatures(this.chainId), 309 | JSON.stringify(Object.fromEntries(cachedRecentSignatures)) 310 | ) 311 | } 312 | 313 | /** 314 | * Remove tokens with few accounts 315 | * @param tokenMap 316 | */ 317 | async filterHolders(tokenMap: TokenSet) { 318 | // Get cached largest tokens, so we can skip RPC requests for them. 319 | let cachedLargeTokens = new Map() 320 | try { 321 | cachedLargeTokens = new Map( 322 | Object.entries( 323 | JSON.parse( 324 | Provider.readCachedJSON( 325 | ProviderLegacyToken.cacheKeyLargeAccounts( 326 | this.chainId 327 | ) 328 | ) 329 | ) 330 | ) 331 | ) 332 | console.log('[LTL] Use cache for large mints') 333 | } catch (e) { 334 | console.log('[LTL] No cache for large mints') 335 | } 336 | 337 | const mints = tokenMap.mints() 338 | 339 | const mintToCheck: string[] = [] 340 | for (const mint of mints) { 341 | if (LARGEST_MINTS.includes(mint) || cachedLargeTokens.has(mint)) { 342 | const token = tokenMap.getByMint(mint, this.chainId) 343 | if (token) { 344 | token.holders = LARGEST_MINTS.includes(mint) 345 | ? 100000 346 | : (cachedLargeTokens.get(mint) as number) 347 | tokenMap.set(token) 348 | } 349 | continue 350 | } 351 | mintToCheck.push(mint) 352 | } 353 | 354 | // Chunk them so we can parallel send multiple RPC requests 355 | let progress = 0 356 | const batches = _.chunk( 357 | mintToCheck, 358 | this.throttleOpts.batchTokenHolders 359 | ) 360 | 361 | for (const batch of batches) { 362 | console.log( 363 | `[LTL] filter by holder ${++progress}/${batches.length}` 364 | ) 365 | 366 | const requests: AxiosPromise[] = [] 367 | 368 | for (const mint of batch) { 369 | requests.push( 370 | axios.post( 371 | this.rpcUrl, 372 | RpcRequestHolders(mint) 373 | ) 374 | ) 375 | } 376 | 377 | // Wait for batch of axios request to finish 378 | const responses = await Promise.allSettled(requests) 379 | for (const response of responses) { 380 | if (response.status === 'fulfilled') { 381 | const mint = response.value.data.id 382 | let count = 0 383 | 384 | if (response.value.data.error) { 385 | console.log( 386 | `[LTL] Failed RPC holders call for ${mint}`, 387 | response.value.data 388 | ) 389 | 390 | if ( 391 | response.value.data.error && 392 | response.value.data.error.data.includes( 393 | 'Exceeded max limit' 394 | ) 395 | ) { 396 | count = 100000 397 | } else { 398 | console.log(`[LTL] Skip holder check for ${mint}`) 399 | continue 400 | } 401 | } 402 | 403 | count = response.value.data.result.length 404 | 405 | if (count < this.minHolders) { 406 | tokenMap.deleteByMint(mint, this.chainId) 407 | continue 408 | } 409 | 410 | if (count >= 1000) { 411 | cachedLargeTokens.set(mint, count) 412 | Provider.saveCachedJSON( 413 | ProviderLegacyToken.cacheKeyLargeAccounts( 414 | this.chainId 415 | ), 416 | JSON.stringify( 417 | Object.fromEntries(cachedLargeTokens) 418 | ) 419 | ) 420 | } 421 | 422 | // Update decimals for token 423 | const token = tokenMap.getByMint(mint, this.chainId) 424 | if (token) { 425 | token.holders = count 426 | tokenMap.set(token) 427 | } 428 | } else { 429 | throw new Error( 430 | `Failed to fetch holders: ${response.reason}` 431 | ) 432 | } 433 | } 434 | 435 | if (this.throttleOpts.throttle > 0) { 436 | await new Promise((f) => 437 | setTimeout(f, this.throttleOpts.throttle) 438 | ) 439 | } 440 | } 441 | } 442 | 443 | public static clearCache(chainId: number) { 444 | this.removeCachedJSON( 445 | ProviderLegacyToken.cacheKeyLargeAccounts(chainId) 446 | ) 447 | this.removeCachedJSON( 448 | ProviderLegacyToken.cacheKeyRecentSignatures(chainId) 449 | ) 450 | } 451 | 452 | private static cacheKeyLargeAccounts(chainId: number) { 453 | return `legacy-list-large-mints-${chainId}` 454 | } 455 | 456 | private static cacheKeyRecentSignatures(chainId: number) { 457 | return `legacy-list-recent-signatures-${chainId}` 458 | } 459 | } 460 | --------------------------------------------------------------------------------