├── .yarnrc ├── .env.dist ├── .prettierignore ├── .gitignore ├── .eslintignore ├── src ├── network-observer │ ├── observation-state.ts │ ├── consensus-timer.ts │ ├── peer-event-handler │ │ ├── stellar-message-handlers │ │ │ ├── scp-envelope │ │ │ │ ├── scp-statement │ │ │ │ │ ├── externalize │ │ │ │ │ │ ├── extract-close-time-from-value.ts │ │ │ │ │ │ ├── __tests__ │ │ │ │ │ │ │ ├── extract-close-time-from-value.test.ts │ │ │ │ │ │ │ ├── slots.test.ts │ │ │ │ │ │ │ ├── slot.test.ts │ │ │ │ │ │ │ └── externalize-statement-handler.test.ts │ │ │ │ │ │ ├── slots.ts │ │ │ │ │ │ ├── map-externalize-statement.ts │ │ │ │ │ │ ├── slot.ts │ │ │ │ │ │ └── externalize-statement-handler.ts │ │ │ │ │ ├── scp-statement-handler.ts │ │ │ │ │ └── __tests__ │ │ │ │ │ │ └── scp-statement-handler.test.ts │ │ │ │ ├── ledger-validator.ts │ │ │ │ ├── __tests__ │ │ │ │ │ ├── ledger-validator.test.ts │ │ │ │ │ └── scp-envelope-handler.test.ts │ │ │ │ └── scp-envelope-handler.ts │ │ │ ├── readme.md │ │ │ ├── __tests__ │ │ │ │ └── stellar-message-handler.test.ts │ │ │ └── stellar-message-handler.ts │ │ ├── on-peer-connection-closed.ts │ │ ├── __tests__ │ │ │ ├── on-connection-closed.test.ts │ │ │ ├── peer-event-handler.test.ts │ │ │ ├── on-peer-data.test.ts │ │ │ └── on-peer-connected.test.ts │ │ ├── peer-event-handler.ts │ │ ├── on-peer-data.ts │ │ └── on-peer-connected.ts │ ├── quorum-set-state.ts │ ├── observation-factory.ts │ ├── straggler-timer.ts │ ├── observation.ts │ ├── __tests__ │ │ ├── straggler-timer.test.ts │ │ ├── observation.test.ts │ │ ├── observation-manager.test.ts │ │ ├── connection-manager.test.ts │ │ └── network-observer.test.ts │ ├── network-observer.ts │ ├── observation-manager.ts │ ├── connection-manager.ts │ └── quorum-set-manager.ts ├── utilities │ ├── timer-factory.ts │ ├── truncate.ts │ ├── timer.ts │ ├── timers.ts │ ├── json-storage.ts │ └── __tests__ │ │ └── timers.test.ts ├── node-address.ts ├── crawl-task.ts ├── crawl-result.ts ├── __fixtures__ │ ├── createDummyErrLoadMessage.ts │ ├── createDummyDontHaveMessage.ts │ ├── createDummyPeersMessage.ts │ ├── createDummyQuorumSetMessage.ts │ ├── createDummyCrawlerConfiguration.ts │ └── createDummyExternalizeMessage.ts ├── max-crawl-time-manager.ts ├── crawl.ts ├── __tests__ │ ├── max-crawl-time-manager.test.ts │ ├── crawl-queue.test.ts │ ├── peer-node.test.ts │ ├── crawl-queue-manager.test.ts │ ├── peer-node-collection.test.ts │ ├── crawler.test.ts │ └── crawler.integration.test.ts ├── crawler-configuration.ts ├── crawl-factory.ts ├── crawl-queue.ts ├── crawl-queue-manager.ts ├── peer-node.ts ├── peer-node-collection.ts ├── README.md ├── crawl-logger.ts ├── index.ts └── crawler.ts ├── prettier.config.js ├── tsconfig.json ├── jest.config.js ├── .eslintrc.js ├── .github └── workflows │ └── test.yml ├── LICENSE ├── seed ├── nodes-single.json └── nodes-testnet.json ├── package.json ├── README.md └── examples └── crawl.js /.yarnrc: -------------------------------------------------------------------------------- 1 | --install.ignore-optional true 2 | ignore-optional true -------------------------------------------------------------------------------- /.env.dist: -------------------------------------------------------------------------------- 1 | LOG_LEVEL=debug 2 | LEDGER_VERSION=21 3 | OVERLAY_VERSION=33 4 | OVERLAY_MIN_VERSION=27 5 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /package.json 2 | /node_modules 3 | /lib 4 | /dist 5 | /src/generated 6 | /src/vendor 7 | /docs -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules 3 | dist/ 4 | flow-typed/ 5 | .env 6 | lib/ 7 | .clinic/ 8 | crawl_result/ 9 | coverage/ -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | # don't ever lint node_modules 2 | node_modules 3 | # don't lint nyc coverage output 4 | coverage 5 | examples 6 | lib -------------------------------------------------------------------------------- /src/network-observer/observation-state.ts: -------------------------------------------------------------------------------- 1 | export enum ObservationState { 2 | Idle, 3 | Syncing, 4 | Synced, 5 | Stopping, 6 | Stopped 7 | } 8 | -------------------------------------------------------------------------------- /src/utilities/timer-factory.ts: -------------------------------------------------------------------------------- 1 | import { Timer } from './timer'; 2 | 3 | export class TimerFactory { 4 | createTimer() { 5 | return new Timer(); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/node-address.ts: -------------------------------------------------------------------------------- 1 | export type NodeAddress = [ip: string, port: number]; 2 | 3 | export function nodeAddressToPeerKey(nodeAddress: NodeAddress) { 4 | return nodeAddress[0] + ':' + nodeAddress[1]; 5 | } 6 | -------------------------------------------------------------------------------- /src/utilities/truncate.ts: -------------------------------------------------------------------------------- 1 | export function truncate(str?: string) { 2 | if (!str) return str; 3 | return str.length > 20 4 | ? str.substring(0, 5) + '...' + str.substring(str.length - 5, str.length) 5 | : str; 6 | } 7 | -------------------------------------------------------------------------------- /src/crawl-task.ts: -------------------------------------------------------------------------------- 1 | import { NodeAddress } from './node-address'; 2 | import { Crawl } from './crawl'; 3 | 4 | export interface CrawlTask { 5 | nodeAddress: NodeAddress; 6 | crawl: Crawl; 7 | connectCallback: () => void; 8 | } 9 | -------------------------------------------------------------------------------- /src/crawl-result.ts: -------------------------------------------------------------------------------- 1 | import { PeerNode } from './peer-node'; 2 | import { Ledger } from './crawler'; 3 | 4 | export interface CrawlResult { 5 | peers: Map; 6 | closedLedgers: bigint[]; 7 | latestClosedLedger: Ledger; 8 | } 9 | -------------------------------------------------------------------------------- /prettier.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | arrowParens: 'always', 3 | bracketSpacing: true, 4 | jsxBracketSameLine: false, 5 | printWidth: 80, 6 | proseWrap: 'always', 7 | semi: true, 8 | singleQuote: true, 9 | tabWidth: 2, 10 | trailingComma: 'none', 11 | useTabs: true 12 | }; 13 | -------------------------------------------------------------------------------- /src/__fixtures__/createDummyErrLoadMessage.ts: -------------------------------------------------------------------------------- 1 | import { xdr } from '@stellar/stellar-base'; 2 | 3 | export function createDummyErrLoadMessage() { 4 | return xdr.StellarMessage.errorMsg( 5 | new xdr.Error({ 6 | code: xdr.ErrorCode.errLoad(), 7 | msg: 'Error loading' 8 | }) 9 | ); 10 | } 11 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "commonjs", 5 | "outDir": "lib", 6 | "sourceMap": true, 7 | "types": ["node", "jest"], 8 | "strict": true 9 | }, 10 | "include": [ 11 | "src/**/*.ts" 12 | ], 13 | "exclude": [ 14 | "node_modules" 15 | ] 16 | } -------------------------------------------------------------------------------- /src/__fixtures__/createDummyDontHaveMessage.ts: -------------------------------------------------------------------------------- 1 | import { xdr } from '@stellar/stellar-base'; 2 | 3 | export function createDummyDontHaveMessage() { 4 | const dontHave = new xdr.DontHave({ 5 | reqHash: Buffer.alloc(32), 6 | type: xdr.MessageType.getScpQuorumset() 7 | }); 8 | return xdr.StellarMessage.dontHave(dontHave); 9 | } 10 | -------------------------------------------------------------------------------- /src/network-observer/consensus-timer.ts: -------------------------------------------------------------------------------- 1 | import { Timer } from '../utilities/timer'; 2 | 3 | export class ConsensusTimer { 4 | constructor(private timer: Timer, private consensusTimeoutMS: number) {} 5 | 6 | start(callback: () => void) { 7 | this.timer.start(this.consensusTimeoutMS, callback); 8 | } 9 | 10 | stop() { 11 | this.timer.stopTimer(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/__fixtures__/createDummyPeersMessage.ts: -------------------------------------------------------------------------------- 1 | import { xdr } from '@stellar/stellar-base'; 2 | 3 | export function createDummyPeersMessage(): xdr.StellarMessage { 4 | const peerAddress = new xdr.PeerAddress({ 5 | ip: xdr.PeerAddressIp.iPv4(Buffer.from([127, 0, 0, 1])), 6 | port: 11625, 7 | numFailures: 0 8 | }); 9 | 10 | return xdr.StellarMessage.peers([peerAddress]); 11 | } 12 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "transform": { 3 | ".(ts|tsx)": "ts-jest" 4 | }, 5 | "testPathIgnorePatterns": [ 6 | "/node_modules/", 7 | "/lib/" 8 | ], 9 | "testRegex": "(/test/.*|\\.(test|spec))\\.(ts|tsx|js)$", 10 | "moduleFileExtensions": [ 11 | "ts", 12 | "tsx", 13 | "js", 14 | "json" 15 | ] 16 | }; -------------------------------------------------------------------------------- /src/max-crawl-time-manager.ts: -------------------------------------------------------------------------------- 1 | export class MaxCrawlTimeManager { 2 | private timer?: NodeJS.Timeout; 3 | 4 | setTimer(maxCrawlTime: number, onMaxCrawlTime: () => void) { 5 | if (this.timer) { 6 | clearTimeout(this.timer); 7 | } 8 | this.timer = setTimeout(onMaxCrawlTime, maxCrawlTime); 9 | } 10 | 11 | clearTimer() { 12 | if (this.timer) { 13 | clearTimeout(this.timer); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/utilities/timer.ts: -------------------------------------------------------------------------------- 1 | export class Timer { 2 | private timer: NodeJS.Timeout | null = null; 3 | 4 | constructor() {} 5 | 6 | start(time: number, callback: () => void) { 7 | if (this.timer) { 8 | clearTimeout(this.timer); 9 | } 10 | this.timer = setTimeout(() => { 11 | callback(); 12 | }, time); 13 | } 14 | 15 | stopTimer() { 16 | if (this.timer) { 17 | clearTimeout(this.timer); 18 | this.timer = null; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/extract-close-time-from-value.ts: -------------------------------------------------------------------------------- 1 | import { xdr } from '@stellar/stellar-base'; 2 | 3 | export function extractCloseTimeFromValue(value: Buffer) { 4 | try { 5 | return new Date( 6 | 1000 * 7 | Number( 8 | xdr.StellarValue.fromXDR(value).closeTime().toXDR().readBigUInt64BE() 9 | ) 10 | ); 11 | } catch (error) { 12 | return new Date(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/network-observer/quorum-set-state.ts: -------------------------------------------------------------------------------- 1 | import { PublicKey } from '@stellarbeat/js-stellarbeat-shared'; 2 | 3 | export type QuorumSetHash = string; 4 | 5 | export class QuorumSetState { 6 | quorumSetOwners: Map> = new Map(); 7 | quorumSetRequestedTo: Map> = new Map(); 8 | quorumSetHashesInProgress: Set = new Set(); 9 | quorumSetRequests: Map< 10 | PublicKey, 11 | { 12 | timeout: NodeJS.Timeout; 13 | hash: QuorumSetHash; 14 | } 15 | > = new Map(); 16 | } 17 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-undef 2 | module.exports = { 3 | root: true, 4 | parser: '@typescript-eslint/parser', 5 | plugins: ['@typescript-eslint'], 6 | extends: [ 7 | 'eslint:recommended', 8 | 'plugin:@typescript-eslint/recommended', 9 | 'prettier' 10 | ], 11 | rules: { 12 | '@typescript-eslint/ban-ts-comment': 'off', 13 | '@typescript-eslint/no-explicit-any': 'off', 14 | '@typescript-eslint/no-unused-vars': 'off', 15 | '@typescript-eslint/no-unsafe-declaration-merging': 'off', 16 | }, 17 | env: { 18 | node: true 19 | } 20 | }; 21 | -------------------------------------------------------------------------------- /src/__fixtures__/createDummyQuorumSetMessage.ts: -------------------------------------------------------------------------------- 1 | import { Keypair, xdr } from '@stellar/stellar-base'; 2 | 3 | export function createDummyQuorumSetMessage(): xdr.StellarMessage { 4 | const keypair1 = Keypair.random(); 5 | const keypair2 = Keypair.random(); 6 | const qSet = new xdr.ScpQuorumSet({ 7 | threshold: 1, 8 | validators: [ 9 | xdr.PublicKey.publicKeyTypeEd25519(keypair1.rawPublicKey()), 10 | xdr.PublicKey.publicKeyTypeEd25519(keypair2.rawPublicKey()) 11 | ], 12 | innerSets: [] 13 | }); 14 | 15 | return xdr.StellarMessage.scpQuorumset(qSet); 16 | } 17 | -------------------------------------------------------------------------------- /src/utilities/timers.ts: -------------------------------------------------------------------------------- 1 | import { Timer } from './timer'; 2 | import { TimerFactory } from './timer-factory'; 3 | 4 | export class Timers { 5 | private timers: Set = new Set(); 6 | 7 | constructor(private timerFactory: TimerFactory) {} 8 | 9 | public startTimer(time: number, callback: () => void) { 10 | const timer = this.timerFactory.createTimer(); 11 | const myCallback = () => { 12 | this.timers.delete(timer); 13 | callback(); 14 | }; 15 | timer.start(time, myCallback); 16 | this.timers.add(timer); 17 | } 18 | 19 | public stopTimers() { 20 | this.timers.forEach((timer) => timer.stopTimer()); 21 | this.timers = new Set(); 22 | } 23 | 24 | public hasActiveTimers() { 25 | return this.timers.size > 0; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | node-version: [20.x, 21.x, 22.x] 15 | # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - uses: pnpm/action-setup@v4 20 | - name: Use Node.js ${{ matrix.node-version }} 21 | uses: actions/setup-node@v2 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | cache: 'pnpm' 25 | - run: pnpm install 26 | - run: pnpm build 27 | - run: pnpm test 28 | -------------------------------------------------------------------------------- /src/crawl.ts: -------------------------------------------------------------------------------- 1 | import { AsyncResultCallback } from 'async'; 2 | import { NodeAddress } from './node-address'; 3 | import { Observation } from './network-observer/observation'; 4 | 5 | type PeerKey = string; //ip:port 6 | 7 | export enum CrawlProcessState { 8 | IDLE, 9 | TOP_TIER_SYNC, 10 | CRAWLING, 11 | STOPPING 12 | } 13 | 14 | export class Crawl { 15 | state: CrawlProcessState = CrawlProcessState.IDLE; 16 | maxCrawlTimeHit = false; 17 | crawlQueueTaskDoneCallbacks: Map> = 18 | new Map(); 19 | crawledNodeAddresses: Set = new Set(); 20 | 21 | failedConnections: string[] = []; 22 | peerAddressesReceivedDuringSync: NodeAddress[] = []; 23 | 24 | constructor( 25 | public nodesToCrawl: NodeAddress[], 26 | public observation: Observation 27 | ) {} 28 | } 29 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/__tests__/extract-close-time-from-value.test.ts: -------------------------------------------------------------------------------- 1 | import { extractCloseTimeFromValue } from '../extract-close-time-from-value'; 2 | import { createDummyExternalizeMessage } from '../../../../../../../__fixtures__/createDummyExternalizeMessage'; 3 | 4 | describe('extract-close-time-from-value', () => { 5 | it('should extract close time from value', () => { 6 | const externalizeMessage = createDummyExternalizeMessage(); 7 | const value = externalizeMessage 8 | .envelope() 9 | .statement() 10 | .pledges() 11 | .externalize() 12 | .commit() 13 | .value(); 14 | const result = extractCloseTimeFromValue(value); 15 | expect(result).toEqual(new Date('2024-02-27T08:36:24.000Z')); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /src/__fixtures__/createDummyCrawlerConfiguration.ts: -------------------------------------------------------------------------------- 1 | import { CrawlerConfiguration } from '../crawler-configuration'; 2 | import { NodeConfig } from '@stellarbeat/js-stellar-node-connector/lib/node-config'; 3 | 4 | export function createDummyCrawlerConfiguration(): CrawlerConfiguration { 5 | const nodeConfig: NodeConfig = { 6 | network: 'test', 7 | nodeInfo: { 8 | ledgerVersion: 1, 9 | overlayVersion: 3, 10 | overlayMinVersion: 2, 11 | versionString: '1', 12 | networkId: 'test' 13 | }, 14 | listeningPort: 11625, 15 | privateKey: 'secret', 16 | receiveTransactionMessages: false, 17 | receiveSCPMessages: true, 18 | peerFloodReadingCapacity: 100, 19 | flowControlSendMoreBatchSize: 100, 20 | peerFloodReadingCapacityBytes: 10000, 21 | flowControlSendMoreBatchSizeBytes: 10000 22 | }; 23 | return new CrawlerConfiguration(nodeConfig); 24 | } 25 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/ledger-validator.ts: -------------------------------------------------------------------------------- 1 | import { Ledger } from '../../../../crawler'; 2 | 3 | const MAX_LEDGER_DRIFT = 5; //how many ledgers can a node fall behind 4 | const MAX_CLOSED_LEDGER_PROCESSING_TIME = 90000; //how long in ms we still process messages of closed ledgers. 5 | 6 | export function isLedgerSequenceValid( 7 | latestClosedLedger: Ledger, 8 | ledgerSequence: bigint 9 | ): boolean { 10 | const latestSequenceDifference = Number( 11 | latestClosedLedger.sequence - ledgerSequence 12 | ); 13 | 14 | if (latestSequenceDifference > MAX_LEDGER_DRIFT) return false; //ledger message older than allowed by pure ledger sequence numbers 15 | 16 | return !( 17 | ledgerSequence <= latestClosedLedger.sequence && 18 | new Date().getTime() - latestClosedLedger.closeTime.getTime() > 19 | MAX_CLOSED_LEDGER_PROCESSING_TIME 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /src/network-observer/observation-factory.ts: -------------------------------------------------------------------------------- 1 | import { Observation } from './observation'; 2 | import { NodeAddress } from '../node-address'; 3 | import { PeerNodeCollection } from '../peer-node-collection'; 4 | import { Slots } from './peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 5 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 6 | import { Ledger } from '../crawler'; 7 | 8 | export class ObservationFactory { 9 | public createObservation( 10 | network: string, 11 | slots: Slots, 12 | topTierAddresses: NodeAddress[], 13 | peerNodes: PeerNodeCollection, 14 | latestConfirmedClosedLedger: Ledger, 15 | quorumSets: Map 16 | ): Observation { 17 | return new Observation( 18 | network, 19 | topTierAddresses, 20 | peerNodes, 21 | latestConfirmedClosedLedger, 22 | quorumSets, 23 | slots 24 | ); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots.ts: -------------------------------------------------------------------------------- 1 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 2 | import * as P from 'pino'; 3 | import { Slot, SlotIndex } from './slot'; 4 | 5 | export class Slots { 6 | protected slots: Map = new Map(); 7 | protected trustedQuorumSet: QuorumSet; 8 | 9 | constructor(trustedQuorumSet: QuorumSet, protected logger: P.Logger) { 10 | this.trustedQuorumSet = trustedQuorumSet; 11 | } 12 | 13 | public getSlot(slotIndex: SlotIndex): Slot { 14 | let slot = this.slots.get(slotIndex); 15 | if (!slot) { 16 | slot = new Slot(slotIndex, this.trustedQuorumSet, this.logger); 17 | this.slots.set(slotIndex, slot); 18 | } 19 | 20 | return slot; 21 | } 22 | 23 | getConfirmedClosedSlotIndexes(): bigint[] { 24 | return Array.from(this.slots.values()) 25 | .filter((slot) => slot.confirmedClosed()) 26 | .map((slot) => slot.index); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/on-peer-connection-closed.ts: -------------------------------------------------------------------------------- 1 | import { ClosePayload } from '../connection-manager'; 2 | import { truncate } from '../../utilities/truncate'; 3 | import { QuorumSetManager } from '../quorum-set-manager'; 4 | import { P } from 'pino'; 5 | import { Observation } from '../observation'; 6 | 7 | export class OnPeerConnectionClosed { 8 | constructor( 9 | private quorumSetManager: QuorumSetManager, 10 | private logger: P.Logger 11 | ) {} 12 | 13 | public handle(data: ClosePayload, observation: Observation) { 14 | this.logIfTopTierDisconnect(data, observation.topTierAddressesSet); 15 | if (data.publicKey) { 16 | this.quorumSetManager.onNodeDisconnected(data.publicKey, observation); 17 | } 18 | } 19 | 20 | private logIfTopTierDisconnect( 21 | data: ClosePayload, 22 | topTierAddresses: Set 23 | ) { 24 | if (topTierAddresses.has(data.address)) { 25 | this.logger.debug( 26 | { pk: truncate(data.publicKey), address: data.address }, 27 | 'Top tier node disconnected' 28 | ); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/__tests__/max-crawl-time-manager.test.ts: -------------------------------------------------------------------------------- 1 | import { MaxCrawlTimeManager } from '../max-crawl-time-manager'; 2 | 3 | describe('MaxCrawlTimeManager', () => { 4 | describe('setTimer', () => { 5 | it('should set a timer', (resolve) => { 6 | // Arrange 7 | const maxCrawlTimeManager = new MaxCrawlTimeManager(); 8 | const onMaxCrawlTime = jest.fn(); 9 | const maxCrawlTime = 200; 10 | // Act 11 | maxCrawlTimeManager.setTimer(maxCrawlTime, onMaxCrawlTime); 12 | setTimeout(() => { 13 | expect(onMaxCrawlTime).toBeCalled(); 14 | resolve(); 15 | }, 300); 16 | // Assert 17 | }); 18 | }); 19 | describe('clearTimer', () => { 20 | it('should clear the timer', (resolve) => { 21 | // Arrange 22 | const maxCrawlTimeManager = new MaxCrawlTimeManager(); 23 | const onMaxCrawlTime = jest.fn(); 24 | const maxCrawlTime = 200; 25 | maxCrawlTimeManager.setTimer(maxCrawlTime, onMaxCrawlTime); 26 | maxCrawlTimeManager.clearTimer(); 27 | setTimeout(() => { 28 | expect(onMaxCrawlTime).not.toBeCalled(); 29 | resolve(); 30 | }, 300); 31 | }); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /src/utilities/json-storage.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | import { Node } from '@stellarbeat/js-stellarbeat-shared'; 3 | 4 | export default { 5 | readFilePromise: function (path: string): Promise { 6 | return new Promise((resolve, reject) => 7 | fs.readFile(path, 'utf8', function callback(err, data) { 8 | if (err) { 9 | reject(err); 10 | } else { 11 | resolve(data); 12 | } 13 | }) 14 | ); 15 | }, 16 | 17 | writeFilePromise: function ( 18 | fileName: string, 19 | data: string 20 | ): Promise { 21 | return new Promise((resolve, reject) => 22 | fs.writeFile(fileName, data, 'utf8', function callback(err) { 23 | if (err) { 24 | reject(err); 25 | } else { 26 | resolve({}); 27 | } 28 | }) 29 | ); 30 | }, 31 | 32 | getNodesFromFile: async function (fileName: string): Promise { 33 | const nodesJson = (await this.readFilePromise(fileName)) as string; 34 | const nodesRaw = JSON.parse(nodesJson); 35 | 36 | return nodesRaw.map((node: Record) => { 37 | //@ts-ignore 38 | return Node.fromNodeV1DTO(node); 39 | }); 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 stellarbeat.io 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/crawler-configuration.ts: -------------------------------------------------------------------------------- 1 | import { NodeConfig } from '@stellarbeat/js-stellar-node-connector/lib/node-config'; 2 | 3 | type PublicKey = string; 4 | 5 | export class CrawlerConfiguration { 6 | constructor( 7 | public nodeConfig: NodeConfig, //How many connections can be open at the same time. The higher the number, the faster the crawl 8 | public maxOpenConnections = 25, //How many (non-top tier) peer connections can be open at the same time. The higher the number, the faster the crawl, but the more risk of higher latencies 9 | public maxCrawlTime = 1800000, //max nr of ms the crawl will last. Safety guard in case crawler is stuck. 10 | public blackList = new Set(), //nodes that are not crawled 11 | public peerStraggleTimeoutMS = 10000, //time in ms that we listen to a node to determine if it is validating a confirmed closed ledger 12 | public syncingTimeoutMS = 10000, //time in ms that the network observer waits for the top tiers to sync 13 | public quorumSetRequestTimeoutMS = 1500, //time in ms that we wait for a quorum set to be received from a single peer 14 | public consensusTimeoutMS = 90000 //time in ms before we declare the network stuck. 15 | ) {} 16 | } 17 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/__tests__/on-connection-closed.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { QuorumSetManager } from '../../quorum-set-manager'; 3 | import { P } from 'pino'; 4 | import { OnPeerConnectionClosed } from '../on-peer-connection-closed'; 5 | import { Observation } from '../../observation'; 6 | 7 | describe('OnConnectionCloseHandler', () => { 8 | const quorumSetManager = mock(); 9 | const logger = mock(); 10 | 11 | beforeEach(() => { 12 | jest.clearAllMocks(); 13 | }); 14 | 15 | function createConnectionCloseHandler() { 16 | return new OnPeerConnectionClosed(quorumSetManager, logger); 17 | } 18 | 19 | it('should stop quorum requests', () => { 20 | const onConnectionCloseHandler = createConnectionCloseHandler(); 21 | const data = { 22 | publicKey: 'publicKey', 23 | address: 'address' 24 | }; 25 | const observation = mock(); 26 | observation.topTierAddressesSet = new Set(); 27 | onConnectionCloseHandler.handle(data, observation); 28 | expect(quorumSetManager.onNodeDisconnected).toHaveBeenCalledWith( 29 | data.publicKey, 30 | observation 31 | ); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /src/crawl-factory.ts: -------------------------------------------------------------------------------- 1 | import { Crawl } from './crawl'; 2 | import { ObservationFactory } from './network-observer/observation-factory'; 3 | import { Slots } from './network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 4 | import { NodeAddress } from './node-address'; 5 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 6 | import { Ledger } from './crawler'; 7 | import { P } from 'pino'; 8 | import { PeerNodeCollection } from './peer-node-collection'; 9 | 10 | export class CrawlFactory { 11 | constructor( 12 | private observationFactory: ObservationFactory, 13 | private network: string, 14 | private logger: P.Logger 15 | ) {} 16 | public createCrawl( 17 | nodesToCrawl: NodeAddress[], 18 | topTierAddresses: NodeAddress[], 19 | topTierQuorumSet: QuorumSet, 20 | latestConfirmedClosedLedger: Ledger, 21 | quorumSets: Map 22 | ): Crawl { 23 | const observation = this.observationFactory.createObservation( 24 | this.network, 25 | new Slots(topTierQuorumSet, this.logger), 26 | topTierAddresses, 27 | new PeerNodeCollection(), 28 | latestConfirmedClosedLedger, 29 | quorumSets 30 | ); 31 | return new Crawl(nodesToCrawl, observation); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/peer-event-handler.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ClosePayload, 3 | ConnectedPayload, 4 | DataPayload 5 | } from '../connection-manager'; 6 | import { Ledger } from '../../crawler'; 7 | import { NodeAddress } from '../../node-address'; 8 | import { OnPeerConnected } from './on-peer-connected'; 9 | import { OnPeerConnectionClosed } from './on-peer-connection-closed'; 10 | import { OnPeerData } from './on-peer-data'; 11 | import { Observation } from '../observation'; 12 | 13 | export class PeerEventHandler { 14 | constructor( 15 | private onConnectedHandler: OnPeerConnected, 16 | private onConnectionCloseHandler: OnPeerConnectionClosed, 17 | private onPeerDataHandler: OnPeerData 18 | ) {} 19 | 20 | public onConnected(data: ConnectedPayload, observation: Observation) { 21 | this.onConnectedHandler.handle(data, observation); 22 | } 23 | 24 | public onConnectionClose(data: ClosePayload, observation: Observation) { 25 | this.onConnectionCloseHandler.handle(data, observation); 26 | } 27 | 28 | public onData( 29 | data: DataPayload, 30 | observation: Observation 31 | ): { 32 | closedLedger: Ledger | null; 33 | peers: Array; 34 | } { 35 | return this.onPeerDataHandler.handle(data, observation); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/map-externalize-statement.ts: -------------------------------------------------------------------------------- 1 | import { xdr } from '@stellar/stellar-base'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import { getPublicKeyStringFromBuffer } from '@stellarbeat/js-stellar-node-connector'; 4 | import { extractCloseTimeFromValue } from './extract-close-time-from-value'; 5 | 6 | export interface ExternalizeData { 7 | publicKey: string; 8 | slotIndex: bigint; 9 | value: string; 10 | closeTime: Date; 11 | } 12 | 13 | export function mapExternalizeStatement( 14 | externalizeStatement: xdr.ScpStatement 15 | ): Result { 16 | const publicKeyResult = getPublicKeyStringFromBuffer( 17 | externalizeStatement.nodeId().value() 18 | ); 19 | if (publicKeyResult.isErr()) { 20 | return err(publicKeyResult.error); 21 | } 22 | 23 | const publicKey = publicKeyResult.value; 24 | const slotIndex = BigInt(externalizeStatement.slotIndex().toString()); 25 | 26 | const value = externalizeStatement.pledges().externalize().commit().value(); 27 | 28 | const closeTime = extractCloseTimeFromValue(value); 29 | 30 | return ok({ 31 | publicKey, 32 | slotIndex, 33 | value: value.toString('base64'), 34 | closeTime 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /src/network-observer/straggler-timer.ts: -------------------------------------------------------------------------------- 1 | import { ConnectionManager } from './connection-manager'; 2 | import { P } from 'pino'; 3 | import { Timers } from '../utilities/timers'; 4 | 5 | export class StragglerTimer { 6 | constructor( 7 | private connectionManager: ConnectionManager, 8 | private timers: Timers, 9 | private straggleTimeoutMS: number, 10 | private logger: P.Logger 11 | ) {} 12 | 13 | public startStragglerTimeoutForActivePeers( 14 | includeTopTier = false, 15 | topTierAddresses: Set, 16 | done?: () => void 17 | ) { 18 | const activePeers = this.connectionManager 19 | .getActiveConnectionAddresses() 20 | .filter((address) => { 21 | return includeTopTier || !topTierAddresses.has(address); 22 | }); 23 | this.startStragglerTimeout(activePeers, done); 24 | } 25 | 26 | public startStragglerTimeout(addresses: string[], done?: () => void) { 27 | if (addresses.length === 0) return; 28 | this.timers.startTimer(this.straggleTimeoutMS, () => { 29 | this.logger.debug({ addresses }, 'Straggler timeout hit'); 30 | addresses.forEach((address) => { 31 | this.connectionManager.disconnectByAddress(address); 32 | }); 33 | if (done) done(); 34 | }); 35 | } 36 | 37 | public stopStragglerTimeouts() { 38 | this.timers.stopTimers(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /seed/nodes-single.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ip": "35.229.188.114", 4 | "port": 11625, 5 | "publicKey": "GDMAU3NHV4H7NZF5PY6O6SULIUKIIHPRYOKM7HMREK4BW65VHMDKNM6M", 6 | "ledgerVersion": 13, 7 | "overlayVersion": 13, 8 | "overlayMinVersion": 11, 9 | "networkId": "esM5l1ROMXXSZr0CJDmyLNsWUIwBFj8m5csqPhBFqXk=", 10 | "versionStr": "stellar-core 13.0.0 (9ed3da29be1c2c932b946025ca2907646a9072f4)", 11 | "active": true, 12 | "overLoaded": false, 13 | "quorumSet": { 14 | "threshold": 9007199254740991, 15 | "validators": [], 16 | "innerQuorumSets": [] 17 | }, 18 | "geoData": { 19 | "dateUpdated": "2020-05-31T12:40:27.290Z" 20 | }, 21 | "statistics": { 22 | "activeInLastCrawl": false, 23 | "overLoadedInLastCrawl": false, 24 | "validatingInLastCrawl": false, 25 | "active30DaysPercentage": 0, 26 | "overLoaded30DaysPercentage": 0, 27 | "validating30DaysPercentage": 0, 28 | "active24HoursPercentage": 0, 29 | "overLoaded24HoursPercentage": 0, 30 | "validating24HoursPercentage": 0, 31 | "has30DayStats": true 32 | }, 33 | "dateDiscovered": "2020-05-31T12:40:27.290Z", 34 | "dateUpdated": "2020-05-31T12:40:27.922Z", 35 | "isValidator": false, 36 | "isFullValidator": false, 37 | "isValidating": false, 38 | "index": 0 39 | } 40 | ] -------------------------------------------------------------------------------- /src/utilities/__tests__/timers.test.ts: -------------------------------------------------------------------------------- 1 | import { TimerFactory } from '../timer-factory'; 2 | import { Timers } from '../timers'; 3 | import { mock } from 'jest-mock-extended'; 4 | import { Timer } from '../timer'; 5 | 6 | describe('timers', () => { 7 | const timerFactory = mock(); 8 | const timers = new Timers(timerFactory); 9 | 10 | beforeEach(() => { 11 | jest.clearAllMocks(); 12 | }); 13 | 14 | it('should start timer', () => { 15 | const callback = jest.fn(); 16 | const timer = mock(); 17 | timerFactory.createTimer.mockReturnValue(timer); 18 | timers.startTimer(1000, callback); 19 | 20 | const calledTime = timer.start.mock.calls[0][0]; 21 | const timerCallback = timer.start.mock.calls[0][1]; 22 | 23 | timerCallback(); 24 | 25 | expect(calledTime).toBe(1000); 26 | expect(timerFactory.createTimer).toHaveBeenCalled(); 27 | expect(timer.start).toHaveBeenCalled(); 28 | expect(timers.hasActiveTimers()).toBeFalsy(); 29 | }); 30 | 31 | it('should stop timers', () => { 32 | const timer = mock(); 33 | timerFactory.createTimer.mockReturnValue(timer); 34 | const callback = jest.fn(); 35 | timers.startTimer(1000, callback); 36 | timers.stopTimers(); 37 | expect(timer.stopTimer).toHaveBeenCalled(); 38 | expect(timers.hasActiveTimers()).toBeFalsy(); 39 | expect(callback).not.toHaveBeenCalled(); 40 | }); 41 | }); 42 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/__tests__/ledger-validator.test.ts: -------------------------------------------------------------------------------- 1 | import { isLedgerSequenceValid } from '../ledger-validator'; 2 | import { Ledger } from '../../../../../crawler'; 3 | 4 | it('should let valid ledger sequences pass', function () { 5 | const latestClosedLedger: Ledger = { 6 | sequence: BigInt('1'), 7 | closeTime: new Date(), 8 | value: '', 9 | localCloseTime: new Date() 10 | }; 11 | expect(isLedgerSequenceValid(latestClosedLedger, BigInt('1'))).toBeTruthy(); 12 | expect(isLedgerSequenceValid(latestClosedLedger, BigInt('2'))).toBeTruthy(); 13 | }); 14 | 15 | it('should not let too old ledger sequences pass', function () { 16 | const latestClosedLedger: Ledger = { 17 | sequence: BigInt('2'), 18 | closeTime: new Date('12/12/2009'), 19 | value: '', 20 | localCloseTime: new Date() 21 | }; 22 | expect(isLedgerSequenceValid(latestClosedLedger, BigInt('2'))).toBeFalsy(); 23 | expect(isLedgerSequenceValid(latestClosedLedger, BigInt('1'))).toBeFalsy(); 24 | }); 25 | 26 | it('should not let ledger sequences older then max seq drift pass', function () { 27 | const latestClosedLedger: Ledger = { 28 | sequence: BigInt('7'), 29 | closeTime: new Date(), 30 | value: '', 31 | localCloseTime: new Date() 32 | }; 33 | 34 | expect(isLedgerSequenceValid(latestClosedLedger, BigInt('1'))).toBeFalsy(); 35 | }); 36 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/readme.md: -------------------------------------------------------------------------------- 1 | # Stellar Message structure (simplified) 2 | 3 | An Externalize Statement is part of an ScpEnvelope that is a type of StellarMessage that contains an ScpStatement 4 | 5 | Source: [Stellar-overlay.x](https://github.com/stellar/stellar-xdr) 6 | 7 | The directory structure mimics this. 8 | 9 | ## StellarMessage Types 10 | * ScpEnvelope 11 | * Peers 12 | * Error 13 | * QuorumSet 14 | * ... 15 | 16 | Message is already verified to be from the sending Node in the NodeConnector package through symmetric key encryption (HMAC). 17 | 18 | ### Peers 19 | Contains the peers advertised by the sending node. These peers are then also crawled to discover the whole network. 20 | 21 | ### Error 22 | Contains an error message after completing a successful handshake. 23 | We are most interested in the ErrLoad messages, which indicate that a node is running at capacity. 24 | 25 | ### ScpEnvelope 26 | Contains a signature and a ScpStatement. Signature is verified through public key encryption to be from the signing node (not necessarily the sender as these messages can be relayed by other nodes) 27 | 28 | An ScpStatement is a message that is part of the consensus process. It contains a slotIndex (=ledger sequence), NodeId, and a type of ScpStatement. 29 | 30 | ### ScpStatement Types 31 | * Nominate 32 | * Prepare 33 | * Confirm 34 | * Externalize 35 | 36 | We use the Externalize messages to determine ledger closing and the validating state of a Node. -------------------------------------------------------------------------------- /src/crawl-queue.ts: -------------------------------------------------------------------------------- 1 | import { queue, QueueObject } from 'async'; 2 | import { CrawlTask } from './crawl-task'; 3 | 4 | export interface CrawlQueue { 5 | push(crawlTask: CrawlTask, error: () => void): void; 6 | onDrain(callback: () => void): void; 7 | activeTasks(): CrawlTask[]; 8 | length(): number; 9 | initialize( 10 | performTask: (task: CrawlTask, done: AsyncResultCallback) => void 11 | ): void; 12 | } 13 | 14 | export interface AsyncResultCallback { 15 | (err?: E | null, result?: T): void; 16 | } 17 | 18 | export class AsyncCrawlQueue implements CrawlQueue { 19 | private _crawlQueue?: QueueObject; 20 | constructor(private maxOpenConnections: number) {} 21 | 22 | private get crawlQueue(): QueueObject { 23 | if (!this._crawlQueue) throw new Error('Crawl queue not set up'); 24 | return this._crawlQueue; 25 | } 26 | 27 | initialize( 28 | performTask: (task: CrawlTask, done: AsyncResultCallback) => void 29 | ) { 30 | this._crawlQueue = queue(performTask, this.maxOpenConnections); 31 | } 32 | 33 | push( 34 | crawlTask: CrawlTask, 35 | callback: AsyncResultCallback 36 | ): void { 37 | this.crawlQueue.push(crawlTask, callback); 38 | } 39 | 40 | onDrain(callback: () => void): void { 41 | this.crawlQueue.drain(callback); 42 | } 43 | 44 | length(): number { 45 | return this.crawlQueue.length(); 46 | } 47 | 48 | activeTasks(): CrawlTask[] { 49 | return this.crawlQueue.workersList().map((worker) => worker.data); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/crawl-queue-manager.ts: -------------------------------------------------------------------------------- 1 | import * as P from 'pino'; 2 | import { AsyncResultCallback, CrawlQueue } from './crawl-queue'; 3 | import { CrawlTask } from './crawl-task'; 4 | 5 | export class CrawlQueueManager { 6 | constructor(private crawlQueue: CrawlQueue, private logger: P.Logger) { 7 | this.crawlQueue.initialize(this.performCrawlQueueTask.bind(this)); 8 | } 9 | 10 | public addCrawlTask(crawlTask: CrawlTask): void { 11 | this.crawlQueue.push(crawlTask, (error?: Error) => { 12 | if (error) { 13 | this.logger.error( 14 | { peer: crawlTask.nodeAddress[0] + ':' + crawlTask.nodeAddress[1] }, 15 | error.message 16 | ); 17 | } 18 | }); 19 | } 20 | 21 | public onDrain(callback: () => void) { 22 | this.crawlQueue.onDrain(callback); 23 | } 24 | 25 | public queueLength(): number { 26 | return this.crawlQueue.length(); 27 | } 28 | 29 | private performCrawlQueueTask( 30 | crawlQueueTask: CrawlTask, 31 | crawlQueueTaskDone: AsyncResultCallback 32 | ): void { 33 | crawlQueueTask.crawl.crawlQueueTaskDoneCallbacks.set( 34 | crawlQueueTask.nodeAddress.join(':'), 35 | crawlQueueTaskDone 36 | ); 37 | 38 | crawlQueueTask.connectCallback(); 39 | } 40 | 41 | public completeCrawlQueueTask( 42 | crawlQueueTaskDoneCallbacks: Map>, 43 | nodeAddress: string 44 | ): void { 45 | const taskDoneCallback = crawlQueueTaskDoneCallbacks.get(nodeAddress); 46 | if (taskDoneCallback) { 47 | taskDoneCallback(); 48 | crawlQueueTaskDoneCallbacks.delete(nodeAddress); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@stellarbeat/js-stellar-node-crawler", 3 | "version": "5.1.0", 4 | "description": "Crawl the network for nodes", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/stellarbeat/js-stellar-node-crawler.git" 8 | }, 9 | "engines": { 10 | "node": "20.*" 11 | }, 12 | "main": "lib/index.js", 13 | "scripts": { 14 | "preversion": "yarn run build", 15 | "build": "tsc --declaration", 16 | "examples:crawl": "yarn run build; NODE_PATH=node_modules node examples/crawl", 17 | "test": "jest" 18 | }, 19 | "types": "lib/index.d.ts", 20 | "files": [ 21 | ".env.dist", 22 | "readme.md", 23 | "lib/**", 24 | "LICENSE", 25 | "examples/**" 26 | ], 27 | "author": "pieterjan84@github", 28 | "license": "MIT", 29 | "dependencies": { 30 | "@stellarbeat/js-stellar-node-connector": "^7.0.1", 31 | "@stellarbeat/js-stellarbeat-shared": "^6.6.1", 32 | "async": "^3.2.6", 33 | "dotenv": "^16.4.5", 34 | "lru-cache": "^11.0.1", 35 | "neverthrow": "^8.0.0", 36 | "pino": "^9.4.0" 37 | }, 38 | "devDependencies": { 39 | "@stellar/stellar-base": "12.1.1", 40 | "@types/async": "^3.2.7", 41 | "@types/jest": "29.5.13", 42 | "@types/node": "20.*", 43 | "eslint": "^9.11.1", 44 | "eslint-config-prettier": "^9.1.0", 45 | "jest": "29.7.0", 46 | "jest-mock-extended": "^3.0.5", 47 | "np": "^10.0.7", 48 | "prettier": "^3.3.3", 49 | "ts-jest": "29.2.5", 50 | "typescript": "^5.6.2" 51 | }, 52 | "packageManager": "pnpm@9.11.0+sha512.0a203ffaed5a3f63242cd064c8fb5892366c103e328079318f78062f24ea8c9d50bc6a47aa3567cabefd824d170e78fa2745ed1f16b132e16436146b7688f19b" 53 | } 54 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/__tests__/slots.test.ts: -------------------------------------------------------------------------------- 1 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { P } from 'pino'; 4 | import { Slots } from '../slots'; 5 | 6 | describe('slots', () => { 7 | it('should create new slot', () => { 8 | const trustedQuorumSet = new QuorumSet(2, ['A', 'B', 'C'], []); 9 | const logger = mock(); 10 | const slots = new Slots(trustedQuorumSet, logger); 11 | const slot = slots.getSlot(BigInt(1)); 12 | expect(slot).toBeDefined(); 13 | expect(slot.index).toEqual(BigInt(1)); 14 | }); 15 | 16 | it('should return same slot if already created', () => { 17 | const trustedQuorumSet = new QuorumSet(2, ['A', 'B', 'C'], []); 18 | const logger = mock(); 19 | const slots = new Slots(trustedQuorumSet, logger); 20 | const slot = slots.getSlot(BigInt(1)); 21 | const slot2 = slots.getSlot(BigInt(1)); 22 | expect(slot).toBe(slot2); 23 | }); 24 | 25 | it('should return empty set if no confirmed closed ledger', () => { 26 | const trustedQuorumSet = new QuorumSet(2, ['A', 'B', 'C'], []); 27 | const logger = mock(); 28 | const slots = new Slots(trustedQuorumSet, logger); 29 | slots.getSlot(BigInt(1)); 30 | expect(slots.getConfirmedClosedSlotIndexes()).toEqual([]); 31 | }); 32 | 33 | it('should return confirmed closed slot indexes', () => { 34 | const trustedQuorumSet = new QuorumSet(1, ['A'], []); 35 | const logger = mock(); 36 | const slots = new Slots(trustedQuorumSet, logger); 37 | const slot = slots.getSlot(BigInt(1)); 38 | slot.addExternalizeValue('A', 'test value', new Date()); 39 | slots.getSlot(BigInt(2)); 40 | 41 | expect(slots.getConfirmedClosedSlotIndexes()).toEqual([BigInt(1)]); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /src/__tests__/crawl-queue.test.ts: -------------------------------------------------------------------------------- 1 | import { AsyncCrawlQueue } from '../crawl-queue'; 2 | 3 | describe('CrawlQueue', () => { 4 | it('should initialize the crawl queue', () => { 5 | const crawlQueue = new AsyncCrawlQueue(10); 6 | crawlQueue.initialize(() => {}); 7 | expect(crawlQueue).toHaveProperty('_crawlQueue'); 8 | }); 9 | 10 | it('should push a crawl task', () => { 11 | const crawlQueue = new AsyncCrawlQueue(10); 12 | crawlQueue.initialize(() => {}); 13 | crawlQueue.push({} as any, () => {}); 14 | expect(crawlQueue.length()).toEqual(1); 15 | }); 16 | 17 | it('should return the length of the queue', () => { 18 | const crawlQueue = new AsyncCrawlQueue(10); 19 | crawlQueue.initialize(() => {}); 20 | crawlQueue.push({} as any, () => {}); 21 | crawlQueue.push({} as any, () => {}); 22 | expect(crawlQueue.length()).toEqual(2); 23 | }); 24 | 25 | it('should throw an error if crawl queue is not set up', () => { 26 | const crawlQueue = new AsyncCrawlQueue(10); 27 | expect(() => crawlQueue.length()).toThrow('Crawl queue not set up'); 28 | }); 29 | 30 | it('should call execute the workers and call the drain function', async () => { 31 | const crawlQueue = new AsyncCrawlQueue(10); 32 | let counter = 0; 33 | 34 | crawlQueue.initialize(() => { 35 | counter++; 36 | }); 37 | crawlQueue.push({} as any, () => { 38 | counter++; 39 | }); 40 | 41 | crawlQueue.onDrain(() => { 42 | expect(counter).toEqual(2); 43 | expect(crawlQueue.length()).toEqual(0); 44 | expect(crawlQueue.activeTasks()).toEqual([]); 45 | }); 46 | }); 47 | 48 | it('should return the active workers', async () => { 49 | const crawlQueue = new AsyncCrawlQueue(10); 50 | crawlQueue.initialize(async () => {}); 51 | 52 | crawlQueue.push({} as any, () => { 53 | setTimeout(() => { 54 | expect(crawlQueue.activeTasks().length).toEqual(1); 55 | }, 1000); 56 | }); 57 | }); 58 | }); 59 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/__tests__/peer-event-handler.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { 3 | ClosePayload, 4 | ConnectedPayload, 5 | DataPayload 6 | } from '../../connection-manager'; 7 | import { OnPeerConnected } from '../on-peer-connected'; 8 | import { OnPeerConnectionClosed } from '../on-peer-connection-closed'; 9 | import { OnPeerData } from '../on-peer-data'; 10 | import { PeerEventHandler } from '../peer-event-handler'; 11 | import { Observation } from '../../observation'; 12 | 13 | describe('PeerConnectionEventHandler', () => { 14 | const onConnectedHandler = mock(); 15 | const onConnectionCloseHandler = mock(); 16 | const onPeerDataHandler = mock(); 17 | const peerConnectionEventHandler = new PeerEventHandler( 18 | onConnectedHandler, 19 | onConnectionCloseHandler, 20 | onPeerDataHandler 21 | ); 22 | 23 | beforeEach(() => { 24 | jest.clearAllMocks(); 25 | }); 26 | 27 | it('should call onConnectedHandler.handle', () => { 28 | const data = mock(); 29 | const observation = mock(); 30 | peerConnectionEventHandler.onConnected(data, observation); 31 | expect(onConnectedHandler.handle).toHaveBeenCalledWith(data, observation); 32 | }); 33 | 34 | it('should call onConnectionCloseHandler.handle', () => { 35 | const data = mock(); 36 | const observation = mock(); 37 | peerConnectionEventHandler.onConnectionClose(data, observation); 38 | expect(onConnectionCloseHandler.handle).toHaveBeenCalledWith( 39 | data, 40 | observation 41 | ); 42 | }); 43 | 44 | it('should call onPeerDataHandler.handle', () => { 45 | const data = mock(); 46 | const observation = mock(); 47 | peerConnectionEventHandler.onData(data, observation); 48 | expect(onPeerDataHandler.handle).toHaveBeenCalledWith(data, observation); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /seed/nodes-testnet.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ip": "core-testnet1.stellar.org", 4 | "port": 11625, 5 | "host": "core-testnet1.stellar.org", 6 | "publicKey": "GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK", 7 | "name": "SDF validator 2", 8 | "ledgerVersion": 10, 9 | "overlayVersion": 8, 10 | "overlayMinVersion": 7, 11 | "networkId": "esM5l1ROMXXSZr0CJDmyLNsWUIwBFj8m5csqPhBFqXk=", 12 | "versionStr": "v10.3.0rc2", 13 | "active": true, 14 | "overLoaded": true, 15 | "quorumSet": { 16 | "hashKey": "7mhz+5gjKbzsEppyQ/VCgZ95mMPPKqmMb7OsvRMWctw=", 17 | "threshold": 3, 18 | "validators": [ 19 | "GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ", 20 | "GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH", 21 | "GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK", 22 | "GAOO3LWBC4XF6VWRP5ESJ6IBHAISVJMSBTALHOQM2EZG7Q477UWA6L7U" 23 | ], 24 | "innerQuorumSets": [ 25 | { 26 | "hashKey": "5OoVvX0yJ4+VlKnXZgr1rzFUgS6QWgRzKunyt9l7bJw=", 27 | "threshold": 2, 28 | "validators": [ 29 | "GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE", 30 | "GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT", 31 | "GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY" 32 | ], 33 | "innerQuorumSets": [] 34 | } 35 | ] 36 | }, 37 | "geoData": { 38 | "countryCode": "US", 39 | "countryName": "United States", 40 | "regionCode": "VA", 41 | "regionName": "Virginia", 42 | "city": "Ashburn", 43 | "zipCode": "20149", 44 | "timeZone": "America/New_York", 45 | "latitude": 39.0853, 46 | "longitude": -77.6452, 47 | "metroCode": 511 48 | }, 49 | "statistics": { 50 | "activeCounter": 300, 51 | "overLoadedCounter": 259, 52 | "activeRating": 5, 53 | "activeInLastCrawl": true, 54 | "overLoadedInLastCrawl": true 55 | }, 56 | "dateDiscovered": "2018-04-28T14:39:01.772Z", 57 | "dateUpdated": "2019-03-28T18:41:22.970Z" 58 | } 59 | ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) 2 | [![test](https://github.com/stellarbeat/js-stellar-node-crawler/actions/workflows/test.yml/badge.svg)](https://github.com/stellarbeat/js-stellar-node-crawler/actions/workflows/test.yml) 3 | 4 | # stellar-js-node-crawler 5 | 6 | Crawl the Stellar Network. Identify the nodes and determine their validating 7 | status, version, lag,.... 8 | 9 | ## How does it work? 10 | 11 | See readme in src/README.md for an overview of the functionality and 12 | architecture. 13 | 14 | ## install 15 | 16 | `pnpm install` 17 | 18 | ## build code 19 | 20 | `pnpm build`: builds code in lib folder 21 | 22 | ## Usage 23 | 24 | ### Create crawler 25 | 26 | ``` 27 | let myCrawler = createCrawler({ 28 | nodeConfig: getConfigFromEnv(), 29 | maxOpenConnections: 25, 30 | maxCrawlTime: 900000 31 | }); 32 | ``` 33 | 34 | The crawler is itself a 35 | [node](https://github.com/stellarbeat/js-stellar-node-connector) and needs to be 36 | configured accordingly. You can limit the number of simultaneous open 37 | connections to not overwhelm your server and set the maxCrawlTime as a safety if 38 | the crawler should be stuck. 39 | 40 | ### Run crawl 41 | 42 | ``` 43 | let result = await myCrawler.crawl( 44 | nodes, // [[ip, port], [ip, port]] 45 | trustedQSet, //a quorumSet the crawler uses the determine the latest closed ledger 46 | latestKnownLedger //a previous detected ledger the crawler can use to ignore older externalize messages 47 | ); 48 | ``` 49 | 50 | ### example script 51 | 52 | Check out `examples/crawl.js` for an example on how to crawl the network. You 53 | can try it out using the bundled seed file with the following command: 54 | `pnpm examples:crawl seed/nodes.json` 55 | 56 | Another example is the 57 | [Stellarbeat backend](https://github.com/stellarbeat/js-stellarbeat-backend/blob/master/src/network/services/CrawlerService.ts) 58 | 59 | ### publish new release 60 | 61 | Uses the [np package](https://github.com/sindresorhus/np) and semantic 62 | versioning. 63 | 64 | ``` 65 | np 66 | ``` 67 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/on-peer-data.ts: -------------------------------------------------------------------------------- 1 | import { ConnectionManager, DataPayload } from '../connection-manager'; 2 | import { Ledger } from '../../crawler'; 3 | import { NodeAddress } from '../../node-address'; 4 | import { StellarMessageHandler } from './stellar-message-handlers/stellar-message-handler'; 5 | import { P } from 'pino'; 6 | import { Observation } from '../observation'; 7 | import { ObservationState } from '../observation-state'; 8 | 9 | export interface OnPeerDataResult { 10 | closedLedger: Ledger | null; 11 | peers: Array; 12 | } 13 | 14 | export class OnPeerData { 15 | constructor( 16 | private stellarMessageHandler: StellarMessageHandler, 17 | private logger: P.Logger, 18 | private connectionManager: ConnectionManager 19 | ) {} 20 | 21 | public handle(data: DataPayload, observation: Observation): OnPeerDataResult { 22 | const attemptLedgerClose = this.attemptLedgerClose(observation); 23 | const result = this.performWork(data, observation, attemptLedgerClose); 24 | 25 | if (result.isErr()) { 26 | this.disconnect(data, result.error); 27 | return this.returnEmpty(); 28 | } 29 | 30 | return this.createOnPeerDataResult(result.value); 31 | } 32 | 33 | private createOnPeerDataResult(result: { 34 | closedLedger: Ledger | null; 35 | peers: Array; 36 | }): OnPeerDataResult { 37 | return { 38 | closedLedger: result.closedLedger, 39 | peers: result.peers 40 | }; 41 | } 42 | 43 | private performWork( 44 | data: DataPayload, 45 | observation: Observation, 46 | attemptLedgerClose: boolean 47 | ) { 48 | const result = this.stellarMessageHandler.handleStellarMessage( 49 | data.publicKey, 50 | data.stellarMessageWork.stellarMessage, 51 | attemptLedgerClose, 52 | observation 53 | ); 54 | 55 | data.stellarMessageWork.done(); 56 | return result; 57 | } 58 | 59 | private attemptLedgerClose(observation: Observation) { 60 | return observation.state === ObservationState.Synced; 61 | } 62 | 63 | private returnEmpty() { 64 | return { 65 | closedLedger: null, 66 | peers: [] 67 | }; 68 | } 69 | 70 | private disconnect(data: DataPayload, error: Error) { 71 | this.logger.info({ peer: data.publicKey }, error.message); 72 | this.connectionManager.disconnectByAddress(data.address, error); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/__tests__/peer-node.test.ts: -------------------------------------------------------------------------------- 1 | import { PeerNode } from '../peer-node'; 2 | 3 | describe('PeerNode', () => { 4 | it('should have a key', () => { 5 | const peerNode = new PeerNode('publicKey'); 6 | peerNode.ip = 'localhost'; 7 | peerNode.port = 8000; 8 | expect(peerNode.key).toBe('localhost:8000'); 9 | }); 10 | 11 | describe('processConfirmedLedgerClose', () => { 12 | test('not externalized', () => { 13 | const peerNode = new PeerNode('publicKey'); 14 | peerNode.processConfirmedLedgerClose({ 15 | sequence: BigInt(1), 16 | localCloseTime: new Date(), 17 | value: 'value', 18 | closeTime: new Date() 19 | }); 20 | expect(peerNode.isValidating).toBe(false); 21 | expect(peerNode.isValidatingIncorrectValues).toBe(false); 22 | expect(peerNode.getMinLagMS()).toBe(undefined); 23 | }); 24 | 25 | test('externalized', () => { 26 | const peerNode = new PeerNode('publicKey'); 27 | peerNode.successfullyConnected = true; 28 | 29 | const closeTime = new Date('2021-01-01'); 30 | const localCloseTime = new Date('2021-01-01'); 31 | const externalizeTime = new Date('2021-01-02'); 32 | peerNode.addExternalizedValue(BigInt(1), externalizeTime, 'value'); 33 | 34 | peerNode.processConfirmedLedgerClose({ 35 | sequence: BigInt(1), 36 | localCloseTime: localCloseTime, 37 | value: 'value', 38 | closeTime: closeTime 39 | }); 40 | expect(peerNode.isValidating).toBe(true); 41 | expect(peerNode.isValidatingIncorrectValues).toBe(false); 42 | expect(peerNode.getMinLagMS()).toBe( 43 | externalizeTime.getTime() - localCloseTime.getTime() 44 | ); 45 | }); 46 | 47 | test('invalid value', () => { 48 | const peerNode = new PeerNode('publicKey'); 49 | peerNode.successfullyConnected = true; 50 | 51 | const closeTime = new Date('2021-01-01'); 52 | const localCloseTime = new Date('2021-02-01'); 53 | const externalizeTime = new Date('2021-03-01'); 54 | peerNode.addExternalizedValue(BigInt(1), externalizeTime, 'value'); 55 | 56 | peerNode.processConfirmedLedgerClose({ 57 | sequence: BigInt(1), 58 | localCloseTime: localCloseTime, 59 | value: 'invalidValue', 60 | closeTime: closeTime 61 | }); 62 | expect(peerNode.isValidating).toBe(false); 63 | expect(peerNode.isValidatingIncorrectValues).toBe(true); 64 | expect(peerNode.getMinLagMS()).toBe(undefined); 65 | }); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /src/network-observer/observation.ts: -------------------------------------------------------------------------------- 1 | import { NodeAddress } from '../node-address'; 2 | import { PeerNodeCollection } from '../peer-node-collection'; 3 | import * as assert from 'assert'; 4 | import { Ledger } from '../crawler'; 5 | import { ObservationState } from './observation-state'; 6 | import { Slots } from './peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 7 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 8 | import { QuorumSetState } from './quorum-set-state'; 9 | import { LRUCache } from 'lru-cache'; 10 | 11 | export class Observation { 12 | public state: ObservationState = ObservationState.Idle; 13 | private networkHalted = false; 14 | public topTierAddressesSet: Set; 15 | public envelopeCache: LRUCache; 16 | public quorumSetState: QuorumSetState = new QuorumSetState(); 17 | 18 | constructor( 19 | public network: string, 20 | public topTierAddresses: NodeAddress[], 21 | public peerNodes: PeerNodeCollection, 22 | public latestConfirmedClosedLedger: Ledger, 23 | public quorumSets: Map, 24 | public slots: Slots 25 | ) { 26 | this.topTierAddressesSet = this.mapTopTierAddresses(topTierAddresses); 27 | this.envelopeCache = new LRUCache({ max: 5000 }); 28 | } 29 | 30 | private mapTopTierAddresses(topTierNodes: NodeAddress[]) { 31 | const topTierAddresses = new Set(); 32 | topTierNodes.forEach((address) => { 33 | topTierAddresses.add(`${address[0]}:${address[1]}`); 34 | }); 35 | return topTierAddresses; 36 | } 37 | 38 | moveToSyncingState() { 39 | assert(this.state === ObservationState.Idle); 40 | this.state = ObservationState.Syncing; 41 | } 42 | 43 | moveToSyncedState() { 44 | assert(this.state === ObservationState.Syncing); 45 | this.state = ObservationState.Synced; 46 | } 47 | 48 | moveToStoppingState() { 49 | assert(this.state !== ObservationState.Idle); 50 | this.state = ObservationState.Stopping; 51 | } 52 | 53 | moveToStoppedState() { 54 | assert(this.state === ObservationState.Stopping); 55 | this.state = ObservationState.Stopped; 56 | } 57 | 58 | ledgerCloseConfirmed(ledger: Ledger) { 59 | this.networkHalted = false; 60 | if (this.state !== ObservationState.Synced) return; 61 | 62 | this.latestConfirmedClosedLedger = ledger; 63 | } 64 | 65 | isNetworkHalted(): boolean { 66 | return this.networkHalted; 67 | } 68 | 69 | setNetworkHalted() { 70 | this.networkHalted = true; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/__tests__/slot.test.ts: -------------------------------------------------------------------------------- 1 | import { Slot } from '../slot'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 4 | import { P } from 'pino'; 5 | import { createDummyValue } from '../../../../../../../__fixtures__/createDummyExternalizeMessage'; 6 | 7 | const mockLogger = mock(); 8 | let quorumSet: QuorumSet; 9 | 10 | describe('slot', () => { 11 | beforeEach(() => { 12 | jest.resetAllMocks(); 13 | quorumSet = new QuorumSet(2, ['A', 'B', 'C'], []); 14 | }); 15 | 16 | it('should return empty set if no confirmed closed ledger', () => { 17 | const slot = new Slot(BigInt(1), quorumSet, mockLogger); 18 | slot.addExternalizeValue('A', 'test value', new Date()); 19 | expect(slot.getNodesAgreeingOnExternalizedValue()).toEqual(new Set()); 20 | }); 21 | 22 | it('should return agreeing and disagreeing nodes when ledger close is confirmed', () => { 23 | const slot = new Slot(BigInt(1), quorumSet, mockLogger); 24 | slot.addExternalizeValue('A', 'test value', new Date()); 25 | slot.addExternalizeValue('B', 'test value', new Date()); 26 | slot.addExternalizeValue('C', 'another value', new Date()); 27 | 28 | expect(slot.getNodesAgreeingOnExternalizedValue()).toEqual( 29 | new Set(['A', 'B']) 30 | ); 31 | expect(slot.getNodesDisagreeingOnExternalizedValue()).toEqual( 32 | new Set(['C']) 33 | ); 34 | }); 35 | 36 | it('should return empty set if no confirmed closed ledger', () => { 37 | const slot = new Slot(BigInt(1), mock(), mockLogger); 38 | expect(slot.getNodesDisagreeingOnExternalizedValue()).toEqual(new Set()); 39 | }); 40 | 41 | it('should close slot and return ledger', () => { 42 | const value = createDummyValue(); 43 | const slot = new Slot(BigInt(100), quorumSet, mockLogger); 44 | const firstObservedTime = new Date('2021-01-01T00:00:00Z'); 45 | const secondObservedTime = new Date('2021-01-02T00:00:01Z'); 46 | 47 | slot.addExternalizeValue('A', value.toString('base64'), firstObservedTime); 48 | slot.addExternalizeValue('B', value.toString('base64'), secondObservedTime); 49 | 50 | expect(slot.getNodesAgreeingOnExternalizedValue()).toEqual( 51 | new Set(['A', 'B']) 52 | ); 53 | expect(slot.getConfirmedClosedLedger()).toEqual({ 54 | value: value.toString('base64'), 55 | closeTime: new Date('2024-02-27T08:36:24.000Z'), 56 | localCloseTime: firstObservedTime, 57 | sequence: BigInt(100) 58 | }); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /src/peer-node.ts: -------------------------------------------------------------------------------- 1 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 2 | import { NodeInfo } from '@stellarbeat/js-stellar-node-connector/lib/node'; 3 | import { Ledger } from './crawler'; 4 | 5 | export class PeerNode { 6 | public ip?: string; 7 | public port?: number; 8 | public publicKey: string; 9 | public nodeInfo?: NodeInfo; 10 | public isValidating = false; 11 | public isValidatingIncorrectValues = false; 12 | public overLoaded = false; 13 | public quorumSetHash: string | undefined; 14 | public quorumSet: QuorumSet | undefined; 15 | public suppliedPeerList = false; 16 | public latestActiveSlotIndex?: string; 17 | public participatingInSCP = false; 18 | public successfullyConnected = false; 19 | private externalizedValues: Map< 20 | bigint, 21 | { 22 | localTime: Date; 23 | value: string; 24 | } 25 | > = new Map(); 26 | private lagMSMeasurement: Map = new Map(); 27 | 28 | constructor(publicKey: string) { 29 | this.publicKey = publicKey; 30 | } 31 | 32 | get key(): string { 33 | return this.ip + ':' + this.port; 34 | } 35 | 36 | processConfirmedLedgerClose(closedLedger: Ledger) { 37 | const externalized = this.externalizedValues.get(closedLedger.sequence); 38 | 39 | if (!externalized) { 40 | return; 41 | } 42 | 43 | if (externalized.value !== closedLedger.value) { 44 | this.isValidatingIncorrectValues = true; 45 | return; 46 | } 47 | 48 | this.isValidating = true; 49 | 50 | this.updateLag(closedLedger, externalized); 51 | } 52 | 53 | public addExternalizedValue( 54 | slotIndex: bigint, 55 | localTime: Date, 56 | value: string 57 | ): void { 58 | this.externalizedValues.set(slotIndex, { 59 | localTime: localTime, 60 | value: value 61 | }); 62 | } 63 | 64 | private updateLag( 65 | closedLedger: Ledger, 66 | externalized: { 67 | localTime: Date; 68 | value: string; 69 | } 70 | ): void { 71 | this.lagMSMeasurement.set( 72 | closedLedger.sequence, 73 | this.determineLag(closedLedger.localCloseTime, externalized.localTime) 74 | ); 75 | } 76 | 77 | private determineLag(localLedgerCloseTime: Date, externalizeTime: Date) { 78 | return externalizeTime.getTime() - localLedgerCloseTime.getTime(); 79 | } 80 | 81 | public getMinLagMS(): number | undefined { 82 | //implement without using spread operator 83 | let minLag: number | undefined; 84 | for (const lag of this.lagMSMeasurement.values()) { 85 | if (minLag === undefined || lag < minLag) { 86 | minLag = lag; 87 | } 88 | } 89 | 90 | return minLag; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-envelope-handler.ts: -------------------------------------------------------------------------------- 1 | import { hash, xdr } from '@stellar/stellar-base'; 2 | import { verifySCPEnvelopeSignature } from '@stellarbeat/js-stellar-node-connector'; 3 | import { err, ok, Result } from 'neverthrow'; 4 | import { isLedgerSequenceValid } from './ledger-validator'; 5 | import { ScpStatementHandler } from './scp-statement/scp-statement-handler'; 6 | import { Ledger } from '../../../../crawler'; 7 | import { Observation } from '../../../observation'; 8 | 9 | /* 10 | * ScpEnvelopeHandler makes sure that no duplicate SCP envelopes are processed, that the signature is valid and 11 | * that the ledger sequence is valid. It then delegates the SCP statement to the ScpStatementHandler. 12 | */ 13 | export class ScpEnvelopeHandler { 14 | constructor(private scpStatementHandler: ScpStatementHandler) {} 15 | 16 | public handle( 17 | scpEnvelope: xdr.ScpEnvelope, 18 | observation: Observation 19 | ): Result< 20 | { 21 | closedLedger: Ledger | null; 22 | }, 23 | Error 24 | > { 25 | if (this.isCached(scpEnvelope, observation)) 26 | return ok({ 27 | closedLedger: null 28 | }); 29 | 30 | if (this.isValidLedger(observation, scpEnvelope)) 31 | return ok({ 32 | closedLedger: null 33 | }); 34 | 35 | const verifiedSignature = this.verifySignature(scpEnvelope, observation); 36 | if (verifiedSignature.isErr()) return err(verifiedSignature.error); 37 | 38 | return this.scpStatementHandler.handle( 39 | scpEnvelope.statement(), 40 | observation 41 | ); 42 | } 43 | 44 | private verifySignature( 45 | scpEnvelope: xdr.ScpEnvelope, 46 | observation: Observation 47 | ): Result { 48 | const verifiedResult = verifySCPEnvelopeSignature( 49 | scpEnvelope, 50 | hash(Buffer.from(observation.network)) 51 | ); 52 | if (verifiedResult.isErr()) 53 | return err(new Error('Error verifying SCP Signature')); 54 | 55 | if (!verifiedResult.value) return err(new Error('Invalid SCP Signature')); 56 | 57 | return ok(undefined); 58 | } 59 | 60 | private isValidLedger( 61 | observation: Observation, 62 | scpEnvelope: xdr.ScpEnvelope 63 | ) { 64 | return !isLedgerSequenceValid( 65 | observation.latestConfirmedClosedLedger, 66 | BigInt(scpEnvelope.statement().slotIndex().toString()) 67 | ); 68 | } 69 | 70 | private isCached( 71 | scpEnvelope: xdr.ScpEnvelope, 72 | observation: Observation 73 | ): boolean { 74 | if (observation.envelopeCache.has(scpEnvelope.signature().toString())) 75 | return true; 76 | observation.envelopeCache.set(scpEnvelope.signature().toString(), 1); 77 | return false; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/network-observer/__tests__/straggler-timer.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { P } from 'pino'; 3 | import { ConnectionManager } from '../connection-manager'; 4 | import { StragglerTimer } from '../straggler-timer'; 5 | import { Timers } from '../../utilities/timers'; 6 | 7 | describe('StragglerTimer', () => { 8 | const logger = mock(); 9 | const connectionManager = mock(); 10 | const timers = mock(); 11 | const stragglerHandler = new StragglerTimer( 12 | connectionManager, 13 | timers, 14 | 1000, 15 | logger 16 | ); 17 | 18 | beforeEach(() => { 19 | jest.clearAllMocks(); 20 | }); 21 | 22 | it('should start straggler timeout', () => { 23 | const addresses = ['address1', 'address2']; 24 | stragglerHandler.startStragglerTimeout(addresses); 25 | const callback1 = timers.startTimer.mock.calls[0][1]; 26 | expect(timers.startTimer).toHaveBeenCalledTimes(1); 27 | callback1(); 28 | 29 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledWith( 30 | 'address1' 31 | ); 32 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledWith( 33 | 'address2' 34 | ); 35 | }); 36 | 37 | it('should not start timeout if addresses is empty', () => { 38 | stragglerHandler.startStragglerTimeout([]); 39 | expect(timers.startTimer).not.toHaveBeenCalled(); 40 | }); 41 | 42 | it('should start straggler timeout for active non top tier peers', () => { 43 | connectionManager.getActiveConnectionAddresses.mockReturnValue([ 44 | 'peerAddress', 45 | 'topTierAddress' 46 | ]); 47 | stragglerHandler.startStragglerTimeoutForActivePeers( 48 | false, 49 | new Set(['topTierAddress']) 50 | ); 51 | const callback1 = timers.startTimer.mock.calls[0][1]; 52 | expect(timers.startTimer).toHaveBeenCalledTimes(1); 53 | callback1(); 54 | 55 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledWith( 56 | 'peerAddress' 57 | ); 58 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledTimes(1); 59 | }); 60 | 61 | it('should start straggler timeout for active top tier peers', () => { 62 | connectionManager.getActiveConnectionAddresses.mockReturnValue([ 63 | 'peerAddress', 64 | 'topTierAddress' 65 | ]); 66 | stragglerHandler.startStragglerTimeoutForActivePeers( 67 | true, 68 | new Set('topTierAddress') 69 | ); 70 | const callback1 = timers.startTimer.mock.calls[0][1]; 71 | expect(timers.startTimer).toHaveBeenCalledTimes(1); 72 | callback1(); 73 | 74 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledTimes(2); 75 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledWith( 76 | 'peerAddress' 77 | ); 78 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledWith( 79 | 'topTierAddress' 80 | ); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/on-peer-connected.ts: -------------------------------------------------------------------------------- 1 | import { ConnectedPayload, ConnectionManager } from '../connection-manager'; 2 | import { StragglerTimer } from '../straggler-timer'; 3 | import { P } from 'pino'; 4 | import { truncate } from '../../utilities/truncate'; 5 | import { PeerNodeCollection } from '../../peer-node-collection'; 6 | import { Observation } from '../observation'; 7 | import { ObservationState } from '../observation-state'; 8 | 9 | export class OnPeerConnected { 10 | constructor( 11 | private stragglerHandler: StragglerTimer, 12 | private connectionManager: ConnectionManager, 13 | private logger: P.Logger 14 | ) {} 15 | public handle(data: ConnectedPayload, observation: Observation) { 16 | this.logIfTopTierConnected(data, observation); 17 | const peerNodeOrError = this.addPeerNode(data, observation.peerNodes); 18 | 19 | if (peerNodeOrError instanceof Error) { 20 | this.disconnect(data.ip, data.port, peerNodeOrError); 21 | return peerNodeOrError; 22 | } 23 | 24 | if (observation.isNetworkHalted()) { 25 | return this.collectMinimalDataAndDisconnect(data); 26 | } 27 | 28 | this.handleConnectedByState(observation, data); 29 | } 30 | 31 | private handleConnectedByState( 32 | observation: Observation, 33 | data: ConnectedPayload 34 | ) { 35 | switch (observation.state) { 36 | case ObservationState.Idle: 37 | return this.disconnectBecauseIdle(data); 38 | case ObservationState.Stopping: 39 | return this.collectMinimalDataAndDisconnect(data); 40 | default: 41 | return; 42 | } 43 | } 44 | 45 | private disconnectBecauseIdle(data: ConnectedPayload) { 46 | return this.disconnect(data.ip, data.port, this.createIdleConnectedError()); 47 | } 48 | 49 | private createIdleConnectedError() { 50 | return new Error('Connected while idle'); 51 | } 52 | 53 | private collectMinimalDataAndDisconnect(data: ConnectedPayload) { 54 | return this.startStragglerTimeout(data); 55 | } 56 | private startStragglerTimeout(data: ConnectedPayload) { 57 | return this.stragglerHandler.startStragglerTimeout([ 58 | data.ip + ':' + data.port 59 | ]); 60 | } 61 | 62 | private disconnect(ip: string, port: number, error?: Error) { 63 | this.connectionManager.disconnectByAddress(`${ip}:${port}`, error); 64 | } 65 | 66 | private addPeerNode(data: ConnectedPayload, peerNodes: PeerNodeCollection) { 67 | return peerNodes.addSuccessfullyConnected( 68 | data.publicKey, 69 | data.ip, 70 | data.port, 71 | data.nodeInfo 72 | ); 73 | } 74 | 75 | private logIfTopTierConnected( 76 | data: ConnectedPayload, 77 | observation: Observation 78 | ) { 79 | if (observation.topTierAddressesSet.has(`${data.ip}:${data.port}`)) { 80 | this.logger.debug( 81 | { pk: truncate(data.publicKey) }, 82 | 'Top tier node connected' 83 | ); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/__tests__/crawl-queue-manager.test.ts: -------------------------------------------------------------------------------- 1 | import { AsyncCrawlQueue } from '../crawl-queue'; 2 | import { CrawlQueueManager } from '../crawl-queue-manager'; 3 | import { mock } from 'jest-mock-extended'; 4 | import { P } from 'pino'; 5 | import { Crawl } from '../crawl'; 6 | import { CrawlTask } from '../crawl-task'; 7 | import { nodeAddressToPeerKey } from '../node-address'; 8 | 9 | describe('CrawlQueueManager', () => { 10 | const crawlQueue = mock(); 11 | const logger = mock(); 12 | const crawlState = mock(); 13 | 14 | beforeEach(() => { 15 | crawlState.crawledNodeAddresses = new Set(); 16 | crawlState.crawlQueueTaskDoneCallbacks = new Map(); 17 | jest.clearAllMocks(); 18 | }); 19 | 20 | it('should add a crawl task', () => { 21 | const crawlQueueManager = new CrawlQueueManager(crawlQueue, logger); 22 | crawlQueueManager.addCrawlTask({ 23 | connectCallback: () => {}, 24 | crawl: crawlState, 25 | nodeAddress: ['localhost', 11625] 26 | }); 27 | 28 | expect(crawlQueue.push).toHaveBeenCalled(); 29 | }); 30 | 31 | it('should call onDrain', () => { 32 | const crawlQueueManager = new CrawlQueueManager(crawlQueue, logger); 33 | crawlQueueManager.onDrain(() => {}); 34 | expect(crawlQueue.onDrain).toHaveBeenCalled(); 35 | }); 36 | 37 | it('should return the queue length', () => { 38 | const crawlQueueManager = new CrawlQueueManager(crawlQueue, logger); 39 | crawlQueueManager.queueLength(); 40 | expect(crawlQueue.length).toHaveBeenCalled(); 41 | }); 42 | 43 | it('should initialize the crawl queue', () => { 44 | new CrawlQueueManager(crawlQueue, logger); 45 | expect(crawlQueue.initialize).toHaveBeenCalled(); 46 | }); 47 | 48 | it('should perform a crawl queue task', () => { 49 | const task: CrawlTask = { 50 | connectCallback: jest.fn(), 51 | crawl: crawlState, 52 | nodeAddress: ['localhost', 11625] 53 | }; 54 | 55 | crawlQueue.initialize.mockImplementation((callback) => { 56 | callback(task, () => {}); 57 | }); 58 | 59 | const crawlQueueManager = new CrawlQueueManager(crawlQueue, logger); 60 | crawlQueueManager.queueLength(); 61 | expect(task.connectCallback).toHaveBeenCalled(); 62 | }); 63 | 64 | it('should complete a crawl task', function () { 65 | const task: CrawlTask = { 66 | connectCallback: jest.fn(), 67 | crawl: crawlState, 68 | nodeAddress: ['localhost', 11625] 69 | }; 70 | 71 | crawlQueue.initialize.mockImplementation((callback) => { 72 | callback(task, () => {}); //execute the async task 73 | }); 74 | 75 | const crawlQueueManager = new CrawlQueueManager(crawlQueue, logger); 76 | expect(crawlState.crawlQueueTaskDoneCallbacks.size).toBe(1); 77 | crawlQueueManager.completeCrawlQueueTask( 78 | crawlState.crawlQueueTaskDoneCallbacks, 79 | nodeAddressToPeerKey(task.nodeAddress) 80 | ); 81 | expect(crawlState.crawlQueueTaskDoneCallbacks.size).toBe(0); 82 | }); 83 | }); 84 | -------------------------------------------------------------------------------- /src/network-observer/network-observer.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ClosePayload, 3 | ConnectedPayload, 4 | ConnectionManager, 5 | DataPayload 6 | } from './connection-manager'; 7 | import { QuorumSetManager } from './quorum-set-manager'; 8 | import { EventEmitter } from 'events'; 9 | import { ObservationManager } from './observation-manager'; 10 | import { PeerEventHandler } from './peer-event-handler/peer-event-handler'; 11 | import { Observation } from './observation'; 12 | import * as assert from 'assert'; 13 | import { ObservationState } from './observation-state'; 14 | import { ObservationFactory } from './observation-factory'; 15 | 16 | export class NetworkObserver extends EventEmitter { 17 | private _observation: Observation | null = null; 18 | 19 | constructor( 20 | private observationFactory: ObservationFactory, 21 | private connectionManager: ConnectionManager, 22 | private quorumSetManager: QuorumSetManager, 23 | private peerEventHandler: PeerEventHandler, 24 | private observationManager: ObservationManager 25 | ) { 26 | super(); 27 | this.setupPeerEventHandlers(); 28 | } 29 | 30 | public async startObservation(observation: Observation): Promise { 31 | this._observation = observation; 32 | await this.observationManager.startSync(this.observation); 33 | return this.connectionManager.getNumberOfActiveConnections(); 34 | } 35 | 36 | public connectToNode(ip: string, port: number) { 37 | assert(this.observation.state === ObservationState.Synced); 38 | this.connectionManager.connectToNode(ip, port); 39 | } 40 | 41 | public async stop() { 42 | return new Promise((resolve) => { 43 | this.observationManager.stopObservation(this.observation, () => 44 | this.onObservationStopped(resolve) 45 | ); 46 | }); 47 | } 48 | 49 | private onObservationStopped( 50 | resolve: (observation: Observation) => void 51 | ): void { 52 | resolve(this.observation); 53 | } 54 | 55 | private setupPeerEventHandlers() { 56 | this.connectionManager.on('connected', (data: ConnectedPayload) => { 57 | this.peerEventHandler.onConnected(data, this.observation); 58 | }); 59 | this.connectionManager.on('close', (data: ClosePayload) => { 60 | this.peerEventHandler.onConnectionClose(data, this.observation); 61 | this.emit('disconnect', data); 62 | }); 63 | this.connectionManager.on('data', (data: DataPayload) => { 64 | this.onPeerData(data); 65 | }); 66 | } 67 | 68 | private onPeerData(data: DataPayload): void { 69 | const result = this.peerEventHandler.onData(data, this.observation); 70 | if (result.closedLedger) { 71 | this.observationManager.ledgerCloseConfirmed( 72 | this.observation, 73 | result.closedLedger 74 | ); 75 | } 76 | 77 | if (result.peers.length > 0) this.emit('peers', result.peers); 78 | } 79 | 80 | private get observation(): Observation { 81 | if (!this._observation) { 82 | throw new Error('Observation not set'); 83 | } 84 | return this._observation; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/scp-statement-handler.ts: -------------------------------------------------------------------------------- 1 | import * as P from 'pino'; 2 | import { xdr } from '@stellar/stellar-base'; 3 | import { getPublicKeyStringFromBuffer } from '@stellarbeat/js-stellar-node-connector'; 4 | import { QuorumSetManager } from '../../../../quorum-set-manager'; 5 | import { err, ok, Result } from 'neverthrow'; 6 | import { ExternalizeStatementHandler } from './externalize/externalize-statement-handler'; 7 | import { mapExternalizeStatement } from './externalize/map-externalize-statement'; 8 | import { Ledger } from '../../../../../crawler'; 9 | import { Observation } from '../../../../observation'; 10 | 11 | export class ScpStatementHandler { 12 | constructor( 13 | private quorumSetManager: QuorumSetManager, 14 | private externalizeStatementHandler: ExternalizeStatementHandler, 15 | private logger: P.Logger 16 | ) {} 17 | 18 | public handle( 19 | scpStatement: xdr.ScpStatement, 20 | observation: Observation 21 | ): Result< 22 | { 23 | closedLedger: Ledger | null; 24 | }, 25 | Error 26 | > { 27 | const publicKeyResult = getPublicKeyStringFromBuffer( 28 | scpStatement.nodeId().value() 29 | ); 30 | if (publicKeyResult.isErr()) { 31 | return err(publicKeyResult.error); 32 | } 33 | 34 | const publicKey = publicKeyResult.value; 35 | const slotIndex = BigInt(scpStatement.slotIndex().toString()); 36 | 37 | this.logger.debug( 38 | { 39 | publicKey: publicKey, 40 | slotIndex: slotIndex.toString() 41 | }, 42 | 'processing new scp statement: ' + scpStatement.pledges().switch().name 43 | ); 44 | 45 | const peer = observation.peerNodes.getOrAdd(publicKey); //maybe we got a relayed message from a peer that we have not crawled yet 46 | peer.participatingInSCP = true; 47 | peer.latestActiveSlotIndex = slotIndex.toString(); 48 | 49 | this.quorumSetManager.processQuorumSetHashFromStatement( 50 | peer, 51 | scpStatement, 52 | observation 53 | ); 54 | 55 | if ( 56 | scpStatement.pledges().switch().value !== 57 | xdr.ScpStatementType.scpStExternalize().value 58 | ) { 59 | //only if node is externalizing, we mark the node as validating 60 | return ok({ 61 | closedLedger: null 62 | }); 63 | } 64 | 65 | const externalizeData = mapExternalizeStatement(scpStatement); 66 | if (!externalizeData.isOk()) { 67 | return err(externalizeData.error); 68 | } 69 | 70 | const closedLedgerOrNull = this.externalizeStatementHandler.handle( 71 | observation.peerNodes, 72 | observation.slots.getSlot(slotIndex), 73 | externalizeData.value, 74 | new Date(), //todo: move up, 75 | observation.latestConfirmedClosedLedger 76 | ); 77 | 78 | if ( 79 | closedLedgerOrNull !== null && 80 | closedLedgerOrNull.sequence > 81 | observation.latestConfirmedClosedLedger.sequence 82 | ) { 83 | return ok({ 84 | closedLedger: closedLedgerOrNull 85 | }); 86 | } 87 | 88 | return ok({ 89 | closedLedger: null 90 | }); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/peer-node-collection.ts: -------------------------------------------------------------------------------- 1 | import { PeerNode } from './peer-node'; 2 | import { NodeInfo } from '@stellarbeat/js-stellar-node-connector/lib/node'; 3 | import { Ledger } from './crawler'; 4 | 5 | type PublicKey = string; 6 | 7 | export class PeerNodeCollection { 8 | constructor(private peerNodes: Map = new Map()) {} 9 | 10 | addExternalizedValueForPeerNode( 11 | publicKey: string, 12 | slotIndex: bigint, 13 | value: string, 14 | localTime: Date 15 | ): void { 16 | const peerNode = this.getOrAdd(publicKey); 17 | peerNode.addExternalizedValue(slotIndex, localTime, value); 18 | } 19 | 20 | getOrAdd(publicKey: string) { 21 | let peerNode = this.peerNodes.get(publicKey); 22 | if (peerNode) return peerNode; 23 | 24 | peerNode = new PeerNode(publicKey); 25 | this.peerNodes.set(publicKey, peerNode); 26 | 27 | return peerNode; 28 | } 29 | 30 | get(publicKey: string) { 31 | return this.peerNodes.get(publicKey); 32 | } 33 | 34 | addSuccessfullyConnected( 35 | publicKey: string, 36 | ip: string, 37 | port: number, 38 | nodeInfo: NodeInfo 39 | ): PeerNode | Error { 40 | let peerNode = this.peerNodes.get(publicKey); 41 | if (peerNode && peerNode.successfullyConnected) { 42 | return new Error('PeerNode reusing publicKey'); 43 | } 44 | 45 | if (!peerNode) { 46 | peerNode = new PeerNode(publicKey); 47 | } 48 | 49 | peerNode.nodeInfo = nodeInfo; 50 | peerNode.ip = ip; 51 | peerNode.port = port; 52 | peerNode.successfullyConnected = true; 53 | 54 | this.peerNodes.set(publicKey, peerNode); 55 | 56 | return peerNode; 57 | } 58 | 59 | getAll() { 60 | return this.peerNodes; 61 | } 62 | 63 | values() { 64 | return this.peerNodes.values(); 65 | } 66 | 67 | get size() { 68 | return this.peerNodes.size; 69 | } 70 | 71 | setPeerOverloaded(publicKey: PublicKey, overloaded: boolean): void { 72 | const peer = this.peerNodes.get(publicKey); 73 | if (peer) { 74 | peer.overLoaded = overloaded; 75 | } 76 | } 77 | 78 | setPeerSuppliedPeerList( 79 | publicKey: PublicKey, 80 | suppliedPeerList: boolean 81 | ): void { 82 | const peer = this.peerNodes.get(publicKey); 83 | if (peer) { 84 | peer.suppliedPeerList = suppliedPeerList; 85 | } 86 | } 87 | 88 | confirmLedgerCloseForNode(publicKey: PublicKey, closedLedger: Ledger): void { 89 | const peer = this.getOrAdd(publicKey); 90 | peer.processConfirmedLedgerClose(closedLedger); 91 | } 92 | 93 | //convenience method to avoid having to loop through all peers 94 | confirmLedgerCloseForDisagreeingNodes( 95 | disagreeingNodes: Set 96 | ): void { 97 | for (const publicKey of disagreeingNodes) { 98 | const peer = this.peerNodes.get(publicKey); 99 | if (peer) { 100 | peer.isValidatingIncorrectValues = true; 101 | } 102 | } 103 | } 104 | 105 | //convenience method to avoid having to loop through all peers 106 | confirmLedgerCloseForValidatingNodes( 107 | validatingNodes: Set, 108 | ledger: Ledger 109 | ): void { 110 | for (const publicKey of validatingNodes) { 111 | const peer = this.peerNodes.get(publicKey); 112 | if (peer) { 113 | peer.processConfirmedLedgerClose(ledger); 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/__fixtures__/createDummyExternalizeMessage.ts: -------------------------------------------------------------------------------- 1 | import { hash, Keypair, Networks, xdr } from '@stellar/stellar-base'; 2 | import { createSCPEnvelopeSignature } from '@stellarbeat/js-stellar-node-connector'; 3 | 4 | export function createDummyValue() { 5 | return Buffer.from( 6 | ' Bdej9XkMRNa5mYeecoR8W1+E10N8cje2irYfmzNh/eIAAAAAZd2fCAAAAAAAAAABAAAAAIwdS0o2ARfVAN/PjN6xZrGaEuD0t7zToaDF6Z5B9peZAAAAQIRy/bWclKwWkxF4qTOg0pBncXfpJhczLQP5D60JlqhgR5Vzcn1KOHTSavxBS8+mZCaXNIe4iJFFfGPnxmRgBQI=', 7 | 'base64' 8 | ); 9 | } 10 | 11 | export function createDummyExternalizeStatement( 12 | keyPair: Keypair = Keypair.random(), 13 | slotIndex = '1' 14 | ) { 15 | const commit = new xdr.ScpBallot({ 16 | counter: 1, 17 | value: createDummyValue() 18 | }); 19 | const externalize = new xdr.ScpStatementExternalize({ 20 | commit: commit, 21 | nH: 1, 22 | commitQuorumSetHash: Buffer.alloc(32) 23 | }); 24 | const pledges = xdr.ScpStatementPledges.scpStExternalize(externalize); 25 | 26 | return new xdr.ScpStatement({ 27 | nodeId: xdr.PublicKey.publicKeyTypeEd25519(keyPair.rawPublicKey()), 28 | slotIndex: xdr.Uint64.fromString(slotIndex), 29 | pledges: pledges 30 | }); 31 | } 32 | 33 | export function createDummyExternalizeScpEnvelope( 34 | keyPair: Keypair = Keypair.random(), 35 | networkHash = hash(Buffer.from(Networks.PUBLIC)) 36 | ) { 37 | const statement = createDummyExternalizeStatement(keyPair); 38 | const signatureResult = createSCPEnvelopeSignature( 39 | statement, 40 | keyPair.rawPublicKey(), 41 | keyPair.rawSecretKey(), 42 | networkHash 43 | ); 44 | 45 | if (signatureResult.isErr()) { 46 | throw signatureResult.error; 47 | } 48 | 49 | return new xdr.ScpEnvelope({ 50 | statement: statement, 51 | signature: signatureResult.value 52 | }); 53 | } 54 | 55 | export function createDummyExternalizeMessage( 56 | keyPair: Keypair = Keypair.random(), 57 | networkHash = hash(Buffer.from(Networks.PUBLIC)) 58 | ) { 59 | return xdr.StellarMessage.scpMessage( 60 | createDummyExternalizeScpEnvelope(keyPair, networkHash) 61 | ); 62 | } 63 | 64 | export function createDummyNominationMessage( 65 | keyPair: Keypair = Keypair.random(), 66 | networkHash = hash(Buffer.from(Networks.PUBLIC)) 67 | ) { 68 | const statement = createDummyNominateStatement(keyPair); 69 | const signatureResult = createSCPEnvelopeSignature( 70 | statement, 71 | keyPair.rawPublicKey(), 72 | keyPair.rawSecretKey(), 73 | networkHash 74 | ); 75 | 76 | if (signatureResult.isErr()) { 77 | throw signatureResult.error; 78 | } 79 | 80 | const envelope = new xdr.ScpEnvelope({ 81 | statement: statement, 82 | signature: signatureResult.value 83 | }); 84 | 85 | return xdr.StellarMessage.scpMessage(envelope); 86 | } 87 | 88 | export function createDummyNominateStatement( 89 | keyPair: Keypair = Keypair.random() 90 | ) { 91 | const nomination = new xdr.ScpNomination({ 92 | quorumSetHash: Buffer.alloc(32), 93 | votes: [Buffer.alloc(32), Buffer.alloc(32)], 94 | accepted: [Buffer.alloc(32), Buffer.alloc(32)] 95 | }); 96 | const pledges = xdr.ScpStatementPledges.scpStNominate(nomination); 97 | 98 | return new xdr.ScpStatement({ 99 | nodeId: xdr.PublicKey.publicKeyTypeEd25519(keyPair.rawPublicKey()), 100 | slotIndex: xdr.Uint64.fromString('1'), 101 | pledges: pledges 102 | }); 103 | } 104 | -------------------------------------------------------------------------------- /src/__tests__/peer-node-collection.test.ts: -------------------------------------------------------------------------------- 1 | //write tests 2 | import { PeerNodeCollection } from '../peer-node-collection'; 3 | import { PeerNode } from '../index'; 4 | import { NodeInfo } from '@stellarbeat/js-stellar-node-connector/lib/node'; 5 | 6 | describe('PeerNodeCollection', () => { 7 | let peerNodeCollection: PeerNodeCollection; 8 | 9 | beforeEach(() => { 10 | peerNodeCollection = new PeerNodeCollection(); 11 | }); 12 | 13 | describe('add', () => { 14 | it('should add a new peer node', () => { 15 | const publicKey = 'publicKey'; 16 | const ip = 'localhost'; 17 | const port = 11625; 18 | const nodeInfo: NodeInfo = { 19 | overlayVersion: 3, 20 | overlayMinVersion: 1, 21 | networkId: 'networkId', 22 | ledgerVersion: 2, 23 | versionString: 'versionString' 24 | }; 25 | const peerNode = peerNodeCollection.addSuccessfullyConnected( 26 | publicKey, 27 | ip, 28 | port, 29 | nodeInfo 30 | ); 31 | expect(peerNode).toBeInstanceOf(PeerNode); 32 | if (peerNode instanceof Error) { 33 | throw peerNode; 34 | } 35 | expect(peerNode.publicKey).toBe(publicKey); 36 | expect(peerNode.ip).toBe(ip); 37 | expect(peerNode.port).toBe(port); 38 | expect(peerNode.nodeInfo).toBe(nodeInfo); 39 | expect(peerNode.successfullyConnected).toBeTruthy(); 40 | }); 41 | 42 | it('should return an error if the peer node already exists and has already successfully connected', () => { 43 | const publicKey = 'publicKey'; 44 | const ip = 'localhost'; 45 | const port = 11625; 46 | const nodeInfo: NodeInfo = { 47 | overlayVersion: 3, 48 | overlayMinVersion: 1, 49 | networkId: 'networkId', 50 | ledgerVersion: 2, 51 | versionString: 'versionString' 52 | }; 53 | peerNodeCollection.addSuccessfullyConnected( 54 | publicKey, 55 | ip, 56 | port, 57 | nodeInfo 58 | ); 59 | const peerNode = peerNodeCollection.addSuccessfullyConnected( 60 | publicKey, 61 | ip, 62 | port, 63 | nodeInfo 64 | ); 65 | expect(peerNode).toBeInstanceOf(Error); 66 | }); 67 | 68 | it('should update an existing peer node', () => { 69 | const publicKey = 'publicKey'; 70 | peerNodeCollection.getOrAdd(publicKey); 71 | const newIp = 'newIp'; 72 | const newPort = 11626; 73 | const newNodeInfo: NodeInfo = { 74 | overlayVersion: 4, 75 | overlayMinVersion: 2, 76 | networkId: 'newNetworkId', 77 | ledgerVersion: 3, 78 | versionString: 'newVersionString' 79 | }; 80 | const peerNode = peerNodeCollection.addSuccessfullyConnected( 81 | publicKey, 82 | newIp, 83 | newPort, 84 | newNodeInfo 85 | ); 86 | expect(peerNode).toBeInstanceOf(PeerNode); 87 | if (peerNode instanceof Error) { 88 | throw peerNode; 89 | } 90 | expect(peerNode.publicKey).toBe(publicKey); 91 | expect(peerNode.ip).toBe(newIp); 92 | expect(peerNode.port).toBe(newPort); 93 | expect(peerNode.nodeInfo).toBe(newNodeInfo); 94 | expect(peerNode.successfullyConnected).toBeTruthy(); 95 | }); 96 | 97 | it('should return an existing peer node', () => { 98 | const publicKey = 'publicKey'; 99 | peerNodeCollection.getOrAdd(publicKey); 100 | const peerNode = peerNodeCollection.getOrAdd(publicKey); 101 | expect(peerNode).toBeInstanceOf(PeerNode); 102 | }); 103 | }); 104 | }); 105 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | # Crawler functionality and architecture 2 | ## Crawler class 3 | Responsibilities: 4 | - Manage the crawl queue. Making sure that we don't connect to too many nodes at once 5 | - Interacts with the Network Observer and direct it which nodes to connect (based on the queue) 6 | - Listens to the Network Observer if new peers are detected and adds them to the queue. 7 | - Returns a crawl result to the caller of this library 8 | 9 | ## Network Observer 10 | Responsibilities: 11 | - Creates an Observation and directs state changes through the Observation Manager. 12 | - Connect to Top Tier nodes and maintain their connections to have the most accurate lag measurements 13 | - Connect to other peer nodes in the network and listen to their messages (directed by the Crawler) 14 | - Sets up a peer event listener to listen to connection, data, close and error events from peers. 15 | - Detect new peers and notify the Crawler class 16 | - Forwards ledger closes to the Observation Manager. 17 | 18 | ## Observation 19 | Responsibilities: 20 | - Stores the state of peer nodes 21 | - Stores the state of the Network Observation and manages transitions between states 22 | - States: 23 | - IDLE: initial state 24 | - SYNCING: connect to top tier nodes 25 | - SYNCED: connected to top tier nodes, ready to connect to peer nodes, confirm ledger closes and measure peer lag 26 | - STOPPING: stop the observation, give (top tier) peers the chance to close the connection gracefully and detect the last information. 27 | - STOPPED: observation is stopped, connections are closed and no more connections are made. No more state changes are allowed. 28 | 29 | ## Observation Manager 30 | Responsibilities: 31 | - Manages the state of the Observation 32 | - Manages the transitions between states 33 | - Starts necessary timers (straggler & consensus) on state transitions and when ledgers are closed 34 | 35 | ### Straggler Timer 36 | - When a ledger is closed, the straggler timer is started for all active (connected) non-top tier nodes. 37 | - This gives the nodes a chance to externalize the closed ledger, indicating its validation state and the time it took to do so (lag). 38 | - It also gives nodes the time to send their QuorumSet to us. 39 | - When the straggler timer expires, the node is disconnected. 40 | - In case of a network halt, the straggler timer is started for all active nodes, to give them the change to send their QuorumSet if needed. 41 | - When the observation is stopping, the straggler timer fires for all nodes (including top tier). 42 | 43 | ### Consensus Timer 44 | - If no ledger is confirmed closed, the consensus timer times out and indicates a network halt. 45 | - Every time a ledger is confirmed closed, the consensus timer is reset. 46 | - If the network is halted, the observation manager will start a straggler timeout for all active (non-top tier) nodes. This gives other peers in the crawl queue a chance to connect to the halted network and detect the last information (Public Keys, version strings, quorumset,...). 47 | - If the network is halted and a new node connects, a straggler timeout is immediately fired for that node. This allows to detect minimal information from the node, disconnect and give others a chance. 48 | 49 | ## Peer Event Handler 50 | Responsibilities: 51 | - Listen to connection, data, close and error events from peers 52 | - Processes data (Stellar Message) events, detects ledger closes or new peers and returns them to the Network Observer. 53 | - See readme in stellar-message-handler directory for more information on handling of data events. 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slot.ts: -------------------------------------------------------------------------------- 1 | import { Ledger } from '../../../../../../crawler'; 2 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 3 | import * as P from 'pino'; 4 | import containsSlice from '@stellarbeat/js-stellarbeat-shared/lib/quorum/containsSlice'; 5 | import { extractCloseTimeFromValue } from './extract-close-time-from-value'; 6 | 7 | export type SlotIndex = bigint; 8 | type NodeId = string; 9 | type SlotValue = string; 10 | 11 | export class Slot { 12 | public index: SlotIndex; 13 | private confirmedClosedLedger?: Ledger; 14 | protected valuesMap: Map> = new Map(); 15 | protected localCloseTimeMap: Map = new Map(); //we store the first time we observed a close time for a value 16 | //we can't wait until we validated the value, because slow nodes could influence this time. 17 | protected trustedQuorumSet: QuorumSet; 18 | protected closeTime?: Date; 19 | protected localCloseTime?: Date; 20 | 21 | constructor( 22 | index: SlotIndex, 23 | trustedQuorumSet: QuorumSet, 24 | private logger: P.Logger 25 | ) { 26 | this.index = index; 27 | this.trustedQuorumSet = trustedQuorumSet; 28 | } 29 | 30 | getNodesAgreeingOnExternalizedValue(): Set { 31 | if (this.confirmedClosedLedger === undefined) return new Set(); 32 | 33 | const nodes = this.valuesMap.get(this.confirmedClosedLedger.value); 34 | if (!nodes) return new Set(); 35 | 36 | return nodes; 37 | } 38 | 39 | getNodesDisagreeingOnExternalizedValue(): Set { 40 | let nodes = new Set(); 41 | if (this.confirmedClosedLedger === undefined) return nodes; 42 | 43 | Array.from(this.valuesMap.keys()) 44 | .filter((value) => value !== this.confirmedClosedLedger?.value) 45 | .forEach((value) => { 46 | const otherNodes = this.valuesMap.get(value); 47 | if (otherNodes) nodes = new Set([...nodes, ...otherNodes]); 48 | }); 49 | 50 | return nodes; 51 | } 52 | 53 | addExternalizeValue( 54 | nodeId: NodeId, 55 | value: SlotValue, 56 | localCloseTime: Date 57 | ): void { 58 | let nodesThatExternalizedValue = this.valuesMap.get(value); 59 | if (!nodesThatExternalizedValue) { 60 | nodesThatExternalizedValue = new Set(); 61 | this.valuesMap.set(value, nodesThatExternalizedValue); 62 | } 63 | 64 | if (this.localCloseTimeMap.get(value) === undefined) { 65 | this.localCloseTimeMap.set(value, localCloseTime); //the first observed close time 66 | } 67 | 68 | if (nodesThatExternalizedValue.has(nodeId)) 69 | //already recorded, no need to check if closed 70 | return; 71 | 72 | nodesThatExternalizedValue.add(nodeId); 73 | 74 | if (this.confirmedClosed()) return; 75 | 76 | if (!QuorumSet.getAllValidators(this.trustedQuorumSet).includes(nodeId)) { 77 | return; 78 | } 79 | 80 | this.logger.debug('Node part of trusted quorumSet, attempting slot close', { 81 | node: nodeId 82 | }); 83 | 84 | if (containsSlice(this.trustedQuorumSet, nodesThatExternalizedValue)) { 85 | //try to close slot 86 | this.confirmedClosedLedger = { 87 | sequence: this.index, 88 | value: value, 89 | closeTime: extractCloseTimeFromValue(Buffer.from(value, 'base64')), 90 | localCloseTime: this.localCloseTimeMap.get(value)! 91 | }; 92 | } 93 | } 94 | 95 | getConfirmedClosedLedger(): Ledger | undefined { 96 | return this.confirmedClosedLedger; 97 | } 98 | 99 | confirmedClosed(): boolean { 100 | return this.getConfirmedClosedLedger() !== undefined; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/network-observer/observation-manager.ts: -------------------------------------------------------------------------------- 1 | import { NodeAddress } from '../node-address'; 2 | import { StragglerTimer } from './straggler-timer'; 3 | import { ConnectionManager } from './connection-manager'; 4 | import { P } from 'pino'; 5 | import { Ledger } from '../crawler'; 6 | import { Observation } from './observation'; 7 | import { ConsensusTimer } from './consensus-timer'; 8 | 9 | export class ObservationManager { 10 | constructor( 11 | private connectionManager: ConnectionManager, 12 | private consensusTimer: ConsensusTimer, 13 | private stragglerTimer: StragglerTimer, 14 | private syncingTimeoutMS: number, 15 | private logger: P.Logger 16 | ) {} 17 | 18 | public async startSync(observation: Observation) { 19 | this.logger.info('Moving to syncing state'); 20 | observation.moveToSyncingState(); 21 | this.connectToTopTierNodes(observation.topTierAddresses); 22 | 23 | await this.timeout(this.syncingTimeoutMS); 24 | return this.syncCompleted(observation); 25 | } 26 | 27 | private syncCompleted(observation: Observation) { 28 | this.logger.info( 29 | { 30 | topTierConnections: 31 | this.connectionManager.getNumberOfActiveConnections() 32 | }, 33 | 'Moving to synced state' 34 | ); 35 | observation.moveToSyncedState(); 36 | this.startNetworkConsensusTimer(observation); 37 | } 38 | 39 | public ledgerCloseConfirmed(observation: Observation, ledger: Ledger) { 40 | observation.ledgerCloseConfirmed(ledger); 41 | this.stragglerTimer.startStragglerTimeoutForActivePeers( 42 | false, 43 | observation.topTierAddressesSet 44 | ); 45 | 46 | this.startNetworkConsensusTimer(observation); 47 | } 48 | 49 | private startNetworkConsensusTimer(observation: Observation) { 50 | this.startNetworkConsensusTimerInternal( 51 | this.onNetworkHalted.bind(this, observation) 52 | ); 53 | } 54 | 55 | private onNetworkHalted(observation: Observation) { 56 | this.logger.info('Network consensus timeout'); 57 | observation.setNetworkHalted(); 58 | this.stragglerTimer.startStragglerTimeoutForActivePeers( 59 | false, 60 | observation.topTierAddressesSet 61 | ); 62 | } 63 | 64 | public stopObservation( 65 | observation: Observation, 66 | onStoppedCallback: () => void 67 | ) { 68 | this.logger.info('Moving to stopping state'); 69 | observation.moveToStoppingState(); 70 | 71 | this.consensusTimer.stop(); 72 | if (this.connectionManager.getNumberOfActiveConnections() === 0) { 73 | return this.onLastNodesDisconnected(observation, onStoppedCallback); 74 | } 75 | 76 | this.stragglerTimer.startStragglerTimeoutForActivePeers( 77 | true, 78 | observation.topTierAddressesSet, 79 | () => this.onLastNodesDisconnected(observation, onStoppedCallback) 80 | ); 81 | } 82 | 83 | private onLastNodesDisconnected( 84 | observation: Observation, 85 | onStopped: () => void 86 | ) { 87 | this.logger.info('Moving to stopped state'); 88 | observation.moveToStoppedState(); 89 | 90 | this.stragglerTimer.stopStragglerTimeouts(); 91 | this.connectionManager.shutdown(); 92 | 93 | onStopped(); 94 | } 95 | 96 | private startNetworkConsensusTimerInternal(onNetworkHalted: () => void) { 97 | this.consensusTimer.start(onNetworkHalted); 98 | } 99 | 100 | private connectToTopTierNodes(topTierNodes: NodeAddress[]) { 101 | topTierNodes.forEach((address) => { 102 | this.connectionManager.connectToNode(address[0], address[1]); 103 | }); 104 | } 105 | 106 | private timeout(ms: number): Promise { 107 | return new Promise((resolve) => setTimeout(resolve, ms)); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/externalize-statement-handler.ts: -------------------------------------------------------------------------------- 1 | import * as P from 'pino'; 2 | import { Ledger } from '../../../../../../crawler'; 3 | import { PeerNodeCollection } from '../../../../../../peer-node-collection'; 4 | import { ExternalizeData } from './map-externalize-statement'; 5 | import { Slot } from './slot'; 6 | import * as assert from 'assert'; 7 | 8 | //attempts slot close confirmation and updates peer statuses accordingly 9 | export class ExternalizeStatementHandler { 10 | constructor(private logger: P.Logger) {} 11 | 12 | //returns ledger if slot is closed 13 | public handle( 14 | peerNodes: PeerNodeCollection, 15 | slot: Slot, 16 | externalizeData: ExternalizeData, 17 | localCloseTime: Date, 18 | latestConfirmedClosedLedger: Ledger 19 | ): Ledger | null { 20 | assert.equal(slot.index, externalizeData.slotIndex, 'Slot index mismatch'); 21 | 22 | this.logExternalizeMessage( 23 | externalizeData.publicKey, 24 | slot.index, 25 | externalizeData.value 26 | ); 27 | 28 | peerNodes.addExternalizedValueForPeerNode( 29 | externalizeData.publicKey, 30 | slot.index, 31 | externalizeData.value, 32 | localCloseTime 33 | ); 34 | 35 | const closedLedger = slot.getConfirmedClosedLedger(); 36 | if (closedLedger) { 37 | peerNodes.confirmLedgerCloseForNode( 38 | externalizeData.publicKey, 39 | closedLedger 40 | ); 41 | return null; 42 | } 43 | 44 | //don't confirm older slots as this could mess with the lag detection 45 | //because nodes could relay/replay old externalize messages 46 | if (externalizeData.slotIndex <= latestConfirmedClosedLedger.sequence) 47 | return null; 48 | 49 | const confirmedClosedSlotOrNull = this.attemptSlotCloseConfirmation( 50 | slot, 51 | externalizeData.publicKey, 52 | externalizeData.value 53 | ); 54 | 55 | if (confirmedClosedSlotOrNull === null) return null; 56 | 57 | this.confirmLedgerCloseForPeersThatHaveExternalized( 58 | confirmedClosedSlotOrNull, 59 | slot, 60 | peerNodes 61 | ); 62 | 63 | return confirmedClosedSlotOrNull; 64 | } 65 | 66 | private attemptSlotCloseConfirmation( 67 | slot: Slot, 68 | publicKey: string, 69 | value: string 70 | ): null | Ledger { 71 | slot.addExternalizeValue(publicKey, value, new Date()); 72 | 73 | const closedLedger = slot.getConfirmedClosedLedger(); 74 | if (!closedLedger) return null; 75 | 76 | return closedLedger; 77 | } 78 | 79 | private confirmLedgerCloseForPeersThatHaveExternalized( 80 | closedLedger: Ledger, 81 | slot: Slot, 82 | peers: PeerNodeCollection 83 | ) { 84 | this.logLedgerClose(closedLedger); 85 | peers.confirmLedgerCloseForValidatingNodes( 86 | slot.getNodesAgreeingOnExternalizedValue(), 87 | closedLedger 88 | ); 89 | peers.confirmLedgerCloseForDisagreeingNodes( 90 | slot.getNodesDisagreeingOnExternalizedValue() 91 | ); 92 | } 93 | 94 | private logExternalizeMessage( 95 | publicKey: string, 96 | slotIndex: bigint, 97 | value: string 98 | ) { 99 | this.logger.debug( 100 | { 101 | publicKey: publicKey, 102 | slotIndex: slotIndex.toString(), 103 | value: value 104 | }, 105 | 'Processing externalize msg' 106 | ); 107 | } 108 | 109 | private logLedgerClose(closedLedger: Ledger) { 110 | this.logger.info( 111 | { 112 | sequence: closedLedger.sequence, 113 | closeTime: closedLedger.closeTime, 114 | localCloseTime: closedLedger.localCloseTime 115 | }, 116 | 'Ledger closed!' 117 | ); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /examples/crawl.js: -------------------------------------------------------------------------------- 1 | const jsonStorage = require('../lib').jsonStorage; 2 | const { QuorumSet } = require('@stellarbeat/js-stellarbeat-shared'); 3 | const { createCrawler, createCrawlFactory } = require('../lib'); 4 | const { getConfigFromEnv } = require('@stellarbeat/js-stellar-node-connector'); 5 | const { CrawlerConfiguration } = require('../lib/crawler-configuration'); 6 | 7 | // noinspection JSIgnoredPromiseFromCall 8 | main(); 9 | 10 | async function main() { 11 | if (process.argv.length <= 2) { 12 | console.log('Usage: ' + __filename + ' NODES.JSON_PATH '); 13 | 14 | process.exit(-1); 15 | } 16 | let nodesJsonPath = process.argv[2]; 17 | 18 | console.log('[MAIN] Reading NODES.JSON_PATH'); 19 | let nodes = await jsonStorage.getNodesFromFile(nodesJsonPath); 20 | 21 | console.log('[MAIN] Crawl!'); 22 | let topTierQSet = new QuorumSet(15, [ 23 | 'GCVJ4Z6TI6Z2SOGENSPXDQ2U4RKH3CNQKYUHNSSPYFPNWTLGS6EBH7I2', 24 | 'GCIXVKNFPKWVMKJKVK2V4NK7D4TC6W3BUMXSIJ365QUAXWBRPPJXIR2Z', 25 | 'GBLJNN3AVZZPG2FYAYTYQKECNWTQYYUUY2KVFN2OUKZKBULXIXBZ4FCT', 26 | 'GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63', 27 | 'GDXQB3OMMQ6MGG43PWFBZWBFKBBDUZIVSUDAZZTRAWQZKES2CDSE5HKJ', 28 | 'GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7', 29 | 'GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J', 30 | 'GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7', 31 | 'GCMSM2VFZGRPTZKPH5OABHGH4F3AVS6XTNJXDGCZ3MKCOSUBH3FL6DOB', 32 | 'GA7DV63PBUUWNUFAF4GAZVXU2OZMYRATDLKTC7VTCG7AU4XUPN5VRX4A', 33 | 'GARYGQ5F2IJEBCZJCBNPWNWVDOFK7IBOHLJKKSG2TMHDQKEEC6P4PE4V', 34 | 'GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE', 35 | 'GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT', 36 | 'GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY', 37 | 'GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN', 38 | 'GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z', 39 | 'GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T', 40 | 'GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK', 41 | 'GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH', 42 | 'GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ', 43 | 'GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4', 44 | 'GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C', 45 | 'GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU' 46 | ]); 47 | 48 | const config = getConfigFromEnv(); 49 | const crawlerConfig = new CrawlerConfiguration(config); 50 | crawlerConfig.maxOpenConnections = 100; 51 | let myCrawler = createCrawler(crawlerConfig); 52 | const factory = createCrawlFactory(crawlerConfig); 53 | 54 | try { 55 | let knownQuorumSets = new Map(); 56 | nodes.forEach((node) => { 57 | knownQuorumSets.set(node.quorumSetHashKey, node.quorumSet); 58 | }); 59 | const addresses = nodes 60 | .filter((node) => node.publicKey) 61 | .map((node) => [node.ip, node.port]); 62 | 63 | const topTierAddresses = nodes 64 | .filter((node) => topTierQSet.validators.includes(node.publicKey)) 65 | .map((node) => [node.ip, node.port]); 66 | 67 | const crawl = factory.createCrawl( 68 | addresses, 69 | topTierAddresses, 70 | topTierQSet, 71 | { 72 | sequence: BigInt(0), 73 | closeTime: new Date(0) 74 | }, 75 | knownQuorumSets 76 | ); 77 | 78 | let result = await myCrawler.startCrawl(crawl); 79 | console.log( 80 | '[MAIN] Writing results to file nodes.json in directory crawl_result' 81 | ); 82 | await jsonStorage.writeFilePromise( 83 | './crawl_result/nodes.json', 84 | JSON.stringify(Array.from(result.peers.values())) 85 | ); 86 | 87 | console.log('[MAIN] Finished'); 88 | } catch (e) { 89 | console.log(e); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/crawl-logger.ts: -------------------------------------------------------------------------------- 1 | import { Crawl } from './crawl'; 2 | import { P } from 'pino'; 3 | import { CrawlQueueManager } from './crawl-queue-manager'; 4 | import { ConnectionManager } from './network-observer/connection-manager'; 5 | import { truncate } from './utilities/truncate'; 6 | 7 | export class CrawlLogger { 8 | private loggingTimer?: NodeJS.Timeout; 9 | private _crawl?: Crawl; 10 | 11 | constructor( 12 | private connectionManager: ConnectionManager, 13 | private crawlQueueManager: CrawlQueueManager, 14 | private logger: P.Logger 15 | ) {} 16 | 17 | get crawl(): Crawl { 18 | if (!this._crawl) { 19 | throw new Error('Crawl not set'); 20 | } 21 | return this._crawl; 22 | } 23 | 24 | start(crawl: Crawl) { 25 | console.time('crawl'); 26 | this._crawl = crawl; 27 | this.logger.info( 28 | 'Starting crawl with seed of ' + crawl.nodesToCrawl.length + 'addresses.' 29 | ); 30 | this.loggingTimer = setInterval(() => { 31 | this.logger.info({ 32 | queueLength: this.crawlQueueManager.queueLength(), 33 | activeConnections: 34 | this.connectionManager.getNumberOfActiveConnections(), 35 | activeTopTiers: this.connectionManager 36 | .getActiveConnectionAddresses() 37 | .filter((address) => 38 | crawl.observation.topTierAddressesSet.has(address) 39 | ).length 40 | }); 41 | }, 5000); 42 | } 43 | 44 | stop() { 45 | this.logger.info('Crawl process complete'); 46 | console.timeEnd('crawl'); 47 | clearInterval(this.loggingTimer); 48 | this.logCrawlState(); 49 | this.logger.info('crawl finished'); 50 | } 51 | 52 | private logCrawlState() { 53 | this.logger.debug( 54 | { peers: this.crawl.failedConnections }, 55 | 'Failed connections' 56 | ); 57 | this.crawl.observation.peerNodes.getAll().forEach((peer) => { 58 | this.logger.info({ 59 | ip: peer.key, 60 | pk: truncate(peer.publicKey), 61 | connected: peer.successfullyConnected, 62 | scp: peer.participatingInSCP, 63 | validating: peer.isValidating, 64 | overLoaded: peer.overLoaded, 65 | lagMS: peer.getMinLagMS(), 66 | incorrect: peer.isValidatingIncorrectValues 67 | }); 68 | }); 69 | this.logger.info( 70 | 'Connection attempts: ' + this.crawl.crawledNodeAddresses.size 71 | ); 72 | this.logger.info( 73 | 'Detected public keys: ' + this.crawl.observation.peerNodes.size 74 | ); 75 | this.logger.info( 76 | 'Successful connections: ' + 77 | Array.from(this.crawl.observation.peerNodes.getAll().values()).filter( 78 | (peer) => peer.successfullyConnected 79 | ).length 80 | ); 81 | this.logger.info( 82 | 'Validating nodes: ' + 83 | Array.from(this.crawl.observation.peerNodes.values()).filter( 84 | (node) => node.isValidating 85 | ).length 86 | ); 87 | this.logger.info( 88 | 'Overloaded nodes: ' + 89 | Array.from(this.crawl.observation.peerNodes.values()).filter( 90 | (node) => node.overLoaded 91 | ).length 92 | ); 93 | 94 | this.logger.info( 95 | Array.from(this.crawl.observation.peerNodes.values()).filter( 96 | (node) => node.suppliedPeerList 97 | ).length + ' supplied us with a peers list.' 98 | ); 99 | this.logger.info( 100 | 'Closed ledgers: ' + 101 | this.crawl.observation.slots.getConfirmedClosedSlotIndexes().length 102 | ); 103 | const slowNodes = Array.from( 104 | this.crawl.observation.peerNodes.values() 105 | ).filter((node) => (node.getMinLagMS() ?? 0) > 2000); 106 | 107 | this.logger.info( 108 | 'Slow nodes: ' + 109 | slowNodes.length + 110 | ' ' + 111 | slowNodes 112 | .map((node) => truncate(node.publicKey) + ':' + node.getMinLagMS()) 113 | .join(', ') 114 | ); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/network-observer/__tests__/observation.test.ts: -------------------------------------------------------------------------------- 1 | import { Observation } from '../observation'; 2 | import { PeerNodeCollection } from '../../peer-node-collection'; 3 | import { mock } from 'jest-mock-extended'; 4 | import { NodeAddress } from '../../node-address'; 5 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 6 | import { P } from 'pino'; 7 | import { ObservationState } from '../observation-state'; 8 | import { Slots } from '../peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 9 | 10 | describe('Observation', () => { 11 | const createObservation = (topTierAddresses: NodeAddress[] = []) => { 12 | return new Observation( 13 | 'test', 14 | topTierAddresses, 15 | mock(), 16 | { 17 | sequence: BigInt(0), 18 | closeTime: new Date(), 19 | value: '', 20 | localCloseTime: new Date() 21 | }, 22 | new Map(), 23 | new Slots(new QuorumSet(1, ['A'], []), mock()) 24 | ); 25 | }; 26 | it('should move to syncing state', () => { 27 | const observation = createObservation(); 28 | observation.moveToSyncingState(); 29 | expect(observation.state).toBe(ObservationState.Syncing); 30 | }); 31 | 32 | it('should map top tier addresses', () => { 33 | const observation = createObservation([ 34 | ['a', 1], 35 | ['b', 2] 36 | ]); 37 | expect(observation.topTierAddressesSet).toEqual(new Set(['a:1', 'b:2'])); 38 | }); 39 | 40 | it('should move to synced state', () => { 41 | const observation = createObservation(); 42 | observation.moveToSyncingState(); 43 | observation.moveToSyncedState(); 44 | expect(observation.state).toBe(ObservationState.Synced); 45 | }); 46 | 47 | it('should move to stopping state', () => { 48 | const observation = createObservation(); 49 | observation.moveToSyncingState(); 50 | observation.moveToSyncedState(); 51 | observation.moveToStoppingState(); 52 | expect(observation.state).toBe(ObservationState.Stopping); 53 | }); 54 | 55 | it('should move to stopped state', () => { 56 | const observation = createObservation(); 57 | observation.moveToSyncingState(); 58 | observation.moveToSyncedState(); 59 | observation.moveToStoppingState(); 60 | observation.moveToStoppedState(); 61 | expect(observation.state).toBe(ObservationState.Stopped); 62 | }); 63 | 64 | it('should confirm ledger close', () => { 65 | const observation = createObservation(); 66 | observation.moveToSyncingState(); 67 | observation.moveToSyncedState(); 68 | const ledger = { 69 | sequence: BigInt(1), 70 | closeTime: new Date(), 71 | value: 'value', 72 | localCloseTime: new Date() 73 | }; 74 | observation.ledgerCloseConfirmed(ledger); 75 | expect(observation.latestConfirmedClosedLedger.sequence).toEqual(BigInt(1)); 76 | }); 77 | 78 | it('should not confirm ledger close if not in synced state', () => { 79 | const observation = createObservation(); 80 | const ledger = { 81 | sequence: BigInt(1), 82 | closeTime: new Date(), 83 | value: 'value', 84 | localCloseTime: new Date() 85 | }; 86 | observation.ledgerCloseConfirmed(ledger); 87 | expect(observation.latestConfirmedClosedLedger.sequence).toBe(BigInt(0)); 88 | }); 89 | 90 | it('should mark network not halted if new ledger is found after network was previously halted', () => { 91 | const observation = createObservation(); 92 | observation.moveToSyncingState(); 93 | observation.moveToSyncedState(); 94 | observation.setNetworkHalted(); 95 | const ledger = { 96 | sequence: BigInt(1), 97 | closeTime: new Date(), 98 | value: 'value', 99 | localCloseTime: new Date() 100 | }; 101 | observation.ledgerCloseConfirmed(ledger); 102 | expect(observation.latestConfirmedClosedLedger.sequence).toBe(BigInt(1)); 103 | expect(observation.isNetworkHalted()).toBeFalsy(); 104 | }); 105 | }); 106 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/__tests__/scp-envelope-handler.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { ScpStatementHandler } from '../scp-statement/scp-statement-handler'; 3 | import { ScpEnvelopeHandler } from '../scp-envelope-handler'; 4 | import { createDummyExternalizeScpEnvelope } from '../../../../../__fixtures__/createDummyExternalizeMessage'; 5 | import { Keypair, Networks } from '@stellar/stellar-base'; 6 | import { ok } from 'neverthrow'; 7 | import { Observation } from '../../../../observation'; 8 | import { LRUCache } from 'lru-cache'; 9 | 10 | describe('scp-envelope-handler', () => { 11 | it('should process valid scp envelope and return closed ledger', () => { 12 | const scpStatementHandler = mock(); 13 | const closedLedger = { 14 | sequence: BigInt(2), 15 | closeTime: new Date(), 16 | value: '', 17 | localCloseTime: new Date() 18 | }; 19 | scpStatementHandler.handle.mockReturnValueOnce(ok({ closedLedger })); 20 | const handler = new ScpEnvelopeHandler(scpStatementHandler); 21 | const scpEnvelope = createDummyExternalizeScpEnvelope(); 22 | const crawlState = createMockObservation(); 23 | const result = handler.handle(scpEnvelope, crawlState); 24 | expect(scpStatementHandler.handle).toHaveBeenCalledTimes(1); 25 | expect(result.isOk()).toBeTruthy(); 26 | if (!result.isOk()) return; 27 | expect(result.value.closedLedger).toEqual(closedLedger); 28 | }); 29 | 30 | it('should not process duplicate scp envelope', () => { 31 | const scpStatementHandler = mock(); 32 | const handler = new ScpEnvelopeHandler(scpStatementHandler); 33 | const scpEnvelope = createDummyExternalizeScpEnvelope(); 34 | const crawlState = createMockObservation(); 35 | handler.handle(scpEnvelope, crawlState); 36 | handler.handle(scpEnvelope, crawlState); 37 | expect(scpStatementHandler.handle).toHaveBeenCalledTimes(1); 38 | }); 39 | 40 | it('should not process scp envelope with invalid (too old) ledger', () => { 41 | const scpStatementHandler = mock(); 42 | const handler = new ScpEnvelopeHandler(scpStatementHandler); 43 | const scpEnvelope = createDummyExternalizeScpEnvelope(); 44 | const crawlState = createMockObservation(BigInt(100)); 45 | handler.handle(scpEnvelope, crawlState); 46 | expect(scpStatementHandler.handle).toHaveBeenCalledTimes(0); 47 | }); 48 | 49 | it('should not process scp envelope with invalid signature', () => { 50 | const scpStatementHandler = mock(); 51 | const handler = new ScpEnvelopeHandler(scpStatementHandler); 52 | const scpEnvelope = createDummyExternalizeScpEnvelope( 53 | Keypair.random(), 54 | Buffer.from('wrong network') 55 | ); 56 | const crawlState = createMockObservation(); 57 | const result = handler.handle(scpEnvelope, crawlState); 58 | expect(scpStatementHandler.handle).toHaveBeenCalledTimes(0); 59 | expect(result.isErr()).toBeTruthy(); 60 | if (!result.isErr()) throw new Error('Expected error but got ok'); 61 | expect(result.error.message).toEqual('Invalid SCP Signature'); 62 | }); 63 | 64 | function createMockObservation(sequence = BigInt(1)) { 65 | const observation = mock(); 66 | observation.latestConfirmedClosedLedger = { 67 | sequence: sequence, 68 | closeTime: new Date(), 69 | value: '', 70 | localCloseTime: new Date() 71 | }; 72 | observation.network = Networks.PUBLIC; 73 | observation.envelopeCache = new LRUCache({ max: 1000 }); 74 | return observation; 75 | } 76 | 77 | it('should not process scp envelope when processing SCP signature fails', () => { 78 | const scpStatementHandler = mock(); 79 | const handler = new ScpEnvelopeHandler(scpStatementHandler); 80 | const scpEnvelope = createDummyExternalizeScpEnvelope(); 81 | scpEnvelope.signature(Buffer.alloc(20)); // invalid signature 82 | const crawlState = createMockObservation(); 83 | const result = handler.handle(scpEnvelope, crawlState); 84 | expect(scpStatementHandler.handle).toHaveBeenCalledTimes(0); 85 | expect(result.isErr()).toBeTruthy(); 86 | if (!result.isErr()) throw new Error('Expected error but got ok'); 87 | expect(result.error.message).toEqual('Error verifying SCP Signature'); 88 | }); 89 | }); 90 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/__tests__/on-peer-data.test.ts: -------------------------------------------------------------------------------- 1 | import { ConnectionManager, DataPayload } from '../../connection-manager'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { P } from 'pino'; 4 | import { OnPeerData } from '../on-peer-data'; 5 | import { StellarMessageHandler } from '../stellar-message-handlers/stellar-message-handler'; 6 | import { createDummyExternalizeMessage } from '../../../__fixtures__/createDummyExternalizeMessage'; 7 | import { err, ok } from 'neverthrow'; 8 | import { PeerNodeCollection } from '../../../peer-node-collection'; 9 | import { Ledger } from '../../../crawler'; 10 | import { NodeAddress } from '../../../node-address'; 11 | import { Observation } from '../../observation'; 12 | import { ObservationState } from '../../observation-state'; 13 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 14 | import { Slots } from '../stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 15 | 16 | describe('OnDataHandler', () => { 17 | const connectionManager = mock(); 18 | const stellarMessageHandler = mock(); 19 | const logger = mock(); 20 | 21 | beforeEach(() => { 22 | jest.clearAllMocks(); 23 | }); 24 | 25 | function createDataHandler() { 26 | return new OnPeerData(stellarMessageHandler, logger, connectionManager); 27 | } 28 | 29 | function createObservation(): Observation { 30 | return new Observation( 31 | 'test', 32 | [], 33 | mock(), 34 | mock(), 35 | new Map(), 36 | new Slots(new QuorumSet(1, ['A'], []), logger) 37 | ); 38 | } 39 | 40 | function createData() { 41 | const data: DataPayload = { 42 | publicKey: 'publicKey', 43 | stellarMessageWork: { 44 | stellarMessage: createDummyExternalizeMessage(), 45 | done: jest.fn() 46 | }, 47 | address: 'address' 48 | }; 49 | return data; 50 | } 51 | 52 | function createSuccessfulResult() { 53 | const result: { 54 | closedLedger: Ledger | null; 55 | peers: Array; 56 | } = { 57 | closedLedger: { 58 | sequence: BigInt(1), 59 | closeTime: new Date(), 60 | value: 'value', 61 | localCloseTime: new Date() 62 | }, 63 | peers: [['address', 11625]] 64 | }; 65 | return result; 66 | } 67 | 68 | it('should handle data successfully in Synced state and attempt slot close', () => { 69 | const onDataHandler = createDataHandler(); 70 | const data = createData(); 71 | const result = createSuccessfulResult(); 72 | 73 | stellarMessageHandler.handleStellarMessage.mockReturnValue(ok(result)); 74 | 75 | const observation = createObservation(); 76 | observation.state = ObservationState.Synced; 77 | const receivedResult = onDataHandler.handle(data, observation); 78 | 79 | expect(stellarMessageHandler.handleStellarMessage).toHaveBeenCalledWith( 80 | data.publicKey, 81 | data.stellarMessageWork.stellarMessage, 82 | true, 83 | observation 84 | ); 85 | expect(data.stellarMessageWork.done).toHaveBeenCalled(); 86 | expect(receivedResult).toEqual(result); 87 | }); 88 | 89 | it('should handle data successfully but not attempt slot close if not in synced mode', () => { 90 | const onDataHandler = createDataHandler(); 91 | const data = createData(); 92 | const observation = createObservation(); 93 | observation.state = ObservationState.Syncing; 94 | const result = createSuccessfulResult(); 95 | stellarMessageHandler.handleStellarMessage.mockReturnValue(ok(result)); 96 | 97 | const receivedResult = onDataHandler.handle(data, observation); 98 | 99 | expect(stellarMessageHandler.handleStellarMessage).toHaveBeenCalledWith( 100 | data.publicKey, 101 | data.stellarMessageWork.stellarMessage, 102 | false, 103 | observation 104 | ); 105 | expect(data.stellarMessageWork.done).toHaveBeenCalled(); 106 | expect(receivedResult).toEqual(result); 107 | }); 108 | 109 | it('should handle data error', () => { 110 | const onDataHandler = createDataHandler(); 111 | const data = createData(); 112 | 113 | stellarMessageHandler.handleStellarMessage.mockReturnValue( 114 | err(new Error('error')) 115 | ); 116 | const result = onDataHandler.handle(data, createObservation()); 117 | expect(data.stellarMessageWork.done).toHaveBeenCalled(); 118 | expect(connectionManager.disconnectByAddress).toHaveBeenCalledWith( 119 | data.address, 120 | new Error('error') 121 | ); 122 | expect(result).toEqual({ closedLedger: null, peers: [] }); 123 | }); 124 | }); 125 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { Crawler } from './crawler'; 2 | import { pino } from 'pino'; 3 | import { createNode } from '@stellarbeat/js-stellar-node-connector'; 4 | import { CrawlerConfiguration } from './crawler-configuration'; 5 | import { ConnectionManager } from './network-observer/connection-manager'; 6 | import { CrawlQueueManager } from './crawl-queue-manager'; 7 | import { AsyncCrawlQueue } from './crawl-queue'; 8 | import { MaxCrawlTimeManager } from './max-crawl-time-manager'; 9 | import { CrawlLogger } from './crawl-logger'; 10 | import { NetworkObserver } from './network-observer/network-observer'; 11 | import { StellarMessageHandler } from './network-observer/peer-event-handler/stellar-message-handlers/stellar-message-handler'; 12 | import { Timer } from './utilities/timer'; 13 | import { ExternalizeStatementHandler } from './network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/externalize-statement-handler'; 14 | import { ScpStatementHandler } from './network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/scp-statement-handler'; 15 | import { ScpEnvelopeHandler } from './network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-envelope-handler'; 16 | import { QuorumSetManager } from './network-observer/quorum-set-manager'; 17 | import { StragglerTimer } from './network-observer/straggler-timer'; 18 | import { OnPeerConnected } from './network-observer/peer-event-handler/on-peer-connected'; 19 | import { OnPeerConnectionClosed } from './network-observer/peer-event-handler/on-peer-connection-closed'; 20 | import { OnPeerData } from './network-observer/peer-event-handler/on-peer-data'; 21 | import { ObservationManager } from './network-observer/observation-manager'; 22 | import { PeerEventHandler } from './network-observer/peer-event-handler/peer-event-handler'; 23 | import { Timers } from './utilities/timers'; 24 | import { TimerFactory } from './utilities/timer-factory'; 25 | import { ConsensusTimer } from './network-observer/consensus-timer'; 26 | import { ObservationFactory } from './network-observer/observation-factory'; 27 | import { CrawlFactory } from './crawl-factory'; 28 | 29 | export { Crawler } from './crawler'; 30 | export { CrawlResult } from './crawl-result'; 31 | export { PeerNode } from './peer-node'; 32 | export { default as jsonStorage } from './utilities/json-storage'; 33 | 34 | export function createLogger(): pino.Logger { 35 | return pino({ 36 | level: process.env.LOG_LEVEL || 'info', 37 | base: undefined 38 | }); 39 | } 40 | 41 | export function createCrawlFactory( 42 | config: CrawlerConfiguration, 43 | logger?: pino.Logger 44 | ) { 45 | if (!logger) { 46 | logger = createLogger(); 47 | } 48 | return new CrawlFactory( 49 | new ObservationFactory(), 50 | config.nodeConfig.network, 51 | logger 52 | ); 53 | } 54 | 55 | export function createCrawler( 56 | config: CrawlerConfiguration, 57 | logger?: pino.Logger 58 | ): Crawler { 59 | if (!logger) { 60 | logger = createLogger(); 61 | } 62 | 63 | const node = createNode(config.nodeConfig, logger); 64 | const connectionManager = new ConnectionManager( 65 | node, 66 | config.blackList, 67 | logger 68 | ); 69 | const quorumSetManager = new QuorumSetManager( 70 | connectionManager, 71 | config.quorumSetRequestTimeoutMS, 72 | logger 73 | ); 74 | const crawlQueueManager = new CrawlQueueManager( 75 | new AsyncCrawlQueue(config.maxOpenConnections), 76 | logger 77 | ); 78 | 79 | const scpEnvelopeHandler = new ScpEnvelopeHandler( 80 | new ScpStatementHandler( 81 | quorumSetManager, 82 | new ExternalizeStatementHandler(logger), 83 | logger 84 | ) 85 | ); 86 | const stellarMessageHandler = new StellarMessageHandler( 87 | scpEnvelopeHandler, 88 | quorumSetManager, 89 | logger 90 | ); 91 | 92 | const timers = new Timers(new TimerFactory()); 93 | const stragglerTimer = new StragglerTimer( 94 | connectionManager, 95 | timers, 96 | config.peerStraggleTimeoutMS, 97 | logger 98 | ); 99 | const peerEventHandler = new PeerEventHandler( 100 | new OnPeerConnected(stragglerTimer, connectionManager, logger), 101 | new OnPeerConnectionClosed(quorumSetManager, logger), 102 | new OnPeerData(stellarMessageHandler, logger, connectionManager) 103 | ); 104 | const consensusTimer = new ConsensusTimer( 105 | new Timer(), 106 | config.consensusTimeoutMS 107 | ); 108 | 109 | const networkObserverStateManager = new ObservationManager( 110 | connectionManager, 111 | consensusTimer, 112 | stragglerTimer, 113 | config.syncingTimeoutMS, 114 | logger 115 | ); 116 | const peerNetworkManager = new NetworkObserver( 117 | new ObservationFactory(), 118 | connectionManager, 119 | quorumSetManager, 120 | peerEventHandler, 121 | networkObserverStateManager 122 | ); 123 | 124 | return new Crawler( 125 | config, 126 | crawlQueueManager, 127 | new MaxCrawlTimeManager(), 128 | peerNetworkManager, 129 | new CrawlLogger(connectionManager, crawlQueueManager, logger), 130 | logger 131 | ); 132 | } 133 | export { CrawlerConfiguration } from './crawler-configuration'; 134 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/__tests__/scp-statement-handler.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { ScpStatementHandler } from '../scp-statement-handler'; 3 | import { QuorumSetManager } from '../../../../../quorum-set-manager'; 4 | import { P } from 'pino'; 5 | import { ExternalizeStatementHandler } from '../externalize/externalize-statement-handler'; 6 | import { 7 | createDummyExternalizeStatement, 8 | createDummyNominationMessage 9 | } from '../../../../../../__fixtures__/createDummyExternalizeMessage'; 10 | import { Keypair } from '@stellar/stellar-base'; 11 | import { PeerNodeCollection } from '../../../../../../peer-node-collection'; 12 | import { Slots } from '../externalize/slots'; 13 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 14 | import { Ledger } from '../../../../../../crawler'; 15 | import { Observation } from '../../../../../observation'; 16 | 17 | describe('scp-statement-handler', () => { 18 | it('should process new scp statement and newly closed ledger', () => { 19 | const quorumSetManager = mock(); 20 | const externalizeStatementHandler = mock(); 21 | const closedLedger: Ledger = { 22 | sequence: BigInt(2), 23 | closeTime: new Date(), 24 | value: '', 25 | localCloseTime: new Date() 26 | }; 27 | externalizeStatementHandler.handle.mockReturnValueOnce(closedLedger); 28 | const handler = new ScpStatementHandler( 29 | quorumSetManager, 30 | externalizeStatementHandler, 31 | mock() 32 | ); 33 | 34 | const keyPair = Keypair.random(); 35 | const scpStatement = createDummyExternalizeStatement(keyPair); 36 | const observation = mock(); 37 | observation.peerNodes = new PeerNodeCollection(); 38 | observation.slots = new Slots( 39 | new QuorumSet(1, ['A'], []), 40 | mock() 41 | ); 42 | observation.latestConfirmedClosedLedger = { 43 | sequence: BigInt(1), 44 | closeTime: new Date(), 45 | value: '', 46 | localCloseTime: new Date() 47 | }; 48 | 49 | const result = handler.handle(scpStatement, observation); 50 | expect(result.isOk()).toBeTruthy(); 51 | if (!result.isOk()) return; 52 | expect(result.value.closedLedger).toEqual(closedLedger); 53 | expect( 54 | observation.peerNodes.get(keyPair.publicKey())?.participatingInSCP 55 | ).toBeTruthy(); 56 | expect( 57 | observation.peerNodes.get(keyPair.publicKey())?.latestActiveSlotIndex 58 | ).toEqual('1'); 59 | expect( 60 | quorumSetManager.processQuorumSetHashFromStatement 61 | ).toHaveBeenCalledTimes(1); 62 | }); 63 | 64 | it('should not return already closed ledger or older ledger', () => { 65 | const quorumSetManager = mock(); 66 | const externalizeStatementHandler = mock(); 67 | const handler = new ScpStatementHandler( 68 | quorumSetManager, 69 | externalizeStatementHandler, 70 | mock() 71 | ); 72 | 73 | const keyPair = Keypair.random(); 74 | const scpStatement = createDummyExternalizeStatement(keyPair, '2'); 75 | const observation = mock(); 76 | observation.peerNodes = new PeerNodeCollection(); 77 | observation.slots = new Slots( 78 | new QuorumSet(1, ['A'], []), 79 | mock() 80 | ); 81 | observation.latestConfirmedClosedLedger = { 82 | sequence: BigInt(2), 83 | closeTime: new Date(), 84 | value: '', 85 | localCloseTime: new Date() 86 | }; 87 | externalizeStatementHandler.handle.mockReturnValueOnce( 88 | observation.latestConfirmedClosedLedger 89 | ); 90 | 91 | const result = handler.handle(scpStatement, observation); 92 | expect(result.isOk()).toBeTruthy(); 93 | if (!result.isOk()) return; 94 | expect(result.value.closedLedger).toBeNull(); 95 | }); 96 | 97 | it('should not use non-externalize statement for ledger close confirmation', () => { 98 | const keyPair = Keypair.random(); 99 | const nominationMessage = createDummyNominationMessage(keyPair); 100 | 101 | const quorumSetManager = mock(); 102 | const externalizeStatementHandler = mock(); 103 | 104 | const handler = new ScpStatementHandler( 105 | quorumSetManager, 106 | externalizeStatementHandler, 107 | mock() 108 | ); 109 | 110 | const observation = mock(); 111 | observation.peerNodes = new PeerNodeCollection(); 112 | 113 | const result = handler.handle( 114 | nominationMessage.envelope().statement(), 115 | observation 116 | ); 117 | expect(result.isOk()).toBeTruthy(); 118 | if (!result.isOk()) return; 119 | expect(result.value.closedLedger).toBeNull(); 120 | expect( 121 | observation.peerNodes.get(keyPair.publicKey())?.participatingInSCP 122 | ).toBeTruthy(); 123 | expect( 124 | observation.peerNodes.get(keyPair.publicKey())?.latestActiveSlotIndex 125 | ).toEqual('1'); 126 | expect( 127 | quorumSetManager.processQuorumSetHashFromStatement 128 | ).toHaveBeenCalledTimes(1); 129 | expect(externalizeStatementHandler.handle).toHaveBeenCalledTimes(0); 130 | }); 131 | }); 132 | -------------------------------------------------------------------------------- /src/network-observer/__tests__/observation-manager.test.ts: -------------------------------------------------------------------------------- 1 | import { ObservationManager } from '../observation-manager'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { ConnectionManager } from '../connection-manager'; 4 | import { ConsensusTimer } from '../consensus-timer'; 5 | import { StragglerTimer } from '../straggler-timer'; 6 | import { P } from 'pino'; 7 | import { Observation } from '../observation'; 8 | import { PeerNodeCollection } from '../../peer-node-collection'; 9 | import { ObservationState } from '../observation-state'; 10 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 11 | import { Ledger } from '../../crawler'; 12 | import { Slots } from '../peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 13 | import { NodeAddress } from '../../node-address'; 14 | 15 | describe('ObservationManager', () => { 16 | const connectionManager = mock(); 17 | const consensusTimer = mock(); 18 | const stragglerTimer = mock(); 19 | const logger = mock(); 20 | 21 | const observationManager = new ObservationManager( 22 | connectionManager, 23 | consensusTimer, 24 | stragglerTimer, 25 | 200, 26 | logger 27 | ); 28 | 29 | const createObservation = (topTierAddresses: NodeAddress[] = []) => { 30 | return new Observation( 31 | 'test', 32 | topTierAddresses, 33 | mock(), 34 | mock(), 35 | new Map(), 36 | new Slots(new QuorumSet(1, ['A'], []), mock()) 37 | ); 38 | }; 39 | 40 | beforeEach(() => { 41 | jest.clearAllMocks(); 42 | }); 43 | 44 | it('should start syncing', (resolve) => { 45 | const observation = createObservation([['localhost', 11625]]); 46 | observationManager.startSync(observation).then(() => { 47 | expect(connectionManager.connectToNode).toHaveBeenCalledWith( 48 | observation.topTierAddresses[0][0], 49 | observation.topTierAddresses[0][1] 50 | ); 51 | expect(consensusTimer.start).toHaveBeenCalled(); 52 | expect(observation.state).toBe(ObservationState.Synced); 53 | resolve(); 54 | }); 55 | 56 | expect(observation.state).toBe(ObservationState.Syncing); 57 | }); 58 | 59 | it('should stop observation immediately if no more active nodes', (resolve) => { 60 | connectionManager.getNumberOfActiveConnections.mockReturnValue(0); 61 | const observation = createObservation(); 62 | observation.moveToSyncingState(); 63 | observation.moveToSyncedState(); 64 | observationManager.stopObservation(observation, () => {}); 65 | 66 | expect(observation.state).toBe(ObservationState.Stopped); 67 | expect(consensusTimer.stop).toHaveBeenCalled(); 68 | expect(stragglerTimer.stopStragglerTimeouts).toHaveBeenCalled(); 69 | expect(connectionManager.shutdown).toHaveBeenCalled(); 70 | resolve(); 71 | }); 72 | 73 | it('should stop observation after all active nodes are disconnected', (resolve) => { 74 | connectionManager.getNumberOfActiveConnections.mockReturnValue(1); 75 | const observation = createObservation(); 76 | observation.moveToSyncingState(); 77 | observation.moveToSyncedState(); 78 | const callback = () => { 79 | expect(observation.state).toBe(ObservationState.Stopped); 80 | expect(stragglerTimer.stopStragglerTimeouts).toHaveBeenCalled(); 81 | expect(connectionManager.shutdown).toHaveBeenCalled(); 82 | resolve(); 83 | }; 84 | observationManager.stopObservation(observation, callback); 85 | expect(observation.state).toBe(ObservationState.Stopping); 86 | expect(consensusTimer.stop).toHaveBeenCalled(); 87 | expect( 88 | stragglerTimer.startStragglerTimeoutForActivePeers 89 | ).toHaveBeenCalled(); 90 | expect(stragglerTimer.startStragglerTimeoutForActivePeers).toBeCalledWith( 91 | true, 92 | observation.topTierAddressesSet, 93 | expect.any(Function) 94 | ); 95 | 96 | const onLastNodesDisconnected = stragglerTimer 97 | .startStragglerTimeoutForActivePeers.mock.calls[0][2] as () => void; 98 | onLastNodesDisconnected(); 99 | }); 100 | 101 | it('should handle ledger close confirmed', () => { 102 | const observation = createObservation(); 103 | observation.moveToSyncingState(); 104 | observation.moveToSyncedState(); 105 | observationManager.ledgerCloseConfirmed(observation, {} as any); 106 | expect( 107 | stragglerTimer.startStragglerTimeoutForActivePeers 108 | ).toHaveBeenCalled(); 109 | expect(consensusTimer.start).toHaveBeenCalled(); 110 | }); 111 | 112 | it('should handle network halted', async () => { 113 | const peerNodes = new PeerNodeCollection(); 114 | const observation = new Observation( 115 | 'test', 116 | [['localhost', 11625]], 117 | peerNodes, 118 | mock(), 119 | new Map(), 120 | new Slots(new QuorumSet(1, ['A'], []), mock()) 121 | ); 122 | await observationManager.startSync(observation); 123 | expect(observation.isNetworkHalted()).toBe(false); 124 | const networkHaltedCallback = consensusTimer.start.mock 125 | .calls[0][0] as () => void; 126 | networkHaltedCallback(); 127 | expect(observation.isNetworkHalted()).toBe(true); 128 | 129 | expect( 130 | stragglerTimer.startStragglerTimeoutForActivePeers 131 | ).toHaveBeenCalled(); 132 | }); 133 | }); 134 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/__tests__/on-peer-connected.test.ts: -------------------------------------------------------------------------------- 1 | import { PeerNodeCollection } from '../../../peer-node-collection'; 2 | import { mock, MockProxy } from 'jest-mock-extended'; 3 | import { P } from 'pino'; 4 | import { ConnectedPayload, ConnectionManager } from '../../connection-manager'; 5 | import { StragglerTimer } from '../../straggler-timer'; 6 | import { OnPeerConnected } from '../on-peer-connected'; 7 | import { Observation } from '../../observation'; 8 | import { ObservationState } from '../../observation-state'; 9 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 10 | import { Ledger } from '../../../crawler'; 11 | import { Slots } from '../stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 12 | 13 | describe('OnPeerConnectedHandler', () => { 14 | const connectionManager = mock(); 15 | const stragglerHandler = mock(); 16 | 17 | beforeEach(() => { 18 | jest.clearAllMocks(); 19 | }); 20 | 21 | const createOnConnectedHandler = () => { 22 | return new OnPeerConnected( 23 | stragglerHandler, 24 | connectionManager, 25 | mock() 26 | ); 27 | }; 28 | const createObservation = (): Observation => { 29 | return new Observation( 30 | 'test', 31 | [], 32 | mock(), 33 | mock(), 34 | new Map(), 35 | new Slots(new QuorumSet(1, ['A'], []), mock()) 36 | ); 37 | }; 38 | 39 | const createData = (): ConnectedPayload => { 40 | return { 41 | ip: 'localhost', 42 | port: 11625, 43 | publicKey: 'publicKey', 44 | nodeInfo: { 45 | overlayVersion: 3, 46 | overlayMinVersion: 1, 47 | networkId: 'networkId', 48 | ledgerVersion: 2, 49 | versionString: 'versionString' 50 | } 51 | }; 52 | }; 53 | 54 | function assertPeerSuccessfullyConnected( 55 | peerNodes: PeerNodeCollection, 56 | data: ConnectedPayload 57 | ) { 58 | expect(peerNodes.addSuccessfullyConnected).toHaveBeenCalledWith( 59 | data.publicKey, 60 | data.ip, 61 | data.port, 62 | data.nodeInfo 63 | ); 64 | 65 | expect(connectionManager.disconnectByAddress).not.toBeCalled(); 66 | } 67 | 68 | it('should handle a successful connection in SYNCING state', () => { 69 | const onConnectedHandler = createOnConnectedHandler(); 70 | const data = createData(); 71 | const observation = createObservation(); 72 | observation.state = ObservationState.Syncing; 73 | onConnectedHandler.handle(data, observation); 74 | 75 | assertPeerSuccessfullyConnected(observation.peerNodes, data); 76 | }); 77 | 78 | it('should handle a successful connection in SYNCED state', () => { 79 | const onConnectedHandler = createOnConnectedHandler(); 80 | const data = createData(); 81 | const observation = createObservation(); 82 | observation.state = ObservationState.Synced; 83 | onConnectedHandler.handle(data, observation); 84 | 85 | assertPeerSuccessfullyConnected(observation.peerNodes, data); 86 | }); 87 | 88 | function assertDisconnected( 89 | data: ConnectedPayload, 90 | observation: Observation, 91 | error?: Error 92 | ) { 93 | expect(observation.peerNodes.addSuccessfullyConnected).toBeCalled(); 94 | expect(connectionManager.disconnectByAddress).toBeCalledWith( 95 | data.ip + ':' + data.port, 96 | error 97 | ); 98 | expect(stragglerHandler.startStragglerTimeout).not.toBeCalled(); 99 | } 100 | 101 | it('should refuse connection in IDLE state', () => { 102 | const onConnectedHandler = createOnConnectedHandler(); 103 | const data = createData(); 104 | const observation = createObservation(); 105 | observation.state = ObservationState.Idle; 106 | onConnectedHandler.handle(data, observation); 107 | assertDisconnected(data, observation, new Error('Connected while idle')); 108 | }); 109 | 110 | function assertStragglerTimeoutStarted(data: ConnectedPayload) { 111 | expect(stragglerHandler.startStragglerTimeout).toHaveBeenCalledWith([ 112 | data.ip + ':' + data.port 113 | ]); 114 | } 115 | 116 | it('should start straggler timeout in STOPPING state', () => { 117 | const onConnectedHandler = createOnConnectedHandler(); 118 | const data = createData(); 119 | const observation = createObservation(); 120 | observation.state = ObservationState.Stopping; 121 | onConnectedHandler.handle(data, observation); 122 | assertPeerSuccessfullyConnected(observation.peerNodes, data); 123 | assertStragglerTimeoutStarted(data); 124 | }); 125 | 126 | it('should start straggler timeout if network is halted ', () => { 127 | const onConnectedHandler = createOnConnectedHandler(); 128 | const data = createData(); 129 | const observation = createObservation(); 130 | observation.state = ObservationState.Synced; 131 | observation.setNetworkHalted(); 132 | onConnectedHandler.handle(data, observation); 133 | assertPeerSuccessfullyConnected(observation.peerNodes, data); 134 | assertStragglerTimeoutStarted(data); 135 | }); 136 | 137 | it('should disconnect if Peer not valid', () => { 138 | const onConnectedHandler = createOnConnectedHandler(); 139 | const data = createData(); 140 | const observation = createObservation(); 141 | const error = new Error('error'); 142 | ( 143 | observation.peerNodes as MockProxy 144 | ).addSuccessfullyConnected.mockReturnValue(error); 145 | onConnectedHandler.handle(data, observation); 146 | 147 | assertDisconnected(data, observation, error); 148 | }); 149 | }); 150 | -------------------------------------------------------------------------------- /src/network-observer/connection-manager.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events'; 2 | import { 3 | Connection, 4 | Node as NetworkNode 5 | } from '@stellarbeat/js-stellar-node-connector'; 6 | import { P } from 'pino'; 7 | import { truncate } from '../utilities/truncate'; 8 | import { StellarMessageWork } from '@stellarbeat/js-stellar-node-connector/lib/connection/connection'; 9 | import { NodeInfo } from '@stellarbeat/js-stellar-node-connector/lib/node'; 10 | 11 | type PublicKey = string; 12 | type Address = string; 13 | 14 | export interface ConnectedPayload { 15 | publicKey: PublicKey; 16 | ip: string; 17 | port: number; 18 | nodeInfo: NodeInfo; 19 | } 20 | 21 | export interface DataPayload { 22 | address: Address; 23 | stellarMessageWork: StellarMessageWork; 24 | publicKey: PublicKey; 25 | } 26 | 27 | export interface ClosePayload { 28 | address: string; 29 | publicKey?: PublicKey; 30 | } 31 | 32 | export class ConnectionManager extends EventEmitter { 33 | private activeConnections: Map; 34 | 35 | constructor( 36 | private node: NetworkNode, 37 | private blackList: Set, 38 | private logger: P.Logger 39 | ) { 40 | super(); 41 | this.activeConnections = new Map(); // Active connections keyed by node public key or address 42 | } 43 | 44 | /** 45 | * Connects to a node at the specified IP and port. 46 | * @param {string} ip The IP address of the node. 47 | * @param {number} port The port number of the node. 48 | */ 49 | connectToNode(ip: string, port: number) { 50 | const address = `${ip}:${port}`; 51 | const connection = this.node.connectTo(ip, port); 52 | this.logger.debug({ peer: connection.remoteAddress }, 'Connecting'); 53 | 54 | // Setup event listeners for the connection 55 | connection.on('connect', (publicKey, nodeInfo) => { 56 | this.logger.trace('Connect event received'); 57 | if (this.blackList.has(publicKey)) { 58 | this.logger.debug({ peer: connection.remoteAddress }, 'Blacklisted'); 59 | this.disconnect(connection); 60 | return; 61 | } 62 | this.logger.debug( 63 | { 64 | pk: truncate(publicKey), 65 | peer: connection.remoteAddress, 66 | local: connection.localAddress 67 | }, 68 | 'Connected' 69 | ); 70 | this.activeConnections.set(address, connection); 71 | connection.remotePublicKey = publicKey; 72 | this.emit('connected', { publicKey, ip, port, nodeInfo }); 73 | }); 74 | 75 | connection.on('error', (error) => { 76 | this.logger.debug(`Connection error with ${address}: ${error.message}`); 77 | this.disconnect(connection, error); 78 | }); 79 | 80 | connection.on('timeout', () => { 81 | this.logger.debug(`Connection timeout for ${address}`); 82 | this.disconnect(connection); 83 | }); 84 | 85 | connection.on('close', (hadError: boolean) => { 86 | this.logger.debug( 87 | { 88 | pk: truncate(connection.remotePublicKey), 89 | peer: connection.remoteAddress, 90 | hadError: hadError, 91 | local: connection.localAddress 92 | }, 93 | 'Node connection closed' 94 | ); 95 | this.activeConnections.delete(address); 96 | const closePayload: ClosePayload = { 97 | address, 98 | publicKey: connection.remotePublicKey 99 | }; 100 | this.emit('close', closePayload); 101 | }); 102 | 103 | connection.on('data', (stellarMessageWork: StellarMessageWork) => { 104 | if (!connection.remotePublicKey) { 105 | this.logger.error(`Received data from unknown peer ${address}`); 106 | return; 107 | } 108 | this.emit('data', { 109 | address, 110 | publicKey: connection.remotePublicKey, 111 | stellarMessageWork 112 | }); 113 | }); 114 | } 115 | 116 | private disconnect(connection: Connection, error?: Error): void { 117 | if (error) { 118 | this.logger.debug( 119 | { 120 | peer: connection.remoteAddress, 121 | pk: truncate(connection.remotePublicKey), 122 | error: error.message 123 | }, 124 | 'Disconnecting' 125 | ); 126 | } else { 127 | this.logger.trace( 128 | { 129 | peer: connection.remoteAddress, 130 | pk: truncate(connection.remotePublicKey) 131 | }, 132 | 'Disconnecting' 133 | ); 134 | } 135 | 136 | connection.destroy(); 137 | } 138 | 139 | /*public broadcast(stellarMessage: xdr.StellarMessage, doNotSendTo: Set
) { 140 | 141 | }*/ 142 | 143 | public disconnectByAddress(address: Address, error?: Error): void { 144 | const connection = this.activeConnections.get(address); 145 | if (!connection) { 146 | return; 147 | } 148 | this.disconnect(connection, error); 149 | } 150 | 151 | getActiveConnection(address: Address) { 152 | return this.activeConnections.get(address); 153 | } 154 | 155 | getActiveConnectionAddresses(): string[] { 156 | return Array.from(this.activeConnections.keys()); 157 | } 158 | 159 | hasActiveConnectionTo(address: Address) { 160 | return this.activeConnections.has(address); 161 | } 162 | 163 | getNumberOfActiveConnections() { 164 | return this.activeConnections.size; 165 | } 166 | 167 | /** 168 | * Shuts down the connection manager, closing all active connections. 169 | */ 170 | shutdown() { 171 | this.activeConnections.forEach((connection) => { 172 | this.disconnect(connection); 173 | }); //what about the in progress connections 174 | this.logger.info('ConnectionManager shutdown: All connections closed.', { 175 | activeConnections: this.activeConnections.size 176 | }); 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/__tests__/stellar-message-handler.test.ts: -------------------------------------------------------------------------------- 1 | import { StellarMessageHandler } from '../stellar-message-handler'; 2 | import { ScpEnvelopeHandler } from '../scp-envelope/scp-envelope-handler'; 3 | import { QuorumSetManager } from '../../../quorum-set-manager'; 4 | import { P } from 'pino'; 5 | import { Keypair } from '@stellar/stellar-base'; 6 | import { mock, MockProxy } from 'jest-mock-extended'; 7 | import { createDummyExternalizeMessage } from '../../../../__fixtures__/createDummyExternalizeMessage'; 8 | import { ok } from 'neverthrow'; 9 | import { createDummyPeersMessage } from '../../../../__fixtures__/createDummyPeersMessage'; 10 | import { createDummyQuorumSetMessage } from '../../../../__fixtures__/createDummyQuorumSetMessage'; 11 | import { createDummyDontHaveMessage } from '../../../../__fixtures__/createDummyDontHaveMessage'; 12 | import { createDummyErrLoadMessage } from '../../../../__fixtures__/createDummyErrLoadMessage'; 13 | import { PeerNodeCollection } from '../../../../peer-node-collection'; 14 | import { Observation } from '../../../observation'; 15 | 16 | describe('StellarMessageHandler', () => { 17 | let scpManager: MockProxy; 18 | let quorumSetManager: MockProxy; 19 | let logger: MockProxy; 20 | let handler: StellarMessageHandler; 21 | let senderPublicKey: string; 22 | 23 | beforeEach(() => { 24 | scpManager = mock(); 25 | quorumSetManager = mock(); 26 | logger = mock(); 27 | handler = new StellarMessageHandler(scpManager, quorumSetManager, logger); 28 | senderPublicKey = 'A'; 29 | }); 30 | 31 | describe('handleStellarMessage', () => { 32 | it('should handle SCP message and attempt ledger close', () => { 33 | const keyPair = Keypair.random(); 34 | const stellarMessage = createDummyExternalizeMessage(keyPair); 35 | const observation = mock(); 36 | const closedLedger = { 37 | sequence: BigInt(2), 38 | closeTime: new Date(), 39 | value: '', 40 | localCloseTime: new Date() 41 | }; 42 | scpManager.handle.mockReturnValueOnce( 43 | ok({ 44 | closedLedger: closedLedger 45 | }) 46 | ); 47 | const result = handler.handleStellarMessage( 48 | senderPublicKey, 49 | stellarMessage, 50 | true, 51 | observation 52 | ); 53 | expect(scpManager.handle).toHaveBeenCalledTimes(1); 54 | expect(result.isOk()).toBeTruthy(); 55 | if (!result.isOk()) return; 56 | expect(result.value).toEqual({ 57 | closedLedger: closedLedger, 58 | peers: [] 59 | }); 60 | }); 61 | 62 | it('should not attempt ledger close', () => { 63 | const stellarMessage = createDummyExternalizeMessage(); 64 | const observation = mock(); 65 | const result = handler.handleStellarMessage( 66 | senderPublicKey, 67 | stellarMessage, 68 | false, 69 | observation 70 | ); 71 | expect(scpManager.handle).toHaveBeenCalledTimes(0); 72 | expect(result.isOk()).toBeTruthy(); 73 | }); 74 | 75 | it('should handle peers message', () => { 76 | const stellarMessage = createDummyPeersMessage(); 77 | const observation = mock(); 78 | const peerNodes = new PeerNodeCollection(); 79 | peerNodes.getOrAdd(senderPublicKey); 80 | observation.peerNodes = peerNodes; 81 | 82 | const result = handler.handleStellarMessage( 83 | senderPublicKey, 84 | stellarMessage, 85 | true, 86 | observation 87 | ); 88 | expect(result.isOk()).toBeTruthy(); 89 | if (!result.isOk()) return; 90 | expect(result.value).toEqual({ 91 | closedLedger: null, 92 | peers: [['127.0.0.1', 11625]] 93 | }); 94 | expect(peerNodes.get(senderPublicKey)?.suppliedPeerList).toBeTruthy(); 95 | }); 96 | 97 | it('should handle SCP quorum set message', () => { 98 | const stellarMessage = createDummyQuorumSetMessage(); 99 | const observation = mock(); 100 | const result = handler.handleStellarMessage( 101 | senderPublicKey, 102 | stellarMessage, 103 | true, 104 | observation 105 | ); 106 | expect(quorumSetManager.processQuorumSet).toHaveBeenCalledTimes(1); 107 | expect(result.isOk()).toBeTruthy(); 108 | if (!result.isOk()) return; 109 | expect(result.value).toEqual({ 110 | closedLedger: null, 111 | peers: [] 112 | }); 113 | }); 114 | 115 | it('should handle dont have message', () => { 116 | const stellarMessage = createDummyDontHaveMessage(); 117 | const observation = mock(); 118 | const result = handler.handleStellarMessage( 119 | senderPublicKey, 120 | stellarMessage, 121 | true, 122 | observation 123 | ); 124 | expect( 125 | quorumSetManager.peerNodeDoesNotHaveQuorumSet 126 | ).toHaveBeenCalledTimes(1); 127 | expect(result.isOk()).toBeTruthy(); 128 | if (!result.isOk()) return; 129 | expect(result.value).toEqual({ 130 | closedLedger: null, 131 | peers: [] 132 | }); 133 | }); 134 | 135 | it('should handle errLoad message', () => { 136 | const stellarMessage = createDummyErrLoadMessage(); 137 | const observation = mock(); 138 | const peerNodes = new PeerNodeCollection(); 139 | peerNodes.getOrAdd(senderPublicKey); 140 | observation.peerNodes = peerNodes; 141 | const result = handler.handleStellarMessage( 142 | senderPublicKey, 143 | stellarMessage, 144 | true, 145 | observation 146 | ); 147 | expect(result.isOk()).toBeTruthy(); 148 | expect( 149 | observation.peerNodes.get(senderPublicKey)?.overLoaded 150 | ).toBeTruthy(); 151 | if (!result.isOk()) return; 152 | expect(result.value).toEqual({ 153 | closedLedger: null, 154 | peers: [] 155 | }); 156 | }); 157 | }); 158 | }); 159 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/__tests__/externalize-statement-handler.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import * as P from 'pino'; 3 | import { PeerNodeCollection } from '../../../../../../../peer-node-collection'; 4 | import { ExternalizeStatementHandler } from '../externalize-statement-handler'; 5 | import { ExternalizeData } from '../map-externalize-statement'; 6 | import { Ledger } from '../../../../../../../crawler'; 7 | import { Slot } from '../slot'; 8 | 9 | const mockLogger = mock(); 10 | 11 | describe('ExternalizeStatementHandler', () => { 12 | let latestConfirmedClosedLedger: Ledger; 13 | 14 | beforeEach(() => { 15 | latestConfirmedClosedLedger = { 16 | sequence: BigInt(0), 17 | closeTime: new Date(), 18 | value: 'test value', 19 | localCloseTime: new Date() 20 | }; 21 | jest.resetAllMocks(); 22 | }); 23 | 24 | it('should not confirm ledger closes for older ledgers even if older slot has not been confirmed yet', () => { 25 | const mockPeerNodes = mock(); 26 | const mockSlot = mock(); 27 | mockSlot.index = BigInt(1); 28 | const slotCloseTime = new Date(); 29 | const localSlotCloseTime = new Date(); 30 | latestConfirmedClosedLedger.sequence = BigInt(2); 31 | const externalizeData: ExternalizeData = { 32 | publicKey: 'A', 33 | value: 'test value', 34 | closeTime: slotCloseTime, 35 | slotIndex: BigInt(1) 36 | }; 37 | 38 | const handler = new ExternalizeStatementHandler(mockLogger); 39 | mockSlot.getConfirmedClosedLedger.mockReturnValueOnce(undefined); 40 | 41 | const result = handler.handle( 42 | mockPeerNodes, 43 | mockSlot, 44 | externalizeData, 45 | localSlotCloseTime, 46 | latestConfirmedClosedLedger 47 | ); 48 | 49 | expect(result).toBe(null); 50 | expect(mockPeerNodes.confirmLedgerCloseForNode).not.toHaveBeenCalled(); 51 | expect(mockPeerNodes.addExternalizedValueForPeerNode).toHaveBeenCalledTimes( 52 | 1 53 | ); 54 | expect(mockSlot.addExternalizeValue).toHaveBeenCalledTimes(0); 55 | }); 56 | 57 | it('should confirm ledger close for peer if slot was already confirmed closed', () => { 58 | const mockPeerNodes = mock(); 59 | const mockSlot = mock(); 60 | mockSlot.index = BigInt(1); 61 | const slotCloseTime = new Date(); 62 | const localSlotCloseTime = new Date(); 63 | const externalizeData: ExternalizeData = { 64 | publicKey: 'A', 65 | value: 'test value', 66 | closeTime: slotCloseTime, 67 | slotIndex: BigInt(1) 68 | }; 69 | 70 | const handler = new ExternalizeStatementHandler(mockLogger); 71 | const closedLedger: Ledger = { 72 | sequence: BigInt(1), 73 | closeTime: slotCloseTime, 74 | value: 'test value', 75 | localCloseTime: localSlotCloseTime 76 | }; 77 | mockSlot.getConfirmedClosedLedger.mockReturnValueOnce(closedLedger); 78 | 79 | const result = handler.handle( 80 | mockPeerNodes, 81 | mockSlot, 82 | externalizeData, 83 | localSlotCloseTime, 84 | latestConfirmedClosedLedger 85 | ); 86 | 87 | expect(result).toBe(null); 88 | expect(mockPeerNodes.confirmLedgerCloseForNode).toHaveBeenCalledWith( 89 | externalizeData.publicKey, 90 | closedLedger 91 | ); 92 | expect(mockPeerNodes.addExternalizedValueForPeerNode).toHaveBeenCalledTimes( 93 | 1 94 | ); 95 | }); 96 | 97 | it('should return null and not update any nodes if slot is not confirmed closed after attempt', () => { 98 | const mockPeerNodes = mock(); 99 | const mockSlot = mock(); 100 | mockSlot.index = BigInt(1); 101 | const slotCloseTime = new Date(); 102 | const localSlotCloseTime = new Date(); 103 | const externalizeData: ExternalizeData = { 104 | publicKey: 'A', 105 | value: 'test value', 106 | closeTime: slotCloseTime, 107 | slotIndex: BigInt(1) 108 | }; 109 | 110 | const handler = new ExternalizeStatementHandler(mockLogger); 111 | mockSlot.getConfirmedClosedLedger.mockReturnValueOnce(undefined); 112 | 113 | const result = handler.handle( 114 | mockPeerNodes, 115 | mockSlot, 116 | externalizeData, 117 | localSlotCloseTime, 118 | latestConfirmedClosedLedger 119 | ); 120 | 121 | expect(result).toBe(null); 122 | expect(mockPeerNodes.confirmLedgerCloseForNode).not.toHaveBeenCalled(); 123 | expect(mockPeerNodes.addExternalizedValueForPeerNode).toHaveBeenCalledTimes( 124 | 1 125 | ); 126 | }); 127 | 128 | it('should return ledger and update nodes if slot is confirmed closed after attempt', () => { 129 | const mockPeerNodes = mock(); 130 | const mockSlot = mock(); 131 | mockSlot.index = BigInt(1); 132 | const slotCloseTime = new Date(); 133 | const localSlotCloseTime = new Date(); 134 | const externalizeData: ExternalizeData = { 135 | publicKey: 'A', 136 | value: 'test value', 137 | closeTime: slotCloseTime, 138 | slotIndex: BigInt(1) 139 | }; 140 | 141 | const handler = new ExternalizeStatementHandler(mockLogger); 142 | const closedLedger: Ledger = { 143 | sequence: BigInt(1), 144 | closeTime: slotCloseTime, 145 | value: 'test value', 146 | localCloseTime: localSlotCloseTime 147 | }; 148 | 149 | mockSlot.getConfirmedClosedLedger.mockReturnValueOnce(undefined); 150 | mockSlot.getConfirmedClosedLedger.mockReturnValueOnce(closedLedger); 151 | 152 | const result = handler.handle( 153 | mockPeerNodes, 154 | mockSlot, 155 | externalizeData, 156 | localSlotCloseTime, 157 | latestConfirmedClosedLedger 158 | ); 159 | 160 | expect(result).toBe(closedLedger); 161 | expect( 162 | mockPeerNodes.confirmLedgerCloseForValidatingNodes 163 | ).toHaveBeenCalledTimes(1); 164 | expect(mockPeerNodes.addExternalizedValueForPeerNode).toHaveBeenCalledTimes( 165 | 1 166 | ); 167 | expect( 168 | mockPeerNodes.confirmLedgerCloseForDisagreeingNodes 169 | ).toHaveBeenCalledTimes(1); 170 | }); 171 | }); 172 | -------------------------------------------------------------------------------- /src/network-observer/__tests__/connection-manager.test.ts: -------------------------------------------------------------------------------- 1 | // connectionManager.test.ts 2 | 3 | import { mock, mockDeep, MockProxy } from 'jest-mock-extended'; 4 | import { 5 | Node as NetworkNode, 6 | Connection 7 | } from '@stellarbeat/js-stellar-node-connector'; 8 | import { ConnectionManager } from '../connection-manager'; 9 | import { P } from 'pino'; 10 | import { EventEmitter } from 'events'; 11 | 12 | describe('ConnectionManager', () => { 13 | let mockNode: MockProxy; 14 | let mockLogger: MockProxy; 15 | let connectionManager: ConnectionManager; 16 | const mockConnection = mockDeep(); 17 | const ip = '127.0.0.1'; 18 | const port = 8001; 19 | const testAddress = ip + ':' + port; 20 | const testPublicKey = 'GABCD1234TEST'; 21 | const nodeInfo = {}; // Simplified for example 22 | 23 | beforeEach(() => { 24 | jest.clearAllMocks(); 25 | mockNode = mock(); 26 | mockLogger = mock(); 27 | connectionManager = new ConnectionManager(mockNode, new Set(), mockLogger); 28 | // Manual implementation to handle event listeners and emitting 29 | const realEmitter = new EventEmitter(); 30 | 31 | // Intercept the `.on` and `.addListener` calls to attach listeners to the real EventEmitter 32 | mockConnection.on.mockImplementation((event, listener) => { 33 | realEmitter.on(event, listener); 34 | return mockConnection; 35 | }); 36 | 37 | mockConnection.addListener.mockImplementation((event, listener) => { 38 | realEmitter.addListener(event, listener); 39 | return mockConnection; 40 | }); 41 | 42 | mockConnection.emit.mockImplementation((event, ...args): boolean => { 43 | return realEmitter.emit(event, ...args); 44 | }); 45 | 46 | // Mock the connectTo method to return a mocked connection object 47 | mockNode.connectTo.mockReturnValue(mockConnection); 48 | }); 49 | 50 | it('should connect to a node and emit "connected" event', () => { 51 | const connectListener = jest.fn(); 52 | connectionManager.on('connected', connectListener); 53 | connectionManager.connectToNode('127.0.0.1', 8001); 54 | mockConnection.emit('connect', testPublicKey, nodeInfo); 55 | 56 | expect(connectListener).toHaveBeenCalledWith({ 57 | publicKey: testPublicKey, 58 | ip: '127.0.0.1', 59 | port: 8001, 60 | nodeInfo 61 | }); 62 | expect(connectionManager.hasActiveConnectionTo(testAddress)).toBeTruthy(); 63 | expect(connectionManager.getActiveConnectionAddresses()).toEqual([ 64 | testAddress 65 | ]); 66 | expect(connectionManager.getNumberOfActiveConnections()).toEqual(1); 67 | expect(connectionManager.getActiveConnection(testAddress)).toEqual( 68 | mockConnection 69 | ); 70 | }); 71 | 72 | it('should close connection with blacklisted node', () => { 73 | connectionManager = new ConnectionManager( 74 | mockNode, 75 | new Set([testPublicKey]), 76 | mockLogger 77 | ); 78 | const connectListener = jest.fn(); 79 | connectionManager.on('connected', connectListener); 80 | connectionManager.connectToNode('127.0.0.1', 8001); 81 | mockConnection.emit('connect', testPublicKey, nodeInfo); 82 | 83 | expect(connectListener).toHaveBeenCalledTimes(0); 84 | expect(connectionManager.getNumberOfActiveConnections()).toEqual(0); 85 | }); 86 | 87 | it('disconnects on connection error', () => { 88 | connectionManager.connectToNode('127.0.0.1', 8001); 89 | const error = new Error('Connection error'); 90 | mockConnection.emit('error', error); 91 | 92 | expect(mockLogger.debug).toHaveBeenCalled(); 93 | expect(mockConnection.destroy).toHaveBeenCalled(); 94 | }); 95 | 96 | it('should shutdown and close all active connections', async () => { 97 | // Simulate a successful connection 98 | connectionManager.connectToNode('127.0.0.1', 8001); 99 | connectionManager.connectToNode('127.0.0.1', 8002); 100 | mockConnection.emit('connect', testPublicKey, nodeInfo); 101 | mockConnection.emit('connect', 'OTHER', nodeInfo); 102 | 103 | connectionManager.shutdown(); 104 | 105 | expect(mockConnection.destroy).toHaveBeenCalledTimes(2); 106 | }); 107 | 108 | it('should disconnect', () => { 109 | connectionManager.connectToNode('127.0.0.1', 8001); 110 | connectionManager.connectToNode('127.0.0.1', 8002); 111 | mockConnection.emit('connect', testPublicKey, nodeInfo); 112 | mockConnection.emit('connect', 'OTHER', nodeInfo); 113 | connectionManager.disconnectByAddress(testAddress); 114 | expect(mockConnection.destroy).toHaveBeenCalledTimes(1); 115 | }); 116 | 117 | it('should not disconnect if connection does not exist', () => { 118 | connectionManager.connectToNode('127.0.0.1', 8001); 119 | mockConnection.emit('connect', testPublicKey, nodeInfo); 120 | connectionManager.disconnectByAddress('127.0.0.1:8002'); 121 | expect(mockConnection.destroy).toHaveBeenCalledTimes(0); 122 | }); 123 | 124 | it('should handle connection close event', () => { 125 | connectionManager.connectToNode('127.0.0.1', 8001); 126 | mockConnection.emit('connect', testPublicKey, nodeInfo); 127 | mockConnection.emit('close'); 128 | expect(connectionManager.getNumberOfActiveConnections()).toEqual(0); 129 | }); 130 | 131 | it('should disconnect on timeout', () => { 132 | connectionManager.connectToNode('127.0.0.1', 8001); 133 | mockConnection.emit('connect', testPublicKey, nodeInfo); 134 | mockConnection.emit('timeout'); 135 | expect(mockConnection.destroy).toHaveBeenCalledTimes(1); 136 | }); 137 | 138 | it('should emit data event on data received', () => { 139 | const dataListener = jest.fn(); 140 | connectionManager.on('data', dataListener); 141 | connectionManager.connectToNode('127.0.0.1', 8001); 142 | mockConnection.emit('connect', testPublicKey, nodeInfo); 143 | mockConnection.emit('data', { content: 'test' }); 144 | 145 | expect(dataListener).toHaveBeenCalledWith({ 146 | address: '127.0.0.1:8001', 147 | publicKey: 'GABCD1234TEST', 148 | stellarMessageWork: { 149 | content: 'test' 150 | } 151 | }); 152 | }); 153 | 154 | it('should not emit data event if remote public key is missing', () => { 155 | const dataListener = jest.fn(); 156 | connectionManager.on('data', dataListener); 157 | connectionManager.connectToNode('127.0.0.1', 8001); 158 | mockConnection.remotePublicKey = undefined; 159 | mockConnection.emit('data', { content: 'test' }); 160 | 161 | expect(dataListener).toHaveBeenCalledTimes(0); 162 | }); 163 | }); 164 | -------------------------------------------------------------------------------- /src/network-observer/peer-event-handler/stellar-message-handlers/stellar-message-handler.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getIpFromPeerAddress, 3 | getQuorumSetFromMessage 4 | } from '@stellarbeat/js-stellar-node-connector'; 5 | import { hash, xdr } from '@stellar/stellar-base'; 6 | import { P } from 'pino'; 7 | import { ScpEnvelopeHandler } from './scp-envelope/scp-envelope-handler'; 8 | import { truncate } from '../../../utilities/truncate'; 9 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 10 | import { QuorumSetManager } from '../../quorum-set-manager'; 11 | import { err, ok, Result } from 'neverthrow'; 12 | import { PeerNodeCollection } from '../../../peer-node-collection'; 13 | import { NodeAddress } from '../../../node-address'; 14 | import { Ledger } from '../../../crawler'; 15 | import { Observation } from '../../observation'; 16 | 17 | type PublicKey = string; 18 | 19 | export class StellarMessageHandler { 20 | constructor( 21 | private scpEnvelopeHandler: ScpEnvelopeHandler, 22 | private quorumSetManager: QuorumSetManager, 23 | private logger: P.Logger 24 | ) {} 25 | 26 | handleStellarMessage( 27 | sender: PublicKey, 28 | stellarMessage: xdr.StellarMessage, 29 | attemptLedgerClose: boolean, 30 | observation: Observation 31 | ): Result< 32 | { 33 | closedLedger: Ledger | null; 34 | peers: Array; 35 | }, 36 | Error 37 | > { 38 | switch (stellarMessage.switch()) { 39 | case xdr.MessageType.scpMessage(): { 40 | if (!attemptLedgerClose) 41 | return ok({ 42 | closedLedger: null, 43 | peers: [] 44 | }); 45 | 46 | const result = this.scpEnvelopeHandler.handle( 47 | stellarMessage.envelope(), 48 | observation 49 | ); 50 | 51 | if (result.isErr()) { 52 | return err(result.error); 53 | } 54 | 55 | return ok({ 56 | closedLedger: result.value.closedLedger, 57 | peers: [] 58 | }); 59 | } 60 | case xdr.MessageType.peers(): { 61 | const result = this.handlePeersMessage( 62 | sender, 63 | stellarMessage.peers(), 64 | observation.peerNodes 65 | ); 66 | 67 | if (result.isErr()) { 68 | return err(result.error); 69 | } 70 | 71 | return ok({ 72 | closedLedger: null, 73 | peers: result.value.peers 74 | }); 75 | } 76 | case xdr.MessageType.scpQuorumset(): { 77 | const result = this.handleScpQuorumSetMessage( 78 | sender, 79 | stellarMessage.qSet(), 80 | observation 81 | ); 82 | 83 | if (result.isErr()) { 84 | return err(result.error); 85 | } 86 | 87 | return ok({ 88 | closedLedger: null, 89 | peers: [] 90 | }); 91 | } 92 | case xdr.MessageType.dontHave(): { 93 | const result = this.handleDontHaveMessage( 94 | sender, 95 | stellarMessage.dontHave(), 96 | observation 97 | ); 98 | 99 | if (result.isErr()) { 100 | return err(result.error); 101 | } 102 | 103 | return ok({ 104 | closedLedger: null, 105 | peers: [] 106 | }); 107 | } 108 | case xdr.MessageType.errorMsg(): { 109 | const result = this.handleErrorMsg( 110 | sender, 111 | stellarMessage.error(), 112 | observation 113 | ); 114 | if (result.isErr()) { 115 | return err(result.error); 116 | } 117 | 118 | return ok({ 119 | closedLedger: null, 120 | peers: [] 121 | }); 122 | } 123 | default: 124 | this.logger.debug( 125 | { type: stellarMessage.switch().name }, 126 | 'Unhandled Stellar message type' 127 | ); 128 | return ok({ 129 | closedLedger: null, 130 | peers: [] 131 | }); 132 | } 133 | } 134 | 135 | private handlePeersMessage( 136 | sender: PublicKey, 137 | peers: xdr.PeerAddress[], 138 | peerNodeCollection: PeerNodeCollection 139 | ): Result< 140 | { 141 | peers: Array; 142 | }, 143 | Error 144 | > { 145 | const peerAddresses: Array = []; 146 | peers.forEach((peer) => { 147 | const ipResult = getIpFromPeerAddress(peer); 148 | if (ipResult.isOk()) peerAddresses.push([ipResult.value, peer.port()]); 149 | }); 150 | 151 | peerNodeCollection.setPeerSuppliedPeerList(sender, true); 152 | 153 | this.logger.debug( 154 | { peer: sender }, 155 | peerAddresses.length + ' peers received' 156 | ); 157 | 158 | return ok({ 159 | peers: peerAddresses 160 | }); 161 | } 162 | 163 | private handleScpQuorumSetMessage( 164 | sender: PublicKey, 165 | quorumSetMessage: xdr.ScpQuorumSet, 166 | observation: Observation 167 | ): Result { 168 | const quorumSetHash = hash(quorumSetMessage.toXDR()).toString('base64'); 169 | const quorumSetResult = getQuorumSetFromMessage(quorumSetMessage); 170 | if (quorumSetResult.isErr()) { 171 | return err(quorumSetResult.error); 172 | } 173 | this.logger.info( 174 | { 175 | pk: truncate(sender), 176 | hash: quorumSetHash 177 | }, 178 | 'QuorumSet received' 179 | ); 180 | this.quorumSetManager.processQuorumSet( 181 | quorumSetHash, 182 | QuorumSet.fromBaseQuorumSet(quorumSetResult.value), 183 | sender, 184 | observation 185 | ); 186 | 187 | return ok(undefined); 188 | } 189 | 190 | private handleDontHaveMessage( 191 | sender: PublicKey, 192 | dontHave: xdr.DontHave, 193 | observation: Observation 194 | ): Result { 195 | this.logger.info( 196 | { 197 | pk: truncate(sender), 198 | type: dontHave.type().name 199 | }, 200 | "Don't have" 201 | ); 202 | if (dontHave.type().value === xdr.MessageType.getScpQuorumset().value) { 203 | this.logger.info( 204 | { 205 | pk: truncate(sender), 206 | hash: dontHave.reqHash().toString('base64') 207 | }, 208 | "Don't have" 209 | ); 210 | this.quorumSetManager.peerNodeDoesNotHaveQuorumSet( 211 | sender, 212 | dontHave.reqHash().toString('base64'), 213 | observation 214 | ); 215 | } 216 | 217 | return ok(undefined); 218 | } 219 | 220 | private handleErrorMsg( 221 | sender: PublicKey, 222 | error: xdr.Error, 223 | observation: Observation 224 | ): Result { 225 | switch (error.code()) { 226 | case xdr.ErrorCode.errLoad(): 227 | return this.onLoadTooHighReceived(sender, observation); 228 | default: 229 | this.logger.info( 230 | { 231 | pk: truncate(sender), 232 | error: error.code().name 233 | }, 234 | error.msg().toString() 235 | ); 236 | return ok(undefined); 237 | } 238 | } 239 | 240 | private onLoadTooHighReceived( 241 | sender: PublicKey, 242 | observation: Observation 243 | ): Result { 244 | this.logger.debug({ peer: sender }, 'Load too high message received'); 245 | observation.peerNodes.setPeerOverloaded(sender, true); 246 | 247 | return ok(undefined); 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /src/network-observer/__tests__/network-observer.test.ts: -------------------------------------------------------------------------------- 1 | import { NetworkObserver } from '../network-observer'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { P } from 'pino'; 4 | import { 5 | ClosePayload, 6 | ConnectionManager, 7 | DataPayload 8 | } from '../connection-manager'; 9 | import { QuorumSetManager } from '../quorum-set-manager'; 10 | import { PeerEventHandler } from '../peer-event-handler/peer-event-handler'; 11 | import { ObservationManager } from '../observation-manager'; 12 | import { NodeAddress } from '../../node-address'; 13 | import { ObservationFactory } from '../observation-factory'; 14 | import { Observation } from '../observation'; 15 | import { ObservationState } from '../observation-state'; 16 | import { EventEmitter } from 'events'; 17 | import { Ledger } from '../../crawler'; 18 | import { nextTick } from 'async'; 19 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 20 | import { PeerNodeCollection } from '../../peer-node-collection'; 21 | import { Slots } from '../peer-event-handler/stellar-message-handlers/scp-envelope/scp-statement/externalize/slots'; 22 | 23 | describe('network-observer', () => { 24 | const observationFactory = mock(); 25 | const connectionManager = mock(); 26 | const connectionManagerEmitter = new EventEmitter(); 27 | connectionManager.on.mockImplementation((event, listener) => { 28 | connectionManagerEmitter.on(event, listener); 29 | return connectionManager; 30 | }); 31 | const quorumSetManager = mock(); 32 | const peerEventHandler = mock(); 33 | const observationManager = mock(); 34 | 35 | const networkObserver = new NetworkObserver( 36 | observationFactory, 37 | connectionManager, 38 | quorumSetManager, 39 | peerEventHandler, 40 | observationManager 41 | ); 42 | 43 | const createObservation = (topTierAddresses: NodeAddress[] = []) => { 44 | return new Observation( 45 | 'test', 46 | topTierAddresses, 47 | mock(), 48 | mock(), 49 | new Map(), 50 | new Slots(new QuorumSet(1, ['A'], []), mock()) 51 | ); 52 | }; 53 | 54 | beforeEach(() => { 55 | jest.clearAllMocks(); 56 | }); 57 | 58 | it('should observe', async () => { 59 | connectionManager.getNumberOfActiveConnections.mockReturnValue(1); 60 | const observation = createObservation(); 61 | observationFactory.createObservation.mockReturnValue(observation); 62 | const result = await networkObserver.startObservation(observation); 63 | expect(result).toBe(1); 64 | expect(observationManager.startSync).toHaveBeenCalledWith(observation); 65 | }); 66 | 67 | it('should connect to node', async () => { 68 | const ip = 'localhost'; 69 | const port = 11625; 70 | const observation = createObservation([['localhost', 11625]]); 71 | observation.state = ObservationState.Synced; 72 | observationFactory.createObservation.mockReturnValue(observation); 73 | await networkObserver.startObservation(observation); 74 | networkObserver.connectToNode(ip, port); 75 | expect(connectionManager.connectToNode).toHaveBeenCalledWith(ip, port); 76 | }); 77 | 78 | it('should stop', async () => { 79 | const observation = createObservation([['localhost', 11625]]); 80 | observation.state = ObservationState.Synced; 81 | observationFactory.createObservation.mockReturnValue(observation); 82 | await networkObserver.startObservation(observation); 83 | observationManager.stopObservation.mockImplementation((observation, cb) => { 84 | cb(); 85 | }); 86 | const result = await networkObserver.stop(); 87 | expect(result).toBe(observation); 88 | }); 89 | 90 | it('should handle peer data', async () => { 91 | const data = mock(); 92 | peerEventHandler.onData.mockReturnValue({ 93 | closedLedger: null, 94 | peers: [] 95 | }); 96 | const observation = createObservation(); 97 | observation.state = ObservationState.Synced; 98 | observationFactory.createObservation.mockReturnValue(observation); 99 | await networkObserver.startObservation(observation); 100 | connectionManagerEmitter.emit('data', data); 101 | expect(peerEventHandler.onData).toHaveBeenCalledWith(data, observation); 102 | }); 103 | 104 | it('should handle closed ledger through peer data event', async () => { 105 | const data = mock(); 106 | peerEventHandler.onData.mockReturnValue({ 107 | closedLedger: mock(), 108 | peers: [] 109 | }); 110 | const observation = createObservation(); 111 | observation.state = ObservationState.Synced; 112 | observationFactory.createObservation.mockReturnValue(observation); 113 | await networkObserver.startObservation(observation); 114 | connectionManagerEmitter.emit('data', data); 115 | expect(peerEventHandler.onData).toHaveBeenCalledWith(data, observation); 116 | expect(observationManager.ledgerCloseConfirmed).toHaveBeenCalledWith( 117 | observation, 118 | peerEventHandler.onData(data, observation).closedLedger 119 | ); 120 | }); 121 | 122 | it('should emit peers event through peer data event', async () => { 123 | const data = mock(); 124 | peerEventHandler.onData.mockReturnValue({ 125 | closedLedger: null, 126 | peers: [['localhost', 11625]] 127 | }); 128 | const observation = createObservation(); 129 | networkObserver.on('peers', (peers) => { 130 | expect(peers).toEqual([['localhost', 11625]]); 131 | }); 132 | observation.state = ObservationState.Synced; 133 | observationFactory.createObservation.mockReturnValue(observation); 134 | await networkObserver.startObservation(observation); 135 | connectionManagerEmitter.emit('data', data); 136 | expect(peerEventHandler.onData).toHaveBeenCalledWith(data, observation); 137 | expect(observationManager.ledgerCloseConfirmed).not.toHaveBeenCalled(); 138 | nextTick(() => {}); //to make sure event is checked 139 | }); 140 | 141 | it('should handle connected event', async () => { 142 | const data = mock(); 143 | const observation = createObservation(); 144 | observation.state = ObservationState.Synced; 145 | observationFactory.createObservation.mockReturnValue(observation); 146 | await networkObserver.startObservation(observation); 147 | connectionManagerEmitter.emit('connected', data); 148 | expect(peerEventHandler.onConnected).toHaveBeenCalledWith( 149 | data, 150 | observation 151 | ); 152 | }); 153 | 154 | it('should handle close event', async () => { 155 | const data = mock(); 156 | const observation = createObservation(); 157 | observation.state = ObservationState.Synced; 158 | observationFactory.createObservation.mockReturnValue(observation); 159 | networkObserver.on('disconnect', (close: ClosePayload) => { 160 | expect(close).toEqual(data); 161 | }); 162 | await networkObserver.startObservation(observation); 163 | connectionManagerEmitter.emit('close', data); 164 | expect(peerEventHandler.onConnectionClose).toHaveBeenCalledWith( 165 | data, 166 | observation 167 | ); 168 | nextTick(() => {}); //to make sure close event is checked 169 | }); 170 | }); 171 | -------------------------------------------------------------------------------- /src/crawler.ts: -------------------------------------------------------------------------------- 1 | import * as P from 'pino'; 2 | import { CrawlProcessState, Crawl } from './crawl'; 3 | import { CrawlResult } from './crawl-result'; 4 | import { CrawlerConfiguration } from './crawler-configuration'; 5 | import { CrawlLogger } from './crawl-logger'; 6 | import { CrawlQueueManager } from './crawl-queue-manager'; 7 | import { NodeAddress, nodeAddressToPeerKey } from './node-address'; 8 | import { CrawlTask } from './crawl-task'; 9 | import { MaxCrawlTimeManager } from './max-crawl-time-manager'; 10 | import { ClosePayload } from './network-observer/connection-manager'; 11 | import { NetworkObserver } from './network-observer/network-observer'; 12 | 13 | export interface Ledger { 14 | sequence: bigint; 15 | closeTime: Date; 16 | localCloseTime: Date; 17 | value: string; 18 | } 19 | 20 | /** 21 | * The crawler is the orchestrator of the crawling process. 22 | * It connects to nodes, delegates the handling of incoming messages to the StellarMessageHandler, 23 | * and manages the crawl state. 24 | */ 25 | export class Crawler { 26 | private _crawl: Crawl | null = null; 27 | 28 | constructor( 29 | private config: CrawlerConfiguration, 30 | private crawlQueueManager: CrawlQueueManager, 31 | private maxCrawlTimeManager: MaxCrawlTimeManager, 32 | private networkObserver: NetworkObserver, 33 | private crawlLogger: CrawlLogger, 34 | public readonly logger: P.Logger 35 | ) { 36 | this.logger = logger.child({ mod: 'Crawler' }); 37 | this.setupPeerListenerEvents(); 38 | } 39 | 40 | async startCrawl(crawl: Crawl): Promise { 41 | return new Promise((resolve, reject) => { 42 | if (this.isCrawlRunning()) { 43 | return reject(new Error('Crawl process already running')); 44 | } 45 | this.crawl = crawl; 46 | 47 | this.syncTopTierAndCrawl(resolve, reject); 48 | }); 49 | } 50 | 51 | private isCrawlRunning() { 52 | return this._crawl && this.crawl.state !== CrawlProcessState.IDLE; 53 | } 54 | 55 | private setupPeerListenerEvents() { 56 | this.networkObserver.on('peers', (peers: NodeAddress[]) => { 57 | this.onPeerAddressesReceived(peers); 58 | }); 59 | this.networkObserver.on('disconnect', (data: ClosePayload) => { 60 | this.crawlQueueManager.completeCrawlQueueTask( 61 | this.crawl.crawlQueueTaskDoneCallbacks, 62 | data.address 63 | ); 64 | 65 | if (!data.publicKey) { 66 | this.crawl.failedConnections.push(data.address); 67 | } 68 | }); 69 | } 70 | 71 | private onPeerAddressesReceived(peerAddresses: NodeAddress[]) { 72 | if (this.crawl.state === CrawlProcessState.TOP_TIER_SYNC) { 73 | this.crawl.peerAddressesReceivedDuringSync = 74 | this.crawl.peerAddressesReceivedDuringSync.concat(peerAddresses); 75 | } else { 76 | peerAddresses.forEach((peerAddress) => this.crawlPeerNode(peerAddress)); 77 | } 78 | } 79 | 80 | private async syncTopTierAndCrawl( 81 | resolve: (value: PromiseLike | CrawlResult) => void, 82 | reject: (reason?: any) => void 83 | ) { 84 | const nrOfActiveTopTierConnections = await this.startTopTierSync(); 85 | this.startCrawlProcess(resolve, reject, nrOfActiveTopTierConnections); 86 | } 87 | 88 | private startCrawlProcess( 89 | resolve: (value: PromiseLike | CrawlResult) => void, 90 | reject: (reason?: any) => void, 91 | nrOfActiveTopTierConnections: number 92 | ) { 93 | const nodesToCrawl = this.crawl.nodesToCrawl.concat( 94 | this.crawl.peerAddressesReceivedDuringSync 95 | ); 96 | 97 | if ( 98 | this.crawl.nodesToCrawl.length === 0 && 99 | nrOfActiveTopTierConnections === 0 100 | ) { 101 | this.logger.warn( 102 | 'No nodes to crawl and top tier connections closed, crawl failed' 103 | ); 104 | reject(new Error('No nodes to crawl and top tier connections failed')); 105 | return; 106 | } 107 | 108 | this.logger.info('Starting crawl process'); 109 | this.crawlLogger.start(this.crawl); 110 | this.crawl.state = CrawlProcessState.CRAWLING; 111 | this.setupCrawlCompletionHandlers(resolve, reject); 112 | 113 | if (nodesToCrawl.length === 0) { 114 | this.logger.warn('No nodes to crawl'); 115 | this.networkObserver.stop().then(() => { 116 | this.finish(resolve, reject); 117 | this.crawl.state = CrawlProcessState.STOPPING; 118 | }); 119 | } else nodesToCrawl.forEach((address) => this.crawlPeerNode(address)); 120 | } 121 | 122 | private async startTopTierSync() { 123 | this.logger.info('Starting Top Tier sync'); 124 | this.crawl.state = CrawlProcessState.TOP_TIER_SYNC; 125 | return this.networkObserver.startObservation(this.crawl.observation); 126 | } 127 | 128 | private setupCrawlCompletionHandlers( 129 | resolve: (value: PromiseLike | CrawlResult) => void, 130 | reject: (reason?: any) => void 131 | ) { 132 | this.startMaxCrawlTimeout(resolve, reject); 133 | this.crawlQueueManager.onDrain(() => { 134 | this.logger.info('Stopping crawl process'); 135 | this.crawl.state = CrawlProcessState.STOPPING; 136 | this.networkObserver.stop().then(() => { 137 | this.finish(resolve, reject); 138 | }); 139 | }); 140 | } 141 | 142 | private startMaxCrawlTimeout( 143 | resolve: (value: CrawlResult | PromiseLike) => void, 144 | reject: (error: Error) => void 145 | ) { 146 | this.maxCrawlTimeManager.setTimer(this.config.maxCrawlTime, () => { 147 | this.logger.fatal('Max crawl time hit, closing all connections'); 148 | this.networkObserver.stop().then(() => this.finish(resolve, reject)); 149 | this.crawl.maxCrawlTimeHit = true; 150 | }); 151 | } 152 | 153 | private finish( 154 | resolve: (value: CrawlResult | PromiseLike) => void, 155 | reject: (error: Error) => void 156 | ): void { 157 | this.crawlLogger.stop(); 158 | this.maxCrawlTimeManager.clearTimer(); 159 | this.crawl.state = CrawlProcessState.IDLE; 160 | 161 | if (this.hasCrawlTimedOut()) { 162 | //todo clean crawl-queue and connections 163 | reject(new Error('Max crawl time hit, shutting down crawler')); 164 | return; 165 | } 166 | 167 | resolve(this.constructCrawlResult()); 168 | } 169 | 170 | private hasCrawlTimedOut(): boolean { 171 | return this.crawl.maxCrawlTimeHit; 172 | } 173 | 174 | private constructCrawlResult(): CrawlResult { 175 | return { 176 | peers: this.crawl.observation.peerNodes.getAll(), 177 | closedLedgers: 178 | this.crawl.observation.slots.getConfirmedClosedSlotIndexes(), 179 | latestClosedLedger: this.crawl.observation.latestConfirmedClosedLedger 180 | }; 181 | } 182 | 183 | private crawlPeerNode(nodeAddress: NodeAddress): void { 184 | const peerKey = nodeAddressToPeerKey(nodeAddress); 185 | 186 | if (!this.canNodeBeCrawled(peerKey)) return; 187 | 188 | this.logNodeAddition(peerKey); 189 | this.crawl.crawledNodeAddresses.add(peerKey); 190 | const crawlTask: CrawlTask = { 191 | nodeAddress: nodeAddress, 192 | crawl: this.crawl, 193 | connectCallback: () => 194 | this.networkObserver.connectToNode(nodeAddress[0], nodeAddress[1]) 195 | }; 196 | 197 | this.crawlQueueManager.addCrawlTask(crawlTask); 198 | } 199 | 200 | private logNodeAddition(peerKey: string): void { 201 | this.logger.debug({ peer: peerKey }, 'Adding address to crawl queue'); 202 | } 203 | 204 | private canNodeBeCrawled(peerKey: string): boolean { 205 | return ( 206 | !this.crawl.crawledNodeAddresses.has(peerKey) && 207 | !this.crawl.observation.topTierAddressesSet.has(peerKey) 208 | ); 209 | } 210 | 211 | private get crawl(): Crawl { 212 | if (!this._crawl) throw new Error('crawl not set'); 213 | return this._crawl; 214 | } 215 | 216 | private set crawl(crawl: Crawl) { 217 | this._crawl = crawl; 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/__tests__/crawler.test.ts: -------------------------------------------------------------------------------- 1 | import { Crawler } from '../index'; 2 | import { createDummyCrawlerConfiguration } from '../__fixtures__/createDummyCrawlerConfiguration'; 3 | import { CrawlQueueManager } from '../crawl-queue-manager'; 4 | import { MaxCrawlTimeManager } from '../max-crawl-time-manager'; 5 | import { P } from 'pino'; 6 | import { mock, MockProxy } from 'jest-mock-extended'; 7 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 8 | import { CrawlLogger } from '../crawl-logger'; 9 | import { CrawlProcessState } from '../crawl'; 10 | import { EventEmitter } from 'events'; 11 | import { AsyncCrawlQueue } from '../crawl-queue'; 12 | import { NetworkObserver } from '../network-observer/network-observer'; 13 | import { ClosePayload } from '../network-observer/connection-manager'; 14 | import { ObservationFactory } from '../network-observer/observation-factory'; 15 | import { CrawlFactory } from '../crawl-factory'; 16 | import { Observation } from '../network-observer/observation'; 17 | 18 | describe('Crawler', () => { 19 | const crawlFactory = new CrawlFactory( 20 | new ObservationFactory(), 21 | 'test', 22 | mock() 23 | ); 24 | beforeEach(() => { 25 | jest.clearAllMocks(); 26 | }); 27 | 28 | function setupSUT() { 29 | const crawlQueueManager = new CrawlQueueManager( 30 | new AsyncCrawlQueue(1), 31 | mock() 32 | ); 33 | const maxCrawlTimeManager = mock(); 34 | const networkObserver = mock(); 35 | const crawlLogger = mock(); 36 | const logger = mock(); 37 | logger.child.mockReturnValue(logger as any); 38 | const networkObserverEventEmitter = new EventEmitter(); 39 | 40 | networkObserver.on.mockImplementation((event, listener) => { 41 | networkObserverEventEmitter.on(event, listener); 42 | return networkObserver; 43 | }); 44 | 45 | const crawler = new Crawler( 46 | createDummyCrawlerConfiguration(), 47 | crawlQueueManager, 48 | maxCrawlTimeManager, 49 | networkObserver, 50 | crawlLogger, 51 | logger 52 | ); 53 | const crawl = crawlFactory.createCrawl( 54 | [['peer', 2]], 55 | [['top', 1]], 56 | new QuorumSet(2, []), 57 | { 58 | closeTime: new Date(0), 59 | localCloseTime: new Date(0), 60 | sequence: BigInt(0), 61 | value: '' 62 | }, 63 | new Map() 64 | ); 65 | 66 | return { 67 | crawler, 68 | crawl, 69 | networkObserver, 70 | networkObserverEventEmitter, 71 | crawlLogger, 72 | maxCrawlTimeManager 73 | }; 74 | } 75 | 76 | it('should create a Crawler', () => { 77 | const crawler = setupSUT().crawler; 78 | expect(crawler).toBeInstanceOf(Crawler); 79 | }); 80 | 81 | it('should return error if no active top tier connections and no node addresses to crawl', async () => { 82 | const { 83 | crawler, 84 | crawl: crawl, 85 | networkObserver, 86 | crawlLogger, 87 | maxCrawlTimeManager 88 | } = setupSUT(); 89 | networkObserver.startObservation.mockResolvedValue(0); 90 | crawl.observation.topTierAddresses = []; 91 | crawl.nodesToCrawl = []; 92 | try { 93 | await crawler.startCrawl(crawl); 94 | } catch (e) { 95 | expect(e).toBeInstanceOf(Error); 96 | expect(crawlLogger.start).not.toHaveBeenCalled(); 97 | expect(crawlLogger.stop).not.toHaveBeenCalled(); 98 | expect(maxCrawlTimeManager.setTimer).not.toHaveBeenCalled(); 99 | expect(maxCrawlTimeManager.clearTimer).not.toHaveBeenCalled(); 100 | } 101 | }); 102 | 103 | function expectCorrectMaxTimer( 104 | maxCrawlTimeManager: MockProxy 105 | ) { 106 | expect(maxCrawlTimeManager.setTimer).toHaveBeenCalled(); 107 | expect(maxCrawlTimeManager.clearTimer).toHaveBeenCalled(); 108 | } 109 | 110 | function expectCorrectLogger(crawlLogger: MockProxy) { 111 | expect(crawlLogger.start).toHaveBeenCalled(); 112 | expect(crawlLogger.stop).toHaveBeenCalled(); 113 | } 114 | 115 | it('should connect to top tier and not crawl if there are no nodes to be crawled', async () => { 116 | const { 117 | crawler, 118 | crawl: crawl, 119 | networkObserver, 120 | crawlLogger, 121 | maxCrawlTimeManager 122 | } = setupSUT(); 123 | networkObserver.startObservation.mockResolvedValue(1); 124 | networkObserver.stop.mockResolvedValue(mock()); 125 | crawl.observation.topTierAddresses = []; 126 | crawl.nodesToCrawl = []; 127 | const result = await crawler.startCrawl(crawl); 128 | expect(result).toEqual({ 129 | closedLedgers: [], 130 | latestClosedLedger: { 131 | closeTime: new Date(0), 132 | localCloseTime: new Date(0), 133 | sequence: BigInt(0), 134 | value: '' 135 | }, 136 | peers: new Map() 137 | }); 138 | expectCorrectMaxTimer(maxCrawlTimeManager); 139 | expectCorrectLogger(crawlLogger); 140 | expect(networkObserver.stop).toHaveBeenCalled(); 141 | }); 142 | 143 | it('should connect to top tier and crawl peer nodes received from top tier', (resolve) => { 144 | const { 145 | crawler, 146 | crawl: crawl, 147 | networkObserver, 148 | networkObserverEventEmitter, 149 | crawlLogger, 150 | maxCrawlTimeManager 151 | } = setupSUT(); 152 | networkObserver.startObservation.mockImplementationOnce(() => { 153 | return new Promise((resolve) => { 154 | networkObserverEventEmitter.emit('peers', [['127.0.0.1', 11625]]); 155 | setTimeout(() => { 156 | resolve(1); 157 | }, 1); 158 | }); 159 | }); 160 | 161 | networkObserver.stop.mockResolvedValue(mock()); 162 | 163 | networkObserver.connectToNode.mockImplementation((address, port) => { 164 | return new Promise((resolve) => { 165 | const disconnectPayload: ClosePayload = { 166 | address: address + ':' + port, 167 | publicKey: 'A' 168 | }; 169 | networkObserverEventEmitter.emit('disconnect', disconnectPayload); 170 | setTimeout(() => { 171 | resolve(undefined); 172 | }, 1); 173 | }); 174 | }); 175 | 176 | crawler 177 | .startCrawl(crawl) 178 | .then((result) => { 179 | expect(result).toEqual({ 180 | closedLedgers: [], 181 | latestClosedLedger: { 182 | closeTime: new Date(0), 183 | localCloseTime: new Date(0), 184 | sequence: BigInt(0), 185 | value: '' 186 | }, 187 | peers: new Map() 188 | }); 189 | expectCorrectMaxTimer(maxCrawlTimeManager); 190 | expectCorrectLogger(crawlLogger); 191 | expect(networkObserver.startObservation).toHaveBeenNthCalledWith( 192 | 1, 193 | crawl.observation 194 | ); 195 | expect(networkObserver.connectToNode).toHaveBeenCalledTimes(2); 196 | expect(crawl.state).toBe(CrawlProcessState.IDLE); 197 | resolve(); 198 | }) 199 | .catch((e) => { 200 | throw e; 201 | }); 202 | }); 203 | 204 | it('should crawl nodes received from peers', (resolve) => { 205 | const { 206 | crawler, 207 | crawl, 208 | networkObserver, 209 | crawlLogger, 210 | maxCrawlTimeManager, 211 | networkObserverEventEmitter 212 | } = setupSUT(); 213 | networkObserver.startObservation.mockResolvedValue(1); 214 | networkObserver.stop.mockResolvedValue(mock()); 215 | networkObserver.connectToNode.mockImplementation((address, port) => { 216 | return new Promise((resolve) => { 217 | const disconnectPayload: ClosePayload = { 218 | address: address + ':' + port, 219 | publicKey: 'A' 220 | }; 221 | networkObserverEventEmitter.emit('peers', [['otherPeer', 2]]); 222 | networkObserverEventEmitter.emit('disconnect', disconnectPayload); 223 | setTimeout(() => { 224 | resolve(undefined); 225 | }, 1); 226 | }); 227 | }); 228 | crawler 229 | .startCrawl(crawl) 230 | .then((result) => { 231 | expect(result).toEqual({ 232 | closedLedgers: [], 233 | latestClosedLedger: { 234 | closeTime: new Date(0), 235 | localCloseTime: new Date(0), 236 | sequence: BigInt(0), 237 | value: '' 238 | }, 239 | peers: new Map() 240 | }); 241 | expectCorrectMaxTimer(maxCrawlTimeManager); 242 | expectCorrectLogger(crawlLogger); 243 | expect(networkObserver.connectToNode).toHaveBeenCalledTimes(2); 244 | resolve(); 245 | }) 246 | .catch((e) => { 247 | throw e; 248 | }); 249 | }); 250 | }); 251 | -------------------------------------------------------------------------------- /src/network-observer/quorum-set-manager.ts: -------------------------------------------------------------------------------- 1 | import { PublicKey, QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 2 | import * as P from 'pino'; 3 | import { xdr } from '@stellar/stellar-base'; 4 | import { PeerNode } from '../peer-node'; 5 | import { err, ok, Result } from 'neverthrow'; 6 | import { truncate } from '../utilities/truncate'; 7 | import { ConnectionManager } from './connection-manager'; 8 | import { Observation } from './observation'; 9 | 10 | type QuorumSetHash = string; 11 | 12 | /** 13 | * Fetches quorumSets in a sequential way from connected nodes. 14 | * Makes sure every peerNode that sent a scp message with a hash, gets the correct quorumSet. 15 | */ 16 | export class QuorumSetManager { 17 | constructor( 18 | private connectionManager: ConnectionManager, 19 | private quorumRequestTimeoutMS: number, 20 | private logger: P.Logger 21 | ) {} 22 | 23 | public onNodeDisconnected( 24 | publicKey: PublicKey, 25 | observation: Observation 26 | ): void { 27 | if (!observation.quorumSetState.quorumSetRequests.has(publicKey)) return; 28 | 29 | this.clearQuorumSetRequest(publicKey, observation); 30 | } 31 | 32 | public processQuorumSetHashFromStatement( 33 | peer: PeerNode, 34 | scpStatement: xdr.ScpStatement, 35 | observation: Observation 36 | ): void { 37 | const quorumSetHashResult = this.getQuorumSetHash(scpStatement); 38 | if (quorumSetHashResult.isErr()) return; 39 | 40 | peer.quorumSetHash = quorumSetHashResult.value; 41 | if ( 42 | !this.getQuorumSetHashOwners(peer.quorumSetHash, observation).has( 43 | peer.publicKey 44 | ) 45 | ) { 46 | this.logger.debug( 47 | { pk: peer.publicKey, hash: peer.quorumSetHash }, 48 | 'Detected quorumSetHash' 49 | ); 50 | } 51 | 52 | this.getQuorumSetHashOwners(peer.quorumSetHash, observation).add( 53 | peer.publicKey 54 | ); 55 | 56 | if (observation.quorumSets.has(peer.quorumSetHash)) 57 | peer.quorumSet = observation.quorumSets.get(peer.quorumSetHash); 58 | else { 59 | this.logger.debug( 60 | { pk: peer.publicKey }, 61 | 'Unknown quorumSet for hash: ' + peer.quorumSetHash 62 | ); 63 | this.requestQuorumSet(peer.quorumSetHash, observation); 64 | } 65 | } 66 | 67 | public processQuorumSet( 68 | quorumSetHash: QuorumSetHash, 69 | quorumSet: QuorumSet, 70 | sender: PublicKey, 71 | observation: Observation 72 | ): void { 73 | observation.quorumSets.set(quorumSetHash, quorumSet); 74 | const owners = this.getQuorumSetHashOwners(quorumSetHash, observation); 75 | 76 | owners.forEach((owner) => { 77 | const peer = observation.peerNodes.get(owner); 78 | if (peer) peer.quorumSet = quorumSet; 79 | }); 80 | 81 | this.clearQuorumSetRequest(sender, observation); 82 | } 83 | 84 | public peerNodeDoesNotHaveQuorumSet( 85 | peerPublicKey: PublicKey, 86 | quorumSetHash: QuorumSetHash, 87 | observation: Observation 88 | ): void { 89 | const request = 90 | observation.quorumSetState.quorumSetRequests.get(peerPublicKey); 91 | if (!request) return; 92 | if (request.hash !== quorumSetHash) return; 93 | 94 | this.clearQuorumSetRequest(peerPublicKey, observation); 95 | this.requestQuorumSet(quorumSetHash, observation); 96 | } 97 | 98 | protected requestQuorumSet( 99 | quorumSetHash: QuorumSetHash, 100 | observation: Observation 101 | ): void { 102 | if (observation.quorumSets.has(quorumSetHash)) return; 103 | 104 | if ( 105 | observation.quorumSetState.quorumSetHashesInProgress.has(quorumSetHash) 106 | ) { 107 | this.logger.debug({ hash: quorumSetHash }, 'Request already in progress'); 108 | return; 109 | } 110 | 111 | this.logger.debug({ hash: quorumSetHash }, 'Requesting quorumSet'); 112 | const alreadyRequestedToResult = 113 | observation.quorumSetState.quorumSetRequestedTo.get(quorumSetHash); 114 | const alreadyRequestedTo: Set = alreadyRequestedToResult 115 | ? alreadyRequestedToResult 116 | : new Set(); 117 | observation.quorumSetState.quorumSetRequestedTo.set( 118 | quorumSetHash, 119 | alreadyRequestedTo 120 | ); 121 | 122 | const owners = this.getQuorumSetHashOwners(quorumSetHash, observation); 123 | const quorumSetMessage = xdr.StellarMessage.getScpQuorumset( 124 | Buffer.from(quorumSetHash, 'base64') 125 | ); 126 | 127 | const sendRequest = (to: string) => { 128 | const connection = this.connectionManager.getActiveConnection(to); //todo: need more separation 129 | if (!connection) { 130 | this.logger.warn( 131 | { hash: quorumSetHash, address: to }, 132 | 'No active connection to request quorumSet from' 133 | ); 134 | return; 135 | } 136 | alreadyRequestedTo.add(to); 137 | this.logger.info( 138 | { hash: quorumSetHash }, 139 | 'Requesting quorumSet from ' + to 140 | ); 141 | 142 | connection.sendStellarMessage(quorumSetMessage); 143 | observation.quorumSetState.quorumSetHashesInProgress.add(quorumSetHash); 144 | observation.quorumSetState.quorumSetRequests.set(to, { 145 | hash: quorumSetHash, 146 | timeout: setTimeout(() => { 147 | this.logger.info( 148 | { pk: truncate(to), hash: quorumSetHash }, 149 | 'Request timeout reached' 150 | ); 151 | observation.quorumSetState.quorumSetRequests.delete(to); 152 | observation.quorumSetState.quorumSetHashesInProgress.delete( 153 | quorumSetHash 154 | ); 155 | 156 | this.requestQuorumSet(quorumSetHash, observation); 157 | }, this.quorumRequestTimeoutMS) 158 | }); 159 | }; 160 | 161 | //first try the owners of the hashes 162 | const notYetRequestedOwnerWithActiveConnection = ( 163 | Array.from(owners.keys()) 164 | .map((owner) => observation.peerNodes.get(owner)) 165 | .filter((owner) => owner !== undefined) as PeerNode[] 166 | ) 167 | .filter((owner) => !alreadyRequestedTo.has(owner.key)) 168 | .find((owner) => this.connectionManager.hasActiveConnectionTo(owner.key)); 169 | if (notYetRequestedOwnerWithActiveConnection) { 170 | sendRequest(notYetRequestedOwnerWithActiveConnection.key); 171 | return; 172 | } 173 | 174 | //try other open connections 175 | const notYetRequestedNonOwnerActiveConnection = this.connectionManager 176 | .getActiveConnectionAddresses() 177 | .find((address) => !alreadyRequestedTo.has(address)); 178 | 179 | if (notYetRequestedNonOwnerActiveConnection) { 180 | sendRequest(notYetRequestedNonOwnerActiveConnection); 181 | return; 182 | } 183 | 184 | this.logger.warn( 185 | { hash: quorumSetHash }, 186 | 'No active connections to request quorumSet from' 187 | ); 188 | } 189 | 190 | protected getQuorumSetHashOwners( 191 | quorumSetHash: QuorumSetHash, 192 | observation: Observation 193 | ): Set { 194 | let quorumSetHashOwners = 195 | observation.quorumSetState.quorumSetOwners.get(quorumSetHash); 196 | if (!quorumSetHashOwners) { 197 | quorumSetHashOwners = new Set(); 198 | observation.quorumSetState.quorumSetOwners.set( 199 | quorumSetHash, 200 | quorumSetHashOwners 201 | ); 202 | } 203 | 204 | return quorumSetHashOwners; 205 | } 206 | 207 | protected getQuorumSetHash( 208 | scpStatement: xdr.ScpStatement 209 | ): Result { 210 | try { 211 | let quorumSetHash: QuorumSetHash | undefined; 212 | switch (scpStatement.pledges().switch()) { 213 | case xdr.ScpStatementType.scpStExternalize(): 214 | quorumSetHash = scpStatement 215 | .pledges() 216 | .externalize() 217 | .commitQuorumSetHash() 218 | .toString('base64'); 219 | break; 220 | case xdr.ScpStatementType.scpStConfirm(): 221 | quorumSetHash = scpStatement 222 | .pledges() 223 | .confirm() 224 | .quorumSetHash() 225 | .toString('base64'); 226 | break; 227 | case xdr.ScpStatementType.scpStPrepare(): 228 | quorumSetHash = scpStatement 229 | .pledges() 230 | .prepare() 231 | .quorumSetHash() 232 | .toString('base64'); 233 | break; 234 | case xdr.ScpStatementType.scpStNominate(): 235 | quorumSetHash = scpStatement 236 | .pledges() 237 | .nominate() 238 | .quorumSetHash() 239 | .toString('base64'); 240 | break; 241 | } 242 | 243 | if (quorumSetHash) return ok(quorumSetHash); 244 | else return err(new Error('Cannot parse quorumSet')); 245 | } catch (e) { 246 | if (e instanceof Error) return err(e); 247 | else return err(new Error('Cannot parse quorumSet')); 248 | } 249 | } 250 | 251 | protected clearQuorumSetRequest( 252 | peerPublicKey: PublicKey, 253 | observation: Observation 254 | ): void { 255 | const result = 256 | observation.quorumSetState.quorumSetRequests.get(peerPublicKey); 257 | if (!result) return; 258 | clearTimeout(result.timeout); 259 | observation.quorumSetState.quorumSetRequests.delete(peerPublicKey); 260 | observation.quorumSetState.quorumSetHashesInProgress.delete(result.hash); 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /src/__tests__/crawler.integration.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Node as NetworkNode, 3 | Connection, 4 | createSCPEnvelopeSignature, 5 | createNode, 6 | getConfigFromEnv 7 | } from '@stellarbeat/js-stellar-node-connector'; 8 | import { xdr, Keypair, hash, Networks } from '@stellar/stellar-base'; 9 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 10 | import { NodeConfig } from '@stellarbeat/js-stellar-node-connector/lib/node-config'; 11 | import { ok, Result, err } from 'neverthrow'; 12 | import { 13 | CrawlerConfiguration, 14 | createCrawler, 15 | createCrawlFactory 16 | } from '../index'; 17 | import { StellarMessageWork } from '@stellarbeat/js-stellar-node-connector/lib/connection/connection'; 18 | import { NodeAddress } from '../node-address'; 19 | 20 | jest.setTimeout(60000); 21 | 22 | let peerNodeAddress: NodeAddress; 23 | let peerNetworkNode: NetworkNode; 24 | 25 | let crawledPeerNetworkNode: NetworkNode; 26 | let crawledPeerNodeAddress: NodeAddress; 27 | 28 | let publicKeyReusingPeerNodeAddress: NodeAddress; 29 | let publicKeyReusingPeerNetworkNode: NetworkNode; 30 | 31 | let qSet: xdr.ScpQuorumSet; 32 | beforeEach(() => { 33 | peerNodeAddress = ['127.0.0.1', 11621]; 34 | peerNetworkNode = getListeningPeerNode(peerNodeAddress); 35 | crawledPeerNodeAddress = ['127.0.0.1', 11622]; 36 | crawledPeerNetworkNode = getListeningPeerNode(crawledPeerNodeAddress); 37 | publicKeyReusingPeerNodeAddress = ['127.0.0.1', 11623]; 38 | publicKeyReusingPeerNetworkNode = getListeningPeerNode( 39 | publicKeyReusingPeerNodeAddress, 40 | peerNetworkNode.keyPair.secret() 41 | ); 42 | qSet = new xdr.ScpQuorumSet({ 43 | threshold: 1, 44 | validators: [ 45 | xdr.PublicKey.publicKeyTypeEd25519( 46 | crawledPeerNetworkNode.keyPair.rawPublicKey() 47 | ), 48 | xdr.PublicKey.publicKeyTypeEd25519(peerNetworkNode.keyPair.rawPublicKey()) 49 | ], 50 | innerSets: [] 51 | }); 52 | }); 53 | 54 | afterEach((done) => { 55 | let counter = 0; 56 | 57 | const cleanup = () => { 58 | counter++; 59 | if (counter === 2) { 60 | done(); 61 | } 62 | }; 63 | peerNetworkNode.stopAcceptingIncomingConnections(cleanup); 64 | crawledPeerNetworkNode.stopAcceptingIncomingConnections(cleanup); 65 | publicKeyReusingPeerNetworkNode.stopAcceptingIncomingConnections(cleanup); 66 | }); 67 | 68 | it('should crawl, listen for validating nodes and harvest quorumSets', async () => { 69 | peerNetworkNode.on('connection', (connection: Connection) => { 70 | connection.on('connect', () => { 71 | const peerAddress = new xdr.PeerAddress({ 72 | ip: xdr.PeerAddressIp.iPv4(Buffer.from([127, 0, 0, 1])), 73 | port: crawledPeerNodeAddress[1], 74 | numFailures: 0 75 | }); 76 | const peers = xdr.StellarMessage.peers([peerAddress]); 77 | connection.sendStellarMessage(peers); 78 | const externalizeResult = createExternalizeMessage(peerNetworkNode); 79 | if (externalizeResult.isOk()) { 80 | connection.sendStellarMessage(externalizeResult.value, (error) => { 81 | if (error) console.log(error); 82 | }); 83 | } else console.log(externalizeResult.error); 84 | }); 85 | connection.on('data', (stellarMessageWork: StellarMessageWork) => { 86 | const stellarMessage = stellarMessageWork.stellarMessage; 87 | switch (stellarMessage.switch()) { 88 | case xdr.MessageType.getScpQuorumset(): { 89 | const dontHave = new xdr.DontHave({ 90 | reqHash: stellarMessage.qSetHash(), 91 | type: xdr.MessageType.getScpQuorumset() 92 | }); 93 | const dontHaveMessage = xdr.StellarMessage.dontHave(dontHave); 94 | connection.sendStellarMessage(dontHaveMessage); 95 | } 96 | } 97 | }); 98 | connection.on('error', (error: Error) => console.log(error)); 99 | 100 | connection.on('close', () => { 101 | return; 102 | }); 103 | connection.on('end', (error?: Error) => { 104 | connection.destroy(error); 105 | }); 106 | }); 107 | peerNetworkNode.on('close', () => { 108 | console.log('seed peer server close'); 109 | }); 110 | 111 | crawledPeerNetworkNode.on('connection', (connection: Connection) => { 112 | connection.on('connect', () => { 113 | const externalizeResult = createExternalizeMessage( 114 | crawledPeerNetworkNode 115 | ); 116 | if (externalizeResult.isOk()) { 117 | connection.sendStellarMessage(externalizeResult.value, (error) => { 118 | if (error) console.log(error); 119 | }); 120 | } 121 | }); 122 | connection.on('data', (stellarMessageWork: StellarMessageWork) => { 123 | const stellarMessage = stellarMessageWork.stellarMessage; 124 | switch (stellarMessage.switch()) { 125 | case xdr.MessageType.getScpQuorumset(): { 126 | const qSetMessage = xdr.StellarMessage.scpQuorumset(qSet); 127 | connection.sendStellarMessage(qSetMessage); 128 | } 129 | } 130 | }); 131 | connection.on('error', (error: Error) => console.log(error)); 132 | 133 | connection.on('close', () => { 134 | return; 135 | }); 136 | connection.on('end', (error?: Error) => { 137 | connection.destroy(error); 138 | }); 139 | }); 140 | crawledPeerNetworkNode.on('close', () => { 141 | console.log('crawled peer server close'); 142 | }); 143 | 144 | const trustedQSet = new QuorumSet(2, [ 145 | peerNetworkNode.keyPair.publicKey(), 146 | crawledPeerNetworkNode.keyPair.publicKey() 147 | ]); 148 | 149 | const nodeConfig: NodeConfig = { 150 | network: Networks.TESTNET, 151 | nodeInfo: { 152 | ledgerVersion: 1, 153 | overlayVersion: 1, 154 | overlayMinVersion: 1, 155 | versionString: '1.0.0', 156 | networkId: Networks.TESTNET 157 | }, 158 | listeningPort: 11026, 159 | receiveTransactionMessages: false, 160 | receiveSCPMessages: true, 161 | peerFloodReadingCapacity: 200, 162 | flowControlSendMoreBatchSize: 40, 163 | peerFloodReadingCapacityBytes: 300000, 164 | flowControlSendMoreBatchSizeBytes: 100000 165 | }; 166 | 167 | const crawlerConfig = new CrawlerConfiguration(nodeConfig); 168 | crawlerConfig.peerStraggleTimeoutMS = 2000; 169 | crawlerConfig.syncingTimeoutMS = 100; 170 | crawlerConfig.quorumSetRequestTimeoutMS = 100; 171 | const crawler = createCrawler(crawlerConfig); 172 | const crawlerFactory = createCrawlFactory(crawlerConfig); 173 | const crawl = crawlerFactory.createCrawl( 174 | [peerNodeAddress, publicKeyReusingPeerNodeAddress], 175 | [], 176 | trustedQSet, 177 | { 178 | sequence: BigInt(0), 179 | closeTime: new Date(0), 180 | value: '', 181 | localCloseTime: new Date(0) 182 | }, 183 | new Map() 184 | ); 185 | 186 | const result = await crawler.startCrawl(crawl); 187 | const peerNode = result.peers.get(peerNetworkNode.keyPair.publicKey()); 188 | expect(peerNode).toBeDefined(); 189 | if (!peerNode) return; 190 | const crawledPeerNode = result.peers.get( 191 | crawledPeerNetworkNode.keyPair.publicKey() 192 | ); 193 | expect(peerNode.successfullyConnected).toBeTruthy(); 194 | expect(peerNode.isValidating).toBeTruthy(); 195 | expect(peerNode.overLoaded).toBeFalsy(); 196 | expect(peerNode.participatingInSCP).toBeTruthy(); 197 | expect(peerNode.latestActiveSlotIndex).toEqual('1'); 198 | expect(peerNode.suppliedPeerList).toBeTruthy(); 199 | expect(peerNode.quorumSetHash).toEqual(hash(qSet.toXDR()).toString('base64')); 200 | expect(peerNode.quorumSet).toBeDefined(); 201 | expect(crawledPeerNode).toBeDefined(); 202 | if (!crawledPeerNode) return; 203 | expect(crawledPeerNode.quorumSetHash).toEqual( 204 | hash(qSet.toXDR()).toString('base64') 205 | ); 206 | expect(crawledPeerNode.quorumSet).toBeDefined(); 207 | expect(crawledPeerNode.isValidating).toBeTruthy(); 208 | expect(crawledPeerNode.participatingInSCP).toBeTruthy(); 209 | expect(crawledPeerNode.latestActiveSlotIndex).toEqual('1'); 210 | }); 211 | 212 | it('should hit the max crawl limit', async function () { 213 | const trustedQSet = new QuorumSet(2, [ 214 | peerNetworkNode.keyPair.publicKey(), 215 | crawledPeerNetworkNode.keyPair.publicKey() 216 | ]); 217 | 218 | const nodeConfig = getConfigFromEnv(); 219 | nodeConfig.network = Networks.TESTNET; 220 | 221 | const crawlerConfig = new CrawlerConfiguration( 222 | nodeConfig, 223 | 25, 224 | 1000, 225 | new Set(), 226 | 1000, 227 | 100, 228 | 100 229 | ); 230 | const crawler = createCrawler(crawlerConfig); 231 | const crawlFactory = createCrawlFactory(crawlerConfig); 232 | 233 | const crawl = crawlFactory.createCrawl( 234 | [peerNodeAddress, publicKeyReusingPeerNodeAddress], 235 | [], 236 | trustedQSet, 237 | { 238 | sequence: BigInt(0), 239 | closeTime: new Date(0), 240 | value: '', 241 | localCloseTime: new Date(0) 242 | }, 243 | new Map() 244 | ); 245 | 246 | try { 247 | expect(await crawler.startCrawl(crawl)).toThrowError(); 248 | } catch (e) { 249 | expect(e).toBeInstanceOf(Error); 250 | } 251 | }); 252 | 253 | function createExternalizeMessage( 254 | node: NetworkNode 255 | ): Result { 256 | const commit = new xdr.ScpBallot({ counter: 1, value: Buffer.alloc(32) }); 257 | const externalize = new xdr.ScpStatementExternalize({ 258 | commit: commit, 259 | nH: 1, 260 | commitQuorumSetHash: hash(qSet.toXDR()) 261 | }); 262 | const pledges = xdr.ScpStatementPledges.scpStExternalize(externalize); 263 | 264 | const statement = new xdr.ScpStatement({ 265 | nodeId: xdr.PublicKey.publicKeyTypeEd25519(node.keyPair.rawPublicKey()), 266 | slotIndex: xdr.Uint64.fromString('1'), 267 | pledges: pledges 268 | }); 269 | const signatureResult = createSCPEnvelopeSignature( 270 | statement, 271 | node.keyPair.rawPublicKey(), 272 | node.keyPair.rawSecretKey(), 273 | hash(Buffer.from(Networks.TESTNET)) 274 | ); 275 | if (signatureResult.isOk()) { 276 | const envelope = new xdr.ScpEnvelope({ 277 | statement: statement, 278 | signature: signatureResult.value 279 | }); 280 | const message = xdr.StellarMessage.scpMessage(envelope); 281 | return ok(message); 282 | } 283 | return err(signatureResult.error); 284 | } 285 | 286 | function getListeningPeerNode(address: NodeAddress, privateKey?: string) { 287 | const peerNodeConfig: NodeConfig = { 288 | network: Networks.TESTNET, 289 | nodeInfo: { 290 | ledgerVersion: 1, 291 | overlayMinVersion: 1, 292 | overlayVersion: 20, 293 | versionString: '1' 294 | }, 295 | listeningPort: address[1], 296 | privateKey: privateKey ? privateKey : Keypair.random().secret(), 297 | receiveSCPMessages: true, 298 | receiveTransactionMessages: false, 299 | peerFloodReadingCapacity: 200, 300 | flowControlSendMoreBatchSize: 40, 301 | peerFloodReadingCapacityBytes: 300000, 302 | flowControlSendMoreBatchSizeBytes: 100000 303 | }; 304 | const peerNetworkNode = createNode(peerNodeConfig); 305 | peerNetworkNode.acceptIncomingConnections(address[1], address[0]); 306 | 307 | return peerNetworkNode; 308 | } 309 | --------------------------------------------------------------------------------