├── .eslintrc
├── .eslintignore
├── tsconfig-build.json
├── src
├── types
│ └── speedometer
│ │ └── index.d.ts
├── logic
│ ├── RtcMessage.ts
│ ├── StorageConfig.ts
│ ├── InstructionCounter.ts
│ ├── LocationManager.ts
│ ├── InstructionRetryManager.ts
│ ├── rtcSignallingHandlers.ts
│ ├── trackerSummaryUtils.ts
│ ├── InstructionThrottler.ts
│ └── PerStreamMetrics.ts
├── helpers
│ ├── MessageEncoder.ts
│ ├── PromiseTools.ts
│ ├── SeenButNotPropagatedSet.ts
│ ├── MessageBuffer.ts
│ ├── Logger.ts
│ └── MetricsContext.ts
├── NameDirectory.ts
├── resend
│ └── proxyRequestStream.ts
├── connection
│ ├── PeerBook.ts
│ ├── IWebRtcEndpoint.ts
│ ├── IWsEndpoint.ts
│ ├── MessageQueue.ts
│ └── NegotiatedProtocolVersions.ts
├── identifiers.ts
└── NetworkNode.ts
├── .idea
├── codeStyles
│ ├── codeStyleConfig.xml
│ └── Project.xml
├── encodings.xml
├── vcs.xml
├── misc.xml
├── modules.xml
├── jsLibraryMappings.xml
├── inspectionProfiles
│ └── Project_Default.xml
├── runConfigurations
│ ├── subscriber__30335_.xml
│ ├── subscriber__30345_.xml
│ ├── test.xml
│ ├── tracker__30300_.xml
│ ├── run_network.xml
│ ├── publisher__30308_.xml
│ ├── publisher__30309_.xml
│ ├── unit_test.xml
│ ├── integration_test.xml
│ └── test__disable_parallel_.xml
└── network.iml
├── typedoc.js
├── .npmignore
├── .gitignore
├── examples
└── system-metrics-pubsub
│ ├── tsconfig.json
│ ├── package.json
│ ├── README.md
│ ├── subscriber.ts
│ └── index.ts
├── test
├── unit
│ ├── NameDirectory.test.ts
│ ├── SeenButNotPropagatedSet.test.ts
│ ├── NumberPair.test.ts
│ ├── encoder.test.ts
│ ├── TrackerServer.test.ts
│ ├── trackerSummaryUtils.test.ts
│ ├── NegotiatedProtocolVersions.test.ts
│ ├── MessageQueue.test.ts
│ ├── PeerInfo.test.ts
│ ├── proxyRequestStream.test.ts
│ ├── Logger.test.ts
│ ├── PerStreamMetrics.test.ts
│ ├── QueueItem.test.ts
│ ├── InstructionThrottler.test.ts
│ └── LocationManager.test.ts
├── integration
│ ├── MockStorageConfig.ts
│ ├── passing-address-between-ws-endpoints.test.ts
│ ├── duplicate-connections-are-closed.test.ts
│ ├── ws-endpoint-back-pressure-handling.test.ts
│ ├── tracker-storage-nodes-response-does-not-contain-self.test.ts
│ ├── nodeMessageBuffering.test.ts
│ ├── killing-dead-connections.test.ts
│ ├── network-stabilization.test.ts
│ ├── latency.test.ts
│ ├── unsubscribe-from-stream.test.ts
│ ├── tracker-node-reconnect-instructions.test.ts
│ ├── webrtc-endpoint-back-pressure-handling.test.ts
│ ├── webrtc-multi-signaller.test.ts
│ ├── do-not-propagate-to-sender-optimization.test.ts
│ ├── storage-config.test.ts
│ ├── resend-request-on-streams-with-no-activity.test.ts
│ ├── tracker-instructions.test.ts
│ ├── l1-resend-requests.test.ts
│ ├── request-resend-from-uninvolved-node.test.ts
│ └── ws-endpoint.test.ts
├── benchmarks
│ ├── overlay-topology-performance.ts
│ ├── overlay-topology-randomness.ts
│ └── tracker.instructions.ts
└── fixtures
│ ├── cert.pem
│ └── key.pem
├── tsconfig.json
├── .github
└── workflows
│ ├── docs.yml
│ └── nodejs.yml
├── jest.config.js
├── bin
├── network.js
├── tracker.js
├── subscriber.js
└── publisher.js
├── package.json
├── LICENSE
└── README.md
/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "eslint-config-streamr-ts"
3 | }
4 |
--------------------------------------------------------------------------------
/.eslintignore:
--------------------------------------------------------------------------------
1 | node_modules/**
2 | examples/**
3 | coverage/**
4 | dist/**
--------------------------------------------------------------------------------
/tsconfig-build.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "./tsconfig.json",
3 | "include": [
4 | "src/**/*"
5 | ]
6 | }
--------------------------------------------------------------------------------
/src/types/speedometer/index.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'speedometer' {
2 | export default function speedometer(seconds?: number): (delta?: number) => number
3 | }
4 |
--------------------------------------------------------------------------------
/.idea/codeStyles/codeStyleConfig.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/typedoc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | entryPoints: [
3 | 'src/composition.ts',
4 | ],
5 | tsconfig: 'tsconfig.json',
6 | excludeInternal: true,
7 | includeVersion: true,
8 | }
9 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | .codedeploy
2 | .eslintignore
3 | .eslintrc.js
4 | examples
5 | .github
6 | .idea
7 | jest.config.js
8 | .npmrc
9 | src
10 | test
11 | .travis.yml
12 | tsconfig.json
13 | *.tsbuildinfo
14 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | dist/
3 |
4 | # IntelliJ IDEA
5 | .idea/workspace.xml
6 | .idea/tasks.xml
7 | .idea/dictionaries
8 | .idea/httpRequests
9 | .idea/git_toolbox_prj.xml
10 | bin/.idea
11 |
12 | .DS_Store
13 | docs
14 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/src/logic/RtcMessage.ts:
--------------------------------------------------------------------------------
1 | export enum RtcSubTypes {
2 | LOCAL_DESCRIPTION = 'localDescription',
3 | LOCAL_CANDIDATE = 'localCandidate',
4 | RTC_CONNECT = 'rtcConnect',
5 | RTC_OFFER = 'rtcOffer',
6 | RTC_ANSWER = 'rtcAnswer',
7 | REMOTE_CANDIDATE = 'remoteCandidate'
8 | }
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/jsLibraryMappings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/examples/system-metrics-pubsub/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es6",
4 | "module": "commonjs",
5 | "lib": ["es6"],
6 | "allowJs": true,
7 | "outDir": "dist",
8 | "rootDir": ".",
9 | "strict": true,
10 | "esModuleInterop": true,
11 | "noImplicitAny": true,
12 | "resolveJsonModule": true
13 | },
14 | "exclude": [
15 | "dist"
16 | ]
17 | }
18 |
--------------------------------------------------------------------------------
/src/logic/StorageConfig.ts:
--------------------------------------------------------------------------------
1 | import { StreamIdAndPartition } from '../identifiers'
2 |
3 | export interface ChangeListener {
4 | onStreamAdded: (stream: StreamIdAndPartition) => void
5 | onStreamRemoved: (stream: StreamIdAndPartition) => void
6 | }
7 |
8 | export interface StorageConfig {
9 | getStreams: () => StreamIdAndPartition[]
10 | addChangeListener: (listener: ChangeListener) => void
11 | }
--------------------------------------------------------------------------------
/test/unit/NameDirectory.test.ts:
--------------------------------------------------------------------------------
1 | import { NameDirectory } from '../../src/NameDirectory'
2 |
3 | describe('NameDirectory', () => {
4 | test('known', () => {
5 | expect(NameDirectory.getName('0xDE33390cC85aBf61d9c27715Fa61d8E5efC61e75')).toBe('T3')
6 | })
7 | test('unknown', () => {
8 | expect(NameDirectory.getName('0x1234567890123456789012345678901234567890')).toBe('0x123456')
9 | })
10 | })
--------------------------------------------------------------------------------
/src/helpers/MessageEncoder.ts:
--------------------------------------------------------------------------------
1 | export function decode(serializedMessage: M, deserializeFn: (serializedMessage: M) => R): R | null | never {
2 | try {
3 | return deserializeFn(serializedMessage)
4 | } catch (e) {
5 | // JSON parsing failed, version parse failed, type parse failed
6 | if (e.name === 'SyntaxError' || e.version != null || e.type != null) {
7 | return null
8 | }
9 | throw e
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es2020",
4 | "module": "commonjs",
5 | "lib": ["es2020"],
6 | "allowJs": true,
7 | "declaration": true,
8 | "outDir": "dist",
9 | "rootDirs": ["src", "test"],
10 | "strict": true,
11 | "esModuleInterop": true,
12 | "noImplicitAny": true,
13 | "resolveJsonModule": true,
14 | "incremental": true,
15 | "sourceMap": true
16 | },
17 | "include": [
18 | "src/**/*",
19 | "test/**/*"
20 | ]
21 | }
22 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/subscriber__30335_.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/subscriber__30345_.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/test.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/tracker__30300_.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/run_network.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/publisher__30308_.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/publisher__30309_.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/unit_test.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/network.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/integration_test.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/test/unit/SeenButNotPropagatedSet.test.ts:
--------------------------------------------------------------------------------
1 | import { MessageLayer } from 'streamr-client-protocol'
2 |
3 | import { SeenButNotPropagatedSet } from '../../src/helpers/SeenButNotPropagatedSet'
4 |
5 | const { MessageIDStrict } = MessageLayer
6 |
7 | describe('SeenButNotPropagatedSet', () => {
8 | it('messageIdToStr', () => {
9 | const messageId = new MessageIDStrict('streamId', 10, 1000000, 0, 'publisherId', 'msgChainId')
10 | const actual = SeenButNotPropagatedSet.messageIdToStr(messageId)
11 | expect(actual).toEqual('streamId-10-1000000-0-publisherId-msgChainId')
12 | })
13 | })
14 |
--------------------------------------------------------------------------------
/src/helpers/PromiseTools.ts:
--------------------------------------------------------------------------------
1 | export function promiseTimeout(ms: number, givenPromise: Promise): Promise {
2 | const timeoutPromise = new Promise((resolve, reject) => {
3 | const timeoutRef = setTimeout(() => {
4 | reject(new Error('timed out in ' + ms + 'ms.'))
5 | }, ms)
6 |
7 | // Clear timeout if promise wins race
8 | givenPromise
9 | .finally(() => clearTimeout(timeoutRef))
10 | .catch(() => null)
11 | })
12 |
13 | return Promise.race([
14 | givenPromise,
15 | timeoutPromise
16 | ]) as Promise
17 | }
18 |
--------------------------------------------------------------------------------
/.idea/runConfigurations/test__disable_parallel_.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.github/workflows/docs.yml:
--------------------------------------------------------------------------------
1 | name: Generate Docs
2 |
3 | on:
4 | push:
5 | branches:
6 | - master # Set a branch name to trigger deployment
7 |
8 | jobs:
9 | build:
10 | name: Run build using Node 14.x
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@v2
14 | - uses: actions/setup-node@v2
15 | with:
16 | node-version: "14.x"
17 | - name: npm ci
18 | run: npm ci
19 | - name: npm run docs
20 | run: npm run docs
21 | - uses: actions/upload-artifact@v2
22 | with:
23 | name: docs
24 | path: docs
25 | - name: Deploy
26 | uses: peaceiris/actions-gh-pages@v3
27 | with:
28 | github_token: ${{ secrets.GITHUB_TOKEN }}
29 | publish_dir: ./docs
30 |
31 |
--------------------------------------------------------------------------------
/jest.config.js:
--------------------------------------------------------------------------------
1 | // For a detailed explanation regarding each configuration property, visit:
2 | // https://jestjs.io/docs/en/configuration.html
3 | module.exports = {
4 |
5 | // Preset ts-jest
6 | preset: 'ts-jest',
7 |
8 | // Automatically clear mock calls and instances between every test
9 | clearMocks: true,
10 |
11 | // An array of glob patterns indicating a set of files for which coverage information should be collected
12 | collectCoverageFrom: ['src/**'],
13 |
14 | // The directory where Jest should output its coverage files
15 | coverageDirectory: 'coverage',
16 |
17 | // The test environment that will be used for testing
18 | testEnvironment: 'node',
19 |
20 | // Default timeout of a test in milliseconds
21 | testTimeout: 15000,
22 |
23 | // This option allows use of a custom test runner
24 | testRunner: 'jest-circus/runner'
25 | }
26 |
--------------------------------------------------------------------------------
/examples/system-metrics-pubsub/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "system-metrics-pubsub",
3 | "version": "1.0.0",
4 | "description": "An example of using streamr-network in a pub/sub setting",
5 | "main": "index.ts",
6 | "scripts": {
7 | "network-init": "ts-node index.ts",
8 | "network-init-with-logging": "LOG_LEVEL=debug ts-node index.ts",
9 | "subscriber": "ts-node subscriber.ts",
10 | "subscriber-with-logging": "LOG_LEVEL=debug ts-node subscriber.ts",
11 | "test": "echo \"Error: no test specified\" && exit 1"
12 | },
13 | "author": "Streamr Network AG ",
14 | "license": "STREAMR NETWORK OPEN SOURCE LICENSE",
15 | "dependencies": {
16 | "streamr-network": "^23.0.20",
17 | "uuid": "^8.3.2"
18 | },
19 | "devDependencies": {
20 | "@types/node": "^14.14.14",
21 | "@types/uuid": "^8.3.0",
22 | "ts-node": "^9.1.1",
23 | "typescript": "^4.1.3"
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/test/integration/MockStorageConfig.ts:
--------------------------------------------------------------------------------
1 | import { ChangeListener, StorageConfig } from '../../src/logic/StorageConfig'
2 | import { StreamIdAndPartition, StreamKey } from '../../src/identifiers'
3 |
4 | export class MockStorageConfig implements StorageConfig {
5 | private streams: Set = new Set()
6 |
7 | private listeners: ChangeListener[] = []
8 |
9 | getStreams(): StreamIdAndPartition[] {
10 | return Array.from(this.streams.values()).map((key) => StreamIdAndPartition.fromKey(key))
11 | }
12 |
13 | addChangeListener(listener: ChangeListener): void {
14 | this.listeners.push(listener)
15 | }
16 |
17 | addStream(stream: StreamIdAndPartition): void {
18 | this.streams.add(stream.key())
19 | this.listeners.forEach((listener) => listener.onStreamAdded(stream))
20 | }
21 |
22 | removeStream(stream: StreamIdAndPartition): void {
23 | this.streams.delete(stream.key())
24 | this.listeners.forEach((listener) => listener.onStreamRemoved(stream))
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/examples/system-metrics-pubsub/README.md:
--------------------------------------------------------------------------------
1 | # System-metric-pubsub example
2 |
3 | An example Streamr Network in a pub/sub setting.
4 |
5 | This example network generates one Tracker and one Publisher Node, connnecting to N Subscriber nodes. The Publisher Node publishes system metrics to the `'system-report'` stream every two seconds. Subscriber nodes subscribe to this `'system-report'` stream and output the arriving messages in stdout. The Tracker assists the nodes in peer discovery (finding and connecting to each other).
6 |
7 | Install
8 | ```
9 | npm ci
10 | ```
11 |
12 | In one terminal window run `network-init`, which starts the tracker and publisher node:
13 | ```
14 | npm run network-init
15 | ```
16 |
17 | In a different terminal window run a subscriber node:
18 | ```
19 | npm run subscriber
20 | ```
21 |
22 | You should see your system metrics stream into the subscriber terminal window.
23 |
24 | ### Debugging
25 |
26 | Run with debugging enabled
27 | ```
28 | npm run network-init-with-logging
29 | npm run subscriber-with-logging
30 | ```
31 |
--------------------------------------------------------------------------------
/.idea/codeStyles/Project.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/src/NameDirectory.ts:
--------------------------------------------------------------------------------
1 | // Get human-readable names for Trackers and Brokers
2 | // Currently contains hardcoded names for all streamr-docker-dev entities
3 | // -> in the future each node receives the peer names from Tracker
4 | // and we can remove the hardcoded values
5 |
6 | const NAMES: Record = {
7 | '0xDE11165537ef6C01260ee89A850a281525A5b63F': 'T1',
8 | '0xDE22222da3F861c2Ec63b03e16a1dce153Cf069c': 'T2',
9 | '0xDE33390cC85aBf61d9c27715Fa61d8E5efC61e75': 'T3',
10 | '0xde1112f631486CfC759A50196853011528bC5FA0': 'S1',
11 | '0xde222E8603FCf641F928E5F66a0CBf4de70d5352': 'B1',
12 | '0xde3331cA6B8B636E0b82Bf08E941F727B8927442': 'B2'
13 | }
14 |
15 | export class NameDirectory {
16 |
17 | static MAX_FALLBACK_NAME_LENGTH = 8
18 |
19 | // if name is not known, creates a short name from the peerId
20 | static getName(peerId: string): string {
21 | const name = NAMES[peerId]
22 | if (name !== undefined) {
23 | return name
24 | } else {
25 | return (peerId.length > NameDirectory.MAX_FALLBACK_NAME_LENGTH)
26 | ? peerId.substring(0, this.MAX_FALLBACK_NAME_LENGTH)
27 | : peerId
28 | }
29 | }
30 | }
--------------------------------------------------------------------------------
/test/unit/NumberPair.test.ts:
--------------------------------------------------------------------------------
1 | import { NumberPair } from '../../src/logic/DuplicateMessageDetector'
2 |
3 | test('equalTo', () => {
4 | expect(new NumberPair(5, 2).equalTo(new NumberPair(5, 3))).toEqual(false)
5 | expect(new NumberPair(5, 2).equalTo(new NumberPair(5, 2))).toEqual(true)
6 | })
7 |
8 | test('greaterThan', () => {
9 | expect(new NumberPair(5, 2).greaterThan(new NumberPair(6, 2))).toEqual(false)
10 | expect(new NumberPair(5, 2).greaterThan(new NumberPair(5, 3))).toEqual(false)
11 | expect(new NumberPair(5, 2).greaterThan(new NumberPair(5, 2))).toEqual(false)
12 | expect(new NumberPair(5, 2).greaterThan(new NumberPair(5, 1))).toEqual(true)
13 | expect(new NumberPair(5, 2).greaterThan(new NumberPair(3, 2))).toEqual(true)
14 | })
15 |
16 | test('greaterThanOrEqual', () => {
17 | expect(new NumberPair(5, 2).greaterThanOrEqual(new NumberPair(6, 2))).toEqual(false)
18 | expect(new NumberPair(5, 2).greaterThanOrEqual(new NumberPair(5, 3))).toEqual(false)
19 | expect(new NumberPair(5, 2).greaterThanOrEqual(new NumberPair(5, 2))).toEqual(true)
20 | expect(new NumberPair(5, 2).greaterThanOrEqual(new NumberPair(5, 1))).toEqual(true)
21 | expect(new NumberPair(5, 2).greaterThanOrEqual(new NumberPair(3, 2))).toEqual(true)
22 | })
23 |
--------------------------------------------------------------------------------
/test/unit/encoder.test.ts:
--------------------------------------------------------------------------------
1 | import { ControlLayer } from 'streamr-client-protocol'
2 |
3 | import { decode } from '../../src/helpers/MessageEncoder'
4 |
5 | describe('encoder', () => {
6 | const controlMessage = new ControlLayer.ResendResponseNoResend({
7 | requestId: 'requestId',
8 | streamId: 'streamId',
9 | streamPartition: 0,
10 | })
11 |
12 | it('decode', () => {
13 | const result = decode(controlMessage.serialize(), ControlLayer.ControlMessage.deserialize)
14 | expect(result).toEqual(controlMessage)
15 | })
16 |
17 | it('decode returns null if controlMessage unparsable', () => {
18 | const result = decode('NOT_A_VALID_CONTROL_MESSAGE', ControlLayer.ControlMessage.deserialize)
19 | expect(result).toBeNull()
20 | })
21 |
22 | it('decode returns null if unknown control message version', () => {
23 | const result = decode('[6666,2,"requestId","streamId",0]', ControlLayer.ControlMessage.deserialize)
24 | expect(result).toBeNull()
25 | })
26 |
27 | it('decode returns null if unknown control message type', () => {
28 | const result = decode('[2,6666,"requestId","streamId",0]', ControlLayer.ControlMessage.deserialize)
29 | expect(result).toBeNull()
30 | })
31 | })
32 |
33 |
--------------------------------------------------------------------------------
/test/unit/TrackerServer.test.ts:
--------------------------------------------------------------------------------
1 | import { TrackerServer } from '../../src/protocol/TrackerServer'
2 | import { Event } from '../../src/connection/IWsEndpoint'
3 | import { WsEndpoint } from '../../src/connection/WsEndpoint'
4 | import { PeerInfo } from '../../src/connection/PeerInfo'
5 |
6 | describe(TrackerServer, () => {
7 | it('getNodeIds', () => {
8 | const trackerServer = new TrackerServer({
9 | on(_event: Event, _args: any): void {
10 | },
11 | getPeerInfo(): Readonly {
12 | return PeerInfo.newNode('nodeZero', 'nodeZero', undefined , undefined,null)
13 | },
14 | getPeerInfos(): PeerInfo[] {
15 | return [
16 | PeerInfo.newNode('nodeOne', 'nodeOne', undefined , undefined,null),
17 | PeerInfo.newNode('nodeTwo', 'nodeTwo',undefined , undefined, null),
18 | PeerInfo.newTracker('tracker', 'tracker', undefined , undefined,null),
19 | PeerInfo.newUnknown('unknownPeer'),
20 | PeerInfo.newStorage('storageNode', 'storageNode', undefined , undefined, null)
21 | ]
22 | }
23 | } as WsEndpoint)
24 | expect(trackerServer.getNodeIds()).toEqual(['nodeOne', 'nodeTwo', 'storageNode'])
25 | })
26 | })
27 |
--------------------------------------------------------------------------------
/test/benchmarks/overlay-topology-performance.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-console */
2 | import { OverlayTopology } from '../../src/logic/OverlayTopology'
3 |
4 | const numOfRounds = 10
5 | const numOfNeighbors = 4
6 | const numOfNodeConfigurations = [10, 100, 200, 500, 1000, 2000, 5000]
7 |
8 | interface Measurements {
9 | [key: number]: any[]
10 | }
11 |
12 | // Run topology experiment
13 | const measurements: Measurements = {}
14 | numOfNodeConfigurations.forEach((k) => {
15 | measurements[k] = []
16 | })
17 |
18 | for (let i = 0; i < numOfRounds; ++i) {
19 | numOfNodeConfigurations.forEach((numOfNodes) => {
20 | const topology = new OverlayTopology(numOfNeighbors)
21 | const startTime = Date.now()
22 | for (let j = 0; j < numOfNodes; ++j) {
23 | const nodeId = `node-${j}`
24 | topology.update(nodeId, [])
25 | topology.formInstructions(nodeId)
26 | }
27 | measurements[numOfNodes].push(Date.now() - startTime)
28 | })
29 | }
30 |
31 | const report = Object.entries(measurements).map(([numOfNodes, values]) => {
32 | const mean = values.reduce((acc: number, v: number) => acc + v, 0) / values.length
33 | const msPerJoinedNode = mean / Number(numOfNodes)
34 | return {
35 | numOfNodes,
36 | mean,
37 | msPerJoinedNode
38 | }
39 | })
40 |
41 | console.table(report)
42 |
--------------------------------------------------------------------------------
/src/resend/proxyRequestStream.ts:
--------------------------------------------------------------------------------
1 | import { ControlLayer } from 'streamr-client-protocol'
2 | import { Readable } from 'stream'
3 | import { ResendRequest } from '../identifiers'
4 |
5 | export function proxyRequestStream(
6 | sendFn: (msg: ControlLayer.ControlMessage) => void,
7 | request: ResendRequest,
8 | requestStream: Readable
9 | ): void {
10 | const { streamId, streamPartition, requestId } = request
11 | let fulfilled = false
12 | requestStream
13 | .once('data', () => {
14 | sendFn(new ControlLayer.ResendResponseResending({
15 | requestId,
16 | streamId,
17 | streamPartition
18 | }))
19 | fulfilled = true
20 | })
21 | .on('data', (unicastMessage: ControlLayer.UnicastMessage) => {
22 | sendFn(unicastMessage)
23 | })
24 | .on('end', () => {
25 | if (fulfilled) {
26 | sendFn(new ControlLayer.ResendResponseResent({
27 | requestId,
28 | streamId,
29 | streamPartition
30 | }))
31 | } else {
32 | sendFn(new ControlLayer.ResendResponseNoResend({
33 | requestId,
34 | streamId,
35 | streamPartition
36 | }))
37 | }
38 | })
39 | }
40 |
--------------------------------------------------------------------------------
/examples/system-metrics-pubsub/subscriber.ts:
--------------------------------------------------------------------------------
1 | import {
2 | startNetworkNode,
3 | Protocol,
4 | NetworkNode
5 | } from "streamr-network"
6 |
7 | /**
8 | * Run a subscriber node that subscribes to stream "system-report" and logs all received messages in stdout.
9 | */
10 | async function runSubscriber(id: string, port: number): Promise {
11 | const subscriberNode: NetworkNode = await startNetworkNode({
12 | host: '127.0.0.1',
13 | port: port,
14 | name: id,
15 | trackers: ['ws://127.0.0.1:30300']
16 | })
17 | subscriberNode.start()
18 | subscriberNode.subscribe('system-report', 0)
19 | subscriberNode.addMessageListener((msg: Protocol.MessageLayer.StreamMessage) => {
20 | const msgAsJson = JSON.stringify(msg.getContent(), null, 2)
21 | console.info(`${id} received ${msgAsJson}`)
22 | })
23 | return subscriberNode
24 | }
25 |
26 | async function main(): Promise {
27 | let SUB_PORT = 30304
28 | let count = 0
29 | let maxTries = 20
30 | let nodeRunning = false
31 |
32 | while(!nodeRunning) {
33 | try {
34 | await runSubscriber(`subscriberNode ${SUB_PORT}`, SUB_PORT)
35 | nodeRunning = true
36 | } catch (e) {
37 | ++SUB_PORT
38 | if (++count == maxTries) throw e;
39 | }
40 | }
41 | }
42 | main().catch((err) => console.error(err))
--------------------------------------------------------------------------------
/src/connection/PeerBook.ts:
--------------------------------------------------------------------------------
1 | import { PeerInfo } from './PeerInfo'
2 |
3 | export class NotFoundInPeerBookError extends Error {
4 | constructor(msg: string) {
5 | super(msg)
6 | Error.captureStackTrace(this, NotFoundInPeerBookError)
7 | }
8 | }
9 |
10 | export class PeerBook {
11 | private readonly peerInfos: { [key: string]: PeerInfo }
12 | constructor() {
13 | this.peerInfos = {}
14 | }
15 |
16 | add(peerAddress: string, peerInfo: PeerInfo): void {
17 | this.peerInfos[peerAddress] = peerInfo
18 | }
19 |
20 | getPeerInfo(peerAddress: string): PeerInfo | null | never {
21 | return this.peerInfos[peerAddress] || null
22 | }
23 |
24 | remove(peerAddress: string): void {
25 | delete this.peerInfos[peerAddress]
26 | }
27 |
28 | getAddress(peerId: string): string | never {
29 | const address = Object.keys(this.peerInfos).find((p) => this.peerInfos[p].peerId === peerId)
30 | if (!address) {
31 | throw new NotFoundInPeerBookError(`PeerId ${peerId} not found in peer book`)
32 | }
33 | return address
34 | }
35 |
36 | getPeerId(address: string): string | never {
37 | const peerInfo = this.peerInfos[address]
38 | if (!peerInfo) {
39 | throw new NotFoundInPeerBookError(`Address ${address} not found in peer book`)
40 | }
41 | return peerInfo.peerId
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/helpers/SeenButNotPropagatedSet.ts:
--------------------------------------------------------------------------------
1 | import LRUCache from 'lru-cache'
2 |
3 | const MAX_ELEMENTS = 50000
4 | const MAX_AGE = 60 * 1000
5 |
6 | interface MessageId {
7 | streamId: string
8 | streamPartition: number
9 | timestamp: number
10 | sequenceNumber: number
11 | publisherId: string
12 | msgChainId: string
13 | }
14 |
15 | type InternalMessageId = string
16 |
17 | /**
18 | * Keeps track of message identifiers that have been seen but not yet propagated to other nodes.
19 | */
20 | export class SeenButNotPropagatedSet {
21 | private readonly cache: LRUCache = new LRUCache({
22 | max: MAX_ELEMENTS,
23 | maxAge: MAX_AGE
24 | })
25 |
26 | add(streamMessage: { messageId: MessageId }): void {
27 | this.cache.set(SeenButNotPropagatedSet.messageIdToStr(streamMessage.messageId))
28 | }
29 |
30 | delete(streamMessage: { messageId: MessageId }): void {
31 | this.cache.del(SeenButNotPropagatedSet.messageIdToStr(streamMessage.messageId))
32 | }
33 |
34 | has(streamMessage: { messageId: MessageId }): boolean {
35 | return this.cache.has(SeenButNotPropagatedSet.messageIdToStr(streamMessage.messageId))
36 | }
37 |
38 | size(): number {
39 | return this.cache.length
40 | }
41 |
42 | static messageIdToStr({
43 | streamId, streamPartition, timestamp, sequenceNumber, publisherId, msgChainId
44 | }: MessageId): InternalMessageId {
45 | return `${streamId}-${streamPartition}-${timestamp}-${sequenceNumber}-${publisherId}-${msgChainId}`
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/connection/IWebRtcEndpoint.ts:
--------------------------------------------------------------------------------
1 | import {PeerInfo} from './PeerInfo'
2 | import { Rtts } from '../identifiers'
3 |
4 | export enum Event {
5 | PEER_CONNECTED = 'streamr:peer:connect',
6 | PEER_DISCONNECTED = 'streamr:peer:disconnect',
7 | MESSAGE_RECEIVED = 'streamr:message-received',
8 | HIGH_BACK_PRESSURE = 'streamr:high-back-pressure',
9 | LOW_BACK_PRESSURE = 'streamr:low-back-pressure'
10 | }
11 |
12 | export interface IWebRtcEndpoint {
13 |
14 | // Declare event handlers
15 | on(event: Event.PEER_CONNECTED, listener: (peerInfo: PeerInfo) => void): this
16 | on(event: Event.PEER_DISCONNECTED, listener: (peerInfo: PeerInfo) => void): this
17 | on(event: Event.MESSAGE_RECEIVED, listener: (peerInfo: PeerInfo, message: string) => void): this
18 | on(event: Event.HIGH_BACK_PRESSURE, listener: (peerInfo: PeerInfo) => void): this
19 | on(event: Event.LOW_BACK_PRESSURE, listener: (peerInfo: PeerInfo) => void): this
20 |
21 | connect(targetPeerId: string, routerId: string, isOffering: boolean|undefined): Promise
22 | send(targetPeerId: string, message: string): Promise
23 | close(receiverNodeId: string, reason: string): void
24 | getRtts(): Readonly
25 | getPeerInfo(): Readonly
26 | getAddress(): string
27 | stop(): void
28 | getNegotiatedMessageLayerProtocolVersionOnNode(peerId: string): number | undefined
29 | getNegotiatedControlLayerProtocolVersionOnNode(peerId: string): number | undefined
30 | getDefaultMessageLayerProtocolVersion(): number
31 | getDefaultControlLayerProtocolVersion(): number
32 | }
--------------------------------------------------------------------------------
/test/integration/passing-address-between-ws-endpoints.test.ts:
--------------------------------------------------------------------------------
1 | import { waitForEvent } from 'streamr-test-utils'
2 |
3 | import { Event } from '../../src/connection/IWsEndpoint'
4 | import { startEndpoint, WsEndpoint } from '../../src/connection/WsEndpoint'
5 | import { PeerInfo } from '../../src/connection/PeerInfo'
6 |
7 | describe('passing address between WsEndpoints', () => {
8 | let wsEndpoint1: WsEndpoint
9 | let wsEndpoint2: WsEndpoint
10 |
11 | beforeEach(async () => {
12 | wsEndpoint1 = await startEndpoint('127.0.0.1', 31960, PeerInfo.newNode('wsEndpoint1'), null)
13 | })
14 |
15 | afterEach(async () => {
16 | await wsEndpoint1.stop()
17 | await wsEndpoint2.stop()
18 | })
19 |
20 | it('bound address is passed to other WsEndpoint if advertisedWsUrl not set', async () => {
21 | wsEndpoint2 = await startEndpoint('127.0.0.1', 31961, PeerInfo.newNode('wsEndpoint2'), null)
22 | wsEndpoint2.connect('ws://127.0.0.1:31960')
23 | await waitForEvent(wsEndpoint1, Event.PEER_CONNECTED)
24 | const address = wsEndpoint1.resolveAddress('wsEndpoint2')
25 | expect(address).toEqual('ws://127.0.0.1:31961')
26 | })
27 |
28 | it('advertised address is passed to other WsEndpoint if advertisedWsUrl set', async () => {
29 | const advertisedWsUrl = 'ws://advertised-ws-url:666'
30 | wsEndpoint2 = await startEndpoint('127.0.0.1', 31961, PeerInfo.newNode('wsEndpoint2'), advertisedWsUrl)
31 | wsEndpoint2.connect('ws://127.0.0.1:31960')
32 | await waitForEvent(wsEndpoint1, Event.PEER_CONNECTED)
33 | const address = wsEndpoint1.resolveAddress('wsEndpoint2')
34 | expect(address).toEqual('ws://advertised-ws-url:666')
35 | })
36 | })
37 |
--------------------------------------------------------------------------------
/src/logic/InstructionCounter.ts:
--------------------------------------------------------------------------------
1 | import { Status, StatusStreams, StreamKey } from '../identifiers'
2 |
3 | interface Counters {
4 | [key: string]: {
5 | [key: string]: number
6 | }
7 | }
8 |
9 | export class InstructionCounter {
10 | private readonly counters: Counters = {}
11 |
12 | constructor() {}
13 |
14 | setOrIncrement(nodeId: string, streamKey: StreamKey): number {
15 | this.getAndSetIfNecessary(nodeId, streamKey)
16 | this.counters[nodeId][streamKey] += 1
17 | return this.counters[nodeId][streamKey]
18 | }
19 |
20 | filterStatus(status: Status, source: string): StatusStreams {
21 | const filteredStreams: StatusStreams = {}
22 | Object.entries(status.streams).forEach(([streamKey, entry]) => {
23 | const currentCounter = this.getAndSetIfNecessary(source, streamKey)
24 | if (entry.counter >= currentCounter || entry.counter === -1) {
25 | filteredStreams[streamKey] = entry
26 | }
27 | })
28 | return filteredStreams
29 | }
30 |
31 | removeNode(nodeId: string): void {
32 | delete this.counters[nodeId]
33 | }
34 |
35 | removeStream(streamKey: StreamKey): void {
36 | Object.keys(this.counters).forEach((nodeId) => {
37 | delete this.counters[nodeId][streamKey]
38 | })
39 | }
40 |
41 | private getAndSetIfNecessary(nodeId: string, streamKey: StreamKey): number {
42 | if (this.counters[nodeId] === undefined) {
43 | this.counters[nodeId] = {}
44 | }
45 | if (this.counters[nodeId][streamKey] === undefined) {
46 | this.counters[nodeId][streamKey] = 0
47 | }
48 | return this.counters[nodeId][streamKey]
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/test/fixtures/cert.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIEpjCCAo4CCQDy+oStl8S8fzANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
3 | b2NhbGhvc3QwIBcNMjAwOTI1MDkwNzQyWhgPMjEyMDA5MDEwOTA3NDJaMBQxEjAQ
4 | BgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
5 | ANSjGuAZlX2k2ZBsVFnn57XcoSoSw4oHfZ5QkCcAdsORWDm/t49KTFq/oLlLhXnn
6 | kFfWwEGzEZdugqvcT8MOfkXrzXnCD5ctqajW8O4GpPkGWDusC+w+/FcKNugyuuDk
7 | 4DxuN5yjp0jnph7VA4znztvILhg9AWNjQ7NL43TxdIoeC5JKGRBvzDnnhMHQA397
8 | /qKzzdKLQDLXyVJr4AsEhoXW//oQNpWMQPZd/vQFj/3FI2IhsrLeK8RY0JuPtGB/
9 | 0KxkXPARChSLS3P2uTzr0xO3Siah99Pqe8bPy7hn2QEYFAXucS/oDCl0x/zNqkZz
10 | X9VqaTs6btX+eOZQvI0u1pHSnpZs7AbndjkpMpk0vueM6oUqTfQlzQDys/EJLJyN
11 | ucz4FmvcsQK1sHQLd3SVE57WPOIZcV2mt6aA0Kjr4c64AQ/9QFTC9acM5IWYa6Q8
12 | 88aFR5T7H8N5cJfamliBMul3EUkDs+447lCe5zpF2uPmpSt8GzxnNFJKYXad3Lzj
13 | PvbvnckJYTIUbqCUHYixDko1KQwgvwavAP1z4GdHtc2ZRM6/1rg26if7rESg3UQZ
14 | 1YUOXcl8vgDOd9IOFEEuKS5qCQSBx8RjOJyb5qRibgy+AEpThyZ5cMCpEeE/cRBu
15 | irRkclrRKpRMPivGLnj5ajRJPNDA/UYxoAvMnqlzlsEfAgMBAAEwDQYJKoZIhvcN
16 | AQELBQADggIBAK3F+DBwNBZZqYCdUiTq7tlT1Wz6Est6GuhNiGlUurba8rsvnkQh
17 | m9+2zgJyqGMifnyMNq/M7Odd44G1mRfFA/RpJvD6QagoVONbrhDW8Qncu3lG9xbe
18 | nCYv5DqDjAH47jRlike+/SkzBLymnhvJDVRsSNaW5D9Ols1DoMPI9xtSDe6joeLM
19 | cNslgkiSvLq8BznbHq4SGwPfDjixoCH/64SQLVFAYoEteketpBJMZ+637Bd3CSaq
20 | skJezxf0nDmrSi1zoes2vTfRiZkuyRnYV0DF+pfFqRwO4c/tuNRuhMkkIn0VptHc
21 | F3s+89Wq92QQa1losYfMbOGiNzQvPSs9dqA6lJnt9gVbxxjtTVpYRa0oCO+WTj0P
22 | 5SkKGMVA7Z5AByhQj4acoA4WGnj3cIYt+ygUT97Qy+cNrPTtdajQUHvK/ewYjAdJ
23 | VRVxPWJE5VGqDbhvLV2Ogq+6gaFNFAmXP6CjevsSh+lDQyn0VzLswpprvgT8fBTW
24 | YHEabZFRJIKt/AXAyY436gOQmjVOzeJdtbAASnEMhRm3im4bLsVEmVJHjyAR3aKx
25 | FZ4PatWIrIFL3HoZnE3XVs5eE99KmUmwi61asi15c06bhMaX7hMQUcaUfzKUQEVG
26 | +6/pOf/yWKft0QknGVpy7eces0rUiriZhOjvECbMF1MTWuksH8eQXRPI
27 | -----END CERTIFICATE-----
28 |
--------------------------------------------------------------------------------
/test/integration/duplicate-connections-are-closed.test.ts:
--------------------------------------------------------------------------------
1 | import { waitForEvent } from 'streamr-test-utils'
2 |
3 | import { DisconnectionReason } from '../../src/connection/IWsEndpoint'
4 | import { startEndpoint, WsEndpoint } from '../../src/connection/WsEndpoint'
5 | import { PeerInfo } from '../../src/connection/PeerInfo'
6 |
7 | describe('duplicate connections are closed', () => {
8 | let wsEndpoint1: WsEndpoint
9 | let wsEndpoint2: WsEndpoint
10 |
11 | beforeEach(async () => {
12 | wsEndpoint1 = await startEndpoint('127.0.0.1', 28501, PeerInfo.newNode('wsEndpoint1'), null)
13 | wsEndpoint2 = await startEndpoint('127.0.0.1', 28502, PeerInfo.newNode('wsEndpoint2'), null)
14 | })
15 |
16 | afterAll(async () => {
17 | await wsEndpoint1.stop()
18 | await wsEndpoint2.stop()
19 | })
20 |
21 | test('if two endpoints open a connection (socket) to each other concurrently, one of them should be closed', async () => {
22 | const connectionsClosedReasons: string[] = []
23 |
24 | await Promise.allSettled([
25 | wsEndpoint1.connect('ws://127.0.0.1:28502'),
26 | wsEndpoint2.connect('ws://127.0.0.1:28501')
27 | ])
28 |
29 | await Promise.race([
30 | waitForEvent(wsEndpoint1, 'close'),
31 | waitForEvent(wsEndpoint2, 'close')
32 | ]).then((res) => {
33 | const reason: any = res[2]
34 | connectionsClosedReasons.push(reason)
35 | return res
36 | })
37 |
38 | expect(connectionsClosedReasons).toEqual([DisconnectionReason.DUPLICATE_SOCKET]) // length === 1
39 |
40 | // to be sure that everything wrong happened
41 | expect(wsEndpoint1.getPeers().size).toEqual(1)
42 | expect(wsEndpoint2.getPeers().size).toEqual(1)
43 | })
44 | })
45 |
--------------------------------------------------------------------------------
/test/unit/trackerSummaryUtils.test.ts:
--------------------------------------------------------------------------------
1 | import { getNodeConnections } from '../../src/logic/trackerSummaryUtils'
2 | import { OverlayTopology } from '../../src/logic/OverlayTopology'
3 |
4 | const createOverlayTopology = (mapping: { [key: string]: string[] }) => {
5 | const overlayTopology = new OverlayTopology(4)
6 | Object.entries(mapping).forEach(([nodeId, neighbors]) => {
7 | // Inform tracker of existence of neighbor
8 | neighbors.forEach((neighbor) => {
9 | overlayTopology.update(neighbor, [])
10 | })
11 | overlayTopology.update(nodeId, neighbors)
12 | })
13 | return overlayTopology
14 | }
15 |
16 | test('getNodeConnections', () => {
17 | const nodes = ['node1', 'node2', 'node3', 'node4', 'node5', 'node6', 'nodeNotInTopology']
18 | const overlayPerStream = {
19 | 'stream-a::0': createOverlayTopology({
20 | node1: ['node2', 'node3']
21 | }),
22 | 'stream-b::0': createOverlayTopology({
23 | node2: ['node4']
24 | }),
25 | 'stream-c::0': createOverlayTopology({}),
26 | 'stream-d::0': createOverlayTopology({
27 | node1: ['node3', 'node5']
28 | }),
29 | 'stream-e::0': createOverlayTopology({
30 | node6: []
31 | })
32 | }
33 | const result = getNodeConnections(nodes, overlayPerStream)
34 | expect(Object.keys(result)).toEqual(nodes)
35 | expect(result.node1).toEqual(new Set(['node2', 'node3', 'node5']))
36 | expect(result.node2).toEqual(new Set(['node1', 'node4']))
37 | expect(result.node3).toEqual(new Set(['node1']))
38 | expect(result.node4).toEqual(new Set(['node2']))
39 | expect(result.node5).toEqual(new Set(['node1']))
40 | expect(result.node6).toEqual(new Set([]))
41 | expect(result.nodeNotInTopology).toEqual(new Set([]))
42 | })
43 |
--------------------------------------------------------------------------------
/test/unit/NegotiatedProtocolVersions.test.ts:
--------------------------------------------------------------------------------
1 | import { NegotiatedProtocolVersions } from '../../src/connection/NegotiatedProtocolVersions'
2 | import { PeerInfo } from '../../src/connection/PeerInfo'
3 |
4 | describe('NegotiatedProtocolVersions', () => {
5 | let negotiatedProtocolVersions: NegotiatedProtocolVersions
6 |
7 | beforeEach(() => {
8 | const peerInfo = PeerInfo.newNode('node', null, [1,2], [30,31,32])
9 | negotiatedProtocolVersions = new NegotiatedProtocolVersions(peerInfo)
10 | negotiatedProtocolVersions.negotiateProtocolVersion('peer2', [1,2,3,4,5], [29,30,31,32,33])
11 | negotiatedProtocolVersions.negotiateProtocolVersion('peer3', [1,5], [29,31])
12 | })
13 |
14 | it('negotiates versions as expected', () => {
15 | expect(negotiatedProtocolVersions.getNegotiatedProtocolVersions('peer2')).toEqual({
16 | controlLayerVersion: 2,
17 | messageLayerVersion: 32
18 | })
19 | expect(negotiatedProtocolVersions.getNegotiatedProtocolVersions('peer3')).toEqual({
20 | controlLayerVersion: 1,
21 | messageLayerVersion: 31
22 | })
23 | })
24 |
25 | it('error is thrown if version negotiation is unsuccessful', () => {
26 | expect(() => negotiatedProtocolVersions.negotiateProtocolVersion(
27 | 'faulty',
28 | [8,9],
29 | [33])
30 | ).toThrow('Supported ControlLayer versions: [1,2]. Are you using an outdated library?')
31 | })
32 |
33 | it('non-existent peerId get request returns undefined', () => {
34 | expect(negotiatedProtocolVersions.getNegotiatedProtocolVersions('peer5')).toEqual(undefined)
35 | })
36 |
37 | it('negotiated versions are removed successfully', () => {
38 | negotiatedProtocolVersions.removeNegotiatedProtocolVersion('peer2')
39 | expect(negotiatedProtocolVersions.getNegotiatedProtocolVersions('peer2')).toEqual(undefined)
40 | })
41 | })
42 |
--------------------------------------------------------------------------------
/bin/network.js:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | const { spawn } = require('child_process')
3 | const path = require('path')
4 |
5 | const program = require('commander')
6 |
7 | const { version: CURRENT_VERSION } = require('../package.json')
8 |
9 | program
10 | .version(CURRENT_VERSION)
11 | .option('--nodes ', 'number of nodes', '10')
12 | .option('--streams ', 'number of streams', '1')
13 | .description('Run local network with stream (-s)')
14 | .parse(process.argv)
15 |
16 | const { nodes: numberOfNodes } = program.opts()
17 | const startingPort = 30000
18 | const trackerPort = 27777
19 | const startingDebugPort = 9200
20 | const streams = []
21 |
22 | for (let i = 0; i < parseInt(program.opts().streams, 10); i++) {
23 | streams.push(`stream-${i}`)
24 | }
25 |
26 | let debug = false
27 |
28 | const productionEnv = Object.create(process.env)
29 | productionEnv.LOG_LEVEL = productionEnv.LOG_LEVEL || 'debug'
30 |
31 | // create tracker
32 | const tracker = path.resolve('./bin/tracker.js')
33 | let args = [tracker, '--port=' + trackerPort]
34 |
35 | if (process.env.NODE_DEBUG_OPTION !== undefined) {
36 | debug = true
37 | args.unshift('--inspect-brk=' + (startingDebugPort - 1))
38 | }
39 |
40 | spawn('node', args, {
41 | env: productionEnv,
42 | stdio: [process.stdin, process.stdout, process.stderr]
43 | })
44 |
45 | setTimeout(() => {
46 | for (let i = 0; i < parseInt(numberOfNodes, 10); i++) {
47 | args = [
48 | path.resolve('./bin/subscriber.js'),
49 | '--streamIds=' + streams,
50 | '--port=' + (startingPort + i),
51 | `--trackers=ws://127.0.0.1:${trackerPort}`
52 | ]
53 |
54 | if (debug) {
55 | args.unshift('--inspect-brk=' + (startingDebugPort + i))
56 | }
57 |
58 | spawn('node', args, {
59 | env: productionEnv,
60 | stdio: [process.stdin, process.stdout, process.stderr]
61 | })
62 | }
63 | }, 1000)
64 |
--------------------------------------------------------------------------------
/src/connection/IWsEndpoint.ts:
--------------------------------------------------------------------------------
1 | import {PeerInfo} from './PeerInfo'
2 | import { Rtts } from '../identifiers'
3 |
4 | export enum Event {
5 | PEER_CONNECTED = 'streamr:peer:connect',
6 | PEER_DISCONNECTED = 'streamr:peer:disconnect',
7 | MESSAGE_RECEIVED = 'streamr:message-received',
8 | HIGH_BACK_PRESSURE = 'streamr:high-back-pressure',
9 | LOW_BACK_PRESSURE = 'streamr:low-back-pressure'
10 | }
11 |
12 | export enum DisconnectionCode {
13 | GRACEFUL_SHUTDOWN = 1000,
14 | DUPLICATE_SOCKET = 1002,
15 | NO_SHARED_STREAMS = 1000,
16 | MISSING_REQUIRED_PARAMETER = 1002,
17 | DEAD_CONNECTION = 1002
18 | }
19 |
20 | export enum DisconnectionReason {
21 | GRACEFUL_SHUTDOWN = 'streamr:node:graceful-shutdown',
22 | DUPLICATE_SOCKET = 'streamr:endpoint:duplicate-connection',
23 | NO_SHARED_STREAMS = 'streamr:node:no-shared-streams',
24 | MISSING_REQUIRED_PARAMETER = 'streamr:node:missing-required-parameter',
25 | DEAD_CONNECTION = 'streamr:endpoint:dead-connection'
26 | }
27 |
28 | export interface IWsEndpoint {
29 |
30 | on(event: Event.PEER_CONNECTED, listener: (peerInfo: PeerInfo) => void): this
31 | on(event: Event.PEER_DISCONNECTED, listener: (peerInfo: PeerInfo, reason: string) => void): this
32 | on(event: Event.MESSAGE_RECEIVED, listener: (peerInfo: PeerInfo, message: string) => void): this
33 | on(event: Event.HIGH_BACK_PRESSURE, listener: (peerInfo: PeerInfo) => void): this
34 | on(event: Event.LOW_BACK_PRESSURE, listener: (peerInfo: PeerInfo) => void): this
35 |
36 | connect(peerAddress: string): Promise
37 | send(recipientId: string, message: string): Promise
38 | close(recipientId: string, reason: DisconnectionReason): void
39 | getRtts(): Rtts
40 | getPeerInfo(): Readonly
41 | getAddress(): string
42 | stop(): Promise
43 |
44 | isConnected(address: string): boolean
45 | getPeerInfos(): PeerInfo[]
46 | resolveAddress(peerId: string): string | never
47 | }
48 |
--------------------------------------------------------------------------------
/src/logic/LocationManager.ts:
--------------------------------------------------------------------------------
1 | import { lookup, Lookup } from 'geoip-lite'
2 | import { Logger } from '../helpers/Logger'
3 | import { Location } from '../identifiers'
4 |
5 | function isValidNodeLocation(location: Location | null) {
6 | return location && (location.country || location.city || location.latitude || location.longitude)
7 | }
8 |
9 | export class LocationManager {
10 | private readonly nodeLocations: {
11 | [key: string]: Location // nodeId => Location
12 | }
13 | private readonly logger: Logger
14 |
15 | constructor() {
16 | this.nodeLocations = {}
17 | this.logger = new Logger(module)
18 | }
19 |
20 | getAllNodeLocations(): Readonly<{[key: string]: Location}> {
21 | return this.nodeLocations
22 | }
23 |
24 | getNodeLocation(nodeId: string): Location {
25 | return this.nodeLocations[nodeId]
26 | }
27 |
28 | updateLocation({ nodeId, location, address }: { nodeId: string, location: Location | null, address: string }): void {
29 | if (isValidNodeLocation(location)) {
30 | this.nodeLocations[nodeId] = location!
31 | } else if (!isValidNodeLocation(this.nodeLocations[nodeId])) {
32 | let geoIpRecord: null | Lookup = null
33 | if (address) {
34 | try {
35 | const ip = address.split(':')[1].replace('//', '')
36 | geoIpRecord = lookup(ip)
37 | } catch (e) {
38 | this.logger.warn('could not parse IP from address %s', address)
39 | }
40 | }
41 | if (geoIpRecord) {
42 | this.nodeLocations[nodeId] = {
43 | country: geoIpRecord.country,
44 | city: geoIpRecord.city,
45 | latitude: geoIpRecord.ll[0],
46 | longitude: geoIpRecord.ll[1]
47 | }
48 | }
49 | }
50 | }
51 |
52 | removeNode(nodeId: string): void {
53 | delete this.nodeLocations[nodeId]
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/test/integration/ws-endpoint-back-pressure-handling.test.ts:
--------------------------------------------------------------------------------
1 | import { Event } from '../../src/connection/IWsEndpoint'
2 | import { startEndpoint, WsEndpoint } from '../../src/connection/WsEndpoint'
3 | import { PeerInfo } from '../../src/connection/PeerInfo'
4 |
5 | describe('WsEndpoint: back pressure handling', () => {
6 | let ep1: WsEndpoint
7 | let ep2: WsEndpoint
8 |
9 | beforeEach(async () => {
10 | ep1 = await startEndpoint('127.0.0.1', 43974, PeerInfo.newNode('ep1'), null)
11 | ep2 = await startEndpoint('127.0.0.1', 43975, PeerInfo.newNode('ep2'), null)
12 | await ep1.connect('ws://127.0.0.1:43975')
13 | })
14 |
15 | afterEach(async () => {
16 | Promise.allSettled([
17 | ep1.stop(),
18 | ep2.stop()
19 | ])
20 | })
21 |
22 | it('emits HIGH_BACK_PRESSURE on high back pressure', (done) => {
23 | let hitHighBackPressure = false
24 | ep1.on(Event.HIGH_BACK_PRESSURE, (peerInfo) => {
25 | hitHighBackPressure = true
26 | expect(peerInfo).toEqual(PeerInfo.newNode('ep2'))
27 | done()
28 | })
29 | while (!hitHighBackPressure) {
30 | ep1.send('ep2', 'aaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbccccccccccccccccdddddddddddeeeeeeeeffffff')
31 | }
32 | })
33 |
34 | it('emits LOW_BACK_PRESSURE after high back pressure', (done) => {
35 | let hitHighBackPressure = false
36 | let sendInterval: ReturnType | null = null
37 | ep1.on(Event.HIGH_BACK_PRESSURE, () => {
38 | hitHighBackPressure = true
39 |
40 | // drain doesn't seem to work, need to send _evaluateBackPressure
41 | sendInterval = setInterval(() => ep1.send('ep2', 'aaaa'), 30)
42 |
43 | ep1.on(Event.LOW_BACK_PRESSURE, (peerInfo) => {
44 | expect(peerInfo).toEqual(PeerInfo.newNode('ep2'))
45 | clearInterval(sendInterval!)
46 | done()
47 | })
48 | })
49 | while (!hitHighBackPressure) {
50 | ep1.send('ep2', 'aaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbccccccccccccccccdddddddddddeeeeeeeeffffff')
51 | }
52 | })
53 | })
54 |
--------------------------------------------------------------------------------
/src/helpers/MessageBuffer.ts:
--------------------------------------------------------------------------------
1 | interface Buffer {
2 | [key: string]: Array
3 | }
4 |
5 | interface Timeouts {
6 | [key: string]: Array
7 | }
8 |
9 | export class MessageBuffer {
10 | private readonly buffer: Buffer = {}
11 | private readonly timeoutRefs: Timeouts = {}
12 | private readonly timeoutInMs: number
13 | private readonly maxSize: number
14 | private readonly onTimeout: (id: string) => void
15 |
16 | constructor(timeoutInMs: number, maxSize = 10000, onTimeout = (_id: string) => {}) {
17 | this.timeoutInMs = timeoutInMs
18 | this.maxSize = maxSize
19 | this.onTimeout = onTimeout
20 | }
21 |
22 | put(id: string, message: M): void {
23 | if (!this.buffer[id]) {
24 | this.buffer[id] = []
25 | this.timeoutRefs[id] = []
26 | }
27 |
28 | if (this.buffer[id].length >= this.maxSize) {
29 | this.pop(id)
30 | }
31 |
32 | this.buffer[id].push(message)
33 | this.timeoutRefs[id].push(setTimeout(() => {
34 | this.pop(id)
35 | this.onTimeout(id)
36 | }, this.timeoutInMs))
37 | }
38 |
39 | pop(id: string): M | null {
40 | if (this.buffer[id]) {
41 | const message = this.buffer[id].shift()!
42 | const ref = this.timeoutRefs[id].shift()!
43 | clearTimeout(ref)
44 |
45 | if (!this.buffer[id].length) {
46 | delete this.buffer[id]
47 | }
48 |
49 | return message
50 | }
51 | return null
52 | }
53 |
54 | popAll(id: string): Array {
55 | if (this.buffer[id]) {
56 | const messages = this.buffer[id]
57 | this.timeoutRefs[id].forEach((ref) => clearTimeout(ref))
58 | delete this.timeoutRefs[id]
59 | delete this.buffer[id]
60 | return messages
61 | }
62 | return []
63 | }
64 |
65 | clear(): void {
66 | Object.keys(this.buffer).forEach((id) => this.popAll(id))
67 | }
68 |
69 | size(): number {
70 | let total = 0
71 | Object.values(this.buffer).forEach((messages) => {
72 | total += messages.length
73 | })
74 | return total
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/test/benchmarks/overlay-topology-randomness.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-console */
2 | import { OverlayTopology, TopologyState } from '../../src/logic/OverlayTopology'
3 |
4 | const numOfNeighbors = 4
5 | const numOfRounds = 1000
6 | const numOfNodes = 1000
7 | const printProgress = true
8 |
9 | const idxToNodeId = (idx: number) => `${idx + 1}`
10 | const nodeIdToIdx = (nodeId: string) => Number.parseInt(nodeId, 10) - 1
11 |
12 | // Run topology experiment
13 | const states: TopologyState[] = []
14 | for (let i = 0; i < numOfRounds; ++i) {
15 | const topology = new OverlayTopology(numOfNeighbors)
16 |
17 | for (let j = 0; j < numOfNodes; ++j) {
18 | const nodeId = idxToNodeId(j)
19 | topology.update(nodeId, [])
20 | topology.formInstructions(nodeId)
21 | }
22 |
23 | states.push(topology.state())
24 | if (printProgress && i % 100 === 0) {
25 | console.warn(`Running topology experiment... ${Math.round((i / numOfRounds) * 100)}%`)
26 | }
27 | }
28 |
29 | /*
30 | // Print raw data as CSV
31 | console.info('round,node,neighbor')
32 | states.forEach((state, round) => {
33 | Object.entries(state).forEach(([nodeId, neighbors]) => {
34 | neighbors.forEach((neighbor) => {
35 | console.info([round, nodeId, neighbor].join(","))
36 | })
37 | })
38 | })
39 | return
40 | */
41 |
42 | // Set up occurrence matrix filled with zeroes
43 | const occurrenceMatrix: number[][] = []
44 | for (let i = 0; i < numOfNodes; ++i) {
45 | occurrenceMatrix[i] = []
46 | for (let j = 0; j < numOfNodes; ++j) {
47 | occurrenceMatrix[i][j] = 0
48 | }
49 | }
50 |
51 | // Tally up numbers
52 | states.forEach((state) => {
53 | Object.entries(state).forEach(([nodeId, neighbors]) => {
54 | const idx = nodeIdToIdx(nodeId)
55 | neighbors.forEach((neighbor) => {
56 | occurrenceMatrix[idx][nodeIdToIdx(neighbor)] += 1
57 | })
58 | })
59 | })
60 |
61 | // Print as grid
62 | console.info(`Pair-wise occurrences with rounds=${numOfRounds}, nodes=${numOfNodes}, neighbors=${numOfNeighbors}`)
63 | occurrenceMatrix.forEach((row) => {
64 | console.info(row.join(' '))
65 | })
66 |
67 | // Print summary statistics
68 | const expectedCount = (1 / (numOfNodes - 1)) * numOfNeighbors * numOfRounds
69 | console.info(`Expected count (per cell): ${expectedCount}`)
70 |
--------------------------------------------------------------------------------
/test/integration/tracker-storage-nodes-response-does-not-contain-self.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { waitForEvent } from 'streamr-test-utils'
3 |
4 | import { PeerInfo } from '../../src/connection/PeerInfo'
5 | import { startEndpoint } from '../../src/connection/WsEndpoint'
6 | import { TrackerNode, Event as TrackerNodeEvent } from '../../src/protocol/TrackerNode'
7 | import { startTracker } from '../../src/composition'
8 | import { StreamIdAndPartition } from '../../src/identifiers'
9 |
10 | /**
11 | * Ensure that when a storage node requests storage nodes from tracker, the list does not contain the
12 | * requesting storage node itself.
13 | */
14 | describe('storage nodes response from tracker does not contain self', () => {
15 | let tracker: Tracker
16 | let storageNodeOne: TrackerNode
17 | let storageNodeTwo: TrackerNode
18 | let storageNodeThree: TrackerNode
19 |
20 | beforeEach(async () => {
21 | tracker = await startTracker({
22 | host: '127.0.0.1',
23 | port: 30460,
24 | id: 'tracker'
25 | })
26 |
27 | const ep1 = await startEndpoint('127.0.0.1', 30461, PeerInfo.newStorage('storageNodeOne'), null)
28 | const ep2 = await startEndpoint('127.0.0.1', 30462, PeerInfo.newStorage('storageNodeTwo'), null)
29 | const ep3 = await startEndpoint('127.0.0.1', 30463, PeerInfo.newStorage('storageNodeThree'), null)
30 |
31 | storageNodeOne = new TrackerNode(ep1)
32 | storageNodeTwo = new TrackerNode(ep2)
33 | storageNodeThree = new TrackerNode(ep3)
34 |
35 | await storageNodeOne.connectToTracker(tracker.getAddress())
36 | await storageNodeTwo.connectToTracker(tracker.getAddress())
37 | await storageNodeThree.connectToTracker(tracker.getAddress())
38 | })
39 |
40 | afterEach(async () => {
41 | await Promise.all([
42 | storageNodeOne.stop(),
43 | storageNodeTwo.stop(),
44 | storageNodeThree.stop(),
45 | tracker.stop()
46 | ])
47 | })
48 |
49 | it('storage node response does not contain self', async () => {
50 | await storageNodeOne.sendStorageNodesRequest('tracker', new StreamIdAndPartition('stream', 0))
51 | const [msg]: any = await waitForEvent(storageNodeOne, TrackerNodeEvent.STORAGE_NODES_RESPONSE_RECEIVED)
52 | expect(msg.nodeIds).toEqual(['storageNodeTwo', 'storageNodeThree'])
53 | })
54 | })
55 |
--------------------------------------------------------------------------------
/test/integration/nodeMessageBuffering.test.ts:
--------------------------------------------------------------------------------
1 | import { NetworkNode } from '../../src/NetworkNode'
2 | import { MessageLayer } from 'streamr-client-protocol'
3 |
4 | import { startNetworkNode, startTracker, Tracker } from '../../src/composition'
5 |
6 | const { StreamMessage, MessageID } = MessageLayer
7 |
8 | /**
9 | * When a node receives a message for a stream it hasn't still subscribed to, it
10 | * subscribes to the stream and then asks the tracker who else is participating
11 | * in the stream. In this test we verify that the initial message that causes
12 | * this whole process is itself eventually delivered.
13 | */
14 | describe('message buffering of Node', () => {
15 | let tracker: Tracker
16 | let sourceNode: NetworkNode
17 | let destinationNode: NetworkNode
18 |
19 | beforeAll(async () => {
20 | tracker = await startTracker({
21 | host: '127.0.0.1',
22 | port: 30320,
23 | id: 'tracker'
24 | })
25 |
26 | sourceNode = await startNetworkNode({
27 | host: '127.0.0.1',
28 | port: 30321,
29 | id: 'source-node',
30 | trackers: [tracker.getAddress()]
31 | })
32 | destinationNode = await startNetworkNode({
33 | host: '127.0.0.1',
34 | port: 30322,
35 | id: 'destination-node',
36 | trackers: [tracker.getAddress()]
37 | })
38 |
39 | sourceNode.start()
40 | destinationNode.start()
41 | })
42 |
43 | afterAll(async () => {
44 | await sourceNode.stop()
45 | await destinationNode.stop()
46 | await tracker.stop()
47 | })
48 |
49 | test('first message to unknown stream eventually gets delivered', (done) => {
50 | destinationNode.addMessageListener((streamMessage) => {
51 | expect(streamMessage.messageId).toEqual(
52 | new MessageID('id', 0, 1, 0, 'publisher-id', 'session-id')
53 | )
54 | expect(streamMessage.getParsedContent()).toEqual({
55 | hello: 'world'
56 | })
57 | done()
58 | })
59 |
60 | destinationNode.subscribe('id', 0)
61 |
62 | // "Client" pushes data
63 | sourceNode.publish(new StreamMessage({
64 | messageId: new MessageID('id', 0, 1, 0, 'publisher-id', 'session-id'),
65 | content: {
66 | hello: 'world'
67 | },
68 | }))
69 | })
70 | })
71 |
--------------------------------------------------------------------------------
/bin/tracker.js:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | const program = require('commander')
3 |
4 | const { Logger } = require('../dist/helpers/Logger')
5 | const { version: CURRENT_VERSION } = require('../package.json')
6 | const { startTracker } = require('../dist/composition')
7 | const { MetricsContext } = require('../dist/helpers/MetricsContext')
8 |
9 | program
10 | .version(CURRENT_VERSION)
11 | .option('--id ', 'Ethereum address / tracker id', undefined)
12 | .option('--trackerName ', 'Human readable name', undefined)
13 | .option('--port ', 'port', '27777')
14 | .option('--ip ', 'ip', '0.0.0.0')
15 | .option('--maxNeighborsPerNode ', 'maxNeighborsPerNode', '4')
16 | .option('--metrics ', 'output metrics to console', false)
17 | .option('--metricsInterval ', 'metrics output interval (ms)', '5000')
18 | .description('Run tracker with reporting')
19 | .parse(process.argv)
20 |
21 | const id = program.opts().id || `TR${program.opts().port}`
22 | const name = program.opts().trackerName || id
23 | const logger = new Logger(module)
24 |
25 | async function main() {
26 | const metricsContext = new MetricsContext(id)
27 | try {
28 | await startTracker({
29 | host: program.opts().ip,
30 | port: Number.parseInt(program.opts().port, 10),
31 | id,
32 | name,
33 | maxNeighborsPerNode: Number.parseInt(program.opts().maxNeighborsPerNode, 10),
34 | metricsContext
35 | })
36 |
37 | const trackerObj = {}
38 | const fields = ['ip', 'port', 'maxNeighborsPerNode', 'metrics', 'metricsInterval']
39 | fields.forEach((prop) => {
40 | trackerObj[prop] = program.opts()[prop]
41 | })
42 |
43 | logger.info('started tracker: %o', {
44 | id,
45 | name,
46 | ...trackerObj
47 | })
48 |
49 | if (program.opts().metrics) {
50 | setInterval(async () => {
51 | const metrics = await metricsContext.report(true)
52 | // output to console
53 | if (program.opts().metrics) {
54 | logger.info(JSON.stringify(metrics, null, 4))
55 | }
56 | }, program.opts().metricsInterval)
57 | }
58 | } catch (err) {
59 | pino.final(logger).error(err, 'tracker bin catch')
60 | process.exit(1)
61 | }
62 | }
63 |
64 | main()
65 |
--------------------------------------------------------------------------------
/test/integration/killing-dead-connections.test.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-underscore-dangle */
2 | import { waitForEvent } from 'streamr-test-utils'
3 |
4 | import { Event, DisconnectionReason, DisconnectionCode } from '../../src/connection/IWsEndpoint'
5 | import { startEndpoint, WsEndpoint } from '../../src/connection/WsEndpoint'
6 | import { PeerInfo } from '../../src/connection/PeerInfo'
7 |
8 | const STATE_OPEN = 1
9 | const STATE_CLOSING = 2
10 |
11 | describe('check and kill dead connections', () => {
12 | let node1: WsEndpoint
13 | let node2: WsEndpoint
14 |
15 | beforeEach(async () => {
16 | node1 = await startEndpoint('127.0.0.1', 43971, PeerInfo.newNode('node1'), null)
17 | node2 = await startEndpoint('127.0.0.1', 43972, PeerInfo.newNode('node2'), null)
18 |
19 | node1.connect('ws://127.0.0.1:43972')
20 | await waitForEvent(node1, Event.PEER_CONNECTED)
21 | })
22 |
23 | afterEach(async () => {
24 | Promise.allSettled([
25 | node1.stop(),
26 | node2.stop()
27 | ])
28 | })
29 |
30 | it('if we find dead connection, we force close it', async () => {
31 | expect(node1.getPeers().size).toBe(1)
32 |
33 | // get alive connection
34 | const connection = node1.getPeers().get('ws://127.0.0.1:43972')
35 | expect(connection!.readyState).toEqual(STATE_OPEN)
36 |
37 | // @ts-expect-error private method
38 | jest.spyOn(node1, 'onClose').mockImplementation()
39 |
40 | // check connections
41 | jest.spyOn(connection!, 'ping').mockImplementation(() => {
42 | throw new Error('mock error message')
43 | })
44 | // @ts-expect-error private method
45 | node1.pingConnections()
46 |
47 | expect(connection!.readyState).toEqual(STATE_CLOSING)
48 |
49 | // @ts-expect-error private method
50 | expect(node1.onClose).toBeCalledTimes(1)
51 | // @ts-expect-error private method
52 | expect(node1.onClose).toBeCalledWith('ws://127.0.0.1:43972', PeerInfo.newNode('node2'),
53 | DisconnectionCode.DEAD_CONNECTION, DisconnectionReason.DEAD_CONNECTION)
54 |
55 | // @ts-expect-error private method
56 | node1.onClose.mockRestore()
57 | // @ts-expect-error private method
58 | node1.pingConnections()
59 |
60 | const [peerInfo] = await waitForEvent(node1, Event.PEER_DISCONNECTED)
61 | expect(peerInfo).toEqual(PeerInfo.newNode('node2'))
62 | })
63 | })
64 |
--------------------------------------------------------------------------------
/.github/workflows/nodejs.yml:
--------------------------------------------------------------------------------
1 | name: Eslint, Test and Publish
2 |
3 | on:
4 | push:
5 | paths-ignore:
6 | - 'README.md'
7 | - 'LICENSE'
8 | - 'typedoc.js'
9 | - '.idea'
10 |
11 | jobs:
12 | test:
13 | name: Run eslint and test
14 | runs-on: ubuntu-latest
15 |
16 | strategy:
17 | matrix:
18 | node-version: [ 14.x ]
19 |
20 | steps:
21 | - uses: actions/checkout@v2
22 | - name: Use Node.js ${{ matrix.node-version }}
23 | uses: actions/setup-node@v1
24 | with:
25 | node-version: ${{ matrix.node-version }}
26 | - name: Cache Node.js modules
27 | uses: actions/cache@v2
28 | with:
29 | path: ~/.npm
30 | key: ${{ runner.OS }}-node-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }}
31 | restore-keys: |
32 | ${{ runner.OS }}-node-${{ matrix.node-version }}
33 | ${{ runner.OS }}-node-
34 | ${{ runner.OS }}-
35 | - run: npm ci
36 | - run: npm run eslint
37 | - run: npm run test-types
38 | - run: npm run build --if-present
39 | - run: npm test
40 | env:
41 | CI: true
42 |
43 | publish:
44 | needs: [test]
45 | name: Publishing master using Node 14
46 | runs-on: ubuntu-latest
47 |
48 | # run job only for tags and skip for cron
49 | if: startsWith(github.ref, 'refs/tags/') && github.event_name != 'schedule'
50 | steps:
51 | - name: Get the release version
52 | id: get_version
53 | run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
54 |
55 | - uses: actions/checkout@master
56 | - uses: actions/setup-node@v1
57 | with:
58 | node-version: 14
59 | registry-url: https://registry.npmjs.org/
60 |
61 | - name: npm ci
62 | run: |
63 | npm ci
64 | npm run build --if-present
65 |
66 | - name: Publish beta ${{ steps.get_version.outputs.VERSION }}
67 | # if tag includes beta keyword
68 | if: contains(steps.get_version.outputs.VERSION, 'beta') == true
69 | run: npm publish --tag beta
70 | env:
71 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
72 |
73 | - name: Publish latest ${{ steps.get_version.outputs.VERSION }}
74 | # if tag doesn't include beta keyword
75 | if: contains(steps.get_version.outputs.VERSION, 'beta') == false
76 | run: npm publish
77 | env:
78 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
79 |
--------------------------------------------------------------------------------
/test/integration/network-stabilization.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { NetworkNode } from '../../src/NetworkNode'
3 | import assert from 'assert'
4 |
5 | import { wait } from 'streamr-test-utils'
6 |
7 | import { startNetworkNode, startTracker } from '../../src/composition'
8 | import { getTopology } from '../../src/logic/trackerSummaryUtils'
9 |
10 | function areEqual(a: any, b: any) {
11 | try {
12 | assert.deepStrictEqual(a, b)
13 | } catch (error) {
14 | if (error.code === 'ERR_ASSERTION') {
15 | return false
16 | }
17 | throw error
18 | }
19 | return true
20 | }
21 |
22 | describe('check network stabilization', () => {
23 | let tracker: Tracker
24 | let nodes: NetworkNode[]
25 | const MAX_NODES = 10
26 | const startingPort = 39001
27 |
28 | beforeEach(async () => {
29 | tracker = await startTracker({
30 | host: '127.0.0.1',
31 | port: 39000,
32 | id: 'tracker'
33 | })
34 |
35 | nodes = []
36 | for (let i = 0; i < MAX_NODES; i++) {
37 | // eslint-disable-next-line no-await-in-loop
38 | const node = await startNetworkNode({
39 | host: '127.0.0.1',
40 | port: startingPort + i,
41 | id: `node-${i}`,
42 | trackers: [tracker.getAddress()]
43 | })
44 | node.subscribe('stream', 0)
45 | nodes.push(node)
46 | }
47 | nodes.forEach((node) => node.start())
48 | })
49 |
50 | afterEach(async () => {
51 | await Promise.allSettled([
52 | tracker.stop(),
53 | ...nodes.map((node) => node.stop())
54 | ])
55 | })
56 |
57 | it('network must become stable in less than 10 seconds', () => {
58 | return new Promise(async (resolve, reject) => {
59 | for (let i = 0; i < 10; ++i) {
60 | const beforeTopology = getTopology(tracker.getOverlayPerStream(), tracker.getOverlayConnectionRtts())
61 | // eslint-disable-next-line no-await-in-loop
62 | await wait(800)
63 | const afterTopology = getTopology(tracker.getOverlayPerStream(), tracker.getOverlayConnectionRtts())
64 | if (areEqual(beforeTopology, afterTopology)) {
65 | resolve(true)
66 | return
67 | }
68 | }
69 | reject(new Error('did not stabilize'))
70 | })
71 | }, 11000)
72 | })
73 |
--------------------------------------------------------------------------------
/examples/system-metrics-pubsub/index.ts:
--------------------------------------------------------------------------------
1 | import os from 'os'
2 | import process from 'process'
3 | import { v4 } from 'uuid'
4 | import {
5 | startTracker,
6 | startNetworkNode,
7 | Protocol,
8 | Tracker,
9 | NetworkNode
10 | } from 'streamr-network'
11 |
12 | /**
13 | * Run a tracker that assist nodes in peer discovery.
14 | */
15 | function runTracker(): Promise {
16 | return startTracker({
17 | host: '127.0.0.1',
18 | port: 30300,
19 | id: 'tracker'
20 | })
21 | }
22 |
23 | /**
24 | * Run a publisher node that publishes system metrics to stream "system-report" every 2 seconds.
25 | */
26 | async function runPublisher(): Promise {
27 | const publisherNode: NetworkNode = await startNetworkNode({
28 | host: '127.0.0.1',
29 | port: 30301,
30 | id: 'publisherNode',
31 | trackers: ['ws://127.0.0.1:30300']
32 | })
33 | publisherNode.start()
34 |
35 | const streamId = 'system-report'
36 | const streamPartition = 0
37 | const sessionId = v4()
38 | let prevMsgRef: Protocol.MessageLayer.MessageRef | null = null
39 | let lastCpuUsage = process.cpuUsage()
40 |
41 | setInterval(() => {
42 | const timestamp = Date.now()
43 | const sequenceNo = 0
44 | const cpuUsage = process.cpuUsage(lastCpuUsage)
45 |
46 | const messageId = new Protocol.MessageLayer.MessageID(
47 | streamId,
48 | streamPartition,
49 | timestamp,
50 | sequenceNo,
51 | 'publisherNode',
52 | sessionId
53 | )
54 | publisherNode.publish(new Protocol.MessageLayer.StreamMessage({
55 | messageId,
56 | prevMsgRef,
57 | content: {
58 | hostname: os.hostname(),
59 | type: os.type(),
60 | release: os.release(),
61 | arch: os.arch(),
62 | loadAvg: os.loadavg(),
63 | upTime: os.uptime(),
64 | mem: {
65 | total: os.totalmem(),
66 | free: os.freemem()
67 | },
68 | process: {
69 | cpuUsage: cpuUsage,
70 | memUsage: process.memoryUsage()
71 | },
72 | }
73 | }))
74 | prevMsgRef = new Protocol.MessageLayer.MessageRef(timestamp, sequenceNo)
75 | lastCpuUsage = cpuUsage
76 | }, 2000)
77 | return publisherNode
78 | }
79 |
80 | async function main(): Promise {
81 | const tracker: Tracker = await runTracker()
82 | const publisherNode: NetworkNode = await runPublisher()
83 | }
84 | main().catch((err) => console.error(err))
85 |
--------------------------------------------------------------------------------
/test/unit/MessageQueue.test.ts:
--------------------------------------------------------------------------------
1 | import { MessageQueue } from '../../src/connection/MessageQueue'
2 | import { wait } from 'streamr-test-utils'
3 |
4 | describe(MessageQueue, () => {
5 | let messageQueue: MessageQueue
6 |
7 | beforeEach(() => {
8 | messageQueue = new MessageQueue(10)
9 | })
10 |
11 | it('starts out empty', () => {
12 | expect(messageQueue.size()).toEqual(0)
13 | expect(messageQueue.empty()).toEqual(true)
14 | })
15 |
16 | it('not empty after adding elements', () => {
17 | messageQueue.add('hello')
18 | messageQueue.add('world')
19 |
20 | expect(messageQueue.size()).toEqual(2)
21 | expect(messageQueue.empty()).toEqual(false)
22 | })
23 |
24 | it('peek does not drop message', () => {
25 | messageQueue.add('hello')
26 | messageQueue.add('world')
27 |
28 | expect(messageQueue.peek().getMessage()).toEqual('hello')
29 | expect(messageQueue.peek().getMessage()).toEqual('hello')
30 | expect(messageQueue.size()).toEqual(2)
31 | })
32 |
33 | it('preserves FIFO insertion order (add & pop)', () => {
34 | messageQueue.add('hello')
35 | messageQueue.add('world')
36 | messageQueue.add('!')
37 | messageQueue.add('lorem')
38 | messageQueue.add('ipsum')
39 | expect(messageQueue.pop().getMessage()).toEqual('hello')
40 | expect(messageQueue.pop().getMessage()).toEqual('world')
41 | expect(messageQueue.pop().getMessage()).toEqual('!')
42 | expect(messageQueue.pop().getMessage()).toEqual('lorem')
43 | expect(messageQueue.pop().getMessage()).toEqual('ipsum')
44 | })
45 |
46 | it('drops message in FIFO order when adding to full queue', async () => {
47 | const recordedErrors: {i: number, err: Error}[] = []
48 | for (let i = 1; i <= 10; ++i) {
49 | messageQueue.add(`message ${i}`).catch((err: Error) => {
50 | recordedErrors.push({
51 | i,
52 | err
53 | })
54 | })
55 | }
56 | messageQueue.add('message 11')
57 | messageQueue.add('message 12')
58 | await wait(0) // yield execution to error handlers
59 |
60 | expect(messageQueue.size()).toEqual(10)
61 | expect(messageQueue.peek().getMessage()).toEqual('message 3')
62 | expect(recordedErrors).toEqual([
63 | {
64 | i: 1,
65 | err: new Error('Message queue full, dropping message.')
66 | },
67 | {
68 | i: 2,
69 | err: new Error('Message queue full, dropping message.')
70 | }
71 | ])
72 | })
73 | })
74 |
--------------------------------------------------------------------------------
/src/helpers/Logger.ts:
--------------------------------------------------------------------------------
1 | import pino from 'pino'
2 | import path from 'path'
3 | import _ from 'lodash'
4 |
5 | const parseBoolean = (value: string|undefined) => {
6 | switch (value) {
7 | case 'true':
8 | return true
9 | case 'false':
10 | return false
11 | case undefined:
12 | return undefined
13 | default:
14 | throw new Error('Invalid boolean value: ${value}')
15 | }
16 | }
17 |
18 | export class Logger {
19 |
20 | static NAME_LENGTH = 20
21 |
22 | private readonly logger: pino.Logger
23 |
24 | constructor(module: NodeJS.Module, context?: string) {
25 | this.logger = pino({
26 | name: Logger.createName(module, context),
27 | enabled: !process.env.NOLOG,
28 | level: process.env.LOG_LEVEL || 'info',
29 | prettyPrint: process.env.NODE_ENV === 'production' ? false : {
30 | colorize: parseBoolean(process.env.LOG_COLORS) ?? true,
31 | translateTime: 'yyyy-mm-dd"T"HH:MM:ss.l',
32 | ignore: 'pid,hostname',
33 | levelFirst: true,
34 | }
35 | })
36 | }
37 |
38 | private static createName(module: NodeJS.Module, context?: string) {
39 | const parsedPath = path.parse(module.filename)
40 | let fileId = parsedPath.name
41 | if (fileId === 'index') {
42 | // file with name "foobar/index.ts" -> "foobar"
43 | const parts = parsedPath.dir.split(path.sep)
44 | fileId = parts[parts.length - 1]
45 | }
46 | const appId = process.env.STREAMR_APPLICATION_ID
47 | const longName = _.without([appId, fileId, context], undefined).join(':')
48 | return _.padEnd(longName.substring(0, Logger.NAME_LENGTH), Logger.NAME_LENGTH, ' ')
49 | }
50 |
51 | fatal(msg: string, ...args: any[]): void {
52 | this.logger.fatal(msg, ...args)
53 | }
54 |
55 | error(msg: string, ...args: any[]): void {
56 | this.logger.error(msg, ...args)
57 | }
58 |
59 | warn(msg: string, ...args: any[]): void {
60 | this.logger.warn(msg, ...args)
61 | }
62 |
63 | info(msg: string, ...args: any[]): void {
64 | this.logger.info(msg, ...args)
65 | }
66 |
67 | debug(msg: string, ...args: any[]): void {
68 | this.logger.debug(msg, ...args)
69 | }
70 |
71 | trace(msg: string, ...args: any[]): void {
72 | this.logger.trace(msg, ...args)
73 | }
74 |
75 | getFinalLogger(): { error: (error: any, origin?: string) => void } {
76 | const finalLogger = pino.final(this.logger)
77 | return {
78 | error: (error: any, origin?: string) => finalLogger.error(error, origin)
79 | }
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/bin/subscriber.js:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | const program = require('commander')
3 |
4 | const { Logger } = require('../dist/helpers/Logger')
5 | const { version: CURRENT_VERSION } = require('../package.json')
6 | const { startNetworkNode } = require('../dist/composition')
7 | const { MetricsContext } = require('../dist/helpers/MetricsContext')
8 | const { Event: NodeEvent } = require('../dist/logic/Node')
9 |
10 | program
11 | .version(CURRENT_VERSION)
12 | .option('--id ', 'Ethereum address / node id', undefined)
13 | .option('--nodeName ', 'Human readble name for node', undefined)
14 | .option('--port ', 'port', '30304')
15 | .option('--ip ', 'ip', '127.0.0.1')
16 | .option('--trackers ', 'trackers', (value) => value.split(','), ['ws://127.0.0.1:27777'])
17 | .option('--streamIds ', 'streamId to publish', (value) => value.split(','), ['stream-0'])
18 | .option('--metrics ', 'log metrics', false)
19 | .description('Run subscriber')
20 | .parse(process.argv)
21 |
22 | const id = program.opts().id || `SU${program.opts().port}`
23 | const name = program.opts().nodeName || id
24 | const logger = new Logger(module)
25 | const metricsContext = new MetricsContext(id)
26 | startNetworkNode({
27 | host: program.opts().ip,
28 | port: program.opts().port,
29 | name,
30 | id,
31 | trackers: program.opts().trackers,
32 | metricsContext
33 | }).then((subscriber) => {
34 | logger.info('started subscriber id: %s, name: %s, port: %d, ip: %s, trackers: %s, streamId: %s, metrics: %s',
35 | id, name, program.opts().port, program.opts().ip, program.opts().trackers.join(', '), program.opts().streamId, program.opts().metrics)
36 | subscriber.start()
37 | program.opts().streamIds.forEach((stream) => subscriber.subscribe(stream, 0))
38 |
39 | let messageNo = 0
40 | let lastReported = 0
41 | subscriber.on(NodeEvent.UNSEEN_MESSAGE_RECEIVED, (streamMessage) => {
42 | messageNo += 1
43 | logger.info('received %j, data %j', streamMessage.getMsgChainId(), streamMessage.getParsedContent())
44 | })
45 |
46 | setInterval(() => {
47 | const newMessages = messageNo - lastReported
48 | logger.info('%s received %d (%d)', id, messageNo, newMessages)
49 | lastReported = messageNo
50 | }, 60 * 1000)
51 |
52 | if (program.opts().metrics) {
53 | setInterval(async () => {
54 | logger.info(JSON.stringify(await metricsContext.report(true), null, 3))
55 | }, 5000)
56 | }
57 |
58 | return true
59 | }).catch((err) => {
60 | throw err
61 | })
62 |
63 | if (process.env.checkUncaughtException === 'true') {
64 | process.on('uncaughtException', (err) => console.error((err && err.stack) ? err.stack : err))
65 | }
66 |
--------------------------------------------------------------------------------
/test/unit/PeerInfo.test.ts:
--------------------------------------------------------------------------------
1 | import { PeerInfo } from '../../src/connection/PeerInfo'
2 |
3 | describe('PeerInfo', () => {
4 | let nodeInfo: PeerInfo
5 | let storageInfo: PeerInfo
6 | let trackerInfo: PeerInfo
7 | let unknownInfo: PeerInfo
8 |
9 | beforeEach(() => {
10 | nodeInfo = PeerInfo.newNode('0x21583691f17b9e36a4577520f8db04a19a2f2a0d', 'NetworkNode')
11 | storageInfo = PeerInfo.newStorage('0xc72e234c716445b1b54f225ec1b13e082d88d74d', 'StorageNode')
12 | trackerInfo = PeerInfo.newTracker('0x4c56dbe52abb0878ee05dc15b86d660e7ef3329e')
13 | unknownInfo = PeerInfo.newUnknown('0xeba1386b00de68dcc514ac5d7de7fcb48495c4c7')
14 | })
15 |
16 | it('isNode', () => {
17 | expect(nodeInfo.isNode()).toEqual(true)
18 | expect(storageInfo.isNode()).toEqual(true)
19 | expect(trackerInfo.isNode()).toEqual(false)
20 | expect(unknownInfo.isNode()).toEqual(false)
21 | })
22 |
23 | it('isStorage', () => {
24 | expect(nodeInfo.isStorage()).toEqual(false)
25 | expect(storageInfo.isStorage()).toEqual(true)
26 | expect(trackerInfo.isStorage()).toEqual(false)
27 | expect(unknownInfo.isStorage()).toEqual(false)
28 | })
29 |
30 | it('isTracker', () => {
31 | expect(nodeInfo.isTracker()).toEqual(false)
32 | expect(storageInfo.isTracker()).toEqual(false)
33 | expect(trackerInfo.isTracker()).toEqual(true)
34 | expect(unknownInfo.isTracker()).toEqual(false)
35 | })
36 |
37 | it('toString', () => {
38 | expect(nodeInfo.toString()).toEqual('NetworkNode<0x215836>')
39 | expect(storageInfo.toString()).toEqual('StorageNode<0xc72e23>')
40 | expect(trackerInfo.toString()).toEqual('<0x4c56db>')
41 | expect(unknownInfo.toString()).toEqual('<0xeba138>')
42 | })
43 |
44 | it('PeerInfo constructor throws if invalid peerType', () => {
45 | expect(() => new PeerInfo('peerId', 'invalidPeerType' as any, [2], [32])).toThrow()
46 | })
47 |
48 | it('fromObject', () => {
49 | const peerInfo = PeerInfo.fromObject({
50 | peerId: 'peerId',
51 | peerType: 'tracker',
52 | controlLayerVersions: [2],
53 | messageLayerVersions: [32]
54 | })
55 | expect(peerInfo.peerId).toEqual('peerId')
56 | expect(peerInfo.isTracker()).toEqual(true)
57 | })
58 |
59 | it('id is null if not given', () => {
60 | const peerInfo = PeerInfo.newNode('nodeId')
61 | expect(peerInfo.peerName).toEqual(null)
62 | })
63 |
64 | it('use default location if not given', () => {
65 | const peerInfo = PeerInfo.newNode('nodeId', 'nodeName',undefined , undefined, null)
66 | expect(peerInfo.location).toEqual({
67 | city: null,
68 | country: null,
69 | latitude: null,
70 | longitude: null
71 | })
72 | })
73 | })
74 |
--------------------------------------------------------------------------------
/test/integration/latency.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { NetworkNode } from '../../src/NetworkNode'
3 | import { MessageLayer } from 'streamr-client-protocol'
4 |
5 | import { MetricsContext, startNetworkNode, startTracker } from '../../src/composition'
6 |
7 | const { StreamMessage, MessageID, MessageRef } = MessageLayer
8 |
9 | describe('latency metrics', () => {
10 | let tracker: Tracker
11 | let metricsContext: MetricsContext
12 | let node: NetworkNode
13 |
14 | beforeEach(async () => {
15 | tracker = await startTracker({
16 | host: '127.0.0.1',
17 | port: 32910,
18 | id: 'tracker'
19 | })
20 |
21 | metricsContext = new MetricsContext('node1')
22 | node = await startNetworkNode({
23 | host: '127.0.0.1',
24 | port: 32911,
25 | id: 'node1',
26 | trackers: [tracker.getAddress()],
27 | metricsContext
28 | })
29 |
30 | node.start()
31 | })
32 |
33 | afterEach(async () => {
34 | await node.stop()
35 | await tracker.stop()
36 | })
37 |
38 | it('should fetch empty metrics', async () => {
39 | const { metrics } = await metricsContext.report()
40 | expect(metrics.node.latency).toEqual(0)
41 | })
42 |
43 | it('should send a single message to Node1 and collect latency', (done) => {
44 | node.addMessageListener(async () => {
45 | const { metrics } = await metricsContext.report()
46 | expect(metrics.node.latency).toBeGreaterThan(0)
47 | done()
48 | })
49 |
50 | node.publish(new StreamMessage({
51 | messageId: new MessageID(
52 | 'stream-1',
53 | 0,
54 | new Date().getTime() - 1,
55 | 0,
56 | 'publisherId',
57 | 'msgChainId'
58 | ),
59 | prevMsgRef: new MessageRef(0, 0),
60 | content: {
61 | messageNo: 1
62 | },
63 | }))
64 | })
65 |
66 | it('should send a bunch of messages to Node1 and collect latency',(done) => {
67 | let receivedMessages = 0
68 |
69 | node.addMessageListener(async () => {
70 | receivedMessages += 1
71 |
72 | if (receivedMessages === 5) {
73 | const { metrics } = await metricsContext.report()
74 | expect(metrics.node.latency).toBeGreaterThan(0)
75 | done()
76 | }
77 | })
78 |
79 | for (let i = 1; i <= 5; i++) {
80 | node.publish(new StreamMessage({
81 | messageId: new MessageID('stream-1', 0, i, 0, 'publisherId', 'msgChainId'),
82 | prevMsgRef: i === 1 ? null : new MessageRef(i - 1, 0),
83 | content: {
84 | messageNo: i
85 | },
86 | }))
87 | }
88 | })
89 | })
90 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "streamr-network",
3 | "version": "24.3.2",
4 | "description": "Minimal and extendable implementation of the Streamr Network network node.",
5 | "repository": {
6 | "type": "git",
7 | "url": "git+https://github.com/streamr-dev/network.git"
8 | },
9 | "bin": {
10 | "network": "bin/network.js",
11 | "publisher": "bin/publisher.js",
12 | "subscriber": "bin/subscriber.js",
13 | "tracker": "bin/tracker.js"
14 | },
15 | "main": "dist/composition.js",
16 | "types": "dist/composition.d.ts",
17 | "scripts": {
18 | "prepare": "tsc -p tsconfig-build.json",
19 | "build": "tsc -p tsconfig-build.json",
20 | "tracker": "node $NODE_DEBUG_OPTION bin/tracker.js",
21 | "pub": "node $NODE_DEBUG_OPTION bin/publisher.js",
22 | "pub-1": "node $NODE_DEBUG_OPTION bin/publisher.js --port=30323",
23 | "pub-2": "node $NODE_DEBUG_OPTION bin/publisher.js --port=30333",
24 | "sub": "node $NODE_DEBUG_OPTION bin/subscriber.js",
25 | "sub-1": "node $NODE_DEBUG_OPTION bin/subscriber.js --port=30335",
26 | "sub-2": "node $NODE_DEBUG_OPTION bin/subscriber.js --port=30345",
27 | "test": "jest --forceExit",
28 | "test-unit": "jest test/unit",
29 | "coverage": "jest --coverage",
30 | "test-integration": "jest test/integration",
31 | "test-types": "tsc --noEmit --project ./tsconfig.json ",
32 | "eslint": "eslint .",
33 | "network": "node $NODE_DEBUG_OPTION bin/network.js",
34 | "docs": "typedoc --options typedoc.js"
35 | },
36 | "author": "Streamr Network AG ",
37 | "license": "STREAMR NETWORK OPEN SOURCE LICENSE",
38 | "dependencies": {
39 | "cancelable-promise": "^3.2.3",
40 | "commander": "^7.2.0",
41 | "geoip-lite": "^1.4.2",
42 | "heap": "^0.2.6",
43 | "lodash": "^4.17.21",
44 | "lru-cache": "^6.0.0",
45 | "node-datachannel": "^0.1.4",
46 | "pino": "^6.11.3",
47 | "speedometer": "^1.1.0",
48 | "streamr-client-protocol": "^8.1.0",
49 | "strict-event-emitter-types": "^2.0.0",
50 | "uWebSockets.js": "github:uNetworking/uWebSockets.js#v19.2.0",
51 | "uuid": "^8.3.2",
52 | "ws": "^7.4.5"
53 | },
54 | "devDependencies": {
55 | "@types/geoip-lite": "^1.4.1",
56 | "@types/heap": "^0.2.28",
57 | "@types/jest": "^26.0.23",
58 | "@types/lodash": "^4.14.169",
59 | "@types/lru-cache": "^5.1.0",
60 | "@types/node": "^14.17.0",
61 | "@types/pino": "^6.3.8",
62 | "@types/uuid": "^8.3.0",
63 | "@types/ws": "^7.4.4",
64 | "@typescript-eslint/eslint-plugin": "^4.24.0",
65 | "@typescript-eslint/parser": "^4.24.0",
66 | "eslint": "^7.26.0",
67 | "eslint-config-streamr-ts": "^3.0.1",
68 | "eslint-plugin-promise": "^5.1.0",
69 | "jest": "^26.6.1",
70 | "jest-circus": "^26.6.3",
71 | "streamr-test-utils": "^1.3.2",
72 | "ts-jest": "^26.5.6",
73 | "ts-node": "^9.1.1",
74 | "typedoc": "^0.20.36",
75 | "typescript": "^4.2.4"
76 | },
77 | "optionalDependencies": {
78 | "bufferutil": "^4.0.1",
79 | "utf-8-validate": "^5.0.2",
80 | "pino-pretty": "^4.7.1"
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/src/connection/MessageQueue.ts:
--------------------------------------------------------------------------------
1 | import Heap from 'heap'
2 | import { Logger } from '../helpers/Logger'
3 |
4 | type ErrorInfo = Record
5 |
6 | export class QueueItem {
7 | private static nextNumber = 0
8 |
9 | private readonly message: M
10 | private readonly onSuccess: () => void
11 | private readonly onError: (err: Error) => void
12 | private readonly errorInfos: ErrorInfo[]
13 | public readonly no: number
14 | private tries: number
15 | private failed: boolean
16 |
17 | constructor(message: M, onSuccess: () => void, onError: (err: Error) => void) {
18 | this.message = message
19 | this.onSuccess = onSuccess
20 | this.onError = onError
21 | this.errorInfos = []
22 | this.no = QueueItem.nextNumber++
23 | this.tries = 0
24 | this.failed = false
25 | }
26 |
27 | getMessage(): M {
28 | return this.message
29 | }
30 |
31 | getErrorInfos(): ReadonlyArray {
32 | return this.errorInfos
33 | }
34 |
35 | isFailed(): boolean {
36 | return this.failed
37 | }
38 |
39 | delivered(): void {
40 | this.onSuccess()
41 | }
42 |
43 | incrementTries(info: ErrorInfo): void | never {
44 | this.tries += 1
45 | this.errorInfos.push(info)
46 | if (this.tries >= MessageQueue.MAX_TRIES) {
47 | this.failed = true
48 | }
49 | if (this.isFailed()) {
50 | this.onError(new Error('Failed to deliver message.'))
51 | }
52 | }
53 |
54 | immediateFail(errMsg: string): void {
55 | this.failed = true
56 | this.onError(new Error(errMsg))
57 | }
58 | }
59 |
60 | export class MessageQueue {
61 | public static readonly MAX_TRIES = 10
62 |
63 | private readonly heap: Heap>
64 | private readonly logger: Logger
65 | private readonly maxSize: number
66 |
67 | constructor(maxSize = 5000) {
68 | this.heap = new Heap>((a, b) => a.no - b.no)
69 | this.logger = new Logger(module)
70 | this.maxSize = maxSize
71 | }
72 |
73 | add(message: M): Promise {
74 | if (this.size() === this.maxSize) {
75 | this.logger.warn("queue is full, dropping message")
76 | this.pop().immediateFail("Message queue full, dropping message.")
77 | }
78 | return new Promise((resolve, reject) => {
79 | this.heap.push(new QueueItem(message, resolve, reject))
80 | })
81 | }
82 |
83 | peek(): QueueItem {
84 | return this.heap.peek()
85 | }
86 |
87 | pop(): QueueItem {
88 | return this.heap.pop()
89 | }
90 |
91 | size(): number {
92 | return this.heap.size()
93 | }
94 |
95 | empty(): boolean {
96 | return this.heap.empty()
97 | }
98 |
99 | clear(): boolean {
100 | // @ts-expect-error clear exists but isn't in typedef
101 | return this.heap.clear()
102 | }
103 | }
104 |
--------------------------------------------------------------------------------
/test/unit/proxyRequestStream.test.ts:
--------------------------------------------------------------------------------
1 | import { ControlLayer, MessageLayer } from 'streamr-client-protocol'
2 | import { toReadableStream } from 'streamr-test-utils'
3 |
4 | import { proxyRequestStream } from '../../src/resend/proxyRequestStream'
5 |
6 | const { ResendResponseResending, ResendResponseNoResend, ResendResponseResent, UnicastMessage, ResendLastRequest } = ControlLayer
7 | const { StreamMessage, MessageID } = MessageLayer
8 |
9 | describe('proxyRequestStream', () => {
10 | let sendFn: any
11 | let request: any
12 |
13 | beforeEach(() => {
14 | sendFn = jest.fn()
15 | request = new ResendLastRequest({
16 | requestId: 'requestId',
17 | streamId: 'streamId',
18 | streamPartition: 0,
19 | numberLast: 10,
20 | sessionToken: 'sessionToken',
21 | })
22 | })
23 |
24 | it('empty requestStream causes only NoResend to be sent', (done) => {
25 | const stream = toReadableStream()
26 | proxyRequestStream(sendFn, request, stream)
27 | stream.on('end', () => {
28 | expect(sendFn.mock.calls).toEqual([
29 | [new ResendResponseNoResend({
30 | requestId: 'requestId',
31 | streamId: 'streamId',
32 | streamPartition: 0,
33 | })]
34 | ])
35 | done()
36 | })
37 | })
38 |
39 | it('requestStream with messages causes Resending, Unicast(s), and Resent to be sent', (done) => {
40 | const firstMessage = new StreamMessage({
41 | messageId: new MessageID('streamId', 0, 10000000, 0, 'publisherId', 'msgChainId'),
42 | content: {
43 | hello: 'world'
44 | },
45 | })
46 | const secondMessage = new StreamMessage({
47 | messageId: new MessageID('streamId', 0, 20000000, 0, 'publisherId', 'msgChainId'),
48 | content: {
49 | moi: 'maailma'
50 | },
51 | })
52 | const stream = toReadableStream(
53 | new UnicastMessage({
54 | requestId: 'requestId', streamMessage: firstMessage
55 | }),
56 | new UnicastMessage({
57 | requestId: 'requestId', streamMessage: secondMessage
58 | })
59 | )
60 |
61 | proxyRequestStream(sendFn, request, stream)
62 |
63 | stream.on('end', () => {
64 | expect(sendFn.mock.calls).toEqual([
65 | [new ResendResponseResending({
66 | streamId: 'streamId', streamPartition: 0, requestId: 'requestId',
67 | })],
68 | [new UnicastMessage({
69 | requestId: 'requestId', streamMessage: firstMessage
70 | })],
71 | [new UnicastMessage({
72 | requestId: 'requestId', streamMessage: secondMessage
73 | })],
74 | [new ResendResponseResent({
75 | streamId: 'streamId', streamPartition: 0, requestId: 'requestId'
76 | })],
77 | ])
78 | done()
79 | })
80 | })
81 | })
82 |
--------------------------------------------------------------------------------
/test/unit/Logger.test.ts:
--------------------------------------------------------------------------------
1 | import path from 'path'
2 | import { Logger } from "../../src/helpers/Logger"
3 | import Mock = jest.Mock
4 |
5 | describe(Logger, () => {
6 | let logger: Logger
7 | let fatalFn: Mock
8 | let errorFn: Mock
9 | let warnFn: Mock
10 | let infoFn: Mock
11 | let debugFn: Mock
12 | let traceFn: Mock
13 |
14 | beforeAll(() => {
15 | logger = new Logger(module)
16 | // @ts-expect-error accessing-private
17 | fatalFn = logger.logger.fatal = jest.fn()
18 | // @ts-expect-error accessing-private
19 | errorFn = logger.logger.error = jest.fn()
20 | // @ts-expect-error accessing-private
21 | warnFn = logger.logger.warn = jest.fn()
22 | // @ts-expect-error accessing-private
23 | infoFn = logger.logger.info = jest.fn()
24 | // @ts-expect-error accessing-private
25 | debugFn = logger.logger.debug = jest.fn()
26 | // @ts-expect-error accessing-private
27 | traceFn = logger.logger.trace = jest.fn()
28 | })
29 |
30 | it('delegates call to fatal to pino.Logger#fatal', () => {
31 | logger.fatal('disaster %s!', 123)
32 | expect(fatalFn).toBeCalledTimes(1)
33 | })
34 |
35 | it('delegates call to error to pino.Logger#error', () => {
36 | logger.error('an error or something %s', 123)
37 | expect(errorFn).toBeCalledTimes(1)
38 | })
39 |
40 | it('delegates call to warn to pino.Logger#warn', () => {
41 | logger.warn('a warning %s!', 123)
42 | expect(warnFn).toBeCalledTimes(1)
43 | })
44 |
45 | it('delegates call to info to pino.Logger#info', () => {
46 | logger.info('here be information %s!', 123)
47 | expect(infoFn).toBeCalledTimes(1)
48 | })
49 |
50 | it('delegates call to debug to pino.Logger#debug', () => {
51 | logger.debug('debugging internals %s...', 123)
52 | expect(debugFn).toBeCalledTimes(1)
53 | })
54 |
55 | it('delegates call to trace to pino.Logger#trace', () => {
56 | logger.trace('tracing %s...', 123)
57 | expect(traceFn).toBeCalledTimes(1)
58 | })
59 |
60 | describe('name', () => {
61 | it('short', () => {
62 | // @ts-expect-error private method
63 | expect(Logger.createName(module)).toBe('Logger.test ')
64 | })
65 | it('short with context', () => {
66 | // @ts-expect-error private method
67 | expect(Logger.createName(module, 'foobar')).toBe('Logger.test:foobar ')
68 | })
69 | it('long with context', () => {
70 | // @ts-expect-error private method
71 | expect(Logger.createName(module, 'loremipsum')).toBe('Logger.test:loremips')
72 | })
73 | it('application id', () => {
74 | process.env.STREAMR_APPLICATION_ID = 'APP'
75 | // @ts-expect-error private method
76 | expect(Logger.createName(module)).toBe('APP:Logger.test ')
77 | delete process.env.STREAMR_APPLICATION_ID
78 | })
79 | it('index', () => {
80 | // @ts-expect-error private method
81 | expect(Logger.createName({
82 | filename: ['foo', 'bar', 'mock', 'index'].join(path.sep)
83 | } as any)).toBe('mock ')
84 | })
85 | })
86 | })
--------------------------------------------------------------------------------
/src/connection/NegotiatedProtocolVersions.ts:
--------------------------------------------------------------------------------
1 | import { PeerInfo } from "./PeerInfo"
2 | import { ControlLayer, MessageLayer } from "streamr-client-protocol"
3 |
4 | const defaultControlLayerVersions = ControlLayer.ControlMessage.getSupportedVersions()
5 | const defaultMessageLayerVersions = MessageLayer.StreamMessage.getSupportedVersions()
6 |
7 | type NegotiatedProtocolVersion = { controlLayerVersion: number, messageLayerVersion: number }
8 |
9 | export class NegotiatedProtocolVersions {
10 |
11 | private readonly peerInfo: PeerInfo
12 | private readonly negotiatedProtocolVersions: { [key: string]: NegotiatedProtocolVersion }
13 | private readonly defaultProtocolVersions: NegotiatedProtocolVersion
14 | constructor(peerInfo: PeerInfo) {
15 | this.negotiatedProtocolVersions = Object.create(null)
16 | this.peerInfo = peerInfo
17 | this.defaultProtocolVersions = {
18 | controlLayerVersion: Math.max(0, ...defaultControlLayerVersions),
19 | messageLayerVersion: Math.max(0, ...defaultMessageLayerVersions)
20 | }
21 | }
22 |
23 | negotiateProtocolVersion(peerId: string, controlLayerVersions: number[], messageLayerVersions: number[]): void | never {
24 | try {
25 | const [controlLayerVersion, messageLayerVersion] = this.validateProtocolVersions(controlLayerVersions, messageLayerVersions)
26 | this.negotiatedProtocolVersions[peerId] = {
27 | controlLayerVersion,
28 | messageLayerVersion
29 | }
30 | } catch (err) {
31 | throw err
32 | }
33 | }
34 |
35 | removeNegotiatedProtocolVersion(peerId: string): void {
36 | delete this.negotiatedProtocolVersions[peerId]
37 | }
38 |
39 | getNegotiatedProtocolVersions(peerId: string): NegotiatedProtocolVersion | undefined {
40 | return this.negotiatedProtocolVersions[peerId]
41 | }
42 |
43 | getDefaultProtocolVersions(): NegotiatedProtocolVersion {
44 | return this.defaultProtocolVersions
45 | }
46 |
47 | private validateProtocolVersions(controlLayerVersions: number[], messageLayerVersions: number[]): [number, number] | never {
48 | if (!controlLayerVersions || !messageLayerVersions || controlLayerVersions.length === 0 || messageLayerVersions.length === 0) {
49 | throw new Error('Missing version negotiation! Must give controlLayerVersions and messageLayerVersions as query parameters!')
50 | }
51 |
52 | const controlLayerVersion = Math.max(...this.peerInfo.controlLayerVersions.filter((version) => controlLayerVersions.includes(version)))
53 | const messageLayerVersion = Math.max(...this.peerInfo.messageLayerVersions.filter((version) => messageLayerVersions.includes(version)))
54 |
55 | // Validate that the requested versions are supported
56 | if (controlLayerVersion < 0) {
57 | throw new Error(`Supported ControlLayer versions: ${
58 | JSON.stringify(defaultControlLayerVersions)
59 | }. Are you using an outdated library?`)
60 | }
61 |
62 | if (messageLayerVersion < 0) {
63 | throw new Error(`Supported MessageLayer versions: ${
64 | JSON.stringify(defaultMessageLayerVersions)
65 | }. Are you using an outdated library?`)
66 | }
67 |
68 | return [controlLayerVersion, messageLayerVersion]
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/test/unit/PerStreamMetrics.test.ts:
--------------------------------------------------------------------------------
1 | import { PerStreamMetrics } from '../../src/logic/PerStreamMetrics'
2 |
3 | describe('PerStreamMetrics', () => {
4 | let perStreamMetrics: PerStreamMetrics
5 |
6 | beforeEach(() => {
7 | perStreamMetrics = new PerStreamMetrics()
8 | })
9 |
10 | it('empty state', () => {
11 | expect(perStreamMetrics.report()).toEqual({})
12 | })
13 |
14 | it('empty state', () => {
15 | perStreamMetrics.recordPropagateMessage('stream-1')
16 | perStreamMetrics.recordIgnoredDuplicate('stream-2')
17 | perStreamMetrics.recordDataReceived('stream-3')
18 | expect(perStreamMetrics.report()).toEqual({
19 | 'stream-1': {
20 | onDataReceived: {
21 | last: 0,
22 | rate: 0,
23 | total: 0
24 | },
25 | 'onDataReceived:ignoredDuplicate': {
26 | last: 0,
27 | rate: 0,
28 | total: 0
29 | },
30 | propagateMessage: {
31 | last: 1,
32 | rate: 1,
33 | total: 1
34 | },
35 | resends: {
36 | last: 0,
37 | rate: 0,
38 | total: 0
39 | },
40 | trackerInstructions: {
41 | last: 0,
42 | rate: 0,
43 | total: 0
44 | }
45 | },
46 | 'stream-2': {
47 | onDataReceived: {
48 | last: 0,
49 | rate: 0,
50 | total: 0
51 | },
52 | 'onDataReceived:ignoredDuplicate': {
53 | last: 1,
54 | rate: 1,
55 | total: 1
56 | },
57 | propagateMessage: {
58 | last: 0,
59 | rate: 0,
60 | total: 0
61 | },
62 | resends: {
63 | last: 0,
64 | rate: 0,
65 | total: 0
66 | },
67 | trackerInstructions: {
68 | last: 0,
69 | rate: 0,
70 | total: 0
71 | }
72 | },
73 | 'stream-3': {
74 | onDataReceived: {
75 | last: 1,
76 | rate: 1,
77 | total: 1
78 | },
79 | 'onDataReceived:ignoredDuplicate': {
80 | last: 0,
81 | rate: 0,
82 | total: 0
83 | },
84 | propagateMessage: {
85 | last: 0,
86 | rate: 0,
87 | total: 0
88 | },
89 | resends: {
90 | last: 0,
91 | rate: 0,
92 | total: 0
93 | },
94 | trackerInstructions: {
95 | last: 0,
96 | rate: 0,
97 | total: 0
98 | }
99 | }
100 | })
101 | })
102 | })
103 |
--------------------------------------------------------------------------------
/test/fixtures/key.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN PRIVATE KEY-----
2 | MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDUoxrgGZV9pNmQ
3 | bFRZ5+e13KEqEsOKB32eUJAnAHbDkVg5v7ePSkxav6C5S4V555BX1sBBsxGXboKr
4 | 3E/DDn5F6815wg+XLamo1vDuBqT5Blg7rAvsPvxXCjboMrrg5OA8bjeco6dI56Ye
5 | 1QOM587byC4YPQFjY0OzS+N08XSKHguSShkQb8w554TB0AN/e/6is83Si0Ay18lS
6 | a+ALBIaF1v/6EDaVjED2Xf70BY/9xSNiIbKy3ivEWNCbj7Rgf9CsZFzwEQoUi0tz
7 | 9rk869MTt0omoffT6nvGz8u4Z9kBGBQF7nEv6AwpdMf8zapGc1/Vamk7Om7V/njm
8 | ULyNLtaR0p6WbOwG53Y5KTKZNL7njOqFKk30Jc0A8rPxCSycjbnM+BZr3LECtbB0
9 | C3d0lROe1jziGXFdpremgNCo6+HOuAEP/UBUwvWnDOSFmGukPPPGhUeU+x/DeXCX
10 | 2ppYgTLpdxFJA7PuOO5Qnuc6Rdrj5qUrfBs8ZzRSSmF2ndy84z72753JCWEyFG6g
11 | lB2IsQ5KNSkMIL8GrwD9c+BnR7XNmUTOv9a4Nuon+6xEoN1EGdWFDl3JfL4AznfS
12 | DhRBLikuagkEgcfEYzicm+akYm4MvgBKU4cmeXDAqRHhP3EQboq0ZHJa0SqUTD4r
13 | xi54+Wo0STzQwP1GMaALzJ6pc5bBHwIDAQABAoICAHgSDigDal3Di7M46LzH7hjM
14 | mBkY9V/o8O9H0M6lPWsblLUvZCi+rLUFxm07jwiSUPi45GF1C3b2SUVgp42ejoFP
15 | MP0TLxiQCWC01uGh0OBpy8MOWMEzo+xGcVDW7J33wAN/vVlvNBQ/8pcwc8vKTg3f
16 | UOAQ+sqzj9QcAznS6prfcmtN+i9E4g7EhDupCYCgdN5NJ+k/BqZvIViwX5f5GzlS
17 | ecMsCpaR11EHLOOAjJmhC2TrYGwOdqpt+IYPNKzVH/Fozu4kwQtuyNGXvWwAQnW8
18 | p1t/VGFO5EwUpJXh1jD+9reOuE1z/AIg1pkTTZZwqenqJ7fxPAsMohpz9rTGgRRY
19 | keAtXQEglZfVprAfX/cnXnWHOKzPVU5FMES2nsRF7zFrNYYpADHzr+aKeTwcmgJo
20 | hfd9QAeJmDXRTI4B4SLLa9lzoziLO7XQ6nsZsgXei+MzyCvj/rnZvIY8e4RUzjM0
21 | gfEmYlx+AyoyyK3tlSmitaJuTsO6iqxxmU1tB7D0nJDgi7twZ4y4Lvtl0RBZNdTV
22 | k7DOmxFdp8A0bVo0jaYIgIGFilkWQGpK///XaXDPqEqXpwjeyFpwG6dSqUzXkuce
23 | 1nnVSqtzxTQgtqrQOYSEh9ifjRuwngVCbACvz+FYq+IpI77awfSwBvh6anQZolDL
24 | a8Lv6UD6xwhLzJ/VDz0pAoIBAQD8BVlMZHREf/QbHx881qgJAf8F0AT7q6rZVm1g
25 | Ca8C8SwnmGmGCJQm+jZscme4DJzttaU/vznoKyLxJGLif5PLltlrSWOdCfsPdoQF
26 | 7+jDKNMkI35MeiDAjG+xEMfTi+2/nE5V1+cQCDw3tgpJIuLIFvFJsm5zNbLwV1F3
27 | PRkqWbu7FoBIbFGfzlRc2f/9NEvDi18SWdwxEffzPu+jkTnSGGxmC8msvBQqQbTl
28 | mUv1ho55wM46+SG27AIuTs9m090aQrUwSnQYZq3ggfTM7G17uYdhk6132+y3C41h
29 | aFmTPm8X8pxGYyak7R0ghTyi4SydE0v9kxTYiVb7gg+1B76LAoIBAQDX/pHWpfOY
30 | vVzSe/Kf+hP6/Z+YOqO20dzWPXAiD1L6I+IClwcPQkJTu70W4flSgFspnQyAbzFF
31 | VxtrhNkWlN6MfajRbBAS2NPy24hVO+SI5UYhVJHJb0VtmrLvtvsFQNLs7dbSqiAL
32 | 9vpb0EKIDz9tWioX1mZjL30StP06H59SD8HiJHWV+HkbuuKOxLSUPJ/5XtHnf5IZ
33 | UPR/bFe3kYvbN4V4nI8H9ZyzaZ1l2hDpwdCG8lPL8l329Oa/m2OJQaOef/XDuTN/
34 | hVIaoum9uRigxn5+HFHKyfoNLK2XCqZuL1QdvleANdEErBGXYjO9JCzXt/Fbtitt
35 | Isbr4GKkEE49AoIBABzr/DiBGrq1uGzYYHxZ4gTOntaq7bd4Fu5ENd8cnWIIDVop
36 | 6opAN8hVVKOTaYW8pNG+080CBAI52RO1ake1/l8R27etP2pJN++pWTNZOewTpk9y
37 | Z08dgN63/yVh2JzVZR5lMLQ3R8QWPdri4EFOaJovz9b2TCialANy8d1uPJIYpCuv
38 | RF+LVr6xLgtN1UvYkY1KVEnF2DglaMAYi+XIh8stlFsNpUJWfzXuPnVO52Gw5G7i
39 | iohICO92HDX2Q2T4kVovJ534HWSje/bU0yQlSdc2DmxLR1AHBB5pi/sOv6DfPF6S
40 | MA7/1/J7Z6lQwQmsL7ODAGiEVZMYB8xrO6Q51EsCggEAGRWEZFQzfQqIzBz+6VtG
41 | lMB9l1VADAxFRFAwRJRZA5nFNp3JgRkl0MmdxRmLNeyYKbYGbKf9Jdte4Na1y2yQ
42 | h+pT/7Y4C6hIvQyHwbZJ7SgLQ/WpVKZqfrcXjsVGU67akA9zAlYlkJ4nJyCBiYGT
43 | 0SRGfSw2CdDLCrAgNG7VttcDojqvuTH5BqJmi0QG4KxFu9UqNWNYWT5QlrfDXBs4
44 | DcZd5srvxXHdt+xNDjYL/sm2aOWdm7LS/MomPhxSq+8GjHK53PgWJfoateMzmAf1
45 | 9z32FLk/OnjIZF1K756aA3PVpfqj+MzHMG63QEh0T8PyvT6sdgUq9+qxBVWyvvDx
46 | 1QKCAQEAk0K6KHPQpB4PMhgN5jFiYf3S9d/qIKnekiBiuzBcCsJFwKi+2gKYCrsm
47 | sxd4+1u1jXff95qinVDMhHau8tqp1aZQRoF9dp5CVHbSrRDhe6vP3UXGvSqemj1f
48 | tedoFa966o02UIgwk3kAVY2zNLbfOdyPjQvIgzHoS/z1JOvnW/HXLl5sOB/vvxc3
49 | XcnMq+sGBdqUjJypDMSmEEhSnPZjL4S1EJ5UdRR0BHLP4TN377lTCrnvk0ECoRFF
50 | gNWFDnEdTfLSVu3BgZiW2kMZ5jL5njHCKWXKXTtb+LGSwiOVrEzNysY4HPp25rYZ
51 | tSXf+Un6sJ8vljFf8zorbJdk966eFw==
52 | -----END PRIVATE KEY-----
53 |
--------------------------------------------------------------------------------
/test/benchmarks/tracker.instructions.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-console */
2 | import path from 'path'
3 | import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
4 |
5 | import { wait } from 'streamr-test-utils'
6 |
7 | async function runNetwork(currentBenchmark: number, numberOfNodes: number, startingPort: number, timeout = 60 * 1000, trackerPort = 27777) {
8 | const productionEnv = Object.create(process.env)
9 | // productionEnv.DEBUG = 'streamr:*,-streamr:connection:*'
10 | productionEnv.checkUncaughtException = true
11 |
12 | const processes: ChildProcessWithoutNullStreams[] = []
13 |
14 | // create tracker
15 | const tracker = path.resolve('../../bin/tracker.js')
16 | let args = [
17 | tracker,
18 | '--port=' + trackerPort,
19 | '--metrics=true',
20 | '--metricsInterval=1000'
21 | ]
22 |
23 | const trackerProcess = spawn('node', args, {
24 | env: productionEnv
25 | })
26 |
27 | let metrics = null
28 |
29 | trackerProcess.stdout.on('data', (data) => {
30 | try {
31 | metrics = JSON.parse(data.toString())
32 | } catch (e) {
33 | //
34 | }
35 | })
36 |
37 | processes.push(trackerProcess)
38 |
39 | for (let j = 0; j < numberOfNodes; j++) {
40 | args = [
41 | path.resolve('../../bin/subscriber.js'),
42 | '--streamId=streamId-1',
43 | '--port=' + (startingPort + j),
44 | `--trackers=ws://127.0.0.1:${trackerPort}`
45 | ]
46 |
47 | const subscriber = spawn('node', args, {
48 | env: productionEnv,
49 | // stdio: [process.stdin, process.stdout, process.stderr]
50 | })
51 |
52 | processes.push(subscriber)
53 | }
54 |
55 | await wait(timeout)
56 | console.info(`Stopping benchmark ${currentBenchmark}`)
57 | processes.forEach((child) => child.kill())
58 | return metrics
59 | }
60 |
61 | interface Benchmark {
62 | sendInstruction: any
63 | memory: any
64 | }
65 |
66 | function extractMetrics(metrics: any): Benchmark {
67 | return {
68 | sendInstruction: metrics.trackerMetrics.metrics.sendInstruction,
69 | memory: metrics.processMetrics.memory
70 | }
71 | }
72 |
73 | const arrMax = (arr: number[]) => Math.max(...arr)
74 | const arrMin = (arr: number[]) => Math.min(...arr)
75 | const arrAvg = (arr: number[]) => arr.reduce((a, b) => a + b, 0) / arr.length
76 | const arrayColumn = (arr: any, n: string) => arr.map((x: any) => x[n])
77 |
78 | async function run(numberOfBenchmarks = 10, numberOfNodes = 100, timeout = 60 * 1000) {
79 | const benchmarks: Benchmark[] = []
80 | console.info('Starting benchmark')
81 | for (let i = 0; i < numberOfBenchmarks; i++) {
82 | console.info(`\nRunning benchmark ${i}`)
83 | // eslint-disable-next-line no-await-in-loop
84 | const metrics = await runNetwork(i, numberOfNodes, 30400, timeout)
85 | benchmarks.push(extractMetrics(metrics))
86 | }
87 | console.info('benchmark stopped\n')
88 | console.info(`\n\nResults for ${numberOfBenchmarks} iterations, running ${numberOfNodes} nodes`)
89 |
90 | const keys = ['sendInstruction', 'memory']
91 | keys.forEach((key) => {
92 | const values = arrayColumn(benchmarks, key)
93 | console.info(`${key} => min: ${arrMin(values)}, max: ${arrMax(values)}, avg: ${arrAvg(values)}`)
94 | })
95 | }
96 |
97 | run(10, 2, 60 * 1000)
98 |
99 |
--------------------------------------------------------------------------------
/src/logic/InstructionRetryManager.ts:
--------------------------------------------------------------------------------
1 | import { StreamIdAndPartition, StreamKey } from "../identifiers"
2 | import { TrackerLayer } from "streamr-client-protocol"
3 | import { Logger } from "../helpers/Logger"
4 |
5 | type HandleFn = (
6 | instructionMessage: TrackerLayer.InstructionMessage,
7 | trackerId: string,
8 | reattempt: boolean
9 | ) => Promise
10 |
11 | export class InstructionRetryManager {
12 | private readonly logger: Logger
13 | private readonly handleFn: HandleFn
14 | private readonly intervalInMs: number
15 | private readonly statusSendCounterLimit: number
16 | private instructionRetryIntervals: { [key: string]: {
17 | interval: NodeJS.Timeout,
18 | counter: number,
19 | }
20 | }
21 |
22 | constructor(handleFn: HandleFn, intervalInMs: number) {
23 | this.logger = new Logger(module)
24 | this.handleFn = handleFn
25 | this.intervalInMs = intervalInMs
26 | this.instructionRetryIntervals = {}
27 | this.statusSendCounterLimit = 9
28 | }
29 |
30 | add(instructionMessage: TrackerLayer.InstructionMessage, trackerId: string): void {
31 | const id = StreamIdAndPartition.fromMessage(instructionMessage).key()
32 | if (this.instructionRetryIntervals[id]) {
33 | clearTimeout(this.instructionRetryIntervals[id].interval)
34 | }
35 | this.instructionRetryIntervals[id] = {
36 | interval: setTimeout(() =>
37 | this.retryFunction(instructionMessage, trackerId)
38 | , this.intervalInMs),
39 | counter: 0
40 | }
41 | }
42 |
43 | async retryFunction(instructionMessage: TrackerLayer.InstructionMessage, trackerId: string): Promise {
44 | const streamId = StreamIdAndPartition.fromMessage(instructionMessage).key()
45 | try {
46 | // First and every nth instruction retries will always send status messages to tracker
47 | await this.handleFn(instructionMessage, trackerId, this.instructionRetryIntervals[streamId].counter !== 0)
48 | } catch (err) {
49 | this.logger.warn('instruction retry threw %s', err)
50 | }
51 | // Check that stream has not been removed
52 | if (this.instructionRetryIntervals[streamId]) {
53 | if (this.instructionRetryIntervals[streamId].counter >= this.statusSendCounterLimit) {
54 | this.instructionRetryIntervals[streamId].counter = 0
55 | } else {
56 | this.instructionRetryIntervals[streamId].counter += 1
57 | }
58 |
59 | clearTimeout(this.instructionRetryIntervals[streamId].interval)
60 | this.instructionRetryIntervals[streamId].interval = setTimeout(() =>
61 | this.retryFunction(instructionMessage, trackerId)
62 | , this.intervalInMs)
63 | }
64 | }
65 |
66 | removeStreamId(streamId: StreamKey): void {
67 | if (streamId in this.instructionRetryIntervals) {
68 | clearTimeout(this.instructionRetryIntervals[streamId].interval)
69 | delete this.instructionRetryIntervals[streamId]
70 | this.logger.debug('stream %s successfully removed', streamId)
71 | }
72 | }
73 |
74 | reset(): void {
75 | Object.values(this.instructionRetryIntervals).forEach((obj) => {
76 | clearTimeout(obj.interval)
77 | obj.counter = 0
78 | })
79 | this.instructionRetryIntervals = {}
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/test/integration/unsubscribe-from-stream.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { NetworkNode } from '../../src/NetworkNode'
3 |
4 | import { MessageLayer } from 'streamr-client-protocol'
5 | import { waitForEvent } from 'streamr-test-utils'
6 |
7 | import { startNetworkNode, startTracker } from '../../src/composition'
8 | import { Event as NodeEvent } from '../../src/logic/Node'
9 |
10 | const { StreamMessage, MessageID } = MessageLayer
11 |
12 | describe('node unsubscribing from a stream', () => {
13 | let tracker: Tracker
14 | let nodeA: NetworkNode
15 | let nodeB: NetworkNode
16 |
17 | beforeEach(async () => {
18 | tracker = await startTracker({
19 | host: '127.0.0.1',
20 | port: 30450,
21 | id: 'tracker'
22 | })
23 | nodeA = await startNetworkNode({
24 | host: '127.0.0.1',
25 | port: 30451,
26 | id: 'a',
27 | trackers: [tracker.getAddress()],
28 | disconnectionWaitTime: 200
29 | })
30 | nodeB = await startNetworkNode({
31 | host: '127.0.0.1',
32 | port: 30452,
33 | id: 'b',
34 | trackers: [tracker.getAddress()],
35 | disconnectionWaitTime: 200
36 | })
37 |
38 | nodeA.start()
39 | nodeB.start()
40 |
41 | nodeA.subscribe('s', 2)
42 | nodeB.subscribe('s', 2)
43 | await Promise.all([
44 | waitForEvent(nodeA, NodeEvent.NODE_SUBSCRIBED),
45 | waitForEvent(nodeB, NodeEvent.NODE_SUBSCRIBED),
46 | ])
47 |
48 | nodeA.subscribe('s', 1)
49 | nodeB.subscribe('s', 1)
50 | await Promise.all([
51 | waitForEvent(nodeA, NodeEvent.NODE_SUBSCRIBED),
52 | waitForEvent(nodeB, NodeEvent.NODE_SUBSCRIBED),
53 | ])
54 | })
55 |
56 | afterEach(async () => {
57 | await nodeA.stop()
58 | await nodeB.stop()
59 | await tracker.stop()
60 | })
61 |
62 | test('node still receives data for subscribed streams thru existing connections', async () => {
63 | const actual: string[] = []
64 | nodeB.addMessageListener((streamMessage) => {
65 | actual.push(`${streamMessage.getStreamId()}::${streamMessage.getStreamPartition()}`)
66 | })
67 |
68 | nodeB.unsubscribe('s', 2)
69 | await waitForEvent(nodeA, NodeEvent.NODE_UNSUBSCRIBED)
70 |
71 | nodeA.publish(new StreamMessage({
72 | messageId: new MessageID('s', 2, 0, 0, 'publisherId', 'msgChainId'),
73 | content: {},
74 | }))
75 | nodeA.publish(new StreamMessage({
76 | messageId: new MessageID('s', 1, 0, 0, 'publisherId', 'msgChainId'),
77 | content: {},
78 | }))
79 | await waitForEvent(nodeB, NodeEvent.UNSEEN_MESSAGE_RECEIVED)
80 | expect(actual).toEqual(['s::1'])
81 | })
82 |
83 | test('connection between nodes is not kept if no shared streams', async () => {
84 | nodeB.unsubscribe('s', 2)
85 | await waitForEvent(nodeA, NodeEvent.NODE_UNSUBSCRIBED)
86 |
87 | nodeA.unsubscribe('s', 1)
88 | await waitForEvent(nodeB, NodeEvent.NODE_UNSUBSCRIBED)
89 |
90 | const [aEventArgs, bEventArgs] = await Promise.all([
91 | waitForEvent(nodeA, NodeEvent.NODE_DISCONNECTED),
92 | waitForEvent(nodeB, NodeEvent.NODE_DISCONNECTED)
93 | ])
94 |
95 | expect(aEventArgs).toEqual(['b'])
96 | expect(bEventArgs).toEqual(['a'])
97 | })
98 | })
99 |
--------------------------------------------------------------------------------
/test/integration/tracker-node-reconnect-instructions.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { NetworkNode } from '../../src/NetworkNode'
3 | import { waitForEvent } from 'streamr-test-utils'
4 | import { TrackerLayer } from 'streamr-client-protocol'
5 |
6 | import { startNetworkNode, startTracker } from '../../src/composition'
7 | import { Event as TrackerServerEvent } from '../../src/protocol/TrackerServer'
8 | import { Event as NodeEvent } from '../../src/logic/Node'
9 |
10 | /**
11 | * This test verifies that tracker can send instructions to node and node will connect and disconnect based on the instructions
12 | */
13 | describe('Check tracker instructions to node', () => {
14 | let tracker: Tracker
15 | let nodeOne: NetworkNode
16 | let nodeTwo: NetworkNode
17 | const streamId = 'stream-1'
18 |
19 | beforeAll(async () => {
20 | tracker = await startTracker({
21 | host: '127.0.0.1',
22 | port: 30950,
23 | id: 'tracker'
24 | })
25 |
26 | nodeOne = await startNetworkNode({
27 | host: '127.0.0.1',
28 | port: 30952,
29 | id: 'node-1',
30 | trackers: [tracker.getAddress()],
31 | disconnectionWaitTime: 200
32 | })
33 | nodeTwo = await startNetworkNode({
34 | host: '127.0.0.1',
35 | port: 30953,
36 | id: 'node-2',
37 | trackers: [tracker.getAddress()],
38 | disconnectionWaitTime: 200
39 | })
40 |
41 | nodeOne.subscribe(streamId, 0)
42 | nodeTwo.subscribe(streamId, 0)
43 |
44 | nodeOne.start()
45 | nodeTwo.start()
46 | })
47 |
48 | afterAll(async () => {
49 | await nodeOne.stop()
50 | await nodeTwo.stop()
51 | await tracker.stop()
52 | })
53 |
54 | it('tracker should receive statuses from both nodes', (done) => {
55 | let receivedTotal = 0
56 | // @ts-expect-error private field
57 | tracker.trackerServer.on(TrackerServerEvent.NODE_STATUS_RECEIVED, () => {
58 | receivedTotal += 1
59 |
60 | if (receivedTotal === 2) {
61 | done()
62 | }
63 | })
64 | })
65 |
66 | it('if tracker sends empty list of nodes, node one will disconnect from node two', async () => {
67 | await Promise.all([
68 | waitForEvent(nodeOne, NodeEvent.NODE_SUBSCRIBED),
69 | waitForEvent(nodeTwo, NodeEvent.NODE_SUBSCRIBED)
70 | ])
71 |
72 | // @ts-expect-error private field
73 | expect(Object.keys(nodeOne.nodeToNode.endpoint.connections).length).toBe(1)
74 | // @ts-expect-error private field
75 | expect(Object.keys(nodeTwo.nodeToNode.endpoint.connections).length).toBe(1)
76 |
77 | // send empty list
78 | // @ts-expect-error private field
79 | await tracker.trackerServer.endpoint.send(
80 | 'node-1',
81 | new TrackerLayer.InstructionMessage({
82 | requestId: 'requestId',
83 | streamId,
84 | streamPartition: 0,
85 | nodeIds: [],
86 | counter: 3
87 | }).serialize()
88 | )
89 | await waitForEvent(nodeOne, NodeEvent.NODE_UNSUBSCRIBED)
90 |
91 | // @ts-expect-error private field
92 | expect(nodeOne.trackerNode.endpoint.getPeers().size).toBe(1)
93 |
94 | nodeOne.unsubscribe(streamId, 0)
95 | await waitForEvent(nodeTwo, NodeEvent.NODE_UNSUBSCRIBED)
96 |
97 | // @ts-expect-error private field
98 | expect(Object.keys(nodeTwo.nodeToNode.endpoint.connections).length).toBe(1)
99 | })
100 | })
101 |
--------------------------------------------------------------------------------
/test/integration/webrtc-endpoint-back-pressure-handling.test.ts:
--------------------------------------------------------------------------------
1 | import { Event } from '../../src/connection/IWebRtcEndpoint'
2 | import { WebRtcEndpoint } from '../../src/connection/WebRtcEndpoint'
3 | import { PeerInfo } from '../../src/connection/PeerInfo'
4 | import { MetricsContext } from '../../src/helpers/MetricsContext'
5 | import { RtcSignaller } from '../../src/logic/RtcSignaller'
6 | import { Tracker } from '../../src/logic/Tracker'
7 | import { startTracker } from '../../src/composition'
8 | import { startEndpoint } from '../../src/connection/WsEndpoint'
9 | import { TrackerNode } from '../../src/protocol/TrackerNode'
10 | import { wait } from 'streamr-test-utils'
11 | import { NegotiatedProtocolVersions } from "../../src/connection/NegotiatedProtocolVersions"
12 |
13 | describe('WebRtcEndpoint: back pressure handling', () => {
14 | let tracker: Tracker
15 | let trackerNode1: TrackerNode
16 | let trackerNode2: TrackerNode
17 | let ep1: WebRtcEndpoint
18 | let ep2: WebRtcEndpoint
19 |
20 | beforeEach(async () => {
21 | tracker = await startTracker({
22 | host: '127.0.0.1',
23 | port: 28710,
24 | id: 'tracker'
25 | })
26 |
27 | const peerInfo1 = PeerInfo.newNode('ep1')
28 | const peerInfo2 = PeerInfo.newNode('ep2')
29 |
30 | // Need to set up TrackerNodes and WsEndpoint(s) to exchange RelayMessage(s) via tracker
31 | const wsEp1 = await startEndpoint('127.0.0.1', 28711, peerInfo1, null, new MetricsContext(peerInfo1.peerId))
32 | const wsEp2 = await startEndpoint('127.0.0.1', 28712, peerInfo2, null, new MetricsContext(peerInfo2.peerId))
33 | trackerNode1 = new TrackerNode(wsEp1)
34 | trackerNode2 = new TrackerNode(wsEp2)
35 | await trackerNode1.connectToTracker(tracker.getAddress())
36 | await trackerNode2.connectToTracker(tracker.getAddress())
37 |
38 | // Set up WebRTC endpoints
39 | ep1 = new WebRtcEndpoint(
40 | peerInfo1,
41 | ['stun:stun.l.google.com:19302'],
42 | new RtcSignaller(peerInfo1, trackerNode1),
43 | new MetricsContext('ep1'),
44 | new NegotiatedProtocolVersions(peerInfo1)
45 | )
46 | ep2 = new WebRtcEndpoint(
47 | peerInfo2,
48 | ['stun:stun.l.google.com:19302'],
49 | new RtcSignaller(peerInfo2, trackerNode2),
50 | new MetricsContext('ep'),
51 | new NegotiatedProtocolVersions(peerInfo2)
52 | )
53 | await ep1.connect('ep2', 'tracker')
54 | })
55 |
56 | afterEach(async () => {
57 | await Promise.allSettled([
58 | tracker.stop(),
59 | trackerNode1.stop(),
60 | trackerNode2.stop(),
61 | ep1.stop(),
62 | ep2.stop()
63 | ])
64 | })
65 |
66 | function inflictHighBackPressure(): Promise {
67 | for (let i = 0; i <= 25; ++i) {
68 | ep1.send('ep2', new Array(1024 * 256).fill('X').join(''))
69 | }
70 | return wait(0) // Relinquish control to allow for setImmediate(() => this.attemptToFlushMessages())
71 | }
72 |
73 | it('emits HIGH_BACK_PRESSURE on high back pressure', (done) => {
74 | ep1.once(Event.HIGH_BACK_PRESSURE, (peerInfo) => {
75 | expect(peerInfo).toEqual(PeerInfo.newNode('ep2'))
76 | done()
77 | })
78 | inflictHighBackPressure()
79 | })
80 |
81 | it('emits LOW_BACK_PRESSURE after high back pressure', (done) => {
82 | ep1.once(Event.HIGH_BACK_PRESSURE, () => {
83 | ep1.once(Event.LOW_BACK_PRESSURE, (peerInfo) => {
84 | expect(peerInfo).toEqual(PeerInfo.newNode('ep2'))
85 | done()
86 | })
87 | })
88 | inflictHighBackPressure()
89 | })
90 | })
91 |
--------------------------------------------------------------------------------
/src/logic/rtcSignallingHandlers.ts:
--------------------------------------------------------------------------------
1 | import { TrackerServer, Event as TrackerServerEvent } from '../protocol/TrackerServer'
2 | import { NotFoundInPeerBookError } from '../connection/PeerBook'
3 | import { LocalCandidateMessage, LocalDescriptionMessage, RelayMessage, RtcConnectMessage } from '../identifiers'
4 | import { RtcSubTypes } from './RtcMessage'
5 | import { Logger } from "../helpers/Logger"
6 |
7 | export function attachRtcSignalling(trackerServer: TrackerServer): void {
8 | if (!(trackerServer instanceof TrackerServer)) {
9 | throw new Error('trackerServer not instance of TrackerServer')
10 | }
11 |
12 | const logger = new Logger(module)
13 |
14 | function handleLocalDescription({ requestId, originator, targetNode, data }: LocalDescriptionMessage & RelayMessage) {
15 | if (data.type === 'answer') {
16 | trackerServer.sendRtcAnswer(
17 | targetNode,
18 | requestId,
19 | originator,
20 | data.description
21 | ).catch((err: Error) => {
22 | logger.debug('failed to sendRtcAnswer to %s due to %s', targetNode, err) // TODO: better?
23 | })
24 | } else if (data.type === 'offer') {
25 | trackerServer.sendRtcOffer(
26 | targetNode,
27 | requestId,
28 | originator,
29 | data.description
30 | ).catch((err: Error) => {
31 | logger.debug('failed to sendRtcOffer to %s due to %s', targetNode, err) // TODO: better?
32 | })
33 | } else {
34 | logger.warn('unrecognized localDescription message: %s', data.type)
35 | }
36 | }
37 |
38 | function handleLocalCandidate({ requestId, originator, targetNode, data }: LocalCandidateMessage & RelayMessage) {
39 | trackerServer.sendRemoteCandidate(
40 | targetNode,
41 | requestId,
42 | originator,
43 | data.candidate,
44 | data.mid
45 | ).catch((err: Error) => {
46 | logger.debug('failed to sendRemoteCandidate to %s due to %s', targetNode, err) // TODO: better?
47 | })
48 | }
49 |
50 | function handleRtcConnect({ requestId, originator, targetNode }: RtcConnectMessage & RelayMessage) {
51 | trackerServer.sendRtcConnect(targetNode, requestId, originator).catch((err: Error) => {
52 | logger.debug('Failed to sendRtcConnect to %s due to %s', targetNode, err) // TODO: better?
53 | })
54 | }
55 |
56 | trackerServer.on(TrackerServerEvent.RELAY_MESSAGE_RECEIVED, (relayMessage: RelayMessage, _source: string) => {
57 | const {
58 | subType,
59 | requestId,
60 | originator,
61 | targetNode,
62 | } = relayMessage
63 | // TODO: validate that source === originator
64 | try {
65 | if (relayMessage.subType === RtcSubTypes.LOCAL_DESCRIPTION) {
66 | handleLocalDescription(relayMessage)
67 | } else if (relayMessage.subType === RtcSubTypes.LOCAL_CANDIDATE) {
68 | handleLocalCandidate(relayMessage)
69 | } else if (relayMessage.subType === RtcSubTypes.RTC_CONNECT) {
70 | handleRtcConnect(relayMessage)
71 | } else {
72 | logger.warn('unrecognized RelayMessage subType %s with contents %o', subType, relayMessage)
73 | }
74 | } catch (err) {
75 | if (err instanceof NotFoundInPeerBookError) {
76 | trackerServer.sendUnknownPeerRtcError(originator.peerId, requestId, targetNode)
77 | .catch((e) => logger.error('failed to sendUnknownPeerRtcError, reason: %s', e))
78 | } else {
79 | throw err
80 | }
81 | }
82 | })
83 | }
84 |
--------------------------------------------------------------------------------
/test/integration/webrtc-multi-signaller.test.ts:
--------------------------------------------------------------------------------
1 | import { MetricsContext, startTracker } from '../../src/composition'
2 | import { startEndpoint } from '../../src/connection/WsEndpoint'
3 | import { TrackerNode } from '../../src/protocol/TrackerNode'
4 | import { Tracker, Event as TrackerEvent } from '../../src/logic/Tracker'
5 | import { PeerInfo } from '../../src/connection/PeerInfo'
6 | import { waitForEvent } from 'streamr-test-utils'
7 | import { Event as EndpointEvent } from '../../src/connection/IWebRtcEndpoint'
8 | import { WebRtcEndpoint } from '../../src/connection/WebRtcEndpoint'
9 | import { RtcSignaller } from '../../src/logic/RtcSignaller'
10 | import { NegotiatedProtocolVersions } from "../../src/connection/NegotiatedProtocolVersions"
11 |
12 | describe('WebRTC multisignaller test', () => {
13 | let tracker1: Tracker
14 | let tracker2: Tracker
15 | let trackerNode1: TrackerNode
16 | let trackerNode2: TrackerNode
17 | let endpoint1: WebRtcEndpoint
18 | let endpoint2: WebRtcEndpoint
19 |
20 | beforeEach(async () => {
21 | tracker1 = await startTracker({
22 | host: '127.0.0.1',
23 | port: 28715,
24 | id: 'tracker1'
25 | })
26 | tracker2 = await startTracker({
27 | host: '127.0.0.1',
28 | port: 28716,
29 | id: 'tracker2'
30 | })
31 |
32 | const ep1 = await startEndpoint('127.0.0.1', 28717, PeerInfo.newNode('node-1'), null, new MetricsContext(''))
33 | const ep2 = await startEndpoint('127.0.0.1', 28718, PeerInfo.newNode('node-2'), null, new MetricsContext(''))
34 |
35 | trackerNode1 = new TrackerNode(ep1)
36 | trackerNode2 = new TrackerNode(ep2)
37 |
38 | trackerNode1.connectToTracker(tracker1.getAddress())
39 | await waitForEvent(tracker1, TrackerEvent.NODE_CONNECTED)
40 | trackerNode2.connectToTracker(tracker1.getAddress())
41 | await waitForEvent(tracker1, TrackerEvent.NODE_CONNECTED)
42 | trackerNode1.connectToTracker(tracker2.getAddress())
43 | await waitForEvent(tracker2, TrackerEvent.NODE_CONNECTED)
44 | trackerNode2.connectToTracker(tracker2.getAddress())
45 | await waitForEvent(tracker2, TrackerEvent.NODE_CONNECTED)
46 |
47 | const peerInfo1 = PeerInfo.newNode('node-1')
48 | const peerInfo2 = PeerInfo.newNode('node-2')
49 | endpoint1 = new WebRtcEndpoint(peerInfo1, ['stun:stun.l.google.com:19302'],
50 | new RtcSignaller(peerInfo1, trackerNode1), new MetricsContext(''), new NegotiatedProtocolVersions(peerInfo1))
51 | endpoint2 = new WebRtcEndpoint(peerInfo2, ['stun:stun.l.google.com:19302'],
52 | new RtcSignaller(peerInfo2, trackerNode2), new MetricsContext(''), new NegotiatedProtocolVersions(peerInfo2))
53 | })
54 |
55 | afterEach(async () => {
56 | await Promise.allSettled([
57 | tracker1.stop(),
58 | tracker2.stop(),
59 | trackerNode1.stop(),
60 | trackerNode2.stop(),
61 | endpoint1.stop(),
62 | endpoint2.stop(),
63 | ])
64 | })
65 |
66 | it('WebRTC connection is established and signalling works if endpoints use different trackers for signalling', async () => {
67 | endpoint1.connect('node-2', 'tracker1', true).catch(() => null)
68 | endpoint2.connect('node-1', 'tracker2', false).catch(() => null)
69 | await Promise.all([
70 | waitForEvent(endpoint1, EndpointEvent.PEER_CONNECTED),
71 | waitForEvent(endpoint2, EndpointEvent.PEER_CONNECTED)
72 | ])
73 |
74 | endpoint1.send('node-2', 'Hello')
75 | await waitForEvent(endpoint2, EndpointEvent.MESSAGE_RECEIVED)
76 | endpoint2.send('node-1', 'Hello')
77 | await waitForEvent(endpoint1, EndpointEvent.MESSAGE_RECEIVED)
78 | })
79 |
80 | })
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | STREAMR NETWORK OPEN SOURCE LICENSE
2 | -----------------------------------
3 |
4 | PREAMBLE
5 |
6 | This software is licensed under the Streamr Network Open
7 | Source License. The Streamr Network Open Source License is
8 | based on the GNU Affero General Public License, version 3
9 | (19 November 2007), but it contains a few additional
10 | limitations and requirements.
11 |
12 | The main addition is the introduction of operating parameters
13 | which apply to the software. These parameters may induce the
14 | use of specific cryptographic tokens (such as the Streamr
15 | DATAcoin) in the operations of the Streamr Network. Doing so
16 | protects the considerable effort already undertaken by the
17 | Streamr community.
18 |
19 | These license terms allow you to create, modify and propagate
20 | new versions of the Streamr Network software (“Program”). Any
21 | copy or modified version shall always comply with certain
22 | network operating parameters, as amended from time to time.
23 | In a manner to be determined in the future, it is Streamr’s
24 | intention to decentralize the governance and decision-making
25 | processes relating to the network operating parameters. The
26 | Streamr Network is also available under a proprietary
27 | commercial license pursuant to a dual licensing scheme.
28 | Please contact Streamr directly at contact@streamr.network if
29 | you are interested in such a proprietary license.
30 |
31 | ADDITIONAL TERMS
32 |
33 | 1. These additional terms (“Additional Terms”) shall be in
34 | addition to and have precedence over the GNU Affero General
35 | Public License, Version 3 (“License Terms”). The License Terms
36 | shall be deemed to be incorporated herein as a material part
37 | of these Additional Terms. Any capitalized terms herein shall
38 | have the meaning given to them in the License Terms.
39 |
40 | 2. Unless explicitly otherwise stated in these Additional
41 | Terms, the License Terms shall be applicable on and govern
42 | your use, modification and propagation of the Program.
43 |
44 | 3. Definitions:
45 |
46 | a. “Supported Parameters” means the operating parameters
47 | of the Program that are indicated in the smart contract
48 | pointed to by Ethereum Name Service (ENS) registry entry
49 | network-parameters.streamr.eth from time to time.
50 |
51 | b. “Reference Versions” means the minimum version number
52 | of the Program that is indicated in the Supported Parameters
53 | as well as any newer version of the Program (as indicated
54 | by a higher version number).
55 |
56 | c. “DATA” means the Streamr DATAcoin cryptographic token,
57 | in each case as set out in and as supported by the Supported
58 | Parameters from time to time.
59 |
60 | 4. Subject to Section 5, any modified version of the Program
61 | shall incorporate and implement all of, and only operate in
62 | accordance with, the Supported Parameters, as implemented in
63 | the Reference Versions. For the avoidance of doubt, as an
64 | example, if the modified version involves the use of crypto-
65 | graphic tokens (such as DATA), such tokens must be set out in
66 | the Supported Parameters. Any breach of this Section 4 is
67 | considered a material breach of these Additional Terms.
68 |
69 | 5. Notwithstanding anything to the contrary in Section 4,
70 | modification and propagation of the Program for development
71 | and testing purposes (meaning e.g., creation of small-scale
72 | prototypes or proofs of concept) is allowed without implementing
73 | the Supported Parameters; provided, however, that the use of
74 | such modified version shall be limited only to the time period
75 | necessary for such development or testing purposes.
76 |
77 | 6. Any violation or breach of these Additional Terms shall be
78 | deemed a violation or breach of the License Terms.
79 |
--------------------------------------------------------------------------------
/src/identifiers.ts:
--------------------------------------------------------------------------------
1 | import { ControlLayer, TrackerLayer } from 'streamr-client-protocol'
2 | import { RtcSubTypes } from './logic/RtcMessage'
3 |
4 | /**
5 | * Uniquely identifies a stream
6 | */
7 | export class StreamIdAndPartition {
8 | public readonly id: string
9 | public readonly partition: number
10 |
11 | constructor(id: string, partition: number) {
12 | if (typeof id !== 'string') {
13 | throw new Error(`invalid id: ${id}`)
14 | }
15 | if (!Number.isInteger(partition)) {
16 | throw new Error(`invalid partition: ${partition}`)
17 | }
18 | this.id = id
19 | this.partition = partition
20 | }
21 |
22 | key(): StreamKey {
23 | return this.toString()
24 | }
25 |
26 | toString(): string {
27 | return `${this.id}::${this.partition}`
28 | }
29 |
30 | static fromMessage(message: { streamId: string, streamPartition: number }): StreamIdAndPartition {
31 | return new StreamIdAndPartition(message.streamId, message.streamPartition)
32 | }
33 |
34 | static fromKey(key: string): StreamIdAndPartition {
35 | const [id, partition] = key.split('::')
36 | return new StreamIdAndPartition(id, Number.parseInt(partition, 10))
37 | }
38 | }
39 |
40 | export type StreamKey = string // Represents format streamId::streamPartition
41 |
42 | export interface Rtts {
43 | [key: string]: number
44 | }
45 |
46 | export interface Location {
47 | latitude: number | null
48 | longitude: number | null
49 | country: string | null
50 | city: string | null
51 | }
52 |
53 | export interface StatusStreams {
54 | [key: string]: { // StreamKey
55 | inboundNodes: string[]
56 | outboundNodes: string[]
57 | counter: number
58 | }
59 | }
60 |
61 | export interface Status {
62 | streams: StatusStreams
63 | rtts: Rtts
64 | location: Location
65 | started: string
66 | singleStream: boolean // indicate whether this is a status update for only a single stream
67 | }
68 |
69 | export type ResendRequest = ControlLayer.ResendLastRequest
70 | | ControlLayer.ResendFromRequest
71 | | ControlLayer.ResendRangeRequest
72 |
73 | export type ResendResponse = ControlLayer.ResendResponseNoResend
74 | | ControlLayer.ResendResponseResending
75 | | ControlLayer.ResendResponseResent
76 |
77 | export type OfferMessage = {
78 | subType: RtcSubTypes.RTC_OFFER
79 | data: {
80 | description: string
81 | }
82 | }
83 |
84 | export type AnswerMessage = {
85 | subType: RtcSubTypes.RTC_ANSWER
86 | data: {
87 | description: string
88 | }
89 | }
90 |
91 | export type RemoteCandidateMessage = {
92 | subType: RtcSubTypes.REMOTE_CANDIDATE
93 | data: {
94 | candidate: string
95 | mid: string
96 | }
97 | }
98 |
99 | export type RtcConnectMessage = {
100 | subType: RtcSubTypes.RTC_CONNECT
101 | data: {
102 | force: boolean
103 | }
104 | }
105 |
106 | export type LocalDescriptionMessage = {
107 | subType: RtcSubTypes.LOCAL_DESCRIPTION
108 | data: {
109 | type: "answer" | "offer"
110 | description: string
111 | }
112 | }
113 |
114 | export type LocalCandidateMessage = {
115 | subType: RtcSubTypes.LOCAL_CANDIDATE
116 | data: {
117 | candidate: string
118 | mid: string
119 | }
120 | }
121 |
122 | export type RelayMessage = (
123 | OfferMessage
124 | | AnswerMessage
125 | | RemoteCandidateMessage
126 | | RtcConnectMessage
127 | | LocalDescriptionMessage
128 | | LocalCandidateMessage
129 | ) & TrackerLayer.RelayMessage
130 |
131 | export interface RtcErrorMessage {
132 | targetNode: string
133 | errorCode: string
134 | }
135 |
--------------------------------------------------------------------------------
/bin/publisher.js:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | const program = require('commander')
3 | const { MessageLayer } = require('streamr-client-protocol')
4 |
5 | const { Logger } = require('../dist/helpers/Logger')
6 | const { version: CURRENT_VERSION } = require('../package.json')
7 | const { startNetworkNode } = require('../dist/composition')
8 | const { MetricsContext } = require('../dist/helpers/MetricsContext')
9 |
10 | const { StreamMessage, MessageID, MessageRef } = MessageLayer
11 |
12 | program
13 | .version(CURRENT_VERSION)
14 | .option('--id ', 'Ethereum address / node id', undefined)
15 | .option('--nodeName ', 'Human readble name for node', undefined)
16 | .option('--port ', 'port', '30302')
17 | .option('--ip ', 'ip', '127.0.0.1')
18 | .option('--trackers ', 'trackers', (value) => value.split(','), ['ws://127.0.0.1:27777'])
19 | .option('--streamIds ', 'streamId to publish', (value) => value.split(','), ['stream-0'])
20 | .option('--metrics ', 'log metrics', false)
21 | .option('--intervalInMs ', 'interval to publish in ms', '2000')
22 | .option('--noise ', 'bytes to add to messages', '64')
23 | .description('Run publisher')
24 | .parse(process.argv)
25 |
26 | const id = program.opts().id || `PU${program.opts().port}`
27 | const name = program.opts().nodeName || id
28 | const logger = new Logger(module)
29 |
30 | const noise = parseInt(program.opts().noise, 10)
31 |
32 | const messageChainId = `message-chain-id-${program.opts().port}`
33 |
34 | function generateString(length) {
35 | let result = ''
36 | const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
37 | const charactersLength = characters.length
38 | for (let i = 0; i < length; i++) {
39 | result += characters.charAt(Math.floor(Math.random() * charactersLength))
40 | }
41 | return result
42 | }
43 |
44 | const metricsContext = new MetricsContext(id)
45 | startNetworkNode({
46 | host: program.opts().ip,
47 | port: program.opts().port,
48 | name,
49 | id,
50 | trackers: program.opts().trackers,
51 | metricsContext
52 | })
53 | .then((publisher) => {
54 | logger.info('started publisher id: %s, name: %s, port: %d, ip: %s, trackers: %s, streamId: %s, intervalInMs: %d, metrics: %s',
55 | id, name, program.opts().port, program.opts().ip, program.opts().trackers.join(', '),
56 | program.opts().streamId, program.opts().intervalInMs, program.opts().metrics)
57 |
58 | publisher.start()
59 |
60 | let lastTimestamp = null
61 | let sequenceNumber = 0
62 |
63 | setInterval(() => {
64 | const timestamp = Date.now()
65 | const msg = 'Hello world, ' + new Date().toLocaleString()
66 | program.opts().streamIds.forEach((streamId) => {
67 | const streamMessage = new StreamMessage({
68 | messageId: new MessageID(streamId, 0, timestamp, sequenceNumber, id, messageChainId),
69 | prevMsgRef: lastTimestamp == null ? null : new MessageRef(lastTimestamp, sequenceNumber - 1),
70 | content: {
71 | msg,
72 | noise: generateString(noise)
73 | },
74 | })
75 | publisher.publish(streamMessage)
76 | })
77 | sequenceNumber += 1
78 | lastTimestamp = timestamp
79 | }, program.opts().intervalInMs)
80 |
81 | if (program.opts().metrics) {
82 | setInterval(async () => {
83 | logger.info(JSON.stringify(await metricsContext.report(true), null, 3))
84 | }, 5000)
85 | }
86 | return true
87 | })
88 | .catch((err) => {
89 | console.error(err)
90 | process.exit(1)
91 | })
92 |
--------------------------------------------------------------------------------
/test/unit/QueueItem.test.ts:
--------------------------------------------------------------------------------
1 | import { MessageQueue, QueueItem } from '../../src/connection/MessageQueue'
2 | import Mock = jest.Mock
3 |
4 | describe(QueueItem, () => {
5 | it('starts as non-failed', () => {
6 | const item = new QueueItem('message', () => {}, () => {})
7 | expect(item.isFailed()).toEqual(false)
8 | expect(item.getErrorInfos()).toEqual([])
9 | })
10 |
11 | it('does not become failed if incrementTries invoked less than MAX_RETRIES times', () => {
12 | const item = new QueueItem('message', () => {}, () => {})
13 | for (let i = 0; i < MessageQueue.MAX_TRIES - 1; ++i) {
14 | item.incrementTries({ error: 'error' })
15 | }
16 | expect(item.isFailed()).toEqual(false)
17 | })
18 |
19 | it('becomes failed if incrementTries invoked MAX_RETRIES times', () => {
20 | const item = new QueueItem('message', () => {}, () => {})
21 | for (let i = 0; i < MessageQueue.MAX_TRIES; ++i) {
22 | item.incrementTries({ error: 'error' })
23 | }
24 | expect(item.isFailed()).toEqual(true)
25 | expect(item.getErrorInfos()).toEqual(Array(MessageQueue.MAX_TRIES).fill({ error: 'error' }))
26 | })
27 |
28 | it('becomes failed immediately if immediateFail invoked', () => {
29 | const item = new QueueItem('message', () => {}, () => {})
30 | item.immediateFail('error')
31 | expect(item.isFailed()).toEqual(true)
32 | expect(item.getErrorInfos()).toEqual([])
33 | })
34 |
35 | describe('when method delivered() invoked', () => {
36 | let successFn: Mock
37 | let errorFn: Mock
38 |
39 | beforeEach(() => {
40 | successFn = jest.fn()
41 | errorFn = jest.fn()
42 | const item = new QueueItem('message', successFn, errorFn)
43 | item.delivered()
44 | })
45 |
46 | it('onSuccess callback invoked', () => {
47 | expect(successFn).toHaveBeenCalledTimes(1)
48 | })
49 |
50 | it('onError callback not invoked', () => {
51 | expect(errorFn).toHaveBeenCalledTimes(0)
52 | })
53 | })
54 |
55 | describe('after method incrementTries invoked() MAX_RETRIES times', () => {
56 | let successFn: Mock
57 | let errorFn: Mock
58 |
59 | beforeEach(() => {
60 | successFn = jest.fn()
61 | errorFn = jest.fn()
62 | const item = new QueueItem('message', successFn, errorFn)
63 | for (let i = 0; i < MessageQueue.MAX_TRIES; ++i) {
64 | item.incrementTries({ error: `error ${i}` })
65 | }
66 | })
67 |
68 | it('onSuccess callback invoked', () => {
69 | expect(successFn).toHaveBeenCalledTimes(0)
70 | })
71 |
72 | it('onError callback invoked with supplied error message', () => {
73 | expect(errorFn.mock.calls).toEqual([
74 | [new Error('Failed to deliver message.')]
75 | ])
76 | })
77 | })
78 |
79 | describe('when method immediateFail() invoked', () => {
80 | let successFn: Mock
81 | let errorFn: Mock
82 |
83 | beforeEach(() => {
84 | successFn = jest.fn()
85 | errorFn = jest.fn()
86 | const item = new QueueItem('message', successFn, errorFn)
87 | item.immediateFail('here is error message')
88 | })
89 |
90 | it('onSuccess callback invoked', () => {
91 | expect(successFn).toHaveBeenCalledTimes(0)
92 | })
93 |
94 | it('onError callback invoked with supplied error message', () => {
95 | expect(errorFn.mock.calls).toEqual([
96 | [new Error('here is error message')]
97 | ])
98 | })
99 | })
100 | })
101 |
--------------------------------------------------------------------------------
/test/integration/do-not-propagate-to-sender-optimization.test.ts:
--------------------------------------------------------------------------------
1 | import { NetworkNode } from '../../src/NetworkNode'
2 | import { Tracker } from '../../src/logic/Tracker'
3 | import { MessageLayer } from 'streamr-client-protocol'
4 | import { waitForCondition } from 'streamr-test-utils'
5 |
6 | import { startNetworkNode, startTracker } from '../../src/composition'
7 |
8 | const { StreamMessage, MessageID, MessageRef } = MessageLayer
9 |
10 | /**
11 | * This test verifies that on receiving a message, the receiver will not propagate the message to the sender as they
12 | * obviously already know about the message.
13 | */
14 | describe('optimization: do not propagate to sender', () => {
15 | let tracker: Tracker
16 | let n1: NetworkNode
17 | let n2: NetworkNode
18 | let n3: NetworkNode
19 |
20 | beforeAll(async () => {
21 | tracker = await startTracker({
22 | host: '127.0.0.1',
23 | port: 30410,
24 | id: 'tracker'
25 | })
26 | n1 = await startNetworkNode({
27 | host: '127.0.0.1',
28 | port: 30411,
29 | id: 'node-1',
30 | trackers: [tracker.getAddress()]
31 | })
32 | n2 = await startNetworkNode({
33 | host: '127.0.0.1',
34 | port: 30412,
35 | id: 'node-2',
36 | trackers: [tracker.getAddress()]
37 | })
38 | n3 = await startNetworkNode({
39 | host: '127.0.0.1',
40 | port: 30413,
41 | id: 'node-3',
42 | trackers: [tracker.getAddress()]
43 | })
44 |
45 | n1.start()
46 | n2.start()
47 | n3.start()
48 |
49 | // Become subscribers (one-by-one, for well connected graph)
50 | n1.subscribe('stream-id', 0)
51 | n2.subscribe('stream-id', 0)
52 | n3.subscribe('stream-id', 0)
53 |
54 | // Wait for fully-connected network
55 | await waitForCondition(() => {
56 | return n1.getNeighbors().length === 2
57 | && n2.getNeighbors().length === 2
58 | && n3.getNeighbors().length === 2
59 | })
60 | })
61 |
62 | afterAll(async () => {
63 | await Promise.allSettled([
64 | tracker.stop(),
65 | n1.stop(),
66 | n2.stop(),
67 | n3.stop()
68 | ])
69 | })
70 |
71 | // In a fully-connected network the number of duplicates should be (n-1)(n-2) instead of (n-1)^2 when not
72 | // propagating received messages back to their source
73 | test('total duplicates == 2 in a fully-connected network of 3 nodes', async () => {
74 | n1.publish(new StreamMessage({
75 | messageId: new MessageID('stream-id', 0, 100, 0, 'publisher', 'session'),
76 | prevMsgRef: new MessageRef(99, 0),
77 | content: {
78 | hello: 'world'
79 | },
80 | }))
81 |
82 | let n1Duplicates = 0
83 | let n2Duplicates = 0
84 | let n3Duplicates = 0
85 |
86 | await waitForCondition(async () => {
87 | // @ts-expect-error private variable
88 | const reportN1 = await n1.metrics.report()
89 | // @ts-expect-error private variable
90 | const reportN2 = await n2.metrics.report()
91 | // @ts-expect-error private variable
92 | const reportN3 = await n3.metrics.report()
93 |
94 | n1Duplicates = (reportN1['onDataReceived:ignoredDuplicate'] as any).total
95 | n2Duplicates = (reportN2['onDataReceived:ignoredDuplicate'] as any).total
96 | n3Duplicates = (reportN3['onDataReceived:ignoredDuplicate'] as any).total
97 |
98 | return n1Duplicates + n2Duplicates + n3Duplicates > 0
99 | })
100 |
101 | expect(n1Duplicates + n2Duplicates + n3Duplicates).toEqual(2)
102 | })
103 | })
104 |
--------------------------------------------------------------------------------
/src/logic/trackerSummaryUtils.ts:
--------------------------------------------------------------------------------
1 | import { StreamIdAndPartition } from '../identifiers'
2 | import { OverlayPerStream, OverlayConnectionRtts } from './Tracker'
3 |
4 | type OverLayWithRtts = { [key: string]: { [key: string]: { neighborId: string, rtt: number | null }[] } }
5 | type OverlaySizes = { streamId: string, partition: number, nodeCount: number }[]
6 |
7 | export function getTopology(
8 | overlayPerStream: OverlayPerStream,
9 | connectionRtts: OverlayConnectionRtts,
10 | streamId: string | null = null,
11 | partition: number | null = null
12 | ): OverLayWithRtts {
13 | const topology: OverLayWithRtts = {}
14 |
15 | const streamKeys = findStreamKeys(overlayPerStream, streamId, partition)
16 |
17 | streamKeys.forEach((streamKey) => {
18 | const streamOverlay = overlayPerStream[streamKey].state()
19 | topology[streamKey] = Object.assign({}, ...Object.entries(streamOverlay).map(([nodeId, neighbors]) => {
20 | return addRttsToNodeConnections(nodeId, neighbors, connectionRtts)
21 | }))
22 | })
23 |
24 | return topology
25 | }
26 |
27 | export function getStreamSizes(overlayPerStream: OverlayPerStream, streamId: string | null = null, partition: number | null = null): OverlaySizes {
28 | const streamKeys = findStreamKeys(overlayPerStream, streamId, partition)
29 |
30 | const streamSizes: OverlaySizes = streamKeys.map((streamKey) => {
31 | const key = StreamIdAndPartition.fromKey(streamKey)
32 | return {
33 | streamId: key.id,
34 | partition: key.partition,
35 | nodeCount: overlayPerStream[streamKey].getNumberOfNodes()
36 | }
37 | })
38 | return streamSizes
39 | }
40 |
41 | export function getNodeConnections(nodes: readonly string[], overlayPerStream: OverlayPerStream): { [key: string]: Set } {
42 | const result: { [key: string]: Set } = {}
43 | nodes.forEach((node) => {
44 | result[node] = new Set()
45 | })
46 | nodes.forEach((node) => {
47 | Object.values(overlayPerStream).forEach((overlayTopology) => {
48 | result[node] = new Set([...result[node], ...overlayTopology.getNeighbors(node)])
49 | })
50 | })
51 | return result
52 | }
53 |
54 | export function addRttsToNodeConnections(
55 | nodeId: string,
56 | neighbors: Array,
57 | connectionRtts: OverlayConnectionRtts
58 | ): { [key: string]: { neighborId: string, rtt: number | null }[] } {
59 | return {
60 | [nodeId]: neighbors.map((neighborId) => {
61 | return {
62 | neighborId,
63 | rtt: getNodeToNodeConnectionRtts(nodeId, neighborId, connectionRtts[nodeId], connectionRtts[neighborId])
64 | }
65 | })
66 | }
67 | }
68 |
69 | function getNodeToNodeConnectionRtts(
70 | nodeOne: string,
71 | nodeTwo: string,
72 | nodeOneRtts: { [key: string]: number },
73 | nodeTwoRtts: { [key: string]: number }
74 | ): number | null {
75 | try {
76 | return nodeOneRtts[nodeTwo] || nodeTwoRtts[nodeOne] || null
77 | } catch (err) {
78 | return null
79 | }
80 | }
81 |
82 | function findStreamKeys(overlayPerStream: OverlayPerStream, streamId: string | null = null, partition: number | null = null): string[] {
83 | let streamKeys
84 |
85 | if (streamId && partition === null) {
86 | streamKeys = Object.keys(overlayPerStream).filter((streamKey) => streamKey.includes(streamId))
87 | } else {
88 | let askedStreamKey: StreamIdAndPartition | null = null
89 | if (streamId && partition != null && Number.isSafeInteger(partition) && partition >= 0) {
90 | askedStreamKey = new StreamIdAndPartition(streamId, partition)
91 | }
92 |
93 | streamKeys = askedStreamKey
94 | ? Object.keys(overlayPerStream).filter((streamKey) => streamKey === askedStreamKey!.toString())
95 | : Object.keys(overlayPerStream)
96 | }
97 |
98 | return streamKeys
99 | }
--------------------------------------------------------------------------------
/test/unit/InstructionThrottler.test.ts:
--------------------------------------------------------------------------------
1 | import { waitForCondition } from 'streamr-test-utils'
2 | import { TrackerLayer } from 'streamr-client-protocol'
3 |
4 | import { InstructionThrottler } from '../../src/logic/InstructionThrottler'
5 |
6 | describe('InstructionThrottler', () => {
7 | let handlerCb: any
8 | let instructionThrottler: InstructionThrottler
9 |
10 | beforeEach(() => {
11 | handlerCb = jest.fn().mockResolvedValue(true)
12 | instructionThrottler = new InstructionThrottler(handlerCb)
13 | })
14 |
15 | function createInstruction(streamId: string, counter: number) {
16 | return new TrackerLayer.InstructionMessage({
17 | requestId: 'requestId',
18 | streamId,
19 | streamPartition: 0,
20 | nodeIds: [],
21 | counter
22 | })
23 | }
24 |
25 | it('all instructions are handled when inserting a burst of them with distinct streams', async () => {
26 | instructionThrottler.add(createInstruction('stream-1', 1), 'tracker-1')
27 | instructionThrottler.add(createInstruction('stream-2', 2), 'tracker-2')
28 | instructionThrottler.add(createInstruction('stream-3', 3), 'tracker-1')
29 | instructionThrottler.add(createInstruction('stream-4', 4), 'tracker-1')
30 | instructionThrottler.add(createInstruction('stream-5', 5), 'tracker-2')
31 |
32 | await waitForCondition(() => instructionThrottler.isIdle())
33 |
34 | expect(handlerCb.mock.calls).toEqual([
35 | [createInstruction('stream-1', 1), 'tracker-1'],
36 | [createInstruction('stream-2', 2), 'tracker-2'],
37 | [createInstruction('stream-3', 3), 'tracker-1'],
38 | [createInstruction('stream-4', 4), 'tracker-1'],
39 | [createInstruction('stream-5', 5), 'tracker-2'],
40 | ])
41 | })
42 |
43 | it('first and last instructions handled when inserting a burst of them with identical keys (throttle)', async () => {
44 | instructionThrottler.add(createInstruction('stream-1', 1), 'tracker-1')
45 | instructionThrottler.add(createInstruction('stream-1', 2), 'tracker-1')
46 | instructionThrottler.add(createInstruction('stream-1', 3), 'tracker-1')
47 | instructionThrottler.add(createInstruction('stream-1', 4), 'tracker-1')
48 | instructionThrottler.add(createInstruction('stream-1', 5), 'tracker-1')
49 |
50 | await waitForCondition(() => instructionThrottler.isIdle())
51 |
52 | expect(handlerCb.mock.calls).toEqual([
53 | [createInstruction('stream-1', 1), 'tracker-1'],
54 | [createInstruction('stream-1', 5), 'tracker-1']
55 | ])
56 | })
57 |
58 | it('all instructions are handled when inserting them slowly with identical keys (no throttle)', async () => {
59 | instructionThrottler.add(createInstruction('stream-1', 1), 'tracker-1')
60 | await waitForCondition(() => instructionThrottler.isIdle())
61 | instructionThrottler.add(createInstruction('stream-1', 2), 'tracker-1')
62 | await waitForCondition(() => instructionThrottler.isIdle())
63 | instructionThrottler.add(createInstruction('stream-1', 3), 'tracker-1')
64 | await waitForCondition(() => instructionThrottler.isIdle())
65 | instructionThrottler.add(createInstruction('stream-1', 4), 'tracker-1')
66 | await waitForCondition(() => instructionThrottler.isIdle())
67 | instructionThrottler.add(createInstruction('stream-1', 5), 'tracker-1')
68 |
69 | await waitForCondition(() => instructionThrottler.isIdle())
70 |
71 | expect(handlerCb.mock.calls).toEqual([
72 | [createInstruction('stream-1', 1), 'tracker-1'],
73 | [createInstruction('stream-1', 2), 'tracker-1'],
74 | [createInstruction('stream-1', 3), 'tracker-1'],
75 | [createInstruction('stream-1', 4), 'tracker-1'],
76 | [createInstruction('stream-1', 5), 'tracker-1'],
77 | ])
78 | })
79 | })
80 |
--------------------------------------------------------------------------------
/test/integration/storage-config.test.ts:
--------------------------------------------------------------------------------
1 | import { MessageLayer } from 'streamr-client-protocol'
2 | import { v4 as uuidv4 } from 'uuid'
3 | import { waitForCondition, waitForEvent } from 'streamr-test-utils'
4 | import { Tracker } from '../../src/logic/Tracker'
5 | import { NetworkNode } from '../../src/NetworkNode'
6 | import { Event as TrackerServerEvent } from '../../src/protocol/TrackerServer'
7 | import { startTracker, startNetworkNode, startStorageNode, Storage } from '../../src/composition'
8 | import { StreamIdAndPartition } from '../../src/identifiers'
9 | import { MockStorageConfig } from './MockStorageConfig'
10 |
11 | const { StreamMessage, MessageID } = MessageLayer
12 |
13 | const HOST = '127.0.0.1'
14 |
15 | const createMockStream = () => {
16 | return new StreamIdAndPartition(uuidv4(), Math.floor(Math.random() * 100))
17 | }
18 |
19 | const createMockMessage = (stream: StreamIdAndPartition) => {
20 | return new StreamMessage({
21 | messageId: new MessageID(stream.id, stream.partition, Date.now(), 0, 'mock-publisherId', 'mock-msgChainId'),
22 | content: {
23 | 'foo': 'bar'
24 | }
25 | })
26 | }
27 |
28 | const createStreamMessageMatcher = (message: MessageLayer.StreamMessage) => {
29 | return expect.objectContaining({
30 | messageId: expect.objectContaining(message.messageId)
31 | })
32 | }
33 |
34 | describe('storage node', () => {
35 | const initialStream = createMockStream()
36 | let storage: Partial
37 | let config: MockStorageConfig
38 | let tracker: Tracker
39 | let relayNode: NetworkNode
40 | let storageNode: NetworkNode
41 |
42 | beforeEach(async () => {
43 | storage = {
44 | store: jest.fn()
45 | }
46 | config = new MockStorageConfig()
47 | config.addStream(initialStream)
48 | tracker = await startTracker({
49 | host: HOST,
50 | port: 49800,
51 | id: 'tracker'
52 | })
53 | relayNode = await startNetworkNode({
54 | host: HOST,
55 | port: 49801,
56 | id: 'relay',
57 | trackers: [tracker.getAddress()]
58 | })
59 | storageNode = await startStorageNode({
60 | host: HOST,
61 | port: 49802,
62 | id: 'storage',
63 | trackers: [tracker.getAddress()],
64 | storages: [storage as Storage],
65 | storageConfig: config
66 | })
67 |
68 | // @ts-expect-error private field
69 | const t1 = waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED)
70 | relayNode.start()
71 | await t1
72 |
73 | // @ts-expect-error private field
74 | const t2 = waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED)
75 | storageNode.start()
76 | await t2
77 | }, 15000)
78 |
79 | afterEach(async () => {
80 | await tracker.stop()
81 | await relayNode.stop()
82 | await storageNode.stop()
83 | })
84 |
85 | it('initial stream', async () => {
86 | const message = createMockMessage(initialStream)
87 | relayNode.publish(message)
88 | await waitForCondition(() => (storage.store as any).mock.calls.length > 0)
89 | expect(storage.store).toHaveBeenCalledWith(createStreamMessageMatcher(message))
90 | })
91 |
92 | it('add stream', async () => {
93 | const stream = createMockStream()
94 | config.addStream(stream)
95 | const message = createMockMessage(stream)
96 | relayNode.publish(message)
97 | await waitForCondition(() => (storage.store as any).mock.calls.length > 0)
98 | expect(storage.store).toHaveBeenCalledWith(createStreamMessageMatcher(message))
99 | })
100 |
101 | it('remove stream', async () => {
102 | config.removeStream(initialStream)
103 | await waitForCondition(() => (storageNode.getStreams().length === 0))
104 | })
105 | })
106 |
--------------------------------------------------------------------------------
/src/NetworkNode.ts:
--------------------------------------------------------------------------------
1 | import { Node, Event as NodeEvent, NodeOptions } from './logic/Node'
2 | import { ForeignResendStrategy, LocalResendStrategy } from './resend/resendStrategies'
3 | import { StreamIdAndPartition } from './identifiers'
4 | import { ControlLayer, MessageLayer } from 'streamr-client-protocol'
5 | import ReadableStream = NodeJS.ReadableStream
6 | import { Storage } from './composition'
7 |
8 | export interface NetworkNodeOptions extends Omit {
9 | storages: Array
10 | }
11 |
12 | /*
13 | Convenience wrapper for building client-facing functionality. Used by broker.
14 | */
15 | export class NetworkNode extends Node {
16 | constructor(opts: NetworkNodeOptions) {
17 | const networkOpts = {
18 | ...opts,
19 | resendStrategies: [
20 | ...opts.storages.map((storage) => new LocalResendStrategy(storage)),
21 | new ForeignResendStrategy(
22 | opts.protocols.trackerNode,
23 | opts.protocols.nodeToNode,
24 | (streamIdAndPartition) => this.getTrackerId(streamIdAndPartition),
25 | (node) => this.isNodePresent(node)
26 | )
27 | ]
28 | }
29 | super(networkOpts)
30 | opts.storages.forEach((storage) => {
31 | this.addMessageListener((msg: MessageLayer.StreamMessage) => storage.store(msg))
32 | })
33 | }
34 |
35 | publish(streamMessage: MessageLayer.StreamMessage): void {
36 | this.onDataReceived(streamMessage)
37 | }
38 |
39 | addMessageListener(cb: (msg: MessageLayer.StreamMessage) => void): void {
40 | this.on(NodeEvent.UNSEEN_MESSAGE_RECEIVED, cb)
41 | }
42 |
43 | subscribe(streamId: string, streamPartition: number): void {
44 | this.subscribeToStreamIfHaveNotYet(new StreamIdAndPartition(streamId, streamPartition))
45 | }
46 |
47 | unsubscribe(streamId: string, streamPartition: number): void {
48 | this.unsubscribeFromStream(new StreamIdAndPartition(streamId, streamPartition))
49 | }
50 |
51 | requestResendLast(
52 | streamId: string,
53 | streamPartition: number,
54 | requestId: string,
55 | numberLast: number
56 | ): ReadableStream {
57 | const request = new ControlLayer.ResendLastRequest({
58 | requestId, streamId, streamPartition, numberLast, sessionToken: null
59 | })
60 | return this.requestResend(request, null)
61 | }
62 |
63 | requestResendFrom(
64 | streamId: string,
65 | streamPartition: number,
66 | requestId: string,
67 | fromTimestamp: number,
68 | fromSequenceNo: number,
69 | publisherId: string | null
70 | ): ReadableStream {
71 | const request = new ControlLayer.ResendFromRequest({
72 | requestId,
73 | streamId,
74 | streamPartition,
75 | fromMsgRef: new MessageLayer.MessageRef(fromTimestamp, fromSequenceNo),
76 | publisherId,
77 | sessionToken: null
78 | })
79 | return this.requestResend(request, null)
80 | }
81 |
82 | requestResendRange(streamId: string,
83 | streamPartition: number,
84 | requestId: string,
85 | fromTimestamp: number,
86 | fromSequenceNo: number,
87 | toTimestamp: number,
88 | toSequenceNo: number,
89 | publisherId: string | null,
90 | msgChainId: string | null
91 | ): ReadableStream {
92 | const request = new ControlLayer.ResendRangeRequest({
93 | requestId,
94 | streamId,
95 | streamPartition,
96 | fromMsgRef: new MessageLayer.MessageRef(fromTimestamp, fromSequenceNo),
97 | toMsgRef: new MessageLayer.MessageRef(toTimestamp, toSequenceNo),
98 | publisherId,
99 | msgChainId,
100 | sessionToken: null
101 | })
102 | return this.requestResend(request, null)
103 | }
104 | }
105 |
--------------------------------------------------------------------------------
/src/logic/InstructionThrottler.ts:
--------------------------------------------------------------------------------
1 | import { cancelable, CancelablePromiseType } from 'cancelable-promise'
2 | import { StreamIdAndPartition, StreamKey } from '../identifiers'
3 | import { TrackerLayer } from 'streamr-client-protocol'
4 | import { Logger } from "../helpers/Logger"
5 |
6 | interface Queue {
7 | [key: string]: {
8 | instructionMessage: TrackerLayer.InstructionMessage
9 | trackerId: string
10 | }
11 | }
12 |
13 | type HandleFn = (instructionMessage: TrackerLayer.InstructionMessage, trackerId: string) => Promise
14 |
15 | /**
16 | * InstructionThrottler makes sure that
17 | * 1. no more than one instruction is handled at a time
18 | * 2. any new instructions arriving while an instruction is being handled are queued in a
19 | * way where only the most latest instruction per streamId is kept in queue.
20 | */
21 | export class InstructionThrottler {
22 | private readonly logger: Logger
23 | private readonly handleFn: HandleFn
24 | private queue: Queue = {} // streamId => instructionMessage
25 | private instructionCounter: { [key: string]: number } = {} // streamId => counter
26 | private ongoingPromises: {
27 | [key: string]: {
28 | promise: CancelablePromiseType | null
29 | handling: boolean
30 | }
31 | }
32 |
33 | constructor(handleFn: HandleFn) {
34 | this.logger = new Logger(module)
35 | this.handleFn = handleFn
36 | this.ongoingPromises = {}
37 | }
38 |
39 | add(instructionMessage: TrackerLayer.InstructionMessage, trackerId: string): void {
40 | const streamId = StreamIdAndPartition.fromMessage(instructionMessage).key()
41 | if (!this.instructionCounter[streamId] || this.instructionCounter[streamId] <= instructionMessage.counter) {
42 | this.instructionCounter[streamId] = instructionMessage.counter
43 | this.queue[StreamIdAndPartition.fromMessage(instructionMessage).key()] = {
44 | instructionMessage,
45 | trackerId
46 | }
47 |
48 | if (!this.ongoingPromises[streamId]) {
49 | this.ongoingPromises[streamId] = {
50 | promise: null,
51 | handling: false
52 | }
53 | }
54 | if (!this.ongoingPromises[streamId].handling) {
55 | this.invokeHandleFnWithLock(streamId).catch((err) => {
56 | this.logger.warn("error handling instruction, reason: %s", err)
57 | })
58 | }
59 | }
60 | }
61 |
62 | removeStreamId(streamId: StreamKey): void {
63 | delete this.queue[streamId]
64 | delete this.instructionCounter[streamId]
65 | if (this.ongoingPromises[streamId]) {
66 | this.ongoingPromises[streamId].promise!.cancel()
67 | }
68 | delete this.ongoingPromises[streamId]
69 | }
70 |
71 | isIdle(): boolean {
72 | return !Object.values(this.ongoingPromises).some((p) => p.handling)
73 | }
74 |
75 | reset(): void {
76 | this.queue = {}
77 | this.instructionCounter = {}
78 | Object.keys(this.ongoingPromises).forEach((streamId) => {
79 | if (this.ongoingPromises[streamId]) {
80 | this.ongoingPromises[streamId].promise!.cancel()
81 | }
82 | delete this.ongoingPromises[streamId]
83 | })
84 | this.ongoingPromises = {}
85 | }
86 |
87 | private async invokeHandleFnWithLock(streamId: string): Promise {
88 | if (!this.queue[streamId]) {
89 | this.ongoingPromises[streamId].handling = false
90 | return
91 | }
92 | this.ongoingPromises[streamId].handling = true
93 |
94 | const { instructionMessage, trackerId } = this.queue[streamId]
95 | delete this.queue[streamId]
96 |
97 | try {
98 | this.ongoingPromises[streamId].promise = cancelable(this.handleFn(instructionMessage, trackerId))
99 | await this.ongoingPromises[streamId].promise
100 | } catch (err) {
101 | this.logger.warn('handling InstructionMessage threw, error %j', err)
102 | } finally {
103 | this.invokeHandleFnWithLock(streamId)
104 | }
105 | }
106 | }
107 |
--------------------------------------------------------------------------------
/test/integration/resend-request-on-streams-with-no-activity.test.ts:
--------------------------------------------------------------------------------
1 | import { NetworkNode } from '../NetworkNode'
2 | import { Tracker } from '../logic/Tracker'
3 | import { MessageLayer } from 'streamr-client-protocol'
4 | import { waitForEvent, waitForStreamToEnd, toReadableStream } from 'streamr-test-utils'
5 |
6 | import { startNetworkNode, startTracker, startStorageNode } from '../../src/composition'
7 | import { Event } from '../../src/protocol/TrackerServer'
8 | import { StreamIdAndPartition } from '../../src/identifiers'
9 | import { MockStorageConfig } from './MockStorageConfig'
10 |
11 | const { StreamMessage, MessageID, MessageRef } = MessageLayer
12 |
13 | describe('resend requests on streams with no activity', () => {
14 | let tracker: Tracker
15 | let subscriberOne: NetworkNode
16 | let subscriberTwo: NetworkNode
17 | let storageNode: NetworkNode
18 |
19 | beforeEach(async () => {
20 | tracker = await startTracker({
21 | host: '127.0.0.1',
22 | port: 32904,
23 | id: 'tracker'
24 | })
25 | subscriberOne = await startNetworkNode({
26 | host: '127.0.0.1',
27 | port: 32905,
28 | trackers: [tracker.getAddress()],
29 | id: 'subscriberOne'
30 | })
31 | subscriberTwo = await startNetworkNode({
32 | host: '127.0.0.1',
33 | port: 32906,
34 | trackers: [tracker.getAddress()],
35 | id: 'subscriberTwo'
36 | })
37 | const storageConfig = new MockStorageConfig()
38 | storageConfig.addStream(new StreamIdAndPartition('streamId', 0))
39 | storageNode = await startStorageNode({
40 | host: '127.0.0.1',
41 | port: 32907,
42 | trackers: [tracker.getAddress()],
43 | id: 'storageNode',
44 | storages: [{
45 | store: () => {},
46 | requestLast: () => toReadableStream(
47 | new StreamMessage({
48 | messageId: new MessageID('streamId', 0, 756, 0, 'publisherId', 'msgChainId'),
49 | prevMsgRef: new MessageRef(666, 50),
50 | content: {},
51 | }),
52 | new StreamMessage({
53 | messageId: new MessageID('streamId', 0, 800, 0, 'publisherId', 'msgChainId'),
54 | prevMsgRef: new MessageRef(756, 0),
55 | content: {},
56 | }),
57 | new StreamMessage({
58 | messageId: new MessageID('streamId', 0, 950, 0, 'publisherId', 'msgChainId'),
59 | prevMsgRef: new MessageRef(800, 0),
60 | content: {},
61 | }),
62 | ),
63 | requestFrom: () => toReadableStream(
64 | new StreamMessage({
65 | messageId: new MessageID('streamId', 0, 666, 0, 'publisherId', 'msgChainId'),
66 | content: {},
67 | }),
68 | ),
69 | requestRange: () => toReadableStream(),
70 | }],
71 | storageConfig
72 | })
73 |
74 | subscriberOne.start()
75 | subscriberTwo.start()
76 | storageNode.start()
77 |
78 | await Promise.all([
79 | // @ts-expect-error private method
80 | waitForEvent(tracker.trackerServer, Event.NODE_STATUS_RECEIVED),
81 | // @ts-expect-error private method
82 | waitForEvent(tracker.trackerServer, Event.NODE_STATUS_RECEIVED),
83 | // @ts-expect-error private method
84 | waitForEvent(tracker.trackerServer, Event.NODE_STATUS_RECEIVED),
85 | ])
86 | })
87 |
88 | afterEach(async () => {
89 | await storageNode.stop()
90 | await subscriberOne.stop()
91 | await subscriberTwo.stop()
92 | await tracker.stop()
93 | })
94 |
95 | it('resend request works on streams that are not subscribed to', async () => {
96 | const stream = subscriberOne.requestResendLast('streamId', 0, 'requestId', 10)
97 | // @ts-expect-error private method
98 | await waitForEvent(tracker.trackerServer, Event.STORAGE_NODES_REQUEST)
99 | const data = await waitForStreamToEnd(stream as any)
100 | expect(data.length).toEqual(3)
101 | })
102 | })
103 |
--------------------------------------------------------------------------------
/test/integration/tracker-instructions.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { NetworkNode } from '../../src/NetworkNode'
3 | import { waitForCondition, waitForEvent } from 'streamr-test-utils'
4 | import { TrackerLayer } from 'streamr-client-protocol'
5 |
6 | import { startNetworkNode, startTracker } from '../../src/composition'
7 | import { Event as TrackerServerEvent } from '../../src/protocol/TrackerServer'
8 | import { Event as NodeEvent } from '../../src/logic/Node'
9 | import { StreamIdAndPartition } from '../../src/identifiers'
10 | import { getTopology } from '../../src/logic/trackerSummaryUtils'
11 |
12 | describe('check tracker, nodes and statuses from nodes', () => {
13 | let tracker: Tracker
14 | const trackerPort = 32900
15 |
16 | let node1: NetworkNode
17 | const port1 = 33971
18 |
19 | let node2: NetworkNode
20 | const port2 = 33972
21 |
22 | const s1 = new StreamIdAndPartition('stream-1', 0)
23 |
24 | beforeEach(async () => {
25 | tracker = await startTracker({
26 | host: '127.0.0.1',
27 | port: trackerPort,
28 | id: 'tracker'
29 | })
30 | // @ts-expect-error private method
31 | tracker.formAndSendInstructions = () => {}
32 | node1 = await startNetworkNode({
33 | host: '127.0.0.1',
34 | port: port1,
35 | id: 'node1',
36 | trackers: [tracker.getAddress()],
37 | disconnectionWaitTime: 200
38 | })
39 | node2 = await startNetworkNode({
40 | host: '127.0.0.1',
41 | port: port2,
42 | id: 'node2',
43 | trackers: [tracker.getAddress()],
44 | disconnectionWaitTime: 200
45 | })
46 |
47 | node1.subscribeToStreamIfHaveNotYet(s1)
48 | node2.subscribeToStreamIfHaveNotYet(s1)
49 |
50 | node1.start()
51 | node2.start()
52 |
53 | await Promise.all([
54 | // @ts-expect-error private variable
55 | waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED),
56 | // @ts-expect-error private variable
57 | waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED)
58 | ])
59 | })
60 |
61 | afterEach(async () => {
62 | await node1.stop()
63 | await node2.stop()
64 | await tracker.stop()
65 | })
66 |
67 | it('if failed to follow tracker instructions, inform tracker about current status', async () => {
68 | const trackerInstruction1 = new TrackerLayer.InstructionMessage({
69 | requestId: 'requestId',
70 | streamId: s1.id,
71 | streamPartition: s1.partition,
72 | nodeIds: ['node2', 'unknown'],
73 | counter: 0
74 | })
75 |
76 | const trackerInstruction2 = new TrackerLayer.InstructionMessage({
77 | requestId: 'requestId',
78 | streamId: s1.id,
79 | streamPartition: s1.partition,
80 | nodeIds: ['node1', 'unknown'],
81 | counter: 0
82 | })
83 |
84 | await Promise.race([
85 | node1.onTrackerInstructionReceived('tracker', trackerInstruction1),
86 | node2.onTrackerInstructionReceived('tracker', trackerInstruction2)
87 | ]).catch(() => {})
88 |
89 | await Promise.race([
90 | waitForEvent(node1, NodeEvent.NODE_SUBSCRIBED),
91 | waitForEvent(node2, NodeEvent.NODE_SUBSCRIBED)
92 | ])
93 |
94 | await Promise.all([
95 | // @ts-expect-error private variable
96 | waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED),
97 | // @ts-expect-error private variable
98 | waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED)
99 | ])
100 |
101 | await waitForCondition(() => node1.getNeighbors().length > 0)
102 | await waitForCondition(() => node2.getNeighbors().length > 0)
103 |
104 | expect(getTopology(tracker.getOverlayPerStream(), tracker.getOverlayConnectionRtts())).toEqual({
105 | 'stream-1::0': {
106 | node1: [{neighborId: 'node2', rtt: null}],
107 | node2: [{neighborId: 'node1', rtt: null}],
108 | }
109 | })
110 |
111 | expect(node1.getNeighbors()).toEqual(['node2'])
112 | expect(node2.getNeighbors()).toEqual(['node1'])
113 | })
114 | })
115 |
--------------------------------------------------------------------------------
/test/integration/l1-resend-requests.test.ts:
--------------------------------------------------------------------------------
1 | import { NetworkNode } from '../../src/NetworkNode'
2 | import { MessageLayer, ControlLayer } from 'streamr-client-protocol'
3 | import { waitForEvent, waitForStreamToEnd, toReadableStream } from 'streamr-test-utils'
4 |
5 | import { startNetworkNode, startTracker, Tracker } from '../../src/composition'
6 | import { Event as TrackerServerEvent } from '../../src/protocol/TrackerServer'
7 |
8 | const { ControlMessage } = ControlLayer
9 | const { StreamMessage, MessageID, MessageRef } = MessageLayer
10 |
11 | const typesOfStreamItems = async (stream: NodeJS.ReadableStream) => {
12 | const arr = await waitForStreamToEnd(stream as any)
13 | return arr.map((msg: any) => msg.type)
14 | }
15 |
16 | /**
17 | * This test verifies that a node can fulfill resend requests at L1. This means
18 | * that the node
19 | * a) understands and handles resend requests,
20 | * b) can respond with resend responses, and finally,
21 | * c) uses its local storage to find messages.
22 | */
23 | describe('resend requests are fulfilled at L1', () => {
24 | let tracker: Tracker
25 | let contactNode: NetworkNode
26 |
27 | beforeEach(async () => {
28 | tracker = await startTracker({
29 | host: '127.0.0.1',
30 | port: 28600,
31 | id: 'tracker'
32 | })
33 | contactNode = await startNetworkNode({
34 | host: '127.0.0.1',
35 | port: 28601,
36 | id: 'contactNode',
37 | trackers: [tracker.getAddress()],
38 | storages: [{
39 | store: () => {},
40 | requestLast: () => toReadableStream(
41 | new StreamMessage({
42 | messageId: new MessageID('streamId', 0, 666, 50, 'publisherId', 'msgChainId'),
43 | content: {},
44 | }),
45 | new StreamMessage({
46 | messageId: new MessageID('streamId', 0, 756, 0, 'publisherId', 'msgChainId'),
47 | prevMsgRef: new MessageRef(666, 50),
48 | content: {},
49 | }),
50 | new StreamMessage({
51 | messageId: new MessageID('streamId', 0, 800, 0, 'publisherId', 'msgChainId'),
52 | prevMsgRef: new MessageRef(756, 0),
53 | content: {},
54 | })
55 | ),
56 | requestFrom: () => toReadableStream(
57 | new StreamMessage({
58 | messageId: new MessageID('streamId', 0, 666, 50, 'publisherId', 'msgChainId'),
59 | content: {},
60 | }),
61 | ),
62 | requestRange: () => toReadableStream(),
63 | }]
64 | })
65 | contactNode.start()
66 | contactNode.subscribe('streamId', 0)
67 |
68 | // @ts-expect-error private field
69 | await waitForEvent(tracker.trackerServer, TrackerServerEvent.NODE_STATUS_RECEIVED)
70 | })
71 |
72 | afterEach(async () => {
73 | await contactNode.stop()
74 | await tracker.stop()
75 | })
76 |
77 | test('requestResendLast', async () => {
78 | const stream = contactNode.requestResendLast('streamId', 0, 'requestId', 10)
79 | const events = await typesOfStreamItems(stream)
80 |
81 | expect(events).toEqual([
82 | ControlMessage.TYPES.UnicastMessage,
83 | ControlMessage.TYPES.UnicastMessage,
84 | ControlMessage.TYPES.UnicastMessage,
85 | ])
86 | })
87 |
88 | test('requestResendFrom', async () => {
89 | const stream = contactNode.requestResendFrom(
90 | 'streamId',
91 | 0,
92 | 'requestId',
93 | 666,
94 | 0,
95 | 'publisherId'
96 | )
97 | const events = await typesOfStreamItems(stream)
98 |
99 | expect(events).toEqual([
100 | ControlMessage.TYPES.UnicastMessage,
101 | ])
102 | })
103 |
104 | test('requestResendRange', async () => {
105 | const stream = contactNode.requestResendRange(
106 | 'streamId',
107 | 0,
108 | 'requestId',
109 | 666,
110 | 0,
111 | 999,
112 | 0,
113 | 'publisherId',
114 | 'msgChainId'
115 | )
116 | const events = await typesOfStreamItems(stream)
117 |
118 | expect(events).toEqual([])
119 | })
120 | })
121 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # NOTICE! This repository has been moved under [network monorepo](https://github.com/streamr-dev/network-monorepo)
2 |
3 |
4 |
5 | 
6 |
7 |
8 |
9 | # streamr-network
10 |
11 | [](https://github.com/streamr-dev/network/actions)
12 | [](https://www.npmjs.com/package/streamr-network)
13 | [](https://github.com/streamr-dev/network/)
14 | [](https://discord.gg/FVtAph9cvz)
15 |
16 |
17 | An extendible implementation of the server-side
18 | [Streamr Protocol](https://github.com/streamr-dev/streamr-specs/blob/master/PROTOCOL.md) logic written in TypeScript.
19 | The package mostly acts as a library for other packages wishing to implement a broker node, but additionally
20 | provides a full tracker executable, and a stripped-down network node executable.
21 |
22 |
23 | The primary executable for running a broker node in the Streamr Network resides in the
24 | [streamr-broker](https://github.com/streamr-dev/broker) package. Although _streamr-network_ contains a
25 | fully-operational minimal network node implementation, we recommend running the node executable found in
26 | _streamr-broker_ as it includes useful client-facing features for interacting with the Streamr Network.
27 |
28 | The [wiki](https://github.com/streamr-dev/network/wiki) outlines the technical and architectural
29 | decisions of the project. It provides thorough explanations of some the more involved features.
30 | A glossary is also included.
31 |
32 | [API Documentation](https://streamr-dev.github.io/network/)
33 |
34 | ## Table of Contents
35 | - [Install](#install)
36 | - [Run](#run)
37 | - [Develop](#develop)
38 | - [Release](#release)
39 |
40 | ## Install
41 |
42 | Prerequisites are [Node.js](https://nodejs.org/) `14.x` and npm version `>=6.14`.
43 |
44 | You can install streamr-network as a library in your project using npm:
45 |
46 | ```bash
47 | npm install streamr-network --save
48 | ```
49 |
50 | To install streamr-network system-wide:
51 | ```bash
52 | npm install streamr-network --global
53 | ```
54 |
55 | ## Run
56 |
57 | Run an example network of 100 nodes (locally):
58 |
59 | npm run network
60 |
61 | ## Develop
62 |
63 | Install dependencies:
64 |
65 | npm ci
66 |
67 | Run the tests:
68 |
69 | npm run test
70 |
71 | To build project:
72 |
73 | npm run build
74 |
75 | We use [eslint](https://github.com/eslint/eslint) for code formatting:
76 |
77 | npm run eslint
78 |
79 | Code coverage:
80 |
81 | npm run coverage
82 |
83 | ### Debug
84 |
85 | To get all debug messages:
86 |
87 | LOG_LEVEL=debug
88 |
89 | ... or adjust debugging to desired level:
90 |
91 | LOG_LEVEL=[debug|info|warn|error]
92 |
93 | To disable all logs
94 |
95 | NOLOG=true
96 |
97 | ### Regenerate self-signed certificate fixture
98 | To regenerate self signed certificate in `./test/fixtures` run:
99 |
100 | ```bash
101 | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 36500 -nodes -subj "/CN=localhost"
102 | ```
103 |
104 | ## Release
105 |
106 | Publishing to NPM is automated via Github Actions. Follow the steps below to publish stable (`latest`) or `beta`.
107 |
108 | ### Publishing stable (latest)
109 | 1. `git checkout master && git pull`
110 | 2. Update version with either `npm version [patch|minor|major]`. Use semantic versioning
111 | https://semver.org/. Files package.json and package-lock.json will be automatically updated, and an appropriate git commit and tag created.
112 | 3. `git push --follow-tags`
113 | 4. Wait for Github Actions to run tests
114 | 5. If tests passed, Github Actions will publish the new version to NPM
115 |
116 | ### Publishing beta
117 | 1. Update version with either `npm version [prepatch|preminor|premajor] --preid=beta`. Use semantic versioning
118 | https://semver.org/. Files package.json and package-lock.json will be automatically updated, and an appropriate git commit and tag created.
119 | 2. `git push --follow-tags`
120 | 3. Wait for Github Actions to run tests
121 | 4. If tests passed, Github Actions will publish the new version to NPM
122 |
--------------------------------------------------------------------------------
/src/logic/PerStreamMetrics.ts:
--------------------------------------------------------------------------------
1 | import speedometer from 'speedometer'
2 |
3 | interface AllMetrics {
4 | resends: M
5 | trackerInstructions: M
6 | onDataReceived: M
7 | "onDataReceived:ignoredDuplicate": M
8 | propagateMessage: M
9 | }
10 |
11 | interface Metric {
12 | total: number
13 | last: number
14 | rate: (delta?: number) => number
15 | }
16 |
17 | interface ReportedMetric {
18 | total: number
19 | last: number
20 | rate: number
21 | }
22 |
23 | export class PerStreamMetrics {
24 | private readonly streams: { [key: string]: AllMetrics } = {}
25 |
26 | recordResend(streamId: string): void {
27 | this.setUpIfNeeded(streamId)
28 | const { resends } = this.streams[streamId]
29 | resends.total += 1
30 | resends.last += 1
31 | resends.rate(1)
32 | }
33 |
34 | recordTrackerInstruction(streamId: string): void {
35 | this.setUpIfNeeded(streamId)
36 | const { trackerInstructions } = this.streams[streamId]
37 | trackerInstructions.total += 1
38 | trackerInstructions.last += 1
39 | trackerInstructions.rate(1)
40 | }
41 |
42 | recordDataReceived(streamId: string): void {
43 | this.setUpIfNeeded(streamId)
44 | const { onDataReceived } = this.streams[streamId]
45 | onDataReceived.total += 1
46 | onDataReceived.last += 1
47 | onDataReceived.rate(1)
48 | }
49 |
50 | recordIgnoredDuplicate(streamId: string): void {
51 | this.setUpIfNeeded(streamId)
52 | const ignoredDuplicate = this.streams[streamId]['onDataReceived:ignoredDuplicate']
53 | ignoredDuplicate.total += 1
54 | ignoredDuplicate.last += 1
55 | ignoredDuplicate.rate(1)
56 | }
57 |
58 | recordPropagateMessage(streamId: string): void {
59 | this.setUpIfNeeded(streamId)
60 | const { propagateMessage } = this.streams[streamId]
61 | propagateMessage.total += 1
62 | propagateMessage.last += 1
63 | propagateMessage.rate(1)
64 | }
65 |
66 | report(): { [key: string]: AllMetrics } {
67 | const result: { [key: string]: AllMetrics } = {}
68 | Object.entries(this.streams).forEach(([streamId, metrics]) => {
69 | result[streamId] = {
70 | resends: {
71 | rate: metrics.resends.rate(),
72 | total: metrics.resends.total,
73 | last: metrics.resends.last
74 | },
75 | trackerInstructions: {
76 | rate: metrics.trackerInstructions.rate(),
77 | total: metrics.trackerInstructions.total,
78 | last: metrics.trackerInstructions.last
79 | },
80 | onDataReceived: {
81 | rate: metrics.onDataReceived.rate(),
82 | total: metrics.onDataReceived.total,
83 | last: metrics.onDataReceived.last
84 | },
85 | "onDataReceived:ignoredDuplicate": {
86 | rate: metrics["onDataReceived:ignoredDuplicate"].rate(),
87 | total: metrics["onDataReceived:ignoredDuplicate"].total,
88 | last: metrics["onDataReceived:ignoredDuplicate"].last
89 | },
90 | propagateMessage: {
91 | rate: metrics.propagateMessage.rate(),
92 | total: metrics.propagateMessage.total,
93 | last: metrics.propagateMessage.last
94 | }
95 | }
96 | })
97 | return result
98 | }
99 |
100 | private setUpIfNeeded(streamId: string): void {
101 | if (!this.streams[streamId]) {
102 | this.streams[streamId] = {
103 | resends: {
104 | rate: speedometer(),
105 | last: 0,
106 | total: 0,
107 | },
108 | trackerInstructions: {
109 | rate: speedometer(),
110 | last: 0,
111 | total: 0
112 | },
113 | onDataReceived: {
114 | rate: speedometer(),
115 | last: 0,
116 | total: 0
117 | },
118 | 'onDataReceived:ignoredDuplicate': {
119 | rate: speedometer(),
120 | last: 0,
121 | total: 0
122 | },
123 | propagateMessage: {
124 | rate: speedometer(),
125 | last: 0,
126 | total: 0
127 | }
128 | }
129 | }
130 | }
131 | }
132 |
--------------------------------------------------------------------------------
/test/integration/request-resend-from-uninvolved-node.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import { NetworkNode } from '../../src/NetworkNode'
3 | import { MessageLayer, ControlLayer } from 'streamr-client-protocol'
4 | import { waitForStreamToEnd, waitForEvent, toReadableStream } from 'streamr-test-utils'
5 |
6 | import { startNetworkNode, startStorageNode, startTracker } from '../../src/composition'
7 | import { Event as NodeEvent } from '../../src/logic/Node'
8 | import { StreamIdAndPartition } from '../../src/identifiers'
9 | import { MockStorageConfig } from './MockStorageConfig'
10 |
11 | const { ControlMessage } = ControlLayer
12 | const { StreamMessage, MessageID, MessageRef } = MessageLayer
13 |
14 | const typesOfStreamItems = async (stream: any) => {
15 | const arr = await waitForStreamToEnd(stream)
16 | return arr.map((msg: any) => msg.type)
17 | }
18 |
19 | /**
20 | * This test verifies that requesting a resend of stream S from a node that is
21 | * not subscribed to S (is uninvolved) works as expected. That is, the resend
22 | * request will be fulfilled via L3 by delegating & proxying through a storage
23 | * node.
24 | */
25 | describe('request resend from uninvolved node', () => {
26 | let tracker: Tracker
27 | let uninvolvedNode: NetworkNode
28 | let involvedNode: NetworkNode
29 | let storageNode: NetworkNode
30 |
31 | beforeAll(async () => {
32 | tracker = await startTracker({
33 | host: '127.0.0.1',
34 | port: 28640,
35 | id: 'tracker'
36 | })
37 | uninvolvedNode = await startNetworkNode({
38 | host: '127.0.0.1',
39 | port: 28641,
40 | id: 'uninvolvedNode',
41 | trackers: [tracker.getAddress()],
42 | storages: [{
43 | store: () => {},
44 | requestLast: () => toReadableStream(),
45 | requestFrom: () => toReadableStream(),
46 | requestRange: () => toReadableStream()
47 | }]
48 | })
49 | involvedNode = await startNetworkNode({
50 | host: '127.0.0.1',
51 | port: 28642,
52 | id: 'involvedNode',
53 | trackers: [tracker.getAddress()],
54 | storages: [{
55 | store: () => {},
56 | requestLast: () => toReadableStream(),
57 | requestFrom: () => toReadableStream(),
58 | requestRange: () => toReadableStream()
59 | }]
60 | })
61 | const mockRequest = () => toReadableStream(
62 | new StreamMessage({
63 | messageId: new MessageID('streamId', 0, 756, 0, 'publisherId', 'msgChainId'),
64 | prevMsgRef: new MessageRef(666, 50),
65 | content: {},
66 | }),
67 | new StreamMessage({
68 | messageId: new MessageID('streamId', 0, 800, 0, 'publisherId', 'msgChainId'),
69 | prevMsgRef: new MessageRef(756, 0),
70 | content: {},
71 | })
72 | )
73 | const storageConfig = new MockStorageConfig()
74 | storageConfig.addStream(new StreamIdAndPartition('streamId', 0))
75 | storageNode = await startStorageNode({
76 | host: '127.0.0.1',
77 | port: 28643,
78 | id: 'storageNode',
79 | trackers: [tracker.getAddress()],
80 | storages: [{
81 | store: () => {},
82 | requestLast: mockRequest,
83 | requestFrom: mockRequest,
84 | requestRange: mockRequest
85 | }],
86 | storageConfig
87 | })
88 |
89 | involvedNode.subscribe('streamId', 0)
90 |
91 | uninvolvedNode.start()
92 | involvedNode.start()
93 | storageNode.start()
94 |
95 | await Promise.all([
96 | waitForEvent(involvedNode, NodeEvent.NODE_SUBSCRIBED),
97 | waitForEvent(storageNode, NodeEvent.NODE_SUBSCRIBED)
98 | ])
99 | })
100 |
101 | afterAll(async () => {
102 | await uninvolvedNode.stop()
103 | await involvedNode.stop()
104 | await storageNode.stop()
105 | await tracker.stop()
106 | })
107 |
108 | test('requesting resend from uninvolved node is fulfilled using l3', async () => {
109 | const stream = uninvolvedNode.requestResendLast('streamId', 0, 'requestId', 10)
110 | const events = await typesOfStreamItems(stream)
111 |
112 | expect(events).toEqual([
113 | ControlMessage.TYPES.UnicastMessage,
114 | ControlMessage.TYPES.UnicastMessage,
115 | ])
116 | // @ts-expect-error private field
117 | expect(uninvolvedNode.streams.getStreamsAsKeys()).toEqual([]) // sanity check
118 | })
119 | })
120 |
--------------------------------------------------------------------------------
/test/unit/LocationManager.test.ts:
--------------------------------------------------------------------------------
1 | import { LocationManager } from '../../src/logic/LocationManager'
2 |
3 | describe('LocationManager', () => {
4 | let locationManager: LocationManager
5 |
6 | beforeEach(() => {
7 | locationManager = new LocationManager()
8 | })
9 |
10 | describe('#updateLocation', () => {
11 | it('passing valid location', () => {
12 | locationManager.updateLocation({
13 | nodeId: 'nodeId',
14 | location: {
15 | city: 'Helsinki',
16 | country: 'Finland',
17 | latitude: null,
18 | longitude: null
19 | },
20 | address: 'ws://193.166.4.1'
21 | })
22 | expect(locationManager.getNodeLocation('nodeId')).toEqual({
23 | city: 'Helsinki',
24 | country: 'Finland',
25 | latitude: null,
26 | longitude: null
27 | })
28 | })
29 |
30 | it('passing empty location but valid address', () => {
31 | locationManager.updateLocation({
32 | nodeId: 'nodeId',
33 | location: null,
34 | address: 'ws://193.166.4.1'
35 | })
36 | expect(locationManager.getNodeLocation('nodeId')).toEqual({
37 | city: '',
38 | country: 'FI',
39 | latitude: 60.1708,
40 | longitude: 24.9375
41 | })
42 | })
43 |
44 | it('passing empty location and address', () => {
45 | locationManager.updateLocation({
46 | nodeId: 'nodeId',
47 | location: null,
48 | address: null as any
49 | })
50 | expect(locationManager.getNodeLocation('nodeId')).toBeUndefined()
51 | })
52 |
53 | it('passing invalid address causes error to be logged', () => {
54 | // @ts-expect-error private field
55 | locationManager.logger.warn = jest.fn()
56 | locationManager.updateLocation({
57 | nodeId: 'nodeId',
58 | location: null,
59 | address: 'dsjklgasdjklgjasdklgj'
60 | })
61 | expect(locationManager.getNodeLocation('nodeId')).toBeUndefined()
62 | // @ts-expect-error private field
63 | expect(locationManager.logger.warn).toHaveBeenCalled()
64 | })
65 |
66 | it('passing invalid location to already set location does not overwrite', () => {
67 | locationManager.updateLocation({
68 | nodeId: 'nodeId',
69 | location: {
70 | city: 'Helsinki',
71 | country: 'Finland',
72 | latitude: null,
73 | longitude: null
74 | },
75 | address: 'ws://193.166.4.1'
76 | })
77 | locationManager.updateLocation({
78 | nodeId: 'nodeId',
79 | location: null,
80 | address: 'ws://193.166.4.1'
81 | })
82 | expect(locationManager.getNodeLocation('nodeId')).toEqual({
83 | city: 'Helsinki',
84 | country: 'Finland',
85 | latitude: null,
86 | longitude: null
87 | })
88 | })
89 | })
90 |
91 | it('getAllNodeLocations', () => {
92 | locationManager.updateLocation({
93 | nodeId: 'node-1',
94 | location: null,
95 | address: 'ws://193.166.4.1'
96 | })
97 | locationManager.updateLocation({
98 | nodeId: 'node-2',
99 | location: null,
100 | address: 'ws://8.8.8.8'
101 | })
102 | expect(locationManager.getAllNodeLocations()).toEqual({
103 | 'node-1': {
104 | city: '',
105 | country: 'FI',
106 | latitude: 60.1708,
107 | longitude: 24.9375
108 | },
109 | 'node-2': {
110 | city: '',
111 | country: 'US',
112 | latitude: 37.751,
113 | longitude: -97.822
114 | },
115 | })
116 | })
117 |
118 | it('removeNode', () => {
119 | locationManager.updateLocation({
120 | nodeId: 'node-1',
121 | location: null,
122 | address: 'ws://193.166.4.1'
123 | })
124 | locationManager.updateLocation({
125 | nodeId: 'node-2',
126 | location: null,
127 | address: 'ws://8.8.8.8'
128 | })
129 | locationManager.removeNode('node-2')
130 | expect(locationManager.getAllNodeLocations()).toEqual({
131 | 'node-1': {
132 | city: '',
133 | country: 'FI',
134 | latitude: 60.1708,
135 | longitude: 24.9375
136 | }
137 | })
138 | })
139 | })
140 |
--------------------------------------------------------------------------------
/src/helpers/MetricsContext.ts:
--------------------------------------------------------------------------------
1 | import speedometer from 'speedometer'
2 |
3 | type QueryFn = () => (Promise | number | Promise> | Record)
4 |
5 | interface IndividualReport {
6 | [key: string]: number | Record | {
7 | rate: number
8 | total: number
9 | last: number
10 | }
11 | }
12 |
13 | interface Report {
14 | peerId: string
15 | startTime: number
16 | currentTime: number
17 | metrics: {
18 | [key: string]: IndividualReport
19 | }
20 | }
21 |
22 | export class Metrics {
23 | private readonly name: string
24 | private readonly queriedMetrics: {
25 | [key: string]: QueryFn
26 | }
27 | private readonly recordedMetrics: {
28 | [key: string]: {
29 | rate: (delta?: number) => number,
30 | last: number,
31 | total: number
32 | }
33 | }
34 | private readonly fixedMetrics: {
35 | [key: string]: number
36 | }
37 |
38 | constructor(name: string) {
39 | this.name = name
40 | this.queriedMetrics = {}
41 | this.recordedMetrics = {}
42 | this.fixedMetrics = {}
43 | }
44 |
45 | addQueriedMetric(name: string, queryFn: QueryFn): Metrics {
46 | this.verifyUniqueness(name)
47 | this.queriedMetrics[name] = queryFn
48 | return this
49 | }
50 |
51 | addRecordedMetric(name: string, windowSizeInSeconds = 5): Metrics {
52 | this.verifyUniqueness(name)
53 | this.recordedMetrics[name] = {
54 | rate: speedometer(windowSizeInSeconds),
55 | last: 0,
56 | total: 0
57 | }
58 | return this
59 | }
60 |
61 | addFixedMetric(name: string, initialValue = 0): Metrics {
62 | this.verifyUniqueness(name)
63 | this.fixedMetrics[name] = initialValue
64 | return this
65 | }
66 |
67 | record(name: string, value: number): Metrics {
68 | if (!this.recordedMetrics[name]) {
69 | throw new Error(`Not a recorded metric "${this.name}.${name}".`)
70 | }
71 | this.recordedMetrics[name].rate(value)
72 | this.recordedMetrics[name].total += value
73 | this.recordedMetrics[name].last += value
74 | return this
75 | }
76 |
77 | set(name: string, value: number): Metrics {
78 | if (this.fixedMetrics[name] === undefined) {
79 | throw new Error(`Not a fixed metric "${this.name}.${name}".`)
80 | }
81 | this.fixedMetrics[name] = value
82 | return this
83 | }
84 |
85 | async report(): Promise {
86 | const queryResults = await Promise.all(
87 | Object.entries(this.queriedMetrics)
88 | .map(async ([name, queryFn]) => [name, await queryFn()])
89 | )
90 | const recordedResults = Object.entries(this.recordedMetrics)
91 | .map(([name, { rate, total, last }]) => [name, {
92 | rate: rate(),
93 | total,
94 | last
95 | }])
96 | const fixedResults = Object.entries(this.fixedMetrics)
97 | return Object.fromEntries([...queryResults, ...recordedResults, ...fixedResults])
98 | }
99 |
100 | clearLast(): void {
101 | Object.values(this.recordedMetrics).forEach((record) => {
102 | // eslint-disable-next-line no-param-reassign
103 | record.last = 0
104 | })
105 | }
106 |
107 | private verifyUniqueness(name: string): void | never {
108 | if (this.queriedMetrics[name] || this.recordedMetrics[name]) {
109 | throw new Error(`Metric "${this.name}.${name}" already registered.`)
110 | }
111 | }
112 | }
113 |
114 | export class MetricsContext {
115 | private readonly peerId: string
116 | private readonly startTime: number
117 | private readonly metrics: {
118 | [key: string]: Metrics
119 | }
120 |
121 | constructor(peerId: string) {
122 | this.peerId = peerId
123 | this.startTime = Date.now()
124 | this.metrics = {}
125 | }
126 |
127 | create(name: string): Metrics {
128 | if (this.metrics[name]) {
129 | throw new Error(`Metrics "${name}" already created.`)
130 | }
131 | this.metrics[name] = new Metrics(name)
132 | return this.metrics[name]
133 | }
134 |
135 | async report(clearLast = false): Promise {
136 | const entries = await Promise.all(
137 | Object.entries(this.metrics)
138 | .map(async ([name, metrics]) => [name, await metrics.report()])
139 | )
140 | if (clearLast) {
141 | Object.values(this.metrics).forEach((metrics) => metrics.clearLast())
142 | }
143 | return {
144 | peerId: this.peerId,
145 | startTime: this.startTime,
146 | currentTime: Date.now(),
147 | metrics: Object.fromEntries(entries),
148 | }
149 | }
150 | }
151 |
--------------------------------------------------------------------------------
/test/integration/ws-endpoint.test.ts:
--------------------------------------------------------------------------------
1 | import { Tracker } from '../../src/logic/Tracker'
2 | import WebSocket from 'ws'
3 | import { waitForEvent, wait } from 'streamr-test-utils'
4 |
5 | import { Event, DisconnectionCode } from '../../src/connection/IWsEndpoint'
6 | import { startEndpoint, WsEndpoint } from '../../src/connection/WsEndpoint'
7 | import { PeerInfo } from '../../src/connection/PeerInfo'
8 | import { startTracker } from '../../src/composition'
9 |
10 | describe('ws-endpoint', () => {
11 | const endpoints: WsEndpoint[] = []
12 |
13 | it('create five endpoints and init connection between them, should be able to start and stop successfully', async () => {
14 | for (let i = 0; i < 5; i++) {
15 | // eslint-disable-next-line no-await-in-loop
16 | const endpoint = await startEndpoint('127.0.0.1', 30690 + i, PeerInfo.newNode(`endpoint-${i}`), null)
17 | .catch((err) => {
18 | throw err
19 | })
20 | endpoints.push(endpoint)
21 | }
22 |
23 | for (let i = 0; i < 5; i++) {
24 | expect(endpoints[i].getPeers().size).toBe(0)
25 | }
26 |
27 | const promises: Promise[] = []
28 |
29 | for (let i = 0; i < 5; i++) {
30 | promises.push(waitForEvent(endpoints[i], Event.PEER_CONNECTED))
31 |
32 | const nextEndpoint = i + 1 === 5 ? endpoints[0] : endpoints[i + 1]
33 |
34 | // eslint-disable-next-line no-await-in-loop
35 | endpoints[i].connect(nextEndpoint.getAddress())
36 | }
37 |
38 | await Promise.all(promises)
39 | await wait(100)
40 |
41 | for (let i = 0; i < 5; i++) {
42 | expect(endpoints[i].getPeers().size).toEqual(2)
43 | }
44 |
45 | for (let i = 0; i < 5; i++) {
46 | // eslint-disable-next-line no-await-in-loop
47 | await endpoints[i].stop()
48 | }
49 | })
50 |
51 | it('peer infos are exchanged between connecting endpoints', async () => {
52 | const endpointOne = await startEndpoint('127.0.0.1', 30695, PeerInfo.newNode('endpointOne'), null)
53 | const endpointTwo = await startEndpoint('127.0.0.1', 30696, PeerInfo.newNode('endpointTwo'), null)
54 |
55 | const e1 = waitForEvent(endpointOne, Event.PEER_CONNECTED)
56 | const e2 = waitForEvent(endpointTwo, Event.PEER_CONNECTED)
57 |
58 | endpointOne.connect(endpointTwo.getAddress())
59 |
60 | const endpointOneArguments = await e1
61 | const endpointTwoArguments = await e2
62 |
63 | expect(endpointOneArguments).toEqual([PeerInfo.newNode('endpointTwo')])
64 | expect(endpointTwoArguments).toEqual([PeerInfo.newNode('endpointOne')])
65 |
66 | await endpointOne.stop()
67 | await endpointTwo.stop()
68 | })
69 |
70 | describe('test direct connections from simple websocket', () => {
71 | const trackerPort = 38481
72 | let tracker: Tracker
73 |
74 | beforeEach(async () => {
75 | tracker = await startTracker({
76 | host: '127.0.0.1',
77 | port: trackerPort,
78 | id: 'tracker'
79 | })
80 | })
81 |
82 | afterEach(async () => {
83 | await tracker.stop()
84 | })
85 |
86 | it('tracker must check all required information for new incoming connection and not crash', async () => {
87 | let ws = new WebSocket(`ws://127.0.0.1:${trackerPort}/ws`)
88 | let close = await waitForEvent(ws, 'close')
89 | expect(close).toEqual([DisconnectionCode.MISSING_REQUIRED_PARAMETER, 'Error: address not given'])
90 |
91 | ws = new WebSocket(`ws://127.0.0.1:${trackerPort}/ws?address`)
92 | close = await waitForEvent(ws, 'close')
93 | expect(close).toEqual([DisconnectionCode.MISSING_REQUIRED_PARAMETER, 'Error: address not given'])
94 |
95 | ws = new WebSocket(`ws://127.0.0.1:${trackerPort}/ws?address=address`)
96 | close = await waitForEvent(ws, 'close')
97 | expect(close).toEqual([DisconnectionCode.MISSING_REQUIRED_PARAMETER, 'Error: peerId not given'])
98 |
99 | ws = new WebSocket(`ws://127.0.0.1:${trackerPort}/ws?address=address`,
100 | undefined,
101 | {
102 | headers: {
103 | 'streamr-peer-id': 'peerId',
104 | }
105 | })
106 | close = await waitForEvent(ws, 'close')
107 | expect(close).toEqual([DisconnectionCode.MISSING_REQUIRED_PARAMETER, 'Error: peerType not given'])
108 |
109 | ws = new WebSocket(`ws://127.0.0.1:${trackerPort}/ws?address=address`,
110 | undefined, {
111 | headers: {
112 | 'streamr-peer-id': 'peerId',
113 | 'streamr-peer-type': 'typiii',
114 | 'control-layer-versions': "2",
115 | 'message-layer-versions': "32"
116 | }
117 | })
118 | close = await waitForEvent(ws, 'close')
119 | expect(close).toEqual([DisconnectionCode.MISSING_REQUIRED_PARAMETER, 'Error: peerType typiii not in peerTypes list'])
120 | })
121 | })
122 | })
123 |
--------------------------------------------------------------------------------