├── .env.dist ├── .eslintignore ├── .eslintrc.js ├── .gitignore ├── .husky └── pre-commit ├── .vim └── coc-settings.json ├── .yarnrc ├── CHANGELOG.md ├── LICENSE ├── Procfile ├── README.md ├── jest.config.js ├── jest.integration.config.js ├── jest.unit.config.js ├── openapi.json ├── package.json ├── pnpm-lock.yaml ├── prettier.config.js ├── seeds ├── nodes-test-network.json └── nodes.json ├── src ├── core │ ├── config │ │ ├── Config.ts │ │ ├── __mocks__ │ │ │ └── configMock.ts │ │ └── __tests__ │ │ │ └── Config.test.ts │ ├── domain │ │ ├── CoreEntity.ts │ │ ├── IUserService.ts │ │ ├── IdentifiedValueObject.ts │ │ ├── Message.ts │ │ ├── Snapshot.ts │ │ ├── Url.ts │ │ ├── ValueObject.ts │ │ ├── VersionedEntity.ts │ │ └── __tests__ │ │ │ ├── Url.test.ts │ │ │ └── VersionedEntity.test.ts │ ├── errors │ │ ├── CustomError.ts │ │ └── __tests__ │ │ │ └── CustomError.test.ts │ ├── infrastructure │ │ ├── Kernel.ts │ │ ├── __tests__ │ │ │ └── Kernel.integration.test.ts │ │ ├── database │ │ │ ├── AppDataSource.ts │ │ │ ├── TestingAppDataSource.ts │ │ │ └── migrations │ │ │ │ ├── 1559296078133-init.ts │ │ │ │ ├── 1559372435536-ledgers.ts │ │ │ │ ├── 1559469975565-index.ts │ │ │ │ ├── 1559975071741-organizations.ts │ │ │ │ ├── 1560152941696-crawl_indexes.ts │ │ │ │ ├── 1563086015292-completed_crawl.ts │ │ │ │ ├── 1570858454796-node-measurement-day.ts │ │ │ │ ├── 1577790139494-time-travel-feature.ts │ │ │ │ ├── 1598083624409-fbas.ts │ │ │ │ ├── 1599197183425-network-rollups.ts │ │ │ │ ├── 1611909155496-horizonUrl.ts │ │ │ │ ├── 1617262226234-fbas_refactoring.ts │ │ │ │ ├── 1625129181262-homeDomain.ts │ │ │ │ ├── 1632481790744-latestLedger.ts │ │ │ │ ├── 1632483433793-latestLedgerCloseTime.ts │ │ │ │ ├── 1632900021069-active-in-scp.ts │ │ │ │ ├── 1634235617874-networkUpdate.ts │ │ │ │ ├── 1637350003769-notify.ts │ │ │ │ ├── 1637591171862-subscription-dates.ts │ │ │ │ ├── 1637837429416-notification-orphan.ts │ │ │ │ ├── 1652883888348-history-scan.ts │ │ │ │ ├── 1653131605238-history-gap.ts │ │ │ │ ├── 1654680277045-notification-enum-history-gap.ts │ │ │ │ ├── 1669033193688-archive-verification.ts │ │ │ │ ├── 1669193270696-archive-verification-v2.ts │ │ │ │ ├── 1669373859398-archive-verification-v3.ts │ │ │ │ ├── 1669729290142-archive-verification-v5.ts │ │ │ │ ├── 1672402482958-versioning.ts │ │ │ │ ├── 1672403715825-versioning-next.ts │ │ │ │ ├── 1672404044492-versioning-seq.ts │ │ │ │ ├── 1672404385799-versioning-indexes.ts │ │ │ │ ├── 1672489594750-organizationId.ts │ │ │ │ ├── 1672914609976-networkQuorumSet.ts │ │ │ │ ├── 1673371281653-update-network.ts │ │ │ │ ├── 1673431104692-passphrase.ts │ │ │ │ ├── 1673434349044-unique-network-id.ts │ │ │ │ ├── 1673440673258-unique-network-id-network-only.ts │ │ │ │ ├── 1673601015030-network-scan.ts │ │ │ │ ├── 1673873199224-org-node.ts │ │ │ │ ├── 1673898386176-last-ip-change.ts │ │ │ │ ├── 1674560008886-node-scan.ts │ │ │ │ ├── 1674643629973-home-domain.ts │ │ │ │ ├── 1674815649104-name-nullable.ts │ │ │ │ ├── 1694520337940-TomlStateMigration.ts │ │ │ │ ├── 1700572695626-connectivity.ts │ │ │ │ ├── 1700658325922-stellar-core-version-behind.ts │ │ │ │ ├── 1701680606580-stellar-core-version-behind.ts │ │ │ │ ├── 1713440843438-lag.ts │ │ │ │ └── 1713780808807-lag-nullable.ts │ │ ├── di │ │ │ ├── container.ts │ │ │ └── di-types.ts │ │ ├── http │ │ │ ├── AxiosHttpService.ts │ │ │ ├── Throttler.ts │ │ │ ├── __tests__ │ │ │ │ └── Throttler.test.ts │ │ │ └── api.ts │ │ └── services │ │ │ ├── LoggerJobMonitor.ts │ │ │ ├── SentryJobMonitor.ts │ │ │ └── __tests__ │ │ │ ├── LoggerJobMonitor.test.ts │ │ │ └── SentryJobMonitor.test.ts │ ├── services │ │ ├── ExceptionLogger.ts │ │ ├── HeartBeater.ts │ │ ├── HttpQueue.ts │ │ ├── HttpService.ts │ │ ├── JobMonitor.ts │ │ ├── LoopTimer.ts │ │ ├── PinoLogger.ts │ │ ├── UserService.ts │ │ ├── __mocks__ │ │ │ ├── ExceptionLoggerMock.ts │ │ │ └── LoggerMock.ts │ │ └── __tests__ │ │ │ ├── HttpQueue.test.ts │ │ │ ├── LoopTimer.test.ts │ │ │ └── UserService.test.ts │ └── utilities │ │ ├── AsyncFunctionStaller.ts │ │ ├── HttpRequestRetry.ts │ │ ├── TestUtils.ts │ │ ├── TypeGuards.ts │ │ ├── __tests__ │ │ ├── FunctionStaller.test.ts │ │ ├── HttpRequestRetry.test.ts │ │ ├── TypeGuards.test.ts │ │ └── getLowestNumber.test.ts │ │ ├── asyncSleep.ts │ │ ├── getDateFromParam.ts │ │ ├── getLowestNumber.ts │ │ ├── getMaximumNumber.ts │ │ ├── isDateString.ts │ │ ├── isZLibError.ts │ │ ├── mapUnknownToError.ts │ │ └── sortDescending.ts ├── history-scan │ ├── README.md │ ├── domain │ │ ├── check-point │ │ │ ├── CheckPointFrequency.ts │ │ │ ├── CheckPointGenerator.ts │ │ │ ├── StandardCheckPointFrequency.ts │ │ │ └── __tests__ │ │ │ │ ├── CheckPointGenerator.test.ts │ │ │ │ └── StandardCheckPointFrequency.test.ts │ │ ├── history-archive │ │ │ ├── Category.ts │ │ │ ├── HASBucketHashExtractor.ts │ │ │ ├── HASValidator.ts │ │ │ ├── HistoryArchiveService.ts │ │ │ ├── HistoryArchiveState.ts │ │ │ ├── UrlBuilder.ts │ │ │ ├── __fixtures__ │ │ │ │ ├── HistoryBaseUrl.ts │ │ │ │ └── getDummyHistoryArchiveState.ts │ │ │ ├── __tests__ │ │ │ │ ├── HASBucketHashExtractor.test.ts │ │ │ │ ├── UrlBuilder.test.ts │ │ │ │ └── hashBucketList.test.ts │ │ │ └── hashBucketList.ts │ │ ├── scan │ │ │ ├── README.md │ │ │ ├── Scan.ts │ │ │ ├── ScanError.ts │ │ │ ├── ScanJob.ts │ │ │ ├── ScanRepository.ts │ │ │ ├── ScanResult.ts │ │ │ ├── ScanSettings.ts │ │ │ ├── ScanSettingsFactory.ts │ │ │ └── __tests__ │ │ │ │ └── ScanSettingsFactory.test.ts │ │ └── scanner │ │ │ ├── ArchivePerformanceTester.ts │ │ │ ├── BucketScanner.ts │ │ │ ├── CategoryScanner.ts │ │ │ ├── CategoryVerificationService.ts │ │ │ ├── CategoryXDRProcessor.ts │ │ │ ├── HasherPool.ts │ │ │ ├── RangeScanner.ts │ │ │ ├── RequestGenerator.ts │ │ │ ├── ScanScheduler.ts │ │ │ ├── ScanState.ts │ │ │ ├── Scanner.ts │ │ │ ├── WorkerPoolLoadTracker.ts │ │ │ ├── XdrStreamReader.ts │ │ │ ├── __fixtures__ │ │ │ ├── bucket.xdr.gz │ │ │ ├── bucket_empty.xdr.gz │ │ │ ├── ledger.xdr.gz │ │ │ ├── ledger_empty.xdr.gz │ │ │ ├── results.xdr.gz │ │ │ ├── results_empty.xdr.gz │ │ │ ├── stellar-history.json │ │ │ ├── transactions.xdr.gz │ │ │ └── transactions_empty.xdr.gz │ │ │ ├── __tests__ │ │ │ ├── BucketScanner.test.ts │ │ │ ├── CategoryScanner.integration.test.ts │ │ │ ├── RangeScanner.test.ts │ │ │ ├── ScanScheduler.test.ts │ │ │ ├── Scanner.test.ts │ │ │ ├── mapHttpQueueErrorToScanError.test.ts │ │ │ └── sortHistoryUrls.test.ts │ │ │ ├── hash-worker.import.js │ │ │ ├── hash-worker.ts │ │ │ ├── mapHttpQueueErrorToScanError.ts │ │ │ ├── sortHistoryUrls.ts │ │ │ └── verification │ │ │ └── empty-transaction-sets │ │ │ ├── EmptyTransactionSetsHashVerifier.ts │ │ │ ├── __tests__ │ │ │ └── EmptyTransactionSetsHashVerifier.test.ts │ │ │ └── hash-policies │ │ │ ├── FirstLedgerHashPolicy.ts │ │ │ ├── GeneralizedTransactionSetHashPolicy.ts │ │ │ ├── IHashCalculationPolicy.ts │ │ │ ├── RegularTransactionSetHashPolicy.ts │ │ │ └── __tests__ │ │ │ ├── FirstLedgerHashPolicy.test.ts │ │ │ ├── GeneralizedTransactionSetPolicy.test.ts │ │ │ └── RegularTransactionSetPolicy.test.ts │ ├── infrastructure │ │ ├── cli │ │ │ ├── verify-archives.ts │ │ │ └── verify-single-archive.ts │ │ ├── database │ │ │ ├── TypeOrmHistoryArchiveScanResultRepository.ts │ │ │ └── __tests__ │ │ │ │ └── TypeOrmHistoryArchiveScanRepository.integration.test.ts │ │ ├── di │ │ │ ├── container.ts │ │ │ └── di-types.ts │ │ ├── http │ │ │ ├── HistoryScanRouter.ts │ │ │ ├── MockHistoryArchive.ts │ │ │ └── __fixtures__ │ │ │ │ ├── bucket-bff0722cb3e89655d3b71c3b517a3bc4b20456298e50073745342f28f6f68b7c.xdr.gz │ │ │ │ ├── history-0000003f.json │ │ │ │ ├── history-0000007f.json │ │ │ │ ├── ledger-0000003f.xdr.gz │ │ │ │ ├── ledger-0000007f.xdr.gz │ │ │ │ ├── results-0000003f.xdr.gz │ │ │ │ ├── results-0000007f.xdr.gz │ │ │ │ ├── stellar-history.json │ │ │ │ ├── transactions-0000003f.xdr.gz │ │ │ │ └── transactions-0000007f.xdr.gz │ │ └── services │ │ │ ├── HistoryArchiveFromNetworkService.ts │ │ │ ├── HistoryArchiveServiceMock.ts │ │ │ └── __tests__ │ │ │ └── HistoryArchiveFromNetworkService.integration.test.ts │ └── use-cases │ │ ├── get-latest-scan │ │ ├── GetLatestScan.ts │ │ ├── GetLatestScanDTO.ts │ │ ├── InvalidUrlError.ts │ │ └── __tests__ │ │ │ └── GetLatestScan.integration.test.ts │ │ ├── verify-archives │ │ ├── VerifyArchives.ts │ │ ├── VerifyArchivesDTO.ts │ │ └── __tests__ │ │ │ └── VerifyArchives.integration.test.ts │ │ └── verify-single-archive │ │ ├── VerifySingleArchive.ts │ │ └── VerifySingleArchiveDTO.ts ├── network-scan │ ├── README.md │ ├── domain │ │ ├── Change.ts │ │ ├── ScanDecouplingTodo.md │ │ ├── ScanRepository.ts │ │ ├── Scanner.ts │ │ ├── __tests__ │ │ │ ├── ScanRepository.test.ts │ │ │ └── Scanner.test.ts │ │ ├── measurement-aggregation │ │ │ ├── MeasurementAggregation.ts │ │ │ ├── MeasurementAggregationRepository.ts │ │ │ ├── MeasurementAggregationRepositoryFactory.ts │ │ │ ├── MeasurementAggregationSourceId.ts │ │ │ ├── MeasurementsRollupService.ts │ │ │ └── __tests__ │ │ │ │ └── MeasurementAggregationRepositoryFactory.test.ts │ │ ├── measurement │ │ │ ├── Measurement.ts │ │ │ └── MeasurementRepository.ts │ │ ├── network │ │ │ ├── Network.ts │ │ │ ├── NetworkId.ts │ │ │ ├── NetworkMeasurement.ts │ │ │ ├── NetworkMeasurementAggregation.ts │ │ │ ├── NetworkMeasurementDay.ts │ │ │ ├── NetworkMeasurementDayRepository.ts │ │ │ ├── NetworkMeasurementMonth.ts │ │ │ ├── NetworkMeasurementMonthRepository.ts │ │ │ ├── NetworkMeasurementRepository.ts │ │ │ ├── NetworkQuorumSetConfiguration.ts │ │ │ ├── NetworkQuorumSetConfigurationMapper.ts │ │ │ ├── NetworkRepository.ts │ │ │ ├── NetworkSnapshot.ts │ │ │ ├── NetworkTopology.ts │ │ │ ├── OverlayVersionRange.ts │ │ │ ├── StellarCoreVersion.ts │ │ │ ├── __fixtures__ │ │ │ │ ├── createDummyNetworkProps.ts │ │ │ │ └── createDummyNetworkQuorumSetConfiguration.ts │ │ │ ├── __tests__ │ │ │ │ ├── Network.test.ts │ │ │ │ ├── OverlayVersionRange.test.ts │ │ │ │ ├── QuorumSetMapper.test.ts │ │ │ │ └── StellarCoreVersion.test.ts │ │ │ ├── change │ │ │ │ ├── NetworkChange.ts │ │ │ │ ├── NetworkMaxLedgerVersionChanged.ts │ │ │ │ ├── NetworkNameChanged.ts │ │ │ │ ├── NetworkOverlayVersionRangeChanged.ts │ │ │ │ ├── NetworkQuorumSetConfigurationChanged.ts │ │ │ │ └── NetworkStellarCoreVersionChanged.ts │ │ │ └── scan │ │ │ │ ├── HorizonService.ts │ │ │ │ ├── NetworkScan.ts │ │ │ │ ├── NetworkScanRepository.ts │ │ │ │ ├── NetworkScanner.ts │ │ │ │ ├── NodesInTransitiveNetworkQuorumSetFinder.ts │ │ │ │ ├── TomlService.ts │ │ │ │ ├── TomlVersionChecker.ts │ │ │ │ ├── __tests__ │ │ │ │ ├── HorizonService.test.ts │ │ │ │ ├── NetworkScan.test.ts │ │ │ │ ├── NetworkScanner.test.ts │ │ │ │ ├── NodesInTransitiveNetworkQuorumSetFinder.test.ts │ │ │ │ ├── TomlService.test.ts │ │ │ │ └── TomlVersionChecker.test.ts │ │ │ │ ├── archiver │ │ │ │ └── Archiver.ts │ │ │ │ └── fbas-analysis │ │ │ │ ├── AnalysisMergedResult.ts │ │ │ │ ├── AnalysisResult.ts │ │ │ │ ├── FbasAnalyzerFacade.ts │ │ │ │ ├── FbasAnalyzerService.ts │ │ │ │ ├── FbasMapper.ts │ │ │ │ ├── FbasMergedByAnalyzer.ts │ │ │ │ └── __tests__ │ │ │ │ ├── FbasAnalyzerFacade.test.ts │ │ │ │ ├── FbasAnalyzerService.test.ts │ │ │ │ ├── FbasMapper.test.ts │ │ │ │ └── FbasMergedByAnalyzer.test.ts │ │ ├── node │ │ │ ├── Node.ts │ │ │ ├── NodeAddress.ts │ │ │ ├── NodeDetails.ts │ │ │ ├── NodeGeoDataLocation.ts │ │ │ ├── NodeMeasurement.ts │ │ │ ├── NodeMeasurementAverage.ts │ │ │ ├── NodeMeasurementDay.ts │ │ │ ├── NodeMeasurementDayRepository.ts │ │ │ ├── NodeMeasurementEvent.ts │ │ │ ├── NodeMeasurementRepository.ts │ │ │ ├── NodeQuorumSet.ts │ │ │ ├── NodeRepository.ts │ │ │ ├── NodeSnapShot.ts │ │ │ ├── NodeSnapShotRepository.ts │ │ │ ├── PublicKey.ts │ │ │ ├── __fixtures__ │ │ │ │ ├── createDummyNode.ts │ │ │ │ ├── createDummyNodeAddress.ts │ │ │ │ └── createDummyPublicKey.ts │ │ │ ├── __tests__ │ │ │ │ ├── Node.test.ts │ │ │ │ ├── NodeAddress.test.ts │ │ │ │ ├── NodeDetails.test.ts │ │ │ │ ├── NodeGeoDataLocation.test.ts │ │ │ │ ├── NodeMeasurement.test.ts │ │ │ │ ├── NodeQuorumSet.test.ts │ │ │ │ └── NodeSnapshot.test.ts │ │ │ ├── archival │ │ │ │ ├── InactiveNodesArchiver.ts │ │ │ │ ├── ValidatorDemoter.ts │ │ │ │ ├── __tests__ │ │ │ │ │ ├── InactiveNodesArchiver.test.ts │ │ │ │ │ ├── ValidatorDemoter.test.ts │ │ │ │ │ └── hasNoActiveTrustingNodes.test.ts │ │ │ │ └── hasNoActiveTrustingNodes.ts │ │ │ └── scan │ │ │ │ ├── GeoDataService.ts │ │ │ │ ├── HistoryArchiveStatusFinder.ts │ │ │ │ ├── HomeDomainFetcher.ts │ │ │ │ ├── MoreThanOneDayApart.ts │ │ │ │ ├── NodeIndexer.ts │ │ │ │ ├── NodeScan.ts │ │ │ │ ├── NodeScanner.ts │ │ │ │ ├── NodeScannerArchivalStep.ts │ │ │ │ ├── NodeScannerCrawlStep.ts │ │ │ │ ├── NodeScannerGeoStep.ts │ │ │ │ ├── NodeScannerHistoryArchiveStep.ts │ │ │ │ ├── NodeScannerHomeDomainStep.ts │ │ │ │ ├── NodeScannerIndexerStep.ts │ │ │ │ ├── NodeScannerTomlStep.ts │ │ │ │ ├── NodeTomlFetcher.ts │ │ │ │ ├── NodeTomlInfo.ts │ │ │ │ ├── TrustGraphFactory.ts │ │ │ │ ├── __tests__ │ │ │ │ ├── HistoryArchiveStatusFinder.test.ts │ │ │ │ ├── HomeDomainFetcher.test.ts │ │ │ │ ├── MoreThanOneDayApart.test.ts │ │ │ │ ├── NodeScan.test.ts │ │ │ │ ├── NodeScanner.test.ts │ │ │ │ ├── NodeScannerArchivalStep.test.ts │ │ │ │ ├── NodeScannerCrawlStep.test.ts │ │ │ │ ├── NodeScannerGeoStep.test.ts │ │ │ │ ├── NodeScannerHistoryArchiveStep.test.ts │ │ │ │ ├── NodeScannerHomeDomainStep.test.ts │ │ │ │ ├── NodeScannerIndexerStep.test.ts │ │ │ │ ├── NodeScannerTomlStep.test.ts │ │ │ │ ├── NodeTomlFetcher.test.ts │ │ │ │ └── TrustGraphFactory.test.ts │ │ │ │ ├── history │ │ │ │ ├── HistoryArchiveScanService.ts │ │ │ │ ├── HistoryService.ts │ │ │ │ └── __tests__ │ │ │ │ │ └── HistoryService.test.ts │ │ │ │ ├── node-crawl │ │ │ │ ├── CrawlerDTOMapper.ts │ │ │ │ ├── CrawlerService.ts │ │ │ │ ├── NodeAddressDTOComposer.ts │ │ │ │ ├── NodeSorter.ts │ │ │ │ ├── PeerNodeToNodeMapper.ts │ │ │ │ └── __tests__ │ │ │ │ │ ├── CrawlerDTOMapper.test.ts │ │ │ │ │ ├── CrawlerService.test.ts │ │ │ │ │ ├── NodeAddressDTOComposer.test.ts │ │ │ │ │ ├── NodeSorter.test.ts │ │ │ │ │ └── PeerNodeToNodeMapper.test.ts │ │ │ │ └── node-index │ │ │ │ ├── __tests__ │ │ │ │ └── node-index │ │ │ │ │ ├── index │ │ │ │ │ ├── active-index.test.ts │ │ │ │ │ ├── age-index.test.ts │ │ │ │ │ ├── trust-index.test.ts │ │ │ │ │ ├── type-index.test.ts │ │ │ │ │ ├── validating-index.test.ts │ │ │ │ │ └── version-index.test.ts │ │ │ │ │ └── node-index.test.ts │ │ │ │ ├── index │ │ │ │ ├── active-index.ts │ │ │ │ ├── age-index.ts │ │ │ │ ├── trust-index.ts │ │ │ │ ├── type-index.ts │ │ │ │ ├── validating-index.ts │ │ │ │ └── version-index.ts │ │ │ │ └── node-index.ts │ │ └── organization │ │ │ ├── Organization.ts │ │ │ ├── OrganizationContactInformation.ts │ │ │ ├── OrganizationId.ts │ │ │ ├── OrganizationMeasurement.ts │ │ │ ├── OrganizationMeasurementAverage.ts │ │ │ ├── OrganizationMeasurementDay.ts │ │ │ ├── OrganizationMeasurementDayRepository.ts │ │ │ ├── OrganizationMeasurementEvent.ts │ │ │ ├── OrganizationMeasurementRepository.ts │ │ │ ├── OrganizationRepository.ts │ │ │ ├── OrganizationSnapShot.ts │ │ │ ├── OrganizationSnapShotRepository.ts │ │ │ ├── OrganizationValidators.ts │ │ │ ├── TierOneCandidatePolicy.ts │ │ │ ├── __fixtures__ │ │ │ └── createDummyOrganizationId.ts │ │ │ ├── __tests__ │ │ │ ├── Organization.test.ts │ │ │ ├── OrganizationContactInformation.test.ts │ │ │ ├── OrganizationId.test.ts │ │ │ ├── OrganizationSnapShot.test.ts │ │ │ └── OrganizationValidators.test.ts │ │ │ └── scan │ │ │ ├── ErrorToTomlStateMapper.ts │ │ │ ├── OrganizationScan.ts │ │ │ ├── OrganizationScanner.ts │ │ │ ├── OrganizationTomlFetcher.ts │ │ │ ├── OrganizationTomlInfo.ts │ │ │ ├── TomlState.ts │ │ │ ├── __tests__ │ │ │ ├── ErrorToTomlStateMapper.test.ts │ │ │ ├── OrganizationScan.test.ts │ │ │ ├── OrganizationScanner.test.ts │ │ │ └── OrganizationTomlFetcher.test.ts │ │ │ └── errors │ │ │ ├── CouldNotRetrieveArchivedOrganizationsError.ts │ │ │ ├── InvalidOrganizationIdError.ts │ │ │ ├── InvalidTomlStateError.ts │ │ │ ├── OrganizationScanError.ts │ │ │ ├── TomlWithoutValidatorsError.ts │ │ │ ├── ValidatorNotSEP20LinkedError.ts │ │ │ └── WrongNodeScanForOrganizationScan.ts │ ├── infrastructure │ │ ├── cli │ │ │ ├── retrieve-home-domain.ts │ │ │ ├── scan-network.ts │ │ │ └── toml-fetch.ts │ │ ├── database │ │ │ ├── entities │ │ │ │ └── MeasurementRollup.ts │ │ │ └── repositories │ │ │ │ ├── TypeOrmNetworkMeasurementDayRepository.ts │ │ │ │ ├── TypeOrmNetworkMeasurementMonthRepository.ts │ │ │ │ ├── TypeOrmNetworkMeasurementRepository.ts │ │ │ │ ├── TypeOrmNetworkRepository.ts │ │ │ │ ├── TypeOrmNetworkScanRepository.ts │ │ │ │ ├── TypeOrmNodeMeasurementDayRepository.ts │ │ │ │ ├── TypeOrmNodeMeasurementRepository.ts │ │ │ │ ├── TypeOrmNodeRepository.ts │ │ │ │ ├── TypeOrmNodeSnapShotRepository.ts │ │ │ │ ├── TypeOrmOrganizationMeasurementDayRepository.ts │ │ │ │ ├── TypeOrmOrganizationMeasurementRepository.ts │ │ │ │ ├── TypeOrmOrganizationRepository.ts │ │ │ │ ├── TypeOrmOrganizationSnapShotRepository.ts │ │ │ │ └── __tests__ │ │ │ │ ├── NetworkMeasurementDayRepository.integration.test.ts │ │ │ │ ├── NetworkMeasurementMonthRepository.integration.test.ts │ │ │ │ ├── NetworkRepository.integration.test.ts │ │ │ │ ├── NetworkScanRepository.integration.test.ts │ │ │ │ ├── NodeMeasurementDayRepository.integration.test.ts │ │ │ │ ├── NodeMeasurementRepository.integration.test.ts │ │ │ │ ├── NodeRepository.integration.test.ts │ │ │ │ ├── NodeSnapShotRepository.integration.test.ts │ │ │ │ ├── OrganizationMeasurementDayRepository.integration.test.ts │ │ │ │ ├── OrganizationMeasurementRepository.integration.test.ts │ │ │ │ ├── OrganizationRepository.integration.test.ts │ │ │ │ └── OrganizationSnapShotRepository.integration.test.ts │ │ ├── di │ │ │ ├── container.ts │ │ │ └── di-types.ts │ │ ├── http │ │ │ ├── NetworkRouter.ts │ │ │ ├── NodeRouter.ts │ │ │ ├── OrganizationRouter.ts │ │ │ └── handleMeasurementsAggregationRequest.ts │ │ └── services │ │ │ ├── DatabaseHistoryArchiveScanService.ts │ │ │ ├── DatabaseMeasurementsRollupService.ts │ │ │ ├── DeadManSnitchHeartBeater.ts │ │ │ ├── DummyHeartBeater.ts │ │ │ ├── IpStackGeoDataService.ts │ │ │ ├── S3Archiver.ts │ │ │ └── __tests__ │ │ │ ├── DatabaseHistoryArchiveScanService.integration.test.ts │ │ │ └── GeoDataService.test.ts │ ├── mappers │ │ ├── BaseQuorumSetDTOMapper.ts │ │ ├── NetworkV1DTOMapper.ts │ │ ├── NodeSnapshotMapper.ts │ │ ├── NodeV1DTOMapper.ts │ │ ├── OrganizationMapper.ts │ │ ├── OrganizationSnapshotMapper.ts │ │ ├── OrganizationV1DTOMapper.ts │ │ └── __tests__ │ │ │ ├── BaseQuorumSetDTOMapper.test.ts │ │ │ ├── NetworkV1DTOMapper.test.ts │ │ │ ├── NodeSnapshotMapper.test.ts │ │ │ ├── NodeV1DTOMapper.test.ts │ │ │ └── OrganizationV1DTOMapper.test.ts │ ├── services │ │ ├── CachedNetworkDTOService.ts │ │ ├── NetworkDTOService.ts │ │ ├── NodeDTOService.ts │ │ ├── OrganizationDTOService.ts │ │ ├── README.md │ │ ├── __fixtures__ │ │ │ ├── createDummyNetworkV1.ts │ │ │ ├── createDummyNodeV1.ts │ │ │ └── createDummyOrganizationV1.ts │ │ └── __tests__ │ │ │ ├── CachedNetworkDTOService.test.ts │ │ │ ├── NetworkDTOService.test.ts │ │ │ ├── NodeDTOService.test.ts │ │ │ └── OrganizationDTOService.test.ts │ └── use-cases │ │ ├── get-latest-node-snapshots │ │ ├── GetLatestNodeSnapshots.ts │ │ ├── GetLatestNodeSnapshotsDTO.ts │ │ └── __tests__ │ │ │ ├── GetLatestNodeSnapshots.integration.test.ts │ │ │ └── GetLatestNodeSnapshots.test.ts │ │ ├── get-latest-organization-snapshots │ │ ├── GetLatestOrganizationSnapshots.ts │ │ ├── GetLatestOrganizationSnapshotsDTO.ts │ │ └── __tests__ │ │ │ ├── GetLatestOrganizationSnapshots.integration.test.ts │ │ │ └── GetLatestOrganizationSnapshots.test.ts │ │ ├── get-measurement-aggregations │ │ ├── GetMeasurementAggregations.ts │ │ ├── GetMeasurementAggregationsDTO.ts │ │ └── __tests__ │ │ │ ├── GetMeasurementAggregation.test.ts │ │ │ └── GetMeasurementAggregations.integration.test.ts │ │ ├── get-measurements │ │ ├── GetMeasurements.ts │ │ ├── GetMeasurementsDTO.ts │ │ ├── GetMeasurementsFactory.ts │ │ └── __tests__ │ │ │ ├── GetMeasurements.integration.test.ts │ │ │ └── GetMeasurements.test.ts │ │ ├── get-network │ │ ├── GetNetwork.ts │ │ ├── GetNetworkDTO.ts │ │ └── __tests__ │ │ │ ├── GetNetwork.integration.test.ts │ │ │ └── GetNetwork.test.ts │ │ ├── get-node-snapshots │ │ ├── GetNodeSnapshots.ts │ │ ├── GetNodeSnapshotsDTO.ts │ │ └── __tests__ │ │ │ ├── GetNodeSnapshots.integration.test.ts │ │ │ └── GetNodeSnapshots.test.ts │ │ ├── get-node │ │ ├── GetNode.ts │ │ ├── GetNodeDTO.ts │ │ └── __tests__ │ │ │ ├── GetNode.integration.test.ts │ │ │ └── GetNode.test.ts │ │ ├── get-nodes │ │ ├── GetNodes.ts │ │ ├── GetNodesDTO.ts │ │ └── __tests__ │ │ │ ├── GetNodes.integration.test.ts │ │ │ └── GetNodes.test.ts │ │ ├── get-organization-snapshots │ │ ├── GetOrganizationSnapshots.ts │ │ ├── GetOrganizationSnapshotsDTO.ts │ │ └── __tests__ │ │ │ ├── GetOrganizationSnapshots.integration.test.ts │ │ │ └── GetOrganizationSnapshots.test.ts │ │ ├── get-organization │ │ ├── GetOrganization.ts │ │ ├── GetOrganizationDTO.ts │ │ └── __tests__ │ │ │ ├── GetOrganization.integration.test.ts │ │ │ └── GetOrganization.test.ts │ │ ├── get-organizations │ │ ├── GetOrganizations.ts │ │ ├── GetOrganizationsDTO.ts │ │ └── __tests__ │ │ │ ├── GetOrganizations.integration.test.ts │ │ │ └── GetOrganizations.test.ts │ │ ├── scan-network-looped │ │ ├── ScanNetworkLooped.ts │ │ ├── ScanNetworkLoopedDTO.ts │ │ └── __tests__ │ │ │ ├── ScanNetwork.integration.test.ts │ │ │ └── ScanNetworkLooped.test.ts │ │ ├── scan-network │ │ ├── InvalidKnownPeersError.ts │ │ ├── NodeAddressMapper.ts │ │ ├── ScanNetwork.ts │ │ ├── ScanNetworkDTO.ts │ │ └── __tests__ │ │ │ ├── NodeAddressMapper.test.ts │ │ │ ├── ScanNetwork.integration.test.ts │ │ │ └── ScanNetwork.test.ts │ │ └── update-network │ │ ├── InvalidOverlayRangeError.ts │ │ ├── InvalidQuorumSetConfigError.ts │ │ ├── InvalidStellarCoreVersionError.ts │ │ ├── InvalidUpdateTimeError.ts │ │ ├── NetworkQuorumSetMapper.ts │ │ ├── RepositoryError.ts │ │ ├── UpdateNetwork.ts │ │ ├── UpdateNetworkDTO.ts │ │ └── __tests__ │ │ ├── NetworkQuorumSetMapper.test.ts │ │ ├── UpdateNetwork.integration.test.ts │ │ └── UpdateNetwork.test.ts └── notifications │ ├── README.md │ ├── domain │ ├── README.md │ ├── event │ │ ├── Event.ts │ │ ├── EventDetector.ts │ │ ├── EventRepository.ts │ │ ├── EventSource.ts │ │ ├── EventSourceId.ts │ │ ├── EventSourceIdFactory.ts │ │ ├── EventSourceService.ts │ │ ├── NetworkEventDetector.ts │ │ ├── NodeEventDetector.ts │ │ └── __tests__ │ │ │ ├── EventDetector.test.ts │ │ │ ├── EventSourceIdFactory.test.ts │ │ │ ├── NetworkEventDetector.test.ts │ │ │ └── NodeEventDetector.test.ts │ ├── notifier │ │ ├── MessageCreator.ts │ │ └── Notifier.ts │ └── subscription │ │ ├── EventNotificationState.ts │ │ ├── Notification.ts │ │ ├── PendingSubscription.ts │ │ ├── Subscriber.ts │ │ ├── SubscriberReference.ts │ │ ├── SubscriberRepository.ts │ │ ├── Subscription.ts │ │ ├── UserId.ts │ │ ├── __fixtures__ │ │ ├── PendingSubscriptionId.fixtures.ts │ │ └── Subscriber.fixtures.ts │ │ └── __tests__ │ │ └── Subscriber.test.ts │ ├── infrastructure │ ├── database │ │ └── repositories │ │ │ ├── TypeOrmEventRepository.ts │ │ │ ├── TypeOrmSubscriberRepository.ts │ │ │ └── __tests__ │ │ │ ├── TypeOrmEventRepository.test.ts │ │ │ └── TypeOrmSubscriberRepository.integration.test.ts │ ├── di │ │ ├── container.ts │ │ └── di-types.ts │ ├── http │ │ └── SubscriptionRouter.ts │ ├── services │ │ ├── EJSMessageCreator.ts │ │ ├── EventSourceFromNetworkService.ts │ │ └── __tests__ │ │ │ ├── EventSourceFromNetworkService.test.ts │ │ │ └── MessageCreator.test.ts │ └── templates │ │ ├── Readme.md │ │ ├── confirm-subscription-notification.ejs │ │ ├── notification.ejs │ │ └── unsubscribe-notification.ejs │ └── use-cases │ ├── confirm-subscription │ ├── ConfirmSubscription.ts │ ├── ConfirmSubscriptionDTO.ts │ ├── ConfirmSubscriptionError.ts │ └── __tests__ │ │ └── ConfirmSubscription.integration.test.ts │ ├── determine-events-and-notify-subscribers │ ├── Notify.ts │ ├── NotifyDTO.ts │ ├── NotifyError.ts │ └── __tests__ │ │ └── Notify.integration.test.ts │ ├── request-unsubscribe-link │ ├── RequestUnsubscribeLink.ts │ ├── RequestUnsubscribeLinkDTO.ts │ └── __tests__ │ │ ├── RequestUnsubscribeLink.integration.test.ts │ │ └── RequestUnsubscribeLink.test.ts │ ├── subscribe │ ├── Subscribe.ts │ ├── SubscribeDTO.ts │ ├── SubscribeError.ts │ └── __tests__ │ │ └── Subscribe.integration.test.ts │ ├── unmute-notification │ ├── UnmuteNotification.ts │ ├── UnmuteNotificationDTO.ts │ ├── UnmuteNotificationError.ts │ └── __tests__ │ │ └── UnmuteNotification.integration.test.ts │ └── unsubscribe │ ├── Unsubscribe.ts │ ├── UnsubscribeDTO.ts │ ├── UnsubscribeError.ts │ └── __tests__ │ └── Unsubscribe.integration.test.ts ├── tsconfig.base.json ├── tsconfig.json └── tsconfig.prod.json /.eslintignore: -------------------------------------------------------------------------------- 1 | # don't ever lint node_modules 2 | node_modules 3 | # don't lint build output (make sure it's set to your correct build folder name) 4 | dist 5 | # don't lint nyc coverage output 6 | coverage 7 | examples 8 | lib 9 | node_modules -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-undef 2 | module.exports = { 3 | root: true, 4 | parser: '@typescript-eslint/parser', 5 | plugins: ['@typescript-eslint'], 6 | extends: [ 7 | 'eslint:recommended', 8 | 'plugin:@typescript-eslint/recommended', 9 | 'prettier' 10 | ], 11 | rules: { 12 | '@typescript-eslint/ban-ts-comment': 'off', 13 | '@typescript-eslint/explicit-module-boundary-types': 'off', 14 | '@typescript-eslint/no-unused-vars': 'off', 15 | '@typescript-eslint/no-explicit-any': 'off' 16 | }, 17 | env: { 18 | node: true 19 | } 20 | }; 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | .idea/ 64 | node_modules 65 | flow-typed/ 66 | config/default.json 67 | .env 68 | lib/ 69 | 70 | .DS_Store 71 | src/**/.DS_Store -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | #pnpm compatibility seems broken? 5 | #pnmp build 6 | #pnpm test:unit 7 | #pnpm test:integration 8 | -------------------------------------------------------------------------------- /.vim/coc-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "tsserver.useLocalTsdk": true, 3 | "tsserver.tsdk": "${workspaceFolder}/node_modules/typescript/lib" 4 | } -------------------------------------------------------------------------------- /.yarnrc: -------------------------------------------------------------------------------- 1 | --install.ignore-optional true 2 | ignore-optional true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 stellarbeat.io 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: pnpm start-api 2 | scan-network: pnpm scan-network 1 -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | rootDir: 'src', 5 | testPathIgnorePatterns: ['/node_modules/', '/lib/'], 6 | transform: { 7 | '^.+\\.tsx?$': [ 8 | 'ts-jest', 9 | { 10 | tsconfig: 'tsconfig.json' 11 | } 12 | ] 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /jest.integration.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | rootDir: 'src', 5 | testPathIgnorePatterns: ['/node_modules/', '/lib/'], 6 | testRegex: '.integration.test.ts', 7 | transform: { 8 | '^.+\\.tsx?$': [ 9 | 'ts-jest', 10 | { 11 | tsconfig: 'tsconfig.json' 12 | } 13 | ] 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /jest.unit.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | rootDir: 'src', 5 | testPathIgnorePatterns: ['/node_modules/', '/lib/', '.integration.'], 6 | transform: { 7 | '^.+\\.tsx?$': [ 8 | 'ts-jest', 9 | { 10 | tsconfig: 'tsconfig.json' 11 | } 12 | ] 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /prettier.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | arrowParens: 'always', 3 | bracketSpacing: true, 4 | jsxBracketSameLine: false, 5 | printWidth: 80, 6 | proseWrap: 'always', 7 | semi: true, 8 | singleQuote: true, 9 | tabWidth: 2, 10 | trailingComma: 'none', 11 | useTabs: true 12 | }; 13 | -------------------------------------------------------------------------------- /seeds/nodes-test-network.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ip": "3.81.61.96", 4 | "port": 11625, 5 | "publicKey": "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y", 6 | "active": true 7 | }, 8 | { 9 | "ip": "18.207.213.63", 10 | "port": 11625, 11 | "publicKey": "GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP", 12 | "active": true 13 | } 14 | ] -------------------------------------------------------------------------------- /src/core/domain/CoreEntity.ts: -------------------------------------------------------------------------------- 1 | import { PrimaryGeneratedColumn } from 'typeorm'; 2 | 3 | //represents a class that has a lifecycle with an internal id, not to be exposed to clients 4 | export abstract class CoreEntity { 5 | @PrimaryGeneratedColumn({ name: 'id' }) 6 | protected readonly id?: number; //marked private as to not expose it to clients 7 | } 8 | -------------------------------------------------------------------------------- /src/core/domain/IUserService.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | import { UserId } from '../../notifications/domain/subscription/UserId'; 3 | import { CustomError } from '../errors/CustomError'; 4 | import { Message } from './Message'; 5 | 6 | export class CreateUserError extends CustomError { 7 | constructor(cause?: Error) { 8 | super('Could not create user', 'CreateUserError', cause); 9 | } 10 | } 11 | 12 | export interface IUserService { 13 | send(userId: UserId, message: Message): Promise>; 14 | 15 | findOrCreateUser(emailAddress: string): Promise>; 16 | 17 | findUser(emailAddress: string): Promise>; 18 | 19 | deleteUser(userId: UserId): Promise>; 20 | } 21 | -------------------------------------------------------------------------------- /src/core/domain/IdentifiedValueObject.ts: -------------------------------------------------------------------------------- 1 | import { PrimaryGeneratedColumn } from 'typeorm'; 2 | import { ValueObject } from './ValueObject'; 3 | 4 | //If you want to store a value object in database in a separate column, but not expose its internal db id, use this class. 5 | export abstract class IdentifiedValueObject extends ValueObject { 6 | @PrimaryGeneratedColumn({ name: 'id' }) 7 | protected readonly id?: number; 8 | 9 | abstract equals(other: this): boolean; 10 | } 11 | -------------------------------------------------------------------------------- /src/core/domain/Message.ts: -------------------------------------------------------------------------------- 1 | export class Message { 2 | constructor(public body: string, public title: string) {} 3 | } 4 | -------------------------------------------------------------------------------- /src/core/domain/Snapshot.ts: -------------------------------------------------------------------------------- 1 | import { Column, Index } from 'typeorm'; 2 | import { CoreEntity } from './CoreEntity'; 3 | 4 | export abstract class Snapshot extends CoreEntity { 5 | static readonly MAX_DATE = new Date(Date.UTC(9999, 11, 31, 23, 59, 59)); 6 | 7 | @Column('timestamptz', { nullable: false }) 8 | @Index() 9 | public readonly startDate: Date; 10 | 11 | @Column('timestamptz', { name: 'endDate', nullable: false }) 12 | @Index() 13 | public endDate: Date = Snapshot.MAX_DATE; 14 | 15 | protected constructor(startDate: Date) { 16 | super(); 17 | this.startDate = startDate; 18 | } 19 | 20 | public abstract copy(startDate: Date): this; 21 | } 22 | -------------------------------------------------------------------------------- /src/core/domain/Url.ts: -------------------------------------------------------------------------------- 1 | import { err, ok, Result } from 'neverthrow'; 2 | import validator from 'validator'; 3 | 4 | export class Url { 5 | public value; 6 | 7 | private constructor(url: string) { 8 | this.value = url; 9 | } 10 | 11 | static create(url: string): Result { 12 | if (!validator.isURL(url)) 13 | return err(new Error('Url is not a proper url: ' + url)); 14 | 15 | url = url.replace(/\/$/, ''); //remove trailing slash 16 | 17 | return ok(new Url(url)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/core/domain/ValueObject.ts: -------------------------------------------------------------------------------- 1 | import { shallowEqual } from 'shallow-equal-object'; 2 | 3 | export abstract class ValueObject { 4 | equals(other: this): boolean { 5 | if (other === null || other === undefined) { 6 | return false; 7 | } 8 | 9 | return shallowEqual(this, other); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/core/domain/__tests__/Url.test.ts: -------------------------------------------------------------------------------- 1 | import { Url } from '../Url'; 2 | 3 | it('should create a valid url object', function () { 4 | const urlString = 'https://my-url.com/455'; 5 | const urlResult = Url.create(urlString); 6 | 7 | expect(urlResult.isOk()).toBeTruthy(); 8 | }); 9 | it('should return an error', function () { 10 | const urlResult = Url.create(' https://url-with-space-in-front.com'); 11 | expect(urlResult.isErr()).toBeTruthy(); 12 | }); 13 | -------------------------------------------------------------------------------- /src/core/errors/CustomError.ts: -------------------------------------------------------------------------------- 1 | export class CustomError extends Error { 2 | public errorType = 'CustomError'; //to allow type inference in err() result 3 | 4 | constructor(message: string, name: string, public cause?: Error) { 5 | super(message); 6 | this.message = CustomError.getExtendedMessage(name, message, cause); 7 | this.cause = cause; 8 | this.name = name; 9 | } 10 | 11 | private static getExtendedMessage( 12 | name: string, 13 | message: string, 14 | cause?: Error 15 | ) { 16 | let extendedMessage = name + ': ' + message; 17 | if (cause instanceof CustomError) extendedMessage += ' => ' + cause.message; 18 | else if (cause) 19 | extendedMessage += ' => ' + cause.name + ': ' + cause.message; 20 | return extendedMessage; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/core/errors/__tests__/CustomError.test.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../CustomError'; 2 | 3 | it('should set the name and cause correctly', function () { 4 | const cause = new Error('The cause'); 5 | const customError = new CustomError('Error', 'custom', cause); 6 | expect(customError.name).toEqual('custom'); 7 | expect(customError.message).toEqual('custom: Error => Error: The cause'); 8 | expect(customError.cause).toEqual(cause); 9 | }); 10 | 11 | it('should set custom name', function () { 12 | const customError = new CustomError('Error', 'CustomError'); 13 | expect(customError.name).toEqual('CustomError'); 14 | }); 15 | 16 | it('should return message with cause info', function () { 17 | const cause1 = new Error('First'); 18 | const cause2 = new CustomError('Second', 'SecondError', cause1); 19 | const error = new CustomError('Third', 'ThirdError', cause2); 20 | 21 | expect(error.message).toEqual( 22 | 'ThirdError: Third => SecondError: Second => Error: First' 23 | ); 24 | }); 25 | -------------------------------------------------------------------------------- /src/core/infrastructure/__tests__/Kernel.integration.test.ts: -------------------------------------------------------------------------------- 1 | import { Connection, DataSource } from 'typeorm'; 2 | import Kernel from '../Kernel'; 3 | import { ConfigMock } from '../../config/__mocks__/configMock'; 4 | import { NodeMeasurementRepository } from '../../../network-scan/domain/node/NodeMeasurementRepository'; 5 | import { NETWORK_TYPES } from '../../../network-scan/infrastructure/di/di-types'; 6 | import { TypeOrmNodeMeasurementRepository } from '../../../network-scan/infrastructure/database/repositories/TypeOrmNodeMeasurementRepository'; 7 | import { TypeOrmOrganizationRepository } from '../../../network-scan/infrastructure/database/repositories/TypeOrmOrganizationRepository'; 8 | 9 | jest.setTimeout(10000); //slow and long integration test 10 | 11 | test('kernel', async () => { 12 | const kernel = await Kernel.getInstance(new ConfigMock()); 13 | const container = kernel.container; 14 | expect( 15 | container.get( 16 | NETWORK_TYPES.NodeMeasurementRepository 17 | ) 18 | ).toBeInstanceOf(TypeOrmNodeMeasurementRepository); 19 | expect(container.get(DataSource)).toBeInstanceOf(DataSource); 20 | expect(container.get(NETWORK_TYPES.OrganizationRepository)).toBeInstanceOf( 21 | TypeOrmOrganizationRepository 22 | ); 23 | 24 | await kernel.close(); 25 | }); 26 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/AppDataSource.ts: -------------------------------------------------------------------------------- 1 | import { config } from 'dotenv'; 2 | import { DataSource } from 'typeorm'; 3 | 4 | config(); 5 | 6 | const AppDataSource = new DataSource({ 7 | type: 'postgres', 8 | logging: false, 9 | synchronize: false, 10 | url: process.env.ACTIVE_DATABASE_URL, 11 | entities: ['lib/**/entities/*.js', 'lib/**/domain/**/!(*.test)*.js'], 12 | migrations: ['lib/**/migrations/*.js'], 13 | migrationsRun: true, 14 | ssl: true, 15 | extra: { 16 | ssl: { 17 | rejectUnauthorized: false 18 | } 19 | }, 20 | poolSize: process.env.DATABASE_POOL_SIZE 21 | ? parseInt(process.env.DATABASE_POOL_SIZE) 22 | : 10 23 | }); 24 | 25 | export { AppDataSource }; 26 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/TestingAppDataSource.ts: -------------------------------------------------------------------------------- 1 | import { DataSource } from 'typeorm'; 2 | import { BaseDataSourceOptions } from 'typeorm/data-source/BaseDataSourceOptions'; 3 | 4 | const TestingAppDataSource: DataSource = new DataSource({ 5 | type: 'postgres', 6 | dropSchema: true, 7 | synchronize: true, 8 | logging: false, 9 | url: process.env.DATABASE_TEST_URL, 10 | entities: ['src/**/entities/*.ts', 'src/**/domain/**/!(*.test)*.ts'], 11 | migrationsRun: false, 12 | ssl: false 13 | }); 14 | 15 | export { TestingAppDataSource }; 16 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1559372435536-ledgers.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class ledgers1559372956880 implements MigrationInterface { 4 | public async up(queryRunner: QueryRunner): Promise { 5 | await queryRunner.query( 6 | `ALTER TABLE "crawl" ADD "ledgers" text NOT NULL DEFAULT ''` 7 | ); 8 | } 9 | 10 | public async down(queryRunner: QueryRunner): Promise { 11 | await queryRunner.query(`ALTER TABLE "crawl" DROP COLUMN "ledgers"`); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1559469975565-index.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class index1559469975565 implements MigrationInterface { 4 | public async up(queryRunner: QueryRunner): Promise { 5 | await queryRunner.query( 6 | `CREATE INDEX "IDX_b33f4eb533966ee0673926cb58" ON "node_measurement" ("publicKey") ` 7 | ); 8 | } 9 | 10 | public async down(queryRunner: QueryRunner): Promise { 11 | await queryRunner.query(`DROP INDEX "IDX_b33f4eb533966ee0673926cb58"`); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1559975071741-organizations.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class organizations1559975071741 implements MigrationInterface { 4 | public async up(queryRunner: QueryRunner): Promise { 5 | await queryRunner.query( 6 | `CREATE TABLE "organization" ("id" SERIAL NOT NULL, "organizationJson" jsonb NOT NULL, "crawlId" integer, CONSTRAINT "PK_472c1f99a32def1b0abb219cd67" PRIMARY KEY ("id"))` 7 | ); 8 | await queryRunner.query( 9 | `ALTER TABLE "organization" ADD CONSTRAINT "FK_7b1195f9c1578dd93507b299a85" FOREIGN KEY ("crawlId") REFERENCES "crawl"("id") ON DELETE NO ACTION ON UPDATE NO ACTION` 10 | ); 11 | } 12 | 13 | public async down(queryRunner: QueryRunner): Promise { 14 | await queryRunner.query( 15 | `ALTER TABLE "organization" DROP CONSTRAINT "FK_7b1195f9c1578dd93507b299a85"` 16 | ); 17 | await queryRunner.query(`DROP TABLE "organization"`); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1560152941696-crawl_indexes.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class crawlIndexes1560152941696 implements MigrationInterface { 4 | public async up(queryRunner: QueryRunner): Promise { 5 | await queryRunner.query( 6 | `CREATE INDEX "IDX_f54faccf9c7f79fcb537b22dd2" ON "node" ("crawlId") ` 7 | ); 8 | await queryRunner.query( 9 | `CREATE INDEX "IDX_7b1195f9c1578dd93507b299a8" ON "organization" ("crawlId") ` 10 | ); 11 | } 12 | 13 | public async down(queryRunner: QueryRunner): Promise { 14 | await queryRunner.query(`DROP INDEX "IDX_7b1195f9c1578dd93507b299a8"`); 15 | await queryRunner.query(`DROP INDEX "IDX_f54faccf9c7f79fcb537b22dd2"`); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1563086015292-completed_crawl.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class completedCrawl1563086015292 implements MigrationInterface { 4 | public async up(queryRunner: QueryRunner): Promise { 5 | await queryRunner.query( 6 | `ALTER TABLE "crawl" ADD "completed" boolean NOT NULL DEFAULT false` 7 | ); 8 | } 9 | 10 | public async down(queryRunner: QueryRunner): Promise { 11 | await queryRunner.query(`ALTER TABLE "crawl" DROP COLUMN "completed"`); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1611909155496-horizonUrl.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class horizonUrl1611909155496 implements MigrationInterface { 4 | name = 'horizonUrl1611909155496'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "organization_snap_shot" ADD "horizonUrl" text`, 9 | undefined 10 | ); 11 | } 12 | 13 | public async down(queryRunner: QueryRunner): Promise { 14 | await queryRunner.query( 15 | `ALTER TABLE "organization_snap_shot" DROP COLUMN "horizonUrl"`, 16 | undefined 17 | ); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1632481790744-latestLedger.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class latestLedger1632481790744 implements MigrationInterface { 4 | name = 'latestLedger1632481790744'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "public"."crawl_v2" ADD "latestLedger" bigint NOT NULL DEFAULT '0'` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise { 13 | await queryRunner.query( 14 | `ALTER TABLE "public"."crawl_v2" DROP COLUMN "latestLedger"` 15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1632483433793-latestLedgerCloseTime.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class latestLedgerCloseTime1632483433793 implements MigrationInterface { 4 | name = 'latestLedgerCloseTime1632483433793'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "public"."crawl_v2" ADD "latestLedgerCloseTime" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT '"1970-01-01T00:00:00.000Z"'` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise { 13 | await queryRunner.query( 14 | `ALTER TABLE "public"."crawl_v2" DROP COLUMN "latestLedgerCloseTime"` 15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1632900021069-active-in-scp.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class activeInScp1632900021069 implements MigrationInterface { 4 | name = 'activeInScp1632900021069'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "public"."node_measurement_v2" ADD "isActiveInScp" boolean NOT NULL DEFAULT FALSE` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise { 13 | await queryRunner.query( 14 | `ALTER TABLE "public"."node_measurement_v2" DROP COLUMN "isActiveInScp"` 15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1634235617874-networkUpdate.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class networkUpdate1634235617874 implements MigrationInterface { 4 | name = 'networkUpdate1634235617874'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query(`alter table crawl_v2 rename to network_update`); 8 | } 9 | 10 | public async down(queryRunner: QueryRunner): Promise { 11 | await queryRunner.query(`alter table network_update rename to crawl_v2`); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1637591171862-subscription-dates.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class subscriptionDates1637591171862 implements MigrationInterface { 4 | name = 'subscriptionDates1637591171862'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "subscription" ADD "subscriptionDate" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT '"2021-11-22T14:26:12.325Z"'` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "subscription_subscriber" ADD "registrationDate" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT '"2021-11-22T14:26:12.325Z"'` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "subscription_subscriber" DROP COLUMN "registrationDate"` 18 | ); 19 | await queryRunner.query( 20 | `ALTER TABLE "subscription" DROP COLUMN "subscriptionDate"` 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1637837429416-notification-orphan.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class notificationOrphan1637837429416 implements MigrationInterface { 4 | name = 'notificationOrphan1637837429416'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "subscription_event_notification_state" DROP CONSTRAINT "FK_ab0ba6414394953ceda810c1001"` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "subscription_event_notification_state" ADD CONSTRAINT "FK_ab0ba6414394953ceda810c1001" FOREIGN KEY ("eventSubscriptionId") REFERENCES "subscription"("id") ON DELETE CASCADE ON UPDATE NO ACTION` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "subscription_event_notification_state" DROP CONSTRAINT "FK_ab0ba6414394953ceda810c1001"` 18 | ); 19 | await queryRunner.query( 20 | `ALTER TABLE "subscription_event_notification_state" ADD CONSTRAINT "FK_ab0ba6414394953ceda810c1001" FOREIGN KEY ("eventSubscriptionId") REFERENCES "subscription"("id") ON DELETE NO ACTION ON UPDATE NO ACTION` 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1652883888348-history-scan.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class historyScan1652883888348 implements MigrationInterface { 4 | name = 'historyScan1652883888348'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `CREATE TABLE "history_archive_scan" ("id" SERIAL NOT NULL, "startDate" TIMESTAMP WITH TIME ZONE NOT NULL, "endDate" TIMESTAMP WITH TIME ZONE, "fromLedger" bigint NOT NULL, "toLedger" bigint NOT NULL, "latestScannedLedger" bigint NOT NULL, "hasGap" boolean NOT NULL, "gapUrl" text, "gapCheckPoint" bigint, "hasError" boolean NOT NULL, "errorMessage" text, "errorStatus" smallint, "errorCode" text, "errorUrl" text, "concurrencyRangeIndex" smallint NOT NULL, "url" text NOT NULL, CONSTRAINT "PK_01635374d215f0f5dee81fd053f" PRIMARY KEY ("id"))` 9 | ); 10 | await queryRunner.query( 11 | `CREATE INDEX "IDX_82d68e5f2b46e4d4cb1406d149" ON "history_archive_scan" ("startDate") ` 12 | ); 13 | await queryRunner.query( 14 | `CREATE INDEX "IDX_b49fc1e2a98573e693d8c7f696" ON "history_archive_scan" ("url") ` 15 | ); 16 | } 17 | 18 | public async down(queryRunner: QueryRunner): Promise { 19 | await queryRunner.query(`DROP TABLE "history_archive_scan"`); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1653131605238-history-gap.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class historyGap1653131605238 implements MigrationInterface { 4 | name = 'historyGap1653131605238'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "node_measurement_day_v2" ADD "historyArchiveGapCount" smallint NOT NULL DEFAULT '0'` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "node_measurement_v2" ADD "historyArchiveGap" boolean NOT NULL DEFAULT false` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "node_measurement_v2" DROP COLUMN "historyArchiveGap"` 18 | ); 19 | await queryRunner.query( 20 | `ALTER TABLE "node_measurement_day_v2" DROP COLUMN "historyArchiveGapCount"` 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1669193270696-archive-verification-v2.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class archiveVerification1669193270696 implements MigrationInterface { 4 | name = 'archiveVerification1669193270696'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "history_archive_scan_v2" ADD "concurrency" smallint NOT NULL default 50` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise { 13 | await queryRunner.query( 14 | `ALTER TABLE "history_archive_scan_v2" DROP COLUMN "concurrency"` 15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1672403715825-versioning-next.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class versioningNext1672403715825 implements MigrationInterface { 4 | name = 'versioningNext1672403715825'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query(`DROP TABLE IF EXISTS crawl cascade;`); 8 | await queryRunner.query(`DROP TABLE IF EXISTS node cascade;`); 9 | await queryRunner.query(`DROP TABLE IF EXISTS organization cascade;`); 10 | 11 | await queryRunner.query( 12 | `CREATE UNIQUE INDEX "IDX_55488aaed4d63c35220746e9fb" ON "node_public_key" ("publicKeyValue") ` 13 | ); 14 | await queryRunner.query( 15 | `CREATE INDEX "IDX_f0378faa8fd3955a3c39b2e711" ON "node_snap_shot" ("NodeId") ` 16 | ); 17 | await queryRunner.query( 18 | `CREATE INDEX "IDX_b364c3a533568016045989c681" ON "organization_snap_shot" ("OrganizationId") ` 19 | ); 20 | await queryRunner.query( 21 | `ALTER TABLE "node_public_key" 22 | RENAME TO "node"` 23 | ); 24 | await queryRunner.query( 25 | `ALTER TABLE "organization_id" 26 | RENAME TO "organization"` 27 | ); 28 | } 29 | 30 | public async down(queryRunner: QueryRunner): Promise {} 31 | } 32 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1672404044492-versioning-seq.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class versioningTest1672404044492 implements MigrationInterface { 4 | name = 'versioningTest1672404044492'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER sequence "node_public_key_id_seq" rename to "node_id_seq"` 9 | ); 10 | await queryRunner.query( 11 | `ALTER sequence "organization_id_id_seq" rename to "organization_id_seq"` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise {} 16 | } 17 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1672489594750-organizationId.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class organizationId1672489594750 implements MigrationInterface { 4 | name = 'organizationId1672489594750'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "organization" RENAME COLUMN "organizationId" TO "organizationIdValue"` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "organization" alter column "organizationIdValue" type varchar(100) using "organizationIdValue"::varchar(100);` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise {} 16 | } 17 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1672914609976-networkQuorumSet.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class networkQuorumSet1672914609976 implements MigrationInterface { 4 | name = 'networkQuorumSet1672914609976'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "network_snapshot" ADD "configurationQuorumset" jsonb NOT NULL` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "network_snapshot" ADD "configurationQuorumsethash" character varying(64) NOT NULL` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "network_snapshot" DROP COLUMN "configurationQuorumsethash"` 18 | ); 19 | await queryRunner.query( 20 | `ALTER TABLE "network_snapshot" DROP COLUMN "configurationQuorumset"` 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1673431104692-passphrase.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class passphrase1673431104692 implements MigrationInterface { 4 | name = 'passphrase1673431104692'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "network" ADD "passphrase" text NOT NULL` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise { 13 | await queryRunner.query(`ALTER TABLE "network" DROP COLUMN "passphrase"`); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1673434349044-unique-network-id.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class uniqueNetworkId1673434349044 implements MigrationInterface { 4 | name = 'uniqueNetworkId1673434349044'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "network_change" ADD CONSTRAINT "UQ_c24068b8094a20a9a4c8a0f7ff0" UNIQUE ("networkIdValue")` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "network" ADD CONSTRAINT "UQ_a6427dcac5713566cbf8b7b0162" UNIQUE ("networkIdValue")` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "network" DROP CONSTRAINT "UQ_a6427dcac5713566cbf8b7b0162"` 18 | ); 19 | await queryRunner.query( 20 | `ALTER TABLE "network_change" DROP CONSTRAINT "UQ_c24068b8094a20a9a4c8a0f7ff0"` 21 | ); 22 | await queryRunner.query( 23 | `CREATE INDEX "IDX_ad5b60bd93fc753f5a5b12bc6f" ON "network_change" ("type") ` 24 | ); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1673440673258-unique-network-id-network-only.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class uniqueNetworkIdNetworkOnly1673440673258 4 | implements MigrationInterface 5 | { 6 | name = 'uniqueNetworkIdNetworkOnly1673440673258'; 7 | 8 | public async up(queryRunner: QueryRunner): Promise { 9 | await queryRunner.query( 10 | `ALTER TABLE "network_change" DROP CONSTRAINT "UQ_c24068b8094a20a9a4c8a0f7ff0"` 11 | ); 12 | } 13 | 14 | public async down(queryRunner: QueryRunner): Promise {} 15 | } 16 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1673601015030-network-scan.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class networkScan1673601015030 implements MigrationInterface { 4 | name = 'networkScan1673601015030'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "network_update" RENAME TO "network_scan"` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise {} 13 | } 14 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1673898386176-last-ip-change.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class lastIpChange1673898386176 implements MigrationInterface { 4 | name = 'lastIpChange1673898386176'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "node_snap_shot" DROP COLUMN "ipChange"` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "node_snap_shot" ADD "lastIpChange" TIMESTAMP WITH TIME ZONE default NULL` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise {} 16 | } 17 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1674643629973-home-domain.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class homeDomain1674643629973 implements MigrationInterface { 4 | name = 'homeDomain1674643629973'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "organization" ALTER COLUMN "homeDomain" SET NOT NULL` 9 | ); 10 | } 11 | 12 | public async down(queryRunner: QueryRunner): Promise { 13 | await queryRunner.query( 14 | `ALTER TABLE "organization" ALTER COLUMN "homeDomain" DROP NOT NULL` 15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1674815649104-name-nullable.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class nameNullable1674815649104 implements MigrationInterface { 4 | name = 'nameNullable1674815649104'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query(`update organization set "homeDomain" = "organizationIdValue" 8 | where "homeDomain" is null`); 9 | await queryRunner.query( 10 | `CREATE UNIQUE INDEX "IDX_b02d3175c52b687e002a4fb5f1" ON "organization" ("homeDomain") ` 11 | ); 12 | await queryRunner.query( 13 | `ALTER TABLE "organization_snap_shot" 14 | ALTER COLUMN "name" DROP NOT NULL` 15 | ); 16 | } 17 | 18 | public async down(queryRunner: QueryRunner): Promise { 19 | await queryRunner.query( 20 | `DROP INDEX "public"."IDX_b02d3175c52b687e002a4fb5f1"` 21 | ); 22 | await queryRunner.query( 23 | `ALTER TABLE "organization_snap_shot" 24 | ALTER COLUMN "name" SET NOT NULL` 25 | ); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1694520337940-TomlStateMigration.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class TomlStateMigration1694520337940 implements MigrationInterface { 4 | name = 'TomlStateMigration1694520337940'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `CREATE TYPE "public"."organization_measurement_tomlstate_enum" AS ENUM('Unknown', 'Ok', 'RequestTimeout', 'DNSLookupFailed', 'HostnameResolutionFailed', 'ConnectionTimeout', 'ConnectionRefused', 'ConnectionResetByPeer', 'SocketClosedPrematurely', 'SocketTimeout', 'HostUnreachable', 'NotFound', 'ParsingError', 'Forbidden', 'ServerError', 'UnsupportedVersion', 'UnspecifiedError', 'ValidatorNotSEP20Linked', 'EmptyValidatorsField')` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "organization_measurement" ADD "tomlState" "public"."organization_measurement_tomlstate_enum" NOT NULL DEFAULT 'Unknown'` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "organization_measurement" DROP COLUMN "tomlState"` 18 | ); 19 | await queryRunner.query( 20 | `DROP TYPE "public"."organization_measurement_tomlstate_enum"` 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1713440843438-lag.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from "typeorm"; 2 | 3 | export class Lag1713440843438 implements MigrationInterface { 4 | name = 'Lag1713440843438' 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query(`ALTER TABLE "node_measurement_v2" ADD "lag" smallint NOT NULL DEFAULT '0'`); 8 | } 9 | 10 | public async down(queryRunner: QueryRunner): Promise { 11 | await queryRunner.query(`ALTER TABLE "node_measurement_v2" DROP COLUMN "lag"`); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/core/infrastructure/database/migrations/1713780808807-lag-nullable.ts: -------------------------------------------------------------------------------- 1 | import { MigrationInterface, QueryRunner } from 'typeorm'; 2 | 3 | export class LagNullable1713780808807 implements MigrationInterface { 4 | name = 'LagNullable1713780808807'; 5 | 6 | public async up(queryRunner: QueryRunner): Promise { 7 | await queryRunner.query( 8 | `ALTER TABLE "node_measurement_v2" ALTER COLUMN "lag" DROP NOT NULL` 9 | ); 10 | await queryRunner.query( 11 | `ALTER TABLE "node_measurement_v2" ALTER COLUMN "lag" DROP DEFAULT` 12 | ); 13 | } 14 | 15 | public async down(queryRunner: QueryRunner): Promise { 16 | await queryRunner.query( 17 | `ALTER TABLE "node_measurement_v2" ALTER COLUMN "lag" SET DEFAULT '0'` 18 | ); 19 | await queryRunner.query( 20 | `ALTER TABLE "node_measurement_v2" ALTER COLUMN "lag" SET NOT NULL` 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/core/infrastructure/di/di-types.ts: -------------------------------------------------------------------------------- 1 | export const CORE_TYPES = { 2 | JobMonitor: Symbol('JobMonitor') 3 | }; 4 | -------------------------------------------------------------------------------- /src/core/infrastructure/http/Throttler.ts: -------------------------------------------------------------------------------- 1 | import * as LRUCache from 'lru-cache'; 2 | 3 | export type Throttle = { 4 | startTime: number; 5 | count: number; 6 | }; 7 | 8 | export class Throttler { 9 | protected cache: LRUCache; 10 | 11 | constructor(protected maxRequestCount: number, protected timeWindow: number) { 12 | this.cache = new LRUCache({ 13 | max: 10000, 14 | ttl: timeWindow * maxRequestCount 15 | }); 16 | } 17 | 18 | processRequest(ip: string, at: Date): void { 19 | const throttle = this.cache.get(ip); 20 | if (!throttle) { 21 | this.cache.set(ip, { 22 | startTime: at.getTime(), 23 | count: 1 24 | }); 25 | } else if (at.getTime() - throttle.startTime >= this.timeWindow) { 26 | this.cache.set(ip, { 27 | startTime: at.getTime(), 28 | count: 1 29 | }); 30 | } else { 31 | throttle.count++; 32 | this.cache.set(ip, throttle); 33 | } 34 | } 35 | 36 | throttled(ip: string): boolean { 37 | const throttle = this.cache.get(ip); 38 | if (!throttle) return false; 39 | 40 | return throttle.count > this.maxRequestCount; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/core/infrastructure/http/__tests__/Throttler.test.ts: -------------------------------------------------------------------------------- 1 | import { Throttler } from '../Throttler'; 2 | 3 | it('should throttle when too many requests in time window', function () { 4 | const throttler = new Throttler(2, 1000 * 60); 5 | const ip = 'localhost'; 6 | throttler.processRequest(ip, new Date()); 7 | expect(throttler.throttled(ip)).toBeFalsy(); 8 | throttler.processRequest(ip, new Date()); 9 | expect(throttler.throttled(ip)).toBeFalsy(); 10 | throttler.processRequest(ip, new Date()); 11 | expect(throttler.throttled(ip)).toBeTruthy(); 12 | 13 | const otherIp = 'other'; 14 | throttler.processRequest(otherIp, new Date()); 15 | expect(throttler.throttled(otherIp)).toBeFalsy(); 16 | }); 17 | 18 | it('should not throttle when max requests span multiple time windows', function () { 19 | const timeWindow = 1000 * 60; 20 | const throttler = new Throttler(1, timeWindow); 21 | 22 | const startTime = new Date(); 23 | const ip = 'localhost'; 24 | throttler.processRequest(ip, startTime); 25 | expect(throttler.throttled(ip)).toBeFalsy(); 26 | 27 | throttler.processRequest(ip, new Date(startTime.getTime() + timeWindow + 1)); 28 | expect(throttler.throttled(ip)).toBeFalsy(); 29 | }); 30 | -------------------------------------------------------------------------------- /src/core/infrastructure/services/LoggerJobMonitor.ts: -------------------------------------------------------------------------------- 1 | import { inject, injectable } from 'inversify'; 2 | import { JobMonitor, MonitoringJob } from '../../services/JobMonitor'; 3 | import { Logger } from '../../services/PinoLogger'; 4 | import 'reflect-metadata'; 5 | import { ok } from 'neverthrow'; 6 | 7 | @injectable() 8 | export class LoggerJobMonitor implements JobMonitor { 9 | constructor(@inject('Logger') private logger: Logger) {} 10 | 11 | async checkIn(job: MonitoringJob) { 12 | this.logger.info('Job check-in', { 13 | job 14 | }); 15 | 16 | return ok(undefined); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/core/infrastructure/services/__tests__/LoggerJobMonitor.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { Logger } from '../../../services/PinoLogger'; 3 | import { MonitoringJob } from '../../../services/JobMonitor'; 4 | import { LoggerJobMonitor } from '../LoggerJobMonitor'; 5 | 6 | describe('LoggerJobMonitor', () => { 7 | test('should log job check in', async () => { 8 | const logger = mock(); 9 | const job: MonitoringJob = { 10 | context: 'context', 11 | status: 'ok' 12 | }; 13 | 14 | const loggerJobMonitor = new LoggerJobMonitor(logger); 15 | await loggerJobMonitor.checkIn(job); 16 | 17 | expect(logger.info).toHaveBeenCalledTimes(1); 18 | expect(logger.info).toHaveBeenCalledWith('Job check-in', { job }); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /src/core/services/ExceptionLogger.ts: -------------------------------------------------------------------------------- 1 | import * as Sentry from '@sentry/node'; 2 | import { inject, injectable } from 'inversify'; 3 | import { Logger } from './PinoLogger'; 4 | 5 | export interface ExceptionLogger { 6 | captureException(error: Error, extra?: Record): void; 7 | } 8 | 9 | export class ConsoleExceptionLogger implements ExceptionLogger { 10 | captureException(error: Error): void { 11 | //console.log('Captured exception'); 12 | //console.error(error); 13 | } 14 | } 15 | 16 | @injectable() 17 | export class SentryExceptionLogger implements ExceptionLogger { 18 | constructor(sentryDSN: string, @inject('Logger') protected logger: Logger) { 19 | Sentry.init({ 20 | dsn: sentryDSN 21 | }); 22 | } 23 | 24 | captureException(error: Error, extra?: Record): void { 25 | this.logger.error(error.message, extra); 26 | Sentry.captureException(error, extra ? { extra: extra } : undefined); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/core/services/HeartBeater.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | 3 | export interface HeartBeater { 4 | tick(): Promise>; 5 | } 6 | -------------------------------------------------------------------------------- /src/core/services/JobMonitor.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | 3 | export interface MonitoringJob { 4 | context: string; 5 | status: 'in_progress' | 'ok' | 'error'; 6 | } 7 | 8 | export interface JobMonitor { 9 | checkIn(job: MonitoringJob): Promise>; 10 | } 11 | -------------------------------------------------------------------------------- /src/core/services/LoopTimer.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import 'reflect-metadata'; 3 | 4 | @injectable() 5 | export class LoopTimer { 6 | private startTime?: number; 7 | private endTime?: number; 8 | private idle = true; 9 | private loopTime = 1000 * 60 * 3; 10 | 11 | start(loopTime?: number) { 12 | if (!this.idle) throw new Error('Timer already started'); 13 | if (loopTime) this.loopTime = loopTime; 14 | this.startTime = Date.now(); 15 | this.idle = false; 16 | } 17 | 18 | stop() { 19 | if (this.idle) throw new Error('Timer not started'); 20 | this.endTime = Date.now(); 21 | this.idle = true; 22 | } 23 | 24 | getElapsedTime(): number { 25 | if (!this.startTime || !this.endTime) 26 | throw new Error('Timer not started and stopped'); 27 | return this.endTime - this.startTime; 28 | } 29 | 30 | getRemainingTime(): number { 31 | return this.loopTime - this.getElapsedTime(); 32 | } 33 | 34 | loopExceededMaxTime(): boolean { 35 | return this.getElapsedTime() > this.loopTime; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/core/services/__mocks__/ExceptionLoggerMock.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | import { ExceptionLogger } from '../ExceptionLogger'; 3 | 4 | export class ExceptionLoggerMock implements ExceptionLogger { 5 | captureException(error: Error): void {} 6 | } 7 | -------------------------------------------------------------------------------- /src/core/services/__mocks__/LoggerMock.ts: -------------------------------------------------------------------------------- 1 | import { logFn, Logger } from '../PinoLogger'; 2 | /* eslint-disable */ 3 | export class LoggerMock implements Logger { 4 | debug: logFn = (message, context) => { 5 | console.log(message, context); 6 | }; 7 | error: logFn = (message, context) => { 8 | console.log(message, context); 9 | }; 10 | fatal: logFn = (message, context) => { 11 | console.log(message, context); 12 | }; 13 | info: logFn = (message, context) => { 14 | //console.log(message, context); 15 | }; 16 | trace: logFn = (message, context) => { 17 | console.log(message, context); 18 | }; 19 | warn: logFn = (message, context) => { 20 | console.log(message, context); 21 | }; 22 | getRawLogger: any; 23 | } 24 | -------------------------------------------------------------------------------- /src/core/services/__tests__/HttpQueue.test.ts: -------------------------------------------------------------------------------- 1 | import { HttpQueue, RequestMethod } from '../HttpQueue'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { LoggerMock } from '../__mocks__/LoggerMock'; 4 | import { HttpService } from '../HttpService'; 5 | import { createDummyHistoryBaseUrl } from '../../../history-scan/domain/history-archive/__fixtures__/HistoryBaseUrl'; 6 | import { ok } from 'neverthrow'; 7 | 8 | it('should bust cache', async function () { 9 | const httpService = mock(); 10 | httpService.get.mockResolvedValue( 11 | ok({ status: 200, data: [], statusText: 'ok', headers: {} }) 12 | ); 13 | const httpQueue = new HttpQueue(httpService, new LoggerMock()); 14 | 15 | await httpQueue.sendRequests( 16 | [ 17 | { 18 | url: createDummyHistoryBaseUrl(), 19 | meta: {}, 20 | method: RequestMethod.GET 21 | } 22 | ][Symbol.iterator](), 23 | { 24 | cacheBusting: true, 25 | concurrency: 1, 26 | rampUpConnections: false, 27 | nrOfRetries: 0, 28 | stallTimeMs: 100, 29 | httpOptions: {} 30 | } 31 | ); 32 | 33 | expect(httpService.get).toHaveBeenCalledTimes(1); 34 | expect( 35 | httpService.get.mock.calls[0][0].value.indexOf('bust') > 0 36 | ).toBeTruthy(); 37 | }); 38 | -------------------------------------------------------------------------------- /src/core/utilities/AsyncFunctionStaller.ts: -------------------------------------------------------------------------------- 1 | import { asyncSleep } from './asyncSleep'; 2 | 3 | export async function stall( 4 | minTimeMs: number, 5 | operation: (...operationParameters: Args) => Return, 6 | ...parameters: Args 7 | ): Promise { 8 | const time = new Date().getTime(); 9 | const result = await operation(...parameters); 10 | const elapsed = new Date().getTime() - time; 11 | if (elapsed < minTimeMs) await asyncSleep(minTimeMs - elapsed); 12 | return result; 13 | } 14 | -------------------------------------------------------------------------------- /src/core/utilities/TestUtils.ts: -------------------------------------------------------------------------------- 1 | import { DataSource } from 'typeorm'; 2 | 3 | export class TestUtils { 4 | private static async getEntities(dataSource: DataSource) { 5 | const entities: { name: string; tableName: string }[] = []; 6 | dataSource.entityMetadatas.forEach((x) => 7 | entities.push({ name: x.name, tableName: x.tableName }) 8 | ); 9 | return entities; 10 | } 11 | 12 | //could be faster if tests supplied a list of entities that should be cleaned. 13 | static async resetDB(dataSource: DataSource) { 14 | if (process.env.NODE_ENV !== 'test') { 15 | throw new Error('Trying to reset DB outside of test environment'); 16 | } 17 | try { 18 | const entities = await TestUtils.getEntities(dataSource); 19 | for (const entity of entities) { 20 | const repository = await dataSource.getRepository(entity.name); 21 | await repository.query(`TRUNCATE TABLE ${entity.tableName} CASCADE;`); 22 | } 23 | } catch (error) { 24 | throw new Error(`ERROR: Reset test db: ${error}`); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/core/utilities/TypeGuards.ts: -------------------------------------------------------------------------------- 1 | export function isString(param: unknown): param is string { 2 | return typeof param === 'string'; 3 | } 4 | 5 | export function isArray(array: unknown): array is unknown[] { 6 | return Array.isArray(array); 7 | } 8 | 9 | export function isObject(obj: unknown): obj is Record { 10 | return typeof obj === 'object' && obj !== null; 11 | } 12 | 13 | export function isNumber(number: unknown): number is number { 14 | return typeof number === 'number'; 15 | } 16 | 17 | export default function isPartOfStringEnum>( 18 | value: unknown, 19 | myEnum: T 20 | ): value is T[keyof T] { 21 | return Object.values(myEnum).includes(value); 22 | } 23 | 24 | export function instanceOfError(object: unknown): object is Error { 25 | return isObject(object) && 'name' in object && 'message' in object; 26 | } 27 | -------------------------------------------------------------------------------- /src/core/utilities/__tests__/FunctionStaller.test.ts: -------------------------------------------------------------------------------- 1 | import { stall } from '../AsyncFunctionStaller'; 2 | 3 | it('should stall a fast async function', async function () { 4 | const increaseWithOneAsync = async (nr: number) => { 5 | return new Promise((resolve) => { 6 | nr++; 7 | setImmediate(() => resolve(nr)); 8 | }); 9 | }; 10 | 11 | const time = new Date().getTime(); 12 | const result = await stall(51, increaseWithOneAsync, 1); 13 | const elapsed = new Date().getTime() - time; 14 | 15 | expect(result).toEqual(2); 16 | expect(elapsed).toBeGreaterThan(50); 17 | }); 18 | -------------------------------------------------------------------------------- /src/core/utilities/__tests__/HttpRequestRetry.test.ts: -------------------------------------------------------------------------------- 1 | import { err } from 'neverthrow'; 2 | import { HttpError } from '../../services/HttpService'; 3 | import { retryHttpRequestIfNeeded } from '../HttpRequestRetry'; 4 | 5 | it('should retry the correct amount of times', async function () { 6 | const actionWrap = createErrorAction('500', 500); 7 | await retryHttpRequestIfNeeded(2, 400, actionWrap.action); 8 | 9 | expect(actionWrap.getCount()).toEqual(2); 10 | }); 11 | 12 | it('should retry on timeout', async function () { 13 | const actionWrap = createErrorAction('ETIMEDOUT'); 14 | await retryHttpRequestIfNeeded(3, 400, actionWrap.action); 15 | 16 | expect(actionWrap.getCount()).toEqual(3); 17 | }); 18 | 19 | function createErrorAction(code: string, status?: number) { 20 | let counter = 0; 21 | const getCount = () => { 22 | return counter; 23 | }; 24 | const action = async () => { 25 | counter++; 26 | return err( 27 | new HttpError( 28 | 'message', 29 | code, 30 | status === undefined 31 | ? undefined 32 | : { 33 | data: null, 34 | status: status, 35 | headers: [], 36 | statusText: 'text' 37 | } 38 | ) 39 | ); 40 | }; 41 | 42 | return { getCount: getCount, action: action }; 43 | } 44 | -------------------------------------------------------------------------------- /src/core/utilities/__tests__/TypeGuards.test.ts: -------------------------------------------------------------------------------- 1 | import isPartOfStringEnum, { isNumber } from '../TypeGuards'; 2 | import { EventType } from '../../../notifications/domain/event/Event'; 3 | 4 | enum Type { 5 | myType = 'myType' 6 | } 7 | 8 | test('enum', function () { 9 | expect(isPartOfStringEnum('myType', Type)).toBeTruthy(); 10 | }); 11 | 12 | test('event part of enum', function () { 13 | expect(isPartOfStringEnum('NodeXUpdatesInactive', EventType)).toBeTruthy(); 14 | }); 15 | 16 | test('is a number', () => { 17 | expect(isNumber(undefined)).toBeFalsy(); 18 | expect(isNumber(0)).toBeTruthy(); 19 | expect(isNumber('0')).toBeFalsy(); 20 | }); 21 | -------------------------------------------------------------------------------- /src/core/utilities/__tests__/getLowestNumber.test.ts: -------------------------------------------------------------------------------- 1 | import { getLowestNumber } from '../getLowestNumber'; 2 | 3 | it('should get lowest number in large array', function () { 4 | const numbers = [...Array(10000000).keys()]; 5 | const lowestNumber = getLowestNumber(numbers.reverse()); 6 | expect(lowestNumber).toBe(0); 7 | }); 8 | -------------------------------------------------------------------------------- /src/core/utilities/asyncSleep.ts: -------------------------------------------------------------------------------- 1 | export async function asyncSleep(time: number) { 2 | return new Promise((resolve) => { 3 | setTimeout(resolve, time); 4 | }); 5 | } 6 | -------------------------------------------------------------------------------- /src/core/utilities/getDateFromParam.ts: -------------------------------------------------------------------------------- 1 | import { isDateString } from './isDateString'; 2 | import { isString } from './TypeGuards'; 3 | 4 | export function getDateFromParam(param: unknown): Date { 5 | let time: Date; 6 | if (!(param && isDateString(param)) || !isString(param)) { 7 | time = new Date(); 8 | } else { 9 | time = new Date(param); 10 | } 11 | 12 | return time; 13 | } 14 | -------------------------------------------------------------------------------- /src/core/utilities/getLowestNumber.ts: -------------------------------------------------------------------------------- 1 | //works for large arrays 2 | export function getLowestNumber(numbers: number[]): number { 3 | let lowest = Number.MAX_SAFE_INTEGER; 4 | for (const nr of numbers) { 5 | if (nr < lowest) lowest = nr; 6 | } 7 | return lowest; 8 | } 9 | -------------------------------------------------------------------------------- /src/core/utilities/getMaximumNumber.ts: -------------------------------------------------------------------------------- 1 | export const getMaximumNumber = (arrayOfNumbers: Array) => { 2 | return arrayOfNumbers.reduce( 3 | (max, currentNumber) => (max >= currentNumber ? max : currentNumber), 4 | -Infinity 5 | ); 6 | }; 7 | -------------------------------------------------------------------------------- /src/core/utilities/isDateString.ts: -------------------------------------------------------------------------------- 1 | export function isDateString(dateString?: any) { 2 | if (dateString === undefined || dateString === null) return false; 3 | 4 | const timestamp = Date.parse(dateString); 5 | 6 | return !isNaN(timestamp); 7 | } 8 | -------------------------------------------------------------------------------- /src/core/utilities/isZLibError.ts: -------------------------------------------------------------------------------- 1 | import { isObject, isString } from './TypeGuards'; 2 | 3 | interface ZLibError { 4 | errno: number; 5 | code: string; 6 | message: string; 7 | } 8 | export function isZLibError(error: unknown): error is ZLibError { 9 | return ( 10 | isObject(error) && 11 | Number.isInteger(error.errno) && 12 | isString(error.code) && 13 | isString(error.message) 14 | ); 15 | } 16 | -------------------------------------------------------------------------------- /src/core/utilities/mapUnknownToError.ts: -------------------------------------------------------------------------------- 1 | import { isString } from './TypeGuards'; 2 | 3 | export function mapUnknownToError(e: unknown): Error { 4 | if (e instanceof Error) { 5 | return e; 6 | } 7 | if (isString(e)) { 8 | return new Error(e); 9 | } 10 | 11 | return new Error('Unspecified error: ' + e); 12 | } 13 | -------------------------------------------------------------------------------- /src/core/utilities/sortDescending.ts: -------------------------------------------------------------------------------- 1 | export function sortDescending(myArray: number[]) { 2 | return [...myArray].sort((a, b) => b - a); 3 | } 4 | -------------------------------------------------------------------------------- /src/history-scan/README.md: -------------------------------------------------------------------------------- 1 | # History scan 2 | Continuously scan history archives for errors. 3 | -------------------------------------------------------------------------------- /src/history-scan/domain/check-point/CheckPointFrequency.ts: -------------------------------------------------------------------------------- 1 | export interface CheckPointFrequency { 2 | get(): number 3 | } -------------------------------------------------------------------------------- /src/history-scan/domain/check-point/CheckPointGenerator.ts: -------------------------------------------------------------------------------- 1 | import { CheckPointFrequency } from './CheckPointFrequency'; 2 | import { inject, injectable } from 'inversify'; 3 | import { TYPES } from '../../infrastructure/di/di-types'; 4 | import 'reflect-metadata'; 5 | 6 | export type CheckPoint = number; 7 | 8 | @injectable() 9 | export class CheckPointGenerator { 10 | constructor( 11 | @inject(TYPES.CheckPointFrequency) 12 | public checkPointFrequency: CheckPointFrequency 13 | ) {} 14 | 15 | *generate( 16 | fromLedger: number, 17 | toLedger: number 18 | ): IterableIterator { 19 | let checkPoint = this.getClosestHigherCheckPoint(fromLedger); 20 | while (checkPoint <= toLedger) { 21 | yield checkPoint; 22 | checkPoint += 64; 23 | } 24 | } 25 | 26 | getClosestHigherCheckPoint(ledger: number): CheckPoint { 27 | return ( 28 | Math.floor( 29 | (ledger + this.checkPointFrequency.get()) / 30 | this.checkPointFrequency.get() 31 | ) * 32 | this.checkPointFrequency.get() - 33 | 1 34 | ); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/history-scan/domain/check-point/StandardCheckPointFrequency.ts: -------------------------------------------------------------------------------- 1 | import { CheckPointFrequency } from './CheckPointFrequency'; 2 | import { injectable } from 'inversify'; 3 | import 'reflect-metadata'; 4 | 5 | @injectable() 6 | export class StandardCheckPointFrequency implements CheckPointFrequency { 7 | //in the future the frequency could change 8 | get(): number { 9 | return 64; //if needed this could come from configuration 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/history-scan/domain/check-point/__tests__/CheckPointGenerator.test.ts: -------------------------------------------------------------------------------- 1 | import { CheckPointGenerator } from '../CheckPointGenerator'; 2 | import { StandardCheckPointFrequency } from '../StandardCheckPointFrequency'; 3 | 4 | it('should generate correct checkpoints in the supplied range', function () { 5 | const generatorClass = new CheckPointGenerator( 6 | new StandardCheckPointFrequency() 7 | ); 8 | const generator = generatorClass.generate(0, 128); 9 | expect(generator.next().value).toEqual(63); 10 | expect(generator.next().value).toEqual(127); 11 | expect(generator.next().done).toEqual(true); 12 | }); 13 | -------------------------------------------------------------------------------- /src/history-scan/domain/check-point/__tests__/StandardCheckPointFrequency.test.ts: -------------------------------------------------------------------------------- 1 | import {StandardCheckPointFrequency} from "../StandardCheckPointFrequency"; 2 | 3 | it('should have a frequency of 64', function () { 4 | const frequency = new StandardCheckPointFrequency(); 5 | expect(frequency.get()).toEqual(64); 6 | }); -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/Category.ts: -------------------------------------------------------------------------------- 1 | export enum Category { 2 | results = 'results', 3 | history = 'history', 4 | transactions = 'transactions', 5 | ledger = 'ledger' 6 | } 7 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/HASBucketHashExtractor.ts: -------------------------------------------------------------------------------- 1 | import { HistoryArchiveState } from './HistoryArchiveState'; 2 | 3 | export class HASBucketHashExtractor { 4 | static getNonZeroHashes(historyArchiveState: HistoryArchiveState): string[] { 5 | const bucketHashes: string[] = []; 6 | historyArchiveState.currentBuckets.forEach((bucket) => { 7 | bucketHashes.push(bucket.curr); 8 | bucketHashes.push(bucket.snap); 9 | 10 | const nextOutput = bucket.next.output; 11 | if (nextOutput) bucketHashes.push(nextOutput); 12 | }); 13 | 14 | return bucketHashes.filter( 15 | (hash) => !HASBucketHashExtractor.isZeroHash(hash) 16 | ); 17 | } 18 | 19 | private static isZeroHash(hash: string) { 20 | return parseInt(hash, 16) === 0; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/HistoryArchiveService.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | import { Url } from '../../../core/domain/Url'; 3 | 4 | export interface HistoryArchiveService { 5 | getHistoryArchiveUrls(): Promise>; 6 | } 7 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/HistoryArchiveState.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchemaType } from 'ajv'; 2 | 3 | export interface HistoryArchiveState { 4 | version: number; 5 | server: string; 6 | currentLedger: number; 7 | networkPassphrase?: string; 8 | currentBuckets: { 9 | curr: string; 10 | snap: string; 11 | next: { 12 | state: number; 13 | output?: string; 14 | }; 15 | }[]; 16 | } 17 | 18 | export const HistoryArchiveStateSchema: JSONSchemaType = { 19 | type: 'object', 20 | properties: { 21 | version: { type: 'integer' }, 22 | server: { type: 'string' }, 23 | currentLedger: { type: 'number' }, 24 | networkPassphrase: { type: 'string', nullable: true }, 25 | currentBuckets: { 26 | type: 'array', 27 | items: { 28 | type: 'object', 29 | properties: { 30 | curr: { type: 'string' }, 31 | snap: { type: 'string' }, 32 | next: { 33 | type: 'object', 34 | properties: { 35 | state: { type: 'number' }, 36 | output: { type: 'string', nullable: true } 37 | }, 38 | required: ['state'] 39 | } 40 | }, 41 | required: ['curr', 'snap', 'next'] 42 | }, 43 | minItems: 0 44 | } 45 | }, 46 | required: ['version', 'server', 'currentLedger', 'currentBuckets'] 47 | }; 48 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/__fixtures__/HistoryBaseUrl.ts: -------------------------------------------------------------------------------- 1 | import { Url } from '../../../../core/domain/Url'; 2 | 3 | let counter = 0; 4 | 5 | export function createDummyHistoryBaseUrl() { 6 | const url = Url.create(`https://history${counter}.stellar.org`); 7 | if (url.isErr()) throw url.error; 8 | 9 | counter++; 10 | return url.value; 11 | } 12 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/__tests__/UrlBuilder.test.ts: -------------------------------------------------------------------------------- 1 | import { UrlBuilder } from '../UrlBuilder'; 2 | import { Url } from '../../../../core/domain/Url'; 3 | import { createDummyHistoryBaseUrl } from '../__fixtures__/HistoryBaseUrl'; 4 | import { Category } from '../Category'; 5 | 6 | it('should return ledger url', function () { 7 | const historyBaseUrl = Url.create('https://history.stellar.org'); 8 | if (historyBaseUrl.isErr()) throw historyBaseUrl.error; 9 | 10 | const url = UrlBuilder.getCategoryUrl( 11 | historyBaseUrl.value, 12 | 39279103, 13 | Category.ledger 14 | ); 15 | 16 | expect(url.value).toEqual( 17 | 'https://history.stellar.org/ledger/02/57/59/ledger-025759ff.xdr.gz' 18 | ); 19 | }); 20 | 21 | it('should generate correct bucket url', function () { 22 | const url = createDummyHistoryBaseUrl(); 23 | expect( 24 | UrlBuilder.getBucketUrl( 25 | url, 26 | 'bd96d76dec3196938aa7acb8116ddb5e442201032ab32dfb5af30fb8563c04d5' 27 | ).value 28 | ).toEqual( 29 | url.value + 30 | '/bucket/bd/96/d7/bucket-bd96d76dec3196938aa7acb8116ddb5e442201032ab32dfb5af30fb8563c04d5.xdr.gz' 31 | ); 32 | }); 33 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/__tests__/hashBucketList.test.ts: -------------------------------------------------------------------------------- 1 | import { hashBucketList } from '../hashBucketList'; 2 | import { getDummyHistoryArchiveState } from '../__fixtures__/getDummyHistoryArchiveState'; 3 | 4 | it('should hash correctly', function () { 5 | const result = hashBucketList(getDummyHistoryArchiveState()); 6 | expect(result.isOk()).toBeTruthy(); 7 | if (result.isErr()) throw result.error; 8 | 9 | expect(result.value.ledger).toEqual(40351615); 10 | expect(result.value.hash).toEqual( 11 | 'vtRf4YP8qFhI3d7AtxQsgMM1AJ60P/6e35Brm4UKJPs=' 12 | ); 13 | }); 14 | -------------------------------------------------------------------------------- /src/history-scan/domain/history-archive/hashBucketList.ts: -------------------------------------------------------------------------------- 1 | import { HistoryArchiveState } from './HistoryArchiveState'; 2 | import { createHash } from 'crypto'; 3 | import { err, ok, Result } from 'neverthrow'; 4 | import { mapUnknownToError } from '../../../core/utilities/mapUnknownToError'; 5 | 6 | export function hashBucketList( 7 | historyArchiveState: HistoryArchiveState 8 | ): Result< 9 | { 10 | ledger: number; 11 | hash: string; 12 | }, 13 | Error 14 | > { 15 | try { 16 | const bucketListHash = createHash('sha256'); 17 | historyArchiveState.currentBuckets.forEach((bucket) => { 18 | const bucketHash = createHash('sha256'); 19 | bucketHash.write(Buffer.from(bucket.curr, 'hex')); 20 | bucketHash.write(Buffer.from(bucket.snap, 'hex')); 21 | bucketListHash.write(bucketHash.digest()); 22 | }); 23 | 24 | return ok({ 25 | ledger: historyArchiveState.currentLedger, 26 | hash: bucketListHash.digest().toString('base64') 27 | }); 28 | } catch (e) { 29 | console.log(e); 30 | return err(mapUnknownToError(e)); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/history-scan/domain/scan/ScanError.ts: -------------------------------------------------------------------------------- 1 | import { IdentifiedValueObject } from '../../../core/domain/IdentifiedValueObject'; 2 | import { Column, Entity } from 'typeorm'; 3 | 4 | export enum ScanErrorType { 5 | TYPE_VERIFICATION, 6 | TYPE_CONNECTION 7 | } 8 | 9 | @Entity({ name: 'history_archive_scan_error' }) 10 | export class ScanError extends IdentifiedValueObject implements Error { 11 | public readonly name = 'ScanError'; 12 | 13 | @Column('enum', { enum: ScanErrorType, nullable: false }) 14 | public readonly type: ScanErrorType; 15 | @Column('text', { nullable: false }) 16 | public readonly url: string; 17 | @Column('text', { nullable: false }) 18 | public readonly message: string; 19 | 20 | constructor(type: ScanErrorType, url: string, message: string) { 21 | super(); 22 | this.type = type; 23 | this.url = url; 24 | this.message = message; 25 | } 26 | 27 | equals(other: this): boolean { 28 | return ( 29 | this.type === other.type && 30 | this.url === other.url && 31 | this.message === other.message 32 | ); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/history-scan/domain/scan/ScanRepository.ts: -------------------------------------------------------------------------------- 1 | import { Scan } from './Scan'; 2 | 3 | export interface ScanRepository { 4 | save(scans: Scan[]): Promise; 5 | findLatestByUrl(url: string): Promise; 6 | findLatest(): Promise; 7 | } 8 | -------------------------------------------------------------------------------- /src/history-scan/domain/scan/ScanResult.ts: -------------------------------------------------------------------------------- 1 | import { ScanError } from './ScanError'; 2 | import { LedgerHeader } from '../scanner/Scanner'; 3 | 4 | export interface ScanResult { 5 | readonly latestLedgerHeader: LedgerHeader; 6 | readonly error?: ScanError; 7 | } 8 | -------------------------------------------------------------------------------- /src/history-scan/domain/scan/ScanSettings.ts: -------------------------------------------------------------------------------- 1 | //Actual settings used for scan, if necessary determined just before starting the scan 2 | export interface ScanSettings { 3 | readonly fromLedger: number; 4 | readonly toLedger: number; 5 | readonly concurrency: number; 6 | readonly isSlowArchive: boolean | null; 7 | readonly latestScannedLedger: number; 8 | readonly latestScannedLedgerHeaderHash: string | null; 9 | } 10 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/HasherPool.ts: -------------------------------------------------------------------------------- 1 | import { WorkerPool } from 'workerpool'; 2 | import * as workerpool from 'workerpool'; 3 | import * as os from 'os'; 4 | 5 | // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging 6 | export class HasherPool { 7 | public workerpool: WorkerPool; 8 | 9 | public terminated = false; 10 | constructor() { 11 | try { 12 | require(__dirname + '/hash-worker.import.js'); 13 | this.workerpool = workerpool.pool(__dirname + '/hash-worker.import.js', { 14 | minWorkers: Math.max((os.cpus().length || 4) - 1, 1) 15 | }); 16 | } catch (e) { 17 | this.workerpool = workerpool.pool(__dirname + '/hash-worker.js', { 18 | minWorkers: Math.max((os.cpus().length || 4) - 1, 1) 19 | }); 20 | } 21 | } 22 | } 23 | 24 | // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging 25 | export interface HasherPool { 26 | terminated: boolean; 27 | workerpool: WorkerPool; 28 | } 29 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/WorkerPoolLoadTracker.ts: -------------------------------------------------------------------------------- 1 | import { HasherPool } from './HasherPool'; 2 | 3 | export class WorkerPoolLoadTracker { 4 | private readonly loadTrackTimer: NodeJS.Timer; 5 | private poolFullCount = 0; 6 | private poolCheckIfFullCount = 0; 7 | 8 | constructor(workerPool: HasherPool, private maxPendingTasks: number) { 9 | this.loadTrackTimer = setInterval(() => { 10 | this.poolCheckIfFullCount++; 11 | if (this.workerPoolIsFull(workerPool)) 12 | //pool 80 percent of max pending is considered full 13 | this.poolFullCount++; 14 | }, 10000); 15 | } 16 | 17 | private workerPoolIsFull(pool: HasherPool) { 18 | return pool.workerpool.stats().pendingTasks >= this.maxPendingTasks * 0.8; 19 | } 20 | 21 | getPoolFullPercentage() { 22 | return this.poolCheckIfFullCount > 0 23 | ? Math.round((this.poolFullCount / this.poolCheckIfFullCount) * 100) 24 | : 0; 25 | } 26 | getPoolFullPercentagePretty() { 27 | return this.getPoolFullPercentage() + '%'; 28 | } 29 | 30 | stop() { 31 | clearTimeout(this.loadTrackTimer); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/bucket.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/bucket.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/bucket_empty.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/bucket_empty.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/ledger.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/ledger.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/ledger_empty.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/ledger_empty.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/results.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/results.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/results_empty.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/results_empty.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/transactions.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/transactions.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__fixtures__/transactions_empty.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/domain/scanner/__fixtures__/transactions_empty.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/__tests__/mapHttpQueueErrorToScanError.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FileNotFoundError, 3 | QueueError, 4 | RequestMethod 5 | } from '../../../../core/services/HttpQueue'; 6 | import { createDummyHistoryBaseUrl } from '../../history-archive/__fixtures__/HistoryBaseUrl'; 7 | import { mapHttpQueueErrorToScanError } from '../mapHttpQueueErrorToScanError'; 8 | import { ScanError, ScanErrorType } from '../../scan/ScanError'; 9 | 10 | it('should map to scan error', function () { 11 | const error = new QueueError({ 12 | url: createDummyHistoryBaseUrl(), 13 | meta: { checkPoint: 100 }, 14 | method: RequestMethod.GET 15 | }); 16 | 17 | const mappedError = mapHttpQueueErrorToScanError(error); 18 | 19 | expect(mappedError).toBeInstanceOf(ScanError); 20 | }); 21 | 22 | it('File not found should map to verification error', function () { 23 | const error = new FileNotFoundError({ 24 | url: createDummyHistoryBaseUrl(), 25 | meta: { checkPoint: 100 }, 26 | method: RequestMethod.GET 27 | }); 28 | 29 | const mappedError = mapHttpQueueErrorToScanError(error); 30 | 31 | expect(mappedError).toBeInstanceOf(ScanError); 32 | expect(mappedError.type).toEqual(ScanErrorType.TYPE_VERIFICATION); 33 | }); 34 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/hash-worker.import.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/no-var-requires 2 | require('ts-node').register(); 3 | // eslint-disable-next-line @typescript-eslint/no-var-requires 4 | const path = require('path'); 5 | // eslint-disable-next-line no-undef 6 | require(path.resolve(__dirname, 'hash-worker.ts')); 7 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/mapHttpQueueErrorToScanError.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FileNotFoundError, 3 | QueueError 4 | } from '../../../core/services/HttpQueue'; 5 | import 'reflect-metadata'; 6 | import { ScanError, ScanErrorType } from '../scan/ScanError'; 7 | 8 | export function mapHttpQueueErrorToScanError(error: QueueError): ScanError { 9 | if (error instanceof FileNotFoundError) { 10 | return new ScanError( 11 | ScanErrorType.TYPE_VERIFICATION, 12 | error.request.url.value, 13 | 'File not found' 14 | ); 15 | } 16 | if (error.cause instanceof ScanError) { 17 | return error.cause; 18 | } 19 | return new ScanError( 20 | ScanErrorType.TYPE_CONNECTION, 21 | error.request.url.value, 22 | error.cause?.message ?? 'Connection error' 23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/sortHistoryUrls.ts: -------------------------------------------------------------------------------- 1 | import { Url } from '../../../core/domain/Url'; 2 | 3 | type urlString = string; 4 | //older or never before scanned urls go to the front 5 | export function sortHistoryUrls( 6 | historyUrls: Url[], 7 | scanDates: Map 8 | ): Url[] { 9 | return historyUrls.sort((a: Url, b: Url): number => { 10 | const aScanDate = scanDates.get(a.value); 11 | const bScanDate = scanDates.get(b.value); 12 | 13 | if (!aScanDate) return -1; 14 | 15 | if (!bScanDate) return 1; 16 | 17 | return aScanDate.getTime() - bScanDate.getTime(); 18 | }); 19 | } 20 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/FirstLedgerHashPolicy.ts: -------------------------------------------------------------------------------- 1 | import { CategoryScanner } from '../../../CategoryScanner'; 2 | import { IHashCalculationPolicy } from './IHashCalculationPolicy'; 3 | 4 | export class FirstLedgerHashPolicy implements IHashCalculationPolicy { 5 | calculateHash() { 6 | return CategoryScanner.ZeroHash; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/GeneralizedTransactionSetHashPolicy.ts: -------------------------------------------------------------------------------- 1 | import { IHashCalculationPolicy } from './IHashCalculationPolicy'; 2 | import { createHash } from 'crypto'; 3 | import { xdr } from '@stellar/stellar-base'; 4 | 5 | export class GeneralizedTransactionSetHashPolicy 6 | implements IHashCalculationPolicy 7 | { 8 | calculateHash(previousLedgerHeaderHash: string): string { 9 | // @ts-ignore 10 | const emptyPhase = new xdr.TransactionPhase(0, []); 11 | const transactionSetV1 = new xdr.TransactionSetV1({ 12 | previousLedgerHash: Buffer.from(previousLedgerHeaderHash, 'base64'), 13 | phases: [emptyPhase, emptyPhase] //protocol 20 has two phases 14 | }); 15 | 16 | const generalized = new xdr.GeneralizedTransactionSet( 17 | //@ts-ignore 18 | 1, 19 | transactionSetV1 20 | ); 21 | 22 | const hash = createHash('sha256'); 23 | hash.update(generalized.toXDR()); 24 | 25 | return hash.digest('base64'); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/IHashCalculationPolicy.ts: -------------------------------------------------------------------------------- 1 | export interface IHashCalculationPolicy { 2 | calculateHash(previousLedgerHeaderHash: string | undefined): string; 3 | } 4 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/RegularTransactionSetHashPolicy.ts: -------------------------------------------------------------------------------- 1 | import { IHashCalculationPolicy } from './IHashCalculationPolicy'; 2 | import { createHash } from 'crypto'; 3 | 4 | export class RegularTransactionSetHashPolicy implements IHashCalculationPolicy { 5 | calculateHash(previousLedgerHeaderHash: string): string { 6 | const previousLedgerHashHashed = createHash('sha256'); 7 | previousLedgerHashHashed.update( 8 | Buffer.from(previousLedgerHeaderHash, 'base64') 9 | ); 10 | return previousLedgerHashHashed.digest('base64'); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/__tests__/FirstLedgerHashPolicy.test.ts: -------------------------------------------------------------------------------- 1 | import { FirstLedgerHashPolicy } from '../FirstLedgerHashPolicy'; 2 | 3 | describe('FirstLedgerHashPolicy', () => { 4 | let policy: FirstLedgerHashPolicy; 5 | 6 | beforeEach(() => { 7 | policy = new FirstLedgerHashPolicy(); 8 | }); 9 | 10 | describe('calculateHash', () => { 11 | it('should return the correct hash for a given ledger header hash', () => { 12 | const expectedHash = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; 13 | 14 | const result = policy.calculateHash(); 15 | 16 | expect(result).toEqual(expectedHash); 17 | }); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/__tests__/GeneralizedTransactionSetPolicy.test.ts: -------------------------------------------------------------------------------- 1 | import { GeneralizedTransactionSetHashPolicy } from '../GeneralizedTransactionSetHashPolicy'; 2 | import { CategoryScanner } from '../../../../CategoryScanner'; 3 | 4 | describe('GeneralizedTransactionSetPolicy', () => { 5 | let policy: GeneralizedTransactionSetHashPolicy; 6 | 7 | beforeEach(() => { 8 | policy = new GeneralizedTransactionSetHashPolicy(); 9 | }); 10 | 11 | describe('calculateHash', () => { 12 | it('should return the correct hash for a given ledger header hash', () => { 13 | const previousLedgerHeaderHash = CategoryScanner.ZeroHash; 14 | const expectedHash = 'g78ujHf/l3cPVozRVmSx41OiXHOINBm8ijIvMcYCmCw='; // replace with the expected result 15 | 16 | const result = policy.calculateHash(previousLedgerHeaderHash); 17 | 18 | expect(result).toEqual(expectedHash); 19 | }); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /src/history-scan/domain/scanner/verification/empty-transaction-sets/hash-policies/__tests__/RegularTransactionSetPolicy.test.ts: -------------------------------------------------------------------------------- 1 | import { RegularTransactionSetHashPolicy } from '../RegularTransactionSetHashPolicy'; 2 | 3 | describe('RegularTransactionSetHashPolicy', () => { 4 | let policy: RegularTransactionSetHashPolicy; 5 | 6 | beforeEach(() => { 7 | policy = new RegularTransactionSetHashPolicy(); 8 | }); 9 | 10 | describe('calculateHash', () => { 11 | it('should return the correct hash for a given ledger header hash', () => { 12 | const previousLedgerHeaderHash = 'test-hash'; 13 | const expectedHash = 'DMYdyBetnwEz/4ZNAfHD/0uRYNuCtoOo0CT5AJPlHIw='; // replace with the expected result 14 | 15 | const result = policy.calculateHash(previousLedgerHeaderHash); 16 | 17 | expect(result).toEqual(expectedHash); 18 | }); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /src/history-scan/infrastructure/cli/verify-archives.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../core/infrastructure/Kernel'; 2 | import { VerifyArchives } from '../../use-cases/verify-archives/VerifyArchives'; 3 | 4 | // noinspection JSIgnoredPromiseFromCall 5 | main(); 6 | 7 | async function main() { 8 | const kernel = await Kernel.getInstance(); 9 | const verifySingleArchive = kernel.container.get(VerifyArchives); 10 | //handle shutdown 11 | process 12 | .on('SIGTERM', async () => { 13 | await kernel.shutdown(); 14 | process.exit(0); 15 | }) 16 | .on('SIGINT', async () => { 17 | await kernel.shutdown(); 18 | process.exit(0); 19 | }); 20 | 21 | let persist = false; 22 | if (process.argv[2] === '1') { 23 | persist = true; 24 | } 25 | 26 | let loop = true; 27 | if (process.argv[3] === '0') { 28 | loop = false; 29 | } 30 | 31 | await verifySingleArchive.execute({ 32 | persist: persist, 33 | loop: loop 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /src/history-scan/infrastructure/di/di-types.ts: -------------------------------------------------------------------------------- 1 | export const TYPES = { 2 | CheckPointFrequency: Symbol('CheckPointFrequency'), 3 | HistoryArchiveScanRepository: Symbol('HistoryArchiveScanRepository'), 4 | ScanScheduler: Symbol('ScanScheduler'), 5 | HistoryArchiveService: Symbol('HistoryArchiveService') 6 | }; 7 | -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/bucket-bff0722cb3e89655d3b71c3b517a3bc4b20456298e50073745342f28f6f68b7c.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/bucket-bff0722cb3e89655d3b71c3b517a3bc4b20456298e50073745342f28f6f68b7c.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/ledger-0000003f.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/ledger-0000003f.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/ledger-0000007f.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/ledger-0000007f.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/results-0000003f.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/results-0000003f.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/results-0000007f.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/results-0000007f.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/transactions-0000003f.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/transactions-0000003f.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/http/__fixtures__/transactions-0000007f.xdr.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stellarbeat/js-stellarbeat-backend/f189235f7dee1ea139b0b5dff90e0c8f49280fd7/src/history-scan/infrastructure/http/__fixtures__/transactions-0000007f.xdr.gz -------------------------------------------------------------------------------- /src/history-scan/infrastructure/services/HistoryArchiveServiceMock.ts: -------------------------------------------------------------------------------- 1 | import { HistoryArchiveService } from '../../domain/history-archive/HistoryArchiveService'; 2 | import { Url } from '../../../core/domain/Url'; 3 | import { ok, Result } from 'neverthrow'; 4 | 5 | export class HistoryArchiveServiceMock implements HistoryArchiveService { 6 | async getHistoryArchiveUrls(): Promise> { 7 | const urlOrError = Url.create('http://127.0.0.1'); 8 | if (urlOrError.isErr()) throw urlOrError.error; 9 | 10 | return Promise.resolve(ok([urlOrError.value])); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/history-scan/use-cases/get-latest-scan/GetLatestScanDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetLatestScanDTO { 2 | url: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/history-scan/use-cases/get-latest-scan/InvalidUrlError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class InvalidUrlError extends CustomError { 4 | constructor(url: string) { 5 | super(`Invalid url: ${url}`, InvalidUrlError.name); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/history-scan/use-cases/verify-archives/VerifyArchivesDTO.ts: -------------------------------------------------------------------------------- 1 | export interface VerifyArchivesDTO { 2 | persist: boolean; 3 | loop: boolean; 4 | } 5 | -------------------------------------------------------------------------------- /src/history-scan/use-cases/verify-single-archive/VerifySingleArchiveDTO.ts: -------------------------------------------------------------------------------- 1 | export interface VerifySingleArchiveDTO { 2 | historyUrl: string; 3 | fromLedger?: number; 4 | toLedger?: number; 5 | persist: boolean; 6 | maxConcurrency?: number; 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/domain/Change.ts: -------------------------------------------------------------------------------- 1 | export interface Change { 2 | readonly time: Date; 3 | readonly from: Record; 4 | readonly to: Record; 5 | } 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/measurement-aggregation/MeasurementAggregation.ts: -------------------------------------------------------------------------------- 1 | export interface MeasurementAggregation { 2 | time: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/domain/measurement-aggregation/MeasurementAggregationRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementAggregation } from './MeasurementAggregation'; 2 | import { MeasurementAggregationSourceId } from './MeasurementAggregationSourceId'; 3 | 4 | export interface MeasurementAggregationRepository< 5 | T extends MeasurementAggregation 6 | > { 7 | rollup(fromNetworkScanId: number, toNetworkScanId: number): Promise; 8 | findBetween( 9 | id: MeasurementAggregationSourceId, 10 | from: Date, 11 | to: Date 12 | ): Promise; 13 | } 14 | -------------------------------------------------------------------------------- /src/network-scan/domain/measurement-aggregation/MeasurementAggregationSourceId.ts: -------------------------------------------------------------------------------- 1 | import PublicKey from '../node/PublicKey'; 2 | import { NetworkId } from '../network/NetworkId'; 3 | import { OrganizationId } from '../organization/OrganizationId'; 4 | 5 | export type MeasurementAggregationSourceId = 6 | | PublicKey 7 | | NetworkId 8 | | OrganizationId; 9 | -------------------------------------------------------------------------------- /src/network-scan/domain/measurement-aggregation/MeasurementsRollupService.ts: -------------------------------------------------------------------------------- 1 | import NetworkScan from '../network/scan/NetworkScan'; 2 | 3 | export interface MeasurementsRollupService { 4 | initializeRollups(): Promise; 5 | 6 | rollupMeasurements(scan: NetworkScan): Promise; 7 | 8 | rollupNodeMeasurements(scan: NetworkScan): Promise; 9 | 10 | rollupOrganizationMeasurements(scan: NetworkScan): Promise; 11 | 12 | rollupNetworkMeasurements(scan: NetworkScan): Promise; 13 | } 14 | -------------------------------------------------------------------------------- /src/network-scan/domain/measurement/Measurement.ts: -------------------------------------------------------------------------------- 1 | export interface Measurement { 2 | time: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/domain/measurement/MeasurementRepository.ts: -------------------------------------------------------------------------------- 1 | import { Measurement } from './Measurement'; 2 | 3 | export interface MeasurementRepository { 4 | findBetween(id: string, from: Date, to: Date): Promise; 5 | findAt(id: string, at: Date): Promise; 6 | findAllAt(at: Date): Promise; 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkId.ts: -------------------------------------------------------------------------------- 1 | import { Column } from 'typeorm'; 2 | import { ValueObject } from '../../../core/domain/ValueObject'; 3 | 4 | export class NetworkId extends ValueObject { 5 | @Column({ type: 'varchar', nullable: false }) 6 | public readonly value: string; 7 | 8 | constructor(value: string) { 9 | super(); 10 | this.value = value; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkMeasurementDay.ts: -------------------------------------------------------------------------------- 1 | import { Entity } from 'typeorm'; 2 | import { NetworkMeasurementAggregation } from './NetworkMeasurementAggregation'; 3 | 4 | @Entity() 5 | export default class NetworkMeasurementDay extends NetworkMeasurementAggregation {} 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkMeasurementDayRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementAggregationRepository } from '../measurement-aggregation/MeasurementAggregationRepository'; 2 | import NetworkMeasurementDay from './NetworkMeasurementDay'; 3 | import { NetworkMeasurementAggregation } from './NetworkMeasurementAggregation'; 4 | import { NetworkId } from './NetworkId'; 5 | 6 | export interface NetworkMeasurementDayRepository 7 | extends MeasurementAggregationRepository { 8 | findBetween( 9 | id: NetworkId, 10 | from: Date, 11 | to: Date 12 | ): Promise; 13 | } 14 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkMeasurementMonth.ts: -------------------------------------------------------------------------------- 1 | import { Entity } from 'typeorm'; 2 | import { NetworkMeasurementAggregation } from './NetworkMeasurementAggregation'; 3 | 4 | @Entity() 5 | export default class NetworkMeasurementMonth extends NetworkMeasurementAggregation {} 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkMeasurementMonthRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementAggregationRepository } from '../measurement-aggregation/MeasurementAggregationRepository'; 2 | import NetworkMeasurementMonth from './NetworkMeasurementMonth'; 3 | import { NetworkId } from './NetworkId'; 4 | 5 | export interface NetworkMeasurementMonthRepository 6 | extends MeasurementAggregationRepository { 7 | findBetween( 8 | networkId: NetworkId, 9 | from: Date, 10 | to: Date 11 | ): Promise; 12 | 13 | rollup(fromCrawlId: number, toCrawlId: number): Promise; 14 | } 15 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkMeasurementRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementRepository } from '../measurement/MeasurementRepository'; 2 | import NetworkMeasurement from './NetworkMeasurement'; 3 | 4 | export interface NetworkMeasurementRepository 5 | extends MeasurementRepository { 6 | save(networkMeasurements: NetworkMeasurement[]): Promise; 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkQuorumSetConfiguration.ts: -------------------------------------------------------------------------------- 1 | import { ValueObject } from '../../../core/domain/ValueObject'; 2 | import PublicKey from '../node/PublicKey'; 3 | import { createHash } from 'crypto'; 4 | import { Type } from 'class-transformer'; 5 | 6 | export class NetworkQuorumSetConfiguration extends ValueObject { 7 | public readonly threshold: number; 8 | @Type(() => PublicKey) 9 | public readonly validators: Array; 10 | @Type(() => NetworkQuorumSetConfiguration) 11 | public readonly innerQuorumSets: Array; 12 | 13 | public constructor( 14 | threshold: number, 15 | validators: Array = [], 16 | innerQuorumSets: Array = [] 17 | ) { 18 | super(); 19 | this.threshold = threshold; 20 | this.validators = validators; 21 | this.innerQuorumSets = innerQuorumSets; 22 | } 23 | 24 | hash(): string { 25 | const hasher = createHash('sha256'); 26 | hasher.update(JSON.stringify(this)); 27 | return hasher.digest('hex'); 28 | } 29 | 30 | equals(other: this): boolean { 31 | return this.hash() === other.hash(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkQuorumSetConfigurationMapper.ts: -------------------------------------------------------------------------------- 1 | import { NetworkQuorumSetConfiguration } from './NetworkQuorumSetConfiguration'; 2 | import { QuorumSet as BasicQuorumSet } from '@stellarbeat/js-stellarbeat-shared/lib/quorum-set'; 3 | 4 | export class NetworkQuorumSetConfigurationMapper { 5 | static toBaseQuorumSet( 6 | quorumSet: NetworkQuorumSetConfiguration 7 | ): BasicQuorumSet { 8 | const crawlerQuorumSet = new BasicQuorumSet(); 9 | crawlerQuorumSet.validators = quorumSet.validators.map( 10 | (validator) => validator.value 11 | ); 12 | crawlerQuorumSet.threshold = quorumSet.threshold; 13 | crawlerQuorumSet.innerQuorumSets = quorumSet.innerQuorumSets.map( 14 | (innerQuorumSet) => this.toBaseQuorumSet(innerQuorumSet) 15 | ); 16 | 17 | return crawlerQuorumSet; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkRepository.ts: -------------------------------------------------------------------------------- 1 | import { Network } from './Network'; 2 | import { NetworkId } from './NetworkId'; 3 | 4 | export interface NetworkRepository { 5 | save(network: Network): Promise; 6 | findActiveByNetworkId(networkId: NetworkId): Promise; 7 | findAtDateByNetworkId( 8 | networkId: NetworkId, 9 | at: Date 10 | ): Promise; 11 | findPassphraseByNetworkId(networkId: NetworkId): Promise; 12 | } 13 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/NetworkTopology.ts: -------------------------------------------------------------------------------- 1 | export class NetworkTopology { 2 | //this is what we measure 3 | //contains the nodes, organizations, network transitive quorumset and such... 4 | //snapshot could contain transitive quorumset changes 5 | } 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/OverlayVersionRange.ts: -------------------------------------------------------------------------------- 1 | import { ValueObject } from '../../../core/domain/ValueObject'; 2 | import { Result, err, ok } from 'neverthrow'; 3 | import { Column } from 'typeorm'; 4 | 5 | export class OverlayVersionRange extends ValueObject { 6 | @Column({ type: 'smallint', nullable: false }) 7 | public readonly min: number; 8 | 9 | @Column({ type: 'smallint', nullable: false }) 10 | public readonly max: number; 11 | 12 | private constructor(min: number, max: number) { 13 | super(); 14 | this.max = max; 15 | this.min = min; 16 | } 17 | 18 | static create( 19 | minVersion: number, 20 | maxVersion: number 21 | ): Result { 22 | if (minVersion > maxVersion) { 23 | return err( 24 | new Error( 25 | 'Min overlay version cannot be greater or equal than max version' 26 | ) 27 | ); 28 | } 29 | 30 | return ok(new OverlayVersionRange(minVersion, maxVersion)); 31 | } 32 | 33 | equals(other: this): boolean { 34 | return this.min === other.min && this.max === other.max; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/StellarCoreVersion.ts: -------------------------------------------------------------------------------- 1 | import { ValueObject } from '../../../core/domain/ValueObject'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import valueValidator from 'validator'; 4 | import { Column } from 'typeorm'; 5 | 6 | export class StellarCoreVersion extends ValueObject { 7 | @Column({ type: 'varchar', nullable: false }) 8 | public readonly value: string; 9 | 10 | private constructor(value: string) { 11 | super(); 12 | this.value = value; 13 | } 14 | 15 | static create(value: string): Result { 16 | if (!valueValidator.isSemVer(value)) { 17 | return err(new Error('Invalid semver string')); 18 | } 19 | 20 | return ok(new StellarCoreVersion(value)); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/__fixtures__/createDummyNetworkProps.ts: -------------------------------------------------------------------------------- 1 | import { NetworkProps } from '../Network'; 2 | import { OverlayVersionRange } from '../OverlayVersionRange'; 3 | import { StellarCoreVersion } from '../StellarCoreVersion'; 4 | import { createDummyNetworkQuorumSetConfiguration } from './createDummyNetworkQuorumSetConfiguration'; 5 | 6 | export function createDummyNetworkProps(): NetworkProps { 7 | const overlayVersionRangeOrError = OverlayVersionRange.create(1, 2); 8 | if (overlayVersionRangeOrError.isErr()) 9 | throw overlayVersionRangeOrError.error; 10 | const stellarCoreVersionStringOrError = StellarCoreVersion.create('1.0.0'); 11 | if (stellarCoreVersionStringOrError.isErr()) 12 | throw stellarCoreVersionStringOrError.error; 13 | const quorumSet = createDummyNetworkQuorumSetConfiguration(); 14 | 15 | return { 16 | name: 'my test network', 17 | maxLedgerVersion: 1, 18 | overlayVersionRange: overlayVersionRangeOrError.value, 19 | stellarCoreVersion: stellarCoreVersionStringOrError.value, 20 | quorumSetConfiguration: quorumSet 21 | }; 22 | } 23 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/__fixtures__/createDummyNetworkQuorumSetConfiguration.ts: -------------------------------------------------------------------------------- 1 | import { NetworkQuorumSetConfiguration } from '../NetworkQuorumSetConfiguration'; 2 | import { createDummyPublicKey } from '../../node/__fixtures__/createDummyPublicKey'; 3 | 4 | export function createDummyNetworkQuorumSetConfiguration() { 5 | const publicKey1 = createDummyPublicKey(); 6 | const publicKey2 = createDummyPublicKey(); 7 | 8 | const innerQuorumSet = new NetworkQuorumSetConfiguration( 9 | 1, 10 | [publicKey1, publicKey2], 11 | [] 12 | ); 13 | return new NetworkQuorumSetConfiguration( 14 | 2, 15 | [publicKey1, publicKey2], 16 | [innerQuorumSet] 17 | ); 18 | } 19 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/__tests__/OverlayVersionRange.test.ts: -------------------------------------------------------------------------------- 1 | import { OverlayVersionRange } from '../OverlayVersionRange'; 2 | 3 | it('should equal other OverlayVersionRange', function () { 4 | const overlayVersionRangeOrError = OverlayVersionRange.create(1, 2); 5 | if (overlayVersionRangeOrError.isErr()) 6 | throw overlayVersionRangeOrError.error; 7 | const otherOverlayVersionRangeOrError = OverlayVersionRange.create(1, 2); 8 | if (otherOverlayVersionRangeOrError.isErr()) 9 | throw otherOverlayVersionRangeOrError.error; 10 | expect( 11 | overlayVersionRangeOrError.value.equals( 12 | otherOverlayVersionRangeOrError.value 13 | ) 14 | ).toBe(true); 15 | }); 16 | 17 | it('should not create OverlayVersionRange with minVersion greater than maxVersion', function () { 18 | const overlayVersionRangeOrError = OverlayVersionRange.create(2, 1); 19 | expect(overlayVersionRangeOrError.isErr()).toBe(true); 20 | }); 21 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/__tests__/QuorumSetMapper.test.ts: -------------------------------------------------------------------------------- 1 | import { createDummyPublicKey } from '../../node/__fixtures__/createDummyPublicKey'; 2 | import { NetworkQuorumSetConfiguration } from '../NetworkQuorumSetConfiguration'; 3 | import { NetworkQuorumSetConfigurationMapper } from '../NetworkQuorumSetConfigurationMapper'; 4 | 5 | it('should map to Basic QuorumSet', function () { 6 | const a = createDummyPublicKey(); 7 | const b = createDummyPublicKey(); 8 | const c = createDummyPublicKey(); 9 | const quorumSet = new NetworkQuorumSetConfiguration( 10 | 2, 11 | [a, b], 12 | [new NetworkQuorumSetConfiguration(1, [c], [])] 13 | ); 14 | 15 | const crawlerQuorumSet = 16 | NetworkQuorumSetConfigurationMapper.toBaseQuorumSet(quorumSet); 17 | expect(crawlerQuorumSet.threshold).toEqual(2); 18 | expect(crawlerQuorumSet.validators).toEqual([a.value, b.value]); 19 | expect(crawlerQuorumSet.innerQuorumSets).toHaveLength(1); 20 | expect(crawlerQuorumSet.innerQuorumSets[0].threshold).toEqual(1); 21 | expect(crawlerQuorumSet.innerQuorumSets[0].validators).toEqual([c.value]); 22 | }); 23 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/__tests__/StellarCoreVersion.test.ts: -------------------------------------------------------------------------------- 1 | import { StellarCoreVersion } from '../StellarCoreVersion'; 2 | it('should only create valid version strings', function () { 3 | const versionStringOrError = StellarCoreVersion.create('1.2.3'); 4 | expect(versionStringOrError.isErr()).toBe(false); 5 | const invalidVersionStringOrError = StellarCoreVersion.create('v1.2.3'); 6 | expect(invalidVersionStringOrError.isErr()).toBe(true); 7 | const longerVersionStringOrError = StellarCoreVersion.create('12.123.4322'); 8 | expect(longerVersionStringOrError.isErr()).toBe(false); 9 | const versionStringWithLeadingZeroOrError = 10 | StellarCoreVersion.create('01.2.3'); 11 | expect(versionStringWithLeadingZeroOrError.isErr()).toBe(true); 12 | }); 13 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/change/NetworkChange.ts: -------------------------------------------------------------------------------- 1 | import { Column, Entity, ManyToOne, TableInheritance } from 'typeorm'; 2 | import { CoreEntity } from '../../../../core/domain/CoreEntity'; 3 | import { Change } from '../../Change'; 4 | import { NetworkId } from '../NetworkId'; 5 | import { Network } from '../Network'; 6 | 7 | @Entity() 8 | @TableInheritance({ column: { type: 'varchar', name: 'type' } }) 9 | export abstract class NetworkChange extends CoreEntity implements Change { 10 | @ManyToOne(() => Network, { 11 | nullable: false 12 | }) 13 | public network?: Network; 14 | 15 | @Column('timestamptz') 16 | time: Date; 17 | 18 | @Column({ type: 'jsonb', nullable: false }) 19 | from: Record; 20 | 21 | @Column({ type: 'jsonb', nullable: false }) 22 | to: Record; 23 | 24 | @Column(() => NetworkId) 25 | public readonly networkId: NetworkId; 26 | 27 | protected constructor( 28 | networkId: NetworkId, 29 | time: Date, 30 | from: Record, 31 | to: Record 32 | ) { 33 | super(); 34 | this.networkId = networkId; 35 | this.time = time; 36 | this.from = from; 37 | this.to = to; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/change/NetworkMaxLedgerVersionChanged.ts: -------------------------------------------------------------------------------- 1 | import { ChildEntity } from 'typeorm'; 2 | import { NetworkChange } from './NetworkChange'; 3 | import { NetworkId } from '../NetworkId'; 4 | 5 | @ChildEntity() 6 | export class NetworkMaxLedgerVersionChanged extends NetworkChange { 7 | constructor(networkId: NetworkId, time: Date, from: number, to: number) { 8 | super( 9 | networkId, 10 | time, 11 | { 12 | value: from 13 | }, 14 | { 15 | value: to 16 | } 17 | ); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/change/NetworkNameChanged.ts: -------------------------------------------------------------------------------- 1 | import { ChildEntity } from 'typeorm'; 2 | import { NetworkId } from '../NetworkId'; 3 | import { NetworkChange } from './NetworkChange'; 4 | 5 | @ChildEntity() 6 | export class NetworkNameChanged extends NetworkChange { 7 | constructor(networkId: NetworkId, time: Date, from: string, to: string) { 8 | super( 9 | networkId, 10 | time, 11 | { 12 | value: from 13 | }, 14 | { 15 | value: to 16 | } 17 | ); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/change/NetworkOverlayVersionRangeChanged.ts: -------------------------------------------------------------------------------- 1 | import { ChildEntity } from 'typeorm'; 2 | import { NetworkChange } from './NetworkChange'; 3 | import { NetworkId } from '../NetworkId'; 4 | import { OverlayVersionRange } from '../OverlayVersionRange'; 5 | 6 | @ChildEntity() 7 | export class NetworkOverlayVersionRangeChanged extends NetworkChange { 8 | constructor( 9 | networkId: NetworkId, 10 | time: Date, 11 | from: OverlayVersionRange, 12 | to: OverlayVersionRange 13 | ) { 14 | super( 15 | networkId, 16 | time, 17 | { 18 | value: from 19 | }, 20 | { 21 | value: to 22 | } 23 | ); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/change/NetworkQuorumSetConfigurationChanged.ts: -------------------------------------------------------------------------------- 1 | import { ChildEntity } from 'typeorm'; 2 | import { NetworkChange } from './NetworkChange'; 3 | import { NetworkId } from '../NetworkId'; 4 | import { NetworkQuorumSetConfiguration } from '../NetworkQuorumSetConfiguration'; 5 | 6 | @ChildEntity() 7 | export class NetworkQuorumSetConfigurationChanged extends NetworkChange { 8 | constructor( 9 | networkId: NetworkId, 10 | time: Date, 11 | from: NetworkQuorumSetConfiguration, 12 | to: NetworkQuorumSetConfiguration 13 | ) { 14 | super( 15 | networkId, 16 | time, 17 | { 18 | value: from 19 | }, 20 | { 21 | value: to 22 | } 23 | ); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/change/NetworkStellarCoreVersionChanged.ts: -------------------------------------------------------------------------------- 1 | import { ChildEntity } from 'typeorm'; 2 | import { NetworkChange } from './NetworkChange'; 3 | import { NetworkId } from '../NetworkId'; 4 | import { StellarCoreVersion } from '../StellarCoreVersion'; 5 | 6 | @ChildEntity() 7 | export class NetworkStellarCoreVersionChanged extends NetworkChange { 8 | constructor( 9 | networkId: NetworkId, 10 | time: Date, 11 | from: StellarCoreVersion, 12 | to: StellarCoreVersion 13 | ) { 14 | super( 15 | networkId, 16 | time, 17 | { 18 | value: from 19 | }, 20 | { 21 | value: to 22 | } 23 | ); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/NetworkScanRepository.ts: -------------------------------------------------------------------------------- 1 | import NetworkScan from './NetworkScan'; 2 | 3 | export interface NetworkScanRepository { 4 | findLatestSuccessfulScanTime(): Promise; 5 | 6 | findLatest(): Promise; 7 | 8 | findAt(at: Date): Promise; 9 | 10 | findPreviousAt(at: Date): Promise; 11 | 12 | saveOne(scan: NetworkScan): Promise; 13 | 14 | save(scans: NetworkScan[]): Promise; 15 | } 16 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/TomlVersionChecker.ts: -------------------------------------------------------------------------------- 1 | import valueValidator from 'validator'; 2 | import { isString } from '../../../../core/utilities/TypeGuards'; 3 | import semver = require('semver/preload'); 4 | 5 | export class TomlVersionChecker { 6 | static isSupportedVersion( 7 | tomlObject: Record, 8 | lowestSupportedVersion: string = '2.0.0' 9 | ): boolean { 10 | if (!valueValidator.isSemVer(lowestSupportedVersion)) return false; 11 | if (!isString(tomlObject.VERSION)) return false; 12 | if (!valueValidator.isSemVer(tomlObject.VERSION)) return false; 13 | 14 | return !semver.lt(tomlObject.VERSION, lowestSupportedVersion); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/__tests__/NodesInTransitiveNetworkQuorumSetFinder.test.ts: -------------------------------------------------------------------------------- 1 | import { createDummyNode } from '../../../node/__fixtures__/createDummyNode'; 2 | import { NodesInTransitiveNetworkQuorumSetFinder } from '../NodesInTransitiveNetworkQuorumSetFinder'; 3 | import { NetworkQuorumSetConfiguration } from '../../NetworkQuorumSetConfiguration'; 4 | 5 | describe('NodesInTransitiveNetworkQuorumSetFinder', () => { 6 | test('should find nodes in transitive network quorum set', () => { 7 | const nodeInTransitiveQuorumSet = createDummyNode(); 8 | const nodeNotInTransitiveQuorumSet = createDummyNode(); 9 | 10 | const nodes = [nodeInTransitiveQuorumSet, nodeNotInTransitiveQuorumSet]; 11 | 12 | const finder = new NodesInTransitiveNetworkQuorumSetFinder(); 13 | const quorumSet = new NetworkQuorumSetConfiguration( 14 | 1, 15 | [nodeInTransitiveQuorumSet.publicKey], 16 | [] 17 | ); 18 | const nodesInTransitiveQuorumSet = finder.find(nodes, quorumSet); 19 | 20 | expect(nodesInTransitiveQuorumSet).toEqual([nodeInTransitiveQuorumSet]); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/archiver/Archiver.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | 3 | export interface Archiver { 4 | archive(time: Date): Promise>; 5 | } 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/fbas-analysis/AnalysisMergedResult.ts: -------------------------------------------------------------------------------- 1 | export interface AnalysisMergedResult { 2 | blockingSetsMinSize: number; 3 | blockingSetsFilteredMinSize: number; 4 | splittingSetsMinSize: number; 5 | topTierSize: number; 6 | } 7 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/fbas-analysis/AnalysisResult.ts: -------------------------------------------------------------------------------- 1 | import { AnalysisMergedResult } from './AnalysisMergedResult'; 2 | 3 | export interface AnalysisResult { 4 | hasQuorumIntersection: boolean; 5 | hasSymmetricTopTier: boolean; 6 | node: AnalysisMergedResult; 7 | organization: AnalysisMergedResult; 8 | isp: AnalysisMergedResult; 9 | country: AnalysisMergedResult; 10 | } 11 | -------------------------------------------------------------------------------- /src/network-scan/domain/network/scan/fbas-analysis/FbasMapper.ts: -------------------------------------------------------------------------------- 1 | import Node from '../../../node/Node'; 2 | import Organization from '../../../organization/Organization'; 3 | import { 4 | FbasAnalysisNode, 5 | FbasAnalysisOrganization 6 | } from './FbasAnalyzerFacade'; 7 | 8 | export class FbasMapper { 9 | static mapToFbasAnalysisNode(node: Node): FbasAnalysisNode { 10 | return { 11 | publicKey: node.publicKey.value, 12 | name: node.details?.name ?? null, 13 | quorumSet: node.quorumSet?.quorumSet ?? null, 14 | geoData: { 15 | countryName: node.geoData?.countryName ?? null 16 | }, 17 | isp: node.isp 18 | }; 19 | } 20 | 21 | static mapToFbasAnalysisOrganization( 22 | organization: Organization 23 | ): FbasAnalysisOrganization { 24 | return { 25 | id: organization.organizationId.value, 26 | validators: organization.validators.value.map( 27 | (validator) => validator.value 28 | ), 29 | name: organization.name 30 | }; 31 | } 32 | 33 | static mapToAnalysisResult() {} 34 | } 35 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeAddress.ts: -------------------------------------------------------------------------------- 1 | import { ValueObject } from '../../../core/domain/ValueObject'; 2 | import { Result, ok, err } from 'neverthrow'; 3 | import validator from 'validator'; 4 | 5 | export class NodeAddress extends ValueObject { 6 | private constructor( 7 | public readonly ip: string, 8 | public readonly port: number 9 | ) { 10 | super(); 11 | } 12 | 13 | static create(ip: string, port: number): Result { 14 | if (!validator.isIP(ip)) return err(new Error('Invalid IP ' + ip)); 15 | if (!validator.isPort(port.toString())) 16 | return err(new Error('Invalid port ' + port)); 17 | 18 | return ok(new NodeAddress(ip, port)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeMeasurementAverage.ts: -------------------------------------------------------------------------------- 1 | export interface NodeMeasurementAverage { 2 | publicKey: string; 3 | activeAvg: number; 4 | validatingAvg: number; 5 | fullValidatorAvg: number; 6 | overLoadedAvg: number; 7 | indexAvg: number; 8 | historyArchiveErrorAvg: number; 9 | } 10 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeMeasurementDayRepository.ts: -------------------------------------------------------------------------------- 1 | import { NodeMeasurementAverage } from './NodeMeasurementAverage'; 2 | import { MeasurementAggregationRepository } from '../measurement-aggregation/MeasurementAggregationRepository'; 3 | import NodeMeasurementDay from './NodeMeasurementDay'; 4 | import PublicKey from './PublicKey'; 5 | 6 | export interface NodeMeasurementDayRepository 7 | extends MeasurementAggregationRepository { 8 | findXDaysAverageAt( 9 | at: Date, 10 | xDays: number 11 | ): Promise; 12 | 13 | findBetween( 14 | publicKey: PublicKey, 15 | from: Date, 16 | to: Date 17 | ): Promise; 18 | 19 | findXDaysInactive( 20 | since: Date, 21 | numberOfDays: number 22 | ): Promise<{ publicKey: string }[]>; 23 | 24 | findXDaysActiveButNotValidating( 25 | since: Date, 26 | numberOfDays: number 27 | ): Promise<{ publicKey: string }[]>; 28 | save(nodeMeasurements: NodeMeasurementDay[]): Promise; 29 | } 30 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeMeasurementEvent.ts: -------------------------------------------------------------------------------- 1 | export interface NodeMeasurementEvent { 2 | time: string; 3 | publicKey: string; 4 | notValidating: boolean; 5 | inactive: boolean; 6 | historyOutOfDate: boolean; 7 | connectivityIssues: boolean; 8 | stellarCoreVersionBehindIssue: boolean; 9 | } 10 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeMeasurementRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementRepository } from '../measurement/MeasurementRepository'; 2 | import NodeMeasurement from './NodeMeasurement'; 3 | import { NodeMeasurementAverage } from './NodeMeasurementAverage'; 4 | import { NodeMeasurementEvent } from './NodeMeasurementEvent'; 5 | 6 | export interface NodeMeasurementRepository 7 | extends MeasurementRepository { 8 | findXDaysAverageAt( 9 | at: Date, 10 | xDays: number 11 | ): Promise; 12 | findEventsForXNetworkScans( 13 | x: number, 14 | at: Date 15 | ): Promise; 16 | save(nodeMeasurements: NodeMeasurement[]): Promise; 17 | findInactiveAt(at: Date): Promise<{ nodeId: number }[]>; 18 | } 19 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeQuorumSet.ts: -------------------------------------------------------------------------------- 1 | import { Entity, Column, Index, ValueTransformer } from 'typeorm'; 2 | import { QuorumSet as QuorumSetDTO } from '@stellarbeat/js-stellarbeat-shared'; 3 | import { IdentifiedValueObject } from '../../../core/domain/IdentifiedValueObject'; 4 | 5 | export const quorumSetTransformer: ValueTransformer = { 6 | from: (dbValue) => { 7 | if (dbValue === null) return null; 8 | return QuorumSetDTO.fromBaseQuorumSet(JSON.parse(dbValue)); 9 | }, 10 | to: (entityValue) => JSON.stringify(entityValue) 11 | }; 12 | 13 | /** 14 | * A quorumSet can be reused between nodes. 15 | */ 16 | @Entity('node_quorum_set') 17 | export default class NodeQuorumSet extends IdentifiedValueObject { 18 | @Index() 19 | @Column('varchar', { length: 64 }) 20 | hash: string; 21 | 22 | @Column('jsonb', { 23 | transformer: quorumSetTransformer 24 | }) 25 | quorumSet: QuorumSetDTO; 26 | 27 | private constructor(hash: string, quorumSet: QuorumSetDTO) { 28 | super(); 29 | this.hash = hash; 30 | this.quorumSet = quorumSet; 31 | } 32 | 33 | static create(hash: string, quorumSet: QuorumSetDTO): NodeQuorumSet { 34 | return new this(hash, quorumSet); 35 | } 36 | 37 | equals(other: this): boolean { 38 | return other.hash === this.hash; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeRepository.ts: -------------------------------------------------------------------------------- 1 | import PublicKey from './PublicKey'; 2 | import Node from './Node'; 3 | 4 | //active means that the node is not archived. i.e. snapshot endDate = SNAPSHOT_MAX_END_DATE 5 | export interface NodeRepository { 6 | save(nodes: Node[], from: Date): Promise; 7 | findActiveAtTimePoint(at: Date): Promise; 8 | findActive(): Promise; 9 | findActiveByPublicKey(publicKeys: string[]): Promise; 10 | findActiveByPublicKeyAtTimePoint( 11 | publicKey: PublicKey, 12 | at: Date 13 | ): Promise; 14 | findByPublicKey(publicKeys: PublicKey[]): Promise; //active or not 15 | findOneByPublicKey(publicKey: PublicKey): Promise; //active or not 16 | } 17 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/NodeSnapShotRepository.ts: -------------------------------------------------------------------------------- 1 | import NodeSnapShot from './NodeSnapShot'; 2 | import PublicKey from './PublicKey'; 3 | 4 | //todo: this repo should be removed when we start storing changes 5 | export interface NodeSnapShotRepository { 6 | //todo: NodeId should not be used in domain 7 | findActiveByNodeId(nodeIds: number[]): Promise; 8 | 9 | archiveInActiveWithMultipleIpSamePort(time: Date): Promise; 10 | 11 | findLatest(at: Date): Promise; 12 | findLatestByPublicKey( 13 | publicKey: PublicKey, 14 | at: Date 15 | ): Promise; 16 | 17 | save(nodeSnapShots: NodeSnapShot[]): Promise; 18 | } 19 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/PublicKey.ts: -------------------------------------------------------------------------------- 1 | import { Column, Index } from 'typeorm'; 2 | 3 | import { err, ok, Result } from 'neverthrow'; 4 | import { ValueObject } from '../../../core/domain/ValueObject'; 5 | 6 | export default class PublicKey extends ValueObject { 7 | @Column('varchar', { length: 56 }) 8 | @Index({ unique: true }) 9 | value: string; 10 | 11 | private constructor(publicKey: string) { 12 | super(); 13 | this.value = publicKey; 14 | } 15 | 16 | static create(publicKey: string): Result { 17 | if (publicKey.length !== 56) { 18 | return err(new Error('Invalid public key length')); 19 | } 20 | if (!publicKey.startsWith('G')) { 21 | return err(new Error('Invalid public key prefix')); 22 | } 23 | return ok(new PublicKey(publicKey)); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/__fixtures__/createDummyNode.ts: -------------------------------------------------------------------------------- 1 | import { createDummyPublicKey } from './createDummyPublicKey'; 2 | import Node from '../Node'; 3 | 4 | export function createDummyNode( 5 | ip?: string, 6 | port?: number, 7 | time = new Date() 8 | ): Node { 9 | return Node.create(time, createDummyPublicKey(), { 10 | ip: ip ?? 'localhost', 11 | port: port ?? 3000 12 | }); 13 | } 14 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/__fixtures__/createDummyNodeAddress.ts: -------------------------------------------------------------------------------- 1 | import { NodeAddress } from '../NodeAddress'; 2 | 3 | let counter = 0; 4 | 5 | export function createDummyNodeAddress() { 6 | const nodeAddress = NodeAddress.create('127.0.0.1', 11625 + counter++); 7 | if (nodeAddress.isErr()) { 8 | throw new Error('Invalid node address'); 9 | } 10 | return nodeAddress.value; 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/__fixtures__/createDummyPublicKey.ts: -------------------------------------------------------------------------------- 1 | import PublicKey from '../PublicKey'; 2 | 3 | export function createDummyPublicKey() { 4 | const publicKeyOrError = PublicKey.create(createDummyPublicKeyString()); 5 | if (publicKeyOrError.isErr()) throw publicKeyOrError.error; 6 | return publicKeyOrError.value; 7 | } 8 | 9 | export function createDummyPublicKeyString() { 10 | let publicKeyString = 'G'; 11 | const allowedChars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; 12 | for (let i = 0; i < 55; i++) { 13 | publicKeyString += allowedChars.charAt( 14 | Math.floor(Math.random() * allowedChars.length) 15 | ); 16 | } 17 | 18 | return publicKeyString; 19 | } 20 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/__tests__/NodeQuorumSet.test.ts: -------------------------------------------------------------------------------- 1 | import NodeQuorumSet from '../NodeQuorumSet'; 2 | import { QuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 3 | 4 | describe('NodeQuorumSet', () => { 5 | it('should equal another NodeQuorumSet when the hashkey is the same', function () { 6 | const quorumSet = NodeQuorumSet.create( 7 | 'hashkey', 8 | new QuorumSet(1, ['a', 'b'], []) 9 | ); 10 | const otherQuorumSet = NodeQuorumSet.create( 11 | 'hashkey', 12 | new QuorumSet(1, ['a', 'b'], []) 13 | ); 14 | expect(quorumSet.equals(otherQuorumSet)).toBe(true); 15 | }); 16 | it('should not equal when the hashkeys are different', function () { 17 | const quorumSet = NodeQuorumSet.create( 18 | 'hashkey', 19 | new QuorumSet(1, ['a', 'b'], []) 20 | ); 21 | const otherQuorumSet = NodeQuorumSet.create( 22 | 'hashkey2', 23 | new QuorumSet(1, ['a', 'b'], []) 24 | ); 25 | expect(quorumSet.equals(otherQuorumSet)).toBe(false); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/archival/hasNoActiveTrustingNodes.ts: -------------------------------------------------------------------------------- 1 | import { TrustGraph } from '@stellarbeat/js-stellarbeat-shared'; 2 | import Node from '../Node'; 3 | 4 | export function hasNoActiveTrustingNodes( 5 | node: Node, 6 | inactiveNodes: string[], 7 | nodesTrustGraph: TrustGraph 8 | ): boolean { 9 | const vertex = nodesTrustGraph.getVertex(node.publicKey.value); 10 | if (!vertex) { 11 | return true; //no trust links, e.g. watcher 12 | } 13 | const trustingNodes = nodesTrustGraph.getParents(vertex); 14 | 15 | const activeTrustingNodes = Array.from(trustingNodes).filter( 16 | (trustingNode) => !inactiveNodes.includes(trustingNode.key) 17 | ); 18 | 19 | return activeTrustingNodes.length === 0; 20 | } 21 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/GeoDataService.ts: -------------------------------------------------------------------------------- 1 | import 'reflect-metadata'; 2 | import { Result } from 'neverthrow'; 3 | import { CustomError } from '../../../../core/errors/CustomError'; 4 | 5 | export interface GeoData { 6 | longitude: number | null; 7 | latitude: number | null; 8 | countryCode: string | null; 9 | countryName: string | null; 10 | isp: string | null; 11 | } 12 | 13 | export class GeoDataUpdateError extends CustomError { 14 | constructor(publicKey: string, cause?: Error) { 15 | super( 16 | 'Failed updating geoData for ' + publicKey, 17 | GeoDataUpdateError.name, 18 | cause 19 | ); 20 | } 21 | } 22 | 23 | export interface GeoDataService { 24 | fetchGeoData(ip: string): Promise>; 25 | } 26 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/MoreThanOneDayApart.ts: -------------------------------------------------------------------------------- 1 | export default (timeA: Date, timeB: Date): boolean => { 2 | return Math.abs(timeA.getTime() - timeB.getTime()) > 1000 * 60 * 60 * 24; 3 | }; 4 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/NodeScannerArchivalStep.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import { NodeScan } from './NodeScan'; 3 | import { ValidatorDemoter } from '../archival/ValidatorDemoter'; 4 | import { InactiveNodesArchiver } from '../archival/InactiveNodesArchiver'; 5 | import { TrustGraphFactory } from './TrustGraphFactory'; 6 | 7 | @injectable() 8 | export class NodeScannerArchivalStep { 9 | constructor( 10 | private validatorDemoter: ValidatorDemoter, 11 | private inactiveNodesArchiver: InactiveNodesArchiver 12 | ) {} 13 | 14 | public async execute(nodeScan: NodeScan): Promise { 15 | const trustGraph = TrustGraphFactory.create(nodeScan.nodes); 16 | await this.validatorDemoter.demote(nodeScan, trustGraph, 2); 17 | await this.inactiveNodesArchiver.archive(nodeScan, trustGraph, 2); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/NodeScannerHistoryArchiveStep.ts: -------------------------------------------------------------------------------- 1 | import { HistoryArchiveStatusFinder } from './HistoryArchiveStatusFinder'; 2 | import { injectable } from 'inversify'; 3 | import { NodeScan } from './NodeScan'; 4 | 5 | @injectable() 6 | export class NodeScannerHistoryArchiveStep { 7 | constructor(private historyArchiveStatusFinder: HistoryArchiveStatusFinder) {} 8 | 9 | public async execute(nodeScan: NodeScan): Promise { 10 | nodeScan.updateHistoryArchiveUpToDateStatus( 11 | await this.historyArchiveStatusFinder.getNodesWithUpToDateHistoryArchives( 12 | nodeScan.getHistoryArchiveUrls(), 13 | nodeScan.latestLedger 14 | ) 15 | ); 16 | nodeScan.updateHistoryArchiveVerificationStatus( 17 | await this.historyArchiveStatusFinder.getNodesWithHistoryArchiveVerificationErrors( 18 | nodeScan.getHistoryArchiveUrls() 19 | ) 20 | ); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/NodeScannerHomeDomainStep.ts: -------------------------------------------------------------------------------- 1 | import { HomeDomainFetcher } from './HomeDomainFetcher'; 2 | import { injectable } from 'inversify'; 3 | import { NodeScan } from './NodeScan'; 4 | 5 | @injectable() 6 | export class NodeScannerHomeDomainStep { 7 | constructor(private homeDomainFetcher: HomeDomainFetcher) {} 8 | 9 | public async execute(nodeScan: NodeScan): Promise { 10 | nodeScan.updateHomeDomains( 11 | await this.homeDomainFetcher.fetchHomeDomains(nodeScan.getPublicKeys()) 12 | ); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/NodeScannerIndexerStep.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import { StellarCoreVersion } from '../../network/StellarCoreVersion'; 3 | import { NodeMeasurementAverage } from '../NodeMeasurementAverage'; 4 | import { NodeScan } from './NodeScan'; 5 | import { NodeIndexer } from './NodeIndexer'; 6 | import 'reflect-metadata'; 7 | 8 | @injectable() 9 | export class NodeScannerIndexerStep { 10 | public execute( 11 | nodeScan: NodeScan, 12 | measurement30DayAverages: NodeMeasurementAverage[], 13 | stellarCoreVersion: StellarCoreVersion 14 | ): void { 15 | nodeScan.updateIndexes( 16 | NodeIndexer.calculateIndexes( 17 | nodeScan.nodes, 18 | measurement30DayAverages, 19 | stellarCoreVersion 20 | ) 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/NodeScannerTomlStep.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import { NodeScan } from './NodeScan'; 3 | import { NodeTomlFetcher } from './NodeTomlFetcher'; 4 | 5 | @injectable() 6 | export class NodeScannerTomlStep { 7 | constructor(private nodeTomlFetcher: NodeTomlFetcher) {} 8 | 9 | public async execute(nodeScan: NodeScan): Promise { 10 | nodeScan.updateWithTomlInfo( 11 | await this.nodeTomlFetcher.fetchNodeTomlInfoCollection( 12 | nodeScan.getHomeDomains() 13 | ) 14 | ); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/NodeTomlInfo.ts: -------------------------------------------------------------------------------- 1 | export interface NodeTomlInfo { 2 | homeDomain: string; 3 | publicKey: string; 4 | historyUrl: string | null; 5 | alias: string | null; 6 | name: string | null; 7 | host: string | null; 8 | } 9 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/__tests__/HomeDomainFetcher.test.ts: -------------------------------------------------------------------------------- 1 | import { HomeDomainFetcher } from '../HomeDomainFetcher'; 2 | import { HorizonService } from '../../../network/scan/HorizonService'; 3 | import { ok } from 'neverthrow'; 4 | import { LoggerMock } from '../../../../../core/services/__mocks__/LoggerMock'; 5 | import { HttpService } from '../../../../../core/services/HttpService'; 6 | 7 | it('should update homeDomains once in a cache period', async function () { 8 | const horizonService = new HorizonService({} as HttpService, { 9 | value: 'url' 10 | }); 11 | jest 12 | .spyOn(horizonService, 'fetchAccount') 13 | .mockResolvedValue(ok({ home_domain: 'myDomain.be' })); 14 | 15 | const homeDomainFetcher = new HomeDomainFetcher( 16 | horizonService, 17 | new LoggerMock() 18 | ); 19 | const domains = await homeDomainFetcher.fetchHomeDomains(['A']); 20 | expect(domains.get('A')).toEqual('myDomain.be'); 21 | jest 22 | .spyOn(horizonService, 'fetchAccount') 23 | .mockResolvedValue(ok({ home_domain: 'myOtherDomain.be' })); 24 | await homeDomainFetcher.fetchHomeDomains(['A']); 25 | expect(domains.get('A')).toEqual('myDomain.be'); 26 | }); 27 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/__tests__/MoreThanOneDayApart.test.ts: -------------------------------------------------------------------------------- 1 | import olderThanOneDay from '../MoreThanOneDayApart'; 2 | 3 | test('older', () => { 4 | const myDate = new Date(1999, 1, 1); 5 | const olderDate = new Date(1998, 1, 1); 6 | expect(olderThanOneDay(myDate, olderDate)).toBeTruthy(); 7 | expect(olderThanOneDay(olderDate, myDate)).toBeTruthy(); 8 | }); 9 | 10 | test('not older', () => { 11 | const timeA = new Date(1999, 1, 1); 12 | const timeB = new Date(1999, 1, 1, 1); 13 | expect(olderThanOneDay(timeA, timeB)).toBeFalsy(); 14 | expect(olderThanOneDay(timeB, timeB)).toBeFalsy(); 15 | }); 16 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/__tests__/NodeScannerArchivalStep.test.ts: -------------------------------------------------------------------------------- 1 | import { NodeScannerHistoryArchiveStep } from '../NodeScannerHistoryArchiveStep'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { NodeScan } from '../NodeScan'; 4 | import { NodeScannerArchivalStep } from '../NodeScannerArchivalStep'; 5 | import { InactiveNodesArchiver } from '../../archival/InactiveNodesArchiver'; 6 | import { ValidatorDemoter } from '../../archival/ValidatorDemoter'; 7 | 8 | describe('NodeScannerHistoryArchiveStep', () => { 9 | const inactiveNodesArchiver = mock(); 10 | const validatorDemoter = mock(); 11 | const nodeScannerArchivalStep = new NodeScannerArchivalStep( 12 | validatorDemoter, 13 | inactiveNodesArchiver 14 | ); 15 | 16 | beforeEach(() => { 17 | jest.clearAllMocks(); 18 | }); 19 | 20 | it('should archive', async () => { 21 | const nodeScan = mock(); 22 | await nodeScannerArchivalStep.execute(nodeScan); 23 | expect(validatorDemoter.demote).toBeCalledTimes(1); 24 | expect(inactiveNodesArchiver.archive).toBeCalledTimes(1); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/__tests__/NodeScannerHomeDomainStep.test.ts: -------------------------------------------------------------------------------- 1 | import { NodeScannerHomeDomainStep } from '../NodeScannerHomeDomainStep'; 2 | import { HomeDomainFetcher } from '../HomeDomainFetcher'; 3 | import { mock } from 'jest-mock-extended'; 4 | import { NodeScan } from '../NodeScan'; 5 | 6 | describe('NodeScannerHomeDomainStep', () => { 7 | const fetcher = mock(); 8 | const homeDomainStep = new NodeScannerHomeDomainStep(fetcher); 9 | 10 | beforeEach(() => { 11 | jest.clearAllMocks(); 12 | }); 13 | 14 | it('should update home domains', async function () { 15 | const nodeScan = mock(); 16 | const homeDomains = new Map(); 17 | fetcher.fetchHomeDomains.mockResolvedValue(homeDomains); 18 | await homeDomainStep.execute(nodeScan); 19 | expect(fetcher.fetchHomeDomains).toBeCalled(); 20 | expect(nodeScan.updateHomeDomains).toBeCalledWith(homeDomains); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/__tests__/NodeScannerIndexerStep.test.ts: -------------------------------------------------------------------------------- 1 | import { NodeScannerIndexerStep } from '../NodeScannerIndexerStep'; 2 | import { NodeScan } from '../NodeScan'; 3 | import { createDummyNode } from '../../__fixtures__/createDummyNode'; 4 | import { StellarCoreVersion } from '../../../network/StellarCoreVersion'; 5 | import 'reflect-metadata'; 6 | import NodeMeasurement from '../../NodeMeasurement'; 7 | 8 | describe('NodeScannerIndexerStep', () => { 9 | const step = new NodeScannerIndexerStep(); 10 | const stellarCoreVersion = StellarCoreVersion.create('13.0.0'); 11 | if (stellarCoreVersion.isErr()) throw new Error('stellarCoreVersion is Err'); 12 | beforeEach(() => { 13 | jest.clearAllMocks(); 14 | }); 15 | 16 | it('should update with indexer info', function () { 17 | const nodeScan = new NodeScan(new Date(), [createDummyNode()]); 18 | nodeScan.nodes[0].addMeasurement( 19 | new NodeMeasurement(new Date(), nodeScan.nodes[0]) 20 | ); 21 | step.execute(nodeScan, [], stellarCoreVersion.value); 22 | expect(nodeScan.nodes[0].latestMeasurement()?.index).toBeGreaterThan(0); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/__tests__/NodeScannerTomlStep.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { NodeScannerTomlStep } from '../NodeScannerTomlStep'; 3 | import { NodeScan } from '../NodeScan'; 4 | import { NodeTomlInfo } from '../NodeTomlInfo'; 5 | import { NodeTomlFetcher } from '../NodeTomlFetcher'; 6 | 7 | describe('NodeScannerTomlStep', () => { 8 | const nodeTomlFetcher = mock(); 9 | const step = new NodeScannerTomlStep(nodeTomlFetcher); 10 | 11 | beforeEach(() => { 12 | jest.clearAllMocks(); 13 | }); 14 | 15 | it('should update with toml info', async function () { 16 | const nodeScan = mock(); 17 | const tomlInfo = new Set(); 18 | nodeTomlFetcher.fetchNodeTomlInfoCollection.mockResolvedValue(tomlInfo); 19 | await step.execute(nodeScan); 20 | expect(nodeTomlFetcher.fetchNodeTomlInfoCollection).toBeCalled(); 21 | expect(nodeScan.updateWithTomlInfo).toBeCalledWith(tomlInfo); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/history/HistoryArchiveScanService.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | import { HistoryArchiveScan } from '@stellarbeat/js-stellarbeat-shared'; 3 | 4 | export interface HistoryArchiveScanService { 5 | findLatestScans(): Promise>; 6 | } 7 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-crawl/NodeSorter.ts: -------------------------------------------------------------------------------- 1 | import Node from '../../Node'; 2 | import { NetworkQuorumSetConfiguration } from '../../../network/NetworkQuorumSetConfiguration'; 3 | import PublicKey from '../../PublicKey'; 4 | 5 | export class NodeSorter { 6 | static sortByNetworkQuorumSetInclusion( 7 | nodes: Node[], 8 | networkQuorumSet: NetworkQuorumSetConfiguration 9 | ): Node[] { 10 | const publicKeys = NodeSorter.getAllValidators(networkQuorumSet); 11 | return nodes.sort((a) => { 12 | if (publicKeys.find((publicKey) => publicKey.equals(a.publicKey))) 13 | return -1; 14 | return 0; 15 | }); 16 | } 17 | 18 | private static getAllValidators( 19 | quorumSet: NetworkQuorumSetConfiguration 20 | ): PublicKey[] { 21 | return quorumSet.innerQuorumSets 22 | .reduce( 23 | (allValidators, innerQS) => 24 | allValidators.concat(this.getAllValidators(innerQS)), 25 | quorumSet.validators 26 | ) 27 | .map((publicKey) => publicKey); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-crawl/__tests__/NodeSorter.test.ts: -------------------------------------------------------------------------------- 1 | import { createDummyNode } from '../../../__fixtures__/createDummyNode'; 2 | import { NodeSorter } from '../NodeSorter'; 3 | import { NetworkQuorumSetConfiguration } from '../../../../network/NetworkQuorumSetConfiguration'; 4 | 5 | describe('NodeSorter', () => { 6 | test('sortByNetworkQuorumSetInclusion', () => { 7 | const a = createDummyNode(); 8 | const b = createDummyNode(); 9 | const c = createDummyNode(); 10 | const d = createDummyNode(); 11 | 12 | const quorumSet = new NetworkQuorumSetConfiguration( 13 | 2, 14 | [a.publicKey], 15 | [new NetworkQuorumSetConfiguration(1, [b.publicKey], [])] 16 | ); 17 | 18 | const nodes = [c, d, b, a]; 19 | NodeSorter.sortByNetworkQuorumSetInclusion(nodes, quorumSet); 20 | 21 | function assertAAndBInFront() { 22 | expect( 23 | [b.publicKey.value, a.publicKey.value].includes( 24 | nodes[0].publicKey.value 25 | ) 26 | ).toBeTruthy(); 27 | expect( 28 | [b.publicKey.value, a.publicKey.value].includes( 29 | nodes[1].publicKey.value 30 | ) 31 | ).toBeTruthy(); 32 | } 33 | 34 | assertAAndBInFront(); 35 | }); 36 | }); 37 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/__tests__/node-index/index/active-index.test.ts: -------------------------------------------------------------------------------- 1 | import { ActiveIndex } from '../../../index/active-index'; 2 | 3 | test('get', () => { 4 | expect(ActiveIndex.get(100)).toEqual(1); 5 | expect(ActiveIndex.get(50)).toEqual(0.5); 6 | }); 7 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/__tests__/node-index/index/age-index.test.ts: -------------------------------------------------------------------------------- 1 | import { AgeIndex } from '../../../index/age-index'; 2 | 3 | test('get', () => { 4 | expect(AgeIndex.get(new Date())).toEqual(0); 5 | let newDate = new Date(); 6 | newDate.setMonth(newDate.getMonth() - 3); 7 | expect(AgeIndex.get(newDate)).toEqual(2 / 6); 8 | 9 | newDate = new Date(); 10 | newDate.setMonth(newDate.getMonth() - 8); 11 | expect(AgeIndex.get(newDate)).toEqual(1); 12 | }); 13 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/__tests__/node-index/index/trust-index.test.ts: -------------------------------------------------------------------------------- 1 | import { Edge, TrustGraph, Vertex } from '@stellarbeat/js-stellarbeat-shared'; 2 | import { StronglyConnectedComponentsFinder } from '@stellarbeat/js-stellarbeat-shared/lib/trust-graph/strongly-connected-components-finder'; 3 | import { NetworkTransitiveQuorumSetFinder } from '@stellarbeat/js-stellarbeat-shared/lib/trust-graph/network-transitive-quorum-set-finder'; 4 | import { TrustIndex } from '../../../index/trust-index'; 5 | 6 | const trustGraph = new TrustGraph( 7 | new StronglyConnectedComponentsFinder(), 8 | new NetworkTransitiveQuorumSetFinder() 9 | ); 10 | 11 | const vertex1 = new Vertex('a', 'a', 1); 12 | trustGraph.addVertex(vertex1); 13 | 14 | const vertex2 = new Vertex('b', 'b', 1); 15 | trustGraph.addVertex(vertex2); 16 | 17 | const vertex3 = new Vertex('c', 'c', 1); 18 | trustGraph.addVertex(vertex3); 19 | 20 | const vertex4 = new Vertex('d', 'd', 1); 21 | trustGraph.addVertex(vertex4); 22 | 23 | trustGraph.addEdge(new Edge(vertex1, vertex2)); 24 | trustGraph.addEdge(new Edge(vertex2, vertex1)); 25 | trustGraph.addEdge(new Edge(vertex3, vertex1)); 26 | trustGraph.addEdge(new Edge(vertex4, vertex2)); 27 | 28 | test('get', () => { 29 | expect(TrustIndex.get(vertex1.key, trustGraph)).toEqual(2 / 3); 30 | }); 31 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/__tests__/node-index/index/type-index.test.ts: -------------------------------------------------------------------------------- 1 | import { TypeIndex } from '../../../index/type-index'; 2 | 3 | test('get', () => { 4 | expect(TypeIndex.get(false, false)).toEqual(0.3); 5 | expect(TypeIndex.get(false, true)).toEqual(0.7); 6 | expect(TypeIndex.get(true, true)).toEqual(1); 7 | }); 8 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/__tests__/node-index/index/validating-index.test.ts: -------------------------------------------------------------------------------- 1 | import { ValidatingIndex } from '../../../index/validating-index'; 2 | 3 | test('get', () => { 4 | expect(ValidatingIndex.get(100)).toEqual(1); 5 | expect(ValidatingIndex.get(50)).toEqual(0.5); 6 | }); 7 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/index/active-index.ts: -------------------------------------------------------------------------------- 1 | export class ActiveIndex { 2 | static get(isActive30DaysPercentage: number): number { 3 | return isActive30DaysPercentage / 100; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/index/age-index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Age index. The more recent, the lower the value 3 | */ 4 | export class AgeIndex { 5 | static get(dateDiscovered: Date): number { 6 | const monthDifference = AgeIndex.monthDifference( 7 | dateDiscovered, 8 | new Date() 9 | ); 10 | if (monthDifference > 6) 11 | //older then 6 months 12 | return 1; 13 | else return monthDifference / 6; 14 | } 15 | 16 | protected static monthDifference(date1: Date, date2: Date) { 17 | let months; 18 | months = (date2.getFullYear() - date1.getFullYear()) * 12; 19 | months -= date1.getMonth() + 1; 20 | months += date2.getMonth(); 21 | return months <= 0 ? 0 : months; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/index/trust-index.ts: -------------------------------------------------------------------------------- 1 | import { TrustGraph } from '@stellarbeat/js-stellarbeat-shared'; 2 | 3 | export class TrustIndex { 4 | static get(vertexId: string, nodesTrustGraph: TrustGraph): number { 5 | const vertex = nodesTrustGraph.getVertex(vertexId); 6 | 7 | if (!vertex) return 0; 8 | 9 | if ( 10 | Array.from(nodesTrustGraph.vertices.values()).filter( 11 | (vertex) => nodesTrustGraph.getOutDegree(vertex) > 0 12 | ).length - 13 | 1 === 14 | 0 15 | ) 16 | return 0; 17 | return ( 18 | Array.from(nodesTrustGraph.getParents(vertex)).filter( 19 | (trustingVertex) => trustingVertex.key !== vertex.key 20 | ).length / 21 | (Array.from(nodesTrustGraph.vertices.values()).filter( 22 | (vertex) => nodesTrustGraph.getOutDegree(vertex) > 0 23 | ).length - 24 | 1) 25 | ); //exclude the node itself 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/index/type-index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Index for node type (full validator, basic validator or watcher node) 3 | */ 4 | export class TypeIndex { 5 | static get(hasUpToDateHistoryArchive: boolean, isValidator: boolean): number { 6 | if (hasUpToDateHistoryArchive) { 7 | return 1; 8 | } 9 | if (isValidator) { 10 | return 0.7; 11 | } 12 | 13 | return 0.3; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/network-scan/domain/node/scan/node-index/index/validating-index.ts: -------------------------------------------------------------------------------- 1 | export class ValidatingIndex { 2 | static get(validating30DaysPercentage: number): number { 3 | return validating30DaysPercentage / 100; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationId.ts: -------------------------------------------------------------------------------- 1 | import { ValueObject } from '../../../core/domain/ValueObject'; 2 | import { Column, Index } from 'typeorm'; 3 | import { err, ok, Result } from 'neverthrow'; 4 | import { createHash } from 'crypto'; 5 | import { mapUnknownToError } from '../../../core/utilities/mapUnknownToError'; 6 | 7 | export class OrganizationId extends ValueObject { 8 | @Column('varchar', { length: 100 }) 9 | @Index('IDX_7867970695572b3f6561516414', { unique: true }) 10 | public readonly value: string; 11 | 12 | private constructor(organizationId: string) { 13 | super(); 14 | this.value = organizationId; 15 | } 16 | 17 | static create( 18 | homeDomain: string, 19 | id?: string 20 | ): Result { 21 | if (id) return ok(new OrganizationId(id)); 22 | try { 23 | const hash = createHash('md5'); 24 | hash.update(homeDomain); 25 | return ok(new OrganizationId(hash.digest('hex'))); 26 | } catch (e) { 27 | return err(mapUnknownToError(e)); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationMeasurement.ts: -------------------------------------------------------------------------------- 1 | import { Entity, Column, ManyToOne, PrimaryColumn } from 'typeorm'; 2 | import Organization from './Organization'; 3 | import { Measurement } from '../measurement/Measurement'; 4 | import { TomlState } from './scan/TomlState'; 5 | 6 | @Entity() 7 | export default class OrganizationMeasurement implements Measurement { 8 | @Column('timestamptz', { primary: true }) 9 | time: Date; 10 | 11 | @PrimaryColumn() 12 | private organizationId?: string; 13 | 14 | @ManyToOne(() => Organization, { 15 | nullable: false, 16 | eager: true 17 | }) 18 | organization: Organization; 19 | 20 | @Column('bool') 21 | isSubQuorumAvailable = false; //todo: rename to isAvailable 22 | 23 | @Column('smallint') 24 | index = 0; //future proof 25 | 26 | @Column('enum', { default: TomlState.Unknown, enum: TomlState }) 27 | tomlState: TomlState = TomlState.Unknown; 28 | 29 | constructor(time: Date, organization: Organization) { 30 | this.time = time; 31 | this.organization = organization; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationMeasurementAverage.ts: -------------------------------------------------------------------------------- 1 | export interface OrganizationMeasurementAverage { 2 | organizationId: string; 3 | isSubQuorumAvailableAvg: number; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationMeasurementDay.ts: -------------------------------------------------------------------------------- 1 | import { Entity, Column, ManyToOne, PrimaryColumn } from 'typeorm'; 2 | import Organization from './Organization'; 3 | import { MeasurementAggregation } from '../measurement-aggregation/MeasurementAggregation'; 4 | 5 | @Entity() 6 | export default class OrganizationMeasurementDay 7 | implements MeasurementAggregation 8 | { 9 | @Column('date', { primary: true, name: 'time' }) 10 | protected _time: string; 11 | 12 | @PrimaryColumn() 13 | private organizationId?: string; 14 | 15 | @ManyToOne(() => Organization, { 16 | nullable: false, 17 | eager: true 18 | }) 19 | organization: Organization; 20 | 21 | @Column('smallint', { default: 0 }) 22 | isSubQuorumAvailableCount = 0; 23 | 24 | @Column('int') 25 | indexSum = 0; //future proof 26 | 27 | @Column('smallint', { default: 0 }) 28 | crawlCount = 0; 29 | 30 | constructor(day: string, organization: Organization) { 31 | this._time = day; 32 | this.organization = organization; 33 | } 34 | 35 | get time() { 36 | return new Date(this._time); 37 | } 38 | 39 | toJSON(): Record { 40 | return { 41 | time: this.time, 42 | isSubQuorumAvailableCount: this.isSubQuorumAvailableCount, 43 | indexSum: this.indexSum, 44 | crawlCount: this.crawlCount 45 | }; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationMeasurementDayRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementAggregationRepository } from '../measurement-aggregation/MeasurementAggregationRepository'; 2 | import { OrganizationMeasurementAverage } from './OrganizationMeasurementAverage'; 3 | import OrganizationMeasurementDay from './OrganizationMeasurementDay'; 4 | import { OrganizationId } from './OrganizationId'; 5 | 6 | export interface OrganizationMeasurementDayRepository 7 | extends MeasurementAggregationRepository { 8 | findXDaysAverageAt( 9 | at: Date, 10 | xDays: number 11 | ): Promise; 12 | 13 | findBetween( 14 | organizationId: OrganizationId, 15 | from: Date, 16 | to: Date 17 | ): Promise; 18 | } 19 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationMeasurementEvent.ts: -------------------------------------------------------------------------------- 1 | export interface OrganizationMeasurementEvent { 2 | time: string; 3 | organizationId: string; 4 | subQuorumUnavailable: boolean; 5 | tomlIssue: boolean; 6 | } 7 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationMeasurementRepository.ts: -------------------------------------------------------------------------------- 1 | import { MeasurementRepository } from '../measurement/MeasurementRepository'; 2 | import OrganizationMeasurement from './OrganizationMeasurement'; 3 | import { OrganizationMeasurementAverage } from './OrganizationMeasurementAverage'; 4 | import { OrganizationMeasurementEvent } from './OrganizationMeasurementEvent'; 5 | 6 | export interface OrganizationMeasurementRepository 7 | extends MeasurementRepository { 8 | findXDaysAverageAt( 9 | at: Date, 10 | xDays: number 11 | ): Promise; 12 | findEventsForXNetworkScans( 13 | x: number, 14 | at: Date 15 | ): Promise; 16 | save(organizationMeasurements: OrganizationMeasurement[]): Promise; 17 | } 18 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationRepository.ts: -------------------------------------------------------------------------------- 1 | import Organization from './Organization'; 2 | 3 | //active means that the organization is not archived. i.e. snapshot endDate = SNAPSHOT_MAX_END_DATE 4 | export interface OrganizationRepository { 5 | findActiveAtTimePoint(at: Date): Promise; //active snapshot at time x 6 | findActive(): Promise; //active snapshot at time now 7 | findByHomeDomains(homeDomains: string[]): Promise; //active or archived 8 | save(organizations: Organization[], from: Date): Promise; 9 | } 10 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationSnapShotRepository.ts: -------------------------------------------------------------------------------- 1 | import OrganizationSnapShot from './OrganizationSnapShot'; 2 | import { OrganizationId } from './OrganizationId'; 3 | 4 | //We need this until we start storing the actual changes. The only reason we need this is to calculate changes on the fly 5 | //@deprecated 6 | export interface OrganizationSnapShotRepository { 7 | findLatest(at: Date): Promise; 8 | findLatestByOrganizationId( 9 | organizationId: OrganizationId, 10 | at: Date 11 | ): Promise; 12 | } 13 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/OrganizationValidators.ts: -------------------------------------------------------------------------------- 1 | import { ValueObject } from '../../../core/domain/ValueObject'; 2 | import PublicKey from '../node/PublicKey'; 3 | import { Column } from 'typeorm'; 4 | import { plainToInstance } from 'class-transformer'; 5 | 6 | export class OrganizationValidators extends ValueObject { 7 | @Column({ 8 | type: 'json', 9 | nullable: true, //after migration set to true 10 | name: 'validators', 11 | transformer: { 12 | to: (value) => value, 13 | from: (publicKeys) => { 14 | //@ts-ignore 15 | return publicKeys.map((publicKey) => 16 | //@ts-ignore 17 | plainToInstance(PublicKey, publicKey) 18 | ); 19 | } 20 | } 21 | }) 22 | value: PublicKey[]; 23 | 24 | constructor(validators: PublicKey[]) { 25 | super(); 26 | this.value = validators; 27 | } 28 | 29 | equals(validators: OrganizationValidators): boolean { 30 | if (this.value.length !== validators.value.length) return false; 31 | return this.value.every((publicKey) => 32 | validators.value 33 | .map((publicKey) => publicKey.value) 34 | .includes(publicKey.value) 35 | ); 36 | } 37 | 38 | contains(publicKey: PublicKey): boolean { 39 | return this.value.some((validator) => validator.equals(publicKey)); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/TierOneCandidatePolicy.ts: -------------------------------------------------------------------------------- 1 | import { OrganizationMeasurementAverage } from './OrganizationMeasurementAverage'; 2 | import Organization from './Organization'; 3 | 4 | export class TierOneCandidatePolicy { 5 | static isTierOneCandidate( 6 | organization: Organization, 7 | measurement30DayAverage?: OrganizationMeasurementAverage 8 | ): boolean { 9 | if (!measurement30DayAverage) return false; 10 | return ( 11 | measurement30DayAverage.isSubQuorumAvailableAvg >= 99 && 12 | organization.validators.value.length >= 3 13 | ); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/__fixtures__/createDummyOrganizationId.ts: -------------------------------------------------------------------------------- 1 | import { OrganizationId } from '../OrganizationId'; 2 | 3 | export function createDummyOrganizationId( 4 | domain = createDummyOrganizationIdString() 5 | ) { 6 | const organizationIdOrError = OrganizationId.create(domain); 7 | if (organizationIdOrError.isErr()) throw organizationIdOrError.error; 8 | return organizationIdOrError.value; 9 | } 10 | 11 | export function createDummyOrganizationIdString() { 12 | let organizationId = ''; 13 | const allowedChars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; 14 | for (let i = 0; i < 55; i++) { 15 | organizationId += allowedChars.charAt( 16 | Math.floor(Math.random() * allowedChars.length) 17 | ); 18 | } 19 | 20 | return organizationId; 21 | } 22 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/__tests__/OrganizationId.test.ts: -------------------------------------------------------------------------------- 1 | import { OrganizationId } from '../OrganizationId'; 2 | 3 | it('should create organization id', function () { 4 | const organizationIdOrError = OrganizationId.create('test'); 5 | expect(organizationIdOrError.isOk()).toBeTruthy(); 6 | if (organizationIdOrError.isErr()) return; 7 | expect(organizationIdOrError.value.value).toEqual( 8 | '098f6bcd4621d373cade4e832627b4f6' 9 | ); 10 | }); 11 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/OrganizationTomlInfo.ts: -------------------------------------------------------------------------------- 1 | import { TomlState } from './TomlState'; 2 | 3 | export interface OrganizationTomlInfo { 4 | state: TomlState; 5 | name: string | null; 6 | physicalAddress: string | null; 7 | twitter: string | null; 8 | github: string | null; 9 | keybase: string | null; 10 | officialEmail: string | null; 11 | horizonUrl: string | null; 12 | dba: string | null; 13 | url: string | null; 14 | description: string | null; 15 | phoneNumber: string | null; 16 | validators: string[]; 17 | } 18 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/TomlState.ts: -------------------------------------------------------------------------------- 1 | export enum TomlState { 2 | Unknown = 'Unknown', 3 | Ok = 'Ok', 4 | RequestTimeout = 'RequestTimeout', 5 | DNSLookupFailed = 'DNSLookupFailed', 6 | HostnameResolutionFailed = 'HostnameResolutionFailed', 7 | ConnectionTimeout = 'ConnectionTimeout', 8 | ConnectionRefused = 'ConnectionRefused', 9 | ConnectionResetByPeer = 'ConnectionResetByPeer', 10 | SocketClosedPrematurely = 'SocketClosedPrematurely', 11 | SocketTimeout = 'SocketTimeout', 12 | HostUnreachable = 'HostUnreachable', 13 | NotFound = 'NotFound', 14 | ParsingError = 'ParsingError', 15 | Forbidden = 'Forbidden', 16 | ServerError = 'ServerError', 17 | UnsupportedVersion = 'UnsupportedVersion', 18 | UnspecifiedError = 'UnspecifiedError', 19 | ValidatorNotSEP20Linked = 'ValidatorNotSEP20Linked', 20 | EmptyValidatorsField = 'EmptyValidatorsField' 21 | } 22 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/CouldNotRetrieveArchivedOrganizationsError.ts: -------------------------------------------------------------------------------- 1 | import { OrganizationScanError } from './OrganizationScanError'; 2 | 3 | export class CouldNotRetrieveArchivedOrganizationsError extends OrganizationScanError { 4 | constructor(cause: Error) { 5 | super( 6 | 'Could not retrieve archived organizations', 7 | CouldNotRetrieveArchivedOrganizationsError.name, 8 | cause 9 | ); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/InvalidOrganizationIdError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../../../core/errors/CustomError'; 2 | import { OrganizationScanError } from './OrganizationScanError'; 3 | 4 | export class InvalidOrganizationIdError extends OrganizationScanError { 5 | constructor(homeDomain: string, cause: Error) { 6 | super( 7 | `Organization id for home-domain ${homeDomain} is invalid`, 8 | InvalidOrganizationIdError.name, 9 | cause 10 | ); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/InvalidTomlStateError.ts: -------------------------------------------------------------------------------- 1 | import { OrganizationScanError } from './OrganizationScanError'; 2 | import { TomlState } from '../TomlState'; 3 | 4 | export class InvalidTomlStateError extends OrganizationScanError { 5 | constructor(homeDomain: string, tomlState: TomlState) { 6 | super( 7 | `Organization toml file for home-domain ${homeDomain} has invalid state ${TomlState[tomlState]}`, 8 | InvalidTomlStateError.name 9 | ); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/OrganizationScanError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../../../core/errors/CustomError'; 2 | 3 | export abstract class OrganizationScanError extends CustomError { 4 | private organizationScanErrorType = 'OrganizationScanError'; //to break duck-typing 5 | 6 | constructor(message: string, name: string, cause?: Error) { 7 | super(message, OrganizationScanError.name, cause); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/TomlWithoutValidatorsError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../../../core/errors/CustomError'; 2 | import { OrganizationScanError } from './OrganizationScanError'; 3 | 4 | export class TomlWithoutValidatorsError extends OrganizationScanError { 5 | constructor(homeDomain: string) { 6 | super( 7 | `Organization toml file for home-domain ${homeDomain} does not have any validators`, 8 | TomlWithoutValidatorsError.name 9 | ); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/ValidatorNotSEP20LinkedError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../../../core/errors/CustomError'; 2 | import PublicKey from '../../../node/PublicKey'; 3 | import { OrganizationScanError } from './OrganizationScanError'; 4 | 5 | export class ValidatorNotSEP20LinkedError extends OrganizationScanError { 6 | constructor( 7 | organizationHomeDomain: string, 8 | validatorHomeDomain: string | null, 9 | validator: PublicKey 10 | ) { 11 | super( 12 | `Cannot add validator ${validator} with home-domain ${validatorHomeDomain} 13 | to organization with home-domain ${organizationHomeDomain} because it is not linked to the organization 14 | through SEP-0020`, 15 | ValidatorNotSEP20LinkedError.name 16 | ); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/network-scan/domain/organization/scan/errors/WrongNodeScanForOrganizationScan.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../../../core/errors/CustomError'; 2 | import { OrganizationScanError } from './OrganizationScanError'; 3 | 4 | export class WrongNodeScanForOrganizationScan extends OrganizationScanError { 5 | constructor(organizationScanTime: Date, nodeScanTime: Date) { 6 | super( 7 | `OrganizationScan time ${organizationScanTime} does not match NodeScan time ${nodeScanTime}`, 8 | WrongNodeScanForOrganizationScan.name 9 | ); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/infrastructure/cli/toml-fetch.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../core/infrastructure/Kernel'; 2 | import { OrganizationTomlFetcher } from '../../domain/organization/scan/OrganizationTomlFetcher'; 3 | import { NodeTomlFetcher } from '../../domain/node/scan/NodeTomlFetcher'; 4 | 5 | main(); 6 | 7 | async function main() { 8 | const kernel = await Kernel.getInstance(); 9 | const organizationTomlFetcher = kernel.container.get(OrganizationTomlFetcher); 10 | const nodeTomlFetcher = kernel.container.get(NodeTomlFetcher); 11 | 12 | const organizationResult = 13 | await organizationTomlFetcher.fetchOrganizationTomlInfoCollection([ 14 | process.argv[2] 15 | ]); 16 | 17 | console.log(organizationResult); 18 | 19 | const nodeResult = await nodeTomlFetcher.fetchNodeTomlInfoCollection([ 20 | process.argv[2] 21 | ]); 22 | 23 | console.log(nodeResult); 24 | } 25 | -------------------------------------------------------------------------------- /src/network-scan/infrastructure/database/entities/MeasurementRollup.ts: -------------------------------------------------------------------------------- 1 | import { Entity, Column, PrimaryGeneratedColumn, Index } from 'typeorm'; 2 | 3 | /* 4 | @Deprecated 5 | */ 6 | @Entity() 7 | export default class MeasurementRollup { 8 | @PrimaryGeneratedColumn() 9 | // @ts-ignore 10 | id: number; 11 | 12 | @Index() 13 | @Column('text') 14 | name: string; 15 | 16 | @Column('text', { nullable: false }) 17 | targetTableName: string; 18 | 19 | @Column('bigint', { default: 0 }) 20 | lastAggregatedCrawlId = 0; 21 | 22 | constructor(name: string, targetTableName: string) { 23 | this.name = name; 24 | this.targetTableName = targetTableName; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/network-scan/infrastructure/database/repositories/TypeOrmNetworkMeasurementRepository.ts: -------------------------------------------------------------------------------- 1 | import { Between, Repository } from 'typeorm'; 2 | import { injectable } from 'inversify'; 3 | import NetworkMeasurement from '../../../domain/network/NetworkMeasurement'; 4 | import { NetworkMeasurementRepository } from '../../../domain/network/NetworkMeasurementRepository'; 5 | 6 | @injectable() 7 | export class TypeOrmNetworkMeasurementRepository 8 | implements NetworkMeasurementRepository 9 | { 10 | constructor(private baseRepository: Repository) {} 11 | 12 | async save(networkMeasurements: NetworkMeasurement[]): Promise { 13 | await this.baseRepository.save(networkMeasurements); 14 | } 15 | 16 | findBetween(id: string, from: Date, to: Date) { 17 | return this.baseRepository.find({ 18 | where: [ 19 | { 20 | time: Between(from, to) 21 | } 22 | ], 23 | order: { time: 'ASC' } 24 | }); 25 | } 26 | 27 | async findAllAt(at: Date): Promise { 28 | return await this.baseRepository.find({ 29 | where: { 30 | time: at 31 | } 32 | }); 33 | } 34 | 35 | async findAt(id: string, at: Date): Promise { 36 | return await this.baseRepository.findOne({ 37 | where: { 38 | time: at 39 | } 40 | }); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/network-scan/infrastructure/services/DeadManSnitchHeartBeater.ts: -------------------------------------------------------------------------------- 1 | import { inject, injectable } from 'inversify'; 2 | import { HttpService } from '../../../core/services/HttpService'; 3 | import { Url } from '../../../core/domain/Url'; 4 | import { err, ok, Result } from 'neverthrow'; 5 | import { CustomError } from '../../../core/errors/CustomError'; 6 | import { HeartBeater } from '../../../core/services/HeartBeater'; 7 | 8 | @injectable() 9 | export class DeadManSnitchHeartBeater implements HeartBeater { 10 | constructor( 11 | @inject('HttpService') protected httpService: HttpService, 12 | protected url: Url 13 | ) { 14 | this.url = url; 15 | this.httpService = httpService; 16 | } 17 | 18 | async tick(): Promise> { 19 | const result = await this.httpService.get(this.url); 20 | if (result.isOk()) return ok(undefined); 21 | 22 | return err( 23 | new CustomError('Heartbeat tick failed', 'HeartbeatError', result.error) 24 | ); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/network-scan/infrastructure/services/DummyHeartBeater.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import { ok, Result } from 'neverthrow'; 3 | import { HeartBeater } from '../../../core/services/HeartBeater'; 4 | 5 | @injectable() 6 | export class DummyHeartBeater implements HeartBeater { 7 | tick() { 8 | return new Promise>((resolve) => 9 | resolve(ok(undefined)) 10 | ); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/network-scan/infrastructure/services/__tests__/DatabaseHistoryArchiveScanService.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { NETWORK_TYPES } from '../../di/di-types'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | test('di', async () => { 16 | const service = kernel.container.get(NETWORK_TYPES.HistoryArchiveScanService); 17 | expect(service).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/mappers/BaseQuorumSetDTOMapper.ts: -------------------------------------------------------------------------------- 1 | import { NetworkQuorumSetConfiguration } from '../domain/network/NetworkQuorumSetConfiguration'; 2 | import { BaseQuorumSet } from '@stellarbeat/js-stellarbeat-shared'; 3 | 4 | export class BaseQuorumSetDTOMapper { 5 | static fromNetworkQuorumSetConfiguration( 6 | networkQuorumSetConfiguration: NetworkQuorumSetConfiguration 7 | ): BaseQuorumSet { 8 | return { 9 | threshold: networkQuorumSetConfiguration.threshold, 10 | validators: networkQuorumSetConfiguration.validators.map( 11 | (validator) => validator.value 12 | ), 13 | innerQuorumSets: networkQuorumSetConfiguration.innerQuorumSets.map( 14 | (innerQuorumSet) => 15 | this.fromNetworkQuorumSetConfiguration(innerQuorumSet) 16 | ) 17 | }; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/network-scan/services/README.md: -------------------------------------------------------------------------------- 1 | Services folder contain code/interfaces to connect between modules through code. -------------------------------------------------------------------------------- /src/network-scan/services/__fixtures__/createDummyOrganizationV1.ts: -------------------------------------------------------------------------------- 1 | import { OrganizationV1 } from '@stellarbeat/js-stellarbeat-shared'; 2 | 3 | export function createDummyOrganizationV1(): OrganizationV1 { 4 | return { 5 | id: 'id', 6 | name: 'name', 7 | validators: [], 8 | dba: 'dba', 9 | url: 'url', 10 | dateDiscovered: new Date().toISOString(), 11 | github: 'github', 12 | twitter: 'twitter', 13 | logo: 'logo', 14 | description: 'description', 15 | homeDomain: 'homeDomain', 16 | has24HourStats: true, 17 | has30DayStats: true, 18 | isTierOneOrganization: true, 19 | keybase: 'keybase', 20 | horizonUrl: 'horizonUrl', 21 | phoneNumber: 'phoneNumber', 22 | officialEmail: 'officialEmail', 23 | physicalAddress: 'physicalAddress', 24 | subQuorum24HoursAvailability: 1, 25 | subQuorum30DaysAvailability: 1, 26 | subQuorumAvailable: true, 27 | tomlState: 'Unknown' 28 | }; 29 | } 30 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-latest-node-snapshots/GetLatestNodeSnapshotsDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetLatestNodeSnapshotsDTO { 2 | at: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-latest-node-snapshots/__tests__/GetLatestNodeSnapshots.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { mock } from 'jest-mock-extended'; 4 | import { GetLatestNodeSnapshots } from '../GetLatestNodeSnapshots'; 5 | import { NodeSnapShotRepository } from '../../../domain/node/NodeSnapShotRepository'; 6 | import { NETWORK_TYPES } from '../../../infrastructure/di/di-types'; 7 | 8 | let kernel: Kernel; 9 | jest.setTimeout(60000); //slow integration tests 10 | beforeAll(async () => { 11 | kernel = await Kernel.getInstance(new ConfigMock()); 12 | }); 13 | 14 | afterAll(async () => { 15 | await kernel.close(); 16 | }); 17 | 18 | it('should fetch latest node snapshots', async () => { 19 | const repo = mock(); 20 | repo.findLatest.mockResolvedValue([]); 21 | kernel.container 22 | .rebind(NETWORK_TYPES.NodeSnapshotRepository) 23 | .toConstantValue(repo); 24 | 25 | const useCase = kernel.container.get(GetLatestNodeSnapshots); 26 | const result = await useCase.execute({ 27 | at: new Date() 28 | }); 29 | expect(result.isOk()).toBe(true); 30 | if (!result.isOk()) return; 31 | }); 32 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-latest-node-snapshots/__tests__/GetLatestNodeSnapshots.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { ExceptionLogger } from '../../../../core/services/ExceptionLogger'; 3 | import { GetLatestNodeSnapshots } from '../GetLatestNodeSnapshots'; 4 | import { NodeSnapShotRepository } from '../../../domain/node/NodeSnapShotRepository'; 5 | 6 | it('should capture and return errors', async function () { 7 | const repo = mock(); 8 | repo.findLatest.mockRejectedValue(new Error('test')); 9 | const exceptionLogger = mock(); 10 | const useCase = new GetLatestNodeSnapshots(repo, exceptionLogger); 11 | const result = await useCase.execute({ 12 | at: new Date() 13 | }); 14 | expect(result.isErr()).toBe(true); 15 | expect(exceptionLogger.captureException).toBeCalledTimes(1); 16 | }); 17 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-latest-organization-snapshots/GetLatestOrganizationSnapshotsDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetLatestOrganizationSnapshotsDTO { 2 | at: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-latest-organization-snapshots/__tests__/GetLatestOrganizationSnapshots.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { mock } from 'jest-mock-extended'; 4 | import { GetLatestOrganizationSnapshots } from '../GetLatestOrganizationSnapshots'; 5 | import { OrganizationSnapShotRepository } from '../../../domain/organization/OrganizationSnapShotRepository'; 6 | import { NETWORK_TYPES } from '../../../infrastructure/di/di-types'; 7 | 8 | let kernel: Kernel; 9 | jest.setTimeout(60000); //slow integration tests 10 | beforeAll(async () => { 11 | kernel = await Kernel.getInstance(new ConfigMock()); 12 | }); 13 | 14 | afterAll(async () => { 15 | await kernel.close(); 16 | }); 17 | 18 | it('should fetch latest snapshots', async () => { 19 | const repo = mock(); 20 | repo.findLatest.mockResolvedValue([]); 21 | kernel.container 22 | .rebind(NETWORK_TYPES.OrganizationSnapshotRepository) 23 | .toConstantValue(repo); 24 | 25 | const useCase = kernel.container.get(GetLatestOrganizationSnapshots); 26 | const result = await useCase.execute({ 27 | at: new Date() 28 | }); 29 | expect(result.isOk()).toBe(true); 30 | if (!result.isOk()) return; 31 | }); 32 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-latest-organization-snapshots/__tests__/GetLatestOrganizationSnapshots.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from 'jest-mock-extended'; 2 | import { ExceptionLogger } from '../../../../core/services/ExceptionLogger'; 3 | import { GetLatestOrganizationSnapshots } from '../GetLatestOrganizationSnapshots'; 4 | import { OrganizationSnapShotRepository } from '../../../domain/organization/OrganizationSnapShotRepository'; 5 | 6 | it('should capture and return errors', async function () { 7 | const repo = mock(); 8 | repo.findLatest.mockRejectedValue(new Error('test')); 9 | const exceptionLogger = mock(); 10 | const useCase = new GetLatestOrganizationSnapshots(repo, exceptionLogger); 11 | const result = await useCase.execute({ 12 | at: new Date() 13 | }); 14 | expect(result.isErr()).toBe(true); 15 | expect(exceptionLogger.captureException).toBeCalledTimes(1); 16 | }); 17 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-measurement-aggregations/GetMeasurementAggregationsDTO.ts: -------------------------------------------------------------------------------- 1 | export enum AggregationTarget { //FUTURE: should map result DTO. For example NodeMeasurementDayDTO. 2 | // For now a simple toJSON is returned, but when api changes we could have to support multiple versions of result DTOs 3 | NodeDay, 4 | OrganizationDay, 5 | NetworkDay, 6 | NetworkMonth 7 | } 8 | 9 | export interface GetMeasurementAggregationsDTO { 10 | aggregationTarget: AggregationTarget; 11 | id: string; 12 | from: Date; 13 | to: Date; 14 | } 15 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-measurement-aggregations/__tests__/GetMeasurementAggregations.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetMeasurementAggregations } from '../GetMeasurementAggregations'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | test('di', () => { 16 | kernel.container.get(GetMeasurementAggregations); 17 | }); 18 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-measurements/GetMeasurements.ts: -------------------------------------------------------------------------------- 1 | import { GetMeasurementsDTO } from './GetMeasurementsDTO'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import { mapUnknownToError } from '../../../core/utilities/mapUnknownToError'; 4 | import { ExceptionLogger } from '../../../core/services/ExceptionLogger'; 5 | import { MeasurementRepository } from '../../domain/measurement/MeasurementRepository'; 6 | import { Measurement } from '../../domain/measurement/Measurement'; 7 | 8 | export class GetMeasurements { 9 | constructor( 10 | private measurementRepository: MeasurementRepository, 11 | private exceptionLogger: ExceptionLogger 12 | ) {} 13 | 14 | public async execute( 15 | dto: GetMeasurementsDTO 16 | ): Promise> { 17 | try { 18 | return ok( 19 | await this.measurementRepository.findBetween(dto.id, dto.from, dto.to) 20 | ); 21 | } catch (error) { 22 | this.exceptionLogger.captureException(mapUnknownToError(error)); 23 | return err(mapUnknownToError(error)); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-measurements/GetMeasurementsDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetMeasurementsDTO { 2 | id: string; 3 | from: Date; 4 | to: Date; 5 | } 6 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-network/GetNetwork.ts: -------------------------------------------------------------------------------- 1 | import { Network, NetworkV1 } from '@stellarbeat/js-stellarbeat-shared'; 2 | import { inject, injectable } from 'inversify'; 3 | import { Result } from 'neverthrow'; 4 | import { GetNetworkDTO } from './GetNetworkDTO'; 5 | import { ExceptionLogger } from '../../../core/services/ExceptionLogger'; 6 | import 'reflect-metadata'; 7 | import { NetworkDTOService } from '../../services/NetworkDTOService'; 8 | import { CachedNetworkDTOService } from '../../services/CachedNetworkDTOService'; 9 | 10 | @injectable() 11 | export class GetNetwork { 12 | constructor( 13 | private readonly networkDTOService: CachedNetworkDTOService, 14 | @inject('ExceptionLogger') protected exceptionLogger: ExceptionLogger 15 | ) {} 16 | 17 | async execute(dto: GetNetworkDTO): Promise> { 18 | let networkOrError: Result; 19 | if (dto.at === undefined) 20 | networkOrError = await this.networkDTOService.getLatestNetworkDTO(); 21 | else networkOrError = await this.networkDTOService.getNetworkDTOAt(dto.at); 22 | 23 | if (networkOrError.isErr()) { 24 | this.exceptionLogger.captureException(networkOrError.error); 25 | } 26 | 27 | return networkOrError; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-network/GetNetworkDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetNetworkDTO { 2 | at?: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-network/__tests__/GetNetwork.test.ts: -------------------------------------------------------------------------------- 1 | import { GetNetwork } from '../GetNetwork'; 2 | import { mock } from 'jest-mock-extended'; 3 | import { err } from 'neverthrow'; 4 | import { ExceptionLogger } from '../../../../core/services/ExceptionLogger'; 5 | import { NetworkDTOService } from '../../../services/NetworkDTOService'; 6 | import { CachedNetworkDTOService } from '../../../services/CachedNetworkDTOService'; 7 | 8 | it('should capture and return network errors', async function () { 9 | const networkDTOService = mock(); 10 | networkDTOService.getNetworkDTOAt.mockResolvedValue(err(new Error('test'))); 11 | const exceptionLogger = mock(); 12 | const getNetwork = new GetNetwork(networkDTOService, exceptionLogger); 13 | const result = await getNetwork.execute({ at: new Date() }); 14 | expect(result.isErr()).toBe(true); 15 | expect(exceptionLogger.captureException).toBeCalledTimes(1); 16 | }); 17 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-node-snapshots/GetNodeSnapshotsDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetNodeSnapshotsDTO { 2 | at: Date; 3 | publicKey: string; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-node-snapshots/__tests__/GetNodeSnapshots.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetNodeSnapshots } from '../GetNodeSnapshots'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(GetNodeSnapshots); 17 | expect(instance).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-node/GetNode.ts: -------------------------------------------------------------------------------- 1 | import { inject, injectable } from 'inversify'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import { ExceptionLogger } from '../../../core/services/ExceptionLogger'; 4 | import 'reflect-metadata'; 5 | import { GetNodeDTO } from './GetNodeDTO'; 6 | import { GetNetwork } from '../get-network/GetNetwork'; 7 | import { NodeV1 } from '@stellarbeat/js-stellarbeat-shared'; 8 | 9 | @injectable() 10 | export class GetNode { 11 | constructor( 12 | private readonly getNetwork: GetNetwork, 13 | @inject('ExceptionLogger') private exceptionLogger: ExceptionLogger 14 | ) {} 15 | 16 | async execute(dto: GetNodeDTO): Promise> { 17 | const networkOrError = await this.getNetwork.execute({ 18 | at: dto.at 19 | }); 20 | 21 | if (networkOrError.isErr()) { 22 | return err(networkOrError.error); 23 | } 24 | 25 | if (networkOrError.value === null) { 26 | return ok(null); 27 | } 28 | 29 | const node = networkOrError.value.nodes.find( 30 | (node) => node.publicKey === dto.publicKey 31 | ); 32 | 33 | if (!node) return ok(null); 34 | 35 | return ok(node); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-node/GetNodeDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetNodeDTO { 2 | at?: Date; 3 | publicKey: string; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-node/__tests__/GetNode.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetNode } from '../GetNode'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(GetNode); 17 | expect(instance).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-nodes/GetNodes.ts: -------------------------------------------------------------------------------- 1 | import { inject, injectable } from 'inversify'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import { ExceptionLogger } from '../../../core/services/ExceptionLogger'; 4 | import 'reflect-metadata'; 5 | import { GetNodesDTO } from './GetNodesDTO'; 6 | import { GetNetwork } from '../get-network/GetNetwork'; 7 | import { NodeV1 } from '@stellarbeat/js-stellarbeat-shared'; 8 | 9 | @injectable() 10 | export class GetNodes { 11 | constructor( 12 | private readonly getNetwork: GetNetwork, 13 | @inject('ExceptionLogger') private exceptionLogger: ExceptionLogger 14 | ) {} 15 | 16 | async execute(dto: GetNodesDTO): Promise> { 17 | const networkOrError = await this.getNetwork.execute({ 18 | at: dto.at 19 | }); 20 | 21 | if (networkOrError.isErr()) { 22 | return err(networkOrError.error); 23 | } 24 | 25 | if (networkOrError.value === null) { 26 | return ok([]); 27 | } 28 | 29 | return ok(networkOrError.value.nodes); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-nodes/GetNodesDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetNodesDTO { 2 | at?: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-nodes/__tests__/GetNodes.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetNodes } from '../GetNodes'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(GetNodes); 17 | expect(instance).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organization-snapshots/GetOrganizationSnapshotsDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetOrganizationSnapshotsDTO { 2 | at: Date; 3 | organizationId: string; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organization-snapshots/__tests__/GetOrganizationSnapshots.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetOrganizationSnapshots } from '../GetOrganizationSnapshots'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(GetOrganizationSnapshots); 17 | expect(instance).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organization/GetOrganization.ts: -------------------------------------------------------------------------------- 1 | import { inject, injectable } from 'inversify'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import { ExceptionLogger } from '../../../core/services/ExceptionLogger'; 4 | import 'reflect-metadata'; 5 | import { GetNetwork } from '../get-network/GetNetwork'; 6 | import { OrganizationV1 } from '@stellarbeat/js-stellarbeat-shared'; 7 | import { GetOrganizationDTO } from './GetOrganizationDTO'; 8 | 9 | @injectable() 10 | export class GetOrganization { 11 | constructor( 12 | private readonly getNetwork: GetNetwork, 13 | @inject('ExceptionLogger') private exceptionLogger: ExceptionLogger 14 | ) {} 15 | 16 | async execute( 17 | dto: GetOrganizationDTO 18 | ): Promise> { 19 | const networkOrError = await this.getNetwork.execute({ 20 | at: dto.at 21 | }); 22 | 23 | if (networkOrError.isErr()) { 24 | return err(networkOrError.error); 25 | } 26 | 27 | if (networkOrError.value === null) { 28 | return ok(null); 29 | } 30 | 31 | const organization = networkOrError.value.organizations.find( 32 | (organization) => organization.id === dto.organizationId 33 | ); 34 | 35 | if (!organization) return ok(null); 36 | 37 | return ok(organization); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organization/GetOrganizationDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetOrganizationDTO { 2 | at?: Date; 3 | organizationId: string; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organization/__tests__/GetOrganization.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetOrganization } from '../GetOrganization'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(GetOrganization); 17 | expect(instance).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organizations/GetOrganizations.ts: -------------------------------------------------------------------------------- 1 | import { inject, injectable } from 'inversify'; 2 | import { err, ok, Result } from 'neverthrow'; 3 | import { ExceptionLogger } from '../../../core/services/ExceptionLogger'; 4 | import 'reflect-metadata'; 5 | import { GetOrganizationsDTO } from './GetOrganizationsDTO'; 6 | import { GetNetwork } from '../get-network/GetNetwork'; 7 | import { OrganizationV1 } from '@stellarbeat/js-stellarbeat-shared'; 8 | 9 | @injectable() 10 | export class GetOrganizations { 11 | constructor( 12 | private readonly getNetwork: GetNetwork, 13 | @inject('ExceptionLogger') private exceptionLogger: ExceptionLogger 14 | ) {} 15 | 16 | async execute( 17 | dto: GetOrganizationsDTO 18 | ): Promise> { 19 | const networkOrError = await this.getNetwork.execute({ 20 | at: dto.at 21 | }); 22 | 23 | if (networkOrError.isErr()) { 24 | return err(networkOrError.error); 25 | } 26 | 27 | if (networkOrError.value === null) { 28 | return ok([]); 29 | } 30 | 31 | return ok(networkOrError.value.organizations); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organizations/GetOrganizationsDTO.ts: -------------------------------------------------------------------------------- 1 | export interface GetOrganizationsDTO { 2 | at?: Date; 3 | } 4 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/get-organizations/__tests__/GetOrganizations.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { GetOrganizations } from '../GetOrganizations'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(GetOrganizations); 17 | expect(instance).toBeDefined(); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/scan-network-looped/ScanNetworkLoopedDTO.ts: -------------------------------------------------------------------------------- 1 | export interface ScanNetworkLoopedDTO { 2 | dryRun: boolean; 3 | loopIntervalMs?: number; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/scan-network-looped/__tests__/ScanNetwork.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { ScanNetworkLooped } from '../ScanNetworkLooped'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(ScanNetworkLooped); 17 | expect(instance).toBeInstanceOf(ScanNetworkLooped); 18 | }); 19 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/scan-network/InvalidKnownPeersError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class InvalidKnownPeersError extends CustomError { 4 | constructor(cause?: Error) { 5 | super( 6 | `Invalid known peer detected in configuration`, 7 | InvalidKnownPeersError.name, 8 | cause 9 | ); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/scan-network/NodeAddressMapper.ts: -------------------------------------------------------------------------------- 1 | import { NodeAddress } from '../../domain/node/NodeAddress'; 2 | import { Result } from 'neverthrow'; 3 | 4 | export class NodeAddressMapper { 5 | static mapToNodeAddresses( 6 | nodeAddressDTOs: [string, number][] 7 | ): Result { 8 | return Result.combine( 9 | nodeAddressDTOs.map((peer) => { 10 | return NodeAddress.create(peer[0], peer[1]); 11 | }) 12 | ); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/scan-network/ScanNetworkDTO.ts: -------------------------------------------------------------------------------- 1 | export interface ScanNetworkDTO { 2 | updateNetwork: boolean; 3 | dryRun: boolean; 4 | } 5 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/scan-network/__tests__/NodeAddressMapper.test.ts: -------------------------------------------------------------------------------- 1 | import { NodeAddressMapper } from '../NodeAddressMapper'; 2 | 3 | describe('NodeAddressMapper', () => { 4 | test('should map to node addresses', () => { 5 | const nodeAddressDTOs: [string, number][] = [ 6 | ['127.0.0.1', 1], 7 | ['127.0.0.2', 2] 8 | ]; 9 | 10 | const result = NodeAddressMapper.mapToNodeAddresses(nodeAddressDTOs); 11 | expect(result.isOk()).toBe(true); 12 | if (result.isErr()) return; 13 | expect(result.value).toHaveLength(2); 14 | expect(result.value[0].ip).toBe('127.0.0.1'); 15 | expect(result.value[0].port).toBe(1); 16 | expect(result.value[1].ip).toBe('127.0.0.2'); 17 | expect(result.value[1].port).toBe(2); 18 | }); 19 | 20 | test('should return error when node address is invalid', () => { 21 | const nodeAddressDTOs: [string, number][] = [ 22 | ['127.0.0.1', 1], 23 | ['127.0.0.2', -2] 24 | ]; 25 | 26 | const result = NodeAddressMapper.mapToNodeAddresses(nodeAddressDTOs); 27 | expect(result.isErr()).toBe(true); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/InvalidOverlayRangeError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class InvalidOverlayRangeError extends CustomError { 4 | constructor() { 5 | super('Invalid overlay range', InvalidOverlayRangeError.name); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/InvalidQuorumSetConfigError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class InvalidQuorumSetConfigError extends CustomError { 4 | constructor() { 5 | super('Invalid quorum set configuration', InvalidQuorumSetConfigError.name); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/InvalidStellarCoreVersionError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class InvalidStellarCoreVersionError extends CustomError { 4 | constructor() { 5 | super('Invalid Stellar Core version', InvalidStellarCoreVersionError.name); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/InvalidUpdateTimeError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class InvalidUpdateTimeError extends CustomError { 4 | constructor() { 5 | super('Invalid update time', InvalidUpdateTimeError.name); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/RepositoryError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class RepositoryError extends CustomError { 4 | constructor(id: string) { 5 | super( 6 | `Error fetching or persisting config for network ${id}`, 7 | RepositoryError.name 8 | ); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/UpdateNetworkDTO.ts: -------------------------------------------------------------------------------- 1 | export interface UpdateNetworkDTO { 2 | time: Date; 3 | name: string; 4 | networkId: string; 5 | passphrase: string; 6 | networkQuorumSet: Array; 7 | overlayVersion: number; 8 | overlayMinVersion: number; 9 | ledgerVersion: number; 10 | stellarCoreVersion: string; 11 | } 12 | -------------------------------------------------------------------------------- /src/network-scan/use-cases/update-network/__tests__/UpdateNetwork.integration.test.ts: -------------------------------------------------------------------------------- 1 | import Kernel from '../../../../core/infrastructure/Kernel'; 2 | import { ConfigMock } from '../../../../core/config/__mocks__/configMock'; 3 | import { UpdateNetwork } from '../UpdateNetwork'; 4 | 5 | let kernel: Kernel; 6 | jest.setTimeout(60000); //slow integration tests 7 | beforeAll(async () => { 8 | kernel = await Kernel.getInstance(new ConfigMock()); 9 | }); 10 | 11 | afterAll(async () => { 12 | await kernel.close(); 13 | }); 14 | 15 | it('should find class instance', async () => { 16 | const instance = kernel.container.get(UpdateNetwork); 17 | expect(instance).toBeInstanceOf(UpdateNetwork); 18 | }); 19 | -------------------------------------------------------------------------------- /src/notifications/README.md: -------------------------------------------------------------------------------- 1 | # Notifications 2 | Notifications are sent to subscribed users for events that occur in the latest network update 3 | 4 | ## Events 5 | Events are calculated for every network update and can trigger notifications. An event describes a _change_ in a certain state. 6 | 7 | Examples of events are: 8 | * a node not validating for three crawls. 9 | * network is in danger of losing liveness when two or more nodes stop validating. 10 | * ... 11 | 12 | -------------------------------------------------------------------------------- /src/notifications/domain/event/EventRepository.ts: -------------------------------------------------------------------------------- 1 | import { Event, MultipleUpdatesEventData } from './Event'; 2 | import { OrganizationId, PublicKey } from './EventSourceId'; 3 | 4 | export interface EventRepository { 5 | findNodeEventsForXNetworkScans( 6 | x: number, 7 | at: Date 8 | ): Promise[]>; 9 | 10 | findOrganizationMeasurementEventsForXNetworkScans( 11 | x: number, 12 | at: Date 13 | ): Promise[]>; 14 | } 15 | -------------------------------------------------------------------------------- /src/notifications/domain/event/EventSource.ts: -------------------------------------------------------------------------------- 1 | import { EventSourceId } from './EventSourceId'; 2 | 3 | export class EventSource { 4 | constructor( 5 | public readonly eventSourceId: EventSourceId, 6 | public readonly name: string 7 | ) {} 8 | } 9 | -------------------------------------------------------------------------------- /src/notifications/domain/event/EventSourceId.ts: -------------------------------------------------------------------------------- 1 | import { err, ok, Result } from 'neverthrow'; 2 | 3 | export abstract class EventSourceId { 4 | public readonly value: string; 5 | protected constructor(value: string) { 6 | this.value = value; 7 | } 8 | 9 | equals(other: EventSourceId): boolean { 10 | return other.constructor === this.constructor && this.value === other.value; 11 | } 12 | } 13 | export class OrganizationId extends EventSourceId { 14 | constructor(value: string) { 15 | super(value); 16 | } 17 | } 18 | 19 | export class PublicKey extends EventSourceId { 20 | static create(publicKey: string): Result { 21 | if (publicKey.length !== 56) return err(new Error('Invalid length')); 22 | 23 | return ok(new PublicKey(publicKey)); 24 | } 25 | } 26 | 27 | export class NetworkId extends EventSourceId { 28 | constructor(value: string) { 29 | super(value); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/notifications/domain/event/EventSourceService.ts: -------------------------------------------------------------------------------- 1 | import { Result } from 'neverthrow'; 2 | import { EventSourceId } from './EventSourceId'; 3 | import { EventSource } from './EventSource'; 4 | 5 | export interface EventSourceService { 6 | isEventSourceIdKnown( 7 | eventSourceId: EventSourceId, 8 | time: Date 9 | ): Promise>; 10 | 11 | findEventSource( 12 | eventSourceId: EventSourceId, 13 | time: Date 14 | ): Promise>; 15 | } 16 | -------------------------------------------------------------------------------- /src/notifications/domain/event/__tests__/EventSourceIdFactory.test.ts: -------------------------------------------------------------------------------- 1 | import { EventSourceIdFactory } from '../EventSourceIdFactory'; 2 | import { EventSourceService } from '../EventSourceService'; 3 | import { Result, ok } from 'neverthrow'; 4 | import { EventSourceId, PublicKey } from '../EventSourceId'; 5 | 6 | it('should create PublicKey', async function () { 7 | jest.mock('../EventSourceService', () => { 8 | return jest.fn().mockImplementation(() => { 9 | return { isEventSourceIdKnown: jest.fn().mockResolvedValue(true) }; 10 | }); 11 | }); 12 | const eventSourceService: EventSourceService = { 13 | isEventSourceIdKnown( 14 | eventSourceId: EventSourceId, 15 | time: Date 16 | ): Promise> { 17 | return Promise.resolve(ok(true)); 18 | } 19 | } as EventSourceService; 20 | const factory = new EventSourceIdFactory(eventSourceService); 21 | 22 | const publicKeyResult = await factory.create( 23 | 'node', 24 | 'GCFXHS4GXL6BVUCXBWXGTITROWLVYXQKQLF4YH5O5JT3YZXCYPAFBJZB', 25 | new Date() 26 | ); 27 | expect(publicKeyResult.isErr()).toBeFalsy(); 28 | if (publicKeyResult.isErr()) return; 29 | expect(publicKeyResult.value).toBeInstanceOf(PublicKey); 30 | expect(publicKeyResult.value.value).toEqual( 31 | 'GCFXHS4GXL6BVUCXBWXGTITROWLVYXQKQLF4YH5O5JT3YZXCYPAFBJZB' 32 | ); 33 | }); 34 | -------------------------------------------------------------------------------- /src/notifications/domain/notifier/MessageCreator.ts: -------------------------------------------------------------------------------- 1 | import { PendingSubscriptionId } from '../subscription/PendingSubscription'; 2 | import { Message } from '../../../core/domain/Message'; 3 | import { Notification } from '../subscription/Notification'; 4 | import { SubscriberReference } from '../subscription/SubscriberReference'; 5 | 6 | export interface MessageCreator { 7 | createConfirmSubscriptionMessage( 8 | pendingSubscriptionId: PendingSubscriptionId 9 | ): Promise; 10 | 11 | createNotificationMessage(notification: Notification): Promise; 12 | 13 | createUnsubscribeMessage( 14 | subscriberReference: SubscriberReference, 15 | time: Date 16 | ): Promise; 17 | } 18 | -------------------------------------------------------------------------------- /src/notifications/domain/subscription/Notification.ts: -------------------------------------------------------------------------------- 1 | import { Event, EventData } from '../event/Event'; 2 | import { EventSourceId } from '../event/EventSourceId'; 3 | import { Subscriber } from './Subscriber'; 4 | 5 | export interface Notification { 6 | subscriber: Subscriber; 7 | events: Event[]; 8 | time: Date; 9 | } 10 | -------------------------------------------------------------------------------- /src/notifications/domain/subscription/SubscriberReference.ts: -------------------------------------------------------------------------------- 1 | import { Column, Index } from 'typeorm'; 2 | import { v4 as uuidv4, validate } from 'uuid'; 3 | import { err, ok, Result } from 'neverthrow'; 4 | 5 | export class SubscriberReference { 6 | @Index() 7 | @Column({ type: 'uuid', nullable: false }) 8 | public readonly value: string; 9 | 10 | private constructor(value: string) { 11 | this.value = value; 12 | } 13 | 14 | static create(): SubscriberReference { 15 | return new SubscriberReference(uuidv4()); 16 | } 17 | 18 | static createFromValue(value: string): Result { 19 | if (!validate(value)) 20 | return err(new Error('Not a valid SubscriberReference')); 21 | else return ok(new SubscriberReference(value)); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/notifications/domain/subscription/SubscriberRepository.ts: -------------------------------------------------------------------------------- 1 | import { UserId } from './UserId'; 2 | import { Subscriber } from './Subscriber'; 3 | import { PendingSubscriptionId } from './PendingSubscription'; 4 | import { SubscriberReference } from './SubscriberReference'; 5 | 6 | export interface SubscriberRepository { 7 | find(): Promise; 8 | findOneByUserId(userId: UserId): Promise; 9 | findOneBySubscriberReference( 10 | subscriberReference: SubscriberReference 11 | ): Promise; 12 | findOneByPendingSubscriptionId( 13 | pendingSubscriptionId: PendingSubscriptionId 14 | ): Promise; 15 | nextPendingSubscriptionId(): PendingSubscriptionId; 16 | save(subscribers: Subscriber[]): Promise; 17 | remove(subscriber: Subscriber): Promise; 18 | } 19 | -------------------------------------------------------------------------------- /src/notifications/domain/subscription/UserId.ts: -------------------------------------------------------------------------------- 1 | import { Column, Index } from 'typeorm'; 2 | import validator from 'validator'; 3 | import { Result, err, ok } from 'neverthrow'; 4 | 5 | export class UserId { 6 | @Index() 7 | @Column({ type: 'uuid', nullable: false, unique: true }) 8 | public readonly value: string; 9 | 10 | private constructor(rawId: string) { 11 | this.value = rawId; 12 | } 13 | 14 | static create(rawId: string): Result { 15 | if (!validator.isUUID(rawId)) 16 | return err(new Error('Invalid userId format')); 17 | return ok(new UserId(rawId)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/notifications/domain/subscription/__fixtures__/PendingSubscriptionId.fixtures.ts: -------------------------------------------------------------------------------- 1 | import { PendingSubscriptionId } from '../PendingSubscription'; 2 | import { v4 as uuidv4 } from 'uuid'; 3 | 4 | export function createDummyPendingSubscriptionId(rawId?: string) { 5 | const pendingSubscriptionIdResult = PendingSubscriptionId.create( 6 | rawId ? rawId : uuidv4() 7 | ); 8 | if (pendingSubscriptionIdResult.isErr()) 9 | throw pendingSubscriptionIdResult.error; 10 | return pendingSubscriptionIdResult.value; 11 | } 12 | -------------------------------------------------------------------------------- /src/notifications/domain/subscription/__fixtures__/Subscriber.fixtures.ts: -------------------------------------------------------------------------------- 1 | import { Subscriber } from '../Subscriber'; 2 | import { UserId } from '../UserId'; 3 | import { randomUUID } from 'crypto'; 4 | import { SubscriberReference } from '../SubscriberReference'; 5 | 6 | export function createDummySubscriber( 7 | userId: UserId | null = null 8 | ): Subscriber { 9 | if (!userId) { 10 | userId = createDummyUserId(); 11 | } 12 | return Subscriber.create({ 13 | userId: userId, 14 | SubscriberReference: SubscriberReference.create(), 15 | registrationDate: new Date() 16 | }); 17 | } 18 | 19 | export function createDummyUserId(): UserId { 20 | const userIdResult = UserId.create(randomUUID()); 21 | if (userIdResult.isErr()) throw userIdResult.error; 22 | return userIdResult.value; 23 | } 24 | -------------------------------------------------------------------------------- /src/notifications/infrastructure/di/di-types.ts: -------------------------------------------------------------------------------- 1 | export const TYPES = { 2 | MessageCreator: Symbol('MessageCreator'), 3 | EventSourceService: Symbol('EventSourceService') 4 | }; 5 | -------------------------------------------------------------------------------- /src/notifications/infrastructure/templates/Readme.md: -------------------------------------------------------------------------------- 1 | Templates based on https://github.com/mailgun/transactional-email-templates (MIT Licence) 2 | 3 | -------------------------------------------------------------------------------- /src/notifications/use-cases/confirm-subscription/ConfirmSubscriptionDTO.ts: -------------------------------------------------------------------------------- 1 | export interface ConfirmSubscriptionDTO { 2 | pendingSubscriptionId: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/notifications/use-cases/confirm-subscription/ConfirmSubscriptionError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class ConfirmSubscriptionError extends CustomError { 4 | errorType = 'ConfirmSubscriptionError'; 5 | } 6 | 7 | export class NoPendingSubscriptionFound extends ConfirmSubscriptionError { 8 | constructor() { 9 | super(`No pending subscription found`, NoPendingSubscriptionFound.name); 10 | } 11 | } 12 | 13 | export class PersistenceError extends ConfirmSubscriptionError { 14 | constructor(cause: Error) { 15 | super(`Persistence error`, PersistenceError.name, cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/notifications/use-cases/determine-events-and-notify-subscribers/NotifyDTO.ts: -------------------------------------------------------------------------------- 1 | export class NotifyDTO { 2 | constructor(public networkUpdateTime: Date) {} 3 | } 4 | -------------------------------------------------------------------------------- /src/notifications/use-cases/request-unsubscribe-link/RequestUnsubscribeLinkDTO.ts: -------------------------------------------------------------------------------- 1 | export interface EventSourceIdDTO { 2 | type: 'node' | 'organization' | 'network'; 3 | id: string; 4 | } 5 | export class RequestUnsubscribeLinkDTO { 6 | constructor( 7 | public readonly emailAddress: string, 8 | public readonly time: Date 9 | ) {} 10 | } 11 | -------------------------------------------------------------------------------- /src/notifications/use-cases/subscribe/SubscribeDTO.ts: -------------------------------------------------------------------------------- 1 | export interface EventSourceIdDTO { 2 | type: 'node' | 'organization' | 'network'; 3 | id: string; 4 | } 5 | export class SubscribeDTO { 6 | constructor( 7 | public readonly emailAddress: string, 8 | public readonly eventSourceIds: EventSourceIdDTO[], 9 | public readonly time: Date 10 | ) {} 11 | } 12 | -------------------------------------------------------------------------------- /src/notifications/use-cases/subscribe/SubscribeError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class SubscribeError extends CustomError { 4 | errorType = 'SubscribeError'; 5 | } 6 | 7 | export class PersistenceError extends SubscribeError { 8 | constructor(cause: Error) { 9 | super(`Persistence error`, PersistenceError.name, cause); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/notifications/use-cases/unmute-notification/UnmuteNotificationDTO.ts: -------------------------------------------------------------------------------- 1 | export interface UnmuteNotificationDTO { 2 | subscriberReference: string; 3 | eventSourceType: 'node' | 'organization' | 'network'; 4 | eventSourceId: string; 5 | eventType: string; 6 | } 7 | -------------------------------------------------------------------------------- /src/notifications/use-cases/unmute-notification/UnmuteNotificationError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | 3 | export class UnmuteNotificationError extends CustomError { 4 | errorType = 'UnmuteNotificationError'; 5 | } 6 | 7 | export class PersistenceError extends UnmuteNotificationError { 8 | constructor(cause: Error) { 9 | super(`Persistence error`, PersistenceError.name, cause); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/notifications/use-cases/unsubscribe/UnsubscribeDTO.ts: -------------------------------------------------------------------------------- 1 | export interface UnsubscribeDTO { 2 | subscriberReference: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/notifications/use-cases/unsubscribe/UnsubscribeError.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from '../../../core/errors/CustomError'; 2 | import { UnmuteNotificationError } from '../unmute-notification/UnmuteNotificationError'; 3 | 4 | export class UnsubscribeError extends CustomError { 5 | errorType = 'UnsubscribeError'; 6 | } 7 | 8 | export class SubscriberNotFoundError extends UnsubscribeError { 9 | constructor(subscriberRef: string) { 10 | super( 11 | `No subscriber found with id ${subscriberRef}`, 12 | SubscriberNotFoundError.name 13 | ); 14 | } 15 | } 16 | 17 | export class PersistenceError extends UnmuteNotificationError { 18 | constructor(cause: Error) { 19 | super(`Persistence error`, PersistenceError.name, cause); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "rootDirs": ["src"], 4 | "target": "ES2016", 5 | "module": "commonjs", 6 | "outDir": "lib", 7 | "types": ["node", "jest", "reflect-metadata"], 8 | "noUnusedParameters": false, 9 | "emitDecoratorMetadata": true, 10 | "experimentalDecorators": true, 11 | "strict": true, 12 | }, 13 | "include": [ 14 | "src/**/*.ts" 15 | ], 16 | "exclude": [ 17 | "node_modules/**/*", 18 | "src/**/__tests__/*", 19 | "src/**/__mocks__/*", 20 | "src/**/__fixtures__/*", 21 | ] 22 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "compilerOptions": { 4 | "sourceMap": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tsconfig.prod.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "compilerOptions": { 4 | "sourceMap": false 5 | } 6 | } --------------------------------------------------------------------------------