├── .dockerignore ├── .editorconfig ├── .github └── CODEOWNERS ├── .gitignore ├── .vscode └── settings.json ├── .vsconfig ├── ARCHITECTURE.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Directory.Build.props ├── Directory.Packages.props ├── LICENSE.txt ├── NuGet.Insights.sln ├── NuGet.config ├── README.md ├── SECURITY.md ├── TESTING.md ├── azurepipelines-coverage.yml ├── deploy ├── bicep │ ├── alerts.bicep │ ├── function-worker-vnet.bicep │ ├── function-worker.bicep │ ├── function-workers.bicep │ ├── key-vault.bicep │ ├── main.bicep │ ├── permissions.bicep │ ├── spot-worker-vnet.bicep │ ├── spot-worker-vnets.bicep │ ├── spot-worker.bicep │ ├── spot-workers.bicep │ ├── storage-network-acls.bicep │ ├── storage.bicep │ ├── vnets.bicep │ └── website.bicep ├── build-ev2.ps1 ├── build-host.ps1 ├── config │ ├── dev-spot.json │ ├── dev.json │ ├── ev2-dev-usnc.json │ ├── msdn-s1.json │ └── msdn-y1.json ├── deploy.ps1 ├── host-version.ps1 ├── prepare.ps1 └── scripts │ ├── Initialize-AadApp.ps1 │ ├── Initialize-AadAppForWebsite.ps1 │ ├── Install-WorkerStandalone.ps1 │ ├── Invoke-Deploy.ps1 │ ├── Invoke-Prepare.ps1 │ ├── NuGet.Insights.psm1 │ └── Set-DeploymentParameters.ps1 ├── docs ├── README.md ├── admin-panel.png ├── azure-resources.png ├── blog-posts.md ├── cost.md ├── drivers │ ├── BuildVersionSet.md │ ├── CatalogDataToCsv.md │ ├── LoadBucketedPackage.md │ ├── LoadLatestPackageLeaf.md │ ├── LoadPackageArchive.md │ ├── LoadPackageManifest.md │ ├── LoadPackageReadme.md │ ├── LoadPackageVersion.md │ ├── LoadSymbolPackageArchive.md │ ├── NuGetPackageExplorerToCsv.md │ ├── PackageArchiveToCsv.md │ ├── PackageAssemblyToCsv.md │ ├── PackageAssetToCsv.md │ ├── PackageCertificateToCsv.md │ ├── PackageCompatibilityToCsv.md │ ├── PackageContentToCsv.md │ ├── PackageFileToCsv.md │ ├── PackageIconToCsv.md │ ├── PackageLicenseToCsv.md │ ├── PackageManifestToCsv.md │ ├── PackageReadmeToCsv.md │ ├── PackageSignatureToCsv.md │ ├── PackageVersionToCsv.md │ ├── README.md │ ├── SymbolPackageArchiveToCsv.md │ └── SymbolPackageFileToCsv.md ├── find-package-files-execution-units.png ├── find-package-files-exucution-count.png ├── local-azure-function.png ├── local-results.png ├── new-driver.md ├── quickstart-admin.png ├── reusable-classes.md └── tables │ ├── CatalogLeafItems.md │ ├── Certificates.md │ ├── ExcludedPackages.md │ ├── GitHubUsages.md │ ├── NuGetPackageExplorerFiles.md │ ├── NuGetPackageExplorers.md │ ├── PackageArchiveEntries.md │ ├── PackageArchives.md │ ├── PackageAssemblies.md │ ├── PackageAssets.md │ ├── PackageCertificates.md │ ├── PackageCompatibilities.md │ ├── PackageContents.md │ ├── PackageDeprecations.md │ ├── PackageDownloads.md │ ├── PackageFiles.md │ ├── PackageIcons.md │ ├── PackageLicenses.md │ ├── PackageManifests.md │ ├── PackageOwners.md │ ├── PackageReadmes.md │ ├── PackageSignatures.md │ ├── PackageVersions.md │ ├── PackageVulnerabilities.md │ ├── PopularityTransfers.md │ ├── README.md │ ├── SymbolPackageArchiveEntries.md │ ├── SymbolPackageArchives.md │ ├── SymbolPackageFiles.md │ └── VerifiedPackages.md ├── global.json ├── markdown-link-check.config.json ├── package-lock.json ├── package.json ├── scripts ├── Add-CurrentUserToStorage.ps1 ├── Get-PerformanceReport.ps1 ├── Invoke-AzureFunctionsHostUpdate.ps1 ├── Invoke-Format.ps1 ├── Invoke-MarkdownValidation.ps1 ├── Invoke-PerformanceTest.ps1 ├── Kusto │ ├── GetAdjacentCommitTimestamps.kql │ ├── GetBucket.kql │ ├── ImportTo-Kusto.ps1 │ └── compare.kql ├── Set-LocalConfig.ps1 └── Wait-StorageEmulator.ps1 ├── src ├── Forks │ ├── 0001-Add-StringComparison-to-IndexOf.patch │ ├── 0002-Remove-Package-make-FrameworkName-settable-in-Packag.patch │ ├── 0003-Add-type-parameter-to-CertificateVerificationResult-.patch │ ├── 0004-Remove-unused-code-from-Utils.cs.patch │ ├── 0005-Remove-unnecessary-usings-in-Forks-Utils.cs.patch │ ├── 0006-Make-AccessTokenCache-reusable.patch │ ├── Azure │ │ └── azure-sdk-for-net │ │ │ ├── LICENSE.txt │ │ │ └── sdk │ │ │ └── core │ │ │ └── Azure.Core │ │ │ └── src │ │ │ ├── Pipeline │ │ │ └── BearerTokenAuthenticationPolicy.cs │ │ │ └── Shared │ │ │ ├── Argument.cs │ │ │ ├── AuthorizationChallengeParser.cs │ │ │ └── TaskExtensions.cs │ ├── Forks.csproj │ ├── NuGet │ │ └── NuGetGallery │ │ │ ├── LICENSE.txt │ │ │ └── src │ │ │ ├── Catalog │ │ │ └── Helpers │ │ │ │ └── Utils.cs │ │ │ ├── NuGet.Services.Entities │ │ │ ├── IEntity.cs │ │ │ └── PackageFramework.cs │ │ │ ├── NuGet.Services.Validation │ │ │ └── Entities │ │ │ │ ├── EndCertificateStatus.cs │ │ │ │ └── EndCertificateUse.cs │ │ │ ├── NuGetGallery.Core │ │ │ ├── Frameworks │ │ │ │ ├── FrameworkCompatibilityService.cs │ │ │ │ ├── FrameworkProductNames.cs │ │ │ │ ├── IPackageFrameworkCompatibilityFactory.cs │ │ │ │ ├── PackageFrameworkCompatibility.cs │ │ │ │ ├── PackageFrameworkCompatibilityBadges.cs │ │ │ │ ├── PackageFrameworkCompatibilityData.cs │ │ │ │ ├── PackageFrameworkCompatibilityFactory.cs │ │ │ │ └── SupportedFrameworks.cs │ │ │ └── Services │ │ │ │ └── AssetFrameworkHelper.cs │ │ │ └── Validation.PackageSigning.ValidateCertificate │ │ │ ├── CertificateVerificationException.cs │ │ │ ├── CertificateVerificationResult.cs │ │ │ ├── ICertificateVerifier.cs │ │ │ ├── OnlineCertificateVerifier.cs │ │ │ └── Primitives.cs │ ├── README.md │ ├── dotnet │ │ └── aspnetcore │ │ │ ├── LICENSE.txt │ │ │ └── src │ │ │ ├── Identity │ │ │ └── Extensions.Core │ │ │ │ └── src │ │ │ │ └── Base32.cs │ │ │ └── Shared │ │ │ └── ThrowHelpers │ │ │ └── ArgumentNullThrowHelper.cs │ └── download.ps1 ├── Logic │ ├── AzureStorage │ │ ├── AccessTokenCache.cs │ │ ├── AsyncPageableExtensions.cs │ │ ├── CachingTokenCredential.cs │ │ ├── CaseInsensitiveDictionary.cs │ │ ├── CaseInsensitiveSortedDictionary.cs │ │ ├── ContainerInitializationState.cs │ │ ├── EntityChangeResult.cs │ │ ├── EntityUpsertStorageService.cs │ │ ├── IBlobClientFactory.cs │ │ ├── IQueueClientFactory.cs │ │ ├── ITableClientFactory.cs │ │ ├── MutableTableTransactionalBatch.cs │ │ ├── ServiceClientFactory.cs │ │ ├── StorageCredentialType.cs │ │ ├── StorageExtensions.cs │ │ ├── StorageId.cs │ │ ├── StorageUtility.cs │ │ ├── TableEntityExtensions.cs │ │ ├── TableEntitySizeCalculator.cs │ │ ├── TableExtensions.cs │ │ └── TableTransactionOperation.cs │ ├── Data │ │ ├── BucketKeyAttribute.cs │ │ ├── CatalogLeafType.cs │ │ ├── CsvRecordAttribute.cs │ │ ├── CsvUtility.cs │ │ ├── ICsvRecord.cs │ │ ├── KustoIgnoreAttribute.cs │ │ ├── KustoPartitionKeyAttribute.cs │ │ ├── KustoTypeAttribute.cs │ │ ├── NoKustoDDLAttribute.cs │ │ ├── PackageIdentity.cs │ │ └── SemVerType.cs │ ├── FileSystemHttpCache │ │ ├── FileSystemHttpCacheEntryType.cs │ │ ├── FileSystemHttpCacheHandler.cs │ │ ├── FileSystemHttpCacheIntegrationTestSettings.cs │ │ ├── FileSystemHttpCacheKey.cs │ │ ├── FileSystemHttpCacheMode.cs │ │ └── FileSystemHttpCacheSettings.cs │ ├── Instrumentation │ │ ├── IMetric.cs │ │ ├── ITelemetryClient.cs │ │ ├── LoggerExtensions.cs │ │ ├── LoggerMetric.cs │ │ ├── LoggerTelemetryClient.cs │ │ ├── MetricNames.cs │ │ ├── MetricWrapper.cs │ │ ├── QueryLoopMetrics.cs │ │ ├── StandardToNuGetLogger.cs │ │ ├── TelemetryClientExtensions.cs │ │ └── TelemetryClientWrapper.cs │ ├── Leasing │ │ ├── AutoRenewingStorageLeaseResult.cs │ │ ├── AutoRenewingStorageLeaseService.cs │ │ ├── BaseLeaseResult.cs │ │ ├── StorageLeaseException.cs │ │ ├── StorageLeaseResult.cs │ │ ├── StorageLeaseService.cs │ │ └── StorageSemaphoreLeaseService.cs │ ├── Logic.csproj │ ├── MemoryStorage │ │ ├── ClientOptionsExtensions.cs │ │ ├── Clients │ │ │ ├── MemoryBlobClient.cs │ │ │ ├── MemoryBlobContainerClient.Response.cs │ │ │ ├── MemoryBlobContainerClient.cs │ │ │ ├── MemoryBlobLeaseClient.cs │ │ │ ├── MemoryBlobServiceClient.Response.cs │ │ │ ├── MemoryBlobServiceClient.cs │ │ │ ├── MemoryBlobStoreExtensions.Helper.cs │ │ │ ├── MemoryBlobStoreExtensions.Response.cs │ │ │ ├── MemoryBlockBlobClient.cs │ │ │ ├── MemoryQueueClient.Response.cs │ │ │ ├── MemoryQueueClient.cs │ │ │ ├── MemoryQueueServiceClient.Response.cs │ │ │ ├── MemoryQueueServiceClient.cs │ │ │ ├── MemoryTableClient.Response.cs │ │ │ ├── MemoryTableClient.cs │ │ │ ├── MemoryTableServiceClient.Response.cs │ │ │ └── MemoryTableServiceClient.cs │ │ ├── DateTimeOffsetExtensions.cs │ │ ├── MemoryResponse.cs │ │ ├── MemoryTokenCredential.cs │ │ ├── StorageResult.cs │ │ ├── Stores │ │ │ ├── MemoryBlobContainerStore.cs │ │ │ ├── MemoryBlobServiceStore.cs │ │ │ ├── MemoryBlobStore.cs │ │ │ ├── MemoryQueueMessageStore.cs │ │ │ ├── MemoryQueueServiceStore.cs │ │ │ ├── MemoryQueueStore.cs │ │ │ ├── MemoryTableServiceStore.cs │ │ │ └── MemoryTableStore.cs │ │ └── UriExtensions.cs │ ├── MessagePack │ │ ├── CsvRecordFormatterResolver.cs │ │ ├── NuGetInsightsFormatterResolver.cs │ │ ├── NuGetInsightsMessagePack.cs │ │ ├── ReadableKey.cs │ │ └── ReadableKeyDictionaryFormatter.cs │ ├── Network │ │ ├── DirectoryHelper.cs │ │ ├── HttpClientExtensions.cs │ │ ├── HttpMessageExtensions.cs │ │ ├── INuGetInsightsHttpMessageHandlerFactory.cs │ │ ├── IThrottle.cs │ │ ├── LoggingHttpHandler.cs │ │ ├── NullThrottle.cs │ │ ├── RedirectResolver.cs │ │ ├── RetryHttpMessageHandler.cs │ │ ├── SemaphoreSlimThrottle.cs │ │ ├── TelemetryHttpHandler.cs │ │ └── TimeoutHttpMessageHandler.cs │ ├── NuGetInsightsSettings.cs │ ├── PackageStorage │ │ ├── PackageFileService.cs │ │ ├── PackageHashService.cs │ │ ├── PackageManifestService.cs │ │ ├── PackageReadmeService.cs │ │ ├── PackageSpecificHashService.cs │ │ ├── PackageWideEntityService.cs │ │ ├── SymbolPackageFileService.cs │ │ └── SymbolPackageHashService.cs │ ├── Protocol │ │ ├── CatalogClient.cs │ │ ├── CatalogCommitTimestampProvider.cs │ │ ├── Data │ │ │ ├── AsOfData.cs │ │ │ ├── CatalogIndex.cs │ │ │ ├── CatalogLeaf.cs │ │ │ ├── CatalogLeafItem.cs │ │ │ ├── CatalogModelExtensions.cs │ │ │ ├── CatalogPackageDependency.cs │ │ │ ├── CatalogPackageDependencyGroup.cs │ │ │ ├── CatalogPage.cs │ │ │ ├── CatalogPageItem.cs │ │ │ ├── ExcludedPackage.cs │ │ │ ├── GitHubRepositoryInfo.cs │ │ │ ├── IAsOfData.cs │ │ │ ├── ICatalogLeafItem.cs │ │ │ ├── IPackageIdentityCommit.cs │ │ │ ├── PackageDeleteCatalogLeaf.cs │ │ │ ├── PackageDeprecation.cs │ │ │ ├── PackageDeprecationAlternatePackage.cs │ │ │ ├── PackageDetailsCatalogLeaf.cs │ │ │ ├── PackageDownloads.cs │ │ │ ├── PackageEntry.cs │ │ │ ├── PackageIdentityCommit.cs │ │ │ ├── PackageOwner.cs │ │ │ ├── PackageVulnerability.cs │ │ │ ├── PopularityTransfer.cs │ │ │ ├── ReadmeType.cs │ │ │ ├── Serialization │ │ │ │ ├── AssumeUniversalDateTimeConverter.cs │ │ │ │ ├── AssumeUniversalDateTimeOffsetConverter.cs │ │ │ │ ├── BaseCatalogLeafConverter.cs │ │ │ │ ├── CatalogLeafItemTypeConverter.cs │ │ │ │ ├── CatalogLeafTypeConverter.cs │ │ │ │ ├── CatalogPackageDependencyRangeConverter.cs │ │ │ │ └── PackageEntryLongConverter.cs │ │ │ ├── ServiceIndexTypes.cs │ │ │ └── VerifiedPackage.cs │ │ ├── ExcludedPackagesClient.cs │ │ ├── ExternalBlobStorageClient.cs │ │ ├── FileDownloader.cs │ │ ├── FlatContainerClient.cs │ │ ├── GitHubUsageClient.cs │ │ ├── IRemoteCursorClient.cs │ │ ├── PackageDownloadsClient.cs │ │ ├── PackageOwnersClient.cs │ │ ├── PopularityTransfersClient.cs │ │ ├── RemoteCursorClient.cs │ │ ├── ServiceIndexCache.cs │ │ ├── SymbolPackageClient.cs │ │ ├── ThrottledDisposable.cs │ │ └── VerifiedPackagesClient.cs │ ├── ReferenceTracking │ │ ├── IReference.cs │ │ ├── OwnerReference.cs │ │ ├── OwnerToSubjectEdges.cs │ │ ├── ReferenceTracker.cs │ │ ├── ReferenceTypes.cs │ │ ├── SubjectEdge.cs │ │ └── SubjectReference.cs │ ├── ServiceCollectionExtensions.cs │ ├── StorageNoOpRetry │ │ ├── DefaultRetryContext.cs │ │ ├── ITableEntityWithClientRequestId.cs │ │ ├── RetryContext.cs │ │ ├── StorageNoOpRetryPolicy.cs │ │ ├── StubResponse.cs │ │ ├── TableClientWithRetryContext.cs │ │ ├── TableRetryBatchContext.cs │ │ ├── TableRetryContext.cs │ │ ├── TableRetryEntityContext.cs │ │ └── TableServiceClientWithRetryContext.cs │ ├── StorageSettings.cs │ ├── Support │ │ ├── AssemblyExtensions.cs │ │ ├── ByteArrayExtensions.cs │ │ ├── CountingWriterStream.cs │ │ ├── DateTimeOffsetExtensions.cs │ │ ├── EnumerableExtensions.cs │ │ ├── HashOutput.cs │ │ ├── IIncrementalHash.cs │ │ ├── IncrementalHash.cs │ │ ├── LimitStream.cs │ │ ├── LimitedConcurrentQueue.cs │ │ ├── NullAsyncDisposable.cs │ │ ├── PathUtility.cs │ │ ├── ReferenceEqualityComparer.cs │ │ ├── UriExtensions.cs │ │ ├── Utf8JsonReaderExtensions.cs │ │ └── XmlUtility.cs │ ├── TablePrefixScan │ │ ├── TablePrefixScanEntitySegment.cs │ │ ├── TablePrefixScanPartitionKeyQuery.cs │ │ ├── TablePrefixScanPrefixQuery.cs │ │ ├── TablePrefixScanStart.cs │ │ ├── TablePrefixScanStep.cs │ │ ├── TablePrefixScanner.cs │ │ └── TableQueryParameters.cs │ ├── TempStream │ │ ├── SlowStreamExtensions.cs │ │ ├── TempStreamDirectory.cs │ │ ├── TempStreamDirectoryLeaseService.cs │ │ ├── TempStreamResult.cs │ │ ├── TempStreamResultType.cs │ │ ├── TempStreamService.cs │ │ └── TempStreamWriter.cs │ ├── Timers │ │ ├── ITimer.cs │ │ ├── SpecificTimerExecutionService.cs │ │ ├── TimerEntity.cs │ │ ├── TimerExecutionService.cs │ │ ├── TimerFrequency.cs │ │ └── TimerState.cs │ └── WideEntities │ │ ├── ChunkStream.cs │ │ ├── WideEntity.cs │ │ ├── WideEntityDeleteOperation.cs │ │ ├── WideEntityInsertOperation.cs │ │ ├── WideEntityInsertOrReplaceOperation.cs │ │ ├── WideEntityOperation.cs │ │ ├── WideEntityReplaceOperation.cs │ │ ├── WideEntitySegment.cs │ │ └── WideEntityService.cs ├── Performance │ ├── Data │ │ └── packagefiles_compact_3.csv.br │ ├── PackageDownloads_Compact_Fake.cs │ ├── PackageFiles_Compact_Real.cs │ ├── Performance.csproj │ └── Program.cs ├── SourceGenerator │ ├── Constants.cs │ ├── CsvPropertyModel.cs │ ├── CsvRecordGenerator.cs │ ├── CsvRecordModel.cs │ ├── CsvRecordResult.cs │ ├── DiagnosticIds.cs │ ├── EquatableList.cs │ ├── GetBucketKeyProperty.cs │ ├── IPropertyVisitor.cs │ ├── IsExternalInit.cs │ ├── KustoMappingBuilder.cs │ ├── KustoPartitioningPolicyBuilder.cs │ ├── KustoTableBuilder.cs │ ├── MessagePackFormatterDeserializeBuilder.cs │ ├── MessagePackFormatterSerializeBuilder.cs │ ├── PropertyHelper.cs │ ├── ReadBuilder.cs │ ├── SetEmptyStringsBuilder.cs │ ├── SourceGenerator.csproj │ ├── ValidatePropertyNullability.cs │ ├── WriteAsyncTextWriterBuilder.cs │ ├── WriteHeaderBuilder.cs │ ├── WriteListBuilder.cs │ └── WriteTextWriterBuilder.cs ├── Tool │ ├── CommandExecutor.cs │ ├── Commands │ │ ├── AnalyzeHttpCacheCommand.cs │ │ ├── ICommand.cs │ │ ├── IngestDownloadsJsonCommand.cs │ │ ├── ProcessMessagesCommand.cs │ │ └── SandboxCommand.cs │ ├── Program.cs │ ├── Support │ │ ├── ConsoleUtility.cs │ │ ├── MinimalConsoleLogger.cs │ │ ├── MinimalConsoleLoggerExtensions.cs │ │ └── MinimalConsoleLoggerProvider.cs │ ├── Tool.csproj │ ├── appsettings.Development.json │ └── appsettings.json ├── Website │ ├── Authorization │ │ ├── AllowListAuthorizationHandler.cs │ │ ├── AllowListRequirement.cs │ │ └── AllowedObject.cs │ ├── Components │ │ ├── CatalogScanCard.razor │ │ ├── CatalogScansCard.razor │ │ ├── CollapseCard.razor │ │ ├── KustoIngestionsCard.razor │ │ ├── TimedReprocessRunsCard.razor │ │ ├── TimerCard.razor │ │ ├── WorkerQueueCard.razor │ │ └── WorkflowRunsCard.razor │ ├── Controllers │ │ ├── AdminController.cs │ │ └── HomeController.cs │ ├── ExtensionMethods.cs │ ├── HostedServices │ │ ├── CachedAdminViewModelService.cs │ │ ├── InitializerHostedService.cs │ │ ├── MoveMessagesHostedService.cs │ │ └── MoveMessagesTaskQueue.cs │ ├── Models │ │ ├── AdminViewModel.cs │ │ ├── CachedAdminViewModel.cs │ │ ├── CatalogScanViewModel.cs │ │ ├── ErrorViewModel.cs │ │ ├── IAdminViewModelCache.cs │ │ ├── MoveQueueMessagesState.cs │ │ ├── QueueViewModel.cs │ │ ├── TimedReprocessRunViewModel.cs │ │ ├── TimedReprocessViewModel.cs │ │ └── ViewModelFactory.cs │ ├── NuGetInsightsWebsiteSettings.cs │ ├── Program.cs │ ├── ServiceCollectionExtensions.cs │ ├── Startup.cs │ ├── Views │ │ ├── Admin │ │ │ └── Index.cshtml │ │ ├── Home │ │ │ ├── AccessDenied.cshtml │ │ │ └── Index.cshtml │ │ ├── Shared │ │ │ ├── Error.cshtml │ │ │ ├── _Layout.cshtml │ │ │ └── _ValidationScriptsPartial.cshtml │ │ ├── _ViewImports.cshtml │ │ └── _ViewStart.cshtml │ ├── Website.csproj │ ├── _Imports.razor │ ├── appsettings.Development.json │ ├── appsettings.json │ └── wwwroot │ │ ├── css │ │ └── site.css │ │ ├── favicon.ico │ │ ├── js │ │ └── site.js │ │ └── lib │ │ ├── bootstrap-icons │ │ ├── LICENSE │ │ ├── bootstrap-icons.css │ │ └── fonts │ │ │ ├── bootstrap-icons.woff │ │ │ └── bootstrap-icons.woff2 │ │ ├── bootstrap │ │ ├── LICENSE │ │ └── dist │ │ │ ├── css │ │ │ ├── bootstrap-grid.css │ │ │ ├── bootstrap-grid.css.map │ │ │ ├── bootstrap-grid.min.css │ │ │ ├── bootstrap-grid.min.css.map │ │ │ ├── bootstrap-reboot.css │ │ │ ├── bootstrap-reboot.css.map │ │ │ ├── bootstrap-reboot.min.css │ │ │ ├── bootstrap-reboot.min.css.map │ │ │ ├── bootstrap.css │ │ │ ├── bootstrap.css.map │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.css.map │ │ │ └── js │ │ │ ├── bootstrap.bundle.js │ │ │ ├── bootstrap.bundle.js.map │ │ │ ├── bootstrap.bundle.min.js │ │ │ ├── bootstrap.bundle.min.js.map │ │ │ ├── bootstrap.js │ │ │ ├── bootstrap.js.map │ │ │ ├── bootstrap.min.js │ │ │ └── bootstrap.min.js.map │ │ ├── date-fns │ │ └── dist │ │ │ ├── LICENSE.txt │ │ │ ├── date-fns.js │ │ │ ├── date-fns.js.map │ │ │ └── date-fns.min.js │ │ └── jquery │ │ ├── LICENSE │ │ └── dist │ │ ├── jquery.js │ │ ├── jquery.min.js │ │ └── jquery.min.map ├── Worker.Logic │ ├── AppendResults │ │ ├── AppendResultStorageService.cs │ │ ├── CsvReaderAdapter.cs │ │ ├── CsvReaderResult.cs │ │ ├── CsvReaderResultType.cs │ │ ├── CsvRecordBlob.cs │ │ ├── CsvRecordContainers.cs │ │ ├── CsvRecordProducer.cs │ │ ├── CsvRecordProducerType.cs │ │ ├── IAggregatedCsvRecord.cs │ │ └── ICsvReader.cs │ ├── CatalogScan │ │ ├── ArchiveEntryHashToCsv │ │ │ ├── FileRecord.cs │ │ │ ├── FileRecordResultType.cs │ │ │ └── FullZipArchiveEntryToCsvDriver.cs │ │ ├── ArchiveToCsv │ │ │ ├── ArchiveEntry.cs │ │ │ ├── ArchiveRecord.cs │ │ │ ├── ArchiveResultType.cs │ │ │ └── ZipArchiveMetadataToCsvDriver.cs │ │ ├── CatalogIndexScan.cs │ │ ├── CatalogIndexScanMessage.cs │ │ ├── CatalogIndexScanMessageProcessor.cs │ │ ├── CatalogIndexScanResult.cs │ │ ├── CatalogIndexScanState.cs │ │ ├── CatalogLeafScan.cs │ │ ├── CatalogLeafScanBatchDriverAdapter.cs │ │ ├── CatalogLeafScanMessage.cs │ │ ├── CatalogLeafScanMessageProcessor.cs │ │ ├── CatalogPageScan.cs │ │ ├── CatalogPageScanMessage.cs │ │ ├── CatalogPageScanMessageProcessor.cs │ │ ├── CatalogPageScanResult.cs │ │ ├── CatalogPageScanState.cs │ │ ├── CatalogScanCursorService.cs │ │ ├── CatalogScanDriverFactory.cs │ │ ├── CatalogScanDriverMetadata.Drivers.CryptoAPI.cs │ │ ├── CatalogScanDriverMetadata.Drivers.NPE.cs │ │ ├── CatalogScanDriverMetadata.Drivers.cs │ │ ├── CatalogScanDriverMetadata.Implementation.cs │ │ ├── CatalogScanDriverType.Drivers.CryptoAPI.cs │ │ ├── CatalogScanDriverType.Drivers.Internal.cs │ │ ├── CatalogScanDriverType.Drivers.NPE.cs │ │ ├── CatalogScanDriverType.Drivers.cs │ │ ├── CatalogScanDriverType.Struct.cs │ │ ├── CatalogScanExpandService.cs │ │ ├── CatalogScanService.cs │ │ ├── CatalogScanServiceResult.cs │ │ ├── CatalogScanServiceResultType.cs │ │ ├── CatalogScanStorageService.cs │ │ ├── CatalogScanToCsv │ │ │ ├── CatalogLeafToCsv │ │ │ │ ├── BaseCatalogLeafScanToCsvAdapter.cs │ │ │ │ ├── BaseCatalogLeafScanToCsvBatchAdapter.cs │ │ │ │ ├── BaseCatalogLeafScanToCsvNonBatchAdapter.cs │ │ │ │ ├── CatalogLeafScanToCsvBatchAdapter.cs │ │ │ │ ├── CatalogLeafScanToCsvNonBatchAdapter.cs │ │ │ │ ├── CatalogLeafToCsvMode.cs │ │ │ │ ├── CsvRecordSets.cs │ │ │ │ ├── ICatalogLeafToCsvBatchDriver.cs │ │ │ │ └── ICatalogLeafToCsvDriver.cs │ │ │ ├── CsvTemporaryStorageFactory.cs │ │ │ └── ICsvTemporaryStorage.cs │ │ ├── CatalogScanUpdateTimer.cs │ │ ├── DriverResult.cs │ │ ├── DriverResultType.cs │ │ ├── ICatalogLeafScanBatchDriver.cs │ │ ├── ICatalogLeafScanNonBatchDriver.cs │ │ ├── ICatalogScanDriver.cs │ │ ├── ICatalogScanDriverFactory.cs │ │ ├── LatestLeaf │ │ │ ├── FindLatestLeafDriver.cs │ │ │ ├── ILatestPackageLeaf.cs │ │ │ ├── ILatestPackageLeafStorage.cs │ │ │ ├── ILatestPackageLeafStorageFactory.cs │ │ │ └── LatestLeafStorageService.cs │ │ └── PackageFilter.cs │ ├── CsvRecordStorage │ │ ├── CsvRecordStorageService.cs │ │ └── ICsvRecordProvider.cs │ ├── Cursors │ │ ├── CursorStorageService.cs │ │ └── CursorTableEntity.cs │ ├── Drivers │ │ ├── BuildVersionSet │ │ │ ├── BuildVersionSetDriver.cs │ │ │ ├── CatalogLeafBatchData.cs │ │ │ ├── CatalogLeafItemData.cs │ │ │ ├── EntityHandle.cs │ │ │ ├── EntityReferenceCounter.cs │ │ │ ├── HistoryVersionSet.cs │ │ │ ├── IVersionSet.cs │ │ │ ├── IVersionSetProvider.cs │ │ │ ├── VersionSet.cs │ │ │ ├── VersionSetAggregateStorageService.cs │ │ │ └── VersionSetService.cs │ │ ├── CatalogDataToCsv │ │ │ ├── CatalogDataToCsvDriver.cs │ │ │ ├── CatalogLeafItemRecord.cs │ │ │ ├── PackageDeprecationRecord.cs │ │ │ ├── PackageDeprecationResultType.cs │ │ │ ├── PackageVulnerabilityRecord.cs │ │ │ └── PackageVulnerabilityResultType.cs │ │ ├── Internal_FindLatestCatalogLeafScan │ │ │ └── LatestCatalogLeafScanStorageFactory.cs │ │ ├── Internal_FindLatestCatalogLeafScanPerId │ │ │ ├── CatalogLeafScanPerId.cs │ │ │ └── LatestCatalogLeafScanPerIdStorageFactory.cs │ │ ├── LoadBucketedPackage │ │ │ ├── BucketRange.cs │ │ │ ├── BucketedPackage.cs │ │ │ ├── BucketedPackageService.cs │ │ │ └── BucketedPackageStorageFactory.cs │ │ ├── LoadLatestPackageLeaf │ │ │ ├── LatestPackageLeaf.cs │ │ │ ├── LatestPackageLeafService.cs │ │ │ └── LatestPackageLeafStorageFactory.cs │ │ ├── LoadPackageArchive │ │ │ └── LoadPackageArchiveDriver.cs │ │ ├── LoadPackageManifest │ │ │ └── LoadPackageManifestDriver.cs │ │ ├── LoadPackageReadme │ │ │ └── LoadPackageReadmeDriver.cs │ │ ├── LoadPackageVersion │ │ │ ├── LoadPackageVersionDriver.cs │ │ │ ├── PackageVersionEntity.cs │ │ │ └── PackageVersionStorageService.cs │ │ ├── LoadSymbolPackageArchive │ │ │ └── LoadSymbolPackageArchiveDriver.cs │ │ ├── NuGetPackageExplorerToCsv │ │ │ ├── GitHubSourceRepo.cs │ │ │ ├── InvalidSourceRepo.cs │ │ │ ├── NuGetPackageExplorerFile.cs │ │ │ ├── NuGetPackageExplorerRecord.cs │ │ │ ├── NuGetPackageExplorerResultType.cs │ │ │ ├── NuGetPackageExplorerToCsvDriver.cs │ │ │ ├── SourceUrlRepo.cs │ │ │ ├── SourceUrlRepoInfo.cs │ │ │ ├── SourceUrlRepoParser.cs │ │ │ ├── TemporaryFileProvider.cs │ │ │ └── UnknownSourceRepo.cs │ │ ├── PackageArchiveToCsv │ │ │ ├── PackageArchiveEntry.cs │ │ │ ├── PackageArchiveRecord.cs │ │ │ └── PackageArchiveToCsvDriver.cs │ │ ├── PackageAssemblyToCsv │ │ │ ├── AssemblyAttributeReader.cs │ │ │ ├── EntityHandleInfo.cs │ │ │ ├── PackageAssembly.cs │ │ │ ├── PackageAssemblyEdgeCases.cs │ │ │ ├── PackageAssemblyResultType.cs │ │ │ ├── PackageAssemblyToCsvDriver.cs │ │ │ ├── SimpleCustomAttributeTypeProvider.cs │ │ │ └── SimpleTypeInfo.cs │ │ ├── PackageAssetToCsv │ │ │ ├── PackageAsset.cs │ │ │ ├── PackageAssetResultType.cs │ │ │ ├── PackageAssetToCsvDriver.cs │ │ │ └── PatternSetType.cs │ │ ├── PackageCertificateToCsv │ │ │ ├── CertificateDataBuilder.cs │ │ │ ├── CertificateInfo.cs │ │ │ ├── CertificateRecord.cs │ │ │ ├── CertificateRelationshipTypes.cs │ │ │ ├── ChainInfo.cs │ │ │ ├── CleanupOrphanCertificateRecordsAdapter.cs │ │ │ ├── FingerprintKustoValidationProvider.cs │ │ │ ├── FingerprintSHA256KustoValidationProvider.cs │ │ │ ├── PackageCertificateRecord.cs │ │ │ ├── PackageCertificateResultType.cs │ │ │ ├── PackageCertificateToCsvDriver.cs │ │ │ ├── X509Certificate2Extensions.cs │ │ │ ├── X509Extensions │ │ │ │ ├── X509BasicConstraintsExtensionInfo.cs │ │ │ │ ├── X509EnhancedKeyUsageExtensionInfo.cs │ │ │ │ ├── X509ExtensionInfo.cs │ │ │ │ ├── X509ExtensionInfoFactory.cs │ │ │ │ ├── X509KeyUsageExtensionInfo.cs │ │ │ │ └── X509SubjectKeyIdentifierExtensionInfo.cs │ │ │ └── X509Policies │ │ │ │ ├── X509CpsPolicyQualifierInfo.cs │ │ │ │ ├── X509PolicyInfo.cs │ │ │ │ ├── X509PolicyQualifierInfo.cs │ │ │ │ └── X509PolicyQualifierInfoFactory.cs │ │ ├── PackageCompatibilityToCsv │ │ │ ├── CompatibilityChecker.cs │ │ │ ├── InMemoryPackageReader.cs │ │ │ ├── PackageCompatibility.cs │ │ │ ├── PackageCompatibilityResultType.cs │ │ │ └── PackageCompatibilityToCsvDriver.cs │ │ ├── PackageContentToCsv │ │ │ ├── PackageContent.cs │ │ │ ├── PackageContentResultType.cs │ │ │ └── PackageContentToCsvDriver.cs │ │ ├── PackageFileToCsv │ │ │ ├── PackageFileRecord.cs │ │ │ └── PackageFileToCsvDriver.cs │ │ ├── PackageIconToCsv │ │ │ ├── FormatDetector.cs │ │ │ ├── PackageIcon.cs │ │ │ ├── PackageIconResultType.cs │ │ │ └── PackageIconToCsvDriver.cs │ │ ├── PackageLicenseToCsv │ │ │ ├── PackageLicense.cs │ │ │ ├── PackageLicenseResultType.cs │ │ │ └── PackageLicenseToCsvDriver.cs │ │ ├── PackageManifestToCsv │ │ │ ├── PackageManifestRecord.cs │ │ │ ├── PackageManifestRecordResultType.cs │ │ │ └── PackageManifestToCsvDriver.cs │ │ ├── PackageReadmeToCsv │ │ │ ├── PackageReadme.cs │ │ │ ├── PackageReadmeResultType.cs │ │ │ └── PackageReadmeToCsvDriver.cs │ │ ├── PackageSignatureToCsv │ │ │ ├── PackageSignature.cs │ │ │ ├── PackageSignatureResultType.cs │ │ │ └── PackageSignatureToCsvDriver.cs │ │ ├── PackageVersionToCsv │ │ │ ├── PackageVersionRecord.cs │ │ │ ├── PackageVersionResultType.cs │ │ │ └── PackageVersionToCsvDriver.cs │ │ ├── SymbolPackageArchiveToCsv │ │ │ ├── SymbolPackageArchiveEntry.cs │ │ │ ├── SymbolPackageArchiveRecord.cs │ │ │ └── SymbolPackageArchiveToCsvDriver.cs │ │ └── SymbolPackageFileToCsv │ │ │ ├── SymbolPackageFileRecord.cs │ │ │ └── SymbolPackageFileToCsvDriver.cs │ ├── Generated │ │ └── NuGet.Insights.SourceGenerator │ │ │ └── NuGet.Insights.CsvRecordGenerator │ │ │ ├── CatalogLeafItemRecord.ICsvRecord.cs │ │ │ ├── CatalogLeafItemRecord.KustoDDL.cs │ │ │ ├── CatalogLeafItemRecord.MessagePack.cs │ │ │ ├── CertificateRecord.ICsvRecord.cs │ │ │ ├── CertificateRecord.KustoDDL.cs │ │ │ ├── CertificateRecord.MessagePack.cs │ │ │ ├── ExcludedPackageRecord.ICsvRecord.cs │ │ │ ├── ExcludedPackageRecord.KustoDDL.cs │ │ │ ├── ExcludedPackageRecord.MessagePack.cs │ │ │ ├── GitHubUsageRecord.ICsvRecord.cs │ │ │ ├── GitHubUsageRecord.KustoDDL.cs │ │ │ ├── GitHubUsageRecord.MessagePack.cs │ │ │ ├── NuGetInsightsWorkerLogicKustoDDL.cs │ │ │ ├── NuGetPackageExplorerFile.ICsvRecord.cs │ │ │ ├── NuGetPackageExplorerFile.KustoDDL.cs │ │ │ ├── NuGetPackageExplorerFile.MessagePack.cs │ │ │ ├── NuGetPackageExplorerRecord.ICsvRecord.cs │ │ │ ├── NuGetPackageExplorerRecord.KustoDDL.cs │ │ │ ├── NuGetPackageExplorerRecord.MessagePack.cs │ │ │ ├── PackageArchiveEntry.ICsvRecord.cs │ │ │ ├── PackageArchiveEntry.KustoDDL.cs │ │ │ ├── PackageArchiveEntry.MessagePack.cs │ │ │ ├── PackageArchiveRecord.ICsvRecord.cs │ │ │ ├── PackageArchiveRecord.KustoDDL.cs │ │ │ ├── PackageArchiveRecord.MessagePack.cs │ │ │ ├── PackageAssembly.ICsvRecord.cs │ │ │ ├── PackageAssembly.KustoDDL.cs │ │ │ ├── PackageAssembly.MessagePack.cs │ │ │ ├── PackageAsset.ICsvRecord.cs │ │ │ ├── PackageAsset.KustoDDL.cs │ │ │ ├── PackageAsset.MessagePack.cs │ │ │ ├── PackageCertificateRecord.ICsvRecord.cs │ │ │ ├── PackageCertificateRecord.KustoDDL.cs │ │ │ ├── PackageCertificateRecord.MessagePack.cs │ │ │ ├── PackageCompatibility.ICsvRecord.cs │ │ │ ├── PackageCompatibility.KustoDDL.cs │ │ │ ├── PackageCompatibility.MessagePack.cs │ │ │ ├── PackageContent.ICsvRecord.cs │ │ │ ├── PackageContent.KustoDDL.cs │ │ │ ├── PackageContent.MessagePack.cs │ │ │ ├── PackageDeprecationRecord.ICsvRecord.cs │ │ │ ├── PackageDeprecationRecord.KustoDDL.cs │ │ │ ├── PackageDeprecationRecord.MessagePack.cs │ │ │ ├── PackageDownloadHistoryRecord.ICsvRecord.cs │ │ │ ├── PackageDownloadHistoryRecord.MessagePack.cs │ │ │ ├── PackageDownloadRecord.ICsvRecord.cs │ │ │ ├── PackageDownloadRecord.KustoDDL.cs │ │ │ ├── PackageDownloadRecord.MessagePack.cs │ │ │ ├── PackageFileRecord.ICsvRecord.cs │ │ │ ├── PackageFileRecord.KustoDDL.cs │ │ │ ├── PackageFileRecord.MessagePack.cs │ │ │ ├── PackageIcon.ICsvRecord.cs │ │ │ ├── PackageIcon.KustoDDL.cs │ │ │ ├── PackageIcon.MessagePack.cs │ │ │ ├── PackageLicense.ICsvRecord.cs │ │ │ ├── PackageLicense.KustoDDL.cs │ │ │ ├── PackageLicense.MessagePack.cs │ │ │ ├── PackageManifestRecord.ICsvRecord.cs │ │ │ ├── PackageManifestRecord.KustoDDL.cs │ │ │ ├── PackageManifestRecord.MessagePack.cs │ │ │ ├── PackageOwnerRecord.ICsvRecord.cs │ │ │ ├── PackageOwnerRecord.KustoDDL.cs │ │ │ ├── PackageOwnerRecord.MessagePack.cs │ │ │ ├── PackageReadme.ICsvRecord.cs │ │ │ ├── PackageReadme.KustoDDL.cs │ │ │ ├── PackageReadme.MessagePack.cs │ │ │ ├── PackageSignature.ICsvRecord.cs │ │ │ ├── PackageSignature.KustoDDL.cs │ │ │ ├── PackageSignature.MessagePack.cs │ │ │ ├── PackageVersionRecord.ICsvRecord.cs │ │ │ ├── PackageVersionRecord.KustoDDL.cs │ │ │ ├── PackageVersionRecord.MessagePack.cs │ │ │ ├── PackageVulnerabilityRecord.ICsvRecord.cs │ │ │ ├── PackageVulnerabilityRecord.KustoDDL.cs │ │ │ ├── PackageVulnerabilityRecord.MessagePack.cs │ │ │ ├── PopularityTransfersRecord.ICsvRecord.cs │ │ │ ├── PopularityTransfersRecord.KustoDDL.cs │ │ │ ├── PopularityTransfersRecord.MessagePack.cs │ │ │ ├── SymbolPackageArchiveEntry.ICsvRecord.cs │ │ │ ├── SymbolPackageArchiveEntry.KustoDDL.cs │ │ │ ├── SymbolPackageArchiveEntry.MessagePack.cs │ │ │ ├── SymbolPackageArchiveRecord.ICsvRecord.cs │ │ │ ├── SymbolPackageArchiveRecord.KustoDDL.cs │ │ │ ├── SymbolPackageArchiveRecord.MessagePack.cs │ │ │ ├── SymbolPackageFileRecord.ICsvRecord.cs │ │ │ ├── SymbolPackageFileRecord.KustoDDL.cs │ │ │ ├── SymbolPackageFileRecord.MessagePack.cs │ │ │ ├── VerifiedPackageRecord.ICsvRecord.cs │ │ │ ├── VerifiedPackageRecord.KustoDDL.cs │ │ │ ├── VerifiedPackageRecord.MessagePack.cs │ │ │ ├── VsAssemblyHash.MessagePack.cs │ │ │ └── VsPackageHash.MessagePack.cs │ ├── IgnoredPackagePattern.cs │ ├── Kusto │ │ └── CachingKustoClientFactory.cs │ ├── LoggerTraceListener.cs │ ├── Management │ │ └── MetricsTimer.cs │ ├── MessageProcessors │ │ ├── AuxiliaryFileUpdater │ │ │ ├── AuxiliaryFileUpdaterMessage.cs │ │ │ ├── AuxiliaryFileUpdaterProcessor.cs │ │ │ ├── AuxiliaryFileUpdaterService.cs │ │ │ ├── AuxiliaryFileUpdaterTimer.cs │ │ │ ├── IAuxiliaryFileCsvRecord.cs │ │ │ ├── IAuxiliaryFileUpdater.cs │ │ │ ├── IAuxiliaryFileUpdaterService.cs │ │ │ └── IAuxiliaryFileUpdaterTimer.cs │ │ ├── BatchMessageProcessorResult.cs │ │ ├── CsvCompact │ │ │ ├── CsvCompactMessage.cs │ │ │ ├── CsvCompactProcessor.cs │ │ │ └── ICsvResultStorage.cs │ │ ├── DownloadsToCsv │ │ │ ├── DownloadsToCsvUpdater.cs │ │ │ ├── IPackageDownloadRecord.cs │ │ │ ├── PackageDownloadHistoryRecord.cs │ │ │ └── PackageDownloadRecord.cs │ │ ├── ExcludedPackagesToCsv │ │ │ ├── ExcludedPackageRecord.cs │ │ │ └── ExcludedPackagesToCsvUpdater.cs │ │ ├── FanOutRecoveryHelper.cs │ │ ├── GenericMessageProcessor.cs │ │ ├── GitHubUsageToCsv │ │ │ ├── GitHubUsageRecord.cs │ │ │ ├── GitHubUsageResultType.cs │ │ │ └── GitHubUsageToCsvUpdater.cs │ │ ├── HeterogeneousBulkEnqueueMessage.cs │ │ ├── HeterogeneousBulkEnqueueMessageProcessor.cs │ │ ├── HomogeneousBatchMessage.cs │ │ ├── HomogeneousBatchMessageProcessor.cs │ │ ├── HomogeneousBulkEnqueueMessage.cs │ │ ├── HomogeneousBulkEnqueueMessageProcessor.cs │ │ ├── IBatchMessageProcessor.cs │ │ ├── IGenericMessageProcessor.cs │ │ ├── IMessageProcessor.cs │ │ ├── ITaskStateMessage.cs │ │ ├── ITaskStateMessageProcessor.cs │ │ ├── KustoIngestion │ │ │ ├── BaseKustoValidationProvider.cs │ │ │ ├── IKustoValidationProvider.cs │ │ │ ├── KustoBlobIngestion.cs │ │ │ ├── KustoBlobIngestionMessage.cs │ │ │ ├── KustoBlobIngestionMessageProcessor.cs │ │ │ ├── KustoBlobIngestionState.cs │ │ │ ├── KustoContainerIngestion.cs │ │ │ ├── KustoContainerIngestionMessage.cs │ │ │ ├── KustoContainerIngestionMessageProcessor.cs │ │ │ ├── KustoContainerIngestionState.cs │ │ │ ├── KustoDataValidator.cs │ │ │ ├── KustoIngestionEntity.cs │ │ │ ├── KustoIngestionMessage.cs │ │ │ ├── KustoIngestionMessageProcessor.cs │ │ │ ├── KustoIngestionService.cs │ │ │ ├── KustoIngestionState.cs │ │ │ ├── KustoIngestionStorageService.cs │ │ │ ├── KustoIngestionTimer.cs │ │ │ ├── PackageIdentityKustoValidationProvider.cs │ │ │ └── PackageLowerIdKustoValidationProvider.cs │ │ ├── OwnersToCsv │ │ │ ├── OwnersToCsvUpdater.cs │ │ │ └── PackageOwnerRecord.cs │ │ ├── PopularityTransfersToCsv │ │ │ ├── PopularityTransfersRecord.cs │ │ │ └── PopularityTransfersToCsvUpdater.cs │ │ ├── ReferenceTracking │ │ │ ├── CleanupOrphanRecordsMessage.cs │ │ │ ├── CleanupOrphanRecordsParameters.cs │ │ │ ├── CleanupOrphanRecordsProcessor.cs │ │ │ ├── CleanupOrphanRecordsService.cs │ │ │ ├── CleanupOrphanRecordsState.cs │ │ │ ├── CleanupOrphanRecordsTimer.cs │ │ │ ├── ICleanupOrphanCsvRecord.cs │ │ │ ├── ICleanupOrphanRecordsAdapter.cs │ │ │ ├── ICleanupOrphanRecordsService.cs │ │ │ └── ICleanupOrphanRecordsTimer.cs │ │ ├── TaskStateMessageProcessor.cs │ │ ├── TaskStateProcessResult.cs │ │ ├── TimedReprocess │ │ │ ├── TimedReprocessBucket.cs │ │ │ ├── TimedReprocessCatalogScan.cs │ │ │ ├── TimedReprocessDetails.cs │ │ │ ├── TimedReprocessMessage.cs │ │ │ ├── TimedReprocessMessageProcessor.cs │ │ │ ├── TimedReprocessRun.cs │ │ │ ├── TimedReprocessService.cs │ │ │ ├── TimedReprocessState.cs │ │ │ ├── TimedReprocessStorageService.cs │ │ │ └── TimedReprocessTimer.cs │ │ ├── VerifiedPackagesToCsv │ │ │ ├── VerifiedPackageRecord.cs │ │ │ └── VerifiedPackagesToCsvUpdater.cs │ │ └── Workflow │ │ │ ├── WorkflowRun.cs │ │ │ ├── WorkflowRunMessage.cs │ │ │ ├── WorkflowRunMessageProcessor.cs │ │ │ ├── WorkflowRunState.cs │ │ │ ├── WorkflowService.cs │ │ │ ├── WorkflowStorageService.cs │ │ │ └── WorkflowTimer.cs │ ├── NuGetInsightsWorkerSettings.AggregateCsv.cs │ ├── NuGetInsightsWorkerSettings.Auxiliary.cs │ ├── NuGetInsightsWorkerSettings.CatalogScan.cs │ ├── NuGetInsightsWorkerSettings.Drivers.CryptoAPI.cs │ ├── NuGetInsightsWorkerSettings.Drivers.NPE.cs │ ├── NuGetInsightsWorkerSettings.Drivers.cs │ ├── NuGetInsightsWorkerSettings.Kusto.cs │ ├── NuGetInsightsWorkerSettings.Queues.cs │ ├── NuGetInsightsWorkerSettings.TimedReprocess.cs │ ├── NuGetInsightsWorkerSettings.Workflow.cs │ ├── NuGetInsightsWorkerSettings.cs │ ├── PackageRecord │ │ ├── IPackageCommitRecord.cs │ │ ├── IPackageEntryRecord.cs │ │ ├── IPackageRecord.cs │ │ ├── PackageEntryKeyComparer.cs │ │ ├── PackageRecord.cs │ │ ├── PackageRecordExtensions.cs │ │ ├── PackageRecordIdVersionComparer.cs │ │ └── PackageRecordIdentityComparer.cs │ ├── Queues │ │ ├── BulkEnqueueStrategy.cs │ │ ├── IMessageBatcher.cs │ │ ├── IMessageEnqueuer.cs │ │ ├── IRawMessageEnqueuer.cs │ │ ├── IWorkerQueueFactory.cs │ │ ├── MessageBatcher.cs │ │ ├── MessageEnqueuer.cs │ │ ├── QueueStorageEnqueuer.cs │ │ ├── QueueType.cs │ │ └── WorkerQueueFactory.cs │ ├── Serialization │ │ ├── CatalogScanDriverTypeConverter.cs │ │ ├── CatalogScanDriverTypeJsonConverter.cs │ │ ├── ISchemaDeserializer.cs │ │ ├── ISchemaSerializer.cs │ │ ├── ISerializedEntity.cs │ │ ├── KustoDynamicSerializer.cs │ │ ├── NameVersionMessage.cs │ │ ├── NameVersionSerializer.cs │ │ ├── SchemaCollection.cs │ │ ├── SchemaCollectionBuilder.cs │ │ ├── SchemaSerializer.cs │ │ ├── SchemaV1.cs │ │ └── SerializedMessage.cs │ ├── ServiceCollectionExtensions.Drivers.CryptoAPI.cs │ ├── ServiceCollectionExtensions.Drivers.NPE.cs │ ├── ServiceCollectionExtensions.Drivers.cs │ ├── ServiceCollectionExtensions.cs │ ├── Support │ │ ├── CryptographicExceptionExtensions.cs │ │ ├── Oids.cs │ │ └── X509Certificate2Extensions.cs │ ├── TableScan │ │ ├── CopyBucketRange │ │ │ ├── CopyBucketRangeDriver.cs │ │ │ └── CopyBucketRangeParameters.cs │ │ ├── EnqueueCatalogLeafScan │ │ │ ├── EnqueueCatalogLeafScanDriver.cs │ │ │ └── EnqueueCatalogLeafScansParameters.cs │ │ ├── ITableScanDriver.cs │ │ ├── TableCopy │ │ │ ├── TableCopyDriver.cs │ │ │ ├── TableCopyParameters.cs │ │ │ ├── TableRowCopyMessage.cs │ │ │ └── TableRowCopyMessageProcessor.cs │ │ ├── TablePrefixScanPartitionKeyQueryParameters.cs │ │ ├── TablePrefixScanPrefixQueryParameters.cs │ │ ├── TablePrefixScanStartParameters.cs │ │ ├── TableScanDriverFactory.cs │ │ ├── TableScanDriverType.cs │ │ ├── TableScanMessage.cs │ │ ├── TableScanMessageProcessor.cs │ │ ├── TableScanService.cs │ │ └── TableScanStrategy.cs │ ├── TaskState │ │ ├── TaskState.cs │ │ ├── TaskStateKey.cs │ │ └── TaskStateStorageService.cs │ ├── TimerComparer.cs │ └── Worker.Logic.csproj └── Worker │ ├── Functions.cs │ ├── Program.cs │ ├── Worker.csproj │ ├── host.json │ └── local.settings.json ├── submodules ├── .editorconfig ├── Directory.Build.props ├── Directory.Build.targets ├── Directory.Packages.props └── NuGet.config ├── test ├── Directory.Build.props ├── Logic.Test │ ├── AzureStorage │ │ ├── MutableTableTransactionalBatchTest.cs │ │ ├── StorageExtensionsTest.cs │ │ ├── StorageUtilityTest.cs │ │ └── TableExtensionsTest.cs │ ├── FileSystemHttpCache │ │ └── FileSystemHttpCacheHandlerTests.cs │ ├── Logic.Test.csproj │ ├── Network │ │ ├── HttpClientExtensionsTest.cs │ │ ├── HttpPipelineIntegrationTest.cs │ │ └── RedirectResolverTest.cs │ ├── Protocol │ │ ├── CatalogClientTest.cs │ │ ├── CatalogCommitTimestampProviderIntegrationTest.cs │ │ ├── CatalogCommitTimestampProviderTest.cs │ │ ├── Data │ │ │ └── CatalogModelExtensionsTest.cs │ │ ├── FileDownloaderIntegrationTest.cs │ │ ├── FileDownloaderTest.cs │ │ ├── GitHubUsageClientTest.cs │ │ ├── PackageDownloadsClientIntegrationTest.cs │ │ ├── PackageDownloadsClientTest.cs │ │ ├── PackageOwnersClientTest.cs │ │ ├── ServiceIndexCacheTest.cs │ │ └── VerifiedPackagesClientTest.cs │ ├── ReferenceTracking │ │ └── ReferenceTrackerTest.cs │ ├── ServiceCollectionExtensionsTest.cs │ ├── Storage │ │ ├── PackageFileServiceIntegrationTest.cs │ │ ├── PackageReadmeIntegrationTest.cs │ │ ├── PackageWideEntityServiceIntegrationTest.cs │ │ ├── StorageLeaseServiceTest.cs │ │ └── SymbolPackageFileServiceIntegrationTest.cs │ ├── StorageNoOpRetry │ │ └── StorageNoOpRetryPolicyTest.cs │ ├── Support │ │ ├── EnumerableExtensionsTest.cs │ │ └── LimitStreamTest.cs │ ├── TablePrefixScan │ │ └── TablePrefixScannerTest.cs │ ├── TempStream │ │ └── TempStreamWriterTest.cs │ ├── TestInput │ │ ├── .gitattributes │ │ ├── Cache │ │ │ ├── .gitattributes │ │ │ └── .gitignore │ │ ├── DownloadsToCsv │ │ │ ├── Step1 │ │ │ │ ├── downloads.v1.json │ │ │ │ └── downloads.v2.json │ │ │ └── Step2 │ │ │ │ ├── downloads.v1.json │ │ │ │ └── downloads.v2.json │ │ ├── DownloadsToCsv_Duplicates │ │ │ └── Step1 │ │ │ │ └── downloads.v1.json │ │ ├── DownloadsToCsv_UnicodeDuplicates │ │ │ └── Step1 │ │ │ │ └── downloads.v1.json │ │ ├── ExcludedPackagesToCsv │ │ │ ├── Step1 │ │ │ │ └── excludedPackages.json │ │ │ └── Step2 │ │ │ │ └── excludedPackages.json │ │ ├── GitHubUsageToCsv │ │ │ ├── Step1 │ │ │ │ └── GitHubUsage.v1.json │ │ │ └── Step2 │ │ │ │ └── GitHubUsage.v1.json │ │ ├── OwnersToCsv │ │ │ ├── Step1 │ │ │ │ └── owners.v2.json │ │ │ └── Step2 │ │ │ │ └── owners.v2.json │ │ ├── PopularityTransfersToCsv │ │ │ ├── Step1 │ │ │ │ └── popularity-transfers.v1.json │ │ │ └── Step2 │ │ │ │ └── popularity-transfers.v1.json │ │ ├── Resources.cs │ │ ├── VerifiedPackagesToCsv │ │ │ ├── Step1 │ │ │ │ └── verifiedPackages.json │ │ │ └── Step2 │ │ │ │ └── verifiedPackages.json │ │ ├── WindowsAzure.Storage.9.3.3.md │ │ ├── behaviorsample.1.0.0.md │ │ ├── behaviorsample.1.0.0.nupkg.testdata │ │ ├── behaviorsample.1.0.0.nuspec │ │ ├── behaviorsample.1.0.0.snupkg.testdata │ │ ├── behaviorsample.project.zip │ │ └── deltax.1.0.0.nupkg.testdata │ ├── TestLevers.cs │ ├── TestSupport │ │ ├── BaseLogicIntegrationTest.cs │ │ ├── DefaultWebApplicationFactory.cs │ │ ├── IterTools.cs │ │ ├── JsonElementComparer.cs │ │ ├── LogicTestSettings.cs │ │ ├── NoInMemoryStorageFactAttribute.cs │ │ ├── OSPlatformFactAttribute.cs │ │ ├── OSPlatformType.cs │ │ ├── RealStorageTokenCredentialFactAttribute.cs │ │ ├── StaticFilesStartup.cs │ │ ├── TestDirectory.cs │ │ ├── TestEntity.cs │ │ ├── TestHttpMessageHandler.cs │ │ ├── TestHttpMessageHandlerFactory.cs │ │ ├── TestOutputHelperExtensions.cs │ │ ├── TestServiceClientFactory.cs │ │ ├── XunitLogger.cs │ │ ├── XunitLoggerFactoryExtensions.cs │ │ └── XunitLoggerProvider.cs │ ├── Timers │ │ └── TimerExecutionServiceTest.cs │ └── WideEntities │ │ ├── ChunkStreamTest.cs │ │ └── WideEntityServiceTest.cs ├── Website.Test │ ├── BaseWebsiteIntegrationTest.cs │ ├── InitializerHostedServiceTest.cs │ └── Website.Test.csproj ├── Worker.Logic.Test │ ├── AppendResults │ │ ├── AppendResultStorageServiceTest.cs │ │ └── CsvRecordContainersTest.cs │ ├── CatalogScan │ │ ├── CatalogLeafScanMessageProcessorTest.cs │ │ ├── CatalogScanDriverFactoryTest.cs │ │ ├── CatalogScanDriverMetadataIntegrationTest.cs │ │ ├── CatalogScanDriverMetadataTest.CryptoAPI.cs │ │ ├── CatalogScanDriverMetadataTest.NPE.cs │ │ ├── CatalogScanDriverMetadataTest.cs │ │ ├── CatalogScanFaultInjectionTest.cs │ │ ├── CatalogScanServiceBucketsIntegrationTest.cs │ │ ├── CatalogScanServiceTest.DriverInfo.CryptoApi.cs │ │ ├── CatalogScanServiceTest.DriverInfo.NPE.cs │ │ ├── CatalogScanServiceTest.DriverInfo.cs │ │ ├── CatalogScanServiceTest.Tests.cs │ │ ├── CatalogScanStorageServiceIntegrationTest.cs │ │ ├── CatalogScanToCsv │ │ │ └── CatalogLeafToCsv │ │ │ │ └── PackageRecordExtensionFacts.cs │ │ ├── CatalogScanUpdateTimerTest.cs │ │ └── LatestLeaf │ │ │ └── LatestLeafStorageServiceTest.cs │ ├── CsvRecordStorage │ │ └── CsvRecordStorageServiceTest.cs │ ├── Cursors │ │ └── CursorStorageServiceTest.cs │ ├── Docs │ │ ├── DocInfo.cs │ │ ├── DriverDocInfo.cs │ │ ├── DriverDocInitializer.cs │ │ ├── DriverDocsTest.AllDrivers.cs │ │ ├── DriverDocsTest.cs │ │ ├── StorageContainerType.cs │ │ ├── TableDocInfo.cs │ │ ├── TableDocInitializer.cs │ │ ├── TableDocsTest.AllTables.cs │ │ └── TableDocsTest.cs │ ├── Drivers │ │ ├── BuildVersionSet │ │ │ ├── BuildVersionSetIntegrationTest.cs │ │ │ ├── VersionSetServiceIntegrationTest.cs │ │ │ └── VersionSetTest.cs │ │ ├── CatalogDataToCsv │ │ │ ├── CatalogDataToCsvDriverTest.cs │ │ │ └── CatalogDataToCsvIntegrationTest.cs │ │ ├── Internal_FindLatestCatalogLeafScan │ │ │ └── FindLatestCatalogLeafScanIntegrationTest.cs │ │ ├── Internal_FindLatestCatalogLeafScanPerId │ │ │ └── FindLatestCatalogLeafScanPerIdIntegrationTest.cs │ │ ├── LoadBucketedPackage │ │ │ ├── BucketRangeTest.cs │ │ │ └── LoadBucketedPackageIntegrationTest.cs │ │ ├── LoadLatestPackageLeaf │ │ │ └── LoadLatestPackageLeafIntegrationTest.cs │ │ ├── LoadPackageArchive │ │ │ └── LoadPackageArchiveIntegrationTest.cs │ │ ├── LoadPackageManifest │ │ │ └── LoadPackageManifestIntegrationTest.cs │ │ ├── LoadPackageReadme │ │ │ └── LoadPackageReadmeIntegrationTest.cs │ │ ├── LoadPackageVersion │ │ │ └── LoadPackageVersionIntegrationTest.cs │ │ ├── LoadSymbolPackageArchive │ │ │ └── LoadSymbolPackageArchiveIntegrationTest.cs │ │ ├── NuGetPackageExplorerToCsv │ │ │ ├── NuGetPackageExplorerToCsvDriverTest.cs │ │ │ └── NuGetPackageExplorerToCsvIntegrationTest.cs │ │ ├── PackageArchiveToCsv │ │ │ ├── PackageArchiveToCsvDriverTest.cs │ │ │ └── PackageArchiveToCsvIntegrationTest.cs │ │ ├── PackageAssemblyToCsv │ │ │ ├── PackageAssemblyToCsvDriverTest.cs │ │ │ └── PackageAssemblyToCsvIntegrationTest.cs │ │ ├── PackageAssetToCsv │ │ │ ├── PackageAssetToCsvDriverTest.cs │ │ │ └── PackageAssetToCsvIntegrationTest.cs │ │ ├── PackageCertificateToCsv │ │ │ ├── PackageCertificateToCsvDriverTest.cs │ │ │ └── PackageCertificateToCsvIntegrationTest.cs │ │ ├── PackageCompatibilityToCsv │ │ │ ├── PackageCompatibilityToCsvDriverTest.cs │ │ │ └── PackageCompatibilityToCsvIntegrationTest.cs │ │ ├── PackageContentToCsv │ │ │ ├── PackageContentToCsvDriverTest.cs │ │ │ └── PackageContentToCsvIntegrationTest.cs │ │ ├── PackageFileToCsv │ │ │ ├── PackageFileToCsvDriverTest.cs │ │ │ └── PackageFileToCsvIntegrationTest.cs │ │ ├── PackageIconToCsv │ │ │ ├── PackageIconToCsvDriverTest.cs │ │ │ └── PackageIconToCsvIntegrationTest.cs │ │ ├── PackageLicenseToCsv │ │ │ ├── PackageLicenseToCsvDriverTest.cs │ │ │ └── PackageLicenseToCsvIntegrationTest.cs │ │ ├── PackageManifestToCsv │ │ │ ├── PackageManifestToCsvDriverTest.cs │ │ │ └── PackageManifestToCsvIntegrationTest.cs │ │ ├── PackageReadmeToCsv │ │ │ ├── PackageReadmeToCsvDriverTest.cs │ │ │ └── PackageReadmeToCsvIntegrationTest.cs │ │ ├── PackageSignatureToCsv │ │ │ └── PackageSignatureToCsvIntegrationTest.cs │ │ ├── PackageVersionToCsv │ │ │ └── PackageVersionToCsvIntegrationTest.cs │ │ ├── SymbolPackageArchiveToCsv │ │ │ ├── SymbolPackageArchiveToCsvDriverTest.cs │ │ │ └── SymbolPackageArchiveToCsvIntegrationTest.cs │ │ └── SymbolPackageFileToCsv │ │ │ └── SymbolPackageFileToCsvIntegrationTest.cs │ ├── Generated │ │ └── NuGet.Insights.SourceGenerator │ │ │ └── NuGet.Insights.CsvRecordGenerator │ │ │ ├── NuGetInsightsWorkerLogicTestKustoDDL.cs │ │ │ ├── TestSubjectRecord.ICsvRecord.cs │ │ │ ├── TestSubjectRecord.KustoDDL.cs │ │ │ └── TestSubjectRecord.MessagePack.cs │ ├── Kusto │ │ └── CachingKustoClientFactoryTest.cs │ ├── MessageProcessors │ │ ├── DownloadsToCsv │ │ │ ├── DownloadsToCsvIntegrationTest.cs │ │ │ └── DownloadsToCsvUpdaterTest.cs │ │ ├── ExcludedPackagesToCsv │ │ │ └── ExcludedPackagesToCsvIntegrationTest.cs │ │ ├── GenericMessageProcessorTest.cs │ │ ├── GitHubUsageToCsv │ │ │ └── GitHubUsageToCsvIntegrationTest.cs │ │ ├── KustoIngestion │ │ │ ├── KustoIngestionTimerTest.cs │ │ │ ├── KustoIngestionWithMockKustoTest.cs │ │ │ └── KustoIngestionWithRealKustoTest.cs │ │ ├── OwnersToCsv │ │ │ └── OwnersToCsvIntegrationTest.cs │ │ ├── PopularityTransfersToCsv │ │ │ └── PopularityTransfersToCsvIntegrationTest.cs │ │ ├── ReferenceTracking │ │ │ ├── CleanupOrphanRecordsServiceTest.cs │ │ │ ├── TestCleanupOrphanRecordsAdapter.cs │ │ │ ├── TestSubjectRecord.cs │ │ │ └── TestSubjectRecordStorage.cs │ │ ├── TimedReprocess │ │ │ ├── TimedReprocessServiceIntegrationTest.cs │ │ │ ├── TimedReprocessServiceTest.cs │ │ │ └── TimedReprocessStorageServiceIntegrationTest.cs │ │ ├── VerifiedPackagesToCsv │ │ │ └── VerifiedPackagesToCsvIntegrationTest.cs │ │ └── Workflow │ │ │ ├── WorkflowIntegrationTest.cs │ │ │ └── WorkflowServiceTest.cs │ ├── Queues │ │ ├── MessageEnqueuerIntegrationTest.cs │ │ └── MessageEnqueuerTest.cs │ ├── Serialization │ │ ├── CatalogScanDriverTypeConverterTest.cs │ │ ├── KustoDynamicSerializerTest.cs │ │ └── SchemaCollectionBuilderTest.cs │ ├── ServiceCollectionExtensionsTest.cs │ ├── TableScan │ │ ├── CopyBucketRange │ │ │ └── CopyBucketRangeDriverIntegrationTest.cs │ │ └── TableCopy │ │ │ └── TableCopyDriverIntegrationTest.cs │ ├── TaskState │ │ └── TaskStateStorageServiceTest.cs │ ├── TestData │ │ ├── .gitattributes │ │ ├── BuildVersionSet │ │ │ └── Step1 │ │ │ │ └── data.json │ │ ├── BuildVersionSet_IgnoredPackages │ │ │ └── Step1 │ │ │ │ └── data.json │ │ ├── BuildVersionSet_WithDelete │ │ │ ├── Step1 │ │ │ │ └── data.json │ │ │ └── Step2 │ │ │ │ └── data.json │ │ ├── BuildVersionSet_WithDuplicates │ │ │ ├── Step1 │ │ │ │ └── data.json │ │ │ └── Step2 │ │ │ │ └── data.json │ │ ├── BuildVersionSet_WithIdCaseChange │ │ │ ├── Step1 │ │ │ │ └── data.json │ │ │ └── Step2 │ │ │ │ └── data.json │ │ ├── BuildVersionSet_WithUnicodeDuplicates │ │ │ ├── Step1 │ │ │ │ └── data.json │ │ │ └── Step2 │ │ │ │ └── data.json │ │ ├── CatalogDataToCsv │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_Deprecation │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_DuplicateLeafUrl │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_IgnoredPackages │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_NoSignatureFile │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_Vulnerabilities │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_WithDuplicates │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogDataToCsv_WithKelvinK_NLS │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── T3 │ │ │ │ └── compact_0.csv │ │ ├── CatalogScanService_TestCachedData │ │ │ ├── Step1 │ │ │ │ ├── PackageReadmes.csv │ │ │ │ ├── SymbolPackageArchiveEntries.csv │ │ │ │ ├── SymbolPackageArchives.csv │ │ │ │ └── SymbolPackageFiles.csv │ │ │ └── Step2 │ │ │ │ ├── PackageReadmes.csv │ │ │ │ ├── SymbolPackageArchiveEntries.csv │ │ │ │ ├── SymbolPackageArchives.csv │ │ │ │ └── SymbolPackageFiles.csv │ │ ├── CatalogScanService_UpdateWithBucketRanges │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── CleanupOrphanRecordsServiceTest_WithClearedCsv │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ ├── compact_1.csv │ │ │ │ ├── compact_2.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ └── Step2 │ │ │ │ ├── compact_0.csv │ │ │ │ ├── compact_2.csv │ │ │ │ └── subject-to-owner.json │ │ ├── CleanupOrphanRecordsServiceTest_WithNoDeletions │ │ │ └── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ ├── compact_1.csv │ │ │ │ ├── compact_2.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ ├── CleanupOrphanRecordsServiceTest_WithNoOrphans │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ ├── compact_1.csv │ │ │ │ ├── compact_2.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ └── Step2 │ │ │ │ └── subject-to-owner.json │ │ ├── CopyBucketRange │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── CopyBucketRange_WithMultiplePartitionKeyWrites │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── DownloadsToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── DownloadsToCsv_Duplicates │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── DownloadsToCsv_JustV2 │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── DownloadsToCsv_NonExistentId │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── DownloadsToCsv_NonExistentVersion │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── DownloadsToCsv_UncheckedIdAndVersion │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── DownloadsToCsv_UnicodeDuplicates │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── ExcludedPackagesToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── ExcludedPackagesToCsv_NonExistentId │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── ExcludedPackagesToCsv_UncheckedId │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── GeneratedStorageNames │ │ │ └── table.md │ │ ├── GitHubUsageToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── GitHubUsageToCsv_NonExistentId │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── GitHubUsageToCsv_UncheckedId │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── Internal_FindLatestCatalogLeafScan │ │ │ └── entities.json │ │ ├── Internal_FindLatestCatalogLeafScanPerId │ │ │ └── entities.json │ │ ├── Internal_FindLatestCatalogLeafScanPerId_WithDuplicates │ │ │ └── entities.json │ │ ├── Internal_FindLatestCatalogLeafScan_WithDuplicates │ │ │ └── entities.json │ │ ├── KustoIngestionWithRealKustoTest │ │ │ ├── CatalogLeafItems.json │ │ │ ├── PackageDeprecations.json │ │ │ └── PackageVulnerabilities.json │ │ ├── LoadBucketedPackage │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadBucketedPackage_WithDuplicatesInCommit │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadLatestPackageLeaf │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadLatestPackageLeaf_WithDuplicatesInCommit │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageArchive │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageArchive_IgnoredPackages │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadPackageArchive_WithCommonReference │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadPackageArchive_WithDelete │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageArchive_WithManyAssemblies │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageArchive_WithManyAssembliesWithDelete │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageManifest │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageManifest_WithDelete │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageReadme │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageReadme_WithDelete │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageVersion │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadPackageVersion_SemVer2 │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadPackageVersion_WithDelete │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadPackageVersion_WithDuplicates │ │ │ └── Step1 │ │ │ │ └── entities.json │ │ ├── LoadSymbolPackageArchive │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── LoadSymbolPackageArchive_WithDelete │ │ │ ├── Step1 │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ └── entities.json │ │ ├── NuGetPackageExplorerToCsv │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── NuGetPackageExplorerToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── OwnersToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── OwnersToCsv_NonExistentId │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── OwnersToCsv_UncheckedId │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageArchiveToCsv │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── PackageArchiveToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── PackageArchiveToCsv_WithDuplicateEntries │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssemblyToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssemblyToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssemblyToCsv_WithDuplicates │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssemblyToCsv_WithUnmanaged │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssetToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssetToCsv_IgnoredPackages │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssetToCsv_IgnoredPackages_PruneCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssetToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssetToCsv_WithDuplicates │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageAssetToCsv_WithSingleBucket │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageCertificateToCsv │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ ├── PackageCertificateToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ ├── Step2 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ └── Step3 │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ └── subject-to-owner.json │ │ ├── PackageCertificateToCsv_WithDollarSignId │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── empty-array.json │ │ ├── PackageCertificateToCsv_WithEVCodeSigningCertificate │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ ├── PackageCertificateToCsv_WithManyCertificates │ │ │ └── Step1 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ ├── PackageCertificateToCsv_WithMultipleBuckets │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ ├── compact_0.csv │ │ │ │ │ ├── compact_1.csv │ │ │ │ │ └── compact_2.csv │ │ │ │ ├── T2 │ │ │ │ │ ├── compact_0.csv │ │ │ │ │ ├── compact_1.csv │ │ │ │ │ └── compact_2.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ ├── Step2 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_2.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ └── Step3 │ │ │ │ ├── T2 │ │ │ │ └── compact_1.csv │ │ │ │ └── subject-to-owner.json │ │ ├── PackageCertificateToCsv_WithSingleDelete │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── T2 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── owner-to-subject.json │ │ │ │ └── subject-to-owner.json │ │ │ ├── Step2 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ ├── empty-array.json │ │ │ │ └── subject-to-owner.json │ │ │ └── Step3 │ │ │ │ └── T2 │ │ │ │ └── empty.csv │ │ ├── PackageCompatibilityToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageCompatibilityToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageCompatibilityToCsv_WithManyAssets │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageCompatibilityToCsv_WithUnsupported │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageContentToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageContentToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageFileToCsv │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ ├── PackageFileToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ ├── PackageFileToCsv_WithDuplicates │ │ │ └── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ ├── PackageIconToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageIconToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageLicenseToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageLicenseToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageManifestToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageManifestToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageManifestToCsv_WithVeryLargeBuffer │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageReadmeToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageReadmeToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageReadmeToCsv_WithVeryLargeBuffer │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageSignatureToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageSignatureToCsv_AuthorSignature │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageSignatureToCsv_BadTimestamp │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageSignatureToCsv_BadTimestampEncoding │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageSignatureToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageSignatureToCsv_WithDuplicatesInCommit │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageVersionToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageVersionToCsv_WithAllLatest │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageVersionToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PackageVersionToCsv_WithDuplicates │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageVersionToCsv_WithFirstCommitTimestamp │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── PackageVersionToCsv_WithMultipleBuckets │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ ├── compact_1.csv │ │ │ │ └── compact_2.csv │ │ │ └── Step2 │ │ │ │ └── compact_2.csv │ │ ├── PopularityTransfersToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PopularityTransfersToCsv_NonExistentId │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── PopularityTransfersToCsv_UncheckedId │ │ │ └── Step1 │ │ │ │ └── compact_0.csv │ │ ├── SymbolPackageArchiveToCsv │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── SymbolPackageArchiveToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── T1 │ │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ ├── T1 │ │ │ │ └── compact_0.csv │ │ │ │ └── T2 │ │ │ │ └── compact_0.csv │ │ ├── SymbolPackageFileToCsv │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ ├── SymbolPackageFileToCsv_WithDelete │ │ │ ├── Step1 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ │ └── Step2 │ │ │ │ ├── compact_0.csv │ │ │ │ └── entities.json │ │ ├── TimedReprocess_AllReprocessDrivers │ │ │ └── Step1 │ │ │ │ ├── PackageReadmes.csv │ │ │ │ ├── PackageReadmes.json │ │ │ │ ├── SymbolPackageArchiveEntries.csv │ │ │ │ ├── SymbolPackageArchives.csv │ │ │ │ ├── SymbolPackageArchives.json │ │ │ │ ├── SymbolPackageFiles.csv │ │ │ │ └── SymbolPackageHashes.json │ │ ├── TimedReprocess_SameBucketRangesDir │ │ │ └── Step1 │ │ │ │ ├── PackageReadmes.csv │ │ │ │ └── PackageReadmes.json │ │ ├── TimedReprocess_SubsequentBucketRanges │ │ │ ├── Step1 │ │ │ │ ├── PackageReadmes.csv │ │ │ │ └── PackageReadmes.json │ │ │ └── Step2 │ │ │ │ ├── PackageReadmes.csv │ │ │ │ └── PackageReadmes.json │ │ ├── VerifiedPackagesToCsv │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ ├── VerifiedPackagesToCsv_NonExistentId │ │ │ ├── Step1 │ │ │ │ └── compact_0.csv │ │ │ └── Step2 │ │ │ │ └── compact_0.csv │ │ └── VerifiedPackagesToCsv_UncheckedId │ │ │ └── Step1 │ │ │ └── compact_0.csv │ ├── TestFrameworkTest.cs │ ├── TestSupport │ │ ├── BaseCatalogLeafScanToCsvIntegrationTest.cs │ │ ├── BaseCatalogScanIntegrationTest.cs │ │ ├── BaseCatalogScanToCsvIntegrationTest.cs │ │ ├── BaseWorkerLogicIntegrationTest.cs │ │ ├── DataContractResolver.cs │ │ ├── EntityExtensions.cs │ │ ├── KustoFactAttribute.cs │ │ ├── ServiceCollectionExtensions.cs │ │ ├── TableEntityJsonConverter.cs │ │ ├── TestDataTest.cs │ │ ├── TestOutputHelperExtensions.cs │ │ └── WorkerLogicTestSettings.cs │ ├── TimerComparerTest.cs │ └── Worker.Logic.Test.csproj └── Worker.Test │ ├── EndToEnd │ ├── AllOutputCanBeReset.cs │ ├── CanRunTimers.cs │ ├── CatalogRangeSameAsBucketRange.cs │ ├── DownloadsExpectedPackageAssets.cs │ ├── EndToEndTest.cs │ └── EntireWorkflowSucceeds.cs │ ├── TestData │ ├── .gitattributes │ └── EntireWorkflowSucceeds │ │ └── Step1 │ │ ├── CatalogLeafItems.csv │ │ ├── Certificates.csv │ │ ├── NuGetPackageExplorerFiles.csv │ │ ├── NuGetPackageExplorers.csv │ │ ├── PackageArchiveEntries.csv │ │ ├── PackageArchives.csv │ │ ├── PackageAssemblies.csv │ │ ├── PackageAssets.csv │ │ ├── PackageCertificates.csv │ │ ├── PackageCompatibilities.csv │ │ ├── PackageContents.csv │ │ ├── PackageDeprecations.csv │ │ ├── PackageFiles.csv │ │ ├── PackageIcons.csv │ │ ├── PackageLicenses.csv │ │ ├── PackageManifests.csv │ │ ├── PackageReadmes.csv │ │ ├── PackageSignatures.csv │ │ ├── PackageVersions.csv │ │ ├── PackageVulnerabilities.csv │ │ ├── SymbolPackageArchiveEntries.csv │ │ ├── SymbolPackageArchives.csv │ │ └── SymbolPackageFiles.csv │ └── Worker.Test.csproj └── tye.yaml /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !src 3 | !Directory.Build.props 4 | !Directory.Build.targets 5 | !global.json 6 | !NuGet.config 7 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in 2 | # the repo. Unless a later match takes precedence, 3 | # review when someone opens a pull request. 4 | # For more on how to customize the CODEOWNERS file - https://help.github.com/en/articles/about-code-owners 5 | * @NuGet/gallery-team 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "yaml.schemas": { 3 | "https://raw.githubusercontent.com/dotnet/tye/main/src/schema/tye-schema.json": [ 4 | "tye.yaml" 5 | ] 6 | } 7 | } -------------------------------------------------------------------------------- /.vsconfig: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "components": [ 4 | "Microsoft.VisualStudio.Workload.Azure", 5 | "Microsoft.VisualStudio.Workload.ManagedDesktop", 6 | "Microsoft.VisualStudio.Workload.NetWeb" 7 | ] 8 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | This project has adopted the code of conduct defined by the Contributor Covenant to clarify expected behavior in our community. For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to the [.NET Foundation Contributor License Agreement (CLA)](https://cla.dotnetfoundation.org) declaring that you have the right to, and actually do, grant us the rights to use your contribution. 4 | 5 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all projects in the .NET Foundation. 6 | 7 | This project has adopted the code of conduct defined by the Contributor Covenant to clarify expected behavior in our community. For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). -------------------------------------------------------------------------------- /azurepipelines-coverage.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: # Code coverage status will be posted to pull requests based on targets defined below. 3 | comments: on # Off by default. When on, details about coverage for each file changed will be posted as a pull request comment. 4 | diff: # diff coverage is code coverage only for the lines changed in a pull request. 5 | target: 50% # set this to a desired %. Default is 50% -------------------------------------------------------------------------------- /deploy/bicep/key-vault.bicep: -------------------------------------------------------------------------------- 1 | param keyVaultName string 2 | param location string 3 | param workspaceId string 4 | 5 | resource keyVault 'Microsoft.KeyVault/vaults@2019-09-01' = { 6 | name: keyVaultName 7 | location: location 8 | properties: { 9 | tenantId: subscription().tenantId 10 | sku: { 11 | family: 'A' 12 | name: 'standard' 13 | } 14 | enableRbacAuthorization: true 15 | accessPolicies: [] 16 | } 17 | } 18 | 19 | resource keyVaultDiagnostics 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = { 20 | scope: keyVault 21 | name: '${keyVaultName}-la-diagnostics' 22 | properties: { 23 | workspaceId: workspaceId 24 | logs: [ 25 | { 26 | category: 'AuditEvent' 27 | enabled: true 28 | } 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /deploy/bicep/spot-worker-vnets.bicep: -------------------------------------------------------------------------------- 1 | param specs array 2 | 3 | // spot workers 4 | var workersDeploymentLongName = '${deployment().name}-' 5 | 6 | // Subtract 10 from the max length to account for the index appended to the module name 7 | var workersDeploymentName = length(workersDeploymentLongName) > (64 - 10) 8 | ? '${guid(deployment().name)}-' 9 | : workersDeploymentLongName 10 | 11 | module workers './spot-worker-vnet.bicep' = [ 12 | for (spec, index) in specs: { 13 | name: '${workersDeploymentName}${index}' 14 | params: { 15 | location: spec.location 16 | nsgName: '${spec.namePrefix}nsg' 17 | vnetName: '${spec.namePrefix}vnet' 18 | index: index 19 | } 20 | } 21 | ] 22 | 23 | output subnetIds array = [for (spec, index) in specs: workers[index].outputs.subnetId] 24 | -------------------------------------------------------------------------------- /deploy/bicep/storage-network-acls.bicep: -------------------------------------------------------------------------------- 1 | param storageAccountName string 2 | param location string 3 | param subnetIds array 4 | param denyTraffic bool 5 | param allowedIpRanges array 6 | 7 | resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' = { 8 | name: storageAccountName 9 | location: location 10 | kind: 'StorageV2' 11 | sku: { 12 | name: 'Standard_LRS' 13 | } 14 | properties: { 15 | networkAcls: { 16 | // Kusto ingestion needs manual setup of a private endpoint to this storage account. 17 | defaultAction: denyTraffic ? 'Deny' : 'Allow' 18 | bypass: 'AzureServices' 19 | virtualNetworkRules: [ 20 | for subnetId in subnetIds: { 21 | id: subnetId 22 | action: 'Allow' 23 | } 24 | ] 25 | ipRules: [ 26 | for item in allowedIpRanges: { 27 | value: item 28 | } 29 | ] 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /deploy/config/dev-spot.json: -------------------------------------------------------------------------------- 1 | { 2 | "Deployment": { 3 | "Location": "West US 2", 4 | "WorkerPlanCount": 1, 5 | "WorkerCountPerPlan": 1, 6 | "WorkerMaxInstances": 1, 7 | "WorkerLogLevel": "Warning", 8 | "WorkerSku": "B1", 9 | "UseSpotWorkers": true, 10 | "SpotWorkerSpecs": [ 11 | { 12 | "MaxInstances": 11 13 | } 14 | ] 15 | } 16 | } -------------------------------------------------------------------------------- /deploy/config/dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "Deployment": { 3 | "Location": "West US", 4 | "WorkerPlanCount": 2, 5 | "WorkerPlanLocations": [ 6 | "West US", 7 | "West US 2" 8 | ], 9 | "WorkerCountPerPlan": 1, 10 | "WorkerLogLevel": "Warning", 11 | "WorkerSku": "S1" 12 | }, 13 | "AppSettings": { 14 | "Shared": { 15 | "NuGetInsights": { 16 | "DisabledDrivers": [ 17 | "NuGetPackageExplorerToCsv" 18 | ] 19 | } 20 | }, 21 | "Worker": { 22 | "NuGetInsights": { 23 | "MoveTempToHome": false 24 | } 25 | }, 26 | "Website": { 27 | "NuGetInsights": { 28 | "RestrictUsers": true, 29 | "ShowAdminLink": true 30 | } 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /deploy/config/msdn-s1.json: -------------------------------------------------------------------------------- 1 | { 2 | "Deployment": { 3 | "SubscriptionId": "7e9b1932-8591-42d8-a02e-7471cf53f915", 4 | "Location": "East US 2", 5 | "WorkerSku": "S1" 6 | } 7 | } -------------------------------------------------------------------------------- /deploy/config/msdn-y1.json: -------------------------------------------------------------------------------- 1 | { 2 | "Deployment": { 3 | "SubscriptionId": "7e9b1932-8591-42d8-a02e-7471cf53f915", 4 | "Location": "East US 2", 5 | "WorkerSku": "Y1" 6 | } 7 | } -------------------------------------------------------------------------------- /deploy/host-version.ps1: -------------------------------------------------------------------------------- 1 | $AzureFunctionsHostVersion = "v4.1039.400" 2 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | See the [other docs](../README.md#other-docs) list for more information. 2 | -------------------------------------------------------------------------------- /docs/admin-panel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/admin-panel.png -------------------------------------------------------------------------------- /docs/azure-resources.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/azure-resources.png -------------------------------------------------------------------------------- /docs/find-package-files-execution-units.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/find-package-files-execution-units.png -------------------------------------------------------------------------------- /docs/find-package-files-exucution-count.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/find-package-files-exucution-count.png -------------------------------------------------------------------------------- /docs/local-azure-function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/local-azure-function.png -------------------------------------------------------------------------------- /docs/local-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/local-results.png -------------------------------------------------------------------------------- /docs/quickstart-admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/docs/quickstart-admin.png -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0.115", 4 | "rollForward": "latestFeature", 5 | "allowPrerelease": true 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /markdown-link-check.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "httpHeaders": [ 3 | { 4 | "urls": [ 5 | "" 6 | ], 7 | "headers": { 8 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44", 9 | "Accept-Encoding": "gzip", 10 | "Accept": "*/*; q=1.0" 11 | } 12 | } 13 | ], 14 | "ignorePatterns": [ 15 | { 16 | "pattern": "^mailto:" 17 | }, 18 | { 19 | "pattern": "^https://aka\\.ms/opensource/" 20 | }, 21 | { 22 | "pattern": "^https://azure\\.microsoft\\.com/" 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nuget-insights", 3 | "version": "1.0.0", 4 | "devDependencies": { 5 | "azurite": "^3.33.0", 6 | "markdown-link-check": "^3.12.2" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /scripts/Invoke-Format.ps1: -------------------------------------------------------------------------------- 1 | [CmdletBinding()] 2 | param ( 3 | [Parameter(Mandatory = $false)] 4 | [switch]$Verify 5 | ) 6 | 7 | Push-Location 8 | Set-Location (Join-Path $PSScriptRoot "..") 9 | try { 10 | $arguments = "format", "--exclude", "src/Forks", "--exclude", "submodules" 11 | if ($Verify) { 12 | $arguments += "--verify-no-changes" 13 | } 14 | 15 | Write-Host "Executing: dotnet $arguments" -ForegroundColor Green 16 | dotnet @arguments 17 | if ($LASTEXITCODE -ne 0) { 18 | Write-Error "Command 'dotnet $arguments' failed with exit code $LASTEXITCODE." 19 | } 20 | } 21 | finally { 22 | Pop-Location 23 | } 24 | -------------------------------------------------------------------------------- /src/Forks/NuGet/NuGetGallery/src/NuGet.Services.Entities/IEntity.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Services.Entities 5 | { 6 | public interface IEntity 7 | { 8 | int Key { get; set; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Forks/NuGet/NuGetGallery/src/NuGet.Services.Validation/Entities/EndCertificateUse.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Services.Validation 5 | { 6 | /// 7 | /// How an end certificate is being used. 8 | /// 9 | public enum EndCertificateUse 10 | { 11 | /// 12 | /// An end certificate used in a . 13 | /// 14 | CodeSigning = 1, 15 | 16 | /// 17 | /// An end certificate used in a . 18 | /// 19 | Timestamping = 2, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Forks/NuGet/NuGetGallery/src/NuGetGallery.Core/Frameworks/FrameworkProductNames.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGetGallery.Frameworks 5 | { 6 | public static class FrameworkProductNames 7 | { 8 | public const string Net = ".NET"; 9 | public const string NetCore = ".NET Core"; 10 | public const string NetStandard = ".NET Standard"; 11 | public const string NetFramework = ".NET Framework"; 12 | public const string NetMicroFramework = ".NET Micro Framework"; 13 | public const string UWP = "Universal Windows Platform"; 14 | public const string WindowsPhone = "Windows Phone"; 15 | public const string WindowsStore = "Windows Store"; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Forks/NuGet/NuGetGallery/src/NuGetGallery.Core/Frameworks/IPackageFrameworkCompatibilityFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Services.Entities; 5 | using System.Collections.Generic; 6 | 7 | namespace NuGetGallery.Frameworks 8 | { 9 | public interface IPackageFrameworkCompatibilityFactory 10 | { 11 | PackageFrameworkCompatibility Create(ICollection packageFrameworks, string packageId, string packageVersion, bool includeComputedBadges = false); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Forks/NuGet/NuGetGallery/src/NuGetGallery.Core/Frameworks/PackageFrameworkCompatibilityData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Frameworks; 5 | 6 | namespace NuGetGallery.Frameworks 7 | { 8 | public class PackageFrameworkCompatibilityData 9 | { 10 | /// 11 | /// Compatible framework. 12 | /// 13 | public NuGetFramework Framework { get; set; } 14 | 15 | /// 16 | /// if the was computed from .

17 | /// if the was retrieved from the package asset frameworks. 18 | /// 19 | public bool IsComputed { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Forks/NuGet/NuGetGallery/src/Validation.PackageSigning.ValidateCertificate/CertificateVerificationException.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System; 5 | 6 | namespace Validation.PackageSigning.ValidateCertificate 7 | { 8 | public class CertificateVerificationException : Exception 9 | { 10 | /// 11 | /// Exception thrown by unexpected failures to validate a certificate. 12 | /// 13 | /// The message describing the failure. 14 | public CertificateVerificationException(string message) 15 | : base(message) 16 | { 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Logic/AzureStorage/CaseInsensitiveDictionary.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CaseInsensitiveDictionary : Dictionary 7 | { 8 | public CaseInsensitiveDictionary() : base(StringComparer.OrdinalIgnoreCase) 9 | { 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Logic/AzureStorage/CaseInsensitiveSortedDictionary.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CaseInsensitiveSortedDictionary : SortedDictionary 7 | { 8 | public CaseInsensitiveSortedDictionary() : base(StringComparer.OrdinalIgnoreCase) 9 | { 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Logic/AzureStorage/StorageCredentialType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights 7 | { 8 | public enum StorageCredentialType 9 | { 10 | DefaultAzureCredential = 1, 11 | UserAssignedManagedIdentityCredential, 12 | ClientCertificateCredentialFromKeyVault, 13 | ClientCertificateCredentialFromPath, 14 | StorageAccessKey, 15 | DevelopmentStorage, 16 | MemoryStorage, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Logic/AzureStorage/StorageId.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class StorageId 7 | { 8 | private readonly string _value; 9 | 10 | public StorageId(string sortable, string unique) 11 | { 12 | Sortable = sortable; 13 | Unique = unique; 14 | _value = sortable + "-" + unique; 15 | } 16 | 17 | public string Sortable { get; } 18 | public string Unique { get; } 19 | 20 | public override string ToString() 21 | { 22 | return _value; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/Logic/Data/BucketKeyAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class BucketKeyAttribute : Attribute 7 | { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/Logic/Data/CatalogLeafType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public enum CatalogLeafType 7 | { 8 | PackageDetails = 1, 9 | PackageDelete = 2, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Logic/Data/CsvRecordAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | [AttributeUsage(AttributeTargets.Class)] 7 | public class CsvRecordAttribute : Attribute 8 | { 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Logic/Data/KustoIgnoreAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class KustoIgnoreAttribute : Attribute 7 | { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/Logic/Data/KustoPartitionKeyAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class KustoPartitionKeyAttribute : Attribute 7 | { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/Logic/Data/KustoTypeAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | [AttributeUsage(AttributeTargets.Property, Inherited = false, AllowMultiple = false)] 7 | public class KustoTypeAttribute : Attribute 8 | { 9 | public KustoTypeAttribute(string type) 10 | { 11 | KustoType = type; 12 | } 13 | 14 | public string KustoType { get; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Logic/Data/NoKustoDDLAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class NoKustoDDLAttribute : Attribute 7 | { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/Logic/Data/SemVerType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | [Flags] 7 | public enum SemVerType 8 | { 9 | SemVer1 = 0, 10 | VersionHasPrereleaseDots = 1 << 0, 11 | VersionHasBuildMetadata = 1 << 1, 12 | DependencyMinHasPrereleaseDots = 1 << 2, 13 | DependencyMinHasBuildMetadata = 1 << 3, 14 | DependencyMaxHasPrereleaseDots = 1 << 4, 15 | DependencyMaxHasBuildMetadata = 1 << 5, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Logic/FileSystemHttpCache/FileSystemHttpCacheEntryType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.FileSystemHttpCache 7 | { 8 | public enum FileSystemHttpCacheEntryType 9 | { 10 | KeyInfo, 11 | RequestHeaders, 12 | RequestBody, 13 | ResponseHeaders, 14 | ResponseBody, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Logic/FileSystemHttpCache/FileSystemHttpCacheKey.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.FileSystemHttpCache 7 | { 8 | public record FileSystemHttpCacheKey( 9 | string Directory, 10 | string Key, 11 | string? KeyInfo, 12 | string? KeyInfoExtension, 13 | string RequestBodyExtension, 14 | string ResponseBodyExtension); 15 | } 16 | -------------------------------------------------------------------------------- /src/Logic/FileSystemHttpCache/FileSystemHttpCacheMode.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.FileSystemHttpCache 7 | { 8 | public enum FileSystemHttpCacheMode 9 | { 10 | Disabled = 1, 11 | ReadAndWriteOnlyMissing, 12 | WriteOnlyMissing, 13 | WriteAlways, 14 | ReadOnly, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Logic/Instrumentation/IMetric.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface IMetric 7 | { 8 | void TrackValue(double metricValue); 9 | bool TrackValue(double metricValue, string dimension1Value); 10 | bool TrackValue(double metricValue, string dimension1Value, string dimension2Value); 11 | bool TrackValue(double metricValue, string dimension1Value, string dimension2Value, string dimension3Value); 12 | bool TrackValue(double metricValue, string dimension1Value, string dimension2Value, string dimension3Value, string dimension4Value); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Logic/Leasing/StorageLeaseException.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class StorageLeaseException : Exception 7 | { 8 | public StorageLeaseException(string message, Exception innerException) : base(message, innerException) 9 | { 10 | } 11 | 12 | public StorageLeaseException(string message) : base(message) 13 | { 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Logic/MemoryStorage/DateTimeOffsetExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | using Azure; 7 | 8 | namespace NuGet.Insights.MemoryStorage 9 | { 10 | public static class DateTimeOffsetExtensions 11 | { 12 | public static ETag ToMemoryETag(this DateTimeOffset lastModified, bool weak) 13 | { 14 | if (weak) 15 | { 16 | return new ETag($"W/\"{lastModified.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.FFFFFFF}Z\""); 17 | } 18 | else 19 | { 20 | return new ETag($"\"{lastModified.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.FFFFFFF}Z\""); 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Logic/MessagePack/ReadableKey.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public static class ReadableKey 7 | { 8 | public static ReadableKey Create(string key, T Value) 9 | { 10 | return new ReadableKey(key, Value); 11 | } 12 | } 13 | 14 | public class ReadableKey 15 | { 16 | public ReadableKey(string key, T value) 17 | { 18 | Key = key; 19 | Value = value; 20 | } 21 | 22 | public string Key { get; set; } 23 | public T Value { get; set; } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/Logic/Network/HttpMessageExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights 7 | { 8 | public static class HttpMessageExtensions 9 | { 10 | public static ILookup GetHeaderLookup(this HttpResponseMessage response) 11 | { 12 | return Enumerable.Empty>() 13 | .Concat(response.Headers.NonValidated) 14 | .Concat(response.Content.Headers.NonValidated) 15 | .SelectMany(x => x.Value.Select(y => new { x.Key, Value = y })) 16 | .ToLookup(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Logic/Network/INuGetInsightsHttpMessageHandlerFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface INuGetInsightsHttpMessageHandlerFactory 7 | { 8 | DelegatingHandler Create(); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Logic/Network/IThrottle.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface IThrottle : Protocol.IThrottle, Knapcode.MiniZip.IThrottle 7 | { 8 | new Task WaitAsync(); 9 | new void Release(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Logic/Network/NullThrottle.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class NullThrottle : IThrottle, Protocol.IThrottle, Knapcode.MiniZip.IThrottle 7 | { 8 | public static NullThrottle Instance { get; } = new NullThrottle(); 9 | 10 | public void Release() 11 | { 12 | } 13 | 14 | public Task WaitAsync() 15 | { 16 | return Task.CompletedTask; 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Logic/Network/SemaphoreSlimThrottle.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class SemaphoreSlimThrottle : IThrottle 7 | { 8 | private readonly SemaphoreSlim _semaphoreSlim; 9 | 10 | public SemaphoreSlimThrottle(SemaphoreSlim semaphoreSlim) 11 | { 12 | _semaphoreSlim = semaphoreSlim; 13 | } 14 | 15 | public int CurrentCount => _semaphoreSlim.CurrentCount; 16 | 17 | public Task WaitAsync() 18 | { 19 | return _semaphoreSlim.WaitAsync(); 20 | } 21 | 22 | public void Release() 23 | { 24 | _semaphoreSlim.Release(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Logic/Network/TimeoutHttpMessageHandler.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Microsoft.Extensions.Http; 5 | using Polly; 6 | 7 | #nullable enable 8 | 9 | namespace NuGet.Insights 10 | { 11 | public class TimeoutHttpMessageHandler : PolicyHttpMessageHandler 12 | { 13 | public TimeoutHttpMessageHandler(IOptions options) : base(GetPolicy(options)) 14 | { 15 | } 16 | 17 | private static IAsyncPolicy GetPolicy(IOptions options) 18 | { 19 | return Policy.TimeoutAsync(options.Value.HttpClientNetworkTimeout); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Logic/PackageStorage/PackageHashService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights 7 | { 8 | public class PackageHashService : PackageSpecificHashService 9 | { 10 | private readonly IOptions _options; 11 | 12 | public PackageHashService( 13 | PackageWideEntityService packageWideEntityService, 14 | IOptions options) : base(packageWideEntityService) 15 | { 16 | _options = options; 17 | } 18 | 19 | protected override string TableName => _options.Value.PackageHashesTableName; 20 | protected override bool MissingHashesIsDeleted => true; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Logic/PackageStorage/SymbolPackageHashService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights 7 | { 8 | public class SymbolPackageHashService : PackageSpecificHashService 9 | { 10 | private readonly IOptions _options; 11 | 12 | 13 | public SymbolPackageHashService( 14 | PackageWideEntityService packageWideEntityService, 15 | IOptions options) : base(packageWideEntityService) 16 | { 17 | _options = options; 18 | } 19 | 20 | protected override string TableName => _options.Value.SymbolPackageHashesTableName; 21 | protected override bool MissingHashesIsDeleted => false; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/CatalogIndex.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CatalogIndex 7 | { 8 | [JsonPropertyName("commitTimeStamp")] 9 | public DateTimeOffset CommitTimestamp { get; set; } 10 | 11 | [JsonPropertyName("count")] 12 | public int Count { get; set; } 13 | 14 | [JsonPropertyName("items")] 15 | public List Items { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/CatalogPackageDependency.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CatalogPackageDependency 7 | { 8 | [JsonPropertyName("id")] 9 | public string Id { get; set; } 10 | 11 | [JsonPropertyName("range")] 12 | [JsonConverter(typeof(CatalogPackageDependencyRangeConverter))] 13 | public string Range { get; set; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/CatalogPackageDependencyGroup.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CatalogPackageDependencyGroup 7 | { 8 | [JsonPropertyName("targetFramework")] 9 | public string TargetFramework { get; set; } 10 | 11 | [JsonPropertyName("dependencies")] 12 | public List Dependencies { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/CatalogPage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CatalogPage 7 | { 8 | [JsonPropertyName("@id")] 9 | public string Url { get; set; } 10 | 11 | [JsonPropertyName("commitTimeStamp")] 12 | public DateTimeOffset CommitTimestamp { get; set; } 13 | 14 | [JsonPropertyName("count")] 15 | public int Count { get; set; } 16 | 17 | [JsonPropertyName("items")] 18 | public List Items { get; set; } 19 | 20 | [JsonPropertyName("parent")] 21 | public string Parent { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/CatalogPageItem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class CatalogPageItem 7 | { 8 | [JsonPropertyName("@id")] 9 | public string Url { get; set; } 10 | 11 | [JsonPropertyName("commitTimeStamp")] 12 | public DateTimeOffset CommitTimestamp { get; set; } 13 | 14 | [JsonPropertyName("count")] 15 | public int Count { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/IAsOfData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface IAsOfData : IAsyncDisposable 7 | { 8 | DateTimeOffset AsOfTimestamp { get; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/ICatalogLeafItem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface ICatalogLeafItem 7 | { 8 | public string PackageId { get; } 9 | public string PackageVersion { get; } 10 | CatalogLeafType LeafType { get; } 11 | public DateTimeOffset CommitTimestamp { get; } 12 | string Url { get; } 13 | string CommitId { get; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/IPackageIdentityCommit.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface IPackageIdentityCommit 7 | { 8 | string PackageId { get; } 9 | string PackageVersion { get; } 10 | CatalogLeafType LeafType { get; } 11 | DateTimeOffset? CommitTimestamp { get; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/PackageDeleteCatalogLeaf.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class PackageDeleteCatalogLeaf : CatalogLeaf 7 | { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/PackageDeprecation.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class PackageDeprecation 7 | { 8 | [JsonPropertyName("message")] 9 | public string Message { get; set; } 10 | 11 | [JsonPropertyName("reasons")] 12 | public List Reasons { get; set; } 13 | 14 | [JsonPropertyName("alternatePackage")] 15 | public PackageDeprecationAlternatePackage AlternatePackage { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/PackageDeprecationAlternatePackage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class PackageDeprecationAlternatePackage 7 | { 8 | [JsonPropertyName("id")] 9 | public string Id { get; set; } 10 | 11 | [JsonPropertyName("range")] 12 | public string Range { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/PackageEntry.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class PackageEntry 7 | { 8 | [JsonPropertyName("compressedLength")] 9 | [JsonConverter(typeof(PackageEntryLongConverter))] 10 | public long CompressedLength { get; set; } 11 | 12 | [JsonPropertyName("fullName")] 13 | public string FullName { get; set; } 14 | 15 | [JsonPropertyName("length")] 16 | [JsonConverter(typeof(PackageEntryLongConverter))] 17 | public long Length { get; set; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/PackageIdentityCommit.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class PackageIdentityCommit : IPackageIdentityCommit 7 | { 8 | public required string PackageId { get; set; } 9 | public required string PackageVersion { get; set; } 10 | public required CatalogLeafType LeafType { get; set; } 11 | public required DateTimeOffset? CommitTimestamp { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/PackageVulnerability.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class PackageVulnerability 7 | { 8 | [JsonPropertyName("@id")] 9 | public string AtId { get; set; } 10 | 11 | [JsonPropertyName("advisoryUrl")] 12 | public string AdvisoryUrl { get; set; } 13 | 14 | [JsonPropertyName("severity")] 15 | public string Severity { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/ReadmeType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public enum ReadmeType 7 | { 8 | None, 9 | Legacy, 10 | Embedded, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Logic/Protocol/Data/ServiceIndexTypes.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public static class ServiceIndexTypes 7 | { 8 | public const string FlatContainer = "PackageBaseAddress/3.0.0"; 9 | public const string Catalog = "Catalog/3.0.0"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Logic/Protocol/IRemoteCursorClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface IRemoteCursorClient 7 | { 8 | Task GetFlatContainerAsync(CancellationToken token = default); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Logic/Protocol/SymbolPackageClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class SymbolPackageClient 7 | { 8 | private readonly IOptions _options; 9 | 10 | public SymbolPackageClient(IOptions options) 11 | { 12 | _options = options; 13 | } 14 | 15 | public string GetSymbolPackageUrl(string id, string version) 16 | { 17 | return $"{_options.Value.SymbolPackagesContainerBaseUrl.TrimEnd('/')}/" + 18 | $"{id.ToLowerInvariant()}." + 19 | $"{NuGetVersion.Parse(version).ToNormalizedString().ToLowerInvariant()}.snupkg"; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Logic/ReferenceTracking/IReference.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.ReferenceTracking 5 | { 6 | public interface IReference 7 | { 8 | string PartitionKey { get; } 9 | string RowKey { get; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Logic/ReferenceTracking/ReferenceTypes.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.ReferenceTracking 5 | { 6 | public static class ReferenceTypes 7 | { 8 | public const string Package = "P"; 9 | public const string Certificate = "C"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Logic/StorageNoOpRetry/DefaultRetryContext.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.StorageNoOpRetry 7 | { 8 | public sealed class DefaultRetryContext : RetryContext 9 | { 10 | public DefaultRetryContext(Guid clientRequestId) : base(clientRequestId) 11 | { 12 | } 13 | 14 | public DefaultRetryContext(RetryContext context) : base(context) 15 | { 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Logic/StorageNoOpRetry/ITableEntityWithClientRequestId.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Azure.Data.Tables; 5 | 6 | #nullable enable 7 | 8 | namespace NuGet.Insights.StorageNoOpRetry 9 | { 10 | public interface ITableEntityWithClientRequestId : ITableEntity 11 | { 12 | /// 13 | /// The client request ID used to last update the entity. This is used for no-oping some failed retries and is 14 | /// used for the x-ms-client-request-id header. 15 | /// 16 | public Guid? ClientRequestId { get; set; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Logic/StorageNoOpRetry/RetryContext.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.StorageNoOpRetry 7 | { 8 | public abstract class RetryContext 9 | { 10 | public RetryContext(Guid clientRequestId) 11 | { 12 | ClientRequestId = clientRequestId; 13 | ClientRequestIdString = clientRequestId.ToString(); 14 | } 15 | 16 | internal RetryContext(RetryContext context) 17 | { 18 | ClientRequestId = context.ClientRequestId; 19 | ClientRequestIdString = context.ClientRequestIdString; 20 | } 21 | 22 | public Guid ClientRequestId { get; } 23 | public string ClientRequestIdString { get; } 24 | public int Attempts { get; set; } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Logic/StorageNoOpRetry/TableRetryContext.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.StorageNoOpRetry 7 | { 8 | public abstract class TableRetryContext : RetryContext 9 | { 10 | public TableRetryContext( 11 | Guid clientRequestId, 12 | TableClientWithRetryContext client, 13 | string clientRequestIdColumn) : base(clientRequestId) 14 | { 15 | Client = client; 16 | ClientRequestIdColumn = clientRequestIdColumn; 17 | } 18 | 19 | public TableClientWithRetryContext Client { get; } 20 | public string ClientRequestIdColumn { get; } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Logic/StorageNoOpRetry/TableRetryEntityContext.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.StorageNoOpRetry 7 | { 8 | public class TableRetryEntityContext : TableRetryContext 9 | { 10 | public TableRetryEntityContext( 11 | Guid clientRequestId, 12 | TableClientWithRetryContext client, 13 | string clientRequestIdColumn, 14 | ITableEntityWithClientRequestId entity) : base(clientRequestId, client, clientRequestIdColumn) 15 | { 16 | Entity = entity; 17 | } 18 | 19 | public ITableEntityWithClientRequestId Entity { get; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Logic/Support/HashOutput.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class HashOutput 7 | { 8 | public byte[] MD5 { get; set; } 9 | public byte[] SHA1 { get; set; } 10 | public byte[] SHA256 { get; set; } 11 | public byte[] SHA512 { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Logic/Support/IIncrementalHash.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public interface IIncrementalHash : IDisposable 7 | { 8 | HashOutput Output { get; } 9 | void TransformBlock(byte[] inputBuffer, int inputOffset, int inputCount); 10 | void TransformFinalBlock(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Logic/Support/NullAsyncDisposable.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class NullAsyncDisposable : IAsyncDisposable 7 | { 8 | public static NullAsyncDisposable Instance { get; } = new NullAsyncDisposable(); 9 | 10 | public ValueTask DisposeAsync() 11 | { 12 | return ValueTask.CompletedTask; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Logic/Support/PathUtility.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public static class PathUtility 7 | { 8 | private static readonly char[] DirectorySeparators = new[] { Path.DirectorySeparatorChar, '/', '\\' }; 9 | 10 | public static string GetTopLevelFolder(string relativePath) 11 | { 12 | var firstSlashIndex = relativePath.IndexOfAny(DirectorySeparators); 13 | if (firstSlashIndex < 0) 14 | { 15 | return null; 16 | } 17 | 18 | return relativePath.Substring(0, firstSlashIndex); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Logic/Support/ReferenceEqualityComparer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class ReferenceEqualityComparer : IEqualityComparer 7 | { 8 | public static ReferenceEqualityComparer Instance { get; } = new ReferenceEqualityComparer(); 9 | 10 | public bool Equals([AllowNull] T x, [AllowNull] T y) 11 | { 12 | return ReferenceEquals(x, y); 13 | } 14 | 15 | public int GetHashCode([DisallowNull] T obj) 16 | { 17 | return obj.GetHashCode(); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Logic/TablePrefixScan/TablePrefixScanStep.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.TablePrefixScan 5 | { 6 | public abstract class TablePrefixScanStep 7 | { 8 | protected TablePrefixScanStep(TableQueryParameters parameters, int depth) 9 | { 10 | Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); 11 | Depth = depth; 12 | } 13 | 14 | public TableQueryParameters Parameters { get; } 15 | public int Depth { get; } 16 | public abstract string DebuggerDisplay { get; } 17 | 18 | public override string ToString() 19 | { 20 | return DebuggerDisplay; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Logic/TempStream/TempStreamResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public enum TempStreamResultType 7 | { 8 | Success, 9 | NeedNewStream, 10 | SemaphoreNotAvailable, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Logic/WideEntities/WideEntityDeleteOperation.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.WideEntities 5 | { 6 | public class WideEntityDeleteOperation : WideEntityOperation 7 | { 8 | public WideEntityDeleteOperation(WideEntity existing) 9 | : base(existing.PartitionKey) 10 | { 11 | Existing = existing; 12 | } 13 | 14 | public WideEntity Existing { get; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Logic/WideEntities/WideEntityInsertOperation.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.WideEntities 5 | { 6 | public class WideEntityInsertOperation : WideEntityOperation 7 | { 8 | public WideEntityInsertOperation(string partitionKey, string rowKey, ReadOnlyMemory content) 9 | : base(partitionKey) 10 | { 11 | RowKey = rowKey; 12 | Content = content; 13 | } 14 | 15 | public string RowKey { get; } 16 | public ReadOnlyMemory Content { get; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Logic/WideEntities/WideEntityInsertOrReplaceOperation.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.WideEntities 5 | { 6 | public class WideEntityInsertOrReplaceOperation : WideEntityOperation 7 | { 8 | public WideEntityInsertOrReplaceOperation(string partitionKey, string rowKey, ReadOnlyMemory content) 9 | : base(partitionKey) 10 | { 11 | RowKey = rowKey; 12 | Content = content; 13 | } 14 | 15 | public string RowKey { get; } 16 | public ReadOnlyMemory Content { get; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Logic/WideEntities/WideEntityReplaceOperation.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.WideEntities 5 | { 6 | public class WideEntityReplaceOperation : WideEntityOperation 7 | { 8 | public WideEntityReplaceOperation(WideEntity existing, ReadOnlyMemory content) 9 | : base(existing.PartitionKey) 10 | { 11 | Existing = existing; 12 | Content = content; 13 | } 14 | 15 | public WideEntity Existing { get; } 16 | public ReadOnlyMemory Content { get; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Performance/Data/packagefiles_compact_3.csv.br: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/src/Performance/Data/packagefiles_compact_3.csv.br -------------------------------------------------------------------------------- /src/Performance/Performance.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | PreserveNewest 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/SourceGenerator/CsvPropertyModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.Collections.Immutable; 5 | using Microsoft.CodeAnalysis; 6 | 7 | #nullable enable 8 | 9 | namespace NuGet.Insights 10 | { 11 | public record CsvPropertyModel( 12 | string FullName, 13 | string Name, 14 | string Type, 15 | string PrettyType, 16 | ImmutableArray Locations, 17 | bool IsNullable, 18 | bool IsNullableEnum, 19 | bool IsEnum, 20 | string? UnderlyingEnumType, 21 | bool IsReferenceType, 22 | bool IsBucketKey, 23 | bool IsKustoIgnore, 24 | bool IsRequired, 25 | bool IsKustoPartitionKey, 26 | string? KustoType); 27 | } 28 | -------------------------------------------------------------------------------- /src/SourceGenerator/CsvRecordModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.Collections.Immutable; 5 | using Microsoft.CodeAnalysis; 6 | 7 | #nullable enable 8 | 9 | namespace NuGet.Insights 10 | { 11 | public record CsvRecordModel( 12 | string? AssemblyName, 13 | string? KustoDDLName, 14 | string Namespace, 15 | string TypeKeyword, 16 | string Name, 17 | ImmutableArray Locations, 18 | EquatableList Properties); 19 | } 20 | -------------------------------------------------------------------------------- /src/SourceGenerator/CsvRecordResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Microsoft.CodeAnalysis; 5 | 6 | #nullable enable 7 | 8 | namespace NuGet.Insights 9 | { 10 | public record CsvRecordResult(Diagnostic? Diagnostic, CsvRecordModel? Model) 11 | { 12 | public CsvRecordResult(Diagnostic diagnostic) : this(diagnostic, null) 13 | { 14 | } 15 | 16 | public CsvRecordResult(CsvRecordModel model) : this(null, model) 17 | { 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/SourceGenerator/IPropertyVisitor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Microsoft.CodeAnalysis; 5 | 6 | namespace NuGet.Insights 7 | { 8 | public interface IPropertyVisitor 9 | { 10 | void OnProperty(SourceProductionContext context, CsvRecordModel model, CsvPropertyModel property); 11 | void Finish(SourceProductionContext context, CsvRecordModel model); 12 | string GetResult(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/SourceGenerator/IsExternalInit.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.ComponentModel; 5 | 6 | #nullable enable 7 | 8 | namespace System.Runtime.CompilerServices 9 | { 10 | [EditorBrowsable(EditorBrowsableState.Never)] 11 | internal class IsExternalInit 12 | { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Tool/Commands/ICommand.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using McMaster.Extensions.CommandLineUtils; 5 | 6 | namespace NuGet.Insights.Tool 7 | { 8 | public interface ICommand 9 | { 10 | void Configure(CommandLineApplication app); 11 | Task ExecuteAsync(CancellationToken token); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Tool/Commands/SandboxCommand.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using McMaster.Extensions.CommandLineUtils; 5 | 6 | namespace NuGet.Insights.Tool 7 | { 8 | public class SandboxCommand : ICommand 9 | { 10 | public SandboxCommand() 11 | { 12 | } 13 | 14 | public void Configure(CommandLineApplication app) 15 | { 16 | } 17 | 18 | public async Task ExecuteAsync(CancellationToken token) 19 | { 20 | await Task.Yield(); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Tool/Support/MinimalConsoleLoggerExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.Tool; 5 | 6 | namespace Microsoft.Extensions.Logging 7 | { 8 | public static class MinimalConsoleLoggerExtensions 9 | { 10 | public static ILoggingBuilder AddMinimalConsole(this ILoggingBuilder builder) 11 | { 12 | builder.AddProvider(new MinimalConsoleLoggerProvider()); 13 | return builder; 14 | } 15 | 16 | public static ILoggerFactory AddMinimalConsole(this ILoggerFactory factory) 17 | { 18 | factory.AddProvider(new MinimalConsoleLoggerProvider()); 19 | return factory; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Tool/Support/MinimalConsoleLoggerProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Tool 5 | { 6 | public class MinimalConsoleLoggerProvider : ILoggerProvider 7 | { 8 | public ILogger CreateLogger(string categoryName) 9 | { 10 | return new MinimalConsoleLogger(categoryName); 11 | } 12 | 13 | public void Dispose() 14 | { 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Tool/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "Microsoft": "Information", 6 | "System": "Information" 7 | } 8 | }, 9 | "NuGetInsights": { 10 | "UseDevelopmentStorage": "true" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Tool/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/Website/Authorization/AllowListRequirement.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Microsoft.AspNetCore.Authorization; 5 | 6 | namespace NuGet.Insights.Website 7 | { 8 | public class AllowListRequirement : IAuthorizationRequirement 9 | { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Website/Authorization/AllowedObject.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Website 5 | { 6 | public class AllowedObject 7 | { 8 | public string TenantId { get; set; } = string.Empty; 9 | public string ObjectId { get; set; } = string.Empty; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Website/Models/CachedAdminViewModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Website 5 | { 6 | public record CachedAdminViewModel(DateTimeOffset AsOfTimestamp, AdminViewModel Data); 7 | } 8 | -------------------------------------------------------------------------------- /src/Website/Models/ErrorViewModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Website 5 | { 6 | public class ErrorViewModel 7 | { 8 | public string RequestId { get; set; } 9 | 10 | public bool ShowRequestId => !string.IsNullOrEmpty(RequestId); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Website/Models/IAdminViewModelCache.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Website 5 | { 6 | public interface IAdminViewModelCache 7 | { 8 | public bool Refreshing { get; } 9 | public CachedAdminViewModel Value { get; } 10 | public NuGetInsightsWebsiteSettings Settings { get; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Website/Models/MoveQueueMessagesState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Website 5 | { 6 | public enum MoveQueueMessagesState 7 | { 8 | None, 9 | Scheduled, 10 | Working, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Website/Models/TimedReprocessRunViewModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.Worker; 5 | using NuGet.Insights.Worker.TimedReprocess; 6 | 7 | namespace NuGet.Insights.Website 8 | { 9 | public class TimedReprocessRunViewModel 10 | { 11 | public TimedReprocessRun Run { get; set; } 12 | public List<(TimedReprocessCatalogScan ReprocessScan, CatalogIndexScan IndexScan)> Scans { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Website/Models/TimedReprocessViewModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.Worker.TimedReprocess; 5 | 6 | namespace NuGet.Insights.Website 7 | { 8 | public class TimedReprocessViewModel 9 | { 10 | public IReadOnlyList StaleBuckets { get; set; } 11 | public TimedReprocessDetails Details { get; set; } 12 | public List Runs { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Website/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Microsoft.AspNetCore; 5 | 6 | namespace NuGet.Insights.Website 7 | { 8 | public class Program 9 | { 10 | public static void Main(string[] args) 11 | { 12 | BuildWebHost(args).Run(); 13 | } 14 | 15 | public static IWebHost BuildWebHost(string[] args) 16 | { 17 | return WebHost 18 | .CreateDefaultBuilder(args) 19 | .UseStartup() 20 | .Build(); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Website/Views/Home/AccessDenied.cshtml: -------------------------------------------------------------------------------- 1 | @{ 2 | ViewData["Title"] = "Access Denied"; 3 | } 4 | 5 |

@ViewData["Title"]

6 | 7 |

Claims:

8 |
    9 | @foreach (var claim in User.Claims.OrderBy(x => x.Type, StringComparer.Ordinal)) 10 | { 11 |
  • 12 | Type: @claim.Type 13 |
      14 |
    • 15 | Value: @claim.Value 16 |
      • 17 |
    18 |
    • 19 | } 20 |
21 | -------------------------------------------------------------------------------- /src/Website/Views/Shared/_ValidationScriptsPartial.cshtml: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /src/Website/Views/_ViewImports.cshtml: -------------------------------------------------------------------------------- 1 | @using Humanizer 2 | @using NuGet.Insights 3 | @using NuGet.Insights.Website 4 | @using NuGet.Insights.Website.Components 5 | @using NuGet.Insights.Worker 6 | @using Microsoft.Extensions.Options 7 | @addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers 8 | -------------------------------------------------------------------------------- /src/Website/Views/_ViewStart.cshtml: -------------------------------------------------------------------------------- 1 | @{ 2 | Layout = "_Layout"; 3 | } 4 | -------------------------------------------------------------------------------- /src/Website/_Imports.razor: -------------------------------------------------------------------------------- 1 | @using Humanizer 2 | @using NuGet.Insights.Worker 3 | @using Microsoft.AspNetCore.Mvc 4 | @using Microsoft.AspNetCore.Mvc.Rendering 5 | @using Microsoft.AspNetCore.Mvc.ViewFeatures 6 | -------------------------------------------------------------------------------- /src/Website/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "AzureAd": { 3 | "ClientId": "", 4 | "Instance": "https://login.microsoftonline.com/", 5 | "TenantId": "common" 6 | }, 7 | "Logging": { 8 | "LogLevel": { 9 | "Default": "Debug", 10 | "Microsoft": "Information", 11 | "System": "Information" 12 | } 13 | }, 14 | "NuGetInsights": { 15 | "RestrictUsers": false, 16 | "UseDevelopmentStorage": "true" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Website/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/Website/wwwroot/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/src/Website/wwwroot/favicon.ico -------------------------------------------------------------------------------- /src/Website/wwwroot/lib/bootstrap-icons/fonts/bootstrap-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/src/Website/wwwroot/lib/bootstrap-icons/fonts/bootstrap-icons.woff -------------------------------------------------------------------------------- /src/Website/wwwroot/lib/bootstrap-icons/fonts/bootstrap-icons.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/src/Website/wwwroot/lib/bootstrap-icons/fonts/bootstrap-icons.woff2 -------------------------------------------------------------------------------- /src/Worker.Logic/AppendResults/CsvReaderResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class CsvReaderResult where T : ICsvRecord 7 | { 8 | public CsvReaderResult(CsvReaderResultType type, List records) 9 | { 10 | Type = type; 11 | Records = records; 12 | } 13 | 14 | public CsvReaderResultType Type { get; } 15 | public List Records { get; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/AppendResults/CsvReaderResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CsvReaderResultType 7 | { 8 | Success, 9 | BufferTooSmall, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/AppendResults/CsvRecordProducer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public record CsvRecordProducer( 7 | CsvRecordProducerType Type, 8 | CatalogScanDriverType? CatalogScanDriverType); 9 | } 10 | -------------------------------------------------------------------------------- /src/Worker.Logic/AppendResults/CsvRecordProducerType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CsvRecordProducerType 7 | { 8 | CatalogScanDriver, 9 | AuxiliaryFileUpdater, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/AppendResults/ICsvReader.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ICsvReader 7 | { 8 | string GetHeader() where T : ICsvRecord; 9 | CsvReaderResult GetRecords(TextReader reader, int bufferSize) where T : ICsvRecord; 10 | IEnumerable GetRecordsEnumerable(TextReader reader, int bufferSize) where T : ICsvRecord; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/ArchiveEntryHashToCsv/FileRecordResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum FileRecordResultType 7 | { 8 | Available, 9 | Deleted, 10 | DoesNotExist, 11 | InvalidZipEntry, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/ArchiveToCsv/ArchiveResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum ArchiveResultType 7 | { 8 | Available, 9 | Deleted, 10 | DoesNotExist, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogIndexScanMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class CatalogIndexScanMessage 7 | { 8 | [JsonPropertyName("t")] 9 | public CatalogScanDriverType DriverType { get; set; } 10 | 11 | [JsonPropertyName("i")] 12 | public string ScanId { get; set; } 13 | 14 | [JsonPropertyName("ac")] 15 | public int AttemptCount { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogIndexScanResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CatalogIndexScanResult 7 | { 8 | ExpandAllLeaves, 9 | ExpandLatestLeaves, 10 | ExpandLatestLeavesPerId, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogLeafScanMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class CatalogLeafScanMessage 7 | { 8 | [JsonPropertyName("s")] 9 | public string StorageSuffix { get; set; } 10 | 11 | [JsonPropertyName("p0")] 12 | public string ScanId { get; set; } 13 | 14 | [JsonPropertyName("p1")] 15 | public string PageId { get; set; } 16 | 17 | [JsonPropertyName("r")] 18 | public string LeafId { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogPageScanMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class CatalogPageScanMessage 7 | { 8 | [JsonPropertyName("s")] 9 | public string StorageSuffix { get; set; } 10 | 11 | [JsonPropertyName("p")] 12 | public string ScanId { get; set; } 13 | 14 | [JsonPropertyName("r")] 15 | public string PageId { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogPageScanResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CatalogPageScanResult 7 | { 8 | Processed, 9 | ExpandAllowDuplicates, 10 | ExpandRemoveDuplicates, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogPageScanState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CatalogPageScanState 7 | { 8 | Created, // Initial state before any work is done 9 | Expanding, // Expanding child entities in storage 10 | Enqueueing, // Enqueueing messages for child entities 11 | Complete, // The scan is complete, no more work is required 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogScanDriverType.Drivers.CryptoAPI.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial struct CatalogScanDriverType 7 | { 8 | /// 9 | /// Implemented by . Loads all certificate 10 | /// chain information from the package signature into storage and tracks the many-to-many relationship between 11 | /// packages and certificates. 12 | /// 13 | public static CatalogScanDriverType PackageCertificateToCsv { get; } = new CatalogScanDriverType(nameof(PackageCertificateToCsv)); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogScanDriverType.Drivers.NPE.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial struct CatalogScanDriverType 7 | { 8 | /// 9 | /// Implemented by . This driver runs 10 | /// NuGet Package Explorer (NPE) assembly and symbol verification logic. 11 | /// 12 | public static CatalogScanDriverType NuGetPackageExplorerToCsv { get; } = new CatalogScanDriverType(nameof(NuGetPackageExplorerToCsv)); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogScanServiceResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class CatalogScanServiceResult 7 | { 8 | public CatalogScanServiceResult(CatalogScanServiceResultType type, string dependencyName, CatalogIndexScan scan) 9 | { 10 | Type = type; 11 | DependencyName = dependencyName; 12 | Scan = scan; 13 | } 14 | 15 | public CatalogScanServiceResultType Type { get; } 16 | public string DependencyName { get; } 17 | public CatalogIndexScan Scan { get; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogScanServiceResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CatalogScanServiceResultType 7 | { 8 | AlreadyStarted, 9 | UnavailableLease, 10 | NewStarted, 11 | BlockedByDependency, 12 | MinAfterMax, 13 | FullyCaughtUpWithDependency, 14 | FullyCaughtUpWithMax, 15 | Disabled, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogScanToCsv/CatalogLeafToCsv/CatalogLeafToCsvMode.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum CatalogLeafToCsvMode 7 | { 8 | AllLeaves, 9 | LatestLeaves, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/CatalogScanToCsv/ICsvTemporaryStorage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ICsvTemporaryStorage 7 | { 8 | Task AppendAsync(string storageSuffix, IReadOnlyList records) where T : class, IAggregatedCsvRecord; 9 | Task FinalizeAsync(string storageSuffix); 10 | Task InitializeAsync(string storageSuffix); 11 | Task IsAggregateCompleteAsync(string aggregatePartitionKeyPrefix, string storageSuffix); 12 | Task StartAggregateAsync(string aggregatePartitionKeyPrefix, string storageSuffix); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/DriverResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum DriverResultType 7 | { 8 | Success, 9 | TryAgainLater, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/ICatalogLeafScanBatchDriver.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ICatalogLeafScanBatchDriver : ICatalogScanDriver 7 | { 8 | Task> ProcessLeavesAsync(IReadOnlyList leafScans); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/ICatalogLeafScanNonBatchDriver.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ICatalogLeafScanNonBatchDriver : ICatalogScanDriver 7 | { 8 | Task ProcessLeafAsync(CatalogLeafScan leafScan); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/ICatalogScanDriverFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface ICatalogScanDriverFactory 9 | { 10 | ICatalogScanDriver Create(CatalogScanDriverType driverType); 11 | ICatalogLeafScanBatchDriver? CreateBatchDriverOrNull(CatalogScanDriverType driverType); 12 | ICatalogLeafScanNonBatchDriver CreateNonBatchDriver(CatalogScanDriverType driverType); 13 | ICatalogLeafScanNonBatchDriver? CreateNonBatchDriverOrNull(CatalogScanDriverType driverType); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/LatestLeaf/ILatestPackageLeaf.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.StorageNoOpRetry; 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface ILatestPackageLeaf : ITableEntityWithClientRequestId 9 | { 10 | string PackageId { get; } 11 | string PackageVersion { get; } 12 | CatalogLeafType LeafType { get; } 13 | DateTimeOffset CommitTimestamp { get; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/LatestLeaf/ILatestPackageLeafStorage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.StorageNoOpRetry; 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface ILatestPackageLeafStorage where T : ILatestPackageLeaf 9 | { 10 | TableClientWithRetryContext Table { get; } 11 | string CommitTimestampColumnName { get; } 12 | EntityUpsertStrategy Strategy { get; } 13 | Task MapAsync(string partitionKey, string rowKey, ICatalogLeafItem item); 14 | (string PartitionKey, string RowKey) GetKey(ICatalogLeafItem item); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Worker.Logic/CatalogScan/LatestLeaf/ILatestPackageLeafStorageFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ILatestPackageLeafStorageFactory where T : ILatestPackageLeaf 7 | { 8 | Task InitializeAsync(); 9 | Task DestroyAsync(); 10 | 11 | Task> CreateAsync( 12 | CatalogPageScan pageScan, 13 | IReadOnlyDictionary leafItemToRank); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/BuildVersionSet/CatalogLeafBatchData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using MessagePack; 5 | 6 | namespace NuGet.Insights.Worker.BuildVersionSet 7 | { 8 | [MessagePackObject] 9 | public record CatalogLeafBatchData 10 | { 11 | public CatalogLeafBatchData(DateTimeOffset maxCommitTimestamp, List leaves) 12 | { 13 | MaxCommitTimestamp = maxCommitTimestamp; 14 | Leaves = leaves; 15 | } 16 | 17 | [Key(0)] 18 | public DateTimeOffset MaxCommitTimestamp { get; } 19 | 20 | [Key(1)] 21 | public List Leaves { get; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/BuildVersionSet/CatalogLeafItemData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using MessagePack; 5 | 6 | namespace NuGet.Insights.Worker.BuildVersionSet 7 | { 8 | [MessagePackObject] 9 | public record CatalogLeafItemData 10 | { 11 | public CatalogLeafItemData(string id, string version, bool isDeleted) 12 | { 13 | Id = id; 14 | Version = version; 15 | IsDeleted = isDeleted; 16 | } 17 | 18 | [Key(0)] 19 | public string Id { get; } 20 | 21 | [Key(1)] 22 | public string Version { get; } 23 | 24 | [Key(2)] 25 | public bool IsDeleted { get; } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/BuildVersionSet/EntityReferenceCounter.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.BuildVersionSet 5 | { 6 | internal abstract class EntityReferenceCounter 7 | { 8 | public int Count; 9 | public abstract void Clear(); 10 | } 11 | 12 | internal class EntityReferenceCounter : EntityReferenceCounter 13 | { 14 | public T Value; 15 | 16 | public override void Clear() 17 | { 18 | Value = default; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/BuildVersionSet/IVersionSet.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.BuildVersionSet 5 | { 6 | public interface IVersionSet 7 | { 8 | DateTimeOffset CommitTimestamp { get; } 9 | IReadOnlyCollection GetUncheckedIds(); 10 | IReadOnlyCollection GetUncheckedVersions(string id); 11 | bool TryGetId(string id, out string outId); 12 | bool TryGetVersion(string id, string version, out string outVersion); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/BuildVersionSet/IVersionSetProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.BuildVersionSet 5 | { 6 | public interface IVersionSetProvider 7 | { 8 | Task> GetAsync(); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/CatalogDataToCsv/PackageDeprecationResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.CatalogDataToCsv 5 | { 6 | public enum PackageDeprecationResultType 7 | { 8 | NotDeprecated, 9 | Deprecated, 10 | Deleted, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/CatalogDataToCsv/PackageVulnerabilityResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.CatalogDataToCsv 5 | { 6 | public enum PackageVulnerabilityResultType 7 | { 8 | NotVulnerable, 9 | Vulnerable, 10 | Deleted, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/Internal_FindLatestCatalogLeafScanPerId/CatalogLeafScanPerId.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.FindLatestCatalogLeafScanPerId 5 | { 6 | public class CatalogLeafScanPerId : CatalogLeafScan 7 | { 8 | public CatalogLeafScanPerId() 9 | { 10 | } 11 | 12 | public CatalogLeafScanPerId(string partitionKey, string rowKey, string storageSuffix, string scanId, string pageId) 13 | : base(partitionKey, rowKey, storageSuffix, scanId, pageId) 14 | { 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/NuGetPackageExplorerToCsv/GitHubSourceRepo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.NuGetPackageExplorerToCsv 5 | { 6 | public record GitHubSourceRepo : SourceUrlRepo 7 | { 8 | public override string Type => "GitHub"; 9 | public string Owner { get; init; } 10 | public string Repo { get; init; } 11 | public string Ref { get; init; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/NuGetPackageExplorerToCsv/InvalidSourceRepo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.NuGetPackageExplorerToCsv 5 | { 6 | public record InvalidSourceRepo : SourceUrlRepo 7 | { 8 | public override string Type => "Invalid"; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/NuGetPackageExplorerToCsv/NuGetPackageExplorerResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.NuGetPackageExplorerToCsv 5 | { 6 | public enum NuGetPackageExplorerResultType 7 | { 8 | Deleted, 9 | Available, 10 | Timeout, 11 | InvalidMetadata, 12 | NothingToValidate, 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/NuGetPackageExplorerToCsv/SourceUrlRepo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.NuGetPackageExplorerToCsv 5 | { 6 | [JsonPolymorphic] 7 | [JsonDerivedType(typeof(GitHubSourceRepo))] 8 | [JsonDerivedType(typeof(InvalidSourceRepo))] 9 | [JsonDerivedType(typeof(UnknownSourceRepo))] 10 | public abstract record SourceUrlRepo 11 | { 12 | public abstract string Type { get; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/NuGetPackageExplorerToCsv/SourceUrlRepoInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.NuGetPackageExplorerToCsv 5 | { 6 | public class SourceUrlRepoInfo 7 | { 8 | public SourceUrlRepo Repo { get; set; } 9 | public int FileCount { get; set; } 10 | public string Example { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/NuGetPackageExplorerToCsv/UnknownSourceRepo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.NuGetPackageExplorerToCsv 5 | { 6 | public record UnknownSourceRepo : SourceUrlRepo 7 | { 8 | public override string Type => "Unknown"; 9 | public string Host { get; init; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageAssemblyToCsv/PackageAssemblyResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageAssemblyToCsv 5 | { 6 | public enum PackageAssemblyResultType 7 | { 8 | NoAssemblies, 9 | Deleted, 10 | ValidAssembly, 11 | NotManagedAssembly, 12 | NoManagedMetadata, 13 | DoesNotContainAssembly, 14 | InvalidZipEntry, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageAssetToCsv/PackageAssetResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageAssetToCsv 5 | { 6 | public enum PackageAssetResultType 7 | { 8 | NoAssets, 9 | Error, 10 | AvailableAssets, 11 | Deleted, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageAssetToCsv/PatternSetType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageAssetToCsv 5 | { 6 | public enum PatternSetType 7 | { 8 | RuntimeAssemblies = 1, 9 | CompileRefAssemblies, 10 | CompileLibAssemblies, 11 | NativeLibraries, 12 | ResourceAssemblies, 13 | MSBuildFiles, 14 | MSBuildMultiTargetingFiles, 15 | ContentFiles, 16 | ToolsAssemblies, 17 | EmbedAssemblies, 18 | MSBuildTransitiveFiles, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCertificateToCsv/ChainInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.Security.Cryptography.X509Certificates; 5 | 6 | namespace NuGet.Insights.Worker.PackageCertificateToCsv 7 | { 8 | public class ChainInfo 9 | { 10 | public ChainInfo(List<(string Fingerprint, X509Certificate2 Certificate)> certificates) 11 | { 12 | Certificates = certificates; 13 | } 14 | 15 | public List<(string Fingerprint, X509Certificate2 Certificate)> Certificates { get; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCertificateToCsv/FingerprintKustoValidationProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.Worker.KustoIngestion; 5 | 6 | namespace NuGet.Insights.Worker.PackageCertificateToCsv 7 | { 8 | public class FingerprintKustoValidationProvider : BaseKustoValidationProvider, IKustoValidationProvider 9 | { 10 | public FingerprintKustoValidationProvider(CsvRecordContainers containers) : base(containers) 11 | { 12 | } 13 | 14 | public async Task> GetValidationsAsync() 15 | { 16 | return await GetSetValidationsAsync(nameof(CertificateRecord.Fingerprint), required: false); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCertificateToCsv/PackageCertificateResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageCertificateToCsv 5 | { 6 | public enum PackageCertificateResultType 7 | { 8 | Available, 9 | Deleted, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCertificateToCsv/X509Extensions/X509KeyUsageExtensionInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.Security.Cryptography.X509Certificates; 5 | 6 | namespace NuGet.Insights.Worker.PackageCertificateToCsv 7 | { 8 | public class X509KeyUsageExtensionInfo : X509ExtensionInfo 9 | { 10 | public X509KeyUsageExtensionInfo(X509KeyUsageExtension extension) : base(extension, recognized: true) 11 | { 12 | KeyUsages = extension.KeyUsages.ToString(); 13 | } 14 | 15 | public string KeyUsages { get; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCertificateToCsv/X509Extensions/X509SubjectKeyIdentifierExtensionInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.Security.Cryptography.X509Certificates; 5 | 6 | namespace NuGet.Insights.Worker.PackageCertificateToCsv 7 | { 8 | public class X509SubjectKeyIdentifierExtensionInfo : X509ExtensionInfo 9 | { 10 | public X509SubjectKeyIdentifierExtensionInfo(X509SubjectKeyIdentifierExtension extension) : base(extension, recognized: true) 11 | { 12 | SubjectKeyIdentifier = extension.SubjectKeyIdentifier; 13 | } 14 | 15 | public string SubjectKeyIdentifier { get; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCertificateToCsv/X509Policies/X509CpsPolicyQualifierInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.Worker.PackageCertificateToCsv 7 | { 8 | public class X509CpsPolicyQualifierInfo : X509PolicyQualifierInfo 9 | { 10 | public X509CpsPolicyQualifierInfo(string policyQualifierId, string qualifier, string cpsUri) : base(policyQualifierId, qualifier, recognized: true) 11 | { 12 | CpsUri = cpsUri; 13 | } 14 | 15 | public string CpsUri { get; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageCompatibilityToCsv/PackageCompatibilityResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageCompatibilityToCsv 5 | { 6 | public enum PackageCompatibilityResultType 7 | { 8 | Available, 9 | Deleted, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageContentToCsv/PackageContentResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageContentToCsv 5 | { 6 | public enum PackageContentResultType 7 | { 8 | Deleted, 9 | NoContent, 10 | AllLoaded, 11 | PartiallyLoaded, 12 | InvalidZipEntry, 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageIconToCsv/PackageIconResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageIconToCsv 5 | { 6 | public enum PackageIconResultType 7 | { 8 | Available, 9 | NoIcon, 10 | Deleted, 11 | Error, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageLicenseToCsv/PackageLicenseResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageLicenseToCsv 5 | { 6 | public enum PackageLicenseResultType 7 | { 8 | Deleted, 9 | None, 10 | Url, 11 | Expression, 12 | File, 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageManifestToCsv/PackageManifestRecordResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageManifestToCsv 5 | { 6 | public enum PackageManifestRecordResultType 7 | { 8 | Deleted, 9 | Available, 10 | Error, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageReadmeToCsv/PackageReadmeResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageReadmeToCsv 5 | { 6 | public enum PackageReadmeResultType 7 | { 8 | Deleted, 9 | None, 10 | Legacy, 11 | Embedded, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageSignatureToCsv/PackageSignatureResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageSignatureToCsv 5 | { 6 | public enum PackageSignatureResultType 7 | { 8 | Deleted, 9 | Available, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/Drivers/PackageVersionToCsv/PackageVersionResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.PackageVersionToCsv 5 | { 6 | public enum PackageVersionResultType 7 | { 8 | Deleted, 9 | Available, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/AuxiliaryFileUpdater/AuxiliaryFileUpdaterMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.AuxiliaryFileUpdater 5 | { 6 | public class AuxiliaryFileUpdaterMessage : ITaskStateMessage where T : IAuxiliaryFileCsvRecord 7 | { 8 | [JsonPropertyName("ts")] 9 | public TaskStateKey TaskStateKey { get; set; } 10 | 11 | [JsonPropertyName("ac")] 12 | public int AttemptCount { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/AuxiliaryFileUpdater/IAuxiliaryFileUpdaterService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.AuxiliaryFileUpdater 5 | { 6 | public interface IAuxiliaryFileUpdaterService : IAuxiliaryFileUpdaterService 7 | where TRecord : IAuxiliaryFileCsvRecord 8 | { 9 | } 10 | 11 | public interface IAuxiliaryFileUpdaterService 12 | { 13 | Task InitializeAsync(); 14 | Task DestroyAsync(); 15 | Task StartAsync(); 16 | Task IsRunningAsync(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/AuxiliaryFileUpdater/IAuxiliaryFileUpdaterTimer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.AuxiliaryFileUpdater 5 | { 6 | public interface IAuxiliaryFileUpdaterTimer : ITimer 7 | { 8 | string ContainerName { get; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/CsvCompact/CsvCompactMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class CsvCompactMessage : ITaskStateMessage where T : ICsvRecord 7 | { 8 | [JsonPropertyName("s")] 9 | public string SourceTable { get; set; } 10 | 11 | [JsonPropertyName("b")] 12 | public int Bucket { get; set; } 13 | 14 | [JsonPropertyName("ts")] 15 | public TaskStateKey TaskStateKey { get; set; } 16 | 17 | [JsonPropertyName("ac")] 18 | public int AttemptCount { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/CsvCompact/ICsvResultStorage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public delegate List Prune(List records, bool isFinalPrune) where T : ICsvRecord; 7 | 8 | public delegate string GetBucketKey(T record) where T : ICsvRecord; 9 | 10 | public interface ICsvResultStorage where T : IAggregatedCsvRecord 11 | { 12 | /// 13 | /// The Azure Blob Storage container name to write CSV results to. 14 | /// 15 | string ResultContainerName { get; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/DownloadsToCsv/IPackageDownloadRecord.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.DownloadsToCsv 5 | { 6 | public interface IPackageDownloadRecord : IAuxiliaryFileCsvRecord where T : IPackageDownloadRecord 7 | { 8 | public DateTimeOffset AsOfTimestamp { get; set; } 9 | 10 | public string LowerId { get; set; } 11 | 12 | [KustoPartitionKey] 13 | public string Identity { get; set; } 14 | 15 | public string Id { get; set; } 16 | public string Version { get; set; } 17 | public long Downloads { get; set; } 18 | public long TotalDownloads { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/GitHubUsageToCsv/GitHubUsageResultType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.GitHubUsageToCsv 5 | { 6 | public enum GitHubUsageResultType 7 | { 8 | NoGitHubDependent, 9 | GitHubDependent, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/HeterogeneousBulkEnqueueMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class HeterogeneousBulkEnqueueMessage 7 | { 8 | [JsonPropertyName("m")] 9 | public List Messages { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/HomogeneousBatchMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class HomogeneousBatchMessage 7 | { 8 | [JsonPropertyName("n")] 9 | public string SchemaName { get; set; } 10 | 11 | [JsonPropertyName("v")] 12 | public int SchemaVersion { get; set; } 13 | 14 | [JsonPropertyName("m")] 15 | public List Messages { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/HomogeneousBulkEnqueueMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class HomogeneousBulkEnqueueMessage 7 | { 8 | [JsonPropertyName("n")] 9 | public string SchemaName { get; set; } 10 | 11 | [JsonPropertyName("v")] 12 | public int SchemaVersion { get; set; } 13 | 14 | [JsonPropertyName("d")] 15 | public TimeSpan? NotBefore { get; set; } 16 | 17 | [JsonPropertyName("m")] 18 | public List Messages { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/IBatchMessageProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface IBatchMessageProcessor 7 | { 8 | Task> ProcessAsync(IReadOnlyList messages, long dequeueCount); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/IGenericMessageProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface IGenericMessageProcessor 7 | { 8 | Task ProcessSingleAsync(QueueType queue, ReadOnlyMemory message, long dequeueCount); 9 | Task ProcessBatchAsync(string schemaName, int schemaVersion, IReadOnlyList data, long dequeueCount); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/IMessageProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface IMessageProcessor 7 | { 8 | Task ProcessAsync(T message, long dequeueCount); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ITaskStateMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ITaskStateMessage 7 | { 8 | TaskStateKey TaskStateKey { get; } 9 | int AttemptCount { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ITaskStateMessageProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ITaskStateMessageProcessor where T : ITaskStateMessage 7 | { 8 | Task ProcessAsync(T message, TaskState taskState, long dequeueCount); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/IKustoValidationProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | /// 7 | /// A delegate to validate the data reader and return an error message if there is any. 8 | /// 9 | /// The data reader. 10 | /// The validation error message, or null if there is no error. 11 | public delegate string ValidateKustoDataReader(IDataReader reader); 12 | 13 | public record KustoValidation(string Label, string Query, ValidateKustoDataReader Validate); 14 | 15 | public interface IKustoValidationProvider 16 | { 17 | Task> GetValidationsAsync(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/KustoBlobIngestionMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public class KustoBlobIngestionMessage 7 | { 8 | public string StorageSuffix { get; set; } 9 | public string ContainerName { get; set; } 10 | public string BlobName { get; set; } 11 | public int AttemptCount { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/KustoBlobIngestionState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public enum KustoBlobIngestionState 7 | { 8 | Created, 9 | Working, 10 | Failed, 11 | TimedOut, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/KustoContainerIngestionMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public class KustoContainerIngestionMessage 7 | { 8 | public string StorageSuffix { get; set; } 9 | public string ContainerName { get; set; } 10 | public int AttemptCount { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/KustoContainerIngestionState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public enum KustoContainerIngestionState 7 | { 8 | Created, 9 | CreatingTable, 10 | Expanding, 11 | Enqueueing, 12 | Requeueing, 13 | Working, 14 | Complete, 15 | Failed, 16 | Retrying, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/KustoIngestionMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public class KustoIngestionMessage 7 | { 8 | [JsonPropertyName("i")] 9 | public string IngestionId { get; set; } 10 | 11 | [JsonPropertyName("ac")] 12 | public int AttemptCount { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/PackageIdentityKustoValidationProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public class PackageIdentityKustoValidationProvider : BaseKustoValidationProvider, IKustoValidationProvider 7 | { 8 | public PackageIdentityKustoValidationProvider(CsvRecordContainers containers) : base(containers) 9 | { 10 | } 11 | 12 | public async Task> GetValidationsAsync() 13 | { 14 | return await GetSetValidationsAsync(nameof(PackageRecord.Identity), required: true); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/KustoIngestion/PackageLowerIdKustoValidationProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.KustoIngestion 5 | { 6 | public class PackageLowerIdKustoValidationProvider : BaseKustoValidationProvider, IKustoValidationProvider 7 | { 8 | public PackageLowerIdKustoValidationProvider(CsvRecordContainers containers) : base(containers) 9 | { 10 | } 11 | 12 | public async Task> GetValidationsAsync() 13 | { 14 | return await GetSetValidationsAsync(nameof(PackageRecord.LowerId), required: true); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/CleanupOrphanRecordsMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public class CleanupOrphanRecordsMessage : ITaskStateMessage 7 | where T : ICleanupOrphanCsvRecord 8 | { 9 | [JsonPropertyName("ts")] 10 | public TaskStateKey TaskStateKey { get; set; } 11 | 12 | [JsonPropertyName("ac")] 13 | public int AttemptCount { get; set; } 14 | 15 | [JsonPropertyName("ci")] 16 | public string CleanupId { get; set; } 17 | 18 | [JsonPropertyName("ss")] 19 | public string StorageSuffix { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/CleanupOrphanRecordsParameters.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public class CleanupOrphanRecordsParameters 7 | { 8 | [JsonPropertyName("s")] 9 | public CleanupOrphanRecordsState State { get; set; } 10 | 11 | [JsonPropertyName("pk")] 12 | public string LastPartitionKey { get; set; } 13 | 14 | [JsonPropertyName("rk")] 15 | public string LastRowKey { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/CleanupOrphanRecordsState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public enum CleanupOrphanRecordsState 7 | { 8 | Created, 9 | FindingOrphans, 10 | StartingAggregate, 11 | Aggregating, 12 | Finalizing, 13 | Deleting, 14 | Complete, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/ICleanupOrphanCsvRecord.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public interface ICleanupOrphanCsvRecord : ICsvRecord 7 | { 8 | static abstract string CleanupOrphanRecordsMessageSchemaName { get; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/ICleanupOrphanRecordsAdapter.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.ReferenceTracking; 5 | 6 | namespace NuGet.Insights.Worker.ReferenceTracking 7 | { 8 | public interface ICleanupOrphanRecordsAdapter where T : ICleanupOrphanCsvRecord 9 | { 10 | string OwnerToSubjectTableName { get; } 11 | string SubjectToOwnerTableName { get; } 12 | string OperationName { get; } 13 | string OwnerType { get; } 14 | string SubjectType { get; } 15 | IReadOnlyList MapToOrphanRecords(IReadOnlyList subjects); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/ICleanupOrphanRecordsService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public interface ICleanupOrphanRecordsService : ICleanupOrphanRecordsService where T : ICleanupOrphanCsvRecord 7 | { 8 | } 9 | 10 | public interface ICleanupOrphanRecordsService 11 | { 12 | Task InitializeAsync(); 13 | Task IsRunningAsync(); 14 | Task StartAsync(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/ReferenceTracking/ICleanupOrphanRecordsTimer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public interface ICleanupOrphanRecordsTimer : ITimer 7 | { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/TaskStateProcessResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum TaskStateProcessResult 7 | { 8 | /// 9 | /// The work is done for this task. The message should be marked as completed. 10 | /// 11 | Complete, 12 | 13 | /// 14 | /// The work is not done and trying again should be delayed. The message should be requeued with a higher delay. 15 | /// 16 | Delay, 17 | 18 | /// 19 | /// The work is not done and trying again should happen soon. The message should be requeued with minimal delay. 20 | /// 21 | Continue, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/TimedReprocess/TimedReprocessMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.TimedReprocess 5 | { 6 | public class TimedReprocessMessage 7 | { 8 | [JsonPropertyName("i")] 9 | public string RunId { get; set; } 10 | 11 | [JsonPropertyName("ac")] 12 | public int AttemptCount { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/TimedReprocess/TimedReprocessState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.TimedReprocess 5 | { 6 | public enum TimedReprocessState 7 | { 8 | Created, 9 | Working, 10 | Finalizing, 11 | Aborted, 12 | Complete, 13 | } 14 | 15 | 16 | public static class TimedReprocessStateExtensions 17 | { 18 | public static bool IsTerminal(this TimedReprocessState state) 19 | { 20 | return state == TimedReprocessState.Aborted 21 | || state == TimedReprocessState.Complete; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/Workflow/WorkflowRunMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.Workflow 5 | { 6 | public class WorkflowRunMessage 7 | { 8 | [JsonPropertyName("i")] 9 | public string RunId { get; set; } 10 | 11 | [JsonPropertyName("ac")] 12 | public int AttemptCount { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/MessageProcessors/Workflow/WorkflowRunState.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.Workflow 5 | { 6 | public enum WorkflowRunState 7 | { 8 | Created, 9 | TimedReprocessWorking, 10 | CatalogScanWorking, 11 | CleanupOrphanRecordsWorking, 12 | AuxiliaryFilesWorking, 13 | KustoIngestionWorking, 14 | Finalizing, 15 | Aborted, 16 | Complete, 17 | } 18 | 19 | public static class WorkflowRunStateExtensions 20 | { 21 | public static bool IsTerminal(this WorkflowRunState state) 22 | { 23 | return state == WorkflowRunState.Complete || state == WorkflowRunState.Aborted; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Worker.Logic/NuGetInsightsWorkerSettings.AggregateCsv.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial class NuGetInsightsWorkerSettings 7 | { 8 | public int AppendResultStorageBucketCount { get; set; } = 1000; // this is the maximum number of blobs fetched in a single request 9 | public int AppendResultBigModeRecordThreshold { get; set; } = 25_000; 10 | public int AppendResultBigModeSubdivisionSize { get; set; } = 10_000; 11 | 12 | public string CsvRecordTableNamePrefix { get; set; } = "csvrecord"; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/NuGetInsightsWorkerSettings.Drivers.CryptoAPI.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial class NuGetInsightsWorkerSettings 7 | { 8 | public string PackageToCertificateTableName { get; set; } = "packagetocertificates"; 9 | public string CertificateToPackageTableName { get; set; } = "certificatetopackages"; 10 | public string PackageCertificateContainerName { get; set; } = "packagecertificates"; 11 | public string CertificateContainerName { get; set; } = "certificates"; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Worker.Logic/NuGetInsightsWorkerSettings.Drivers.NPE.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial class NuGetInsightsWorkerSettings 7 | { 8 | public string NuGetPackageExplorerContainerName { get; set; } = "nugetpackageexplorer"; 9 | public string NuGetPackageExplorerFileContainerName { get; set; } = "nugetpackageexplorerfiles"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/NuGetInsightsWorkerSettings.Queues.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial class NuGetInsightsWorkerSettings 7 | { 8 | public bool UseBulkEnqueueStrategy { get; set; } = true; 9 | public int BulkEnqueueThreshold { get; set; } = 10; 10 | public int EnqueueWorkers { get; set; } = 1; 11 | public int MaxBulkEnqueueMessageCount { get; set; } = 50; 12 | public bool AllowBatching { get; set; } = true; 13 | public TimeSpan MaxMessageDelay { get; set; } = TimeSpan.MaxValue; 14 | 15 | public string WorkQueueName { get; set; } = "work"; 16 | public string ExpandQueueName { get; set; } = "expand"; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Worker.Logic/NuGetInsightsWorkerSettings.Workflow.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial class NuGetInsightsWorkerSettings 7 | { 8 | public string WorkflowRunTableName { get; set; } = "workflowruns"; 9 | 10 | public int OldWorkflowRunsToKeep { get; set; } = 49; 11 | 12 | public int WorkflowMaxAttempts { get; set; } = 5; 13 | 14 | public string WorkflowFrequency { get; set; } = TimeSpan.FromDays(1).ToString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Worker.Logic/PackageRecord/IPackageCommitRecord.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface IPackageCommitRecord 9 | { 10 | string Id { get; set; } 11 | string Version { get; set; } 12 | DateTimeOffset CatalogCommitTimestamp { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/PackageRecord/IPackageEntryRecord.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface IPackageEntryRecord : IPackageRecord 9 | { 10 | int? SequenceNumber { get; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/PackageRecord/IPackageRecord.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | #nullable enable 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface IPackageRecord : IPackageCommitRecord 9 | { 10 | Guid? ScanId { get; set; } 11 | 12 | DateTimeOffset? ScanTimestamp { get; set; } 13 | 14 | string LowerId { get; set; } 15 | 16 | string Identity { get; set; } 17 | 18 | DateTimeOffset? Created { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/Queues/IMessageBatcher.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface IMessageBatcher 7 | { 8 | Task> BatchOrNullAsync(IReadOnlyList messages, ISchemaSerializer serializer); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Worker.Logic/Queues/IWorkerQueueFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Azure.Storage.Queues; 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public interface IWorkerQueueFactory 9 | { 10 | Task InitializeAsync(); 11 | Task GetQueueAsync(QueueType type); 12 | Task GetPoisonQueueAsync(QueueType type); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/Queues/QueueType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum QueueType 7 | { 8 | /// 9 | /// The main queue used for processing units of work. When in doubt, enqueue to this one. 10 | /// 11 | Work, 12 | 13 | /// 14 | /// The queue used for expanding messages into more messages, with relatively low effort. Ideally this queue 15 | /// remains relatively small and workers can pull messages off this queue to fill up the 16 | /// queue. 17 | /// 18 | Expand, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Worker.Logic/Serialization/ISchemaDeserializer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ISchemaDeserializer 7 | { 8 | string Name { get; } 9 | Type Type { get; } 10 | object Deserialize(int schemaVersion, JsonElement data); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/Serialization/ISchemaSerializer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ISchemaSerializer 7 | { 8 | string Name { get; } 9 | int LatestVersion { get; } 10 | ISerializedEntity SerializeData(T message); 11 | ISerializedEntity SerializeMessage(T message); 12 | } 13 | 14 | public interface ISchemaSerializer 15 | { 16 | public string Name { get; } 17 | ISerializedEntity SerializeMessage(object message); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Worker.Logic/Serialization/ISerializedEntity.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ISerializedEntity 7 | { 8 | JsonElement AsJsonElement(); 9 | string AsString(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/Serialization/NameVersionMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class NameVersionMessage 7 | { 8 | [JsonConstructor] 9 | public NameVersionMessage(string schemaName, int schemaVersion, T data) 10 | { 11 | SchemaName = schemaName ?? throw new ArgumentNullException(nameof(schemaName)); 12 | SchemaVersion = schemaVersion; 13 | Data = data; 14 | } 15 | 16 | [JsonPropertyName("n")] 17 | public string SchemaName { get; } 18 | 19 | [JsonPropertyName("v")] 20 | public int SchemaVersion { get; } 21 | 22 | [JsonPropertyName("d")] 23 | public T Data { get; } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/Worker.Logic/ServiceCollectionExtensions.Drivers.CryptoAPI.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.Worker.PackageCertificateToCsv; 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public static partial class ServiceCollectionExtensions 9 | { 10 | /// 11 | /// Invoked via reflection in . 12 | /// 13 | private static void SetupPackageCertificateToCsvDriver(IServiceCollection serviceCollection) 14 | { 15 | serviceCollection.AddSingleton(); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Worker.Logic/ServiceCollectionExtensions.Drivers.NPE.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using NuGet.Insights.Worker.NuGetPackageExplorerToCsv; 5 | 6 | namespace NuGet.Insights.Worker 7 | { 8 | public static partial class ServiceCollectionExtensions 9 | { 10 | /// 11 | /// Invoked via reflection in . 12 | /// 13 | private static void SetupNuGetPackageExplorerToCsv(IServiceCollection serviceCollection) 14 | { 15 | serviceCollection.AddSingleton(); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/CopyBucketRange/CopyBucketRangeParameters.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.CopyBucketRange 5 | { 6 | public class CopyBucketRangeParameters 7 | { 8 | [JsonPropertyName("t")] 9 | public CatalogScanDriverType DriverType { get; set; } 10 | 11 | [JsonPropertyName("i")] 12 | public string ScanId { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/EnqueueCatalogLeafScan/EnqueueCatalogLeafScansParameters.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.EnqueueCatalogLeafScan 5 | { 6 | public class EnqueueCatalogLeafScansParameters 7 | { 8 | [JsonPropertyName("o")] 9 | public bool OneMessagePerId { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/ITableScanDriver.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public interface ITableScanDriver 7 | { 8 | IList SelectColumns { get; } 9 | Task InitializeAsync(JsonElement? parameters); 10 | Task ProcessEntitySegmentAsync(string tableName, JsonElement? parameters, IReadOnlyList entities); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/TableCopy/TableCopyParameters.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.TableCopy 5 | { 6 | public class TableCopyParameters 7 | { 8 | [JsonPropertyName("d")] 9 | public string DestinationTableName { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/TableCopy/TableRowCopyMessage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using Azure.Data.Tables; 5 | 6 | namespace NuGet.Insights.Worker.TableCopy 7 | { 8 | public class TableRowCopyMessage where T : class, ITableEntity, new() 9 | { 10 | [JsonPropertyName("s")] 11 | public string SourceTableName { get; set; } 12 | 13 | [JsonPropertyName("d")] 14 | public string DestinationTableName { get; set; } 15 | 16 | [JsonPropertyName("p")] 17 | public string PartitionKey { get; set; } 18 | 19 | [JsonPropertyName("r")] 20 | public List RowKeys { get; set; } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/TablePrefixScanPartitionKeyQueryParameters.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class TablePrefixScanPartitionKeyQueryParameters 7 | { 8 | [JsonPropertyName("sf")] 9 | public int SegmentsPerFirstPrefix { get; set; } 10 | 11 | [JsonPropertyName("ss")] 12 | public int SegmentsPerSubsequentPrefix { get; set; } 13 | 14 | [JsonPropertyName("d")] 15 | public int Depth { get; set; } 16 | 17 | [JsonPropertyName("p")] 18 | public string PartitionKey { get; set; } 19 | 20 | [JsonPropertyName("r")] 21 | public string RowKeySkip { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/TablePrefixScanStartParameters.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public class TablePrefixScanStartParameters 7 | { 8 | [JsonPropertyName("sf")] 9 | public int SegmentsPerFirstPrefix { get; set; } 10 | 11 | [JsonPropertyName("ss")] 12 | public int SegmentsPerSubsequentPrefix { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/TableScanDriverType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum TableScanDriverType 7 | { 8 | TableCopy, 9 | EnqueueCatalogLeafScans, 10 | CopyBucketRange, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Worker.Logic/TableScan/TableScanStrategy.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum TableScanStrategy 7 | { 8 | PrefixScan, 9 | Serial, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Worker/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensions": { 3 | "http": { 4 | "routePrefix": "" 5 | }, 6 | "queues": { 7 | "maxDequeueCount": 10, 8 | "messageEncoding": "none" 9 | } 10 | }, 11 | "functionTimeout": "00:10:00", 12 | "logging": { 13 | "applicationInsights": { 14 | "enableDependencyTracking": false, 15 | "enablePerformanceCountersCollection": false, 16 | "samplingSettings": { 17 | "isEnabled": false 18 | } 19 | }, 20 | "logLevel": { 21 | "Azure.Core": "Error", 22 | "default": "Information", 23 | "LoggerTraceListener": "Information", 24 | "Host.Results": "Critical", 25 | "Host.Startup": "Error" 26 | } 27 | }, 28 | "version": "2.0" 29 | } 30 | -------------------------------------------------------------------------------- /src/Worker/local.settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "IsEncrypted": false, 3 | "Values": { 4 | "AzureWebJobsStorage": "UseDevelopmentStorage=true", 5 | "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", 6 | "NUGET_INSIGHTS_ALLOW_ICU": "true", 7 | "NuGetInsights__ExpandQueueName": "expand", 8 | "NuGetInsights__UseDevelopmentStorage": "true", 9 | "NuGetInsights__WorkQueueName": "work", 10 | "QueueTriggerConnection": "UseDevelopmentStorage=true" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /submodules/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | -------------------------------------------------------------------------------- /submodules/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | NU1901;NU1902;NU1903;NU1904 4 | 5 | -------------------------------------------------------------------------------- /submodules/Directory.Build.targets: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /submodules/Directory.Packages.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | 5 | -------------------------------------------------------------------------------- /submodules/NuGet.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /test/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/.gitattributes: -------------------------------------------------------------------------------- 1 | *.nuspec binary 2 | *.json binary 3 | *.csv binary 4 | *.md binary 5 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/Cache/.gitattributes: -------------------------------------------------------------------------------- 1 | * binary 2 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/Cache/.gitignore: -------------------------------------------------------------------------------- 1 | !**/* -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/DownloadsToCsv/Step1/downloads.v1.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "Knapcode.TorSharp", 4 | [ 5 | "2.0.7", 6 | 138 7 | ], 8 | [ 9 | "foo", 10 | 13 11 | ] 12 | ], 13 | [ 14 | "Newtonsoft.Json", 15 | [ 16 | "10.5.0.0", 17 | 137 18 | ], 19 | [ 20 | "2.4.3.700", 21 | 406 22 | ] 23 | ] 24 | ] 25 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/DownloadsToCsv/Step1/downloads.v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Knapcode.TorSharp": { 3 | "2.0.7": 139, 4 | "foo": 14 5 | }, 6 | "Newtonsoft.Json": { 7 | "10.5.0.0": 138, 8 | "2.4.3.700": 407 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/DownloadsToCsv/Step2/downloads.v1.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "Knapcode.TorSharp", 4 | [ 5 | "2.0.7", 6 | 138 7 | ], 8 | [ 9 | "foo", 10 | 13 11 | ] 12 | ], 13 | [ 14 | "Newtonsoft.Json", 15 | [ 16 | "10.5.0.0", 17 | 139 18 | ], 19 | [ 20 | "2.4.3.700", 21 | 409 22 | ] 23 | ], 24 | [ 25 | "Knapcode.KitchenSink", 26 | [ 27 | "1.0.0-beta", 28 | 1 29 | ] 30 | ] 31 | ] 32 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/DownloadsToCsv/Step2/downloads.v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Knapcode.TorSharp": { 3 | "2.0.7": 139, 4 | "foo": 14 5 | }, 6 | "Newtonsoft.Json": { 7 | "10.5.0.0": 140, 8 | "2.4.3.700": 410 9 | }, 10 | "Knapcode.KitchenSink": { 11 | "1.0.0-beta": 2 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/DownloadsToCsv_Duplicates/Step1/downloads.v1.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "YourPackageId", 4 | [ 5 | "1.0.0", 6 | 3 7 | ] 8 | ], 9 | [ 10 | "foo", 11 | [ 12 | "2.0.0", 13 | 23 14 | ] 15 | ], 16 | [ 17 | "yourpackageid", 18 | [ 19 | "1.0.0", 20 | 62 21 | ] 22 | ] 23 | ] 24 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/DownloadsToCsv_UnicodeDuplicates/Step1/downloads.v1.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "Newtonsoft.Json", 4 | [ 5 | "9.0.1", 6 | 137 7 | ] 8 | ], 9 | [ 10 | "Sem-Tostão", 11 | [ 12 | "1.0.1", 13 | 100 14 | ] 15 | ], 16 | [ 17 | "Sem-Tostão", 18 | [ 19 | "1.0.1", 20 | 200 21 | ] 22 | ] 23 | ] 24 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/ExcludedPackagesToCsv/Step1/excludedPackages.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Microsoft.Extensions.Logging", 3 | "Knapcode.TorSharp", 4 | "Castle.Core", 5 | "Newtonsoft.Json" 6 | ] 7 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/ExcludedPackagesToCsv/Step2/excludedPackages.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Microsoft.Extensions.Logging", 3 | "Knapcode.TorSharp", 4 | "Knapcode.KitchenSink", 5 | "Newtonsoft.Json" 6 | ] 7 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/GitHubUsageToCsv/Step1/GitHubUsage.v1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Url": "https://github.com/xunit/xunit", 4 | "Stars": 4273, 5 | "Id": "xunit/xunit", 6 | "Description": "xUnit.net is a free, open source, community-focused unit testing tool for .NET.", 7 | "Dependencies": [ 8 | "Nerdbank.GitVersioning", 9 | "Microsoft.NET.Test.Sdk", 10 | "Moq" 11 | ] 12 | }, 13 | { 14 | "Url": "https://github.com/NuGet/NuGetGallery", 15 | "Stars": 1561, 16 | "Id": "NuGet/NuGetGallery", 17 | "Description": "NuGet Gallery is a package repository that powers https://www.nuget.org. Use this repo for reporting NuGet.org issues.", 18 | "Dependencies": [ 19 | "Knapcode.MiniZip", 20 | "CsvHelper", 21 | "Moq", 22 | "WindowsAzure.Storage", 23 | "NuGet.Versioning" 24 | ] 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/OwnersToCsv/Step1/owners.v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Microsoft.Extensions.Logging": [ 3 | "Microsoft", 4 | "aspnet", 5 | "dotnetframework" 6 | ], 7 | "Knapcode.TorSharp": [ 8 | "joelverhagen" 9 | ], 10 | "OwnerlessPackage": [], 11 | "Castle.Core": [ 12 | "castleproject" 13 | ], 14 | "Newtonsoft.Json": [ 15 | "newtonsoft", 16 | "jamesnk" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/OwnersToCsv/Step2/owners.v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Microsoft.Extensions.Logging": [ 3 | "Microsoft", 4 | "dotnetframework" 5 | ], 6 | "Knapcode.TorSharp": [ 7 | "joelverhagen" 8 | ], 9 | "OwnerlessPackage": [], 10 | "Knapcode.KitchenSink": [ 11 | "joelverhagen", 12 | "nuget" 13 | ], 14 | "Newtonsoft.Json": [ 15 | "newtonsoft", 16 | "jamesnk" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/PopularityTransfersToCsv/Step1/popularity-transfers.v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "WindowsAzure.Storage": [ 3 | "Azure.Storage.Blobs" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/PopularityTransfersToCsv/Step2/popularity-transfers.v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "WindowsAzure.ServiceBus": [ 3 | "Azure.Messaging.EventHubs", 4 | "Azure.Messaging.ServiceBus" 5 | ], 6 | "WindowsAzure.Storage": [ 7 | "Azure.Storage.Blobs", 8 | "Azure.Storage.Queues" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/VerifiedPackagesToCsv/Step1/verifiedPackages.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Microsoft.Extensions.Logging", 3 | "Knapcode.TorSharp", 4 | "Castle.Core", 5 | "Newtonsoft.Json" 6 | ] 7 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/VerifiedPackagesToCsv/Step2/verifiedPackages.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Microsoft.Extensions.Logging", 3 | "Knapcode.TorSharp", 4 | "Knapcode.KitchenSink", 5 | "Newtonsoft.Json" 6 | ] 7 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/WindowsAzure.Storage.9.3.3.md: -------------------------------------------------------------------------------- 1 | Development on this library has shifted focus to the Azure Unified SDK. You can find more information [here]( https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/README.md). These packages can be found below: 2 | 3 | [Azure.Storage.Blobs](https://www.nuget.org/packages/Azure.Storage.Blobs/) 4 | [Azure.Storage.Queues](https://www.nuget.org/packages/Azure.Storage.Queues/) 5 | [Azure.Storage.Blobs.Batch](https://www.nuget.org/packages/Azure.Storage.Blobs.Batch/) 6 | [Azure.Storage.Files.Shares](https://www.nuget.org/packages/Azure.Storage.Files.Shares/) 7 | **It is recommended that you move to the new package.** 8 | 9 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/behaviorsample.1.0.0.md: -------------------------------------------------------------------------------- 1 | # Example README 2 | 3 | This is an example readme. 4 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/behaviorsample.1.0.0.nupkg.testdata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/test/Logic.Test/TestInput/behaviorsample.1.0.0.nupkg.testdata -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/behaviorsample.1.0.0.nuspec: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | BehaviorSample 5 | 1.0.0 6 | BehaviorSample 7 | README.md 8 | Package Description 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/behaviorsample.1.0.0.snupkg.testdata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/test/Logic.Test/TestInput/behaviorsample.1.0.0.snupkg.testdata -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/behaviorsample.project.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/test/Logic.Test/TestInput/behaviorsample.project.zip -------------------------------------------------------------------------------- /test/Logic.Test/TestInput/deltax.1.0.0.nupkg.testdata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NuGet/Insights/1ccb450889e01cdadefa0ebd9826aa24ba224c80/test/Logic.Test/TestInput/deltax.1.0.0.nupkg.testdata -------------------------------------------------------------------------------- /test/Logic.Test/TestSupport/JsonElementComparer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class JsonElementComparer : IEqualityComparer 7 | { 8 | public static JsonElementComparer Instance { get; } = new JsonElementComparer(); 9 | 10 | public bool Equals(JsonElement x, JsonElement y) 11 | { 12 | return x.ToString() == y.ToString(); 13 | } 14 | 15 | public int GetHashCode([DisallowNull] JsonElement obj) 16 | { 17 | return obj.ToString().GetHashCode(StringComparison.Ordinal); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /test/Logic.Test/TestSupport/NoInMemoryStorageFactAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | public class NoInMemoryStorageFactAttribute : FactAttribute 7 | { 8 | public NoInMemoryStorageFactAttribute() 9 | { 10 | if (LogicTestSettings.UseMemoryStorage) 11 | { 12 | Skip = "This Fact is skipped because it is not compatible with in-memory storage."; 13 | } 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/Logic.Test/TestSupport/OSPlatformType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights 5 | { 6 | [Flags] 7 | public enum OSPlatformType 8 | { 9 | Windows = 1 << 0, 10 | Linux = 1 << 1, 11 | OSX = 1 << 2, 12 | FreeBSD = 1 << 3, 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /test/Website.Test/Website.Test.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | NuGet.Insights.Website.Test 4 | NuGet.Insights.Website 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | all 13 | runtime; build; native; contentfiles; analyzers 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/CatalogScan/CatalogScanDriverMetadataTest.NPE.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public partial class CatalogScanDriverMetadataTest : BaseWorkerLogicIntegrationTest 7 | { 8 | [Fact] 9 | public void GetTitleReturnsTitleOverride() 10 | { 11 | var actual = CatalogScanDriverMetadata.GetTitle(CatalogScanDriverType.NuGetPackageExplorerToCsv); 12 | Assert.Equal("NuGet Package Explorer to CSV", actual); 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/Docs/StorageContainerType.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker 5 | { 6 | public enum StorageContainerType 7 | { 8 | BlobContainer, 9 | Queue, 10 | Table, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/MessageProcessors/ReferenceTracking/TestSubjectRecordStorage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | namespace NuGet.Insights.Worker.ReferenceTracking 5 | { 6 | public class TestSubjectRecordStorage : ICsvResultStorage 7 | { 8 | private readonly CleanupOrphanRecordsServiceTest _test; 9 | private readonly IOptions _options; 10 | 11 | public TestSubjectRecordStorage( 12 | CleanupOrphanRecordsServiceTest test, 13 | IOptions options) 14 | { 15 | _test = test; 16 | _options = options; 17 | } 18 | 19 | public string ResultContainerName => _test.ResultContainerName; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/.gitattributes: -------------------------------------------------------------------------------- 1 | *.nuspec binary 2 | *.json binary 3 | *.csv binary 4 | *.md binary 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/BuildVersionSet_IgnoredPackages/Step1/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "V1": { 3 | "CommitTimestamp": "2025-04-23T21:22:16.2507724+00:00", 4 | "IdToVersionToDeleted": { 5 | "FluidSharp": { 6 | "3.0.1-alpha": false 7 | }, 8 | "Pepperdash.Essentials.Plugins.Utilities.Timer": { 9 | "2.0.0-4-series-updates.1": false 10 | }, 11 | "RequestGuard": { 12 | "1.0.1": false 13 | }, 14 | "TencentCloudSDKIntlEN": { 15 | "3.0.1116": false 16 | }, 17 | "TencentCloudSDKIntlEN.Aiart": { 18 | "3.0.1116": false 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/BuildVersionSet_WithDelete/Step1/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "V1": { 3 | "CommitTimestamp": "2020-12-20T03:01:57.2082154+00:00", 4 | "IdToVersionToDeleted": { 5 | "AmazonKS3": { 6 | "1.5.0": false 7 | }, 8 | "BehaviorSample": { 9 | "1.0.0": false 10 | }, 11 | "LunaBot.Commands.Public": { 12 | "1.2.0": false 13 | }, 14 | "LunaBot.Core": { 15 | "1.0.4": false 16 | }, 17 | "Nut.MediatR.Behaviors": { 18 | "0.0.0-preview.0.44": false 19 | }, 20 | "Nut.MediatR.Behaviors.FluentValidation": { 21 | "0.0.0-preview.0.44": false 22 | }, 23 | "Nut.MediatR.ServiceLike.DependencyInjection": { 24 | "0.0.0-preview.0.44": false 25 | }, 26 | "ServiceLikeSample": { 27 | "1.0.0": false 28 | } 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/BuildVersionSet_WithDelete/Step2/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "V1": { 3 | "CommitTimestamp": "2020-12-20T03:03:53.7885893+00:00", 4 | "IdToVersionToDeleted": { 5 | "AmazonKS3": { 6 | "1.5.0": false 7 | }, 8 | "BehaviorSample": { 9 | "1.0.0": true 10 | }, 11 | "LunaBot.Commands.Public": { 12 | "1.2.0": false 13 | }, 14 | "LunaBot.Core": { 15 | "1.0.4": false 16 | }, 17 | "Nut.MediatR.Behaviors": { 18 | "0.0.0-preview.0.44": false 19 | }, 20 | "Nut.MediatR.Behaviors.FluentValidation": { 21 | "0.0.0-preview.0.44": false 22 | }, 23 | "Nut.MediatR.ServiceLike.DependencyInjection": { 24 | "0.0.0-preview.0.44": false 25 | }, 26 | "ServiceLikeSample": { 27 | "1.0.0": false 28 | } 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/BuildVersionSet_WithIdCaseChange/Step1/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "V1": { 3 | "CommitTimestamp": "2018-11-16T04:13:15.2076282+00:00", 4 | "IdToVersionToDeleted": { 5 | "Byzan.Sdk.All": { 6 | "1.0.0-preview0041": false 7 | }, 8 | "UtfprLibrary.EntityFramework": { 9 | "1.0.142-alpha": false 10 | } 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/BuildVersionSet_WithIdCaseChange/Step2/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "V1": { 3 | "CommitTimestamp": "2018-11-16T04:17:16.6086735+00:00", 4 | "IdToVersionToDeleted": { 5 | "Byzan.Sdk": { 6 | "1.0.0-preview0041": false 7 | }, 8 | "Byzan.Sdk.All": { 9 | "1.0.0-preview0041": false 10 | }, 11 | "MoxiWorks.Platform": { 12 | "1.2.11": false 13 | }, 14 | "UTFPRLibrary.EntityFramework": { 15 | "1.0.0.97": false, 16 | "1.0.142-alpha": false 17 | } 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogDataToCsv_WithKelvinK_NLS/Step1/T1/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Message,Reasons,AlternatePackageId,AlternateVersionRange 2 | ,,awssdK.core,awssdK.core/1.0.0,AWSSDK.Core,1.0.0,2021-08-11T23:39:31.9024782Z,,Deleted,,,, 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogDataToCsv_WithKelvinK_NLS/Step1/T2/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,GitHubDatabaseKey,AdvisoryUrl,Severity 2 | ,,awssdK.core,awssdK.core/1.0.0,AWSSDK.Core,1.0.0,2021-08-11T23:39:31.9024782Z,,Deleted,,, 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogDataToCsv_WithKelvinK_NLS/Step1/T3/compact_0.csv: -------------------------------------------------------------------------------- 1 | CommitId,CommitTimestamp,LowerId,Identity,Id,Version,Type,Url,PageUrl,Published,IsListed,Created,LastEdited,PackageSize,PackageHash,PackageHashAlgorithm,Deprecation,Vulnerabilities,HasRepositoryProperty,PackageEntryCount,NuspecPackageEntry,SignaturePackageEntry 2 | d3c98ee3-8cf4-42d7-8c73-455431225003,2021-08-11T23:39:31.9024782Z,awssdK.core,awssdK.core/1.0.0,AWSSDK.Core,1.0.0,PackageDelete,https://api.nuget.org/v3/catalog0/data/2021.08.11.23.39.31/awssd%E2%84%AA.core.1.0.0.json,https://api.nuget.org/v3/catalog0/page13272.json,2021-08-11T23:39:11.7067065Z,,,,,,,,,,,, 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogScanService_TestCachedData/Step1/PackageReadmes.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Size,LastModified,SHA256,Content 2 | ,,nlog.extensions.logging,nlog.extensions.logging/1.6.3,NLog.Extensions.Logging,1.6.3,2020-05-11T20:12:56.9143404Z,2020-05-11T20:10:44.683Z,Embedded,12,,L6l7fL/m2S9QPFvFYgJDIVQ2TWbfvQ9I5/MQPd9FEyE=,# My package 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogScanService_TestCachedData/Step1/SymbolPackageArchives.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Size,OffsetAfterEndOfCentralDirectory,CentralDirectorySize,OffsetOfCentralDirectory,EntryCount,Comment,HeaderMD5,HeaderSHA512,MD5,SHA1,SHA256,SHA512 2 | ,,nlog.extensions.logging,nlog.extensions.logging/1.6.3,NLog.Extensions.Logging,1.6.3,2020-05-11T20:12:56.9143404Z,2020-05-11T20:10:44.683Z,Available,46807,46789,860,45925,10,,,e6SDtEvGITWhUTkic6c6fqtVD+UVliW9rvEyfYm6SaOa/xp1lEywytkQmQfWtgN0mpaOJxviXFIZG9BmUsxMyA==,0X2sqNbjfDYeMkDd0QsyGQ==,q7Rn51vXnBWgqPmXEbFYonBaFN4=,xcsaCApZNEaGs62A/7XTA+VnX1tJd0hoceM4Xfv+l5E=,e6SDtEvGITWhUTkic6c6fqtVD+UVliW9rvEyfYm6SaOa/xp1lEywytkQmQfWtgN0mpaOJxviXFIZG9BmUsxMyA== 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogScanService_TestCachedData/Step2/PackageReadmes.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Size,LastModified,SHA256,Content 2 | ,,nlog.extensions.logging,nlog.extensions.logging/1.6.3,NLog.Extensions.Logging,1.6.3,2020-05-11T20:12:56.9143404Z,2020-05-11T20:10:44.683Z,Embedded,48,2021-01-14T20:00:00Z,7xige+pW0rIfUinunqBuVXzLZ8TUs9uaeknBIjVsDHU=,"# Example README 3 | 4 | This is an example readme. 5 | " 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CatalogScanService_TestCachedData/Step2/SymbolPackageArchives.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Size,OffsetAfterEndOfCentralDirectory,CentralDirectorySize,OffsetOfCentralDirectory,EntryCount,Comment,HeaderMD5,HeaderSHA512,MD5,SHA1,SHA256,SHA512 2 | ,,nlog.extensions.logging,nlog.extensions.logging/1.6.3,NLog.Extensions.Logging,1.6.3,2020-05-11T20:12:56.9143404Z,2020-05-11T20:10:44.683Z,Available,13654,13636,473,13159,6,,,,uOano6u6YpWBVxA3XZWdUg==,AUvV8RQGS5aq5GyA6Znso9uH5no=,Iufewnf48YyRObZge21hMxI+iqp1lLcUdrywVvzLK98=,ihFkShylFVhanyS+lVmr0vYmVnAkUB70VJVvGEMWcKAs2gIjlSsWYhWMZYU9/UzyKhRBVxqUe5PIAoTjpbcfCQ== 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithClearedCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 0,Subject0,false 3 | 6,Subject6,false 4 | 8,Subject8,false 5 | 9,Subject9,false 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithClearedCsv/Step1/compact_1.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 1,Subject1,false 3 | 2,Subject2,false 4 | 3,Subject3,false 5 | 4,Subject4,false 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithClearedCsv/Step1/compact_2.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 5,Subject5,false 3 | 7,Subject7,false 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithClearedCsv/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 0,Subject0,false 3 | 6,Subject6,false 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithClearedCsv/Step2/compact_2.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithNoDeletions/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 0,Subject0,false 3 | 6,Subject6,false 4 | 8,Subject8,false 5 | 9,Subject9,false 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithNoDeletions/Step1/compact_1.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 1,Subject1,false 3 | 2,Subject2,false 4 | 3,Subject3,false 5 | 4,Subject4,false 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithNoDeletions/Step1/compact_2.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 5,Subject5,false 3 | 7,Subject7,false 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithNoOrphans/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 0,Subject0,false 3 | 6,Subject6,false 4 | 8,Subject8,false 5 | 9,Subject9,false 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithNoOrphans/Step1/compact_1.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 1,Subject1,false 3 | 2,Subject2,false 4 | 3,Subject3,false 5 | 4,Subject4,false 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/CleanupOrphanRecordsServiceTest_WithNoOrphans/Step1/compact_2.csv: -------------------------------------------------------------------------------- 1 | BucketKey,Id,IsOrphan 2 | 5,Subject5,false 3 | 7,Subject7,false 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-14T18:00:00Z,knapcode.torsharp,knapcode.torsharp/2.0.7,Knapcode.TorSharp,2.0.7,138,138 3 | 2021-01-14T18:00:00Z,newtonsoft.json,newtonsoft.json/10.5.0,Newtonsoft.Json,10.5.0,137,543 4 | 2021-01-14T18:00:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,406,543 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,knapcode.kitchensink/1.0.0-beta,Knapcode.KitchenSink,1.0.0-beta,1,1 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,knapcode.torsharp/2.0.7,Knapcode.TorSharp,2.0.7,138,138 4 | 2021-01-15T19:00:00Z,newtonsoft.json,newtonsoft.json/10.5.0,Newtonsoft.Json,10.5.0,139,548 5 | 2021-01-15T19:00:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,409,548 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_Duplicates/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-14T18:00:00Z,foo,foo/2.0.0,foo,2.0.0,23,23 3 | 2021-01-14T18:00:00Z,yourpackageid,yourpackageid/1.0.0,yourpackageid,1.0.0,62,62 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_JustV2/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-14T18:30:00Z,knapcode.torsharp,knapcode.torsharp/2.0.7,Knapcode.TorSharp,2.0.7,139,139 3 | 2021-01-14T18:30:00Z,newtonsoft.json,newtonsoft.json/10.5.0,Newtonsoft.Json,10.5.0,138,545 4 | 2021-01-14T18:30:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,407,545 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_JustV2/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-15T19:30:00Z,knapcode.kitchensink,knapcode.kitchensink/1.0.0-beta,Knapcode.KitchenSink,1.0.0-beta,2,2 3 | 2021-01-15T19:30:00Z,knapcode.torsharp,knapcode.torsharp/2.0.7,Knapcode.TorSharp,2.0.7,139,139 4 | 2021-01-15T19:30:00Z,newtonsoft.json,newtonsoft.json/10.5.0,Newtonsoft.Json,10.5.0,140,550 5 | 2021-01-15T19:30:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,410,550 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_NonExistentId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_NonExistentId/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,knapcode.kitchensink/1.0.0-beta,Knapcode.KitchenSink,1.0.0-beta,1,1 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,knapcode.torsharp/2.0.7,knapcode.TORSHARP,2.0.7,138,138 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_NonExistentVersion/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-14T18:00:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,406,406 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_NonExistentVersion/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,knapcode.kitchensink/1.0.0-beta,Knapcode.KitchenSink,1.0.0-beta,1,1 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,knapcode.torsharp/2.0.7,Knapcode.TorSharp,2.0.7,138,138 4 | 2021-01-15T19:00:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,409,409 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_UncheckedIdAndVersion/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-14T18:00:00Z,knapcode.torsharp,knapcode.torsharp/0.0.1,Knapcode.TorSharp,0.0.1,0,0 3 | 2021-01-14T18:00:00Z,newtonsoft.json,newtonsoft.json/10.5.0,Newtonsoft.Json,10.5.0,137,543 4 | 2021-01-14T18:00:00Z,newtonsoft.json,newtonsoft.json/2.4.3.700,Newtonsoft.Json,2.4.3.700,406,543 5 | 2021-01-14T18:00:00Z,uncheckeda,uncheckeda/1.0.0,UncheckedA,1.0.0,0,0 6 | 2021-01-14T18:00:00Z,uncheckeda,uncheckeda/2.0.0,UncheckedA,2.0.0,0,0 7 | 2021-01-14T18:00:00Z,uncheckedb,uncheckedb/3.0.0,UncheckedB,3.0.0,0,0 8 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/DownloadsToCsv_UnicodeDuplicates/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Identity,Id,Version,Downloads,TotalDownloads 2 | 2021-01-14T18:00:00Z,newtonsoft.json,newtonsoft.json/9.0.1,Newtonsoft.Json,9.0.1,137,137 3 | 2021-01-14T18:00:00Z,sem-tostão,sem-tostão/1.0.1,Sem-Tostão,1.0.1,100,100 4 | 2021-01-14T18:00:00Z,sem-tostão,sem-tostão/1.0.1,Sem-Tostão,1.0.1,200,200 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/ExcludedPackagesToCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsExcluded 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,true 3 | 2021-01-14T18:00:00Z,knapcode.torsharp,Knapcode.TorSharp,true 4 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | 2021-01-14T18:00:00Z,newtonsoft.json,Newtonsoft.Json,true 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/ExcludedPackagesToCsv/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsExcluded 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,Knapcode.KitchenSink,true 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,Knapcode.TorSharp,true 4 | 2021-01-15T19:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | 2021-01-15T19:00:00Z,newtonsoft.json,Newtonsoft.Json,true 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/ExcludedPackagesToCsv_NonExistentId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsExcluded 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,true 3 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/ExcludedPackagesToCsv_NonExistentId/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsExcluded 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,Knapcode.KitchenSink,true 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,knapcode.TORSHARP,true 4 | 2021-01-15T19:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/ExcludedPackagesToCsv_UncheckedId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsExcluded 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,true 3 | 2021-01-14T18:00:00Z,knapcode.torsharp,Knapcode.TorSharp,true 4 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | 2021-01-14T18:00:00Z,newtonsoft.json,Newtonsoft.Json,true 6 | 2021-01-14T18:00:00Z,uncheckeda,UncheckedA,false 7 | 2021-01-14T18:00:00Z,uncheckedb,UncheckedB,false 8 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/GitHubUsageToCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,ResultType,Repository,Stars 2 | 2021-01-16T20:00:00Z,csvhelper,CsvHelper,GitHubDependent,NuGet/NuGetGallery,1561 3 | 2021-01-16T20:00:00Z,knapcode.minizip,Knapcode.MiniZip,GitHubDependent,NuGet/NuGetGallery,1561 4 | 2021-01-16T20:00:00Z,microsoft.net.test.sdk,Microsoft.NET.Test.Sdk,GitHubDependent,xunit/xunit,4273 5 | 2021-01-16T20:00:00Z,moq,Moq,GitHubDependent,NuGet/NuGetGallery,1561 6 | 2021-01-16T20:00:00Z,moq,Moq,GitHubDependent,xunit/xunit,4273 7 | 2021-01-16T20:00:00Z,nerdbank.gitversioning,Nerdbank.GitVersioning,GitHubDependent,xunit/xunit,4273 8 | 2021-01-16T20:00:00Z,nuget.versioning,NuGet.Versioning,GitHubDependent,NuGet/NuGetGallery,1561 9 | 2021-01-16T20:00:00Z,windowsazure.storage,WindowsAzure.Storage,GitHubDependent,NuGet/NuGetGallery,1561 10 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/GitHubUsageToCsv_NonExistentId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,ResultType,Repository,Stars 2 | 2021-01-16T20:00:00Z,csvhelper,CsvHelper,GitHubDependent,NuGet/NuGetGallery,1561 3 | 2021-01-16T20:00:00Z,knapcode.minizip,Knapcode.MiniZip,GitHubDependent,NuGet/NuGetGallery,1561 4 | 2021-01-16T20:00:00Z,microsoft.net.test.sdk,Microsoft.NET.Test.Sdk,GitHubDependent,xunit/xunit,4273 5 | 2021-01-16T20:00:00Z,moq,Moq,GitHubDependent,NuGet/NuGetGallery,1561 6 | 2021-01-16T20:00:00Z,moq,Moq,GitHubDependent,xunit/xunit,4273 7 | 2021-01-16T20:00:00Z,nerdbank.gitversioning,Nerdbank.GitVersioning,GitHubDependent,xunit/xunit,4273 8 | 2021-01-16T20:00:00Z,nuget.versioning,NuGet.Versioning,GitHubDependent,NuGet/NuGetGallery,1561 9 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/LoadSymbolPackageArchive/Step1/entities.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Entity": { 4 | "Available": true, 5 | "CommitTimestamp": "2021-03-22T20:13:54.6075418+00:00", 6 | "HttpHeaders": { 7 | "Content-Type": [ 8 | "binary/octet-stream" 9 | ], 10 | "Last-Modified": [ 11 | "Mon, 22 Mar 2021 20:12:48 GMT" 12 | ], 13 | "x-ms-blob-type": [ 14 | "BlockBlob" 15 | ], 16 | "x-ms-lease-status": [ 17 | "unlocked" 18 | ], 19 | "x-ms-meta-SHA512": [ 20 | "vaWuAPufkOrRT0f48FuX90NnymmB/hrLUvg\u002BXsQUHkaht/cwpqkLYcNlUPrjiak7Uhw4uX14dcXUT/NznHHHHg==" 21 | ] 22 | }, 23 | "MZipHash": "7f60ee52e75640227bdcea907723c1b7d1c15b18de2700a79dd16d26f96b3ee1" 24 | }, 25 | "PartitionKey": "newtonsoft.json", 26 | "RowKey": "13.0.1" 27 | } 28 | ] -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/OwnersToCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,Owners 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,"[""castleproject""]" 3 | 2021-01-14T18:00:00Z,knapcode.torsharp,Knapcode.TorSharp,"[""joelverhagen""]" 4 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,"[""aspnet"",""dotnetframework"",""Microsoft""]" 5 | 2021-01-14T18:00:00Z,newtonsoft.json,Newtonsoft.Json,"[""jamesnk"",""newtonsoft""]" 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/OwnersToCsv/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,Owners 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,Knapcode.KitchenSink,"[""joelverhagen"",""nuget""]" 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,Knapcode.TorSharp,"[""joelverhagen""]" 4 | 2021-01-15T19:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,"[""dotnetframework"",""Microsoft""]" 5 | 2021-01-15T19:00:00Z,newtonsoft.json,Newtonsoft.Json,"[""jamesnk"",""newtonsoft""]" 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/OwnersToCsv_NonExistentId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,Owners 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,"[""castleproject""]" 3 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,"[""aspnet"",""dotnetframework"",""Microsoft""]" 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/OwnersToCsv_NonExistentId/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,Owners 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,Knapcode.KitchenSink,"[""joelverhagen"",""nuget""]" 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,knapcode.TORSHARP,"[""joelverhagen""]" 4 | 2021-01-15T19:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,"[""dotnetframework"",""Microsoft""]" 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/OwnersToCsv_UncheckedId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,Owners 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,"[""castleproject""]" 3 | 2021-01-14T18:00:00Z,knapcode.torsharp,Knapcode.TorSharp,"[""joelverhagen""]" 4 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,"[""aspnet"",""dotnetframework"",""Microsoft""]" 5 | 2021-01-14T18:00:00Z,newtonsoft.json,Newtonsoft.Json,"[""jamesnk"",""newtonsoft""]" 6 | 2021-01-14T18:00:00Z,uncheckeda,UncheckedA,[] 7 | 2021-01-14T18:00:00Z,uncheckedb,UncheckedB,[] 8 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageCertificateToCsv_WithDollarSignId/Step1/T1/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Fingerprint,RelationshipTypes 2 | ,,$id$,$id$/0.1.1,$id$,0.1.1,2016-03-21T06:41:26.9274378Z,,Deleted,, 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageCertificateToCsv_WithDollarSignId/Step1/empty-array.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageCertificateToCsv_WithSingleDelete/Step2/T1/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Fingerprint,RelationshipTypes 2 | ,,deltax,deltax/1.0.0,DeltaX,1.0.0,2020-04-10T18:18:43.6788949Z,,Deleted,, 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageCertificateToCsv_WithSingleDelete/Step2/empty-array.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageCertificateToCsv_WithSingleDelete/Step3/T2/empty.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,ResultType,Fingerprint,FingerprintSHA256Hex,FingerprintSHA1Hex,Subject,Issuer,NotBefore,NotAfter,SerialNumber,SignatureAlgorithmOid,Version,Extensions,PublicKeyOid,RawDataLength,RawData,IssuerFingerprint,RootFingerprint,ChainLength,CodeSigningCommitTimestamp,CodeSigningStatus,CodeSigningStatusFlags,CodeSigningStatusUpdateTime,CodeSigningRevocationTime,TimestampingCommitTimestamp,TimestampingStatus,TimestampingStatusFlags,TimestampingStatusUpdateTime,TimestampingRevocationTime,Policies 2 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageCompatibilityToCsv_WithUnsupported/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,HasError,DoesNotRoundTrip,HasAny,HasUnsupported,HasAgnostic,BrokenFrameworks,NuspecReader,NU1202,NuGetGallery,NuGetGalleryEscaped,NuGetGallerySupported,NuGetGalleryBadges 2 | ,,flexmonster.accelerator,flexmonster.accelerator/2.7.19,Flexmonster.Accelerator,2.7.19,2019-11-18T17:19:38.2541574Z,2019-11-18T17:16:48.877Z,Available,false,true,true,true,false,"[""flexmonster"",""flexmonster,Version=v0.0""]","[""any""]","[""unsupported""]","[""unsupported""]","[""unsupported""]",[],[] 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageVersionToCsv_WithMultipleBuckets/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,OriginalVersion,FullVersion,Major,Minor,Patch,Revision,Release,ReleaseLabels,Metadata,IsPrerelease,IsListed,IsSemVer2,SemVerType,SemVerOrder,IsLatest,IsLatestStable,IsLatestSemVer2,IsLatestStableSemVer2,Published,LastEdited 2 | ,,nut.mediatr.behaviors.fluentvalidation,nut.mediatr.behaviors.fluentvalidation/0.0.0-preview.0.44,Nut.MediatR.Behaviors.FluentValidation,0.0.0-preview.0.44,2020-12-20T02:37:52.9649363Z,2020-12-20T02:35:16.733Z,Available,0.0.0-preview.0.44,0.0.0-preview.0.44,0,0,0,0,preview.0.44,"[""preview"",""0"",""44""]",,true,true,true,VersionHasPrereleaseDots,0,false,false,true,false,2020-12-20T02:35:16.733Z,2020-12-20T02:37:50.413Z 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PackageVersionToCsv_WithMultipleBuckets/Step1/compact_1.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,OriginalVersion,FullVersion,Major,Minor,Patch,Revision,Release,ReleaseLabels,Metadata,IsPrerelease,IsListed,IsSemVer2,SemVerType,SemVerOrder,IsLatest,IsLatestStable,IsLatestSemVer2,IsLatestStableSemVer2,Published,LastEdited 2 | ,,nut.mediatr.servicelike.dependencyinjection,nut.mediatr.servicelike.dependencyinjection/0.0.0-preview.0.44,Nut.MediatR.ServiceLike.DependencyInjection,0.0.0-preview.0.44,2020-12-20T02:37:52.9649363Z,2020-12-20T02:35:16.043Z,Available,0.0.0-preview.0.44,0.0.0-preview.0.44,0,0,0,0,preview.0.44,"[""preview"",""0"",""44""]",,true,true,true,"VersionHasPrereleaseDots, DependencyMinHasPrereleaseDots",0,false,false,true,false,2020-12-20T02:35:16.043Z,2020-12-20T02:37:50.103Z 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PopularityTransfersToCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,TransferIds,TransferLowerIds 2 | 2021-01-16T18:00:00Z,windowsazure.storage,WindowsAzure.Storage,"[""Azure.Storage.Blobs""]","[""azure.storage.blobs""]" 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PopularityTransfersToCsv/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,TransferIds,TransferLowerIds 2 | 2021-01-17T19:00:00Z,windowsazure.servicebus,WindowsAzure.ServiceBus,"[""Azure.Messaging.EventHubs"",""Azure.Messaging.ServiceBus""]","[""azure.messaging.eventhubs"",""azure.messaging.servicebus""]" 3 | 2021-01-17T19:00:00Z,windowsazure.storage,WindowsAzure.Storage,"[""Azure.Storage.Blobs"",""Azure.Storage.Queues""]","[""azure.storage.blobs"",""azure.storage.queues""]" 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PopularityTransfersToCsv_NonExistentId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,TransferIds,TransferLowerIds 2 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PopularityTransfersToCsv_NonExistentId/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,TransferIds,TransferLowerIds 2 | 2021-01-17T19:00:00Z,windowsazure.servicebus,WindowsAzure.ServiceBus,"[""Azure.Messaging.EventHubs"",""Azure.Messaging.ServiceBus""]","[""azure.messaging.eventhubs"",""azure.messaging.servicebus""]" 3 | 2021-01-17T19:00:00Z,windowsazure.storage,windowsAZURE.StorAGE,"[""Azure.Storage.Blobs"",""Azure.Storage.Queues""]","[""azure.storage.blobs"",""azure.storage.queues""]" 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/PopularityTransfersToCsv_UncheckedId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,TransferIds,TransferLowerIds 2 | 2021-01-16T18:00:00Z,uncheckeda,UncheckedA,[],[] 3 | 2021-01-16T18:00:00Z,uncheckedb,UncheckedB,[],[] 4 | 2021-01-16T18:00:00Z,windowsazure.storage,WindowsAzure.Storage,"[""Azure.Storage.Blobs""]","[""azure.storage.blobs""]" 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/SymbolPackageArchiveToCsv/Step1/T1/compact_0.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Size,OffsetAfterEndOfCentralDirectory,CentralDirectorySize,OffsetOfCentralDirectory,EntryCount,Comment,HeaderMD5,HeaderSHA512,MD5,SHA1,SHA256,SHA512 2 | ,,newtonsoft.json,newtonsoft.json/13.0.1,Newtonsoft.Json,13.0.1,2021-03-22T20:13:54.6075418Z,2021-03-22T20:10:49.407Z,Available,750829,750811,869,749938,11,,,vaWuAPufkOrRT0f48FuX90NnymmB/hrLUvg+XsQUHkaht/cwpqkLYcNlUPrjiak7Uhw4uX14dcXUT/NznHHHHg==,jRzvirf3Q1udIjn2m1L6vQ==,Z0LHaGXHLUmrGbGzISv1stCGByE=,9WljKAdpVsA7D4Odw+cmHxZH2p3tFTfWPhIsVHH8h8Y=,vaWuAPufkOrRT0f48FuX90NnymmB/hrLUvg+XsQUHkaht/cwpqkLYcNlUPrjiak7Uhw4uX14dcXUT/NznHHHHg== 3 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/TimedReprocess_SubsequentBucketRanges/Step1/PackageReadmes.csv: -------------------------------------------------------------------------------- 1 | ScanId,ScanTimestamp,LowerId,Identity,Id,Version,CatalogCommitTimestamp,Created,ResultType,Size,LastModified,SHA256,Content 2 | ,,mgcframework.blazor,mgcframework.blazor/1.2.0,MgcFramework.Blazor,1.2.0,2024-04-25T02:13:04.3170295Z,2024-04-25T02:10:05.677Z,Embedded,45,2024-04-25T02:12:45Z,UPLkk8p5oXSuAAwOztB3iEiOxy/tZRl0kGV3Pn/pIhg=,# Librería interna para aplicaciones .Net 3 | ,,rocket.surgery.launchpad.aspnetcore.blazor,rocket.surgery.launchpad.aspnetcore.blazor/2.0.0-beta.4,Rocket.Surgery.LaunchPad.AspNetCore.Blazor,2.0.0-beta.4,2024-04-25T02:13:04.3170295Z,2024-04-25T02:10:00.45Z,None,,,, 4 | ,,rocket.surgery.launchpad.foundation.newtonsoftjson,rocket.surgery.launchpad.foundation.newtonsoftjson/2.0.0-beta.4,Rocket.Surgery.LaunchPad.Foundation.NewtonsoftJson,2.0.0-beta.4,2024-04-25T02:13:04.3170295Z,2024-04-25T02:09:56.127Z,None,,,, 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/VerifiedPackagesToCsv/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsVerified 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,true 3 | 2021-01-14T18:00:00Z,knapcode.torsharp,Knapcode.TorSharp,true 4 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | 2021-01-14T18:00:00Z,newtonsoft.json,Newtonsoft.Json,true 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/VerifiedPackagesToCsv/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsVerified 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,Knapcode.KitchenSink,true 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,Knapcode.TorSharp,true 4 | 2021-01-15T19:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | 2021-01-15T19:00:00Z,newtonsoft.json,Newtonsoft.Json,true 6 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/VerifiedPackagesToCsv_NonExistentId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsVerified 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,true 3 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 4 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/VerifiedPackagesToCsv_NonExistentId/Step2/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsVerified 2 | 2021-01-15T19:00:00Z,knapcode.kitchensink,Knapcode.KitchenSink,true 3 | 2021-01-15T19:00:00Z,knapcode.torsharp,knapcode.TORSHARP,true 4 | 2021-01-15T19:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | -------------------------------------------------------------------------------- /test/Worker.Logic.Test/TestData/VerifiedPackagesToCsv_UncheckedId/Step1/compact_0.csv: -------------------------------------------------------------------------------- 1 | AsOfTimestamp,LowerId,Id,IsVerified 2 | 2021-01-14T18:00:00Z,castle.core,Castle.Core,true 3 | 2021-01-14T18:00:00Z,knapcode.torsharp,Knapcode.TorSharp,true 4 | 2021-01-14T18:00:00Z,microsoft.extensions.logging,Microsoft.Extensions.Logging,true 5 | 2021-01-14T18:00:00Z,newtonsoft.json,Newtonsoft.Json,true 6 | 2021-01-14T18:00:00Z,uncheckeda,UncheckedA,false 7 | 2021-01-14T18:00:00Z,uncheckedb,UncheckedB,false 8 | -------------------------------------------------------------------------------- /test/Worker.Test/TestData/.gitattributes: -------------------------------------------------------------------------------- 1 | *.nuspec binary 2 | *.json binary 3 | *.csv binary 4 | *.md binary 5 | -------------------------------------------------------------------------------- /tye.yaml: -------------------------------------------------------------------------------- 1 | name: NuGet.Insights 2 | services: 3 | - name: website 4 | project: src/Website/Website.csproj 5 | bindings: 6 | - protocol: "http" 7 | - name: worker 8 | azureFunction: src/Worker/ 9 | - name: storage 10 | image: mcr.microsoft.com/azure-storage/azurite:latest 11 | volumes: 12 | - target: /data 13 | source: ./azurite/ 14 | bindings: 15 | - name: blobs 16 | port: 10000 17 | - name: queues 18 | port: 10001 19 | - name: tables 20 | port: 10002 21 | --------------------------------------------------------------------------------