├── .github └── workflows │ └── gradle.yml ├── .gitignore ├── LICENSE ├── README.md ├── analysis ├── checkstyle │ └── checkstyle.xml └── pmd │ └── ruleset.xml ├── build.gradle ├── documentation ├── application │ ├── application-administration.adoc │ ├── application-architectural-overview.adoc │ ├── application-conductor.adoc │ ├── application-downloader.adoc │ ├── application-generator.adoc │ ├── application-indexer.adoc │ ├── application-parameters.adoc │ ├── application-queue.adoc │ ├── application-staging.adoc │ ├── application-vault.adoc │ ├── application-web.adoc │ ├── application.adoc │ └── beacon-application.adoc ├── assets │ └── images │ │ ├── architecture.svg │ │ └── logo-background.png ├── diagram │ └── architecture.drawio ├── disc-usage │ └── disc-usage.adoc ├── domain │ └── domain.adoc ├── index.adoc ├── installation │ ├── installation-conductor-application.adoc │ ├── installation-downloader-application.adoc │ ├── installation-elasticsearch.adoc │ ├── installation-generator-application.adoc │ ├── installation-indexer-application.adoc │ ├── installation-java.adoc │ ├── installation-mongodb.adoc │ ├── installation-queue-application.adoc │ ├── installation-staging-application.adoc │ ├── installation-vault-application.adoc │ ├── installation-web-application.adoc │ └── installation.adoc ├── prerequisites │ └── prerequisites.adoc ├── service │ └── conductor │ │ └── conductor-configuration.adoc └── system-requirements │ ├── system-requirements-collecting.adoc │ ├── system-requirements-indexing.adoc │ └── system-requirements.adoc ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── loa-application ├── build.gradle ├── loa-administrator-application │ ├── build.gradle │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── github │ │ │ │ └── bottomlessarchive │ │ │ │ └── loa │ │ │ │ └── administrator │ │ │ │ ├── LibraryAdministratorApplication.java │ │ │ │ ├── command │ │ │ │ ├── cleanup │ │ │ │ │ └── CleanupCommand.java │ │ │ │ ├── compressor │ │ │ │ │ ├── SilentCompressorCommand.java │ │ │ │ │ └── configuration │ │ │ │ │ │ └── SilentCompressorConfigurationProperties.java │ │ │ │ ├── recollect │ │ │ │ │ ├── DocumentRecollectorService.java │ │ │ │ │ ├── RecollectCorruptDocumentsCommand.java │ │ │ │ │ └── SourceLocationRecrawlerService.java │ │ │ │ ├── reindex │ │ │ │ │ └── ReindexCommand.java │ │ │ │ ├── user │ │ │ │ │ ├── RegisterUserCommand.java │ │ │ │ │ └── configuration │ │ │ │ │ │ └── RegisterUserConfigurationProperties.java │ │ │ │ └── validator │ │ │ │ │ └── DocumentValidatorCommand.java │ │ │ │ ├── configuration │ │ │ │ ├── AdministratorCommandConfigurationProperties.java │ │ │ │ ├── AdministratorConfigurationProperties.java │ │ │ │ └── StageConfiguration.java │ │ │ │ └── service │ │ │ │ └── AdministratorInstancePropertyExtensionProvider.java │ │ └── resources │ │ │ ├── application-local.properties │ │ │ ├── application.properties │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── administrator │ │ └── command │ │ └── recollect │ │ ├── DocumentRecollectorServiceTest.java │ │ └── SourceLocationRecrawlerServiceTest.java ├── loa-beacon-application │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── beacon │ │ │ ├── LibraryBeaconApplication.java │ │ │ ├── command │ │ │ ├── OfflineDownloaderCommand.java │ │ │ └── configuration │ │ │ │ ├── OfflineDownloaderConfigurationProperties.java │ │ │ │ └── OfflineDownloaderStatisticsConfiguration.java │ │ │ ├── configuration │ │ │ ├── BeaconConfigurationProperties.java │ │ │ └── StageConfiguration.java │ │ │ ├── service │ │ │ ├── DocumentLocationProcessor.java │ │ │ ├── DocumentLocationVisitor.java │ │ │ ├── StoragePathFactory.java │ │ │ └── domain │ │ │ │ ├── DocumentLocation.java │ │ │ │ ├── DocumentLocationProcessingException.java │ │ │ │ └── DocumentLocationResult.java │ │ │ └── view │ │ │ └── beacon │ │ │ ├── BeaconController.java │ │ │ ├── request │ │ │ ├── DocumentLocationPartialRequest.java │ │ │ └── VisitDocumentLocationsRequest.java │ │ │ └── response │ │ │ ├── DocumentLocationResultPartialResponse.java │ │ │ └── VisitDocumentLocationsResponse.java │ │ └── resources │ │ ├── application-local.properties │ │ ├── application.properties │ │ └── banner.txt ├── loa-conductor-application │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── conductor │ │ │ ├── LibraryConductorApplication.java │ │ │ ├── configuration │ │ │ ├── ConductorWebMvcConfigurer.java │ │ │ ├── IndexDatabaseConfigurationProperties.java │ │ │ └── RepositoryConfigurationProperties.java │ │ │ ├── service │ │ │ └── DocumentDatabaseRegistrar.java │ │ │ └── view │ │ │ ├── controller │ │ │ └── ServiceRegistryController.java │ │ │ ├── converter │ │ │ └── StringToApplicationTypeConverter.java │ │ │ ├── request │ │ │ ├── ServiceInstancePropertyRequest.java │ │ │ ├── ServiceInstanceRefreshRequest.java │ │ │ └── ServiceInstanceRegistrationRequest.java │ │ │ ├── response │ │ │ ├── ServiceInstancePropertyResponse.java │ │ │ ├── ServiceInstanceRegistrationResponse.java │ │ │ ├── ServiceInstanceResponse.java │ │ │ └── ServiceResponse.java │ │ │ └── service │ │ │ └── ServiceInstanceEntityTransformer.java │ │ └── resources │ │ ├── application-local.properties │ │ ├── application.properties │ │ └── banner.txt ├── loa-downloader-application │ ├── build.gradle │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── github │ │ │ │ └── bottomlessarchive │ │ │ │ └── loa │ │ │ │ └── downloader │ │ │ │ ├── LibraryDownloaderApplication.java │ │ │ │ ├── configuration │ │ │ │ ├── DownloaderConfigurationProperties.java │ │ │ │ ├── DownloaderParallelismConfiguration.java │ │ │ │ ├── DownloaderStatisticsConfiguration.java │ │ │ │ ├── SourceLocation.java │ │ │ │ └── StageConfiguration.java │ │ │ │ └── service │ │ │ │ ├── DocumentLocationCreationContextFactory.java │ │ │ │ ├── conductor │ │ │ │ └── ParallelismInstancePropertyExtensionProvider.java │ │ │ │ ├── document │ │ │ │ ├── DocumentArchiver.java │ │ │ │ ├── DocumentArchivingMessageFactory.java │ │ │ │ ├── DocumentIdFactory.java │ │ │ │ ├── DocumentLocationEvaluator.java │ │ │ │ ├── DocumentLocationProcessingExecutor.java │ │ │ │ ├── DocumentLocationProcessorTask.java │ │ │ │ ├── DocumentLocationProcessorTaskFactory.java │ │ │ │ ├── DocumentLocationProcessorWrapper.java │ │ │ │ └── domain │ │ │ │ │ ├── DocumentArchivingContext.java │ │ │ │ │ └── exception │ │ │ │ │ ├── ArchivingException.java │ │ │ │ │ └── NotEnoughSpaceException.java │ │ │ │ └── source │ │ │ │ ├── beacon │ │ │ │ ├── accumulating │ │ │ │ │ └── AccumulatingBeaconDownloader.java │ │ │ │ ├── configuration │ │ │ │ │ └── BeaconDownloaderConfigurationProperties.java │ │ │ │ ├── loader │ │ │ │ │ ├── BeaconDocumentArchiver.java │ │ │ │ │ ├── BeaconDocumentLoader.java │ │ │ │ │ └── domain │ │ │ │ │ │ └── LoaderException.java │ │ │ │ ├── loadocloader │ │ │ │ │ ├── BeaconLoadocDocumentArchiver.java │ │ │ │ │ ├── BeaconLoadocDocumentLoader.java │ │ │ │ │ └── configuration │ │ │ │ │ │ └── LoadocLoaderBeaconConfigurationProperties.java │ │ │ │ ├── offline │ │ │ │ │ ├── OfflineBeaconDownloader.java │ │ │ │ │ └── configuration │ │ │ │ │ │ └── OfflineBeaconDownloaderConfigurationProperties.java │ │ │ │ ├── reply │ │ │ │ │ └── ReplyBeaconDownloader.java │ │ │ │ └── service │ │ │ │ │ ├── BeaconDocumentLocationFactory.java │ │ │ │ │ └── DocumentLocationCollector.java │ │ │ │ ├── configuration │ │ │ │ └── DownloaderFolderSourceConfigurationProperties.java │ │ │ │ ├── folder │ │ │ │ └── DocumentFolderReader.java │ │ │ │ └── queue │ │ │ │ ├── DownloadQueueListener.java │ │ │ │ └── QueueMessageHandler.java │ │ └── resources │ │ │ ├── application-local.properties │ │ │ ├── application.properties │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── downloader │ │ ├── integration │ │ └── QueueSourceIntegrationTest.java │ │ └── service │ │ ├── DocumentLocationCreationContextFactoryTest.java │ │ └── document │ │ ├── DocumentArchiverTest.java │ │ └── DocumentArchivingMessageFactoryTest.java ├── loa-generator-application │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── generator │ │ │ ├── LibraryGeneratorApplication.java │ │ │ └── command │ │ │ └── GeneratorCommand.java │ │ └── resources │ │ ├── application-local.properties │ │ ├── application.properties │ │ └── banner.txt ├── loa-indexer-application │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── indexer │ │ │ ├── DocumentEntityIndexer.java │ │ │ ├── LibraryIndexerApplication.java │ │ │ ├── command │ │ │ └── IndexerCommand.java │ │ │ ├── configuration │ │ │ ├── IndexerConfigurationProperties.java │ │ │ └── IndexerParallelismConfiguration.java │ │ │ └── service │ │ │ └── conductor │ │ │ └── ParallelismInstancePropertyExtensionProvider.java │ │ └── resources │ │ ├── application-local.properties │ │ ├── application.properties │ │ └── banner.txt ├── loa-queue-application │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── queue │ │ │ ├── LibraryQueueApplication.java │ │ │ └── service │ │ │ ├── ArtemisQueueInitializer.java │ │ │ └── conductor │ │ │ └── QueueInstancePropertyExtensionProvider.java │ │ └── resources │ │ ├── application-local.properties │ │ ├── application.properties │ │ ├── banner.txt │ │ └── logback-spring.xml ├── loa-staging-application │ ├── build.gradle │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── github │ │ │ │ └── bottomlessarchive │ │ │ │ └── loa │ │ │ │ └── stage │ │ │ │ ├── LibraryStagingApplication.java │ │ │ │ ├── configuration │ │ │ │ └── StagingConfigurationProperties.java │ │ │ │ ├── service │ │ │ │ ├── StageLocationFactory.java │ │ │ │ ├── StagingFreeSpaceInstancePropertyExtensionProvider.java │ │ │ │ └── location │ │ │ │ │ ├── StageAccessException.java │ │ │ │ │ └── StageLocation.java │ │ │ │ └── view │ │ │ │ └── document │ │ │ │ └── controller │ │ │ │ └── StageDocumentController.java │ │ └── resources │ │ │ └── application.properties │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── stage │ │ ├── integration │ │ └── StageViewIntegrationTest.java │ │ └── service │ │ └── StageLocationFactoryTest.java ├── loa-vault-application │ ├── build.gradle │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── github │ │ │ │ └── bottomlessarchive │ │ │ │ └── loa │ │ │ │ └── vault │ │ │ │ ├── LibraryVaultApplication.java │ │ │ │ ├── configuration │ │ │ │ ├── MongoClientConfiguration.java │ │ │ │ ├── StageConfiguration.java │ │ │ │ ├── VaultConfigurationProperties.java │ │ │ │ └── VaultParallelismConfiguration.java │ │ │ │ ├── service │ │ │ │ ├── VaultNameInstanceRegistrationExtensionProvider.java │ │ │ │ ├── conductor │ │ │ │ │ └── VaultInstancePropertyExtensionProvider.java │ │ │ │ ├── listener │ │ │ │ │ ├── ArchivingMessageProcessor.java │ │ │ │ │ └── VaultQueueListener.java │ │ │ │ └── transformer │ │ │ │ │ └── DocumentArchivingContextTransformer.java │ │ │ │ └── view │ │ │ │ ├── controller │ │ │ │ └── VaultController.java │ │ │ │ ├── domain │ │ │ │ └── InvalidRequestException.java │ │ │ │ ├── request │ │ │ │ └── domain │ │ │ │ │ └── RecompressDocumentRequest.java │ │ │ │ └── response │ │ │ │ └── domain │ │ │ │ ├── DocumentExistsResponse.java │ │ │ │ └── QueryDocumentResponse.java │ │ └── resources │ │ │ ├── application-local.properties │ │ │ ├── application.properties │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── vault │ │ ├── integration │ │ ├── VaultProcessingIntegrationTest.java │ │ ├── VaultViewDefaultIntegrationTest.java │ │ └── VaultViewModificationDisabledIntegrationTest.java │ │ └── view │ │ └── controller │ │ └── VaultControllerTest.java └── loa-web-application │ ├── build.gradle │ └── src │ └── main │ ├── java │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── web │ │ ├── LibraryWebApplication.java │ │ └── view │ │ ├── dashboard │ │ ├── controller │ │ │ └── DashboardController.java │ │ ├── response │ │ │ ├── AdministratorApplicationInstance.java │ │ │ ├── DashboardApplicationsResponse.java │ │ │ ├── DashboardDocumentStatisticsResponse.java │ │ │ ├── DownloaderApplicationInstance.java │ │ │ ├── GeneratorApplicationInstance.java │ │ │ ├── IndexerApplicationInstance.java │ │ │ ├── QueueApplicationInstance.java │ │ │ ├── StagingApplicationInstance.java │ │ │ └── VaultApplicationInstance.java │ │ └── service │ │ │ ├── ApplicationStatisticsResponseFactory.java │ │ │ └── DocumentStatisticsResponseFactory.java │ │ ├── document │ │ ├── controller │ │ │ ├── DocumentDebugController.java │ │ │ ├── DocumentImageController.java │ │ │ ├── DocumentQueryController.java │ │ │ ├── DocumentSearchController.java │ │ │ └── DocumentStatisticsController.java │ │ ├── response │ │ │ ├── DocumentDebugResponse.java │ │ │ ├── DocumentSearchResponse.java │ │ │ ├── DocumentStatisticsResponse.java │ │ │ └── SearchDocumentEntityResponse.java │ │ └── service │ │ │ ├── DocumentDebugResponseFactory.java │ │ │ ├── SearchDocumentEntityResponseTransformer.java │ │ │ └── SearchDocumentEntityResponseTransformerTest.java │ │ ├── info │ │ ├── controller │ │ │ └── InfoController.java │ │ └── response │ │ │ └── InfoResponse.java │ │ ├── location │ │ ├── controller │ │ │ └── DocumentLocationDebugController.java │ │ ├── response │ │ │ └── DocumentLocationDebugResponse.java │ │ └── service │ │ │ └── DocumentLocationDebugResponseFactory.java │ │ └── user │ │ ├── controller │ │ └── UserController.java │ │ ├── request │ │ └── LoginRequest.java │ │ └── response │ │ ├── InfoResponse.java │ │ ├── LoginResponse.java │ │ └── domain │ │ └── LoginResult.java │ └── resources │ ├── application-local.properties │ ├── application.properties │ └── banner.txt ├── loa-frontend ├── .editorconfig ├── .eslintrc.json ├── .gitignore ├── README.md ├── angular.json ├── build.gradle ├── e2e │ ├── protractor.conf.js │ ├── src │ │ ├── app.e2e-spec.ts │ │ └── app.po.ts │ └── tsconfig.json ├── karma.conf.js ├── package-lock.json ├── package.json ├── src │ ├── app │ │ ├── app-routing.module.ts │ │ ├── app.module.ts │ │ ├── bytes.pipe.ts │ │ ├── shared │ │ │ ├── application │ │ │ │ ├── domain │ │ │ │ │ ├── administrator-application-instance.ts │ │ │ │ │ ├── application-info-entity.ts │ │ │ │ │ ├── downloader-application-instance.ts │ │ │ │ │ ├── generator-application-instance.ts │ │ │ │ │ ├── indexer-application-instance.ts │ │ │ │ │ ├── queue-application-instance.ts │ │ │ │ │ ├── vault-application-instance.ts │ │ │ │ │ └── vault-pplication-instance.ts │ │ │ │ └── service │ │ │ │ │ └── application.service.ts │ │ │ ├── debug │ │ │ │ └── service │ │ │ │ │ ├── debug.service.ts │ │ │ │ │ └── domain │ │ │ │ │ ├── debug-document.ts │ │ │ │ │ └── debug-location.ts │ │ │ ├── info │ │ │ │ └── service │ │ │ │ │ ├── domain │ │ │ │ │ └── site-info.ts │ │ │ │ │ └── site-info.service.ts │ │ │ ├── search │ │ │ │ └── service │ │ │ │ │ ├── domain │ │ │ │ │ ├── search-hit.ts │ │ │ │ │ └── search-result.ts │ │ │ │ │ └── search.service.ts │ │ │ ├── statistics │ │ │ │ └── service │ │ │ │ │ └── statistics-service.ts │ │ │ └── user │ │ │ │ └── service │ │ │ │ ├── domain │ │ │ │ ├── login-response.ts │ │ │ │ └── user-info.ts │ │ │ │ └── user-service.ts │ │ └── view │ │ │ ├── app │ │ │ ├── app.component.html │ │ │ ├── app.component.scss │ │ │ └── app.component.ts │ │ │ ├── dashboard-applications │ │ │ ├── dashboard-applications.component.html │ │ │ ├── dashboard-applications.component.scss │ │ │ └── dashboard-applications.component.ts │ │ │ ├── dashboard-statistics │ │ │ ├── dashboard-statistics.component.html │ │ │ ├── dashboard-statistics.component.scss │ │ │ └── dashboard-statistics.component.ts │ │ │ ├── debug-document │ │ │ ├── debug-document.component.html │ │ │ ├── debug-document.component.scss │ │ │ └── debug-document.component.ts │ │ │ ├── debug-location │ │ │ ├── debug-location.component.html │ │ │ ├── debug-location.component.scss │ │ │ └── debug-location.component.ts │ │ │ ├── home │ │ │ ├── home.component.html │ │ │ ├── home.component.scss │ │ │ └── home.component.ts │ │ │ ├── login │ │ │ ├── login.component.html │ │ │ ├── login.component.scss │ │ │ └── login.component.ts │ │ │ └── search │ │ │ ├── search.component.html │ │ │ ├── search.component.scss │ │ │ └── search.component.ts │ ├── assets │ │ ├── background.png │ │ ├── sphinx.ico │ │ ├── sphinx.svg │ │ └── spinner.css │ ├── environments │ │ ├── environment.prod.ts │ │ └── environment.ts │ ├── favicon.ico │ ├── index.html │ ├── main.ts │ ├── polyfills.ts │ ├── styles.scss │ └── test.ts ├── tsconfig.app.json ├── tsconfig.json └── tsconfig.spec.json ├── loa-library ├── build.gradle ├── loa-application-domain │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── application │ │ └── domain │ │ └── ApplicationType.java ├── loa-beacon-client-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── beacon │ │ └── service │ │ └── client │ │ ├── BeaconClient.java │ │ ├── configuration │ │ └── BeaconClientConfiguration.java │ │ ├── domain │ │ ├── BeaconClientException.java │ │ ├── BeaconDocumentLocation.java │ │ └── BeaconDocumentLocationResult.java │ │ ├── request │ │ ├── DocumentLocationPartialRequest.java │ │ └── VisitDocumentLocationsRequest.java │ │ └── response │ │ ├── DocumentLocationResultPartialResponse.java │ │ └── VisitDocumentLocationsResponse.java ├── loa-checksum-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── checksum │ │ │ ├── configuration │ │ │ └── ChecksumConfigurationProperties.java │ │ │ ├── domain │ │ │ ├── ChecksumCalculationException.java │ │ │ └── ChecksumType.java │ │ │ └── service │ │ │ ├── ChecksumProvider.java │ │ │ └── Sha256ChecksumProvider.java │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── checksum │ │ │ └── service │ │ │ └── Sha256ChecksumProviderTest.java │ │ └── resources │ │ └── checksum_test.txt ├── loa-compression-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── compression │ │ │ ├── configuration │ │ │ └── CompressionConfigurationProperties.java │ │ │ ├── domain │ │ │ ├── CompressionException.java │ │ │ └── DocumentCompression.java │ │ │ └── service │ │ │ ├── compressor │ │ │ ├── BrotliCompressorService.java │ │ │ ├── CompressorService.java │ │ │ ├── GZIPCompressorService.java │ │ │ ├── LZMACompressorService.java │ │ │ └── provider │ │ │ │ ├── CompressorServiceProvider.java │ │ │ │ └── domain │ │ │ │ └── MissingCompressionServiceException.java │ │ │ └── file │ │ │ └── FileCompressionService.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── compression │ │ └── service │ │ └── provider │ │ └── CompressorServiceProviderTest.java ├── loa-conductor-client-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── conductor │ │ └── service │ │ ├── InstanceRefreshSchedulerService.java │ │ ├── NetworkAddressCalculator.java │ │ ├── client │ │ ├── ConductorClient.java │ │ ├── configuration │ │ │ ├── ConductorClientConfiguration.java │ │ │ └── ConductorClientConfigurationProperties.java │ │ ├── domain │ │ │ └── ConductorClientException.java │ │ ├── extension │ │ │ ├── InstancePropertyExtensionProvider.java │ │ │ └── domain │ │ │ │ └── InstanceExtensionContext.java │ │ ├── request │ │ │ ├── ServiceInstancePropertyRequest.java │ │ │ ├── ServiceInstanceRefreshRequest.java │ │ │ └── ServiceInstanceRegistrationRequest.java │ │ └── response │ │ │ ├── ServiceInstancePropertyResponse.java │ │ │ ├── ServiceInstanceRegistrationResponse.java │ │ │ ├── ServiceInstanceResponse.java │ │ │ └── ServiceResponse.java │ │ └── domain │ │ └── NetworkAddressCalculationException.java ├── loa-conductor-client-test │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── conductor │ │ └── service │ │ └── ConductorClientTestUtility.java ├── loa-conductor-domain │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── conductor │ │ └── service │ │ └── domain │ │ ├── ServiceInstanceEntity.java │ │ └── ServiceInstanceEntityProperty.java ├── loa-conductor-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── conductor │ │ └── service │ │ ├── ServiceInstanceContainer.java │ │ └── domain │ │ ├── ServiceInstanceProperty.java │ │ ├── ServiceInstanceRefreshContext.java │ │ └── ServiceInstanceRegistrationContext.java ├── loa-document-location-repository │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── location │ │ └── repository │ │ ├── DocumentLocationRepository.java │ │ ├── configuration │ │ └── DocumentLocationMongoConfiguration.java │ │ └── domain │ │ └── DocumentLocationDatabaseEntity.java ├── loa-document-location-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── location │ │ ├── domain │ │ ├── DocumentLocation.java │ │ └── DocumentLocationResultType.java │ │ └── service │ │ ├── DocumentLocationManipulator.java │ │ ├── factory │ │ ├── DocumentLocationEntityFactory.java │ │ ├── DocumentLocationFactory.java │ │ └── domain │ │ │ ├── DocumentLocation.java │ │ │ └── DocumentLocationCreationContext.java │ │ └── id │ │ └── factory │ │ ├── DocumentLocationIdFactory.java │ │ └── Sha256DocumentLocationIdFactory.java ├── loa-document-parser-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── parser │ │ ├── configuration │ │ ├── DocumentParserConfiguration.java │ │ └── LoggerConfiguration.java │ │ ├── domain │ │ ├── ParsingException.java │ │ └── ParsingResult.java │ │ └── service │ │ ├── DocumentDataParser.java │ │ └── parser │ │ └── ExceptionParser.java ├── loa-document-repository │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── document │ │ └── repository │ │ ├── DocumentRepository.java │ │ ├── configuration │ │ └── DocumentMongoConfiguration.java │ │ └── domain │ │ ├── DocumentDatabaseEntity.java │ │ ├── DocumentStatusAggregateEntity.java │ │ └── DocumentTypeAggregateEntity.java ├── loa-document-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── document │ │ │ └── service │ │ │ ├── DocumentManipulator.java │ │ │ ├── domain │ │ │ ├── DocumentEntity.java │ │ │ ├── DocumentStatus.java │ │ │ └── DuplicateDocumentException.java │ │ │ └── entity │ │ │ ├── factory │ │ │ ├── DocumentEntityFactory.java │ │ │ └── domain │ │ │ │ └── DocumentCreationContext.java │ │ │ └── transformer │ │ │ └── DocumentEntityTransformer.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── document │ │ └── service │ │ ├── DocumentTypeCalculatorTest.java │ │ └── entity │ │ └── factory │ │ └── DocumentEntityFactoryTest.java ├── loa-document-type-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── type │ │ ├── DocumentTypeCalculator.java │ │ └── domain │ │ └── DocumentType.java ├── loa-document-view │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── document │ │ └── view │ │ └── service │ │ └── MediaTypeCalculator.java ├── loa-downloader-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── io │ │ │ └── service │ │ │ ├── collector │ │ │ ├── DocumentCollector.java │ │ │ └── domain │ │ │ │ └── DocumentCollectionException.java │ │ │ ├── configuration │ │ │ ├── DownloaderClientConfiguration.java │ │ │ ├── HttpClientConfigurationProperties.java │ │ │ └── ssl │ │ │ │ └── TrustAllTrustManager.java │ │ │ └── downloader │ │ │ └── FileDownloadManager.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── io │ │ └── service │ │ └── collector │ │ └── DocumentCollectorTest.java ├── loa-file-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── file │ │ │ ├── FileManipulatorService.java │ │ │ └── zip │ │ │ └── ZipFileManipulatorService.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── file │ │ ├── FileManipulatorServiceTest.java │ │ └── zip │ │ └── ZipFileManipulatorServiceTest.java ├── loa-indexer-client-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── indexer │ │ │ └── service │ │ │ ├── configuration │ │ │ └── IndexDatabaseConfiguration.java │ │ │ ├── indexer │ │ │ ├── IndexDocumentFactory.java │ │ │ ├── IndexRequestFactory.java │ │ │ ├── IndexerClient.java │ │ │ └── domain │ │ │ │ └── IndexingContext.java │ │ │ └── search │ │ │ ├── DocumentSearchClient.java │ │ │ ├── domain │ │ │ ├── DocumentLength.java │ │ │ ├── DocumentSearchEntity.java │ │ │ ├── DocumentSearchResult.java │ │ │ ├── IndexerAccessException.java │ │ │ ├── SearchContext.java │ │ │ ├── SearchDatabaseEntity.java │ │ │ └── SearchField.java │ │ │ ├── request │ │ │ ├── IndexerRequestFactory.java │ │ │ └── QueryBuilderFactory.java │ │ │ └── transformer │ │ │ └── DocumentSearchEntityTransformer.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── indexer │ │ └── service │ │ ├── indexer │ │ ├── IndexDocumentFactoryTest.java │ │ ├── IndexRequestFactoryTest.java │ │ └── IndexerClientTest.java │ │ └── search │ │ ├── DocumentSearchClientTest.java │ │ ├── request │ │ └── IndexerRequestFactoryTest.java │ │ └── transformer │ │ └── DocumentSearchEntityTransformerTest.java ├── loa-io-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── url │ │ │ └── service │ │ │ └── downloader │ │ │ ├── DocumentLocationResultCalculator.java │ │ │ └── domain │ │ │ └── DownloadResult.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── url │ │ └── service │ │ └── downloader │ │ └── DocumentLocationResultCalculatorTest.java ├── loa-loadoc-domain │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── loadoc │ │ └── domain │ │ └── LoadocMetadata.java ├── loa-logging-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── logging │ │ └── service │ │ ├── LoggingMeterRegistry.java │ │ └── MetricLogger.java ├── loa-number-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── number │ │ └── service │ │ ├── HexConverter.java │ │ └── domain │ │ └── HexConversionException.java ├── loa-queue-artemis-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── queue │ │ │ └── artemis │ │ │ ├── configuration │ │ │ ├── QueueClientConfiguration.java │ │ │ ├── QueueServerConfiguration.java │ │ │ └── QueueServerConfigurationProperties.java │ │ │ └── service │ │ │ ├── ArtemisQueueManipulator.java │ │ │ ├── consumer │ │ │ ├── deserializer │ │ │ │ ├── DocumentArchivingMessageDeserializer.java │ │ │ │ ├── DocumentLocationMessageDeserializer.java │ │ │ │ ├── MessageDeserializer.java │ │ │ │ └── MessageDeserializerProvider.java │ │ │ └── pool │ │ │ │ ├── QueueConsumerProvider.java │ │ │ │ └── domain │ │ │ │ └── QueueConsumer.java │ │ │ └── producer │ │ │ ├── pool │ │ │ ├── QueueProducerProvider.java │ │ │ └── domain │ │ │ │ └── QueueProducer.java │ │ │ └── serializer │ │ │ ├── DocumentArchivingMessageSerializer.java │ │ │ ├── DocumentLocationMessageSerializer.java │ │ │ ├── MessageSerializer.java │ │ │ └── MessageSerializerProvider.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── queue │ │ └── artemis │ │ └── service │ │ ├── ArtemisQueueManipulatorTest.java │ │ ├── consumer │ │ └── deserializer │ │ │ └── DocumentArchivingMessageDeserializerTest.java │ │ └── producer │ │ └── serializer │ │ └── DocumentArchivingMessageSerializerTest.java ├── loa-queue-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── queue │ │ └── service │ │ ├── QueueManipulator.java │ │ └── domain │ │ ├── Queue.java │ │ ├── QueueException.java │ │ └── message │ │ ├── DocumentArchivingMessage.java │ │ └── DocumentLocationMessage.java ├── loa-renderer-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── renderer │ │ └── service │ │ ├── DocumentRendererService.java │ │ ├── PageRendererService.java │ │ ├── ThumbnailService.java │ │ └── domain │ │ └── ImageRenderingException.java ├── loa-repository │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── repository │ │ ├── configuration │ │ ├── RepositoryConfiguration.java │ │ └── RepositoryMetadataContainer.java │ │ └── document │ │ └── Error.java ├── loa-source-file-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── source │ │ │ └── file │ │ │ ├── configuration │ │ │ ├── FileDocumentSourceConfigurationProperties.java │ │ │ └── GeneratorStatisticsConfiguration.java │ │ │ └── service │ │ │ ├── FileSourceFactory.java │ │ │ ├── domain │ │ │ ├── FileEncodingType.java │ │ │ └── FileHandlingException.java │ │ │ └── location │ │ │ └── FileDocumentLocationSource.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── source │ │ └── file │ │ └── service │ │ ├── FileSourceFactoryTest.java │ │ └── location │ │ └── FileDocumentLocationSourceTest.java ├── loa-source-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── source │ │ ├── configuration │ │ └── DocumentSourceConfigurationProperties.java │ │ ├── domain │ │ └── DocumentSourceType.java │ │ └── source │ │ └── DocumentLocationSource.java ├── loa-stage-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── stage │ │ │ └── service │ │ │ ├── StageLocationFactory.java │ │ │ └── domain │ │ │ ├── StageLocation.java │ │ │ └── exception │ │ │ └── StageAccessException.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── stage │ │ └── service │ │ └── StageLocationFactoryTest.java ├── loa-staging-client-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── staging │ │ └── service │ │ └── client │ │ ├── StagingClient.java │ │ ├── configuration │ │ └── StagingClientConfiguration.java │ │ └── domain │ │ └── StagingException.java ├── loa-threading-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── threading │ │ ├── executor │ │ └── BlockingExecutor.java │ │ ├── task │ │ ├── CallbackWrapperTask.java │ │ ├── CounterIncrementingWrapperTask.java │ │ └── MDCWrapperTask.java │ │ └── thread │ │ └── ThreadManipulator.java ├── loa-url-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── url │ │ │ └── service │ │ │ └── encoder │ │ │ └── UrlEncoder.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── url │ │ └── service │ │ └── encoder │ │ └── UrlEncoderTest.java ├── loa-user-repository │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── user │ │ └── repository │ │ ├── UserRepository.java │ │ ├── configuration │ │ └── UserMongoConfiguration.java │ │ └── domain │ │ ├── UserAlreadyExistsInDatabaseException.java │ │ └── UserDatabaseEntity.java ├── loa-user-service │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── user │ │ ├── configuration │ │ └── UserConfigurationProperties.java │ │ └── service │ │ ├── UserEntityFactory.java │ │ ├── UserEntityTransformer.java │ │ └── domain │ │ ├── UserAlreadyExistsException.java │ │ └── UserEntity.java ├── loa-validator-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── validator │ │ │ ├── configuration │ │ │ └── FileValidationConfigurationProperties.java │ │ │ └── service │ │ │ └── DocumentFileValidator.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── validator │ │ └── service │ │ └── DocumentFileValidatorTest.java ├── loa-vault-client-service │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bottomlessarchive │ │ │ └── loa │ │ │ └── vault │ │ │ └── client │ │ │ ├── configuration │ │ │ └── VaultClientConfiguration.java │ │ │ └── service │ │ │ ├── VaultClientService.java │ │ │ ├── VaultLocationContainer.java │ │ │ ├── VaultLocationFetcher.java │ │ │ ├── domain │ │ │ ├── VaultAccessException.java │ │ │ └── VaultLocation.java │ │ │ ├── request │ │ │ └── RecompressDocumentRequest.java │ │ │ └── response │ │ │ ├── DocumentExistsResponse.java │ │ │ └── QueryDocumentResponse.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── vault │ │ └── client │ │ └── service │ │ └── VaultClientServiceTest.java └── loa-vault-service │ ├── build.gradle │ └── src │ ├── main │ └── java │ │ └── com │ │ └── github │ │ └── bottomlessarchive │ │ └── loa │ │ └── vault │ │ ├── domain │ │ ├── VaultType.java │ │ └── exception │ │ │ └── StorageAccessException.java │ │ └── service │ │ ├── RecompressorService.java │ │ ├── VaultDocumentManager.java │ │ ├── archive │ │ ├── ArchivingService.java │ │ └── DocumentCreationContextFactory.java │ │ ├── backend │ │ ├── domain │ │ │ ├── VaultPersistenceException.java │ │ │ └── VaultStorageBackend.java │ │ └── service │ │ │ └── VaultDocumentStorage.java │ │ ├── domain │ │ └── DocumentArchivingContext.java │ │ └── location │ │ ├── VaultLocation.java │ │ ├── VaultLocationFactory.java │ │ ├── configuration │ │ └── VaultLocationConfigurationProperties.java │ │ ├── file │ │ ├── FileVaultLocationFactory.java │ │ ├── configuration │ │ │ └── FileConfigurationProperties.java │ │ └── domain │ │ │ └── FileVaultLocation.java │ │ └── s3 │ │ ├── S3VaultLocationFactory.java │ │ ├── configuration │ │ ├── S3Configuration.java │ │ └── S3ConfigurationProperties.java │ │ └── domain │ │ └── S3VaultLocation.java │ └── test │ └── java │ └── com │ └── github │ └── bottomlessarchive │ └── loa │ └── vault │ └── service │ ├── archive │ ├── ArchivingServiceTest.java │ └── DocumentCreationContextFactoryTest.java │ ├── backend │ └── service │ │ └── VaultDocumentStorageTest.java │ └── location │ ├── file │ ├── FileVaultLocationFactoryTest.java │ └── domain │ │ └── FileVaultLocationTest.java │ └── s3 │ └── domain │ └── S3VaultLocationTest.java ├── lombok.config └── settings.gradle /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | name: Java CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v1 11 | - name: Set up JDK 21 12 | uses: actions/setup-java@v1 13 | with: 14 | java-version: 21 15 | - name: Build with Gradle 16 | run: ./gradlew clean build 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .gradle 3 | *.iml 4 | */build/ 5 | */*/build/ 6 | */out/ 7 | */*/out/ 8 | data/ 9 | /documentation/index.html 10 | -------------------------------------------------------------------------------- /documentation/application/application-architectural-overview.adoc: -------------------------------------------------------------------------------- 1 | = Architectural overview 2 | 3 | The collection of documents starts with the <>. The <> is responsible to create *document locations* (known URLs where documents could be available). These locations are passed to the <> where they are stored until a <> picks them up and visit the location to get new *documents*. Then the downloaded document is sent to a new queue in the <>. After a while a <> will pick up a document and *archive* it. If indexing is being done then every archived document will be indexed by the <> and made searchable by the <>. The communication of the connection information between these applications are made available by the <> using https://en.wikipedia.org/wiki/Service_discovery[service discovery]. 4 | 5 | image::architecture.svg[align="center"] -------------------------------------------------------------------------------- /documentation/application/application-parameters.adoc: -------------------------------------------------------------------------------- 1 | = Application parameters 2 | 3 | This section describes each supported parameter for every application. Supplying these runtime parameters can be done by adding them to the runtime arguments of the application with the `--` prefix. For example: 4 | 5 | .... 6 | java -jar loa-downloader-application-{release-number}.jar --loa.downloader.parallelism=10 7 | .... 8 | -------------------------------------------------------------------------------- /documentation/application/application.adoc: -------------------------------------------------------------------------------- 1 | = Applications 2 | 3 | The Library of Alexandria project consists of more than one (usually) scalable applications. 4 | Not all of them are mandatory for the archiving effort. 5 | Some of them are created for administrating or maintenance purposes. 6 | 7 | include::application-architectural-overview.adoc[leveloffset=+1] 8 | include::application-parameters.adoc[leveloffset=+1] 9 | include::application-conductor.adoc[leveloffset=+1] 10 | include::application-queue.adoc[leveloffset=+1] 11 | include::application-vault.adoc[leveloffset=+1] 12 | include::application-generator.adoc[leveloffset=+1] 13 | include::application-staging.adoc[leveloffset=+1] 14 | include::application-downloader.adoc[leveloffset=+1] 15 | include::application-indexer.adoc[leveloffset=+1] 16 | include::application-web.adoc[leveloffset=+1] 17 | include::application-administration.adoc[leveloffset=+1] 18 | -------------------------------------------------------------------------------- /documentation/application/beacon-application.adoc: -------------------------------------------------------------------------------- 1 | :application-type: BEACON_APPLICATION 2 | :application-name: Beacon Application 3 | 4 | = Beacon Application 5 | 6 | The Beacon Application is experimental and should not be used in production. 7 | -------------------------------------------------------------------------------- /documentation/assets/images/logo-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/documentation/assets/images/logo-background.png -------------------------------------------------------------------------------- /documentation/domain/domain.adoc: -------------------------------------------------------------------------------- 1 | = Domain Language 2 | 3 | .Language elements 4 | |=== 5 | | Name | Description 6 | 7 | | **Vault** 8 | | The location where the collected documents are archived. 9 | 10 | | **Document** 11 | | A document collected from the internet. 12 | 13 | | **Staging area** 14 | | A temporary location where the collected documents placed for post processing before going to the archive. 15 | 16 | | **Source** 17 | | A source of document locations. At the moment, it can only be a file. 18 | 19 | | **Failure rate** 20 | | How many documents fail to download compared to the successfully downloaded ones. 21 | 22 | | **Archiving** 23 | | Saving a document into the vault. 24 | |=== 25 | -------------------------------------------------------------------------------- /documentation/installation/installation-conductor-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Conductor Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-conductor-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-downloader-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Downloader Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-downloader-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-generator-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Generator Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-generator-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-indexer-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Indexer Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-indexer-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-java.adoc: -------------------------------------------------------------------------------- 1 | = Installing Java 2 | 3 | First, you need to download the Java 17 Runtime Environment. It's available https://www.oracle.com/technetwork/java/javase/downloads/index.html[here]. After the download is complete you should run the installer and follow the directions it provides until the installation is complete. 4 | 5 | Once it's done, if you open a command line (write cmd to the Start menu's search bar) you will be able to use the java command. Try to write `java -version`. You should get something similar: 6 | 7 | .... 8 | java version "15" 2020-09-15 9 | Java(TM) SE Runtime Environment (build 15+36-1562) 10 | Java HotSpot(TM) 64-Bit Server VM (build 15+36-1562, mixed mode, sharing) 11 | .... 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-mongodb.adoc: -------------------------------------------------------------------------------- 1 | = Installing MongoDB 2 | 3 | Download MongoDB 5.0 from https://www.mongodb.com/download-center/community[here]. After the download is complete run the installer and follow the directions it provides. If it's possible, install the MongoDB Compass tool as well because you will need it later for administrative tasks. 4 | -------------------------------------------------------------------------------- /documentation/installation/installation-queue-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Queue Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-queue-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-staging-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Staging Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-staging-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-vault-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Vault Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-vault-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/installation/installation-web-application.adoc: -------------------------------------------------------------------------------- 1 | = Running the Web Application 2 | 3 | You can download the <> files at our https://github.com/bottomless-archive-project/library-of-alexandria/releases[release page]. Please take care to choose a non "pre-release" version! 4 | 5 | After the download is complete run the application via the following command: 6 | 7 | .... 8 | java -jar loa-web-application-{release-number}.jar ... 9 | .... 10 | 11 | In the place of the ... you should write the various parameters. For the available parameters check the parameter list under the <>. 12 | -------------------------------------------------------------------------------- /documentation/prerequisites/prerequisites.adoc: -------------------------------------------------------------------------------- 1 | = Prerequisites 2 | 3 | Special emphasis was placed on to keep the prerequisites of the project to the minimum. This is because one of the main goals of the project is to make it easy to start archiving even with limited technical knowledge and resources. 4 | 5 | List of the required software to start archiving: 6 | 7 | - https://www.oracle.com/technetwork/java/javase/downloads/index.html[Java 21] 8 | - https://www.mongodb.com/download-center/community[MongoDB 6.0] 9 | 10 | List of the required software to be able to search the archived documents: 11 | 12 | - https://www.elastic.co/downloads/past-releases/elasticsearch-8-3-1[Elasticsearch 8.3.1] 13 | -------------------------------------------------------------------------------- /documentation/service/conductor/conductor-configuration.adoc: -------------------------------------------------------------------------------- 1 | | **loa.conductor.host** 2 | | The location (ip address or domain) of the <>. *(Default value: localhost)* 3 | 4 | | **loa.conductor.port** 5 | | The port where the <> is listening for new connections. *(Default value: 8092)* 6 | 7 | | **loa.conductor.application-type** 8 | | Sets the type of the application. If this property is set to `{application-type}` then this application will be registered as a {application-name}. This property is pre-configured for each service. **Don't change it!** 9 | 10 | | **loa.conductor.application-port** 11 | | Sets the port that this application reports as the listening port. This property is pre-configured for each service. **Don't change it!** 12 | -------------------------------------------------------------------------------- /documentation/system-requirements/system-requirements.adoc: -------------------------------------------------------------------------------- 1 | = System Requirements 2 | 3 | It's important to mention the hardware requirements of collecting and handling documents. If you only want to gather a few million documents then that could be done on a home PC. However, if you want to collect & store a massive amount, then purpose-built hardware is necessary. Also, if you want to collect 24/7 then having a machine that doesn't consume a lot of power is recommended where electricity prices are high. 4 | 5 | Collecting and indexing have quite different requirements as well, so we split the hardware requirements section into two separate chapters. 6 | 7 | The requirements mentioned here could be built from used/commodity hardware if necessary to keep the prices low. 8 | 9 | include::system-requirements-collecting.adoc[leveloffset=+1] 10 | include::system-requirements-indexing.adoc[leveloffset=+1] 11 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | group=com.github.bottomlessarchive.loa 2 | version=2.1.0-milestone.1 3 | org.gradle.parallel=true 4 | dockerUser= 5 | dockerPassword= 6 | dockerEmail= 7 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /loa-application/build.gradle: -------------------------------------------------------------------------------- 1 | subprojects { 2 | dependencies { 3 | testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: '1.19.7' 4 | testImplementation group: 'org.testcontainers', name: 'testcontainers', version: '1.19.7' 5 | 6 | testImplementation group: 'org.wiremock', name: 'wiremock-standalone', version: '3.5.2' 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/java/com/github/bottomlessarchive/loa/administrator/command/compressor/configuration/SilentCompressorConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.administrator.command.compressor.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.compression.domain.DocumentCompression; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | 6 | @ConfigurationProperties("loa.command.silent-compressor") 7 | public record SilentCompressorConfigurationProperties( 8 | 9 | DocumentCompression algorithm 10 | ) { 11 | 12 | public boolean hasAlgorithm() { 13 | return algorithm != null; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/java/com/github/bottomlessarchive/loa/administrator/command/user/configuration/RegisterUserConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.administrator.command.user.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | @ConfigurationProperties("loa.command.register-user") 6 | public record RegisterUserConfigurationProperties( 7 | String name, 8 | String password 9 | ) { 10 | 11 | public boolean isNameValid() { 12 | return name != null && name.length() > 3; 13 | } 14 | 15 | public boolean isPasswordValid() { 16 | return password != null && password.length() > 3; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/java/com/github/bottomlessarchive/loa/administrator/configuration/AdministratorCommandConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.administrator.configuration; 2 | 3 | import jakarta.validation.constraints.NotBlank; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | @Validated 8 | @ConfigurationProperties("loa.command") 9 | public record AdministratorCommandConfigurationProperties( 10 | 11 | @NotBlank 12 | String name 13 | ) { 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/java/com/github/bottomlessarchive/loa/administrator/configuration/AdministratorConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.administrator.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.nio.file.Path; 6 | 7 | @ConfigurationProperties("loa.administrator") 8 | public record AdministratorConfigurationProperties( 9 | 10 | Path stagingDirectory 11 | ) { 12 | } 13 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/java/com/github/bottomlessarchive/loa/administrator/configuration/StageConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.administrator.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.stage.service.StageLocationFactory; 4 | import lombok.RequiredArgsConstructor; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | @RequiredArgsConstructor 10 | public class StageConfiguration { 11 | 12 | private final AdministratorConfigurationProperties administratorConfigurationProperties; 13 | 14 | @Bean 15 | public StageLocationFactory stageLocationFactory() { 16 | return new StageLocationFactory(administratorConfigurationProperties.stagingDirectory()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-application/loa-administrator-application/src/main/resources/application-local.properties -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # Server 2 | spring.main.web-application-type=none 3 | 4 | # Compression 5 | loa.compression.algorithm=none 6 | 7 | # Logging 8 | logging.level.org.apache.pdfbox=FATAL 9 | logging.level.org.apache.fontbox=FATAL 10 | logging.level.org.apache.tika.parser.SQLite3Parser=FATAL 11 | logging.level.org.apache.tika.parsers.PDFParser=FATAL 12 | 13 | # Stage 14 | loa.administrator.staging-directory=${java.io.tmpdir} 15 | 16 | # Validation 17 | loa.validation.maximum-archive-size=8589934592 18 | 19 | # Conductor 20 | loa.conductor.host=localhost 21 | loa.conductor.port=8092 22 | loa.conductor.application-type=ADMINISTRATOR_APPLICATION 23 | loa.conductor.application-port=0 24 | -------------------------------------------------------------------------------- /loa-application/loa-administrator-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/LibraryBeaconApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 6 | import org.springframework.scheduling.annotation.EnableScheduling; 7 | 8 | /** 9 | * The runner class of the beacon application. 10 | */ 11 | @EnableScheduling 12 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 13 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 14 | public class LibraryBeaconApplication { 15 | 16 | public static void main(final String[] args) { 17 | SpringApplication.run(LibraryBeaconApplication.class, args); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/command/configuration/OfflineDownloaderConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.command.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.nio.file.Path; 6 | 7 | @ConfigurationProperties("loa.beacon.offline") 8 | public record OfflineDownloaderConfigurationProperties( 9 | 10 | Path sourceFile, 11 | long skip 12 | ) { 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/command/configuration/OfflineDownloaderStatisticsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.command.configuration; 2 | 3 | import io.micrometer.core.instrument.Counter; 4 | import io.micrometer.core.instrument.MeterRegistry; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | public class OfflineDownloaderStatisticsConfiguration { 10 | 11 | @Bean 12 | public Counter processedDocumentCount(final MeterRegistry meterRegistry) { 13 | return Counter.builder("downloader.processed-document-count") 14 | .tag("printed-name", "processed document count") 15 | .baseUnit("documents") 16 | .register(meterRegistry); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/configuration/BeaconConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.configuration; 2 | 3 | import jakarta.validation.constraints.NotNull; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | import java.nio.file.Path; 8 | 9 | @Validated 10 | @ConfigurationProperties("loa.beacon") 11 | public record BeaconConfigurationProperties( 12 | 13 | @NotNull 14 | Path storageDirectory, 15 | 16 | @NotNull 17 | Path stagingDirectory 18 | ) { 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/configuration/StageConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.stage.service.StageLocationFactory; 4 | import lombok.RequiredArgsConstructor; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | @RequiredArgsConstructor 10 | public class StageConfiguration { 11 | 12 | private final BeaconConfigurationProperties beaconConfigurationProperties; 13 | 14 | @Bean 15 | public StageLocationFactory stageLocationFactory() { 16 | return new StageLocationFactory(beaconConfigurationProperties.stagingDirectory()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/service/StoragePathFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service; 2 | 3 | import com.github.bottomlessarchive.loa.beacon.configuration.BeaconConfigurationProperties; 4 | import lombok.RequiredArgsConstructor; 5 | import org.springframework.stereotype.Service; 6 | 7 | import java.nio.file.Path; 8 | import java.util.UUID; 9 | 10 | @Service 11 | @RequiredArgsConstructor 12 | public class StoragePathFactory { 13 | 14 | private final BeaconConfigurationProperties beaconConfigurationProperties; 15 | 16 | public Path buildStoragePath(final UUID documentId) { 17 | return beaconConfigurationProperties.storageDirectory() 18 | .resolve(documentId.toString()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/service/domain/DocumentLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | @Getter 8 | @Builder 9 | public class DocumentLocation { 10 | 11 | private final String id; 12 | private final String location; 13 | private final DocumentType type; 14 | private final String sourceName; 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/service/domain/DocumentLocationProcessingException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class DocumentLocationProcessingException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/service/domain/DocumentLocationResult.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import com.github.bottomlessarchive.loa.url.service.downloader.domain.DownloadResult; 5 | import lombok.Builder; 6 | 7 | import java.util.UUID; 8 | 9 | @Builder 10 | public record DocumentLocationResult( 11 | 12 | String id, 13 | UUID documentId, 14 | String checksum, 15 | long size, 16 | DownloadResult resultType, 17 | String sourceName, 18 | DocumentType type 19 | ) { 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/view/beacon/request/DocumentLocationPartialRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.view.beacon.request; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.extern.jackson.Jacksonized; 7 | 8 | @Getter 9 | @Builder 10 | @Jacksonized 11 | public class DocumentLocationPartialRequest { 12 | 13 | private final String id; 14 | private final String location; 15 | private final DocumentType type; 16 | private final String sourceName; 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/view/beacon/request/VisitDocumentLocationsRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.view.beacon.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.List; 8 | 9 | @Getter 10 | @Builder 11 | @Jacksonized 12 | public class VisitDocumentLocationsRequest { 13 | 14 | private final List locations; 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/view/beacon/response/DocumentLocationResultPartialResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.view.beacon.response; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import com.github.bottomlessarchive.loa.url.service.downloader.domain.DownloadResult; 5 | import lombok.Builder; 6 | import lombok.Getter; 7 | 8 | @Getter 9 | @Builder 10 | public class DocumentLocationResultPartialResponse { 11 | 12 | private final String id; 13 | private final String documentId; 14 | private final String checksum; 15 | private final long size; 16 | private final DownloadResult resultType; 17 | private final String sourceName; 18 | private final DocumentType type; 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/java/com/github/bottomlessarchive/loa/beacon/view/beacon/response/VisitDocumentLocationsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.view.beacon.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.List; 7 | 8 | @Getter 9 | @Builder 10 | public class VisitDocumentLocationsResponse { 11 | 12 | private final List results; 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- 1 | # Stage 2 | loa.beacon.storage-directory=C:\\loa\\beacon 3 | loa.beacon.staging-directory=C:\\loa\\stage 4 | -------------------------------------------------------------------------------- /loa-application/loa-beacon-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'org.springframework.boot' 2 | 3 | bootJar { 4 | manifest { 5 | attributes( 6 | 'Implementation-Title': 'Library of Alexandria - Conductor Application', 7 | 'Implementation-Version': archiveVersion 8 | ) 9 | } 10 | } 11 | 12 | bootBuildImage { 13 | imageName = "bottomlessarchive/loa-conductor-application:${rootProject.version.toString()}" 14 | publish = true 15 | docker { 16 | publishRegistry { 17 | username = dockerUser 18 | password = dockerPassword 19 | email = dockerEmail 20 | url = "https://index.docker.io/v2/" 21 | } 22 | } 23 | } 24 | 25 | dependencies { 26 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web' 27 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 28 | 29 | implementation project(':loa-library:loa-conductor-service') 30 | } 31 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/LibraryConductorApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 6 | import org.springframework.scheduling.annotation.EnableScheduling; 7 | 8 | @EnableScheduling 9 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 10 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 11 | public class LibraryConductorApplication { 12 | 13 | public static void main(final String[] args) { 14 | SpringApplication.run(LibraryConductorApplication.class, args); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/configuration/ConductorWebMvcConfigurer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.conductor.view.converter.StringToApplicationTypeConverter; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.format.FormatterRegistry; 6 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 7 | 8 | @Configuration 9 | public class ConductorWebMvcConfigurer implements WebMvcConfigurer { 10 | 11 | @Override 12 | public void addFormatters(final FormatterRegistry registry) { 13 | registry.addConverter(new StringToApplicationTypeConverter()); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/configuration/IndexDatabaseConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.configuration; 2 | 3 | import jakarta.validation.constraints.Max; 4 | import jakarta.validation.constraints.Min; 5 | import jakarta.validation.constraints.NotBlank; 6 | import org.springframework.boot.context.properties.ConfigurationProperties; 7 | import org.springframework.validation.annotation.Validated; 8 | 9 | @Validated 10 | @ConfigurationProperties("loa.indexer.database") 11 | public record IndexDatabaseConfigurationProperties( 12 | 13 | @NotBlank 14 | String host, 15 | 16 | @Min(1) 17 | @Max(65535) 18 | int port 19 | ) { 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/configuration/RepositoryConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | @ConfigurationProperties("loa.database") 6 | public record RepositoryConfigurationProperties( 7 | 8 | String host, 9 | int port, 10 | boolean noCursorTimeout, 11 | String uri 12 | ) { 13 | 14 | public boolean isUriConfiguration() { 15 | return uri != null && !uri.isEmpty(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/converter/StringToApplicationTypeConverter.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.converter; 2 | 3 | import com.github.bottomlessarchive.loa.application.domain.ApplicationType; 4 | import org.springframework.core.convert.converter.Converter; 5 | 6 | import java.util.Locale; 7 | 8 | public class StringToApplicationTypeConverter implements Converter { 9 | 10 | @Override 11 | public ApplicationType convert(final String source) { 12 | return ApplicationType.valueOf(source.replaceAll("-", "_").toUpperCase(Locale.ENGLISH)); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/request/ServiceInstancePropertyRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Getter 8 | @Builder 9 | @Jacksonized 10 | public class ServiceInstancePropertyRequest { 11 | 12 | private final String name; 13 | private final String value; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/request/ServiceInstanceRefreshRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.request; 2 | 3 | import lombok.Builder; 4 | import lombok.RequiredArgsConstructor; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.Collections; 8 | import java.util.List; 9 | 10 | @Builder 11 | @Jacksonized 12 | @RequiredArgsConstructor 13 | public class ServiceInstanceRefreshRequest { 14 | 15 | private final List properties; 16 | 17 | public List getProperties() { 18 | return Collections.unmodifiableList(properties); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/request/ServiceInstanceRegistrationRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.request; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | import java.util.Collections; 7 | import java.util.List; 8 | 9 | @Getter 10 | @RequiredArgsConstructor 11 | public class ServiceInstanceRegistrationRequest { 12 | 13 | private final String location; 14 | private final int port; 15 | private final List properties; 16 | 17 | public List getProperties() { 18 | return Collections.unmodifiableList(properties); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/response/ServiceInstancePropertyResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Getter 8 | @Builder 9 | @Jacksonized 10 | public class ServiceInstancePropertyResponse { 11 | 12 | private final String name; 13 | private final String value; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/response/ServiceInstanceRegistrationResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.UUID; 7 | 8 | @Getter 9 | @Builder 10 | public class ServiceInstanceRegistrationResponse { 11 | 12 | private final UUID id; 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/response/ServiceInstanceResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.time.Instant; 7 | import java.util.List; 8 | import java.util.UUID; 9 | 10 | @Getter 11 | @Builder 12 | public class ServiceInstanceResponse { 13 | 14 | private final UUID id; 15 | private final String location; 16 | private final int port; 17 | private final Instant lastHeartbeat; 18 | private final List properties; 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/java/com/github/bottomlessarchive/loa/conductor/view/response/ServiceResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.view.response; 2 | 3 | import com.github.bottomlessarchive.loa.application.domain.ApplicationType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.List; 8 | 9 | @Getter 10 | @Builder 11 | public class ServiceResponse { 12 | 13 | private final ApplicationType applicationType; 14 | private final List instances; 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-application/loa-conductor-application/src/main/resources/application-local.properties -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=8092 2 | 3 | # Database 4 | loa.database.host=127.0.0.1 5 | loa.database.port=27017 6 | loa.database.no-cursor-timeout=true 7 | 8 | # Indexer 9 | loa.indexer.database.host=127.0.0.1 10 | loa.indexer.database.port=9200 -------------------------------------------------------------------------------- /loa-application/loa-conductor-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/LibraryDownloaderApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 6 | import org.springframework.retry.annotation.EnableRetry; 7 | import org.springframework.scheduling.annotation.EnableScheduling; 8 | 9 | /** 10 | * The runner class of the downloader application. 11 | */ 12 | @EnableRetry 13 | @EnableScheduling 14 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 15 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 16 | public class LibraryDownloaderApplication { 17 | 18 | public static void main(final String[] args) { 19 | SpringApplication.run(LibraryDownloaderApplication.class, args); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/configuration/SourceLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.configuration; 2 | 3 | public enum SourceLocation { 4 | 5 | QUEUE, 6 | FOLDER, 7 | BEACON_REPLY, 8 | BEACON_OFFLINE, 9 | LOADOC_LOADER 10 | } 11 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/configuration/StageConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.stage.service.StageLocationFactory; 4 | import lombok.RequiredArgsConstructor; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | @RequiredArgsConstructor 10 | public class StageConfiguration { 11 | 12 | private final DownloaderConfigurationProperties downloaderConfigurationProperties; 13 | 14 | @Bean 15 | public StageLocationFactory stageLocationFactory() { 16 | return new StageLocationFactory(downloaderConfigurationProperties.stagingDirectory()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/document/DocumentIdFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.document; 2 | 3 | import org.springframework.stereotype.Service; 4 | 5 | import java.util.UUID; 6 | 7 | @Service 8 | public class DocumentIdFactory { 9 | 10 | public UUID newDocumentId() { 11 | return UUID.randomUUID(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/document/DocumentLocationProcessorTask.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.document; 2 | 3 | import com.github.bottomlessarchive.loa.location.domain.DocumentLocation; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | @RequiredArgsConstructor 7 | public class DocumentLocationProcessorTask implements Runnable { 8 | 9 | private final DocumentLocation documentLocation; 10 | private final DocumentLocationProcessingExecutor documentLocationProcessingExecutor; 11 | 12 | @Override 13 | public void run() { 14 | documentLocationProcessingExecutor.executeProcessing(documentLocation); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/document/domain/DocumentArchivingContext.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.document.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.nio.file.Path; 8 | import java.util.UUID; 9 | 10 | @Getter 11 | @Builder 12 | public class DocumentArchivingContext { 13 | 14 | private final UUID id; 15 | private final DocumentType type; 16 | private final String source; 17 | private final String sourceLocationId; 18 | private final Path contents; 19 | private final boolean fromBeacon; 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/document/domain/exception/ArchivingException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.document.domain.exception; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class ArchivingException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/document/domain/exception/NotEnoughSpaceException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.document.domain.exception; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class NotEnoughSpaceException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/source/beacon/configuration/BeaconDownloaderConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.source.beacon.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.nio.file.Path; 6 | 7 | @ConfigurationProperties("loa.downloader.beacon") 8 | public record BeaconDownloaderConfigurationProperties( 9 | 10 | String name, 11 | String host, 12 | int port, 13 | int requestSize, 14 | Path location //TODO: This should be loader.location in a different prop config 15 | ) { 16 | } 17 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/source/beacon/loader/domain/LoaderException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.source.beacon.loader.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class LoaderException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/source/beacon/loadocloader/configuration/LoadocLoaderBeaconConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.source.beacon.loadocloader.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.nio.file.Path; 6 | 7 | @ConfigurationProperties("loa.downloader.loadoc") 8 | public record LoadocLoaderBeaconConfigurationProperties( 9 | 10 | Path location 11 | ) { 12 | } 13 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/source/beacon/offline/configuration/OfflineBeaconDownloaderConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.source.beacon.offline.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.nio.file.Path; 6 | 7 | @ConfigurationProperties("loa.downloader.beacon.offline") 8 | public record OfflineBeaconDownloaderConfigurationProperties( 9 | 10 | long resultSize, 11 | Path resultLocation 12 | ) { 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/source/beacon/service/BeaconDocumentLocationFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.source.beacon.service; 2 | 3 | import com.github.bottomlessarchive.loa.beacon.service.client.domain.BeaconDocumentLocation; 4 | import com.github.bottomlessarchive.loa.location.domain.DocumentLocation; 5 | import org.springframework.stereotype.Service; 6 | 7 | @Service 8 | public class BeaconDocumentLocationFactory { 9 | 10 | public BeaconDocumentLocation newBeaconDocumentLocation(final DocumentLocation location) { 11 | return BeaconDocumentLocation.builder() 12 | .id(location.getId()) 13 | .type(location.getType()) 14 | .location(location.getLocation()) 15 | .sourceName(location.getSourceName()) 16 | .build(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/java/com/github/bottomlessarchive/loa/downloader/service/source/configuration/DownloaderFolderSourceConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.downloader.service.source.configuration; 2 | 3 | import jakarta.validation.constraints.NotNull; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | @Validated 8 | @ConfigurationProperties("loa.downloader.source.folder") 9 | public record DownloaderFolderSourceConfigurationProperties( 10 | 11 | String location, 12 | boolean shouldRemove, 13 | 14 | @NotNull 15 | String sourceName 16 | ) { 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- 1 | # Downloader properties 2 | loa.downloader.staging-directory=C:/loa/stage 3 | -------------------------------------------------------------------------------- /loa-application/loa-downloader-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-generator-application/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'org.springframework.boot' 2 | 3 | bootJar { 4 | manifest { 5 | attributes( 6 | 'Implementation-Title': 'Library of Alexandria - Generator Application', 7 | 'Implementation-Version': archiveVersion 8 | ) 9 | } 10 | } 11 | 12 | bootBuildImage { 13 | imageName = "bottomlessarchive/loa-generator-application:${rootProject.version.toString()}" 14 | publish = true 15 | docker { 16 | publishRegistry { 17 | username = dockerUser 18 | password = dockerPassword 19 | email = dockerEmail 20 | url = "https://index.docker.io/v2/" 21 | } 22 | } 23 | } 24 | 25 | dependencies { 26 | implementation project(':loa-library:loa-source-file-service') 27 | implementation project(':loa-library:loa-queue-artemis-service') 28 | implementation project(':loa-library:loa-conductor-client-service') 29 | implementation project(':loa-library:loa-io-service') 30 | } 31 | -------------------------------------------------------------------------------- /loa-application/loa-generator-application/src/main/java/com/github/bottomlessarchive/loa/generator/LibraryGeneratorApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.generator; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 6 | import org.springframework.scheduling.annotation.EnableScheduling; 7 | 8 | /** 9 | * The runner class of the generator application. 10 | */ 11 | @EnableScheduling 12 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 13 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 14 | public class LibraryGeneratorApplication { 15 | 16 | public static void main(final String[] args) { 17 | SpringApplication.run(LibraryGeneratorApplication.class, args); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-generator-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- 1 | # Source 2 | loa.source.file.location=C:/loa/source.txt 3 | -------------------------------------------------------------------------------- /loa-application/loa-generator-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # Server 2 | spring.main.web-application-type=none 3 | 4 | # Conductor 5 | loa.conductor.host=localhost 6 | loa.conductor.port=8092 7 | loa.conductor.application-type=GENERATOR_APPLICATION 8 | loa.conductor.application-port=0 9 | 10 | # Source 11 | loa.source.name=unknown 12 | loa.source.type=file 13 | loa.source.file.location= 14 | loa.source.file.encoding=none 15 | loa.source.file.skip-lines=0 16 | -------------------------------------------------------------------------------- /loa-application/loa-generator-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-indexer-application/src/main/java/com/github/bottomlessarchive/loa/indexer/LibraryIndexerApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration; 6 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 7 | import org.springframework.scheduling.annotation.EnableScheduling; 8 | 9 | @EnableScheduling 10 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa", exclude = ElasticsearchRestClientAutoConfiguration.class) 11 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 12 | public class LibraryIndexerApplication { 13 | 14 | public static void main(final String[] args) { 15 | SpringApplication.run(LibraryIndexerApplication.class, args); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-indexer-application/src/main/java/com/github/bottomlessarchive/loa/indexer/configuration/IndexerConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.configuration; 2 | 3 | import jakarta.validation.constraints.Min; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | @Validated 8 | @ConfigurationProperties("loa.indexer") 9 | public record IndexerConfigurationProperties( 10 | 11 | @Min(1) 12 | int batchSize, 13 | 14 | @Min(1) 15 | int parallelism 16 | ) { 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-indexer-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- 1 | loa.indexer.concurrent-indexer-threads=32 2 | -------------------------------------------------------------------------------- /loa-application/loa-indexer-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-queue-application/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'org.springframework.boot' 2 | 3 | bootJar { 4 | manifest { 5 | attributes( 6 | 'Implementation-Title': 'Library of Alexandria - Queue Application', 7 | 'Implementation-Version': archiveVersion 8 | ) 9 | } 10 | } 11 | 12 | bootBuildImage { 13 | imageName = "bottomlessarchive/loa-queue-application:${rootProject.version.toString()}" 14 | publish = true 15 | docker { 16 | publishRegistry { 17 | username = dockerUser 18 | password = dockerPassword 19 | email = dockerEmail 20 | url = "https://index.docker.io/v2/" 21 | } 22 | } 23 | } 24 | 25 | dependencies { 26 | implementation project(':loa-library:loa-queue-artemis-service') 27 | implementation project(':loa-library:loa-conductor-client-service') 28 | 29 | implementation group: 'org.apache.activemq', name: 'artemis-server' 30 | implementation group: 'org.codehaus.janino', name: 'janino', version: '3.1.12' 31 | } 32 | -------------------------------------------------------------------------------- /loa-application/loa-queue-application/src/main/java/com/github/bottomlessarchive/loa/queue/LibraryQueueApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 6 | 7 | /** 8 | * An application that is an intermediate between the Generator and the Downloader Application. 9 | */ 10 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 11 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 12 | public class LibraryQueueApplication { 13 | 14 | public static void main(final String[] args) { 15 | SpringApplication.run(LibraryQueueApplication.class, args); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-queue-application/src/main/java/com/github/bottomlessarchive/loa/queue/service/ArtemisQueueInitializer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.service; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import org.apache.activemq.artemis.core.server.embedded.EmbeddedActiveMQ; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.stereotype.Component; 7 | 8 | @Component 9 | @RequiredArgsConstructor 10 | public class ArtemisQueueInitializer implements CommandLineRunner { 11 | 12 | private final EmbeddedActiveMQ embeddedActiveMQ; 13 | 14 | @Override 15 | public void run(final String... args) throws Exception { 16 | embeddedActiveMQ.start(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-queue-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- 1 | # Queue 2 | loa.queue.data-directory=C:\\loa\\artemis 3 | -------------------------------------------------------------------------------- /loa-application/loa-queue-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | loa.queue.port=61616 2 | 3 | # Artemis 4 | spring.artemis.embedded.enabled=false 5 | 6 | # Logging 7 | logging.level.org.apache.activemq.audit.base=WARN 8 | logging.level.org.apache.activemq.audit.message=WARN 9 | logging.level.org.apache.activemq.artemis.core.server=WARN 10 | 11 | # Conductor 12 | loa.conductor.host=localhost 13 | loa.conductor.port=8092 14 | loa.conductor.application-type=QUEUE_APPLICATION 15 | loa.conductor.application-port=61616 16 | 17 | spring.main.web-application-type=none 18 | -------------------------------------------------------------------------------- /loa-application/loa-queue-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-staging-application/src/main/java/com/github/bottomlessarchive/loa/stage/LibraryStagingApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.stage; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 6 | 7 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 8 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 9 | public class LibraryStagingApplication { 10 | 11 | public static void main(final String[] args) { 12 | SpringApplication.run(LibraryStagingApplication.class, args); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-staging-application/src/main/java/com/github/bottomlessarchive/loa/stage/configuration/StagingConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.stage.configuration; 2 | 3 | import jakarta.validation.constraints.NotNull; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | import java.nio.file.Path; 8 | 9 | /** 10 | * Contains the configuration properties for the stage feature. 11 | * 12 | * @param location the location where the staged documents should be saved to 13 | */ 14 | @Validated 15 | @ConfigurationProperties("loa.staging") 16 | public record StagingConfigurationProperties( 17 | 18 | @NotNull 19 | Path location 20 | ) { 21 | } 22 | -------------------------------------------------------------------------------- /loa-application/loa-staging-application/src/main/java/com/github/bottomlessarchive/loa/stage/service/StagingFreeSpaceInstancePropertyExtensionProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.stage.service; 2 | 3 | import com.github.bottomlessarchive.loa.conductor.service.client.extension.InstancePropertyExtensionProvider; 4 | import com.github.bottomlessarchive.loa.conductor.service.client.extension.domain.InstanceExtensionContext; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.SneakyThrows; 7 | import org.springframework.stereotype.Component; 8 | 9 | @Component 10 | @RequiredArgsConstructor 11 | public class StagingFreeSpaceInstancePropertyExtensionProvider implements InstancePropertyExtensionProvider { 12 | 13 | private final StageLocationFactory stageLocationFactory; 14 | 15 | @Override 16 | @SneakyThrows 17 | public void extendInstanceWithProperty(final InstanceExtensionContext instanceExtensionContext) { 18 | instanceExtensionContext.setProperty("freeSpace", stageLocationFactory.getAvailableSpace()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-staging-application/src/main/java/com/github/bottomlessarchive/loa/stage/service/location/StageAccessException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.stage.service.location; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | /** 6 | * Thrown when an error happens while accessing content on the staging area. 7 | */ 8 | @StandardException 9 | public class StageAccessException extends RuntimeException { 10 | } 11 | -------------------------------------------------------------------------------- /loa-application/loa-staging-application/src/main/java/com/github/bottomlessarchive/loa/stage/service/location/StageLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.stage.service.location; 2 | 3 | import lombok.Builder; 4 | 5 | import java.nio.file.Path; 6 | 7 | @Builder 8 | public record StageLocation( 9 | 10 | Path location 11 | ) { 12 | } 13 | -------------------------------------------------------------------------------- /loa-application/loa-staging-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=8099 2 | 3 | # Conductor 4 | loa.conductor.host=localhost 5 | loa.conductor.port=8092 6 | loa.conductor.application-type=STAGING_APPLICATION 7 | loa.conductor.application-port=${server.port} 8 | 9 | # Stage 10 | loa.staging.location=${java.io.tmpdir} -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/LibraryVaultApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault; 2 | 3 | import com.github.bottomlessarchive.loa.vault.service.listener.VaultQueueListener; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 7 | 8 | /** 9 | * This application is responsible for storing and retrieving the documents. 10 | *

11 | * The main logic of the archiving is found in the {@link VaultQueueListener}. 12 | */ 13 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa") 14 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 15 | public class LibraryVaultApplication { 16 | 17 | public static void main(final String[] args) { 18 | SpringApplication.run(LibraryVaultApplication.class, args); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/configuration/MongoClientConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.configuration; 2 | 3 | import org.springframework.boot.autoconfigure.mongo.MongoClientSettingsBuilderCustomizer; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | import java.util.concurrent.TimeUnit; 8 | 9 | @Configuration 10 | public class MongoClientConfiguration { 11 | 12 | @Bean 13 | public MongoClientSettingsBuilderCustomizer mongoClientSettingsBuilderCustomizer() { 14 | return clientSettingsBuilder -> clientSettingsBuilder 15 | .applyToConnectionPoolSettings(connectionPoolSettings -> { 16 | connectionPoolSettings.maxSize(100); 17 | connectionPoolSettings.minSize(100); 18 | connectionPoolSettings.maxWaitTime(10, TimeUnit.MINUTES); 19 | }); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/configuration/StageConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.stage.service.StageLocationFactory; 4 | import lombok.RequiredArgsConstructor; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | @RequiredArgsConstructor 10 | public class StageConfiguration { 11 | 12 | private final VaultConfigurationProperties vaultConfigurationProperties; 13 | 14 | @Bean 15 | public StageLocationFactory stageLocationFactory() { 16 | return new StageLocationFactory(vaultConfigurationProperties.stagingDirectory()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/view/domain/InvalidRequestException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.view.domain; 2 | 3 | import org.springframework.http.HttpStatus; 4 | import org.springframework.web.server.ResponseStatusException; 5 | 6 | public class InvalidRequestException extends ResponseStatusException { 7 | 8 | public InvalidRequestException(final String message) { 9 | super(HttpStatus.BAD_REQUEST, message); 10 | } 11 | 12 | public InvalidRequestException(final String message, final Throwable throwable) { 13 | super(HttpStatus.BAD_REQUEST, message, throwable); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/view/request/domain/RecompressDocumentRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.view.request.domain; 2 | 3 | import com.github.bottomlessarchive.loa.compression.domain.DocumentCompression; 4 | import lombok.Builder; 5 | import lombok.Value; 6 | import lombok.extern.jackson.Jacksonized; 7 | 8 | @Value 9 | @Builder 10 | @Jacksonized 11 | public class RecompressDocumentRequest { 12 | 13 | DocumentCompression compression; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/view/response/domain/DocumentExistsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.view.response.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Value 8 | @Builder 9 | @Jacksonized 10 | public class DocumentExistsResponse { 11 | 12 | boolean exists; 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/java/com/github/bottomlessarchive/loa/vault/view/response/domain/QueryDocumentResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.view.response.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Value 8 | @Builder 9 | @Jacksonized 10 | public class QueryDocumentResponse { 11 | 12 | byte[] payload; 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- 1 | # Vault 2 | loa.vault.staging-directory=C:\\loa\\stage\\ 3 | loa.vault.location.file.path=C:\\loa\\vault\\ 4 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # Server 2 | server.port=8093 3 | server.error.include-message=always 4 | 5 | spring.rsocket.server.port=${server.port} 6 | spring.rsocket.server.fragment-size=8MB 7 | 8 | # Conductor 9 | loa.conductor.host=localhost 10 | loa.conductor.port=8092 11 | loa.conductor.application-type=VAULT_APPLICATION 12 | loa.conductor.application-port=${spring.rsocket.server.port} 13 | 14 | # Vault 15 | loa.vault.name=default 16 | loa.vault.archiving=true 17 | loa.vault.parallelism=20 18 | loa.vault.modification-enabled=true 19 | loa.vault.version-number=7 20 | loa.vault.location.type=file 21 | loa.vault.location.file.path= 22 | loa.vault.location.s3.bucket-name=document-archive 23 | loa.vault.staging-directory=${java.io.tmpdir} 24 | 25 | # Compression 26 | loa.compression.algorithm=NONE 27 | -------------------------------------------------------------------------------- /loa-application/loa-vault-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/LibraryWebApplication.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration; 6 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan; 7 | import org.springframework.scheduling.annotation.EnableScheduling; 8 | 9 | @EnableScheduling 10 | @SpringBootApplication(scanBasePackages = "com.github.bottomlessarchive.loa", exclude = ElasticsearchRestClientAutoConfiguration.class) 11 | @ConfigurationPropertiesScan(basePackages = "com.github.bottomlessarchive.loa") 12 | public class LibraryWebApplication { 13 | 14 | public static void main(final String[] args) { 15 | SpringApplication.run(LibraryWebApplication.class, args); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/AdministratorApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class AdministratorApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | private final String command; 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/DashboardApplicationsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.List; 7 | 8 | @Getter 9 | @Builder 10 | public class DashboardApplicationsResponse { 11 | 12 | private final List administrators; 13 | private final List downloaders; 14 | private final List generators; 15 | private final List indexers; 16 | private final List queues; 17 | private final List vaults; 18 | private final List stagings; 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/DashboardDocumentStatisticsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import com.github.bottomlessarchive.loa.document.service.domain.DocumentStatus; 4 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 5 | import lombok.Builder; 6 | import lombok.Getter; 7 | 8 | import java.util.Map; 9 | 10 | @Getter 11 | @Builder 12 | public class DashboardDocumentStatisticsResponse { 13 | 14 | private final long documentCount; 15 | private final Map documentCountByType; 16 | private final Map documentCountByStatus; 17 | } 18 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/DownloaderApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class DownloaderApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | private final int parallelism; 13 | } 14 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/GeneratorApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class GeneratorApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | } 13 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/IndexerApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class IndexerApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | private final int parallelism; 13 | private final int batchSize; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/QueueApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class QueueApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | 13 | private final long documentLocationQueueCount; 14 | private final long documentArchivingQueueCount; 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/StagingApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class StagingApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | 13 | private final long freeSpace; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/dashboard/response/VaultApplicationInstance.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.dashboard.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class VaultApplicationInstance { 9 | 10 | private final String host; 11 | private final int port; 12 | 13 | private final String name; 14 | private final long freeSpace; 15 | } 16 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/document/response/DocumentDebugResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.document.response; 2 | 3 | import com.github.bottomlessarchive.loa.compression.domain.DocumentCompression; 4 | import com.github.bottomlessarchive.loa.document.service.domain.DocumentStatus; 5 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 6 | import lombok.Builder; 7 | import lombok.Value; 8 | 9 | import java.time.Instant; 10 | import java.util.Set; 11 | import java.util.UUID; 12 | 13 | @Value 14 | @Builder 15 | public class DocumentDebugResponse { 16 | 17 | UUID id; 18 | String vault; 19 | DocumentType type; 20 | DocumentStatus status; 21 | DocumentCompression compression; 22 | String checksum; 23 | long fileSize; 24 | Instant downloadDate; 25 | int downloaderVersion; 26 | boolean isInIndex; 27 | boolean isInVault; 28 | Set sourceLocations; 29 | } 30 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/document/response/DocumentSearchResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.document.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.List; 7 | 8 | @Getter 9 | @Builder 10 | public class DocumentSearchResponse { 11 | 12 | private final long totalHitCount; 13 | private final List searchHits; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/document/response/DocumentStatisticsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.document.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class DocumentStatisticsResponse { 9 | 10 | private final long documentCount; 11 | private final long indexedDocumentCount; 12 | } 13 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/document/response/SearchDocumentEntityResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.document.response; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.time.Instant; 8 | import java.util.List; 9 | import java.util.Set; 10 | 11 | @Getter 12 | @Builder 13 | public class SearchDocumentEntityResponse { 14 | 15 | private final String id; 16 | private final String title; 17 | private final String author; 18 | private final List description; 19 | private final String language; 20 | private final int pageCount; 21 | private final DocumentType type; 22 | private final String vault; 23 | private final String source; 24 | private final Instant downloadDate; 25 | private final Set sourceLocations; 26 | } 27 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/info/controller/InfoController.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.info.controller; 2 | 3 | import com.github.bottomlessarchive.loa.user.configuration.UserConfigurationProperties; 4 | import com.github.bottomlessarchive.loa.web.view.info.response.InfoResponse; 5 | import lombok.RequiredArgsConstructor; 6 | import org.springframework.web.bind.annotation.GetMapping; 7 | import org.springframework.web.bind.annotation.RestController; 8 | 9 | @RestController 10 | @RequiredArgsConstructor 11 | public class InfoController { 12 | 13 | private final UserConfigurationProperties userConfigurationProperties; 14 | 15 | @GetMapping("/info") 16 | public InfoResponse info() { 17 | return InfoResponse.builder() 18 | .usersEnabled(userConfigurationProperties.enabled()) 19 | .build(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/info/response/InfoResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.info.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class InfoResponse { 9 | 10 | private final boolean usersEnabled; 11 | } 12 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/location/response/DocumentLocationDebugResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.location.response; 2 | 3 | import com.github.bottomlessarchive.loa.location.domain.DocumentLocationResultType; 4 | import lombok.Builder; 5 | import lombok.Value; 6 | 7 | @Value 8 | @Builder 9 | public class DocumentLocationDebugResponse { 10 | 11 | String id; 12 | String url; 13 | String source; 14 | int downloaderVersion; 15 | DocumentLocationResultType downloadResultCode; 16 | } 17 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/location/service/DocumentLocationDebugResponseFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.location.service; 2 | 3 | import com.github.bottomlessarchive.loa.location.service.factory.domain.DocumentLocation; 4 | import com.github.bottomlessarchive.loa.web.view.location.response.DocumentLocationDebugResponse; 5 | import org.springframework.stereotype.Service; 6 | 7 | @Service 8 | public class DocumentLocationDebugResponseFactory { 9 | 10 | public DocumentLocationDebugResponse newDocumentLocationDebugResponse(final DocumentLocation documentLocation) { 11 | return DocumentLocationDebugResponse.builder() 12 | .id(documentLocation.id()) 13 | .url(documentLocation.url()) 14 | .source(documentLocation.source()) 15 | .downloaderVersion(documentLocation.downloaderVersion()) 16 | .downloadResultCode(documentLocation.downloadResultCode()) 17 | .build(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/user/request/LoginRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.user.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Getter 8 | @Builder 9 | @Jacksonized 10 | public class LoginRequest { 11 | 12 | private final String username; 13 | private final String password; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/user/response/InfoResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.user.response; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | @Getter 8 | @Builder 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | public class InfoResponse { 11 | 12 | private final String name; 13 | private final boolean isLoggedIn; 14 | } 15 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/user/response/LoginResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.user.response; 2 | 3 | import com.github.bottomlessarchive.loa.web.view.user.response.domain.LoginResult; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | @Getter 8 | @Builder 9 | public class LoginResponse { 10 | 11 | private final LoginResult result; 12 | } 13 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/java/com/github/bottomlessarchive/loa/web/view/user/response/domain/LoginResult.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.web.view.user.response.domain; 2 | 3 | public enum LoginResult { 4 | 5 | SUCCESSFUL, 6 | INVALID_CREDENTIALS 7 | } 8 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/resources/application-local.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-application/loa-web-application/src/main/resources/application-local.properties -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # Server 2 | server.port=80 3 | server.error.include-message=always 4 | 5 | # Conductor 6 | loa.conductor.host=localhost 7 | loa.conductor.port=8092 8 | loa.conductor.application-type=WEB_APPLICATION 9 | loa.conductor.application-port=${server.port} 10 | 11 | # Compression 12 | loa.compression.algorithm=none 13 | 14 | # Users 15 | loa.user.enabled=false 16 | -------------------------------------------------------------------------------- /loa-application/loa-web-application/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ __ _ _ _ _ 2 | | | (_) |__ _ _ __ _ _ _ _ _ ___ / _| /_\ | |_____ ____ _ _ _ __| |_ _(_)__ _ 3 | | |__| | '_ \ '_/ _` | '_| || | / _ \ _| / _ \| / -_) \ / _` | ' \/ _` | '_| / _` | 4 | |____|_|_.__/_| \__,_|_| \_, | \___/_| /_/ \_\_\___/_\_\__,_|_||_\__,_|_| |_\__,_| 5 | |__/ ${application.formatted-version} 6 | "If you have a garden and a library, you have everything you need." - Marcus Tullius Cicero 7 | -------------------------------------------------------------------------------- /loa-frontend/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.ts] 12 | quote_type = single 13 | 14 | [*.md] 15 | max_line_length = off 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /loa-frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # compiled output 4 | /dist 5 | /tmp 6 | /out-tsc 7 | # Only exists if Bazel was run 8 | /bazel-out 9 | .angular 10 | 11 | # dependencies 12 | /node_modules 13 | 14 | # profiling files 15 | chrome-profiler-events*.json 16 | speed-measure-plugin*.json 17 | 18 | # IDEs and editors 19 | /.idea 20 | .project 21 | .classpath 22 | .c9/ 23 | *.launch 24 | .settings/ 25 | *.sublime-workspace 26 | 27 | # IDE - VSCode 28 | .vscode/* 29 | !.vscode/settings.json 30 | !.vscode/tasks.json 31 | !.vscode/launch.json 32 | !.vscode/extensions.json 33 | .history/* 34 | 35 | # misc 36 | /.sass-cache 37 | /connect.lock 38 | /coverage 39 | /libpeerconnection.log 40 | npm-debug.log 41 | yarn-error.log 42 | testem.log 43 | /typings 44 | 45 | # System Files 46 | .DS_Store 47 | Thumbs.db 48 | -------------------------------------------------------------------------------- /loa-frontend/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'com.github.node-gradle.node' version '7.0.1' 4 | } 5 | 6 | node { 7 | version = '20.10.0' 8 | npmVersion = '8.10.0' 9 | download = true 10 | } 11 | 12 | jar.dependsOn 'npm_run_build' 13 | 14 | jar { 15 | from 'dist/loa-frontend' into 'static' 16 | } 17 | -------------------------------------------------------------------------------- /loa-frontend/e2e/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | // Protractor configuration file, see link for more information 3 | // https://github.com/angular/protractor/blob/master/lib/config.ts 4 | 5 | const { SpecReporter, StacktraceOption } = require('jasmine-spec-reporter'); 6 | 7 | /** 8 | * @type { import("protractor").Config } 9 | */ 10 | exports.config = { 11 | allScriptsTimeout: 11000, 12 | specs: [ 13 | './src/**/*.e2e-spec.ts' 14 | ], 15 | capabilities: { 16 | browserName: 'chrome' 17 | }, 18 | directConnect: true, 19 | SELENIUM_PROMISE_MANAGER: false, 20 | baseUrl: 'http://localhost:4200/', 21 | framework: 'jasmine', 22 | jasmineNodeOpts: { 23 | showColors: true, 24 | defaultTimeoutInterval: 30000, 25 | print: function() {} 26 | }, 27 | onPrepare() { 28 | require('ts-node').register({ 29 | project: require('path').join(__dirname, './tsconfig.json') 30 | }); 31 | jasmine.getEnv().addReporter(new SpecReporter({ 32 | spec: { 33 | displayStacktrace: StacktraceOption.PRETTY 34 | } 35 | })); 36 | } 37 | }; -------------------------------------------------------------------------------- /loa-frontend/e2e/src/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { browser, logging } from 'protractor'; 2 | import { AppPage } from './app.po'; 3 | 4 | describe('workspace-project App', () => { 5 | let page: AppPage; 6 | 7 | beforeEach(() => { 8 | page = new AppPage(); 9 | }); 10 | 11 | it('should display welcome message', async () => { 12 | await page.navigateTo(); 13 | expect(await page.getTitleText()).toEqual('loa-frontend app is running!'); 14 | }); 15 | 16 | afterEach(async () => { 17 | // Assert that there are no errors emitted from the browser 18 | const logs = await browser.manage().logs().get(logging.Type.BROWSER); 19 | expect(logs).not.toContain(jasmine.objectContaining({ 20 | level: logging.Level.SEVERE, 21 | } as logging.Entry)); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /loa-frontend/e2e/src/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | async navigateTo(): Promise { 5 | return browser.get(browser.baseUrl); 6 | } 7 | 8 | async getTitleText(): Promise { 9 | return element(by.css('app-root .content span')).getText(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /loa-frontend/e2e/tsconfig.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "../tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "../out-tsc/e2e", 6 | "module": "commonjs", 7 | "target": "es2018", 8 | "types": [ 9 | "jasmine", 10 | "node" 11 | ] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/administrator-application-instance.ts: -------------------------------------------------------------------------------- 1 | export class AdministratorApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | command: string; 6 | } 7 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/application-info-entity.ts: -------------------------------------------------------------------------------- 1 | import {AdministratorApplicationInstance} from "./administrator-application-instance"; 2 | import {DownloaderApplicationInstance} from "./downloader-application-instance"; 3 | import {GeneratorApplicationInstance} from "./generator-application-instance"; 4 | import {IndexerApplicationInstance} from "./indexer-application-instance"; 5 | import {QueueApplicationInstance} from "./queue-application-instance"; 6 | import {VaultApplicationInstance} from "./vault-application-instance"; 7 | import {StagingApplicationInstance} from "./vault-pplication-instance"; 8 | 9 | export class ApplicationInfoEntity { 10 | 11 | administrators: AdministratorApplicationInstance[]; 12 | downloaders: DownloaderApplicationInstance[]; 13 | generators: GeneratorApplicationInstance[]; 14 | indexers: IndexerApplicationInstance[]; 15 | queues: QueueApplicationInstance[]; 16 | vaults: VaultApplicationInstance[]; 17 | stagings: StagingApplicationInstance[]; 18 | } 19 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/downloader-application-instance.ts: -------------------------------------------------------------------------------- 1 | export class DownloaderApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | parallelism: number; 6 | } 7 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/generator-application-instance.ts: -------------------------------------------------------------------------------- 1 | export class GeneratorApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | } 6 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/indexer-application-instance.ts: -------------------------------------------------------------------------------- 1 | export class IndexerApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | parallelism: number; 6 | batchSize: number; 7 | } 8 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/queue-application-instance.ts: -------------------------------------------------------------------------------- 1 | export class QueueApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | 6 | documentLocationQueueCount: number; 7 | documentArchivingQueueCount: number; 8 | } 9 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/vault-application-instance.ts: -------------------------------------------------------------------------------- 1 | export class VaultApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | 6 | name: string; 7 | freeSpace: number; 8 | } 9 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/domain/vault-pplication-instance.ts: -------------------------------------------------------------------------------- 1 | export class StagingApplicationInstance { 2 | 3 | host: string; 4 | port: number; 5 | 6 | freeSpace: number; 7 | } 8 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/application/service/application.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from "@angular/core"; 2 | import {HttpClient} from "@angular/common/http"; 3 | import {Observable} from "rxjs"; 4 | import {ApplicationInfoEntity} from "../domain/application-info-entity"; 5 | 6 | @Injectable({ 7 | providedIn: 'root' 8 | }) 9 | export class ApplicationService { 10 | 11 | constructor(private http: HttpClient) { 12 | } 13 | 14 | getApplications(): Observable { 15 | return this.http.get('/dashboard/applications'); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/debug/service/debug.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {HttpClient} from '@angular/common/http'; 3 | import {Observable} from 'rxjs'; 4 | import {DebugDocument} from './domain/debug-document'; 5 | import {DebugLocation} from './domain/debug-location'; 6 | 7 | @Injectable({ 8 | providedIn: 'root' 9 | }) 10 | export class DebugService { 11 | 12 | constructor(private http: HttpClient) { 13 | } 14 | 15 | queryDocument(documentId: string): Observable { 16 | return this.http.get('/document/' + documentId + '/debug'); 17 | } 18 | 19 | queryLocation(locationId: string): Observable { 20 | return this.http.get('/location/' + locationId + '/debug'); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/debug/service/domain/debug-document.ts: -------------------------------------------------------------------------------- 1 | export class DebugDocument { 2 | 3 | constructor( 4 | public id: string, 5 | public vault: string, 6 | public type: string, 7 | public status: string, 8 | public compression: string, 9 | public checksum: string, 10 | public fileSize: number, 11 | public downloadDate: string, 12 | public downloaderVersion: number, 13 | public inVault: boolean, 14 | public inIndex: boolean, 15 | public sourceLocations: string[] 16 | ) { 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/debug/service/domain/debug-location.ts: -------------------------------------------------------------------------------- 1 | export class DebugLocation { 2 | 3 | constructor( 4 | public id: string, 5 | public url: string, 6 | public source: string, 7 | public downloaderVersion: number, 8 | public downloadResultCode: string 9 | ) { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/info/service/domain/site-info.ts: -------------------------------------------------------------------------------- 1 | export class SiteInfo { 2 | 3 | constructor( 4 | public usersEnabled: boolean 5 | ) { 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/info/service/site-info.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from "@angular/core"; 2 | import {HttpClient} from "@angular/common/http"; 3 | import {Observable} from "rxjs"; 4 | import {SiteInfo} from "./domain/site-info"; 5 | 6 | @Injectable({ 7 | providedIn: 'root' 8 | }) 9 | export class SiteInfoService { 10 | 11 | constructor(private http: HttpClient) { 12 | } 13 | 14 | querySiteInfo(): Observable { 15 | console.log("Querying site info!") 16 | 17 | return this.http.get('/info'); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/search/service/domain/search-hit.ts: -------------------------------------------------------------------------------- 1 | export class SearchHit { 2 | 3 | constructor( 4 | public id: string, 5 | public title: string, 6 | public author: string, 7 | public description: string[], 8 | public type: string, 9 | public language: string, 10 | public pageCount: number, 11 | public vault: string, 12 | public source: string, 13 | public downloadDate: string, 14 | public sourceLocations: string[] 15 | ) { 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/search/service/domain/search-result.ts: -------------------------------------------------------------------------------- 1 | import {SearchHit} from "./search-hit"; 2 | 3 | export class SearchResult { 4 | 5 | constructor( 6 | public searchHits: SearchHit[], 7 | public totalHitCount: number 8 | ) { 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/statistics/service/statistics-service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {HttpClient} from '@angular/common/http'; 3 | import {Observable} from "rxjs"; 4 | import {ApplicationInfoEntity} from "../../application/domain/application-info-entity"; 5 | 6 | @Injectable({ 7 | providedIn: 'root' 8 | }) 9 | export class StatisticsService { 10 | 11 | constructor(private http: HttpClient) { 12 | } 13 | 14 | getStatistics(): Observable { 15 | return this.http.get('/dashboard/statistics'); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/user/service/domain/login-response.ts: -------------------------------------------------------------------------------- 1 | export class LoginResponse { 2 | 3 | constructor( 4 | public result: string 5 | ) { 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/user/service/domain/user-info.ts: -------------------------------------------------------------------------------- 1 | export class UserInfo { 2 | 3 | constructor( 4 | public name: string, 5 | public loggedIn: boolean 6 | ) { 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-frontend/src/app/shared/user/service/user-service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from "@angular/core"; 2 | import {HttpClient} from "@angular/common/http"; 3 | import {Observable, ReplaySubject} from "rxjs"; 4 | import {UserInfo} from "./domain/user-info"; 5 | import {LoginResponse} from "./domain/login-response"; 6 | 7 | @Injectable({ 8 | providedIn: 'root' 9 | }) 10 | export class UserService { 11 | 12 | public userInfo: ReplaySubject = new ReplaySubject(); 13 | 14 | constructor(private http: HttpClient) { 15 | } 16 | 17 | updateUserInfo(): void { 18 | console.log("Querying site info!") 19 | 20 | this.http.get('/user/info') 21 | .subscribe(result => { 22 | this.userInfo.next(result); 23 | }); 24 | } 25 | 26 | login(name: string, password: string): Observable { 27 | return this.http.post('/user/login', { 28 | username: name, 29 | password: password 30 | }); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/app/app.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/app/app.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/dashboard-applications/dashboard-applications.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/dashboard-applications/dashboard-applications.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/dashboard-applications/dashboard-applications.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnInit} from '@angular/core'; 2 | import {ActivatedRoute} from '@angular/router'; 3 | import {ApplicationInfoEntity} from "../../shared/application/domain/application-info-entity"; 4 | 5 | @Component({ 6 | selector: 'app-dashboard', 7 | templateUrl: './dashboard-applications.component.html', 8 | styleUrls: ['./dashboard-applications.component.scss'] 9 | }) 10 | export class DashboardApplicationsComponent implements OnInit { 11 | 12 | applications: ApplicationInfoEntity; 13 | 14 | constructor(private route: ActivatedRoute) { 15 | } 16 | 17 | ngOnInit(): void { 18 | this.applications = this.route.snapshot.data.applications; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/dashboard-statistics/dashboard-statistics.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/dashboard-statistics/dashboard-statistics.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/dashboard-statistics/dashboard-statistics.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnInit} from '@angular/core'; 2 | import {ActivatedRoute} from '@angular/router'; 3 | 4 | @Component({ 5 | selector: 'app-dashboard', 6 | templateUrl: './dashboard-statistics.component.html', 7 | styleUrls: ['./dashboard-statistics.component.scss'] 8 | }) 9 | export class DashboardStatisticsComponent implements OnInit { 10 | 11 | statistics: any; 12 | 13 | constructor(private route: ActivatedRoute) { 14 | } 15 | 16 | ngOnInit(): void { 17 | this.statistics = this.route.snapshot.data.statistics; 18 | } 19 | 20 | keepOrder(a: any, b: any): any { 21 | return a; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/debug-document/debug-document.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/debug-document/debug-document.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/debug-location/debug-location.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/debug-location/debug-location.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/home/home.component.html: -------------------------------------------------------------------------------- 1 |

2 |
3 |
4 |
5 | 9 |
10 |
11 | 15 |
16 |
17 | 21 |
22 |
23 |
24 |
25 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/home/home.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/home/home.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/home/home.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-home', 5 | templateUrl: './home.component.html', 6 | styleUrls: ['./home.component.scss'] 7 | }) 8 | export class HomeComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit(): void { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/login/login.component.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
6 |
7 | Invalid credentials! 8 |
9 |
10 | 11 | 12 |
13 |
14 | 15 | 16 |
17 |
18 | 19 |
20 |
21 |
22 |
23 |
24 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/login/login.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/login/login.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/app/view/login/login.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {UserService} from "../../shared/user/service/user-service"; 3 | import {Router} from "@angular/router"; 4 | 5 | @Component({ 6 | selector: 'app-login', 7 | templateUrl: './login.component.html', 8 | styleUrls: ['./login.component.scss'] 9 | }) 10 | export class LoginComponent { 11 | 12 | username: string; 13 | password: string; 14 | 15 | invalidCredentials: boolean = false; 16 | 17 | constructor(private userService: UserService, private router: Router) { 18 | } 19 | 20 | login(): void { 21 | this.userService.login(this.username, this.password) 22 | .subscribe(loginResponse => { 23 | if (loginResponse.result === 'SUCCESSFUL') { 24 | this.userService.updateUserInfo(); 25 | this.router.navigateByUrl('/home'); 26 | } else { 27 | this.invalidCredentials = true; 28 | } 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /loa-frontend/src/app/view/search/search.component.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/app/view/search/search.component.scss -------------------------------------------------------------------------------- /loa-frontend/src/assets/background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/assets/background.png -------------------------------------------------------------------------------- /loa-frontend/src/assets/sphinx.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/assets/sphinx.ico -------------------------------------------------------------------------------- /loa-frontend/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true 3 | }; 4 | -------------------------------------------------------------------------------- /loa-frontend/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false 7 | }; 8 | 9 | /* 10 | * For easier debugging in development mode, you can import the following file 11 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 12 | * 13 | * This import should be commented out in production mode because it will have a negative impact 14 | * on performance if an error is thrown. 15 | */ 16 | // import 'zone.js/plugins/zone-error'; // Included with Angular CLI. 17 | -------------------------------------------------------------------------------- /loa-frontend/src/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-frontend/src/favicon.ico -------------------------------------------------------------------------------- /loa-frontend/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Bottomless Archive Project - Library of Alexandria 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 |
19 | 20 | 21 | -------------------------------------------------------------------------------- /loa-frontend/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.error(err)); 13 | -------------------------------------------------------------------------------- /loa-frontend/src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/testing'; 4 | import { getTestBed } from '@angular/core/testing'; 5 | import { 6 | BrowserDynamicTestingModule, 7 | platformBrowserDynamicTesting 8 | } from '@angular/platform-browser-dynamic/testing'; 9 | 10 | // First, initialize the Angular testing environment. 11 | getTestBed().initTestEnvironment( 12 | BrowserDynamicTestingModule, 13 | platformBrowserDynamicTesting() 14 | ); 15 | -------------------------------------------------------------------------------- /loa-frontend/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/app", 6 | "types": [] 7 | }, 8 | "files": [ 9 | "src/main.ts", 10 | "src/polyfills.ts" 11 | ], 12 | "include": [ 13 | "src/**/*.d.ts" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /loa-frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "compileOnSave": false, 4 | "compilerOptions": { 5 | "baseUrl": "./", 6 | "outDir": "./dist/out-tsc", 7 | "forceConsistentCasingInFileNames": true, 8 | "strictPropertyInitialization": false, 9 | "strict": true, 10 | "noImplicitReturns": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "sourceMap": true, 13 | "declaration": false, 14 | "downlevelIteration": true, 15 | "experimentalDecorators": true, 16 | "moduleResolution": "node", 17 | "importHelpers": true, 18 | "target": "ES2022", 19 | "module": "es2020", 20 | "lib": [ 21 | "es2018", 22 | "dom" 23 | ], 24 | "useDefineForClassFields": false 25 | }, 26 | "angularCompilerOptions": { 27 | "enableI18nLegacyMessageIdFormat": false, 28 | "strictInjectionParameters": true, 29 | "strictInputAccessModifiers": true, 30 | "strictTemplates": false, 31 | "fullTemplateTypeCheck": false 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /loa-frontend/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/spec", 6 | "types": [ 7 | "jasmine" 8 | ] 9 | }, 10 | "files": [ 11 | "src/test.ts", 12 | "src/polyfills.ts" 13 | ], 14 | "include": [ 15 | "src/**/*.spec.ts", 16 | "src/**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /loa-library/build.gradle: -------------------------------------------------------------------------------- 1 | subprojects { 2 | apply plugin: 'java-library' 3 | } -------------------------------------------------------------------------------- /loa-library/loa-application-domain/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-library/loa-application-domain/build.gradle -------------------------------------------------------------------------------- /loa-library/loa-application-domain/src/main/java/com/github/bottomlessarchive/loa/application/domain/ApplicationType.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.application.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | @Getter 7 | @RequiredArgsConstructor 8 | public enum ApplicationType { 9 | 10 | ADMINISTRATOR_APPLICATION, 11 | CONDUCTOR_APPLICATION, 12 | DOWNLOADER_APPLICATION, 13 | BEACON_APPLICATION, 14 | GENERATOR_APPLICATION, 15 | INDEXER_APPLICATION, 16 | QUEUE_APPLICATION, 17 | VAULT_APPLICATION, 18 | WEB_APPLICATION, 19 | STAGING_APPLICATION, 20 | DOCUMENT_DATABASE, 21 | DOCUMENT_INDEX 22 | } 23 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-json' 3 | 4 | implementation group: 'org.springframework.retry', name: 'spring-retry', version: '2.0.6' 5 | 6 | implementation group: 'com.squareup.okhttp3', name: 'okhttp' 7 | 8 | implementation project(':loa-library:loa-io-service') 9 | implementation project(':loa-library:loa-downloader-service') 10 | implementation project(':loa-library:loa-document-type-service') 11 | } 12 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/configuration/BeaconClientConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.configuration; 2 | 3 | import okhttp3.OkHttpClient; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | import java.time.Duration; 8 | 9 | @Configuration 10 | public class BeaconClientConfiguration { 11 | 12 | @Bean 13 | public OkHttpClient beaconWebClient() { 14 | return new OkHttpClient.Builder() 15 | .connectTimeout(Duration.ofSeconds(30)) 16 | .callTimeout(Duration.ofMinutes(30)) 17 | .readTimeout(Duration.ofMinutes(30)) 18 | .writeTimeout(Duration.ofMinutes(30)) 19 | .build(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/domain/BeaconClientException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class BeaconClientException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/domain/BeaconDocumentLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | @Getter 8 | @Builder 9 | public class BeaconDocumentLocation { 10 | 11 | private final String id; 12 | private final String location; 13 | private final DocumentType type; 14 | private final String sourceName; 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/domain/BeaconDocumentLocationResult.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.Optional; 8 | import java.util.UUID; 9 | 10 | @Getter 11 | @Builder 12 | public class BeaconDocumentLocationResult { 13 | 14 | private final String id; 15 | private final UUID documentId; 16 | private final String checksum; 17 | private final long size; 18 | private final String resultType; 19 | private final String sourceName; 20 | private final DocumentType type; 21 | 22 | public Optional getDocumentId() { 23 | return Optional.ofNullable(documentId); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/request/DocumentLocationPartialRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.request; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.extern.jackson.Jacksonized; 7 | 8 | @Getter 9 | @Builder 10 | @Jacksonized 11 | public class DocumentLocationPartialRequest { 12 | 13 | private final String id; 14 | private final String location; 15 | private final DocumentType type; 16 | private final String sourceName; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/request/VisitDocumentLocationsRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.List; 8 | 9 | @Getter 10 | @Builder 11 | @Jacksonized 12 | public class VisitDocumentLocationsRequest { 13 | 14 | private final List locations; 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/response/DocumentLocationResultPartialResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.response; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.extern.jackson.Jacksonized; 7 | 8 | @Getter 9 | @Builder 10 | @Jacksonized 11 | public class DocumentLocationResultPartialResponse { 12 | 13 | private final String id; 14 | private final String documentId; 15 | private final String checksum; 16 | private final long size; 17 | private final String resultType; 18 | private final String sourceName; 19 | private final DocumentType type; 20 | } 21 | -------------------------------------------------------------------------------- /loa-library/loa-beacon-client-service/src/main/java/com/github/bottomlessarchive/loa/beacon/service/client/response/VisitDocumentLocationsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.beacon.service.client.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.List; 8 | 9 | @Getter 10 | @Builder 11 | @Jacksonized 12 | public class VisitDocumentLocationsResponse { 13 | 14 | private final List results; 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-checksum-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter' 3 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 4 | 5 | implementation group: 'commons-codec', name: 'commons-codec', version: '1.17.0' 6 | } 7 | -------------------------------------------------------------------------------- /loa-library/loa-checksum-service/src/main/java/com/github/bottomlessarchive/loa/checksum/configuration/ChecksumConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.checksum.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.checksum.domain.ChecksumType; 4 | import jakarta.validation.constraints.NotNull; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.validation.annotation.Validated; 7 | 8 | /** 9 | * Contains the configuration properties for the checksum. 10 | */ 11 | @Validated 12 | @ConfigurationProperties("loa.checksum") 13 | public record ChecksumConfigurationProperties( 14 | 15 | @NotNull 16 | ChecksumType type 17 | ) { 18 | } 19 | -------------------------------------------------------------------------------- /loa-library/loa-checksum-service/src/main/java/com/github/bottomlessarchive/loa/checksum/domain/ChecksumCalculationException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.checksum.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | /** 6 | * This exception is thrown whenever an error happens in the checksum calculation. 7 | */ 8 | @StandardException 9 | public class ChecksumCalculationException extends RuntimeException { 10 | } 11 | -------------------------------------------------------------------------------- /loa-library/loa-checksum-service/src/main/java/com/github/bottomlessarchive/loa/checksum/domain/ChecksumType.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.checksum.domain; 2 | 3 | /** 4 | * The available types for the document checksum. 5 | */ 6 | public enum ChecksumType { 7 | 8 | /** 9 | * @see https://en.wikipedia.org/wiki/SHA-2 10 | */ 11 | SHA256 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-checksum-service/src/main/java/com/github/bottomlessarchive/loa/checksum/service/ChecksumProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.checksum.service; 2 | 3 | import java.io.InputStream; 4 | 5 | /** 6 | * This interface is responsible for providing checksum values for documents in the stage location. 7 | * 8 | * @see https://en.wikipedia.org/wiki/Checksum 9 | */ 10 | public interface ChecksumProvider { 11 | 12 | /** 13 | * Generate a checksum for the provided document. 14 | * 15 | * @param documentContents the contents of the document 16 | * @return the checksum for the document 17 | */ 18 | String checksum(InputStream documentContents); 19 | } 20 | -------------------------------------------------------------------------------- /loa-library/loa-checksum-service/src/test/resources/checksum_test.txt: -------------------------------------------------------------------------------- 1 | asdfghjk -------------------------------------------------------------------------------- /loa-library/loa-compression-service/src/main/java/com/github/bottomlessarchive/loa/compression/configuration/CompressionConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.compression.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.compression.domain.DocumentCompression; 4 | import jakarta.validation.constraints.NotNull; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.validation.annotation.Validated; 7 | 8 | /** 9 | * Contains the configuration properties for the compression. 10 | */ 11 | @Validated 12 | @ConfigurationProperties("loa.compression") 13 | public record CompressionConfigurationProperties( 14 | 15 | @NotNull 16 | DocumentCompression algorithm 17 | ) { 18 | } 19 | -------------------------------------------------------------------------------- /loa-library/loa-compression-service/src/main/java/com/github/bottomlessarchive/loa/compression/domain/CompressionException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.compression.domain; 2 | 3 | public class CompressionException extends RuntimeException { 4 | 5 | public CompressionException(final String message, final Throwable throwable) { 6 | super(message, throwable); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-compression-service/src/main/java/com/github/bottomlessarchive/loa/compression/service/compressor/provider/domain/MissingCompressionServiceException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.compression.service.compressor.provider.domain; 2 | 3 | public class MissingCompressionServiceException extends RuntimeException { 4 | 5 | public MissingCompressionServiceException(final String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-json' 3 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 4 | 5 | implementation group: 'com.squareup.okhttp3', name: 'okhttp' 6 | 7 | api project(':loa-library:loa-conductor-domain') 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/configuration/ConductorClientConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.configuration; 2 | 3 | import okhttp3.OkHttpClient; 4 | import org.springframework.cache.annotation.EnableCaching; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.scheduling.annotation.EnableScheduling; 8 | 9 | import java.time.Duration; 10 | 11 | @Configuration 12 | @EnableCaching 13 | @EnableScheduling 14 | public class ConductorClientConfiguration { 15 | 16 | @Bean 17 | public OkHttpClient conductorWebClient() { 18 | return new OkHttpClient.Builder() 19 | .callTimeout(Duration.ofSeconds(30)) 20 | .connectTimeout(Duration.ofSeconds(30)) 21 | .readTimeout(Duration.ofSeconds(30)) 22 | .writeTimeout(Duration.ofSeconds(30)) 23 | .build(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/domain/ConductorClientException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class ConductorClientException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/extension/InstancePropertyExtensionProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.extension; 2 | 3 | import com.github.bottomlessarchive.loa.conductor.service.client.extension.domain.InstanceExtensionContext; 4 | 5 | public interface InstancePropertyExtensionProvider { 6 | 7 | void extendInstanceWithProperty(InstanceExtensionContext instanceExtensionContext); 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/request/ServiceInstancePropertyRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class ServiceInstancePropertyRequest { 9 | 10 | private final String name; 11 | private final String value; 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/request/ServiceInstanceRefreshRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.List; 8 | 9 | @Value 10 | @Builder 11 | @Jacksonized 12 | public class ServiceInstanceRefreshRequest { 13 | 14 | String location; 15 | int port; 16 | List properties; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/request/ServiceInstanceRegistrationRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.request; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.List; 8 | 9 | @Value 10 | @Builder 11 | @Jacksonized 12 | public class ServiceInstanceRegistrationRequest { 13 | 14 | String location; 15 | int port; 16 | List properties; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/response/ServiceInstancePropertyResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Getter 8 | @Builder 9 | @Jacksonized 10 | public class ServiceInstancePropertyResponse { 11 | 12 | private final String name; 13 | private final String value; 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/response/ServiceInstanceRegistrationResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.util.UUID; 8 | 9 | @Value 10 | @Builder 11 | @Jacksonized 12 | public class ServiceInstanceRegistrationResponse { 13 | 14 | UUID id; 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/response/ServiceInstanceResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | import java.time.Instant; 8 | import java.util.List; 9 | import java.util.UUID; 10 | 11 | @Getter 12 | @Builder 13 | @Jacksonized 14 | public class ServiceInstanceResponse { 15 | 16 | private final UUID id; 17 | private final String location; 18 | private final int port; 19 | private final Instant lastHeartbeat; 20 | private final List properties; 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/client/response/ServiceResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.client.response; 2 | 3 | import com.github.bottomlessarchive.loa.application.domain.ApplicationType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.extern.jackson.Jacksonized; 7 | 8 | import java.util.List; 9 | 10 | @Getter 11 | @Builder 12 | @Jacksonized 13 | public class ServiceResponse { 14 | 15 | private final ApplicationType applicationType; 16 | private final List instances; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/domain/NetworkAddressCalculationException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class NetworkAddressCalculationException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-client-test/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-application-domain') 3 | 4 | implementation group: 'com.github.tomakehurst', name: 'wiremock-jre8-standalone', version: '3.0.1' 5 | } 6 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-domain/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-application-domain') 3 | } 4 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-domain/src/main/java/com/github/bottomlessarchive/loa/conductor/service/domain/ServiceInstanceEntityProperty.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class ServiceInstanceEntityProperty { 9 | 10 | private final String name; 11 | private final String value; 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-conductor-domain') 3 | 4 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-json' 5 | } 6 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/domain/ServiceInstanceProperty.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Getter 8 | @Builder 9 | @Jacksonized 10 | public class ServiceInstanceProperty { 11 | 12 | private final String name; 13 | private final String value; 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/domain/ServiceInstanceRefreshContext.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.domain; 2 | 3 | import com.github.bottomlessarchive.loa.application.domain.ApplicationType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.List; 8 | import java.util.UUID; 9 | 10 | @Getter 11 | @Builder 12 | public class ServiceInstanceRefreshContext { 13 | 14 | private final UUID instanceId; 15 | private final ApplicationType applicationType; 16 | private final List properties; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-conductor-service/src/main/java/com/github/bottomlessarchive/loa/conductor/service/domain/ServiceInstanceRegistrationContext.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.conductor.service.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.Collections; 7 | import java.util.List; 8 | 9 | @Getter 10 | @Builder 11 | public class ServiceInstanceRegistrationContext { 12 | 13 | private final String location; 14 | private final int port; 15 | 16 | @Builder.Default 17 | private final List properties = Collections.emptyList(); 18 | 19 | public List getProperties() { 20 | return Collections.unmodifiableList(properties); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-repository/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-repository') 3 | } 4 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-repository/src/main/java/com/github/bottomlessarchive/loa/location/repository/configuration/DocumentLocationMongoConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.repository.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.location.repository.domain.DocumentLocationDatabaseEntity; 4 | import com.mongodb.client.MongoCollection; 5 | import com.mongodb.client.MongoDatabase; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | 9 | @Configuration 10 | public class DocumentLocationMongoConfiguration { 11 | 12 | @Bean 13 | public MongoCollection documentLocationDatabaseEntityMongoCollection( 14 | final MongoDatabase mongoDatabase) { 15 | return mongoDatabase.getCollection("documentLocation", DocumentLocationDatabaseEntity.class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-repository/src/main/java/com/github/bottomlessarchive/loa/location/repository/domain/DocumentLocationDatabaseEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.repository.domain; 2 | 3 | import lombok.Builder; 4 | import org.bson.codecs.pojo.annotations.BsonId; 5 | 6 | @Builder 7 | public record DocumentLocationDatabaseEntity( 8 | 9 | @BsonId 10 | byte[] id, 11 | String url, 12 | String source, 13 | int downloaderVersion, 14 | String downloadResultCode 15 | ) { 16 | } 17 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation project(':loa-library:loa-url-service') 3 | implementation project(':loa-library:loa-number-service') 4 | implementation project(':loa-library:loa-document-type-service') 5 | implementation project(':loa-library:loa-document-location-repository') 6 | 7 | implementation group: 'commons-codec', name: 'commons-codec', version: '1.17.0' 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/src/main/java/com/github/bottomlessarchive/loa/location/domain/DocumentLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | @Getter 8 | @Builder 9 | public class DocumentLocation { 10 | 11 | private final String id; 12 | private final String location; 13 | private final DocumentType type; 14 | private final String sourceName; 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/src/main/java/com/github/bottomlessarchive/loa/location/service/DocumentLocationManipulator.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.service; 2 | 3 | import com.github.bottomlessarchive.loa.location.domain.DocumentLocationResultType; 4 | import com.github.bottomlessarchive.loa.location.repository.DocumentLocationRepository; 5 | import com.github.bottomlessarchive.loa.number.service.HexConverter; 6 | import lombok.RequiredArgsConstructor; 7 | import org.springframework.stereotype.Service; 8 | 9 | @Service 10 | @RequiredArgsConstructor 11 | public class DocumentLocationManipulator { 12 | 13 | private final HexConverter hexConverter; 14 | private final DocumentLocationRepository documentLocationRepository; 15 | 16 | public void updateDownloadResultCode(final String documentLocationId, final DocumentLocationResultType downloadResultCode) { 17 | documentLocationRepository.updateDownloadResultCode(hexConverter.decode(documentLocationId), downloadResultCode.toString()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/src/main/java/com/github/bottomlessarchive/loa/location/service/factory/domain/DocumentLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.service.factory.domain; 2 | 3 | import com.github.bottomlessarchive.loa.location.domain.DocumentLocationResultType; 4 | import lombok.Builder; 5 | 6 | @Builder 7 | public record DocumentLocation( 8 | 9 | String id, 10 | String url, 11 | String source, 12 | int downloaderVersion, 13 | DocumentLocationResultType downloadResultCode 14 | ) { 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/src/main/java/com/github/bottomlessarchive/loa/location/service/factory/domain/DocumentLocationCreationContext.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.service.factory.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class DocumentLocationCreationContext { 9 | 10 | private final String id; 11 | private final String url; 12 | private final String source; 13 | private final int downloaderVersion; 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/src/main/java/com/github/bottomlessarchive/loa/location/service/id/factory/DocumentLocationIdFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.service.id.factory; 2 | 3 | import java.net.URL; 4 | 5 | /** 6 | * This factory creates semi-unique ids for document locations. 7 | */ 8 | public interface DocumentLocationIdFactory { 9 | 10 | /** 11 | * Create an semi-unique document location id based on the url. 12 | * 13 | * @param documentLocation the location of the document to create the id for 14 | * @return the id for the document 15 | */ 16 | String newDocumentLocationId(URL documentLocation); 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-document-location-service/src/main/java/com/github/bottomlessarchive/loa/location/service/id/factory/Sha256DocumentLocationIdFactory.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.location.service.id.factory; 2 | 3 | import org.apache.commons.codec.digest.DigestUtils; 4 | import org.springframework.stereotype.Service; 5 | 6 | import java.net.URL; 7 | 8 | /** 9 | * An SHA-256 based implementation for the {@link DocumentLocationIdFactory}. 10 | */ 11 | @Service 12 | public class Sha256DocumentLocationIdFactory implements DocumentLocationIdFactory { 13 | 14 | /** 15 | * Create a semi-unique document id based on the SHA-256 hash of the document's location. 16 | * 17 | * @param documentLocation the location of the document to create the id for 18 | * @return the id for the document 19 | */ 20 | @Override 21 | public String newDocumentLocationId(final URL documentLocation) { 22 | return DigestUtils.sha256Hex(documentLocation.toString()); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /loa-library/loa-document-parser-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter' 3 | 4 | implementation group: 'com.github.pemistahl', name: 'lingua', version: '1.2.2' 5 | 6 | implementation group: 'org.apache.tika', name: 'tika-core', version: '3.0.0-BETA' 7 | implementation group: 'org.apache.tika', name: 'tika-parser-pdf-module', version: '3.0.0-BETA' 8 | implementation group: 'org.apache.tika', name: 'tika-parser-microsoft-module', version: '3.0.0-BETA' 9 | implementation group: 'org.apache.tika', name: 'tika-parser-miscoffice-module', version: '3.0.0-BETA' 10 | 11 | implementation group: 'com.fasterxml.woodstox', name: 'woodstox-core', version: '7.0.0' 12 | 13 | api project(':loa-library:loa-document-type-service') 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-document-parser-service/src/main/java/com/github/bottomlessarchive/loa/parser/domain/ParsingException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.parser.domain; 2 | 3 | public class ParsingException extends RuntimeException { 4 | 5 | public ParsingException(final String message, final Throwable cause) { 6 | super(message, cause); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-document-parser-service/src/main/java/com/github/bottomlessarchive/loa/parser/domain/ParsingResult.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.parser.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.UUID; 8 | 9 | @Getter 10 | @Builder 11 | public class ParsingResult { 12 | 13 | private final UUID id; 14 | private final String content; 15 | private final String title; 16 | private final String author; 17 | private final String date; 18 | private final String language; 19 | private final DocumentType type; 20 | private final int pageCount; 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-document-repository/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-repository') 3 | 4 | implementation project(':loa-library:loa-number-service') 5 | } 6 | -------------------------------------------------------------------------------- /loa-library/loa-document-repository/src/main/java/com/github/bottomlessarchive/loa/document/repository/domain/DocumentStatusAggregateEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.document.repository.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import org.bson.codecs.pojo.annotations.BsonId; 6 | 7 | @Getter 8 | @Setter 9 | public class DocumentStatusAggregateEntity { 10 | 11 | @BsonId 12 | private String id; 13 | private int count; 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-document-repository/src/main/java/com/github/bottomlessarchive/loa/document/repository/domain/DocumentTypeAggregateEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.document.repository.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import org.bson.codecs.pojo.annotations.BsonId; 6 | 7 | @Getter 8 | @Setter 9 | public class DocumentTypeAggregateEntity { 10 | 11 | @BsonId 12 | private String id; 13 | private int count; 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-document-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-json' 3 | 4 | api project(':loa-library:loa-compression-service') 5 | api project(':loa-library:loa-document-type-service') 6 | 7 | implementation project(':loa-library:loa-number-service') 8 | implementation project(':loa-library:loa-document-repository') 9 | } 10 | -------------------------------------------------------------------------------- /loa-library/loa-document-service/src/main/java/com/github/bottomlessarchive/loa/document/service/domain/DocumentStatus.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.document.service.domain; 2 | 3 | public enum DocumentStatus { 4 | 5 | /** 6 | * The document was created in the database but not yet fully moved into the vault. 7 | */ 8 | CREATED, 9 | 10 | /** 11 | * The document was successfully downloaded and moved to the vault. 12 | */ 13 | DOWNLOADED, 14 | 15 | /** 16 | * The document is indexed to the search engine. 17 | */ 18 | INDEXED, 19 | 20 | /** 21 | * The document is not in the vaults but on one of the beacon machines. 22 | */ 23 | ON_BEACON, 24 | 25 | /** 26 | * The document is not available in the vault, or the data that is available are corrupt in any for or shape. This means that a corrupt 27 | * document mostly likely can't be opened, indexed or restored. 28 | */ 29 | CORRUPT 30 | } 31 | -------------------------------------------------------------------------------- /loa-library/loa-document-service/src/main/java/com/github/bottomlessarchive/loa/document/service/domain/DuplicateDocumentException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.document.service.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class DuplicateDocumentException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-document-type-service/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-library/loa-document-type-service/build.gradle -------------------------------------------------------------------------------- /loa-library/loa-document-type-service/src/main/java/com/github/bottomlessarchive/loa/type/domain/DocumentType.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.type.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | @Getter 7 | @RequiredArgsConstructor 8 | public enum DocumentType { 9 | 10 | PDF("pdf", "application/pdf"), 11 | DOC("doc", "application/msword"), 12 | DOCX("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"), 13 | PPT("ppt", "application/vnd.ms-powerpoint"), 14 | PPTX("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"), 15 | XLS("xls", "application/vnd.ms-excel"), 16 | XLSX("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), 17 | RTF("rtf", "application/rtf"), 18 | MOBI("mobi", "application/x-mobipocket-ebook"), 19 | EPUB("epub", "application/epub+zip"), 20 | FB2("fb2", "text/fb2+xml"), 21 | TXT("txt", "text/plain"); 22 | 23 | private final String fileExtension; 24 | private final String mimeType; 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-document-view/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api group: 'org.springframework', name: 'spring-web' 3 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter' 4 | 5 | api project(':loa-library:loa-document-service') 6 | implementation project(':loa-library:loa-document-type-service') 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-document-view/src/main/java/com/github/bottomlessarchive/loa/document/view/service/MediaTypeCalculator.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.document.view.service; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import org.springframework.http.MediaType; 5 | import org.springframework.stereotype.Service; 6 | 7 | @Service 8 | public class MediaTypeCalculator { 9 | 10 | public MediaType calculateMediaType(final DocumentType documentType) { 11 | return MediaType.valueOf(documentType.getMimeType()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /loa-library/loa-downloader-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 3 | 4 | implementation group: 'com.squareup.okhttp3', name: 'okhttp' 5 | 6 | implementation project(':loa-library:loa-io-service') 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-downloader-service/src/main/java/com/github/bottomlessarchive/loa/io/service/collector/domain/DocumentCollectionException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.io.service.collector.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class DocumentCollectionException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-downloader-service/src/main/java/com/github/bottomlessarchive/loa/io/service/configuration/HttpClientConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.io.service.configuration; 2 | 3 | import jakarta.validation.constraints.Min; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | @Validated 8 | @ConfigurationProperties("loa.http") 9 | public record HttpClientConfigurationProperties( 10 | 11 | @Min(1) 12 | int parallelism 13 | ) { 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-downloader-service/src/main/java/com/github/bottomlessarchive/loa/io/service/configuration/ssl/TrustAllTrustManager.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.io.service.configuration.ssl; 2 | 3 | import org.springframework.stereotype.Component; 4 | 5 | import javax.net.ssl.X509TrustManager; 6 | 7 | @Component 8 | public class TrustAllTrustManager implements X509TrustManager { 9 | 10 | @Override 11 | public void checkClientTrusted(final java.security.cert.X509Certificate[] chain, final String authType) { 12 | } 13 | 14 | @Override 15 | public void checkServerTrusted(final java.security.cert.X509Certificate[] chain, final String authType) { 16 | } 17 | 18 | @Override 19 | public java.security.cert.X509Certificate[] getAcceptedIssuers() { 20 | return new java.security.cert.X509Certificate[]{}; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /loa-library/loa-file-service/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-library/loa-file-service/build.gradle -------------------------------------------------------------------------------- /loa-library/loa-file-service/src/test/java/com/github/bottomlessarchive/loa/file/FileManipulatorServiceTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.file; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import java.io.File; 6 | import java.nio.file.Path; 7 | 8 | import static org.assertj.core.api.Assertions.assertThat; 9 | 10 | class FileManipulatorServiceTest { 11 | 12 | private final FileManipulatorService underTest = new FileManipulatorService(); 13 | 14 | @Test 15 | void testNewFile() { 16 | final Path result = underTest.newFile("testpath", "testname.zip"); 17 | 18 | assertThat(result.toString()) 19 | .isEqualTo("testpath" + File.separator + "testname.zip"); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation project(':loa-library:loa-document-service') 3 | implementation project(':loa-library:loa-conductor-client-service') 4 | 5 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-json' 6 | 7 | implementation group: 'jakarta.json', name: 'jakarta.json-api', version: '2.1.3' 8 | implementation group: 'co.elastic.clients', name: 'elasticsearch-java', version: '8.14.1' 9 | } 10 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/indexer/domain/IndexingContext.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.indexer.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.UUID; 8 | 9 | @Getter 10 | @Builder 11 | public class IndexingContext { 12 | 13 | private final UUID id; 14 | private final String content; 15 | private final String title; 16 | private final String author; 17 | private final String date; 18 | private final String language; 19 | private final DocumentType type; 20 | private final int pageCount; 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/search/domain/DocumentLength.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.search.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | @Getter 7 | @RequiredArgsConstructor 8 | public enum DocumentLength { 9 | 10 | SHORT_STORY(1, 10), 11 | NOVELETTE(11, 50), 12 | NOVELLA(51, 150), 13 | NOVEL(151, Integer.MAX_VALUE); 14 | 15 | private final int minimumPageCount; 16 | private final int maximumPageCount; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/search/domain/DocumentSearchEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.search.domain; 2 | 3 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.List; 8 | 9 | @Getter 10 | @Builder 11 | public class DocumentSearchEntity { 12 | 13 | private final String id; 14 | private final String title; 15 | private final String author; 16 | private final List description; 17 | private final String language; 18 | private final int pageCount; 19 | private final DocumentType type; 20 | } 21 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/search/domain/DocumentSearchResult.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.search.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Contains the result of a document search in the indexer repository. 10 | */ 11 | @Getter 12 | @Builder 13 | public class DocumentSearchResult { 14 | 15 | private final long totalHitCount; 16 | private final List searchHits; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/search/domain/IndexerAccessException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.search.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class IndexerAccessException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/search/domain/SearchDatabaseEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.search.domain; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.github.bottomlessarchive.loa.type.domain.DocumentType; 6 | import lombok.Data; 7 | 8 | @Data 9 | @JsonIgnoreProperties(ignoreUnknown = true) 10 | public class SearchDatabaseEntity { 11 | 12 | private String title; 13 | private String language; 14 | private String author; 15 | private String date; 16 | private DocumentType type; 17 | 18 | @JsonProperty("page_count") 19 | private int pageCount; 20 | } 21 | -------------------------------------------------------------------------------- /loa-library/loa-indexer-client-service/src/main/java/com/github/bottomlessarchive/loa/indexer/service/search/domain/SearchField.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.indexer.service.search.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | /** 7 | * This enum contains the fields that are available in the search engine. 8 | */ 9 | @Getter 10 | @RequiredArgsConstructor 11 | public enum SearchField { 12 | 13 | /** 14 | * The content of the document. 15 | */ 16 | CONTENT("content"), 17 | 18 | /** 19 | * The language of the document. 20 | */ 21 | LANGUAGE("language"), 22 | 23 | /** 24 | * The type of the document. 25 | */ 26 | DOCUMENT_TYPE("type"), 27 | 28 | /** 29 | * The page count of the document. 30 | */ 31 | PAGE_COUNT("page_count"); 32 | 33 | /** 34 | * The name of the field in the search engine. 35 | */ 36 | private final String name; 37 | } 38 | -------------------------------------------------------------------------------- /loa-library/loa-io-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-file-service') 3 | api project(':loa-library:loa-document-type-service') 4 | } 5 | -------------------------------------------------------------------------------- /loa-library/loa-loadoc-domain/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-library/loa-loadoc-domain/build.gradle -------------------------------------------------------------------------------- /loa-library/loa-loadoc-domain/src/main/java/com/github/bottomlessarchive/loa/loadoc/domain/LoadocMetadata.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.loadoc.domain; 2 | 3 | import lombok.Builder; 4 | 5 | @Builder 6 | public record LoadocMetadata( 7 | 8 | String id, 9 | String type, 10 | String sourceName, 11 | String documentId, 12 | String downloadResultCode 13 | ) { 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-logging-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | //TODO: This should be implementation! We should abstract that we are using micrometer in the background! 3 | api group: 'io.micrometer', name: 'micrometer-core' 4 | } -------------------------------------------------------------------------------- /loa-library/loa-logging-service/src/main/java/com/github/bottomlessarchive/loa/logging/service/LoggingMeterRegistry.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.logging.service; 2 | 3 | import io.micrometer.core.instrument.Counter; 4 | import io.micrometer.core.instrument.Meter; 5 | import io.micrometer.core.instrument.simple.SimpleMeterRegistry; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.scheduling.annotation.Scheduled; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Slf4j 11 | @Service 12 | public class LoggingMeterRegistry extends SimpleMeterRegistry { 13 | 14 | private final MetricLogger metricLogger; 15 | 16 | public LoggingMeterRegistry(final MetricLogger metricLogger) { 17 | this.metricLogger = metricLogger; 18 | } 19 | 20 | @Scheduled(fixedDelay = 6000) 21 | public void printMetrics() { 22 | getMeters().forEach(meter -> { 23 | if (meter.getId().getType() == Meter.Type.COUNTER) { 24 | metricLogger.logCounter((Counter) meter); 25 | } 26 | }); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /loa-library/loa-logging-service/src/main/java/com/github/bottomlessarchive/loa/logging/service/MetricLogger.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.logging.service; 2 | 3 | import io.micrometer.core.instrument.Counter; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.stereotype.Service; 6 | 7 | import java.util.Arrays; 8 | 9 | @Slf4j 10 | @Service 11 | public class MetricLogger { 12 | 13 | public void logCounters(final Counter... counter) { 14 | Arrays.stream(counter) 15 | .forEach(this::logCounter); 16 | } 17 | 18 | public void logCounter(final Counter counter) { 19 | if (log.isInfoEnabled()) { 20 | log.info("The {} are {} {}!", counter.getId().getTag("printed-name"), (int) counter.count(), 21 | counter.getId().getBaseUnit()); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /loa-library/loa-number-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'commons-codec', name: 'commons-codec', version: '1.17.0' 3 | } 4 | -------------------------------------------------------------------------------- /loa-library/loa-number-service/src/main/java/com/github/bottomlessarchive/loa/number/service/HexConverter.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.number.service; 2 | 3 | import com.github.bottomlessarchive.loa.number.service.domain.HexConversionException; 4 | import org.apache.commons.codec.DecoderException; 5 | import org.apache.commons.codec.binary.Hex; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Service 9 | public class HexConverter { 10 | 11 | public String encode(final byte[] target) { 12 | return Hex.encodeHexString(target); 13 | } 14 | 15 | public byte[] decode(final String target) { 16 | try { 17 | return Hex.decodeHex(target); 18 | } catch (final DecoderException e) { 19 | throw new HexConversionException("Unable to decode: " + target + "!", e); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /loa-library/loa-number-service/src/main/java/com/github/bottomlessarchive/loa/number/service/domain/HexConversionException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.number.service.domain; 2 | 3 | public class HexConversionException extends RuntimeException { 4 | 5 | public HexConversionException(final String message, final Throwable throwable) { 6 | super(message, throwable); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-queue-artemis-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 3 | 4 | api project(':loa-library:loa-queue-service') 5 | implementation project(':loa-library:loa-conductor-client-service') 6 | 7 | implementation group: 'org.apache.activemq', name: 'artemis-core-client' 8 | compileOnly group: 'org.apache.activemq', name: 'artemis-server' 9 | } 10 | -------------------------------------------------------------------------------- /loa-library/loa-queue-artemis-service/src/main/java/com/github/bottomlessarchive/loa/queue/artemis/configuration/QueueServerConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.artemis.configuration; 2 | 3 | import jakarta.validation.constraints.Max; 4 | import jakarta.validation.constraints.Min; 5 | import jakarta.validation.constraints.NotNull; 6 | import org.apache.activemq.artemis.core.server.embedded.EmbeddedActiveMQ; 7 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 8 | import org.springframework.boot.context.properties.ConfigurationProperties; 9 | import org.springframework.validation.annotation.Validated; 10 | 11 | import java.nio.file.Path; 12 | 13 | @Validated 14 | @ConfigurationProperties("loa.queue") 15 | @ConditionalOnClass(EmbeddedActiveMQ.class) 16 | public record QueueServerConfigurationProperties( 17 | 18 | @Min(1) 19 | @Max(65535) 20 | int port, 21 | 22 | @NotNull 23 | Path dataDirectory 24 | ) { 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-queue-artemis-service/src/main/java/com/github/bottomlessarchive/loa/queue/artemis/service/consumer/deserializer/MessageDeserializer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.artemis.service.consumer.deserializer; 2 | 3 | import com.github.bottomlessarchive.loa.queue.service.domain.Queue; 4 | import org.apache.activemq.artemis.api.core.client.ClientMessage; 5 | 6 | public interface MessageDeserializer { 7 | 8 | T deserialize(ClientMessage clientMessage); 9 | 10 | Queue supports(); 11 | } 12 | -------------------------------------------------------------------------------- /loa-library/loa-queue-artemis-service/src/main/java/com/github/bottomlessarchive/loa/queue/artemis/service/consumer/pool/domain/QueueConsumer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.artemis.service.consumer.pool.domain; 2 | 3 | import com.github.bottomlessarchive.loa.queue.service.domain.QueueException; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import org.apache.activemq.artemis.api.core.ActiveMQException; 7 | import org.apache.activemq.artemis.api.core.client.ClientConsumer; 8 | import org.apache.activemq.artemis.api.core.client.ClientSession; 9 | 10 | @Getter 11 | @Builder 12 | public class QueueConsumer implements AutoCloseable { 13 | 14 | private final ClientSession clientSession; 15 | private final ClientConsumer clientConsumer; 16 | 17 | @Override 18 | public void close() { 19 | try { 20 | clientConsumer.close(); 21 | clientSession.close(); 22 | } catch (final ActiveMQException e) { 23 | throw new QueueException("Failed to close queue consumer!", e); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /loa-library/loa-queue-artemis-service/src/main/java/com/github/bottomlessarchive/loa/queue/artemis/service/producer/pool/domain/QueueProducer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.artemis.service.producer.pool.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import org.apache.activemq.artemis.api.core.ActiveMQException; 6 | import org.apache.activemq.artemis.api.core.client.ClientProducer; 7 | import org.apache.activemq.artemis.api.core.client.ClientSession; 8 | 9 | @Getter 10 | @Builder 11 | public class QueueProducer implements AutoCloseable { 12 | 13 | private final ClientSession clientSession; 14 | private final ClientProducer clientProducer; 15 | 16 | @Override 17 | public void close() throws ActiveMQException { 18 | clientProducer.close(); 19 | clientSession.close(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-queue-artemis-service/src/main/java/com/github/bottomlessarchive/loa/queue/artemis/service/producer/serializer/MessageSerializer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.artemis.service.producer.serializer; 2 | 3 | import com.github.bottomlessarchive.loa.queue.service.domain.Queue; 4 | import org.apache.activemq.artemis.api.core.client.ClientMessage; 5 | 6 | public interface MessageSerializer { 7 | 8 | ClientMessage serialize(T message); 9 | 10 | Queue supports(); 11 | } 12 | -------------------------------------------------------------------------------- /loa-library/loa-queue-service/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bottomless-archive-project/library-of-alexandria/70d4a157012ab1ca6fe4fdebdffe1bf8e83c5678/loa-library/loa-queue-service/build.gradle -------------------------------------------------------------------------------- /loa-library/loa-queue-service/src/main/java/com/github/bottomlessarchive/loa/queue/service/domain/Queue.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.service.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | @Getter 7 | @RequiredArgsConstructor 8 | public enum Queue { 9 | 10 | /** 11 | * This queue is responsible to hold the document locations created by the Generator Application. 12 | */ 13 | DOCUMENT_LOCATION_QUEUE("loa-document-location", "loa-document-location"), 14 | /** 15 | * This queue is responsible to hold the documents downloaded by the Downloader Application. 16 | */ 17 | DOCUMENT_ARCHIVING_QUEUE("loa-document-archiving", "loa-document-archiving"); 18 | 19 | private final String name; 20 | private final String address; 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-queue-service/src/main/java/com/github/bottomlessarchive/loa/queue/service/domain/QueueException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.service.domain; 2 | 3 | public class QueueException extends RuntimeException { 4 | 5 | public QueueException(final String message) { 6 | super(message); 7 | } 8 | 9 | public QueueException(final String message, final Throwable throwable) { 10 | super(message, throwable); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-queue-service/src/main/java/com/github/bottomlessarchive/loa/queue/service/domain/message/DocumentArchivingMessage.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.service.domain.message; 2 | 3 | import lombok.Builder; 4 | import lombok.NonNull; 5 | 6 | import java.util.Optional; 7 | 8 | @Builder 9 | public record DocumentArchivingMessage( 10 | 11 | @NonNull 12 | String id, 13 | 14 | @NonNull 15 | String type, 16 | 17 | @NonNull 18 | String source, 19 | 20 | @NonNull 21 | String checksum, 22 | 23 | @NonNull 24 | Optional sourceLocationId, 25 | 26 | @NonNull 27 | String compression, 28 | 29 | boolean fromBeacon, 30 | long contentLength, 31 | long originalContentLength 32 | ) { 33 | } 34 | -------------------------------------------------------------------------------- /loa-library/loa-queue-service/src/main/java/com/github/bottomlessarchive/loa/queue/service/domain/message/DocumentLocationMessage.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.queue.service.domain.message; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class DocumentLocationMessage { 9 | 10 | private final String id; 11 | private final String sourceName; 12 | private final String documentLocation; 13 | private final String type; 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-renderer-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.apache.pdfbox', name: 'pdfbox', version: '3.0.2' 3 | implementation group: 'net.coobird', name: 'thumbnailator', version: '0.4.20' 4 | } 5 | -------------------------------------------------------------------------------- /loa-library/loa-renderer-service/src/main/java/com/github/bottomlessarchive/loa/renderer/service/PageRendererService.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.renderer.service; 2 | 3 | import lombok.SneakyThrows; 4 | import org.apache.pdfbox.pdmodel.PDDocument; 5 | import org.apache.pdfbox.rendering.PDFRenderer; 6 | import org.springframework.stereotype.Service; 7 | 8 | import java.awt.image.BufferedImage; 9 | 10 | @Service 11 | public class PageRendererService { 12 | 13 | @SneakyThrows 14 | public BufferedImage renderPage(final PDDocument document, final int pageIndex) { 15 | final PDFRenderer pdfRenderer = new PDFRenderer(document); 16 | 17 | return pdfRenderer.renderImageWithDPI(pageIndex, 300); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /loa-library/loa-renderer-service/src/main/java/com/github/bottomlessarchive/loa/renderer/service/ThumbnailService.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.renderer.service; 2 | 3 | import lombok.SneakyThrows; 4 | import net.coobird.thumbnailator.Thumbnails; 5 | import org.springframework.stereotype.Service; 6 | 7 | import java.awt.image.BufferedImage; 8 | import java.io.ByteArrayOutputStream; 9 | 10 | @Service 11 | public class ThumbnailService { 12 | 13 | @SneakyThrows 14 | public byte[] createThumbnail(final BufferedImage image) { 15 | try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { 16 | Thumbnails.of(image) 17 | .size(248, 350) 18 | .outputFormat("png") 19 | .toOutputStream(outputStream); 20 | 21 | return outputStream.toByteArray(); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /loa-library/loa-renderer-service/src/main/java/com/github/bottomlessarchive/loa/renderer/service/domain/ImageRenderingException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.renderer.service.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class ImageRenderingException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-repository/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api group: 'org.mongodb', name: 'mongodb-driver-sync' 3 | 4 | implementation project(':loa-library:loa-conductor-client-service') 5 | } 6 | -------------------------------------------------------------------------------- /loa-library/loa-repository/src/main/java/com/github/bottomlessarchive/loa/repository/document/Error.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.repository.document; 2 | 3 | import lombok.Getter; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | @Getter 7 | @RequiredArgsConstructor 8 | public enum Error { 9 | 10 | DUPLICATE(11000); 11 | 12 | private final int errorCode; 13 | 14 | public boolean hasErrorCode(final int expectedErrorCode) { 15 | return errorCode == expectedErrorCode; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-source-file-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 3 | 4 | api project(':loa-library:loa-source-service') 5 | implementation project(':loa-library:loa-file-service') 6 | implementation project(':loa-library:loa-logging-service') 7 | 8 | testImplementation group: 'org.apache.commons', name: 'commons-compress', version: '1.26.2' 9 | } 10 | -------------------------------------------------------------------------------- /loa-library/loa-source-file-service/src/main/java/com/github/bottomlessarchive/loa/source/file/configuration/GeneratorStatisticsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.source.file.configuration; 2 | 3 | import io.micrometer.core.instrument.Counter; 4 | import io.micrometer.core.instrument.MeterRegistry; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | public class GeneratorStatisticsConfiguration { 10 | 11 | @Bean 12 | public Counter processedDocumentLocationCount(final MeterRegistry meterRegistry) { 13 | return Counter.builder("generator.processed-document-location-count") 14 | .tag("printed-name", "processed document location count") 15 | .baseUnit("urls") 16 | .register(meterRegistry); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /loa-library/loa-source-file-service/src/main/java/com/github/bottomlessarchive/loa/source/file/service/domain/FileEncodingType.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.source.file.service.domain; 2 | 3 | public enum FileEncodingType { 4 | 5 | NONE, 6 | GZIP 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-source-file-service/src/main/java/com/github/bottomlessarchive/loa/source/file/service/domain/FileHandlingException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.source.file.service.domain; 2 | 3 | public class FileHandlingException extends RuntimeException { 4 | 5 | public FileHandlingException(final String message, final Throwable throwable) { 6 | super(message, throwable); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /loa-library/loa-source-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter' 3 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 4 | 5 | api project(':loa-library:loa-document-location-service') 6 | } 7 | -------------------------------------------------------------------------------- /loa-library/loa-source-service/src/main/java/com/github/bottomlessarchive/loa/source/domain/DocumentSourceType.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.source.domain; 2 | 3 | /** 4 | * Describe the various type of sources where document locations could be gathered. 5 | */ 6 | public enum DocumentSourceType { 7 | 8 | /** 9 | * The source data are loaded from a file. 10 | */ 11 | FILE 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-source-service/src/main/java/com/github/bottomlessarchive/loa/source/source/DocumentLocationSource.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.source.source; 2 | 3 | import com.github.bottomlessarchive.loa.location.domain.DocumentLocation; 4 | 5 | import java.util.stream.Stream; 6 | 7 | /** 8 | * This class is streaming new {@link DocumentLocation}s that should be checked for new documents. 9 | */ 10 | public interface DocumentLocationSource { 11 | 12 | /** 13 | * This method is streaming new document locations that should be checked for new documents. 14 | * 15 | * @return document locations that should be checked for new documents 16 | */ 17 | Stream streamLocations(); 18 | } 19 | -------------------------------------------------------------------------------- /loa-library/loa-stage-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation project(':loa-library:loa-file-service') 3 | 4 | testImplementation group: 'com.google.jimfs', name: 'jimfs', version: '1.3.0' 5 | } 6 | -------------------------------------------------------------------------------- /loa-library/loa-stage-service/src/main/java/com/github/bottomlessarchive/loa/stage/service/domain/exception/StageAccessException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.stage.service.domain.exception; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | /** 6 | * Thrown when an error happens while accessing content on the staging area. 7 | */ 8 | @StandardException 9 | public class StageAccessException extends RuntimeException { 10 | } 11 | -------------------------------------------------------------------------------- /loa-library/loa-staging-client-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'com.squareup.okhttp3', name: 'okhttp' 3 | 4 | implementation project(':loa-library:loa-conductor-client-service') 5 | } -------------------------------------------------------------------------------- /loa-library/loa-staging-client-service/src/main/java/com/github/bottomlessarchive/loa/staging/service/client/configuration/StagingClientConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.staging.service.client.configuration; 2 | 3 | import okhttp3.OkHttpClient; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | import java.time.Duration; 8 | 9 | @Configuration 10 | public class StagingClientConfiguration { 11 | 12 | @Bean 13 | public OkHttpClient stagingWebClient() { 14 | return new OkHttpClient.Builder() 15 | .callTimeout(Duration.ofSeconds(30)) 16 | .connectTimeout(Duration.ofSeconds(30)) 17 | .readTimeout(Duration.ofSeconds(30)) 18 | .writeTimeout(Duration.ofSeconds(30)) 19 | .build(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-staging-client-service/src/main/java/com/github/bottomlessarchive/loa/staging/service/client/domain/StagingException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.staging.service.client.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class StagingException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-threading-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'io.micrometer', name: 'micrometer-core' 3 | } -------------------------------------------------------------------------------- /loa-library/loa-threading-service/src/main/java/com/github/bottomlessarchive/loa/threading/task/CallbackWrapperTask.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.threading.task; 2 | 3 | import lombok.Builder; 4 | import lombok.NonNull; 5 | 6 | @Builder 7 | public class CallbackWrapperTask implements Runnable { 8 | 9 | @NonNull 10 | private final Runnable task; 11 | private final Runnable callback; 12 | 13 | @Override 14 | public void run() { 15 | task.run(); 16 | 17 | if (callback != null) { 18 | callback.run(); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-threading-service/src/main/java/com/github/bottomlessarchive/loa/threading/task/CounterIncrementingWrapperTask.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.threading.task; 2 | 3 | import io.micrometer.core.instrument.Counter; 4 | import lombok.Builder; 5 | import lombok.NonNull; 6 | 7 | @Builder 8 | public class CounterIncrementingWrapperTask implements Runnable { 9 | 10 | @NonNull 11 | private final Counter counter; 12 | @NonNull 13 | private final Runnable task; 14 | 15 | @Override 16 | public void run() { 17 | counter.increment(); 18 | 19 | task.run(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /loa-library/loa-threading-service/src/main/java/com/github/bottomlessarchive/loa/threading/task/MDCWrapperTask.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.threading.task; 2 | 3 | import lombok.Builder; 4 | import lombok.NonNull; 5 | import org.slf4j.MDC; 6 | 7 | import java.util.Map; 8 | 9 | @Builder 10 | public class MDCWrapperTask implements Runnable { 11 | 12 | @NonNull 13 | private final Map mdcParameters; 14 | @NonNull 15 | private final Runnable task; 16 | 17 | @Override 18 | public void run() { 19 | mdcParameters.forEach(MDC::put); 20 | 21 | task.run(); 22 | 23 | MDC.clear(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-threading-service/src/main/java/com/github/bottomlessarchive/loa/threading/thread/ThreadManipulator.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.threading.thread; 2 | 3 | import org.springframework.stereotype.Service; 4 | 5 | @Service 6 | public class ThreadManipulator { 7 | 8 | public void runInNewThread(final Runnable runnable) { 9 | new Thread(runnable).start(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /loa-library/loa-url-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'io.mola.galimatias', name: 'galimatias', version: '0.2.1' 3 | } -------------------------------------------------------------------------------- /loa-library/loa-user-repository/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | api project(':loa-library:loa-repository') 3 | } 4 | -------------------------------------------------------------------------------- /loa-library/loa-user-repository/src/main/java/com/github/bottomlessarchive/loa/user/repository/domain/UserAlreadyExistsInDatabaseException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.user.repository.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class UserAlreadyExistsInDatabaseException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-user-repository/src/main/java/com/github/bottomlessarchive/loa/user/repository/domain/UserDatabaseEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.user.repository.domain; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import org.bson.codecs.pojo.annotations.BsonId; 6 | 7 | import java.util.UUID; 8 | 9 | @Getter 10 | @Setter 11 | public class UserDatabaseEntity { 12 | 13 | @BsonId 14 | private UUID id; 15 | private String name; 16 | private String password; 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-user-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation project(':loa-library:loa-user-repository') 3 | } -------------------------------------------------------------------------------- /loa-library/loa-user-service/src/main/java/com/github/bottomlessarchive/loa/user/configuration/UserConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.user.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | @ConfigurationProperties("loa.user") 6 | public record UserConfigurationProperties( 7 | 8 | boolean enabled 9 | ) { 10 | } 11 | -------------------------------------------------------------------------------- /loa-library/loa-user-service/src/main/java/com/github/bottomlessarchive/loa/user/service/UserEntityTransformer.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.user.service; 2 | 3 | import com.github.bottomlessarchive.loa.user.repository.domain.UserDatabaseEntity; 4 | import com.github.bottomlessarchive.loa.user.service.domain.UserEntity; 5 | import org.springframework.stereotype.Service; 6 | 7 | @Service 8 | public class UserEntityTransformer { 9 | 10 | public UserEntity transform(final UserDatabaseEntity userDatabaseEntity) { 11 | return UserEntity.builder() 12 | .id(userDatabaseEntity.getId()) 13 | .name(userDatabaseEntity.getName()) 14 | .password(userDatabaseEntity.getPassword()) 15 | .build(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /loa-library/loa-user-service/src/main/java/com/github/bottomlessarchive/loa/user/service/domain/UserAlreadyExistsException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.user.service.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class UserAlreadyExistsException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-user-service/src/main/java/com/github/bottomlessarchive/loa/user/service/domain/UserEntity.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.user.service.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | import java.util.UUID; 7 | 8 | @Getter 9 | @Builder 10 | public class UserEntity { 11 | 12 | private final UUID id; 13 | private final String name; 14 | private final String password; 15 | } 16 | -------------------------------------------------------------------------------- /loa-library/loa-validator-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 3 | 4 | implementation project(':loa-library:loa-stage-service') 5 | implementation project(':loa-library:loa-document-parser-service') 6 | } 7 | -------------------------------------------------------------------------------- /loa-library/loa-validator-service/src/main/java/com/github/bottomlessarchive/loa/validator/configuration/FileValidationConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.validator.configuration; 2 | 3 | import jakarta.validation.constraints.Min; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.validation.annotation.Validated; 6 | 7 | /** 8 | * @param maximumArchiveSize Maximum file size to archive in bytes. Documents bigger than this in size are not archived. 9 | */ 10 | @Validated 11 | @ConfigurationProperties("loa.validation") 12 | public record FileValidationConfigurationProperties( 13 | 14 | @Min(0) 15 | long maximumArchiveSize 16 | ) { 17 | 18 | public FileValidationConfigurationProperties(final long maximumArchiveSize) { 19 | if (maximumArchiveSize == 0) { 20 | this.maximumArchiveSize = 8589934592L; 21 | } else { 22 | this.maximumArchiveSize = maximumArchiveSize; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-json' 3 | 4 | implementation group: 'com.squareup.okhttp3', name: 'okhttp' 5 | implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' 6 | 7 | implementation project(':loa-library:loa-document-service') 8 | implementation project(':loa-library:loa-conductor-client-service') 9 | } 10 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/src/main/java/com/github/bottomlessarchive/loa/vault/client/configuration/VaultClientConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.client.configuration; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import lombok.extern.slf4j.Slf4j; 5 | import okhttp3.OkHttpClient; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | 9 | import java.time.Duration; 10 | 11 | @Slf4j 12 | @Configuration 13 | @RequiredArgsConstructor 14 | public class VaultClientConfiguration { 15 | 16 | @Bean 17 | public OkHttpClient vaultOkHttpClient() { 18 | return new OkHttpClient.Builder() 19 | .callTimeout(Duration.ofSeconds(30)) 20 | .connectTimeout(Duration.ofSeconds(30)) 21 | .readTimeout(Duration.ofSeconds(30)) 22 | .writeTimeout(Duration.ofSeconds(30)) 23 | .build(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/src/main/java/com/github/bottomlessarchive/loa/vault/client/service/domain/VaultAccessException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.client.service.domain; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | @StandardException 6 | public class VaultAccessException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/src/main/java/com/github/bottomlessarchive/loa/vault/client/service/domain/VaultLocation.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.client.service.domain; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @Builder 8 | public class VaultLocation { 9 | 10 | private final String location; 11 | private final int port; 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/src/main/java/com/github/bottomlessarchive/loa/vault/client/service/request/RecompressDocumentRequest.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.client.service.request; 2 | 3 | import com.github.bottomlessarchive.loa.compression.domain.DocumentCompression; 4 | import lombok.Builder; 5 | import lombok.Value; 6 | 7 | @Value 8 | @Builder 9 | public class RecompressDocumentRequest { 10 | 11 | DocumentCompression compression; 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/src/main/java/com/github/bottomlessarchive/loa/vault/client/service/response/DocumentExistsResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.client.service.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Value 8 | @Builder 9 | @Jacksonized 10 | public class DocumentExistsResponse { 11 | 12 | boolean exists; 13 | } 14 | -------------------------------------------------------------------------------- /loa-library/loa-vault-client-service/src/main/java/com/github/bottomlessarchive/loa/vault/client/service/response/QueryDocumentResponse.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.client.service.response; 2 | 3 | import lombok.Builder; 4 | import lombok.Value; 5 | import lombok.extern.jackson.Jacksonized; 6 | 7 | @Value 8 | @Builder 9 | @Jacksonized 10 | public class QueryDocumentResponse { 11 | 12 | byte[] payload; 13 | } 14 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation' 3 | 4 | implementation group: 'commons-io', name: 'commons-io', version: '2.16.1' 5 | implementation group: 'software.amazon.awssdk', name: 's3', version: '2.26.15' 6 | 7 | implementation project(':loa-library:loa-file-service') 8 | implementation project(':loa-library:loa-stage-service') 9 | implementation project(':loa-library:loa-number-service') 10 | implementation project(':loa-library:loa-document-service') 11 | implementation project(':loa-library:loa-staging-client-service') 12 | 13 | testImplementation group: 'com.google.jimfs', name: 'jimfs', version: '1.3.0' 14 | } 15 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/domain/VaultType.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.domain; 2 | 3 | public enum VaultType { 4 | 5 | /** 6 | * The vault location is directly on the disks. 7 | */ 8 | FILE, 9 | 10 | /** 11 | * An AWS S3 compatible backend. 12 | * 13 | * @see https://aws.amazon.com/s3/features/?nc=sn&loc=2 14 | */ 15 | S3 16 | } 17 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/domain/exception/StorageAccessException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.domain.exception; 2 | 3 | import lombok.experimental.StandardException; 4 | 5 | /** 6 | * This exception is thrown when the underlying storage is inaccessible. 7 | */ 8 | @StandardException 9 | public class StorageAccessException extends RuntimeException { 10 | } 11 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/service/backend/domain/VaultPersistenceException.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.service.backend.domain; 2 | 3 | /** 4 | * Thrown when the application fails to persist a document to the vault. The reason could vary but 5 | * predominantly IO related. 6 | */ 7 | public class VaultPersistenceException extends RuntimeException { 8 | 9 | public VaultPersistenceException(final String message, final Throwable throwable) { 10 | super(message, throwable); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/service/backend/domain/VaultStorageBackend.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.service.backend.domain; 2 | 3 | public enum VaultStorageBackend { 4 | 5 | DISC, 6 | AWS 7 | } 8 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/service/location/configuration/VaultLocationConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.service.location.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.vault.domain.VaultType; 4 | import jakarta.validation.constraints.NotNull; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.validation.annotation.Validated; 7 | 8 | /** 9 | * Holds the configuration for the vault location. 10 | * 11 | * @param type The type of the vault. 12 | */ 13 | @Validated 14 | @ConfigurationProperties("loa.vault.location") 15 | public record VaultLocationConfigurationProperties( 16 | 17 | @NotNull 18 | VaultType type 19 | ) { 20 | } 21 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/service/location/file/configuration/FileConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.service.location.file.configuration; 2 | 3 | import com.github.bottomlessarchive.loa.vault.domain.VaultType; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | 6 | import java.nio.file.Path; 7 | 8 | /** 9 | * A {@link ConfigurationProperties} class that contains the configuration of the vault location if the vault type 10 | * property is {@link VaultType#FILE} at runtime. 11 | * 12 | * @param path The path to the vault on the local filesystem. 13 | */ 14 | @ConfigurationProperties("loa.vault.location.file") 15 | public record FileConfigurationProperties( 16 | 17 | Path path 18 | ) { 19 | } 20 | -------------------------------------------------------------------------------- /loa-library/loa-vault-service/src/main/java/com/github/bottomlessarchive/loa/vault/service/location/s3/configuration/S3ConfigurationProperties.java: -------------------------------------------------------------------------------- 1 | package com.github.bottomlessarchive.loa.vault.service.location.s3.configuration; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.net.URI; 6 | 7 | @ConfigurationProperties("loa.vault.location.s3") 8 | public record S3ConfigurationProperties( 9 | 10 | String host, 11 | String port, 12 | String accessKey, 13 | String secretKey, 14 | String region, 15 | String bucketName 16 | ) { 17 | 18 | public URI getEndpointLocation() { 19 | return URI.create("http://" + host + ":" + port + "/"); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /lombok.config: -------------------------------------------------------------------------------- 1 | lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Qualifier --------------------------------------------------------------------------------