├── .gitattributes ├── .gitignore ├── .gitlab-ci.yml ├── .gitlab ├── issue_templates │ ├── bug.md │ └── feature.md └── merge_request_templates │ └── standard.md ├── .jvmopts ├── .lab.yml ├── .scalafmt.conf ├── LICENSE ├── README.md ├── RELEASE NOTES.md ├── WASP_SCRUM.md ├── aws └── auth │ └── temporary-credentials │ └── src │ └── main │ └── java │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── aws │ └── auth │ └── v2 │ ├── ConfigurationLoader.java │ ├── CredentialsSerde.java │ ├── DriverSideAssumeRoleCredentialsProvider.java │ ├── ExecutorSideAssumeRoleCredentialsProvider.java │ ├── PlacementAwareCredentialsProvider.java │ ├── ProviderConfiguration.java │ └── WebIdentityProvider.java ├── baseVersion.version ├── build.sbt ├── censor.sh ├── ci └── build-image │ ├── Dockerfile │ └── build-image.sh ├── compiler └── src │ ├── main │ ├── scala-2.11 │ │ └── scala │ │ │ └── tools │ │ │ └── nsc │ │ │ └── symtab │ │ │ └── classfile │ │ │ └── ClassfileParser.scala │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── compiler │ │ └── utils │ │ ├── Compiler.scala │ │ ├── CompilerPool.scala │ │ ├── FreeCodeCompiler.scala │ │ └── Reporter.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── compiler │ └── utils │ └── FreeCodeCompilerTest.scala ├── consumers-spark ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── reference.conf │ ├── scala-spark-3.4 │ │ └── it │ │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── consumers │ │ │ └── spark │ │ │ └── utils │ │ │ └── RowEncoderUtils.scala │ ├── scala-spark-3.5 │ │ └── it │ │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── consumers │ │ │ └── spark │ │ │ └── utils │ │ │ └── RowEncoderUtils.scala │ └── scala │ │ ├── it │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── consumers │ │ │ └── spark │ │ │ ├── MlModels │ │ │ ├── MlModelsDB.scala │ │ │ └── TransformerWithInfo.scala │ │ │ ├── SparkSingletons.scala │ │ │ ├── batch │ │ │ ├── BatchJobActor.scala │ │ │ ├── BatchJobEtlExecution.scala │ │ │ ├── SparkConsumersBatchMasterGuardian.scala │ │ │ └── StartBatchJobSender.scala │ │ │ ├── eventengine │ │ │ ├── EventStrategy.scala │ │ │ └── MailStrategy.scala │ │ │ ├── http │ │ │ ├── Enricher.scala │ │ │ ├── HttpEnricher.scala │ │ │ ├── request │ │ │ │ ├── JacksonToHttpRequest.scala │ │ │ │ └── ToHttpRequest.scala │ │ │ ├── response │ │ │ │ ├── FromHttpResponse.scala │ │ │ │ └── JacksonFromHttpResponse.scala │ │ │ └── utils │ │ │ │ ├── FormattedPathParams.scala │ │ │ │ └── HttpEnricherUtils.scala │ │ │ ├── launcher │ │ │ ├── SparkConsumersBatchNodeLauncher.scala │ │ │ └── SparkConsumersStreamingNodeLauncher.scala │ │ │ ├── metadata │ │ │ └── Metadata.scala │ │ │ ├── plugins │ │ │ └── WaspConsumersSparkPlugin.scala │ │ │ ├── readers │ │ │ ├── SparkReaderFactory.scala │ │ │ └── SparkReaders.scala │ │ │ ├── strategies │ │ │ ├── DropKafkaMetadata.scala │ │ │ ├── EnrichmentStrategy.scala │ │ │ ├── EventIndexingStrategy.scala │ │ │ ├── FreeCodeStrategy.scala │ │ │ ├── HasPostMaterializationHook.scala │ │ │ ├── InternalStrategy.scala │ │ │ ├── Strategy.scala │ │ │ ├── TelemetryIndexingStrategy.scala │ │ │ ├── cdc │ │ │ │ ├── CdcMapper.scala │ │ │ │ ├── DebeziumMutationStrategy.scala │ │ │ │ └── GoldenGateAdapterFlatModelStrategy.scala │ │ │ └── gdpr │ │ │ │ ├── DeletionOutput.scala │ │ │ │ ├── GdprStrategy.scala │ │ │ │ ├── config │ │ │ │ └── DeletionConfig.scala │ │ │ │ ├── exception │ │ │ │ ├── ConfigExceptions.scala │ │ │ │ └── GdprException.scala │ │ │ │ ├── hbase │ │ │ │ └── HBaseDeletionHandler.scala │ │ │ │ ├── hdfs │ │ │ │ ├── HdfsBackupHandler.scala │ │ │ │ ├── HdfsDataDeletion.scala │ │ │ │ └── HdfsDeletionHandler.scala │ │ │ │ └── utils │ │ │ │ ├── ConfigUtils.scala │ │ │ │ ├── GdprUtils.scala │ │ │ │ ├── hbase │ │ │ │ └── HBaseUtils.scala │ │ │ │ └── hdfs │ │ │ │ └── HdfsUtils.scala │ │ │ ├── streaming │ │ │ └── actor │ │ │ │ ├── collaborator │ │ │ │ └── CollaboratorActor.scala │ │ │ │ ├── etl │ │ │ │ ├── ActivationSteps.scala │ │ │ │ ├── Data.scala │ │ │ │ ├── MaterializationSteps.scala │ │ │ │ ├── MetricsTelemetryMessage.scala │ │ │ │ ├── MonitorOutcome.scala │ │ │ │ ├── MonitoringStep.scala │ │ │ │ ├── Protocol.scala │ │ │ │ ├── State.scala │ │ │ │ ├── StoppingStep.scala │ │ │ │ └── StructuredStreamingETLActor.scala │ │ │ │ ├── master │ │ │ │ ├── Data.scala │ │ │ │ ├── DatabaseOperations.scala │ │ │ │ ├── FifoSchedulingStrategy.scala │ │ │ │ ├── FifoSchedulingStrategyFactory.scala │ │ │ │ ├── LowestUUIDNodeSchedulingStrategy.scala │ │ │ │ ├── NodeLabelsSchedulingStrategy.scala │ │ │ │ ├── Protocol.scala │ │ │ │ ├── RetrySupport.scala │ │ │ │ ├── SchedulingStrategy.scala │ │ │ │ ├── SchedulingStrategyFactory.scala │ │ │ │ ├── SparkConsumersStreamingMasterGuardian.scala │ │ │ │ └── State.scala │ │ │ │ ├── pipegraph │ │ │ │ ├── Data.scala │ │ │ │ ├── PipegraphGuardian.scala │ │ │ │ ├── Protocol.scala │ │ │ │ └── State.scala │ │ │ │ ├── telemetry │ │ │ │ ├── CompatibilityTelemetryActor.scala │ │ │ │ └── TelemetryActor.scala │ │ │ │ └── watchdog │ │ │ │ └── SparkContextWatchDog.scala │ │ │ ├── utils │ │ │ ├── AvroDeserializerExpression.scala │ │ │ ├── AvroEncoders.scala │ │ │ ├── AvroSerializerExpression.scala │ │ │ ├── CompatibilityAvroDeserializerExpression.scala │ │ │ ├── CompatibilityAvroSerializerExpression.scala │ │ │ ├── CompatibilityCompressExpression.scala │ │ │ ├── CompatibilityDecodeUsingAvro.scala │ │ │ ├── CompatibilityEncodeUsingAvro.scala │ │ │ ├── CompatibilityEncoders.scala │ │ │ ├── CompressExpression.scala │ │ │ ├── DecodeUsingAvro.scala │ │ │ ├── EncodeUsingAvro.scala │ │ │ ├── HBaseConnection.scala │ │ │ ├── HadoopConfiguration.scala │ │ │ ├── MetadataUtils.scala │ │ │ ├── Quartz2Utils.scala │ │ │ ├── SchemaFlatteners.scala │ │ │ ├── SparkUtils.scala │ │ │ └── ToolBoxUtils.scala │ │ │ └── writers │ │ │ ├── SparkWriterFactory.scala │ │ │ └── SparkWriters.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ ├── CompatibilitySparkHadoopUtil.scala │ │ └── UtilsForwarder.scala │ └── test │ ├── resources │ ├── application.conf │ ├── gdpr │ │ ├── data.txt │ │ ├── data │ │ │ ├── .part-00000-155c21d9-3623-439d-9943-02754fe3bb80-c000.snappy.parquet.crc │ │ │ ├── _SUCCESS │ │ │ └── part-00000-155c21d9-3623-439d-9943-02754fe3bb80-c000.snappy.parquet │ │ ├── datawithdate │ │ │ ├── _SUCCESS │ │ │ ├── category=111 │ │ │ │ ├── .part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc │ │ │ │ └── part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet │ │ │ ├── category=222 │ │ │ │ ├── .part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc │ │ │ │ └── part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet │ │ │ └── category=333 │ │ │ │ ├── .part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc │ │ │ │ └── part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet │ │ └── input │ │ │ └── runId=1 │ │ │ ├── .part-00000-57e1fc7f-43e4-4f77-9fbe-580c63a751a9-c000.snappy.parquet.crc │ │ │ └── part-00000-57e1fc7f-43e4-4f77-9fbe-580c63a751a9-c000.snappy.parquet │ ├── it │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── consumers │ │ │ └── spark │ │ │ └── eventengine │ │ │ ├── inner_event_strategy.conf │ │ │ └── inner_mail_strategy.conf │ ├── log4j2.xml │ └── velocitytemplates │ │ ├── template1.vm │ │ └── template2.vm │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ ├── DatastoreModelsForTesting.scala │ └── consumers │ └── spark │ ├── batch │ └── SparkConsumersBatchMasterGuardianSpec.scala │ ├── eventengine │ ├── FakeData.scala │ ├── InnerEventStrategySpec.scala │ └── InnerMailStrategySpec.scala │ ├── http │ ├── CustomEnricher.scala │ ├── HttpEnricherTest.scala │ ├── data │ │ ├── EnrichedData.scala │ │ ├── FromKafka.scala │ │ └── SampleData.scala │ ├── etl │ │ ├── CustomEnrichmentStrategy.scala │ │ ├── HttpEnrichmentStrategy.scala │ │ ├── StrategiesBLTest.scala │ │ └── package.scala │ ├── package.scala │ ├── request │ │ └── JacksonToHttpRequestTest.scala │ ├── response │ │ └── JacksonFromHttpResponseTest.scala │ ├── topic │ │ └── package.scala │ └── utils │ │ ├── SampleEnrichmentUtil.scala │ │ └── StrategiesUtil.scala │ ├── strategies │ ├── FreeCodeStrategyTest.scala │ ├── PackageTest.scala │ ├── cdc │ │ ├── GoldenGateAdapterFlatModelStrategyTest.scala │ │ └── table │ │ │ └── OrderTable.scala │ └── gdpr │ │ ├── ConfigUtilsSpec.scala │ │ ├── HdfsDataDeletionSpec.scala │ │ └── MockFileSystem.java │ ├── streaming │ └── actor │ │ ├── IntegrationSpec.scala │ │ ├── MockPipegraphBl.scala │ │ ├── MockPipegraphInstanceBl.scala │ │ ├── etl │ │ └── ActivationStepsTest.scala │ │ ├── master │ │ ├── FifoSchedulingStrategySpec.scala │ │ ├── LowestNodeUUIDSchedulingStrategySpec.scala │ │ ├── MultiMasterSpec.scala │ │ ├── NodeLabelsSchedulingStrategySpec.scala │ │ └── SparkConsumersStreamingMasterGuardianSpec.scala │ │ └── pipegraph │ │ └── PipegraphGuardianSpec.scala │ └── utils │ ├── AvroDeserializerExpressionSpec.scala │ ├── AvroEncodersSpec.scala │ ├── AvroSerializerExpressionSpec.scala │ ├── CodegenTester.scala │ ├── CompressExpressionSpec.scala │ ├── RowToAvroExpressionTestDataGenerator.scala │ ├── SparkSuite.scala │ ├── TestClasses.scala │ └── ToolBoxUtilsTest.scala ├── core └── src │ ├── main │ ├── java │ │ ├── it │ │ │ └── agilelab │ │ │ │ └── bigdata │ │ │ │ └── wasp │ │ │ │ └── core │ │ │ │ └── utils │ │ │ │ ├── AvroConversionException.java │ │ │ │ ├── AvroTypeExceptions.java │ │ │ │ ├── CompatibilityJsonGenerator.java │ │ │ │ ├── CompatibilityObjectMapper.java │ │ │ │ ├── JsonAvroConverter.java │ │ │ │ ├── JsonGenericRecordReader.java │ │ │ │ └── SimpleUnionJsonEncoder.java │ │ └── org │ │ │ └── apache │ │ │ └── avro │ │ │ └── io │ │ │ └── NoWrappingJsonEncoder.java │ ├── resources │ │ ├── it_agilelab_bigdata_wasp_core_eventengine_event.avsc │ │ └── reference.conf │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ ├── utils │ │ ├── ClasspathUtils.scala │ │ ├── FileSystemUtils.scala │ │ └── NonEmptyList.scala │ │ └── wasp │ │ ├── consumers │ │ └── spark │ │ │ └── strategies │ │ │ ├── FreeCodeGenerator.scala │ │ │ ├── ReaderKey.scala │ │ │ └── strategies.scala │ │ ├── core │ │ ├── AroundLaunch.scala │ │ ├── SystemPipegraphs.scala │ │ ├── WaspMessage.scala │ │ ├── WaspSystem.scala │ │ ├── cluster │ │ │ └── ClusterListenerActor.scala │ │ ├── consumers │ │ │ └── BaseConsumersMasterGuadian.scala │ │ ├── eventengine │ │ │ ├── Event.scala │ │ │ ├── EventEngineConstants.scala │ │ │ ├── EventTopicModelFactory.scala │ │ │ ├── eventconsumers │ │ │ │ ├── MailingPipegraphModel.scala │ │ │ │ └── MailingRule.scala │ │ │ ├── eventproducers │ │ │ │ ├── EventPipegraphModel.scala │ │ │ │ └── EventRule.scala │ │ │ └── settings │ │ │ │ ├── EventEngineSettings.scala │ │ │ │ └── EventEngineSettingsFactory.scala │ │ ├── exceptions │ │ │ └── ModelNotFound.scala │ │ ├── hbase │ │ │ └── HBaseAdminActor.scala │ │ ├── kafka │ │ │ ├── KafkaAdminMessage.scala │ │ │ ├── NewKafkaAdminActor.scala │ │ │ ├── WaspKafkaReader.scala │ │ │ └── WaspKafkaWriter.scala │ │ ├── launcher │ │ │ ├── ClusterSingletonLauncher.scala │ │ │ ├── MasterCommandLineOptions.scala │ │ │ ├── MultipleClusterSingletonsLauncher.scala │ │ │ ├── WaspCommandLineOptions.scala │ │ │ └── WaspLauncher.scala │ │ ├── logging │ │ │ ├── GuardedLogging.scala │ │ │ ├── Logging.scala │ │ │ └── WaspLogger.scala │ │ ├── messages │ │ │ ├── BatchMessages.scala │ │ │ ├── ClusterMessages.scala │ │ │ ├── ConsumersMessages.scala │ │ │ ├── GenericMessages.scala │ │ │ ├── MasterGuardianMessages.scala │ │ │ ├── PipegraphMessages.scala │ │ │ ├── ProducersMessages.scala │ │ │ ├── TelemetryMessageJsonProtocol.scala │ │ │ └── TelemetryMessages.scala │ │ ├── models │ │ │ └── configuration │ │ │ │ └── ValidationRule.scala │ │ └── utils │ │ │ ├── AvroSchemaConverters.scala │ │ │ ├── AvroToJsonUtil.scala │ │ │ ├── CanOverrideName.scala │ │ │ ├── CanOverrideNameInstances.scala │ │ │ ├── CliUtils.scala │ │ │ ├── ConfUtils.scala │ │ │ ├── ConfigManager.scala │ │ │ ├── FreeCodeCompilerUtils.scala │ │ │ ├── JsonConverter.scala │ │ │ ├── JsonOps.scala │ │ │ ├── StringToByteArrayUtil.scala │ │ │ ├── SubjectUtils.scala │ │ │ ├── TimeFormatter.scala │ │ │ └── Utils.scala │ │ └── spark │ │ └── sql │ │ └── kafka011 │ │ ├── KafkaHeader.scala │ │ └── KafkaSparkSQLSchemas .scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ ├── utils │ └── NonEmptyListSpec.scala │ └── wasp │ └── core │ └── utils │ ├── ConfigManagerSpec.scala │ ├── FreeCodeCompilerUtilsTest.scala │ ├── JsonAvroConverterTest.scala │ └── JsonGenericRecordReaderSpec.scala ├── documentation ├── Architecture.md ├── CONTRIBUTING.md ├── Ext_points.md ├── Getting_started.md ├── Glossary.md ├── Parallel_write.md ├── README.md ├── RELEASE.md ├── SUMMARY.md ├── Services.md ├── Using_WASP.md ├── api.md ├── building.md ├── building │ ├── restart_and_refresh.png │ ├── select_sbt_settings.png │ └── set_vm_parameters.png ├── configuration.md ├── delegation_tokens.md ├── dev.md ├── diagrams │ ├── Wasp1.png │ ├── Wasp2.png │ ├── actor_system.png │ ├── components.png │ ├── pipegraph.png │ ├── pipegraph_model.png │ └── statemachines.png ├── hbase_writer_dynamic_fields.md ├── icons │ ├── WASP_logo.jpg │ ├── WASP_logo.pdf │ └── WASP_logo.png ├── images │ ├── EventEngine.png │ ├── EventEngineOverview.png │ ├── EventPipegraphExample.png │ ├── MailingPipegraphExample.png │ ├── wasp-branching-model.png │ └── wasp-branching-model.svg ├── kafka_integration.md ├── monitoring.md ├── ops.md ├── plain_hbase_writer.md ├── spark-structured-streaming-checkpointing.md ├── testing.md ├── troubleshooting.md ├── wasp-openapi.yaml └── whitelabel │ ├── cdh6.md │ ├── cdh7.md │ ├── cdp.md │ ├── inspection.md │ ├── structure.md │ ├── vanilla2.md │ └── whitelabel.md ├── kernel └── src │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── master ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── reference.conf │ └── scala │ │ ├── akka │ │ └── http │ │ │ └── extension │ │ │ ├── ApiKeyCredentials.scala │ │ │ └── SecurityUtils.scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── master │ │ ├── MasterGuardian.scala │ │ ├── launcher │ │ └── MasterNodeLauncher.scala │ │ ├── security │ │ ├── ApiKeyAuthenticationProvider.scala │ │ ├── ApiKeyAuthenticationVerifier.scala │ │ ├── ApiKeyAuthenticationVerifierImpl.scala │ │ ├── NoSecurity.scala │ │ └── common │ │ │ ├── AuthenticationProvider.scala │ │ │ ├── AuthenticationService.scala │ │ │ ├── CredentialsVerifier.scala │ │ │ └── Identity.scala │ │ └── web │ │ ├── controllers │ │ ├── BatchJobController.scala │ │ ├── BatchJobService.scala │ │ ├── CdcController.scala │ │ ├── Configuration_C.scala │ │ ├── DefaultBatchJobService.scala │ │ ├── Document_C.scala │ │ ├── EditorController.scala │ │ ├── EventController.scala │ │ ├── EventsService.scala │ │ ├── FreeCodeController.scala │ │ ├── FreeCodeDBService.scala │ │ ├── GenericController.scala │ │ ├── Index_C.scala │ │ ├── KeyValueController.scala │ │ ├── LogsController.scala │ │ ├── LogsService.scala │ │ ├── MlModels_C.scala │ │ ├── NifiEditorService.scala │ │ ├── NifiProxyController.scala │ │ ├── PipegraphEditorService.scala │ │ ├── Pipegraph_C.scala │ │ ├── Producer_C.scala │ │ ├── RawController.scala │ │ ├── SolrClient.scala │ │ ├── StatsController.scala │ │ ├── StatsService.scala │ │ ├── Status_C.scala │ │ ├── StrategyController.scala │ │ ├── TelemetryController.scala │ │ ├── TelemetryService.scala │ │ └── Topic_C.scala │ │ ├── models │ │ ├── PaginationInfo.scala │ │ └── RestProducerModel.scala │ │ └── utils │ │ ├── ContentResultsHelper.scala │ │ └── JsonResultsHelper.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── master │ └── web │ └── controllers │ ├── BatchJobControllerSpec.scala │ ├── EditorControllerSpec.scala │ ├── EditorPipegraphControllerSpec.scala │ ├── EventsControllerSpec.scala │ ├── FreeCodeControllerSpec.scala │ ├── LogsControllerSpec.scala │ └── SecurityApiKeyAuthenticationSpec.scala ├── model └── src │ ├── main │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ ├── datastores │ │ └── Datastores.scala │ │ ├── models │ │ ├── BatchJobModel.scala │ │ ├── BatchSchedulerModel.scala │ │ ├── CdcModel.scala │ │ ├── CompletionModel.scala │ │ ├── CountEntry.scala │ │ ├── Counts.scala │ │ ├── DataStoreConf.scala │ │ ├── DatastoreModel.scala │ │ ├── DocumentModel.scala │ │ ├── ErrorModel.scala │ │ ├── EventEntry.scala │ │ ├── FreeCodeModel.scala │ │ ├── GenericCdcMutationFields.scala │ │ ├── GenericModel.scala │ │ ├── HttpModel.scala │ │ ├── IndexModel.scala │ │ ├── IndexModelBuilder.scala │ │ ├── IndexType.scala │ │ ├── KeyValueModel.scala │ │ ├── LogEntry.scala │ │ ├── MetadataModel.scala │ │ ├── MetricsEntry.scala │ │ ├── MlModelOnlyInfo.scala │ │ ├── Model.scala │ │ ├── MultiTopicModel.scala │ │ ├── PipegraphModel.scala │ │ ├── ProcessGroupModel.scala │ │ ├── ProducerModel.scala │ │ ├── RawModel.scala │ │ ├── ReaderModel.scala │ │ ├── SQLSinkModel.scala │ │ ├── SqlSourceModel.scala │ │ ├── StreamingReaderModel.scala │ │ ├── TopicModel.scala │ │ ├── WebMailModel.scala │ │ ├── WebsocketModel.scala │ │ ├── WriterModel.scala │ │ ├── builder │ │ │ ├── KVColumn.scala │ │ │ ├── KVColumnFamily.scala │ │ │ ├── KVSchemaBuilder.scala │ │ │ └── KVType.scala │ │ ├── configuration │ │ │ ├── CompilerConfigModel.scala │ │ │ ├── ConnectionConfig.scala │ │ │ ├── ElasticConfigModel.scala │ │ │ ├── HBaseConfigModel.scala │ │ │ ├── JdbcConnectionConfig.scala │ │ │ ├── KafkaConfigModel.scala │ │ │ ├── MongoDBConfigModel.scala │ │ │ ├── NifiConfigModel.scala │ │ │ ├── ParsingMode.scala │ │ │ ├── PostgresDBConfigModel.scala │ │ │ ├── RestEnrichmentConfigModel.scala │ │ │ ├── SolrConfigModel.scala │ │ │ ├── SparkConfigModel.scala │ │ │ ├── TelemetryConfigModel.scala │ │ │ ├── WaspConfigModel.scala │ │ │ └── ZookeeperConnectionsConfig.scala │ │ └── editor │ │ │ ├── ErrorDTO.scala │ │ │ ├── NifiStatelessInstanceModel.scala │ │ │ └── PipegraphDTO.scala │ │ └── utils │ │ ├── BsonConvertToSprayJson.scala │ │ ├── ConfigManagerHelper.scala │ │ ├── DatastoreProductJsonFormat.scala │ │ ├── DatastoreProductSerde.scala │ │ └── EitherUtils.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── models │ ├── IndexModelBuilderSpec.scala │ ├── MultiTopicModelTest.scala │ └── builder │ ├── KVColumnFamilyBuilderTest.scala │ ├── KVColumnSpec.scala │ └── KVSchemaBuilderTest.scala ├── nifi-client └── src │ ├── main │ └── scala │ │ ├── it │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── nifi │ │ │ └── client │ │ │ ├── Client.scala │ │ │ ├── api │ │ │ ├── AccessApi.scala │ │ │ ├── ConnectionsApi.scala │ │ │ ├── ControllerApi.scala │ │ │ ├── ControllerServicesApi.scala │ │ │ ├── CountersApi.scala │ │ │ ├── DataTransferApi.scala │ │ │ ├── EnumsSerializers.scala │ │ │ ├── FlowApi.scala │ │ │ ├── FlowfileQueuesApi.scala │ │ │ ├── FunnelApi.scala │ │ │ ├── InputPortsApi.scala │ │ │ ├── LabelsApi.scala │ │ │ ├── OutputPortsApi.scala │ │ │ ├── ParameterContextsApi.scala │ │ │ ├── PoliciesApi.scala │ │ │ ├── ProcessGroupsApi.scala │ │ │ ├── ProcessorsApi.scala │ │ │ ├── ProvenanceApi.scala │ │ │ ├── ProvenanceEventsApi.scala │ │ │ ├── RemoteProcessGroupsApi.scala │ │ │ ├── ReportingTasksApi.scala │ │ │ ├── ResourcesApi.scala │ │ │ ├── SiteToSiteApi.scala │ │ │ ├── SnippetsApi.scala │ │ │ ├── SystemDiagnosticsApi.scala │ │ │ ├── TemplatesApi.scala │ │ │ ├── TenantsApi.scala │ │ │ └── VersionsApi.scala │ │ │ ├── core │ │ │ ├── ApiInvoker.scala │ │ │ ├── Serializers.scala │ │ │ └── requests.scala │ │ │ └── model │ │ │ ├── AboutDTO.scala │ │ │ ├── AboutEntity.scala │ │ │ ├── AccessConfigurationDTO.scala │ │ │ ├── AccessConfigurationEntity.scala │ │ │ ├── AccessPolicyDTO.scala │ │ │ ├── AccessPolicyEntity.scala │ │ │ ├── AccessPolicySummaryDTO.scala │ │ │ ├── AccessPolicySummaryEntity.scala │ │ │ ├── AccessStatusDTO.scala │ │ │ ├── AccessStatusEntity.scala │ │ │ ├── ActionDTO.scala │ │ │ ├── ActionEntity.scala │ │ │ ├── ActivateControllerServicesEntity.scala │ │ │ ├── AffectedComponentDTO.scala │ │ │ ├── AffectedComponentEntity.scala │ │ │ ├── AllowableValueDTO.scala │ │ │ ├── AllowableValueEntity.scala │ │ │ ├── AttributeDTO.scala │ │ │ ├── BannerDTO.scala │ │ │ ├── BannerEntity.scala │ │ │ ├── BatchSettingsDTO.scala │ │ │ ├── BatchSize.scala │ │ │ ├── Bucket.scala │ │ │ ├── BucketDTO.scala │ │ │ ├── BucketEntity.scala │ │ │ ├── BucketsEntity.scala │ │ │ ├── BulletinBoardDTO.scala │ │ │ ├── BulletinBoardEntity.scala │ │ │ ├── BulletinDTO.scala │ │ │ ├── BulletinEntity.scala │ │ │ ├── Bundle.scala │ │ │ ├── BundleDTO.scala │ │ │ ├── ClusteSummaryEntity.scala │ │ │ ├── ClusterDTO.scala │ │ │ ├── ClusterEntity.scala │ │ │ ├── ClusterSearchResultsEntity.scala │ │ │ ├── ClusterSummaryDTO.scala │ │ │ ├── ComponentDifferenceDTO.scala │ │ │ ├── ComponentHistoryDTO.scala │ │ │ ├── ComponentHistoryEntity.scala │ │ │ ├── ComponentReferenceDTO.scala │ │ │ ├── ComponentReferenceEntity.scala │ │ │ ├── ComponentRestrictionPermissionDTO.scala │ │ │ ├── ComponentSearchResultDTO.scala │ │ │ ├── ComponentStateDTO.scala │ │ │ ├── ComponentStateEntity.scala │ │ │ ├── ComponentValidationResultDTO.scala │ │ │ ├── ComponentValidationResultEntity.scala │ │ │ ├── ComponentValidationResultsEntity.scala │ │ │ ├── ConnectableComponent.scala │ │ │ ├── ConnectableDTO.scala │ │ │ ├── ConnectionDTO.scala │ │ │ ├── ConnectionEntity.scala │ │ │ ├── ConnectionStatisticsDTO.scala │ │ │ ├── ConnectionStatisticsEntity.scala │ │ │ ├── ConnectionStatisticsSnapshotDTO.scala │ │ │ ├── ConnectionStatusDTO.scala │ │ │ ├── ConnectionStatusEntity.scala │ │ │ ├── ConnectionStatusPredictionsSnapshotDTO.scala │ │ │ ├── ConnectionStatusSnapshotDTO.scala │ │ │ ├── ConnectionStatusSnapshotEntity.scala │ │ │ ├── ConnectionsEntity.scala │ │ │ ├── ControllerBulletinsEntity.scala │ │ │ ├── ControllerConfigurationDTO.scala │ │ │ ├── ControllerConfigurationEntity.scala │ │ │ ├── ControllerDTO.scala │ │ │ ├── ControllerEntity.scala │ │ │ ├── ControllerServiceAPI.scala │ │ │ ├── ControllerServiceApiDTO.scala │ │ │ ├── ControllerServiceDTO.scala │ │ │ ├── ControllerServiceEntity.scala │ │ │ ├── ControllerServiceReferencingComponentDTO.scala │ │ │ ├── ControllerServiceReferencingComponentEntity.scala │ │ │ ├── ControllerServiceReferencingComponentsEntity.scala │ │ │ ├── ControllerServiceRunStatusEntity.scala │ │ │ ├── ControllerServiceStatusDTO.scala │ │ │ ├── ControllerServiceTypesEntity.scala │ │ │ ├── ControllerServicesEntity.scala │ │ │ ├── ControllerStatusDTO.scala │ │ │ ├── ControllerStatusEntity.scala │ │ │ ├── CopySnippetRequestEntity.scala │ │ │ ├── CounterDTO.scala │ │ │ ├── CounterEntity.scala │ │ │ ├── CountersDTO.scala │ │ │ ├── CountersEntity.scala │ │ │ ├── CountersSnapshotDTO.scala │ │ │ ├── CreateActiveRequestEntity.scala │ │ │ ├── CreateTemplateRequestEntity.scala │ │ │ ├── CurrentUserEntity.scala │ │ │ ├── DifferenceDTO.scala │ │ │ ├── DimensionsDTO.scala │ │ │ ├── DocumentedTypeDTO.scala │ │ │ ├── DropRequestDTO.scala │ │ │ ├── DropRequestEntity.scala │ │ │ ├── ExplicitRestrictionDTO.scala │ │ │ ├── ExternalControllerServiceReference.scala │ │ │ ├── FlowBreadcrumbDTO.scala │ │ │ ├── FlowBreadcrumbEntity.scala │ │ │ ├── FlowComparisonEntity.scala │ │ │ ├── FlowConfigurationDTO.scala │ │ │ ├── FlowConfigurationEntity.scala │ │ │ ├── FlowDTO.scala │ │ │ ├── FlowEntity.scala │ │ │ ├── FlowFileDTO.scala │ │ │ ├── FlowFileEntity.scala │ │ │ ├── FlowFileSummaryDTO.scala │ │ │ ├── FlowSnippetDTO.scala │ │ │ ├── FunnelDTO.scala │ │ │ ├── FunnelEntity.scala │ │ │ ├── FunnelsEntity.scala │ │ │ ├── GarbageCollectionDTO.scala │ │ │ ├── HistoryDTO.scala │ │ │ ├── HistoryEntity.scala │ │ │ ├── InputPortsEntity.scala │ │ │ ├── InstantiateTemplateRequestEntity.scala │ │ │ ├── JaxbLink.scala │ │ │ ├── LabelDTO.scala │ │ │ ├── LabelEntity.scala │ │ │ ├── LabelsEntity.scala │ │ │ ├── LineageDTO.scala │ │ │ ├── LineageEntity.scala │ │ │ ├── LineageRequestDTO.scala │ │ │ ├── LineageResultsDTO.scala │ │ │ ├── ListingRequestDTO.scala │ │ │ ├── ListingRequestEntity.scala │ │ │ ├── NodeConnectionStatisticsSnapshotDTO.scala │ │ │ ├── NodeConnectionStatusSnapshotDTO.scala │ │ │ ├── NodeCountersSnapshotDTO.scala │ │ │ ├── NodeDTO.scala │ │ │ ├── NodeEntity.scala │ │ │ ├── NodeEventDTO.scala │ │ │ ├── NodePortStatusSnapshotDTO.scala │ │ │ ├── NodeProcessGroupStatusSnapshotDTO.scala │ │ │ ├── NodeProcessorStatusSnapshotDTO.scala │ │ │ ├── NodeRemoteProcessGroupStatusSnapshotDTO.scala │ │ │ ├── NodeSearchResultDTO.scala │ │ │ ├── NodeStatusSnapshotsDTO.scala │ │ │ ├── NodeSystemDiagnosticsSnapshotDTO.scala │ │ │ ├── OutputPortsEntity.scala │ │ │ ├── ParameterContextDTO.scala │ │ │ ├── ParameterContextEntity.scala │ │ │ ├── ParameterContextReferenceDTO.scala │ │ │ ├── ParameterContextReferenceEntity.scala │ │ │ ├── ParameterContextUpdateRequestDTO.scala │ │ │ ├── ParameterContextUpdateRequestEntity.scala │ │ │ ├── ParameterContextUpdateStepDTO.scala │ │ │ ├── ParameterContextValidationRequestDTO.scala │ │ │ ├── ParameterContextValidationRequestEntity.scala │ │ │ ├── ParameterContextValidationStepDTO.scala │ │ │ ├── ParameterContextsEntity.scala │ │ │ ├── ParameterDTO.scala │ │ │ ├── ParameterEntity.scala │ │ │ ├── PeerDTO.scala │ │ │ ├── PeersEntity.scala │ │ │ ├── Permissions.scala │ │ │ ├── PermissionsDTO.scala │ │ │ ├── PortDTO.scala │ │ │ ├── PortEntity.scala │ │ │ ├── PortRunStatusEntity.scala │ │ │ ├── PortStatusDTO.scala │ │ │ ├── PortStatusEntity.scala │ │ │ ├── PortStatusSnapshotDTO.scala │ │ │ ├── PortStatusSnapshotEntity.scala │ │ │ ├── Position.scala │ │ │ ├── PositionDTO.scala │ │ │ ├── PreviousValueDTO.scala │ │ │ ├── PrioritizerTypesEntity.scala │ │ │ ├── ProcessGroupDTO.scala │ │ │ ├── ProcessGroupEntity.scala │ │ │ ├── ProcessGroupFlowDTO.scala │ │ │ ├── ProcessGroupFlowEntity.scala │ │ │ ├── ProcessGroupNameDTO.scala │ │ │ ├── ProcessGroupStatusDTO.scala │ │ │ ├── ProcessGroupStatusEntity.scala │ │ │ ├── ProcessGroupStatusSnapshotDTO.scala │ │ │ ├── ProcessGroupStatusSnapshotEntity.scala │ │ │ ├── ProcessGroupsEntity.scala │ │ │ ├── ProcessorConfigDTO.scala │ │ │ ├── ProcessorDTO.scala │ │ │ ├── ProcessorEntity.scala │ │ │ ├── ProcessorRunStatusEntity.scala │ │ │ ├── ProcessorStatusDTO.scala │ │ │ ├── ProcessorStatusEntity.scala │ │ │ ├── ProcessorStatusSnapshotDTO.scala │ │ │ ├── ProcessorStatusSnapshotEntity.scala │ │ │ ├── ProcessorTypesEntity.scala │ │ │ ├── ProcessorsEntity.scala │ │ │ ├── PropertyDescriptorDTO.scala │ │ │ ├── PropertyDescriptorEntity.scala │ │ │ ├── PropertyHistoryDTO.scala │ │ │ ├── ProvenanceDTO.scala │ │ │ ├── ProvenanceEntity.scala │ │ │ ├── ProvenanceEventDTO.scala │ │ │ ├── ProvenanceEventEntity.scala │ │ │ ├── ProvenanceLinkDTO.scala │ │ │ ├── ProvenanceNodeDTO.scala │ │ │ ├── ProvenanceOptionsDTO.scala │ │ │ ├── ProvenanceOptionsEntity.scala │ │ │ ├── ProvenanceRequestDTO.scala │ │ │ ├── ProvenanceResultsDTO.scala │ │ │ ├── ProvenanceSearchableFieldDTO.scala │ │ │ ├── QueueSizeDTO.scala │ │ │ ├── RegistryClientEntity.scala │ │ │ ├── RegistryClientsEntity.scala │ │ │ ├── RegistryDTO.scala │ │ │ ├── RelationshipDTO.scala │ │ │ ├── RemotePortRunStatusEntity.scala │ │ │ ├── RemoteProcessGroupContentsDTO.scala │ │ │ ├── RemoteProcessGroupDTO.scala │ │ │ ├── RemoteProcessGroupEntity.scala │ │ │ ├── RemoteProcessGroupPortDTO.scala │ │ │ ├── RemoteProcessGroupPortEntity.scala │ │ │ ├── RemoteProcessGroupStatusDTO.scala │ │ │ ├── RemoteProcessGroupStatusEntity.scala │ │ │ ├── RemoteProcessGroupStatusSnapshotDTO.scala │ │ │ ├── RemoteProcessGroupStatusSnapshotEntity.scala │ │ │ ├── RemoteProcessGroupsEntity.scala │ │ │ ├── ReportingTaskDTO.scala │ │ │ ├── ReportingTaskEntity.scala │ │ │ ├── ReportingTaskRunStatusEntity.scala │ │ │ ├── ReportingTaskStatusDTO.scala │ │ │ ├── ReportingTaskTypesEntity.scala │ │ │ ├── ReportingTasksEntity.scala │ │ │ ├── RequiredPermissionDTO.scala │ │ │ ├── ResourceDTO.scala │ │ │ ├── ResourcesEntity.scala │ │ │ ├── RevisionDTO.scala │ │ │ ├── Runner.scala │ │ │ ├── ScheduleComponentsEntity.scala │ │ │ ├── SearchResultGroupDTO.scala │ │ │ ├── SearchResultsDTO.scala │ │ │ ├── SearchResultsEntity.scala │ │ │ ├── SnippetDTO.scala │ │ │ ├── SnippetEntity.scala │ │ │ ├── StartVersionControlRequestEntity.scala │ │ │ ├── StateEntryDTO.scala │ │ │ ├── StateMapDTO.scala │ │ │ ├── StatusDescriptorDTO.scala │ │ │ ├── StatusHistoryDTO.scala │ │ │ ├── StatusHistoryEntity.scala │ │ │ ├── StatusSnapshotDTO.scala │ │ │ ├── StorageUsageDTO.scala │ │ │ ├── SubmitReplayRequestEntity.scala │ │ │ ├── SystemDiagnosticsDTO.scala │ │ │ ├── SystemDiagnosticsEntity.scala │ │ │ ├── SystemDiagnosticsSnapshotDTO.scala │ │ │ ├── TemplateDTO.scala │ │ │ ├── TemplateEntity.scala │ │ │ ├── TemplatesEntity.scala │ │ │ ├── TenantDTO.scala │ │ │ ├── TenantEntity.scala │ │ │ ├── TenantsEntity.scala │ │ │ ├── TransactionResultEntity.scala │ │ │ ├── UpdateControllerServiceReferenceRequestEntity.scala │ │ │ ├── UserDTO.scala │ │ │ ├── UserEntity.scala │ │ │ ├── UserGroupDTO.scala │ │ │ ├── UserGroupEntity.scala │ │ │ ├── UserGroupsEntity.scala │ │ │ ├── UsersEntity.scala │ │ │ ├── VariableDTO.scala │ │ │ ├── VariableEntity.scala │ │ │ ├── VariableRegistryDTO.scala │ │ │ ├── VariableRegistryEntity.scala │ │ │ ├── VariableRegistryUpdateRequestDTO.scala │ │ │ ├── VariableRegistryUpdateRequestEntity.scala │ │ │ ├── VariableRegistryUpdateStepDTO.scala │ │ │ ├── VersionControlComponentMappingEntity.scala │ │ │ ├── VersionControlInformationDTO.scala │ │ │ ├── VersionControlInformationEntity.scala │ │ │ ├── VersionInfoDTO.scala │ │ │ ├── VersionedConnection.scala │ │ │ ├── VersionedControllerService.scala │ │ │ ├── VersionedFlow.scala │ │ │ ├── VersionedFlowCoordinates.scala │ │ │ ├── VersionedFlowDTO.scala │ │ │ ├── VersionedFlowEntity.scala │ │ │ ├── VersionedFlowSnapshot.scala │ │ │ ├── VersionedFlowSnapshotEntity.scala │ │ │ ├── VersionedFlowSnapshotMetadata.scala │ │ │ ├── VersionedFlowSnapshotMetadataEntity.scala │ │ │ ├── VersionedFlowSnapshotMetadataSetEntity.scala │ │ │ ├── VersionedFlowUpdateRequestDTO.scala │ │ │ ├── VersionedFlowUpdateRequestEntity.scala │ │ │ ├── VersionedFlowsEntity.scala │ │ │ ├── VersionedFunnel.scala │ │ │ ├── VersionedLabel.scala │ │ │ ├── VersionedParameter.scala │ │ │ ├── VersionedParameterContext.scala │ │ │ ├── VersionedPort.scala │ │ │ ├── VersionedProcessGroup.scala │ │ │ ├── VersionedProcessor.scala │ │ │ ├── VersionedPropertyDescriptor.scala │ │ │ ├── VersionedRemoteGroupPort.scala │ │ │ └── VersionedRemoteProcessGroup.scala │ │ └── sttp │ │ └── client │ │ └── akkahttp │ │ ├── AkkaHttpBackend.scala │ │ └── AkkaHttpClient.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── openapi └── src │ ├── main │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── master │ │ └── web │ │ └── openapi │ │ ├── AngularResponse.scala │ │ ├── AngularResponseOpenApiComponentSupport.scala │ │ ├── BatchJobRoutesOpenApiDefinition.scala │ │ ├── BatchJobStartResult.scala │ │ ├── BatchOpenApiComponentsSupport.scala │ │ ├── BsonDocumentOpenApiDefinition.scala │ │ ├── CollectionsOpenApi.scala │ │ ├── ConfigRoutesOpenApiDefinition.scala │ │ ├── DataStoreOpenApiComponentSupport.scala │ │ ├── DocumentRoutesOpenApiDefinition.scala │ │ ├── EditorOpenApiComponentSupport.scala │ │ ├── EditorRoutesOpenApiDefinition.scala │ │ ├── EnumOpenApi.scala │ │ ├── EventsRoutesOpenApiDefinition.scala │ │ ├── FreeCodeModelOpenApiSupport.scala │ │ ├── FreeCodeRoutesOpenApiDefinition.scala │ │ ├── GenerateOpenApi.scala │ │ ├── GenerateProductFunctions.scala │ │ ├── GenericRoutesOpenApiDefinition.scala │ │ ├── IndicesRoutesOpenApiDefinition.scala │ │ ├── KeyValueRoutesOpenApiDefinition.scala │ │ ├── LangOpenApi.scala │ │ ├── LogsRoutesOpenApiDefinition.scala │ │ ├── MlModelOnlyInfoComponentSupport.scala │ │ ├── MlModelsRoutesOpenApiDefinition.scala │ │ ├── OpenApiRenderer.scala │ │ ├── OpenApiSchemaSupport.scala │ │ ├── PipegraphOpenApiComponentSupport.scala │ │ ├── PipegraphRoutesOpenApiDefinition.scala │ │ ├── ProducerOpenApiComponentSupport.scala │ │ ├── ProducersRoutesOpenApiDefinition.scala │ │ ├── ProductOpenApi.scala │ │ ├── RawRoutesOpenApiDefinition.scala │ │ ├── ReaderModelOpenApiComponentSupport.scala │ │ ├── ResultIndicator.scala │ │ ├── Schemas.scala │ │ ├── StatsRoutesOpenApi.scala │ │ ├── StrategyModelOpenApiComponentSupport.scala │ │ ├── StrategyOpenApiDefinition.scala │ │ ├── TelemetryRoutesOpenApiDefinition.scala │ │ ├── ToOpenApiSchema.scala │ │ ├── TopicModelOpenApiComponentSupport.scala │ │ ├── TopicRoutesOpenApiDefinition.scala │ │ └── WriterModelOpenApiComponentSupport.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── openshift ├── batch │ ├── Dockerfile │ ├── coursier.exclusions │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── etc │ │ ├── hadoop │ │ │ └── conf │ │ │ │ ├── core-site.xml │ │ │ │ └── yarn-site.xml │ │ └── hbase │ │ │ └── conf │ │ │ └── hbase-site.xml │ ├── licenses.csv │ ├── log4j2.properties │ └── wasp-license ├── docker-compose.yml ├── master │ ├── Dockerfile │ ├── coursier.exclusions │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── etc │ │ └── hbase │ │ │ └── conf │ │ │ └── hbase-site.xml │ ├── licenses.csv │ ├── log4j2.properties │ └── wasp-license ├── mongodb │ ├── Dockerfile │ ├── README.md │ ├── docker-entrypoint.sh │ ├── mongo.repo │ └── mongodb.txt ├── producers │ ├── Dockerfile │ ├── coursier.exclusions │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── etc │ │ └── hbase │ │ │ └── conf │ │ │ └── hbase-site.xml │ ├── licenses.csv │ ├── log4j2.properties │ └── wasp-license ├── services │ ├── Dockerfile │ └── docker-entrypoint.sh └── streaming │ ├── Dockerfile │ ├── coursier.exclusions │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── core-site.xml │ │ │ └── yarn-site.xml │ └── hbase │ │ └── conf │ │ └── hbase-site.xml │ ├── licenses.csv │ ├── log4j2.properties │ └── wasp-license ├── plugin-cdc-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── cdc │ │ ├── CdcConsumersSpark.scala │ │ ├── CdcSparkStructuredStreamingWriter.scala │ │ └── DeltaLakeWriter.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── cdc │ ├── CdcSparkStructuredStreamingTest.scala │ └── TempDirectoryEach.scala ├── plugin-console-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── console │ │ ├── ConsoleConsumersSpark.scala │ │ └── ConsoleWriters.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── plugin-elastic-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── elastic │ │ ├── ElasticAdminActor.scala │ │ ├── ElasticAdminMessage.scala │ │ ├── ElasticConsumersSpark.scala │ │ ├── ElasticRestClient.scala │ │ ├── ElasticWriters.scala │ │ ├── ElasticsearchSparkBatchReader.scala │ │ └── HighLevelElasticRestClient.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── plugin-hbase-spark └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── hadoop │ │ │ └── hbase │ │ │ └── spark │ │ │ └── SparkSQLPushDownFilter.java │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ ├── it │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── consumers │ │ │ └── spark │ │ │ └── plugins │ │ │ └── hbase │ │ │ ├── HBaseAdminActor.scala │ │ │ ├── HBaseConsumerSpark.scala │ │ │ ├── HBaseSparkBatchReader.scala │ │ │ └── HBaseWriters.scala │ │ └── org │ │ └── apache │ │ ├── hadoop │ │ └── hbase │ │ │ ├── HBaseInterfaceAudienceSpark.java │ │ │ └── spark │ │ │ ├── BulkLoadPartitioner.scala │ │ │ ├── ByteArrayComparable.scala │ │ │ ├── ByteArrayWrapper.scala │ │ │ ├── ColumnFamilyQualifierMapKeyWrapper.scala │ │ │ ├── DefaultSource.scala │ │ │ ├── DynamicFieldStructure.scala │ │ │ ├── DynamicLogicExpression.scala │ │ │ ├── FamiliesQualifiersValues.scala │ │ │ ├── FamilyHFileWriteOptions.scala │ │ │ ├── HBaseConnectionCache.scala │ │ │ ├── HBaseContext.scala │ │ │ ├── HBaseRDDFunctions.scala │ │ │ ├── HBaseSink.scala │ │ │ ├── JavaHBaseContext.scala │ │ │ ├── KeyFamilyQualifier.scala │ │ │ ├── NewHBaseRDD.scala │ │ │ ├── PutConverterFactory.scala │ │ │ ├── datasources │ │ │ ├── Bound.scala │ │ │ ├── HBaseResources.scala │ │ │ ├── HBaseSparkConf.scala │ │ │ ├── HBaseTableScanRDD.scala │ │ │ ├── JavaBytesEncoder.scala │ │ │ ├── NaiveEncoder.scala │ │ │ ├── SchemaConverters.scala │ │ │ ├── SerDes.scala │ │ │ ├── SerializableConfiguration.scala │ │ │ └── package.scala │ │ │ └── example │ │ │ ├── datasources │ │ │ ├── AvroSource.scala │ │ │ ├── DataType.scala │ │ │ └── HBaseSource.scala │ │ │ ├── hbasecontext │ │ │ ├── HBaseBulkDeleteExample.scala │ │ │ ├── HBaseBulkGetExample.scala │ │ │ ├── HBaseBulkPutExample.scala │ │ │ ├── HBaseBulkPutExampleFromFile.scala │ │ │ ├── HBaseBulkPutTimestampExample.scala │ │ │ └── HBaseDistributedScanExample.scala │ │ │ └── rdd │ │ │ ├── HBaseBulkDeleteExample.scala │ │ │ ├── HBaseBulkGetExample.scala │ │ │ ├── HBaseBulkPutExample.scala │ │ │ ├── HBaseForeachPartitionExample.scala │ │ │ └── HBaseMapPartitionExample.scala │ │ └── spark │ │ └── sql │ │ └── datasources │ │ └── hbase │ │ ├── DataTypeParserWrapper.scala │ │ ├── HBaseTableCatalog.scala │ │ └── Utils.scala │ ├── protobuf │ └── SparkFilter.proto │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── org │ └── apache │ ├── hadoop │ └── hbase │ │ └── spark │ │ └── HBaseContextSerializationSuite.scala │ └── spark │ └── sql │ └── datasources │ └── hbase │ └── HBaseTableCatalogTest.scala ├── plugin-http-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── http │ │ ├── HttpConsumersSparkPlugin.scala │ │ ├── HttpWaspWriter.scala │ │ └── HttpWriter.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── http │ ├── HttpTestUtils.scala │ ├── HttpWaspWriterSpec.scala │ ├── HttpWriterSpec.scala │ └── WebAssertions.scala ├── plugin-jdbc-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── jdbc │ │ ├── JDBCSparkBatchReader.scala │ │ └── JdbcConsumerSpark.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── plugin-kafka-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── kafka │ │ ├── KafkaBatchWriter.scala │ │ ├── KafkaConsumersSpark.scala │ │ ├── KafkaSparkStructuredStreamingReader.scala │ │ ├── KafkaSparkStructuredStreamingWriter.scala │ │ ├── KafkaWriters.scala │ │ └── TopicModelUtils.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── kafka │ ├── ConvertDataframeTest.scala │ └── KafkaSparkStructuredStreamingReaderSpec.scala ├── plugin-mailer-spark └── src │ └── main │ ├── resources │ └── META-INF │ │ └── services │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── mailer │ ├── Mail.scala │ ├── MailAgent.scala │ ├── MailWriter.scala │ └── MailerConsumerSpark.scala ├── plugin-mongo-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ ├── com │ │ └── mongodb │ │ │ └── spark │ │ │ └── sql │ │ │ └── MongoForeachRddWriter.scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── mongo │ │ ├── MongoConsumersSpark.scala │ │ ├── MongoSparkBatchReader.scala │ │ ├── MongoSparkBatchWriter.scala │ │ └── MongoSparkStructuredStreamingWriter.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── plugin-parallel-write-spark └── src │ ├── main │ ├── resources │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ │ └── reference.conf │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── parallel │ │ ├── ParallelWriteConsumersSpark.scala │ │ ├── ParallelWriteWriters.scala │ │ ├── catalog │ │ ├── EntityCatalogBuilder.scala │ │ ├── EntityCatalogService.scala │ │ ├── MicroserviceClient.scala │ │ └── entity │ │ │ ├── EntityApi.scala │ │ │ └── ParallelWriteEntity.scala │ │ ├── model │ │ ├── ParallelWriteModel.scala │ │ ├── ParallelWriteModelParser.scala │ │ └── WriterDetails.scala │ │ ├── utils │ │ ├── CommitStatus.scala │ │ ├── DataCatalogService.scala │ │ ├── DataframeSchemaUtils.scala │ │ ├── HadoopS3Utils.scala │ │ ├── MetastoreCatalogService.scala │ │ └── SchemaChecker.scala │ │ └── writers │ │ ├── ColdAreaCredentialsPersister.scala │ │ ├── ColdParallelWriter.scala │ │ ├── ContinuousUpdateWriter.scala │ │ ├── CredentialsConfigurator.scala │ │ ├── DeltaParallelWriter.scala │ │ ├── DeltaParallelWriterTrait.scala │ │ ├── HotParallelWriter.scala │ │ ├── ParallelWriter.scala │ │ ├── ParallelWriterFactory.scala │ │ └── ParquetParallelWriter.scala │ └── test │ ├── resources │ └── reference.conf │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── parallel │ └── tools │ ├── ColdAreaCredentialsPersisterSpec.scala │ ├── ContinuousUpdateWriterSpec.scala │ ├── DeltaParallelWriterSpec.scala │ ├── HotParallelWriterSpec.scala │ ├── MockCredentialProvider.scala │ ├── ParallelWriteModelParserSpec.scala │ ├── ParallelWriterSpec.scala │ ├── TestModels.scala │ ├── catalog │ ├── MicroserviceCatalogBuilderSpec.scala │ └── builders │ │ ├── catalogservices │ │ ├── MSEntityCatalogService.scala │ │ ├── MockPlatformCatalogService.scala │ │ ├── ParameterConstructorService.scala │ │ └── WrongCatalogService.scala │ │ └── mockbuilders │ │ ├── MSEntityMockBuilder.scala │ │ ├── NotExistingServiceBuilder.scala │ │ ├── ParameterBuilder.scala │ │ ├── RightMockBuilder.scala │ │ ├── WrongCatalogMockBuilder.scala │ │ ├── WrongConfigurationPathBuilder.scala │ │ └── WrongMicroserviceMockBuilder.scala │ └── utils │ ├── DataframeSchemaUtilsTest.scala │ ├── DeltaTableTest.scala │ ├── MetastoreCatalogTableNameBuilder.scala │ ├── MockCatalogService.scala │ ├── ParallelWriteTest.scala │ ├── ParallelWriteTestUtils.scala │ ├── SchemaCheckerTest.scala │ ├── TempDirectoryTest.scala │ └── WebAssertions.scala ├── plugin-plain-hbase-writer-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── plain │ │ └── hbase │ │ ├── PlainHBaseWriterConsumer.scala │ │ └── integration │ │ ├── DefaultSource.scala │ │ ├── HBaseConnectionCache.scala │ │ ├── HBaseContext.scala │ │ ├── HBaseCredentialsManager.scala │ │ ├── HBaseSparkConf.scala │ │ ├── HBaseTableCatalog.scala │ │ └── sink │ │ ├── HBaseSink.scala │ │ ├── HBaseWriter.scala │ │ └── HBaseWriterTask.scala │ └── test │ ├── resources │ ├── hbase │ │ ├── hbase-policy-local.xml │ │ └── hbase-site-local.xml │ ├── log4j.properties │ └── mockito-extensions │ │ └── org.mockito.plugins.MockMaker │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── plain │ └── hbase │ └── integration │ ├── DefaultSourceTest.scala │ ├── TestFixture.scala │ └── sink │ └── HBaseWriterTest.scala ├── plugin-postgresql-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── postgresql │ │ ├── JDBCConnectionInfoProvider.scala │ │ ├── JDBCMetadataFetcherService.scala │ │ ├── JDBCMetadataFetcherServiceImpl.scala │ │ ├── JDBCPooledConnectionSupport.scala │ │ ├── PostgreSQLConsumerSparkPlugin.scala │ │ ├── PostgreSQLSparkBaseWriter.scala │ │ ├── PostgreSQLSparkBatchWriter.scala │ │ ├── PostgreSQLSparkStructuredStreamingWriter.scala │ │ ├── PostgreSQLUpsertGenerationSupport.scala │ │ ├── PostgreSQLUpsertWriter.scala │ │ └── TableMetadata.scala │ └── test │ ├── resources │ └── log4j2.properties │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── postgresql │ ├── PostgreSQLBatchWriterSpec.scala │ ├── PostgreSQLPluginSpec.scala │ ├── PostgreSQLStructuredStreamingWriterSpec.scala │ ├── PostgresSuite.scala │ ├── PostgresTestSupport.scala │ └── TestData.scala ├── plugin-raw-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── raw │ │ ├── RawConsumersSpark.scala │ │ ├── RawSparkBatchReader.scala │ │ ├── RawSparkReaderUtils.scala │ │ ├── RawSparkStructuredStreamingReader.scala │ │ ├── RawWriters.scala │ │ └── tools │ │ ├── FolderCompaction.scala │ │ ├── FolderCompactionUtils.scala │ │ ├── PartitionColumnValues.scala │ │ ├── PartitionDiscoveryUtils.scala │ │ └── WhereCondition.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── consumers │ └── spark │ └── plugins │ └── raw │ ├── RawConsumersSparkSpec.scala │ └── tools │ ├── FolderCompactionSpec.scala │ ├── FolderCompactionUtilsSpec.scala │ └── PartitionDiscoveryUtilsSpec.scala ├── plugin-solr-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── consumers │ │ └── spark │ │ └── plugins │ │ └── solr │ │ ├── SolrAdminActor.scala │ │ ├── SolrAdminMessage.scala │ │ ├── SolrConsumersSpark.scala │ │ ├── SolrSparkBatchReader.scala │ │ └── SolrWriters.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── plugin └── src │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── producers ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── reference.conf │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ ├── data │ │ ├── RingBuffer.scala │ │ └── RingBufferQueue.scala │ │ └── producers │ │ ├── InternalLogProducerGuardian.scala │ │ ├── KafkaProducer.scala │ │ ├── NifiProducerGuardian.scala │ │ ├── NifiProducerModel.scala │ │ ├── NifiRequestJsonProtocol.scala │ │ ├── ProducerActor.scala │ │ ├── ProducerGuardian.scala │ │ ├── ProducersMasterGuardian.scala │ │ ├── launcher │ │ ├── NifiProducersNodeLauncher.scala │ │ └── ProducersNodeLauncher.scala │ │ ├── metrics │ │ └── kafka │ │ │ ├── KafkaCheckOffsetsActor.scala │ │ │ ├── KafkaCheckOffsetsGuardian.scala │ │ │ ├── KafkaOffsetActorAlive.scala │ │ │ ├── KafkaOffsets.scala │ │ │ ├── KafkaOffsetsRequest.scala │ │ │ ├── WrongKafkaOffsetsRequest.scala │ │ │ ├── backlog │ │ │ ├── BacklogAnalyzerConfig.scala │ │ │ ├── BacklogInfo.scala │ │ │ ├── BacklogSizeAnalyzerProducerActor.scala │ │ │ └── BacklogSizeAnalyzerProducerGuardian.scala │ │ │ └── throughput │ │ │ ├── KafkaThroughputConfig.scala │ │ │ ├── KafkaThroughputProducerActor.scala │ │ │ └── KafkaThroughputProducerGuardian.scala │ │ └── remote │ │ └── package.scala │ └── test │ ├── resources │ ├── log4j.xml │ └── log4j2.xml │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── producers │ └── metrics │ └── kafka │ ├── TestActors.scala │ ├── backlog │ ├── BacklogSizeAnalyzerSpec.scala │ └── TestBacklogActors.scala │ └── throughput │ ├── KafkaThroughputSpec.scala │ └── TestThroughputActors.scala ├── project ├── BranchingModelSupport.scala ├── Dependencies.scala ├── Flavor.scala ├── Settings.scala ├── Spark3Dependencies.scala ├── Spark3Exclusions.scala ├── Spark3Versions.scala ├── Utils.scala ├── VersionBumper.scala ├── build.properties └── plugins.sbt ├── repository ├── core │ └── src │ │ ├── main │ │ └── scala │ │ │ └── it │ │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── repository │ │ │ └── core │ │ │ ├── bl │ │ │ ├── AllBLsTestWrapper.scala │ │ │ ├── BatchJobBL.scala │ │ │ ├── BatchSchedulersBL.scala │ │ │ ├── CdcBL.scala │ │ │ ├── ConfigBL.scala │ │ │ ├── ConfigManagerBL.scala │ │ │ ├── DBConfigBL.scala │ │ │ ├── DocumentBL.scala │ │ │ ├── FreeCodeBL.scala │ │ │ ├── GenericBL.scala │ │ │ ├── HttpBL.scala │ │ │ ├── IndexBL.scala │ │ │ ├── KeyValueBL.scala │ │ │ ├── MlModelBL.scala │ │ │ ├── PipegraphBL.scala │ │ │ ├── ProcessGroupBL.scala │ │ │ ├── ProducerBL.scala │ │ │ ├── RawBL.scala │ │ │ ├── SQLSinkBL.scala │ │ │ ├── SqlSourceBl.scala │ │ │ ├── TopicBL.scala │ │ │ ├── WebsocketBL.scala │ │ │ └── package.scala │ │ │ ├── db │ │ │ ├── RepositoriesFactory.scala │ │ │ └── WaspDB.scala │ │ │ ├── dbModels │ │ │ ├── BatchJobDBModel.scala │ │ │ ├── BatchSchedulersDBModel.scala │ │ │ ├── CdcDBModel.scala │ │ │ ├── ConfigDBModel.scala │ │ │ ├── DocumentDBModel.scala │ │ │ ├── FreeCodeDBModel.scala │ │ │ ├── GenericDBModel.scala │ │ │ ├── HttpDBModel.scala │ │ │ ├── IndexDBModel.scala │ │ │ ├── KeyValueDBModel.scala │ │ │ ├── MlDBModelOnlyInfo.scala │ │ │ ├── MultiTopicDBModel.scala │ │ │ ├── PipegraphDBModel.scala │ │ │ ├── ProcessGroupDBModel.scala │ │ │ ├── ProducerDBModel.scala │ │ │ ├── RawDBModel.scala │ │ │ ├── SqlSourceDBModel.scala │ │ │ ├── TopicDBModel.scala │ │ │ └── WebsocketDBModel.scala │ │ │ └── mappers │ │ │ ├── BatchJobModelMapper.scala │ │ │ ├── BatchSchedulersMapper.scala │ │ │ ├── CdcMapper.scala │ │ │ ├── ConfigMapper.scala │ │ │ ├── DocumentMapper.scala │ │ │ ├── FreeCodeMapper.scala │ │ │ ├── GenericMapper.scala │ │ │ ├── HBaseConfigMapper.scala │ │ │ ├── HttpMapper.scala │ │ │ ├── IndexMapper.scala │ │ │ ├── KeyValueMapper.scala │ │ │ ├── Mapper.scala │ │ │ ├── MlDBModelMapper.scala │ │ │ ├── MultiModelTopicMapper.scala │ │ │ ├── PipegraphMapper.scala │ │ │ ├── ProcessGroupMapper.scala │ │ │ ├── ProducerMapper.scala │ │ │ ├── RawMapper.scala │ │ │ ├── SqlSourceMapper.scala │ │ │ ├── TopicMapper.scala │ │ │ └── WebsocketMapper.scala │ │ └── test │ │ └── resources │ │ ├── log4j.xml │ │ └── log4j2.xml ├── mongo │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── it.agilelab.bigdata.wasp.repository.core.db.RepositoriesFactory │ │ └── scala │ │ │ └── it │ │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── repository │ │ │ └── mongo │ │ │ ├── MongoRepositoriesFactory.scala │ │ │ ├── WaspMongoDB.scala │ │ │ ├── bl │ │ │ ├── BatchJobBLImp.scala │ │ │ ├── BatchJobInstanceBlImp.scala │ │ │ ├── BatchSchedulersBLImp.scala │ │ │ ├── CdcBLImp.scala │ │ │ ├── ConfigManagerBLImpl.scala │ │ │ ├── DocumentBLImpl.scala │ │ │ ├── FreeCodeBLImpl.scala │ │ │ ├── GenericBLImp.scala │ │ │ ├── HttpBlImpl.scala │ │ │ ├── IndexBLImp.scala │ │ │ ├── KeyValueBLImp.scala │ │ │ ├── MlModelBLImp.scala │ │ │ ├── MongoFactoryBL.scala │ │ │ ├── PipegraphBLImp.scala │ │ │ ├── ProcessGroupBLImpl.scala │ │ │ ├── ProducerBLImp.scala │ │ │ ├── RawBLImp.scala │ │ │ ├── SQLSinkBLImpl.scala │ │ │ ├── SqlSourceBlImpl.scala │ │ │ ├── TopicBLImp.scala │ │ │ └── WebsocketBLImp.scala │ │ │ ├── providers │ │ │ ├── AbstractCodecProvider.scala │ │ │ ├── BatchETLCodecProvider.scala │ │ │ ├── BatchJobDBProvider.scala │ │ │ ├── DataStoreConfCodecProviders.scala │ │ │ ├── DatastoreProductCodecProvider.scala │ │ │ ├── HttpCompressionCodecProvider.scala │ │ │ ├── PipegraphInstanceDBModelProvider.scala │ │ │ ├── SealedTraitCodecProvider.scala │ │ │ ├── SubjectStrategyCodecProvider.scala │ │ │ ├── TopicCompressionCodecProvider.scala │ │ │ ├── VersionedCodecProvider.scala │ │ │ └── VersionedProviders.scala │ │ │ └── utils │ │ │ └── MongoDBHelper.scala │ │ └── test │ │ ├── resources │ │ ├── application.conf │ │ ├── log4j.xml │ │ └── log4j2.xml │ │ └── scala │ │ ├── com │ │ └── github │ │ │ └── simplyscala │ │ │ └── MongoEmbedDatabase.scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── repository │ │ └── mongo │ │ ├── BatchGDPRJobTest.scala │ │ ├── BatchJobBLImplTest.scala │ │ ├── BatchJobInstanceBLImplTest.scala │ │ ├── BatchSchedulerBLImplTest.scala │ │ ├── CdcBLImplTest.scala │ │ ├── ConfigManagerBLImplTest.scala │ │ ├── DocumentBLImplTest.scala │ │ ├── HttpBLImplTest.scala │ │ ├── IndexBLImplTest.scala │ │ ├── KeyValueBLImplTest.scala │ │ ├── MlModelBLImplTest.scala │ │ ├── MongoRepositoriesFactoryTest.scala │ │ ├── PipegraphBLImplTest.scala │ │ ├── ProcessGroupBLImplTest.scala │ │ ├── ProducerBLImplTest.scala │ │ ├── ProvidersTest.scala │ │ ├── RawBLImplTest.scala │ │ ├── SqlSourceBLImplTest.scala │ │ ├── TopicBLImplTest.scala │ │ ├── WebsocketBLImplTest.scala │ │ └── providers │ │ ├── BatchETLCodecProviderTest.scala │ │ └── TopicCompressionCodecProviderTest.scala ├── postgres │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── it.agilelab.bigdata.wasp.repository.core.db.RepositoriesFactory │ │ └── scala │ │ │ └── it │ │ │ └── agilelab │ │ │ └── bigdata │ │ │ └── wasp │ │ │ └── repository │ │ │ └── postgres │ │ │ ├── PostgresRepositoryFactory.scala │ │ │ ├── WaspPostgresDB.scala │ │ │ ├── WaspPostgresDBImpl.scala │ │ │ ├── bl │ │ │ ├── BatchJobBLImpl.scala │ │ │ ├── BatchJobInstanceBLImpl.scala │ │ │ ├── BatchSchedulersBLImpl.scala │ │ │ ├── CdcBLImpl.scala │ │ │ ├── ConfigManagerBLImpl.scala │ │ │ ├── DocumentBLImpl.scala │ │ │ ├── FreeCodeBLImpl.scala │ │ │ ├── GenericBLImpl.scala │ │ │ ├── HttpBLImpl.scala │ │ │ ├── IndexBLImpl.scala │ │ │ ├── KeyValueBLImpl.scala │ │ │ ├── MlModelBLImpl.scala │ │ │ ├── PipegraphBLImpl.scala │ │ │ ├── PipegraphInstanceBlImpl.scala │ │ │ ├── PostgresBL.scala │ │ │ ├── PostgresFactoryBL.scala │ │ │ ├── ProcessGroupBLImpl.scala │ │ │ ├── ProducerBLImpl.scala │ │ │ ├── RawBLImpl.scala │ │ │ ├── SQLSinkBLImpl.scala │ │ │ ├── SqlSourceBLImpl.scala │ │ │ ├── TopicBLImpl.scala │ │ │ └── WebsocketBLImpl.scala │ │ │ ├── tables │ │ │ ├── BatchJobInstanceTableDefinition.scala │ │ │ ├── BatchJobTableDefinition.scala │ │ │ ├── BatchSchedulersTableDefinition.scala │ │ │ ├── CdcTableDefinition.scala │ │ │ ├── ConfigManagerTableDefinition.scala │ │ │ ├── DocumentTableDefinition.scala │ │ │ ├── FreeCodeTableDefinition.scala │ │ │ ├── GenericTableDefinition.scala │ │ │ ├── HttpTableDefinition.scala │ │ │ ├── IndexTableDefinition.scala │ │ │ ├── KeyValueTableDefinition.scala │ │ │ ├── MlModelOnlyDataTableDefinition.scala │ │ │ ├── MlModelOnlyInfoTableDefinition.scala │ │ │ ├── ModelTableDefinition.scala │ │ │ ├── PipegraphInstanceTableDefinition.scala │ │ │ ├── PipegraphTableDefinition.scala │ │ │ ├── ProcessGroupTableDefinition.scala │ │ │ ├── ProducerTableDefinition.scala │ │ │ ├── RawTableDefinition.scala │ │ │ ├── SQLSinkTableDefinition.scala │ │ │ ├── SimpleModelTableDefinition.scala │ │ │ ├── SqlSourceTableDefinition.scala │ │ │ ├── TableDefinition.scala │ │ │ ├── TopicTableDefinition.scala │ │ │ ├── WebSocketTableDefinition.scala │ │ │ └── package.scala │ │ │ └── utils │ │ │ ├── ConnectionInfoProvider.scala │ │ │ ├── ConnectionSupport.scala │ │ │ └── PostgresDBHelper.scala │ │ └── test │ │ ├── resources │ │ ├── log4j.xml │ │ └── log4j2.xml │ │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── repository │ │ └── postgres │ │ ├── AllPostgresTests.scala │ │ ├── bl │ │ ├── BatchJobBLImplTest.scala │ │ ├── BatchJobInstanceBLImplTest.scala │ │ ├── BatchSchedulersBLImplTest.scala │ │ ├── ConfigManagerBLImplTest.scala │ │ ├── DocumentBLImplTest.scala │ │ ├── FreeCodeBLImplTest.scala │ │ ├── GenericBLImplTest.scala │ │ ├── HttpBLImplTest.scala │ │ ├── IndexBLImplTest.scala │ │ ├── KeyValueBLImplTest.scala │ │ ├── MlModelBLImplTest.scala │ │ ├── PipegraphBLImplTest.scala │ │ ├── PipegraphInstanceBLImplTest.scala │ │ ├── ProcessGroupBLImplTest.scala │ │ ├── ProducerBLImplTest.scala │ │ ├── RawBLImplTest.scala │ │ ├── SqlSourceBlImplTest.scala │ │ ├── TopicBLImplTest.scala │ │ └── WebsocketBLImplTest.scala │ │ ├── tables │ │ └── packageTest.scala │ │ └── utils │ │ └── PostgresSuite.scala └── src │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── run-sbt-unprivileged.sh ├── sbin └── release.sh ├── spark ├── nifi-plugin-bridge │ └── src │ │ └── main │ │ └── java │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── spark │ │ └── plugins │ │ └── nifi │ │ └── Bridge.java ├── nifi-plugin │ └── src │ │ ├── main │ │ ├── java │ │ │ └── it │ │ │ │ └── agilelab │ │ │ │ └── bigdata │ │ │ │ └── wasp │ │ │ │ └── spark │ │ │ │ └── plugins │ │ │ │ └── nifi │ │ │ │ └── ReflectiveCall.java │ │ └── scala │ │ │ ├── it │ │ │ └── agilelab │ │ │ │ └── bigdata │ │ │ │ └── wasp │ │ │ │ └── spark │ │ │ │ └── plugins │ │ │ │ └── nifi │ │ │ │ └── CompatibilityNifiPlugin.scala │ │ │ └── nifi │ │ │ └── NifiPlugin.scala │ │ └── test │ │ └── resources │ │ ├── log4j.xml │ │ └── log4j2.xml ├── src │ └── test │ │ └── resources │ │ ├── log4j.xml │ │ └── log4j2.xml └── telemetry-plugin │ └── src │ ├── main │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── spark │ │ └── plugins │ │ └── telemetry │ │ ├── CompatibilityExecutorPlugin.scala │ │ ├── CompatibilityTelemetryPluginProducer.scala │ │ ├── ConfigurationSupport.scala │ │ ├── JmxTelemetry.scala │ │ ├── SchedulingSupport.scala │ │ ├── TelemetryMetadataProducerConfig.scala │ │ ├── TelemetryPlugin.scala │ │ ├── TelemetryPluginConfiguration.scala │ │ ├── TelemetryPluginConnectionConfig.scala │ │ ├── TelemetryPluginKafkaConfig.scala │ │ ├── TelemetryPluginProducer.scala │ │ └── TelemetryPluginTopicConfigModel.scala │ └── test │ └── resources │ ├── log4j.xml │ └── log4j2.xml ├── tools └── release-note-generator │ ├── .gitignore │ ├── README.md │ ├── requirements.txt │ ├── setup.py │ └── wasprng │ ├── __init__.py │ └── cli.py ├── version.sbt ├── waspctl ├── .gitignore ├── Dockerfile ├── README.md ├── build-for-all-archs.bash ├── main.go └── wasp-openapi.go └── whitelabel ├── .gitignore ├── consumers-spark ├── build.sbt └── src │ ├── main │ └── scala │ │ └── it │ │ └── agilelab │ │ └── bigdata │ │ └── wasp │ │ └── whitelabel │ │ └── consumers │ │ └── spark │ │ ├── aws │ │ └── MinioCredentialsProvider.scala │ │ ├── catalog │ │ ├── EntityCatalog.scala │ │ └── WhitelabelCatalogTableNameBuilder.scala │ │ ├── launcher │ │ ├── SparkConsumersBatchNodeLauncher.scala │ │ └── SparkConsumersStreamingNodeLauncher.scala │ │ ├── serializer │ │ └── kryo │ │ │ ├── CustomKyroRegistrator1.scala │ │ │ └── CustomKyroRegistrator2.scala │ │ └── strategies │ │ └── test │ │ ├── TestAvroEncoderStrategy.scala │ │ ├── TestCheckpointStrategy.scala │ │ ├── TestContinuousUpdateStrategy.scala │ │ ├── TestEchoStrategy.scala │ │ ├── TestErrorStrategy.scala │ │ ├── TestHttpEnrichmentStrategy.scala │ │ ├── TestHttpHeaders.scala │ │ ├── TestIdentityStrategy.scala │ │ ├── TestIdentityStrategyPostHook.scala │ │ ├── TestJdbcMySqlStrategy.scala │ │ ├── TestKafkaAvroSchemaManager.scala │ │ ├── TestKafkaBinary.scala │ │ ├── TestKafkaHeaders.scala │ │ ├── TestKafkaMetadata.scala │ │ ├── TestKafkaMultitopicWrite.scala │ │ ├── TestKafkaPlaintext.scala │ │ ├── TestMultiTopicWriteMixedStrategy.scala │ │ ├── TestSetShufflePartitionsTo10Strategy.scala │ │ ├── TestStuckQueryStrategy.scala │ │ └── TestToKafkaPlainStrategy.scala │ └── test │ ├── resources │ └── 3.delta │ └── scala │ └── it.agilelab.bigdata.wasp.whitelabel.consumers.spark.strategies.test │ └── TestAvroEncoderStrategySpec.scala ├── docker ├── cdp711 │ ├── build-and-push.sh │ ├── build │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── envsubst │ │ │ └── envsubst.go │ │ ├── install-scripts │ │ │ ├── common │ │ │ │ ├── bash-defaults.sh │ │ │ │ └── yum-clean-caches.sh │ │ │ ├── configure-cdp.sh │ │ │ ├── configure-hadoop.sh │ │ │ ├── install-mongo.sh │ │ │ ├── install-prerequisites.sh │ │ │ └── install-supervisord.sh │ │ ├── runtime │ │ │ └── entrypoint.sh │ │ └── templates │ │ │ ├── hadoop │ │ │ ├── capacity-scheduler.xml │ │ │ ├── configuration.xsl │ │ │ ├── container-executor.cfg │ │ │ ├── core-site.xml │ │ │ ├── hadoop-metrics2.properties │ │ │ ├── hadoop-policy.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── log4j.properties │ │ │ ├── mapred-queues.xml.template │ │ │ ├── mapred-site.xml │ │ │ ├── ssl-client.xml.example │ │ │ ├── ssl-server.xml.example │ │ │ ├── workers │ │ │ ├── yarn-env.sh │ │ │ └── yarn-site.xml │ │ │ ├── hbase │ │ │ ├── hadoop-metrics2-hbase.properties │ │ │ ├── hbase-env.cmd │ │ │ ├── hbase-env.sh │ │ │ ├── hbase-policy.xml │ │ │ ├── hbase-site.xml │ │ │ ├── log4j-hbtop.properties │ │ │ ├── log4j.properties │ │ │ └── regionservers │ │ │ ├── kafka │ │ │ ├── connect-console-sink.properties │ │ │ ├── connect-console-source.properties │ │ │ ├── connect-distributed.properties │ │ │ ├── connect-file-sink.properties │ │ │ ├── connect-file-source.properties │ │ │ ├── connect-log4j.properties │ │ │ ├── connect-mirror-maker.properties │ │ │ ├── connect-standalone.properties │ │ │ ├── log4j.properties │ │ │ ├── server.properties │ │ │ ├── tools-log4j.properties │ │ │ └── trogdor.conf │ │ │ ├── kafka2 │ │ │ └── server.properties │ │ │ ├── mongo │ │ │ └── mongodb.repo │ │ │ ├── resolve-hadoop-templates.sh │ │ │ ├── resolve-hbase-templates.sh │ │ │ ├── resolve-kafka-templates.sh │ │ │ ├── resolve-polynote-templates.sh │ │ │ ├── resolve-solr-templates.sh │ │ │ ├── resolve-spark-templates.sh │ │ │ ├── resolve-zookeeper-templates.sh │ │ │ ├── solr │ │ │ ├── bin │ │ │ │ └── solr.in.sh │ │ │ └── conf │ │ │ │ ├── README.txt │ │ │ │ ├── configsets │ │ │ │ ├── _default │ │ │ │ │ └── conf │ │ │ │ │ │ ├── lang │ │ │ │ │ │ ├── contractions_ca.txt │ │ │ │ │ │ ├── contractions_fr.txt │ │ │ │ │ │ ├── contractions_ga.txt │ │ │ │ │ │ ├── contractions_it.txt │ │ │ │ │ │ ├── hyphenations_ga.txt │ │ │ │ │ │ ├── stemdict_nl.txt │ │ │ │ │ │ ├── stoptags_ja.txt │ │ │ │ │ │ ├── stopwords_ar.txt │ │ │ │ │ │ ├── stopwords_bg.txt │ │ │ │ │ │ ├── stopwords_ca.txt │ │ │ │ │ │ ├── stopwords_cz.txt │ │ │ │ │ │ ├── stopwords_da.txt │ │ │ │ │ │ ├── stopwords_de.txt │ │ │ │ │ │ ├── stopwords_el.txt │ │ │ │ │ │ ├── stopwords_en.txt │ │ │ │ │ │ ├── stopwords_es.txt │ │ │ │ │ │ ├── stopwords_et.txt │ │ │ │ │ │ ├── stopwords_eu.txt │ │ │ │ │ │ ├── stopwords_fa.txt │ │ │ │ │ │ ├── stopwords_fi.txt │ │ │ │ │ │ ├── stopwords_fr.txt │ │ │ │ │ │ ├── stopwords_ga.txt │ │ │ │ │ │ ├── stopwords_gl.txt │ │ │ │ │ │ ├── stopwords_hi.txt │ │ │ │ │ │ ├── stopwords_hu.txt │ │ │ │ │ │ ├── stopwords_hy.txt │ │ │ │ │ │ ├── stopwords_id.txt │ │ │ │ │ │ ├── stopwords_it.txt │ │ │ │ │ │ ├── stopwords_ja.txt │ │ │ │ │ │ ├── stopwords_lv.txt │ │ │ │ │ │ ├── stopwords_nl.txt │ │ │ │ │ │ ├── stopwords_no.txt │ │ │ │ │ │ ├── stopwords_pt.txt │ │ │ │ │ │ ├── stopwords_ro.txt │ │ │ │ │ │ ├── stopwords_ru.txt │ │ │ │ │ │ ├── stopwords_sv.txt │ │ │ │ │ │ ├── stopwords_th.txt │ │ │ │ │ │ ├── stopwords_tr.txt │ │ │ │ │ │ └── userdict_ja.txt │ │ │ │ │ │ ├── managed-schema │ │ │ │ │ │ ├── protwords.txt │ │ │ │ │ │ ├── solrconfig.xml │ │ │ │ │ │ ├── stopwords.txt │ │ │ │ │ │ └── synonyms.txt │ │ │ │ └── sample_techproducts_configs │ │ │ │ │ └── conf │ │ │ │ │ ├── _rest_managed.json │ │ │ │ │ ├── _schema_analysis_stopwords_english.json │ │ │ │ │ ├── _schema_analysis_synonyms_english.json │ │ │ │ │ ├── clustering │ │ │ │ │ └── carrot2 │ │ │ │ │ │ ├── README.txt │ │ │ │ │ │ ├── kmeans-attributes.xml │ │ │ │ │ │ ├── lingo-attributes.xml │ │ │ │ │ │ └── stc-attributes.xml │ │ │ │ │ ├── currency.xml │ │ │ │ │ ├── elevate.xml │ │ │ │ │ ├── lang │ │ │ │ │ ├── contractions_ca.txt │ │ │ │ │ ├── contractions_fr.txt │ │ │ │ │ ├── contractions_ga.txt │ │ │ │ │ ├── contractions_it.txt │ │ │ │ │ ├── hyphenations_ga.txt │ │ │ │ │ ├── stemdict_nl.txt │ │ │ │ │ ├── stoptags_ja.txt │ │ │ │ │ ├── stopwords_ar.txt │ │ │ │ │ ├── stopwords_bg.txt │ │ │ │ │ ├── stopwords_ca.txt │ │ │ │ │ ├── stopwords_ckb.txt │ │ │ │ │ ├── stopwords_cz.txt │ │ │ │ │ ├── stopwords_da.txt │ │ │ │ │ ├── stopwords_de.txt │ │ │ │ │ ├── stopwords_el.txt │ │ │ │ │ ├── stopwords_en.txt │ │ │ │ │ ├── stopwords_es.txt │ │ │ │ │ ├── stopwords_et.txt │ │ │ │ │ ├── stopwords_eu.txt │ │ │ │ │ ├── stopwords_fa.txt │ │ │ │ │ ├── stopwords_fi.txt │ │ │ │ │ ├── stopwords_fr.txt │ │ │ │ │ ├── stopwords_ga.txt │ │ │ │ │ ├── stopwords_gl.txt │ │ │ │ │ ├── stopwords_hi.txt │ │ │ │ │ ├── stopwords_hu.txt │ │ │ │ │ ├── stopwords_hy.txt │ │ │ │ │ ├── stopwords_id.txt │ │ │ │ │ ├── stopwords_it.txt │ │ │ │ │ ├── stopwords_ja.txt │ │ │ │ │ ├── stopwords_lv.txt │ │ │ │ │ ├── stopwords_nl.txt │ │ │ │ │ ├── stopwords_no.txt │ │ │ │ │ ├── stopwords_pt.txt │ │ │ │ │ ├── stopwords_ro.txt │ │ │ │ │ ├── stopwords_ru.txt │ │ │ │ │ ├── stopwords_sv.txt │ │ │ │ │ ├── stopwords_th.txt │ │ │ │ │ ├── stopwords_tr.txt │ │ │ │ │ └── userdict_ja.txt │ │ │ │ │ ├── managed-schema │ │ │ │ │ ├── mapping-FoldToASCII.txt │ │ │ │ │ ├── mapping-ISOLatin1Accent.txt │ │ │ │ │ ├── params.json │ │ │ │ │ ├── protwords.txt │ │ │ │ │ ├── solrconfig.xml │ │ │ │ │ ├── spellings.txt │ │ │ │ │ ├── stopwords.txt │ │ │ │ │ ├── synonyms.txt │ │ │ │ │ ├── update-script.js │ │ │ │ │ ├── velocity │ │ │ │ │ ├── README.txt │ │ │ │ │ ├── VM_global_library.vm │ │ │ │ │ ├── browse.vm │ │ │ │ │ ├── cluster.vm │ │ │ │ │ ├── cluster_results.vm │ │ │ │ │ ├── debug.vm │ │ │ │ │ ├── did_you_mean.vm │ │ │ │ │ ├── error.vm │ │ │ │ │ ├── facet_fields.vm │ │ │ │ │ ├── facet_pivot.vm │ │ │ │ │ ├── facet_queries.vm │ │ │ │ │ ├── facet_ranges.vm │ │ │ │ │ ├── facets.vm │ │ │ │ │ ├── footer.vm │ │ │ │ │ ├── head.vm │ │ │ │ │ ├── header.vm │ │ │ │ │ ├── hit.vm │ │ │ │ │ ├── hit_grouped.vm │ │ │ │ │ ├── hit_plain.vm │ │ │ │ │ ├── join_doc.vm │ │ │ │ │ ├── jquery.autocomplete.css │ │ │ │ │ ├── jquery.autocomplete.js │ │ │ │ │ ├── layout.vm │ │ │ │ │ ├── main.css │ │ │ │ │ ├── mime_type_lists.vm │ │ │ │ │ ├── pagination_bottom.vm │ │ │ │ │ ├── pagination_top.vm │ │ │ │ │ ├── product_doc.vm │ │ │ │ │ ├── query.vm │ │ │ │ │ ├── query_form.vm │ │ │ │ │ ├── query_group.vm │ │ │ │ │ ├── query_spatial.vm │ │ │ │ │ ├── results_list.vm │ │ │ │ │ ├── richtext_doc.vm │ │ │ │ │ ├── suggest.vm │ │ │ │ │ └── tabs.vm │ │ │ │ │ └── xslt │ │ │ │ │ ├── example.xsl │ │ │ │ │ ├── example_atom.xsl │ │ │ │ │ ├── example_rss.xsl │ │ │ │ │ ├── luke.xsl │ │ │ │ │ └── updateXml.xsl │ │ │ │ ├── solr.xml │ │ │ │ └── zoo.cfg │ │ │ ├── spark │ │ │ ├── docker.properties.template │ │ │ ├── fairscheduler.xml.template │ │ │ ├── log4j.properties.template │ │ │ ├── metrics.properties.template │ │ │ ├── slaves.template │ │ │ ├── spark-defaults.conf │ │ │ ├── spark-defaults.conf.template │ │ │ ├── spark-env.sh │ │ │ └── spark-env.sh.template │ │ │ └── zookeeper │ │ │ ├── configuration.xsl │ │ │ ├── log4j.properties │ │ │ ├── zoo.cfg │ │ │ └── zoo_sample.cfg │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── get-docker-cmd.sh │ ├── log4j.xml │ ├── start-wasp.sh │ └── supervisord.conf ├── debezium │ └── docker-compose-debezium.yaml ├── vanilla-hadoop2 │ ├── build-and-tag.sh │ ├── docker-entrypoint-one.sh │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── docker-environment_reference.conf │ ├── docker │ │ ├── Dockerfile │ │ ├── install-scripts │ │ │ ├── common │ │ │ │ ├── apt-clean-caches.sh │ │ │ │ └── bash-defaults.sh │ │ │ ├── configure-hadoop.sh │ │ │ ├── configure-spark.sh │ │ │ ├── install-hadoop.sh │ │ │ ├── install-hbase.sh │ │ │ ├── install-kafka.sh │ │ │ ├── install-mongo.sh │ │ │ ├── install-prerequisites.sh │ │ │ ├── install-solr.sh │ │ │ └── install-spark.sh │ │ ├── runtime │ │ │ └── entrypoint.sh │ │ └── templates │ │ │ ├── hadoop │ │ │ ├── core-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ │ │ ├── hbase │ │ │ ├── hbase-site.xml │ │ │ └── regionservers │ │ │ ├── kafka │ │ │ └── server.properties │ │ │ ├── mongo │ │ │ └── mongod.conf │ │ │ ├── resolve-templates.sh │ │ │ └── solr │ │ │ └── solr.in.sh │ ├── get-docker-cmd.sh │ ├── log4j-consumer.properties │ ├── log4j-master.properties │ ├── log4j-producer.properties │ ├── log4j-single.properties │ ├── start-wasp-one.sh │ ├── start-wasp.sh │ └── supervisord.conf ├── vanilla-hadoop3 │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── docker-environment_reference.conf │ ├── docker │ │ ├── Dockerfile │ │ ├── install-scripts │ │ │ ├── common │ │ │ │ ├── apt-clean-caches.sh │ │ │ │ └── bash-defaults.sh │ │ │ ├── configure-hadoop.sh │ │ │ ├── configure-spark.sh │ │ │ ├── install-hadoop.sh │ │ │ ├── install-hbase.sh │ │ │ ├── install-kafka.sh │ │ │ ├── install-minio.sh │ │ │ ├── install-mongo.sh │ │ │ ├── install-prerequisites.sh │ │ │ ├── install-solr.sh │ │ │ └── install-spark.sh │ │ ├── runtime │ │ │ └── entrypoint.sh │ │ └── templates │ │ │ ├── hadoop │ │ │ ├── core-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ │ │ ├── hbase │ │ │ ├── hbase-site.xml │ │ │ └── regionservers │ │ │ ├── kafka │ │ │ └── server.properties │ │ │ ├── mongo │ │ │ └── mongod.conf │ │ │ ├── resolve-templates.sh │ │ │ └── solr │ │ │ └── solr.in.sh │ ├── example-dashboard.json │ ├── flask-parallel-write-entity.py │ ├── get-docker-cmd.sh │ ├── log4j-consumer.properties │ ├── log4j-master.properties │ ├── log4j-producer.properties │ ├── start-wasp.sh │ └── supervisord.conf └── vanilla2.12-hadoop2 │ ├── build-and-tag.sh │ ├── docker-entrypoint-one.sh │ ├── docker-entrypoint.sh │ ├── docker-environment.conf │ ├── docker-environment_reference.conf │ ├── docker │ ├── Dockerfile │ ├── install-scripts │ │ ├── common │ │ │ ├── apt-clean-caches.sh │ │ │ └── bash-defaults.sh │ │ ├── configure-hadoop.sh │ │ ├── configure-spark.sh │ │ ├── install-hadoop.sh │ │ ├── install-hbase.sh │ │ ├── install-kafka.sh │ │ ├── install-mongo.sh │ │ ├── install-prerequisites.sh │ │ ├── install-solr.sh │ │ └── install-spark.sh │ ├── runtime │ │ └── entrypoint.sh │ └── templates │ │ ├── hadoop │ │ ├── core-site.xml │ │ ├── mapred-site.xml │ │ └── yarn-site.xml │ │ ├── hbase │ │ ├── hbase-site.xml │ │ └── regionservers │ │ ├── kafka │ │ └── server.properties │ │ ├── mongo │ │ └── mongod.conf │ │ ├── resolve-templates.sh │ │ └── solr │ │ └── solr.in.sh │ ├── get-docker-cmd.sh │ ├── log4j-consumer.properties │ ├── log4j-master.properties │ ├── log4j-producer.properties │ ├── log4j-single.properties │ ├── start-wasp-one.sh │ ├── start-wasp.sh │ └── supervisord.conf ├── master ├── build.sbt └── src │ └── main │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── whitelabel │ └── master │ └── launcher │ └── MasterNodeLauncher.scala ├── models └── src │ └── main │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ └── whitelabel │ └── models │ ├── example │ ├── ExamplePipegraphModel.scala │ ├── ExampleTopicModel.scala │ ├── FakeDataProducerModel.scala │ ├── FakeDataTopicModel.scala │ └── iot │ │ ├── IoTIndustrialPlantIndexModel.scala │ │ ├── IoTIndustrialPlantPipegraphModel.scala │ │ ├── IoTIndustrialPlantProducerModel.scala │ │ └── IoTIndustrialPlantTopicModel.scala │ └── test │ ├── TestBatchJobModels.scala │ ├── TestCdcModel.scala │ ├── TestCheckpointState.scala │ ├── TestDocument.scala │ ├── TestDocumentEncoder.scala │ ├── TestEnrichmentModel.scala │ ├── TestFreeCodeModels.scala │ ├── TestGdprBatchJobModels.scala │ ├── TestHttpModel.scala │ ├── TestIndexModel.scala │ ├── TestKeyValueModel.scala │ ├── TestMongoModel.scala │ ├── TestParallelWriteModel.scala │ ├── TestPipegraphs.scala │ ├── TestProducerModel.scala │ ├── TestRawModel.scala │ ├── TestSchemaAvroManager.scala │ ├── TestSqlSourceModel.scala │ ├── TestState.scala │ ├── TestStrategies.scala │ └── TestTopicModel.scala ├── producers ├── build.sbt └── src │ └── main │ └── scala │ └── it │ └── agilelab │ └── bigdata │ └── wasp │ ├── producers │ └── metrics │ │ └── kafka │ │ ├── backlog │ │ └── ImplBacklogSizeAnalyzerProducer.scala │ │ └── throughput │ │ └── ImplKafkaThroughputProducer.scala │ └── whitelabel │ └── producers │ ├── TestProducerAvroSchemaManager_v1.scala │ ├── TestProducerAvroSchemaManager_v2.scala │ ├── eventengine │ └── FakeDataProducerGuardian.scala │ ├── iot │ └── IoTIndustrialPlantProducerGuardian.scala │ ├── launcher │ └── ProducersNodeLauncher.scala │ └── test │ ├── TestCheckpointProducerGuardian.scala │ ├── TestDocumentWithMetadataProducerGuardian.scala │ ├── TestHbaseMultiClusteringProducerGuardian.scala │ ├── TestProducerDeduplication.scala │ └── TestProducerGuardian.scala └── single-node ├── build.sbt └── src └── main └── scala └── it └── agilelab └── bigdata └── wasp └── whitelabel └── launcher └── SingleNodeLauncher.scala /.gitattributes: -------------------------------------------------------------------------------- 1 | * text eol=lf 2 | 3 | # Denote all files that are truly binary and should not be modified. 4 | *.png binary 5 | *.jpg binary 6 | -------------------------------------------------------------------------------- /.gitlab/issue_templates/bug.md: -------------------------------------------------------------------------------- 1 | # Scenario summary 2 | 3 | Summarize the scenario and provide a history reference, such as a commit or tag. 4 | 5 | # Problem statement 6 | 7 | Describe the problem, ie what happens. 8 | 9 | # Expected behaviour 10 | 11 | Describe what should happen if everything worked correctly. 12 | 13 | # Steps to reproduce 14 | 15 | Provide information needed to reproduce the problem. 16 | 17 | # Proposed solution 18 | 19 | Propose a solution. -------------------------------------------------------------------------------- /.gitlab/issue_templates/feature.md: -------------------------------------------------------------------------------- 1 | # Scenario summary 2 | 3 | Summarize the scenario and provide a history reference, such as a commit or tag. 4 | 5 | # Problem statement 6 | 7 | Describe what problem the feature would solve. 8 | 9 | # Proposed solution 10 | 11 | Propose a solution. -------------------------------------------------------------------------------- /.gitlab/merge_request_templates/standard.md: -------------------------------------------------------------------------------- 1 | # New features and improvements 2 | 3 | List any new features and improvements made here. 4 | 5 | # Breaking changes 6 | 7 | List any breaking changes here, along with a brief reason why they were made (eg to add a new feature). 8 | 9 | # Migration 10 | 11 | If any breaking changes were made, list how to handle them here. 12 | 13 | # Bug fixes 14 | 15 | List any bug fixes made here. 16 | 17 | # How this feature was tested 18 | 19 | List test methodology and reference to them (i.e. unit test name) 20 | 21 | # Related issue 22 | 23 | Closes #{{issue.iid}} -------------------------------------------------------------------------------- /.jvmopts: -------------------------------------------------------------------------------- 1 | -Xmx16G 2 | -Xms2G 3 | #-XX:+UseConcMarkSweepGC 4 | #-XX:+CMSClassUnloadingEnabled 5 | #-XX:MaxPermSize=2G 6 | -Xss2M 7 | -Duser.timezone=GMT 8 | -XX:ReservedCodeCacheSize=1G 9 | --add-opens=java.base/java.lang=ALL-UNNAMED 10 | --add-exports=java.base/sun.nio.ch=ALL-UNNAMED 11 | --add-opens=java.base/sun.util.calendar=ALL-UNNAMED 12 | --add-opens=java.base/java.lang.reflect=ALL-UNNAMED 13 | --add-opens=java.base/java.util=ALL-UNNAMED 14 | --add-opens=java.base/java.net=ALL-UNNAMED 15 | --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED -------------------------------------------------------------------------------- /.lab.yml: -------------------------------------------------------------------------------- 1 | group: AgileFactory 2 | project_name: Agile.Wasp2 3 | config: 4 | main_branch_name: develop 5 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 2.3.2 2 | style = defaultWithAlign 3 | maxColumn = 120 4 | project.git = true 5 | align.tokens.add = [ 6 | {code = "=>", owner = "Case"}, 7 | {code = "%", owner = "Term.ApplyInfix"}, 8 | {code = "%%", owner = "Term.ApplyInfix"} 9 | ] 10 | align.arrowEnumeratorGenerator = true 11 | rewrite.rules = [ 12 | SortImports, 13 | SortModifiers 14 | ] 15 | -------------------------------------------------------------------------------- /baseVersion.version: -------------------------------------------------------------------------------- 1 | 3.1.0 -------------------------------------------------------------------------------- /censor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | if [ -z "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ] 4 | then 5 | echo "Not something that can be merged with this MR!" 6 | exit -1 7 | fi 8 | echo "base-commit: ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" 9 | echo "target-commit ${CI_COMMIT_SHA}" 10 | 11 | censor ${CENSOR_ARGS} \ 12 | --repo-path '.' \ 13 | --base-commit origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} \ 14 | --target-commit ${CI_COMMIT_SHA} \ 15 | --exclude 'documentation/icons/WASP_logo.pdf' \ 16 | --exclude 'documentation/diagrams/statemachines.png' \ 17 | --exclude 'icons/WASP_logo.pdf' \ 18 | --exclude 'diagrams/statemachines.png' \ 19 | --exclude 'censor.sh' 20 | -------------------------------------------------------------------------------- /ci/build-image/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SBT_VERSION=1.10.7 3 | BASE_IMAGE_TAG=17.0.13_11-jdk-jammy 4 | NAME="registry.gitlab.com/agilefactory/agile.wasp2/sbt:${SBT_VERSION}-${BASE_IMAGE_TAG}" 5 | docker buildx build --push --platform linux/amd64,linux/arm64 \ 6 | --build-arg BASE_IMAGE_TAG=${BASE_IMAGE_TAG} \ 7 | --build-arg SBT_VERSION=${SBT_VERSION} \ 8 | --build-arg SCALA_VERSION=2.12.10 \ 9 | . -t "${NAME}" -------------------------------------------------------------------------------- /compiler/src/main/scala/it/agilelab/bigdata/wasp/compiler/utils/CompilerPool.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.compiler.utils 2 | 3 | import io.github.andrebeat.pool.Pool 4 | 5 | class CompilerPool(capacity: Int) extends AutoCloseable { 6 | override def close(): Unit = pool.close() 7 | 8 | val pool: Pool[Compiler] = Pool( 9 | capacity = this.capacity, 10 | factory = () => new Compiler(), 11 | healthCheck = _ => true, //always dispose compilers 12 | dispose = _.close(), 13 | reset = _.reset() 14 | ) 15 | 16 | def use[A](action: Compiler => A): A = pool.acquire().apply(action) 17 | } 18 | 19 | -------------------------------------------------------------------------------- /compiler/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /compiler/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /consumers-spark/build.sbt: -------------------------------------------------------------------------------- 1 | /* Custom Node Launcher usage */ 2 | Compile / mainClass := Some("thisClassNotExist") 3 | //Compile / mainClass := Some("it.agilelab.bigdata.wasp.consumers.spark.launcher.SparkConsumersStreamingNodeLauncher") 4 | //Compile / mainClass := Some("it.agilelab.bigdata.wasp.consumers.spark.launcher.SparkConsumersBatchNodeLauncher") -------------------------------------------------------------------------------- /consumers-spark/src/main/scala-spark-3.4/it/agilelab/bigdata/wasp/consumers/spark/utils/RowEncoderUtils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import org.apache.spark.sql.catalyst.encoders.RowEncoder 4 | import org.apache.spark.sql.types.StructType 5 | import org.apache.spark.sql.{Encoder, Row} 6 | 7 | object RowEncoderUtils { 8 | def encoderFor(schema: StructType): Encoder[Row] = { 9 | RowEncoder.apply(schema) 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/http/Enricher.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.http 2 | 3 | import scala.reflect.ClassTag 4 | 5 | trait Enricher extends AutoCloseable { 6 | def call[A: ClassTag, B: ClassTag]( 7 | body: A, 8 | params: Map[String, String], 9 | headers: Map[String, String] 10 | ): B 11 | } 12 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/http/response/FromHttpResponse.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.http.response 2 | 3 | import org.apache.http.HttpResponse 4 | 5 | import scala.reflect.ClassTag 6 | 7 | trait FromHttpResponse { 8 | def fromResponse[A: ClassTag](response: HttpResponse): A 9 | } 10 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/http/utils/FormattedPathParams.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.http.utils 2 | 3 | case class FormattedPathParams( 4 | varKey: String, 5 | valueKey: String 6 | ) { 7 | def formattedVarKey: String = "${"+varKey+"}" 8 | } -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/metadata/Metadata.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.metadata 2 | 3 | import org.apache.spark.sql.Row 4 | 5 | object Path{ 6 | def apply(r: Row): Path = { 7 | Path(r.getString(0), r.getLong(1)) 8 | } 9 | } 10 | 11 | case class Path(name: String, ts: Long) 12 | case class Metadata(id: String, 13 | sourceId: String, 14 | arrivalTimestamp: Long, 15 | lastSeenTimestamp: Long, 16 | path: Array[Path]) 17 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/DropKafkaMetadata.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.strategies 2 | 3 | import org.apache.spark.sql.DataFrame 4 | 5 | /** 6 | * A simple strategy that drops the kafkaMetadata column. 7 | */ 8 | class DropKafkaMetadata extends Strategy { 9 | override def transform(dataFrames: Map[ReaderKey, DataFrame]): DataFrame = { 10 | val df = dataFrames.head._2 11 | df.drop("kafkaMetadata") 12 | } 13 | } 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/InternalStrategy.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.strategies 2 | 3 | trait InternalStrategy extends Strategy 4 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/gdpr/exception/ConfigExceptions.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.strategies.gdpr.exception 2 | 3 | import com.typesafe.config.ConfigException 4 | 5 | object ConfigExceptions { 6 | 7 | case class RawDataConfigException(configException: ConfigException, 8 | message: String = "Missing mandatory configuration key") extends Exception(message, configException) 9 | 10 | case class KeyValueConfigException(throwable: Throwable, message: String) extends Exception(message, throwable) 11 | 12 | } 13 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/gdpr/utils/hbase/HBaseUtils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.strategies.gdpr.utils.hbase 2 | 3 | import org.apache.hadoop.hbase.TableName 4 | import org.apache.hadoop.hbase.client.{Connection, Delete, Table} 5 | 6 | import scala.util.Try 7 | 8 | object HBaseUtils { 9 | 10 | def getTable(connection: Connection)(tableName: String): Try[Table] = { 11 | Try(connection.getTable(TableName.valueOf(tableName))) 12 | } 13 | 14 | def deleteRow(table: Table)(rowKey: Array[Byte]): Try[Unit] = { 15 | Try(table.delete(new Delete(rowKey))) 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/streaming/actor/etl/MonitorOutcome.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.streaming.actor.etl 2 | 3 | import org.apache.spark.sql.streaming.{StreamingQueryException, StreamingQueryProgress, StreamingQueryStatus} 4 | 5 | case class MonitorOutcome(isActive: Boolean, 6 | status: StreamingQueryStatus, 7 | progress: Option[StreamingQueryProgress], 8 | option: Option[StreamingQueryException]) 9 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/streaming/actor/master/FifoSchedulingStrategyFactory.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.streaming.actor.master 2 | 3 | import scala.collection.immutable.Queue 4 | 5 | class FifoSchedulingStrategyFactory extends SchedulingStrategyFactory { 6 | override def create: SchedulingStrategy = FifoSchedulingStrategy(Queue.empty) 7 | } 8 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/streaming/actor/master/SchedulingStrategy.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.streaming.actor.master 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.streaming.actor.master.Data.Collaborator 4 | import it.agilelab.bigdata.wasp.models.PipegraphModel 5 | 6 | object SchedulingStrategy { 7 | 8 | type SchedulingStrategyOutcome = Either[(String, SchedulingStrategy), (Collaborator, SchedulingStrategy)] 9 | 10 | } 11 | 12 | trait SchedulingStrategy { 13 | 14 | def choose(members: Set[Collaborator], pipegraph: PipegraphModel): SchedulingStrategy.SchedulingStrategyOutcome 15 | 16 | } 17 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/streaming/actor/master/SchedulingStrategyFactory.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.streaming.actor.master 2 | 3 | import com.typesafe.config.Config 4 | 5 | trait SchedulingStrategyFactory { 6 | 7 | def inform(factoryParams: Config): SchedulingStrategyFactory = this 8 | 9 | def create: SchedulingStrategy 10 | } 11 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/utils/CompatibilityAvroDeserializerExpression.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import org.apache.spark.sql.catalyst.expressions.Expression 4 | 5 | trait CompatibilityAvroDeserializerExpression { 6 | self: AvroDeserializerExpression => 7 | override protected def withNewChildInternal(newChild: Expression): Expression = { 8 | copy(child = newChild) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/utils/CompatibilityCompressExpression.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import org.apache.spark.sql.catalyst.expressions.Expression 4 | 5 | trait CompatibilityCompressExpression { 6 | self: CompressExpression => 7 | override protected def withNewChildInternal(newChild: Expression): Expression = { 8 | copy(child = newChild) 9 | } 10 | } -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/utils/CompatibilityDecodeUsingAvro.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import org.apache.spark.sql.catalyst.expressions.Expression 4 | 5 | trait CompatibilityDecodeUsingAvro[A] { 6 | self: DecodeUsingAvro[A] => 7 | override protected def withNewChildInternal(newChild: Expression): Expression = { 8 | copy(child = newChild) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/utils/CompatibilityEncodeUsingAvro.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import org.apache.spark.sql.catalyst.expressions.Expression 4 | 5 | trait CompatibilityEncodeUsingAvro[A] { 6 | self: EncodeUsingAvro[A] => 7 | override protected def withNewChildInternal(newChild: Expression): Expression = { 8 | copy(child = newChild) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/utils/HadoopConfiguration.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import java.io.{ ObjectInputStream, ObjectOutputStream } 4 | 5 | import org.apache.hadoop.conf.Configuration 6 | 7 | /** 8 | * [[Serializable]] wrapper for a Hadoop [[Configuration]] 9 | */ 10 | class HadoopConfiguration(@transient var value: Configuration) extends Serializable { 11 | private def writeObject(out: ObjectOutputStream): Unit = 12 | value.write(out) 13 | 14 | private def readObject(in: ObjectInputStream): Unit = { 15 | value = new Configuration(false) 16 | value.readFields(in) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/utils/ToolBoxUtils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.utils 2 | 3 | import scala.tools.reflect.ToolBox 4 | import scala.reflect.runtime.universe 5 | 6 | object ToolBoxUtils { 7 | 8 | private lazy val toolBox: ToolBox[universe.type] = universe.runtimeMirror(getClass.getClassLoader).mkToolBox() 9 | 10 | def compileCode[T](code : String) : T = toolBox.eval(toolBox.parse(s"""$code""".stripMargin)).asInstanceOf[T] 11 | 12 | } 13 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/writers/SparkWriters.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.writers 2 | 3 | import org.apache.spark.sql.{DataFrame, Row} 4 | import org.apache.spark.sql.streaming.DataStreamWriter 5 | 6 | trait SparkStructuredStreamingWriter { 7 | def write(stream: DataFrame): DataStreamWriter[Row] 8 | } 9 | 10 | trait SparkBatchWriter { 11 | def write(data: DataFrame) : Unit 12 | } 13 | -------------------------------------------------------------------------------- /consumers-spark/src/main/scala/org/apache/spark/UtilsForwarder.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark 2 | 3 | import java.net.URI 4 | 5 | object UtilsForwarder { 6 | 7 | def resolveURI(path: String) : URI= 8 | org.apache.spark.util.Utils.resolveURI(path) 9 | 10 | } 11 | -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/data/.part-00000-155c21d9-3623-439d-9943-02754fe3bb80-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/data/.part-00000-155c21d9-3623-439d-9943-02754fe3bb80-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/data/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/data/_SUCCESS -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/data/part-00000-155c21d9-3623-439d-9943-02754fe3bb80-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/data/part-00000-155c21d9-3623-439d-9943-02754fe3bb80-c000.snappy.parquet -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/_SUCCESS -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/category=111/.part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/category=111/.part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/category=111/part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/category=111/part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/category=222/.part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/category=222/.part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/category=222/part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/category=222/part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/category=333/.part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/category=333/.part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/datawithdate/category=333/part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/datawithdate/category=333/part-00000-40d6f8a5-dc70-423c-8c31-a000703657be.c000.snappy.parquet -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/input/runId=1/.part-00000-57e1fc7f-43e4-4f77-9fbe-580c63a751a9-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/consumers-spark/src/test/resources/gdpr/input/runId=1/.part-00000-57e1fc7f-43e4-4f77-9fbe-580c63a751a9-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/gdpr/input/runId=1/part-00000-57e1fc7f-43e4-4f77-9fbe-580c63a751a9-c000.snappy.parquet: -------------------------------------------------------------------------------- 1 | PAR102,k3k14k1k2k3,H spark_schema %key%& 5keyjl&<k3k1j)org.apache.spark.sql.parquet.row.metadataY{"type":"struct","fields":[{"name":"key","type":"string","nullable":true,"metadata":{}}]}9parquet-mr version 1.5.0-cdh5.13.3 (build ${buildNumber})PAR1 -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/velocitytemplates/template1.vm: -------------------------------------------------------------------------------- 1 | TEMPLATE 1 2 | eventId = ${eventId} 3 | eventType = ${eventType} 4 | severity = ${severity} 5 | payload = ${payload} 6 | timestamp = ${timestamp} 7 | source = ${source} 8 | sourceId = ${sourceId} 9 | ruleName = ${ruleName} 10 | 11 | -------------------------------------------------------------------------------- /consumers-spark/src/test/resources/velocitytemplates/template2.vm: -------------------------------------------------------------------------------- 1 | TEMPLATE 1 2 | eventId = ${eventId} 3 | eventType = ${eventType} 4 | severity = ${severity} 5 | payload = ${payload} 6 | timestamp = ${timestamp} 7 | source = ${source} 8 | sourceId = ${sourceId} 9 | ruleName = ${ruleName} -------------------------------------------------------------------------------- /consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/http/data/EnrichedData.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.http.data 2 | 3 | case class EnrichedData(id: String, exampleAuthor: String, text: String, timestamp: Long) 4 | -------------------------------------------------------------------------------- /core/src/main/java/it/agilelab/bigdata/wasp/core/utils/AvroConversionException.java: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.utils; 2 | 3 | 4 | import org.apache.avro.AvroRuntimeException; 5 | 6 | public class AvroConversionException extends AvroRuntimeException { 7 | 8 | public AvroConversionException(String message) { 9 | super(message); 10 | } 11 | 12 | public AvroConversionException(String message, Throwable cause) { 13 | super(message, cause); 14 | } 15 | } -------------------------------------------------------------------------------- /core/src/main/java/it/agilelab/bigdata/wasp/core/utils/CompatibilityJsonGenerator.java: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonGenerator; 4 | 5 | public abstract class CompatibilityJsonGenerator extends JsonGenerator { 6 | public JsonGenerator getJsonGenerator() { 7 | return this; 8 | } 9 | } -------------------------------------------------------------------------------- /core/src/main/java/it/agilelab/bigdata/wasp/core/utils/CompatibilityObjectMapper.java: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.utils; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | 5 | public class CompatibilityObjectMapper extends ObjectMapper {} -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/utils/ClasspathUtils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.utils 2 | 3 | object ClasspathUtils { 4 | def locations(clz: Class[_], classLoader:ClassLoader): List[String] = { 5 | val enums = classLoader.getResources(clz.getCanonicalName.replace(".","/")) 6 | val list = List.newBuilder[String] 7 | while (enums.hasMoreElements){ 8 | list. += (enums.nextElement().toString) 9 | } 10 | list.result() 11 | } 12 | def locations(clz: Class[_]): List[String] = { 13 | locations(clz, getClass.getClassLoader) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/utils/NonEmptyList.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.utils 2 | 3 | case class NonEmptyList[+A](head: A, tail: List[A]) { 4 | val size: Int = tail.size + 1 5 | 6 | def map[B](f: A => B): NonEmptyList[B] = { 7 | NonEmptyList(f(head), tail.map(f)) 8 | } 9 | 10 | def mkString(sep: String): String = { 11 | if (tail.isEmpty) { 12 | "" + head 13 | } else { 14 | head + sep + tail.mkString(sep) 15 | } 16 | } 17 | 18 | def exists(p: A => Boolean): Boolean = { 19 | p(head) || tail.exists(p) 20 | } 21 | } 22 | object NonEmptyList { 23 | def one[A](a: A): NonEmptyList[A] = NonEmptyList(a, Nil) 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/ReaderKey.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.strategies 2 | 3 | case class ReaderKey(sourceTypeName: String, name: String) 4 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/AroundLaunch.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core 2 | 3 | trait AroundLaunch { 4 | def beforeLaunch(): Unit 5 | 6 | def afterLaunch(): Unit 7 | } 8 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/WaspMessage.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core 2 | 3 | trait WaspMessage {} -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/exceptions/ModelNotFound.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.exceptions 2 | 3 | /** 4 | * Created by Agile Lab s.r.l. on 04/11/2017. 5 | */ 6 | class ModelNotFound(s: String) extends RuntimeException(s: String) {} 7 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/hbase/HBaseAdminActor.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.hbase 2 | 3 | import akka.actor.Actor 4 | import it.agilelab.bigdata.wasp.core.logging.Logging 5 | 6 | object HBaseAdminActor { 7 | 8 | val name = "HBaseAdminActor" 9 | } 10 | 11 | class HBaseAdminActor extends Actor with Logging { 12 | 13 | override def receive: Actor.Receive = { 14 | case message: Any => logger.error("unknown message: " + message) 15 | } 16 | } -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/launcher/MasterCommandLineOptions.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.launcher 2 | 3 | import org.apache.commons.cli.{Option => CliOption} 4 | 5 | 6 | /** 7 | * @author Nicolò Bidotti 8 | */ 9 | object MasterCommandLineOptions { 10 | def dropDb: CliOption = new CliOption("d", "drop-db", false, "Drop MongoDB database") 11 | 12 | def allOptions: Seq[CliOption] = Seq(dropDb) 13 | } 14 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/launcher/WaspCommandLineOptions.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.launcher 2 | 3 | import org.apache.commons.cli.{Option => CliOption} 4 | 5 | 6 | object WaspCommandLineOptions { 7 | def version: CliOption = new CliOption("v", "version", false, "Print version and exit") 8 | def help: CliOption = new CliOption("h", "help", false, "Print help and exit") 9 | 10 | val requiredOptions = Seq.empty[CliOption] 11 | val otherOptions = Seq( 12 | version, 13 | help 14 | ) 15 | 16 | def allOptions: Seq[CliOption] = requiredOptions ++ otherOptions 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/logging/GuardedLogging.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.logging 2 | 3 | /** 4 | * Trait useful to log in spark. this traits correctly handles spark serializability of closures. 5 | */ 6 | trait GuardedLogging extends Serializable { 7 | 8 | protected def loggerName: String = this.getClass.getCanonicalName 9 | 10 | @transient protected lazy val log: WaspLogger = WaspLogger(this.getClass) 11 | 12 | } 13 | 14 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/messages/ClusterMessages.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.messages 2 | 3 | import it.agilelab.bigdata.wasp.core.WaspMessage 4 | 5 | case object DownUnreachableMembers extends WaspMessage 6 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/messages/ConsumersMessages.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.messages 2 | 3 | import it.agilelab.bigdata.wasp.core.WaspMessage 4 | 5 | 6 | case object RestartConsumers extends WaspMessage 7 | 8 | case object OutputStreamInitialized extends WaspMessage 9 | 10 | case object StopProcessingComponent extends WaspMessage 11 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/messages/GenericMessages.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.messages 2 | 3 | import akka.http.scaladsl.model.HttpMethod 4 | import it.agilelab.bigdata.wasp.core.WaspMessage 5 | import spray.json.JsValue 6 | 7 | case object Start extends WaspMessage 8 | 9 | case object Stop extends WaspMessage 10 | 11 | case class ModelKey(name: String, version: String, timestamp: Long) 12 | 13 | case class RestRequest(httpMethod: HttpMethod, data: JsValue, model: ModelKey) extends WaspMessage -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/messages/ProducersMessages.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.messages 2 | 3 | import it.agilelab.bigdata.wasp.core.WaspMessage 4 | 5 | 6 | case class WaspMessageEnvelope[K, V](topic: String, key: K, messages: V*) extends WaspMessage -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/messages/TelemetryMessageJsonProtocol.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.messages 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import spray.json.{DefaultJsonProtocol, RootJsonFormat} 5 | 6 | /** 7 | * @author Eugenio Liso 8 | */ 9 | object TelemetryMessageJsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { 10 | implicit val telemetryMessageSource: RootJsonFormat[TelemetryMessageSource] = jsonFormat6(TelemetryMessageSource.apply) 11 | implicit val telemetryMessageSources: RootJsonFormat[TelemetryMessageSourcesSummary] = jsonFormat1(TelemetryMessageSourcesSummary.apply) 12 | } 13 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/models/configuration/ValidationRule.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.models.configuration 2 | 3 | import it.agilelab.bigdata.wasp.core.utils.ConfigManager 4 | 5 | class ValidationRule(val key: String, val func: (ConfigManager) => Either[String, Unit]) 6 | 7 | object ValidationRule { 8 | def apply(key: String) 9 | (func: (ConfigManager) => Either[String, Unit]): ValidationRule = new ValidationRule(key, func) 10 | } -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/utils/CanOverrideName.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.utils 2 | 3 | trait CanOverrideName[T] { 4 | def named(instance: T, name: String): T 5 | } 6 | 7 | object CanOverrideName { 8 | def apply[T](f: (T, String) => T) = new CanOverrideName[T] { 9 | override def named(instance: T, name: String): T = f(instance, name) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/utils/JsonConverter.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.utils 2 | 3 | import org.mongodb.scala.bson.BsonDocument 4 | 5 | object JsonConverter { 6 | 7 | def toString(doc: BsonDocument): String = { 8 | doc.toJson() 9 | } 10 | 11 | def fromString(doc: String): Option[BsonDocument] = { 12 | try { 13 | Some(BsonDocument.apply(doc)) 14 | }catch { 15 | case e: Exception => None 16 | } 17 | 18 | } 19 | 20 | 21 | } -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/core/utils/Utils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.core.utils 2 | 3 | object Utils { 4 | def using[A <: AutoCloseable, B](a: A)(f: A => B): B = 5 | try { 6 | f(a) 7 | } finally { 8 | a.close() 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /core/src/main/scala/it/agilelab/bigdata/wasp/spark/sql/kafka011/KafkaHeader.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.sql.kafka011 2 | 3 | case class KafkaHeader(headerKey: String, headerValue: Array[Byte]) 4 | -------------------------------------------------------------------------------- /core/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /core/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /documentation/Using_WASP.md: -------------------------------------------------------------------------------- 1 | ## Using WASP 2 | 3 | * [Getting started](Getting_started.md) 4 | * [Extension points](Ext_points.md) 5 | * [REST API](REST_API.md) 6 | * [Operations](ops.md) 7 | * [Monitoring](monitoring.md) 8 | * [Configuration](configuration.md) 9 | * [Parallel Write Sink](Parallel_write.md) -------------------------------------------------------------------------------- /documentation/building/restart_and_refresh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/building/restart_and_refresh.png -------------------------------------------------------------------------------- /documentation/building/select_sbt_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/building/select_sbt_settings.png -------------------------------------------------------------------------------- /documentation/building/set_vm_parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/building/set_vm_parameters.png -------------------------------------------------------------------------------- /documentation/diagrams/Wasp1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/Wasp1.png -------------------------------------------------------------------------------- /documentation/diagrams/Wasp2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/Wasp2.png -------------------------------------------------------------------------------- /documentation/diagrams/actor_system.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/actor_system.png -------------------------------------------------------------------------------- /documentation/diagrams/components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/components.png -------------------------------------------------------------------------------- /documentation/diagrams/pipegraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/pipegraph.png -------------------------------------------------------------------------------- /documentation/diagrams/pipegraph_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/pipegraph_model.png -------------------------------------------------------------------------------- /documentation/diagrams/statemachines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/diagrams/statemachines.png -------------------------------------------------------------------------------- /documentation/icons/WASP_logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/icons/WASP_logo.jpg -------------------------------------------------------------------------------- /documentation/icons/WASP_logo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/icons/WASP_logo.pdf -------------------------------------------------------------------------------- /documentation/icons/WASP_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/icons/WASP_logo.png -------------------------------------------------------------------------------- /documentation/images/EventEngine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/images/EventEngine.png -------------------------------------------------------------------------------- /documentation/images/EventEngineOverview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/images/EventEngineOverview.png -------------------------------------------------------------------------------- /documentation/images/EventPipegraphExample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/images/EventPipegraphExample.png -------------------------------------------------------------------------------- /documentation/images/MailingPipegraphExample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/images/MailingPipegraphExample.png -------------------------------------------------------------------------------- /documentation/images/wasp-branching-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/documentation/images/wasp-branching-model.png -------------------------------------------------------------------------------- /documentation/troubleshooting.md: -------------------------------------------------------------------------------- 1 | ### Troubleshooting 2 | 3 | IN PROGRESS -------------------------------------------------------------------------------- /documentation/whitelabel/cdp.md: -------------------------------------------------------------------------------- 1 | ## CDP Private Deployment on a Single node - NOTES 2 | 3 | TBD -------------------------------------------------------------------------------- /documentation/whitelabel/whitelabel.md: -------------------------------------------------------------------------------- 1 | ## Whitelabel 2 | 3 | Whitelabel is an example project used to test the WASP framework. 4 | 5 | Applications using WASP should resemble the whitelable structure. -------------------------------------------------------------------------------- /kernel/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /kernel/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /master/build.sbt: -------------------------------------------------------------------------------- 1 | Compile / mainClass := Some("it.agilelab.bigdata.wasp.master.launcher.MasterNodeLauncher") -------------------------------------------------------------------------------- /master/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | wasp { 2 | akka { 3 | cluster { 4 | roles = ["master"] 5 | } 6 | } 7 | 8 | } 9 | -------------------------------------------------------------------------------- /master/src/main/scala/akka/http/extension/SecurityUtils.scala: -------------------------------------------------------------------------------- 1 | package akka.http.extension 2 | 3 | import akka.http.impl.util.EnhancedString 4 | import java.nio.charset.StandardCharsets 5 | import java.security.MessageDigest 6 | 7 | trait SecurityUtils { 8 | /** redefined from akka.http.impl.utils since it is private. 9 | */ 10 | implicit def enhanceString_(s: String): EnhancedString = new EnhancedString(s) 11 | 12 | def hash(input: String): String = MessageDigest 13 | .getInstance("SHA-256") 14 | .digest(input.getBytes(StandardCharsets.UTF_8)) 15 | .map("%02X" format _) 16 | .mkString 17 | } 18 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/security/ApiKeyAuthenticationVerifierImpl.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.security 2 | 3 | /** Implementation of [[ApiKeyAuthenticationVerifier]] that verifies on a list of hashed keys 4 | */ 5 | class ApiKeyAuthenticationVerifierImpl(keyList: Iterable[String]) extends ApiKeyAuthenticationVerifier { 6 | override protected lazy val allowedKeys: Iterable[String] = keyList 7 | } 8 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/security/common/AuthenticationProvider.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.security.common 2 | 3 | import akka.http.scaladsl.server.directives.AuthenticationDirective 4 | 5 | trait AuthenticationProvider { 6 | self: CredentialsVerifier => 7 | def authentication: AuthenticationDirective[Identity] 8 | } 9 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/security/common/AuthenticationService.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.security.common 2 | 3 | import akka.http.scaladsl.server.directives.AuthenticationDirective 4 | 5 | trait AuthenticationService { 6 | self: AuthenticationProvider with CredentialsVerifier => 7 | def authenticate: AuthenticationDirective[Identity] = authentication 8 | } 9 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/security/common/CredentialsVerifier.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.security.common 2 | 3 | import akka.http.scaladsl.server.Directives.AsyncAuthenticatorPF 4 | 5 | trait CredentialsVerifier { 6 | def verifyCredentials: AsyncAuthenticatorPF[Identity] 7 | } 8 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/security/common/Identity.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.security.common 2 | 3 | case class Identity(identifier: String) 4 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/web/models/PaginationInfo.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.models 2 | 3 | case class PaginationInfo(page: Integer, 4 | rows : Integer, 5 | numFound : Long) 6 | -------------------------------------------------------------------------------- /master/src/main/scala/it/agilelab/bigdata/wasp/master/web/models/RestProducerModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.models 2 | 3 | import it.agilelab.bigdata.wasp.core.messages.ModelKey 4 | import spray.json.{DefaultJsonProtocol, JsValue, RootJsonFormat} 5 | 6 | case class RestProducerModel(httpMethod: String, data: JsValue, mlModel: ModelKey) 7 | 8 | object RestProducerModelJsonProtocol extends DefaultJsonProtocol { 9 | 10 | implicit val modelKey = jsonFormat3(ModelKey) 11 | implicit def nifiPlatform: RootJsonFormat[RestProducerModel] = jsonFormat3(RestProducerModel.apply) 12 | } -------------------------------------------------------------------------------- /master/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /master/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/BatchSchedulerModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import org.mongodb.scala.bson.BsonDocument 4 | 5 | case class BatchSchedulerModel (override val name: String, 6 | cronExpression: String, 7 | batchJob: Option[String], 8 | options: Option[BsonDocument] = None, 9 | isActive: Boolean = true) extends Model -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/CompletionModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | case class CompletionModel(toComplete : String,info : String ) { 4 | override def toString(): String = s"$toComplete : $info" 5 | } 6 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/CountEntry.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import java.time.Instant 4 | 5 | case class CountEntry(timestamp: Instant, count: Map[String, Int]) 6 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/Counts.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | case class Counts(logs: Seq[CountEntry], telemetry: Seq[CountEntry], events: Seq[CountEntry]) 4 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/DatastoreModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import it.agilelab.bigdata.wasp.datastores.{DatastoreProduct} 4 | 5 | /** 6 | * Base datastore model. 7 | * 8 | * @author Nicolò Bidotti 9 | */ 10 | abstract class DatastoreModel extends Model { 11 | def datastoreProduct: DatastoreProduct 12 | } 13 | 14 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/DocumentModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import it.agilelab.bigdata.wasp.datastores.DatastoreProduct.{MongoDbProduct} 4 | import it.agilelab.bigdata.wasp.datastores.DatastoreProduct 5 | 6 | case class DocumentModel(override val name: String, connectionString: String, schema: String) extends DatastoreModel { 7 | override def datastoreProduct: DatastoreProduct = MongoDbProduct 8 | } 9 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/ErrorModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | case class ErrorModel(fileName: String, where : String,errorType : String, msg :String,content:String,indicator :String) { 4 | override def toString() : String = {f"$fileName:$where: $errorType:$msg%n$content%n$indicator"} 5 | } 6 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/EventEntry.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import java.time.Instant 4 | 5 | 6 | case class EventEntry(eventType: String, 7 | eventId: String, 8 | severity: String, 9 | payload: String, 10 | timestamp: Instant, 11 | source:String, 12 | sourceId: String, 13 | eventRuleName: String) 14 | 15 | case class Events(found: Long, entries: Seq[EventEntry]) 16 | 17 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/FreeCodeModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | 4 | case class FreeCodeModel(override val name : String, code : String) extends Model 5 | 6 | case class FreeCode(code : String) 7 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/IndexType.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | 4 | object IndexType extends Enumeration { 5 | type Type = Value 6 | val SOLR, ELASTIC = Value 7 | } -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/LogEntry.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import java.time.Instant 4 | 5 | case class LogEntry(log_source: String, 6 | log_level: String, 7 | message: String, 8 | timestamp: Instant, 9 | thread: String, 10 | cause: Option[String] = None, 11 | stacktrace: Option[String] = None) 12 | 13 | case class Logs(found: Long, entries: Seq[LogEntry]) -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/MetadataModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | case class PathModel ( 4 | val name: String, 5 | val ts: Long 6 | ) 7 | 8 | case class MetadataModel( 9 | val id: String, 10 | val sourceId: String, 11 | val arrivalTimestamp: Long, 12 | val lastSeenTimestamp: Long, 13 | val path: Array[PathModel] 14 | ) 15 | 16 | trait Metadata{ 17 | val metadata: MetadataModel 18 | } -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/MlModelOnlyInfo.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import org.mongodb.scala.bson.BsonObjectId 4 | 5 | case class MlModelOnlyInfo( 6 | name: String, 7 | version: String, 8 | className: Option[String] = None, 9 | timestamp: Option[Long] = None, 10 | modelFileId: Option[BsonObjectId] = None, 11 | favorite: Boolean = false, 12 | description: String = "" 13 | ) extends Model 14 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/Model.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | trait Model { 4 | val name: String 5 | } 6 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/ProcessGroupModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import org.mongodb.scala.bson.BsonDocument 4 | 5 | case class ProcessGroupModel(name: String, content: BsonDocument, errorPort: String) extends Model -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/WebMailModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import it.agilelab.bigdata.wasp.datastores.DatastoreProduct.WebMailProduct 4 | import it.agilelab.bigdata.wasp.datastores.{DatastoreProduct} 5 | 6 | case class WebMailModel (override val name: String) 7 | extends DatastoreModel { 8 | override val datastoreProduct: DatastoreProduct = WebMailProduct 9 | } -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/WebsocketModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models 2 | 3 | import it.agilelab.bigdata.wasp.datastores.DatastoreProduct.WebSocketProduct 4 | import it.agilelab.bigdata.wasp.datastores.{DatastoreProduct} 5 | import org.mongodb.scala.bson.BsonDocument 6 | 7 | 8 | case class WebsocketModel (override val name: String, 9 | host: String, 10 | port: String, 11 | resourceName: String, 12 | options: Option[BsonDocument] = None) 13 | extends DatastoreModel { 14 | override def datastoreProduct: DatastoreProduct = WebSocketProduct 15 | } -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/CompilerConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | case class CompilerConfigModel(compilerInstances: Int, name: String) extends Model 6 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/ElasticConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | /** 6 | * Configuration model for ElasticSearch. 7 | * 8 | */ 9 | case class ElasticConfigModel( 10 | connections: Seq[ConnectionConfig], 11 | name: String 12 | ) extends Model -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/HBaseConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | /** 6 | * Configuration model for HBase. 7 | * 8 | */ 9 | case class HBaseConfigModel( 10 | coreSiteXmlPath: String, 11 | hbaseSiteXmlPath: String, 12 | others: Seq[HBaseEntryConfig], 13 | name: String 14 | ) extends Model 15 | 16 | case class HBaseEntryConfig( 17 | key: String, 18 | value: String 19 | ) -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/MongoDBConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | case class MongoDBConfigModel(address: String, 4 | databaseName: String, 5 | username: String, 6 | password: String, 7 | credentialDb: String, 8 | millisecondsTimeoutConnection: Int, 9 | collectionPrefix: String) 10 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/NifiConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | case class NifiConfigModel( 6 | nifiBaseUrl: String, 7 | nifiApiPath: String, 8 | nifiUiPath: String, 9 | name: String 10 | ) extends Model 11 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/ParsingMode.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | sealed abstract class ParsingMode(val mode: String) 4 | 5 | case object Strict extends ParsingMode("strict") 6 | 7 | case object Ignore extends ParsingMode("ignore") 8 | 9 | case object Handle extends ParsingMode("handle") 10 | 11 | object ParsingMode { 12 | val map = List(Strict, Ignore, Handle).map(x => (x.mode, x)).toMap 13 | 14 | def asString(parsingMode: ParsingMode): String = parsingMode.mode 15 | 16 | def fromString(parsingMode: String): Option[ParsingMode] = map.get(parsingMode) 17 | } 18 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/PostgresDBConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | case class PostgresDBConfigModel(url: String, 4 | user: String, 5 | password: String, 6 | driver: String, 7 | poolSize : Int ) 8 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/RestEnrichmentConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | case class RestEnrichmentConfigModel(sources: Map[String, RestEnrichmentSource]) 4 | 5 | case class RestEnrichmentSource( 6 | kind: String, 7 | parameters: Map[String, String], 8 | headers: Map[String, String] = Map.empty 9 | ) -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/SolrConfigModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | /** 6 | * Configuration model for Solr. 7 | * 8 | */ 9 | case class SolrConfigModel( 10 | zookeeperConnections: ZookeeperConnectionsConfig, 11 | name: String 12 | ) extends Model -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/configuration/ZookeeperConnectionsConfig.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.configuration 2 | 3 | case class ZookeeperConnectionsConfig(connections: Seq[ConnectionConfig], chRoot: String) { 4 | override def toString = connections.map(conn => s"${conn.host}:${conn.port}").mkString(",") + s"${chRoot}" 5 | } 6 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/editor/ErrorDTO.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.editor 2 | 3 | case class ErrorDTO(msg: String) 4 | 5 | object ErrorDTO { 6 | def alreadyExists(entity: String, value: String): ErrorDTO = ErrorDTO(s"$entity already exists: $value") 7 | def unknownArgument(entity: String, value: String): ErrorDTO = ErrorDTO(s"Unknown $entity type: $value") 8 | def notFound(entity: String, value: String): ErrorDTO = ErrorDTO(s"$entity not found: $value") 9 | def illegalArgument(entity: String, value: String): ErrorDTO = ErrorDTO(s"$entity has incorrect value: $value") 10 | } 11 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/models/editor/NifiStatelessInstanceModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.models.editor 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | import org.json4s.JsonAST.JObject 5 | 6 | case class NifiStatelessInstanceModel( 7 | name: String, 8 | url: String, 9 | processGroupId: String 10 | ) extends Model 11 | 12 | case class ProcessGroupResponse( 13 | id: String, 14 | content: JObject 15 | ) 16 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/utils/ConfigManagerHelper.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.utils 2 | 3 | import java.text.SimpleDateFormat 4 | import java.util.Date 5 | 6 | object ConfigManagerHelper { 7 | def buildTimedName(prefix: String): String = { 8 | val result = prefix + "-" + new SimpleDateFormat("yyyy.MM.dd") 9 | .format(new Date()) 10 | 11 | result 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /model/src/main/scala/it/agilelab/bigdata/wasp/utils/DatastoreProductSerde.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.utils 2 | 3 | /** 4 | * Base trait for `DataStoreProduct` serde. 5 | * 6 | * @author Nicolò Bidotti 7 | */ 8 | trait DatastoreProductSerde { 9 | // field names 10 | protected val categoryField = "category" 11 | protected val productField = "product" 12 | } 13 | -------------------------------------------------------------------------------- /model/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /model/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /nifi-client/src/main/scala/it/agilelab/bigdata/nifi/client/model/Runner.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.nifi.client.model 2 | 3 | import it.agilelab.bigdata.nifi.client.api.FlowApi 4 | import it.agilelab.bigdata.nifi.client.core.SttpSerializer 5 | 6 | object Runner { 7 | def main(args: Array[String]): Unit = { 8 | import sttp.client._ 9 | implicit val sttpBackend = HttpURLConnectionBackend() 10 | implicit val s = new SttpSerializer() 11 | println(FlowApi("http://localhost:8080/nifi-api").getRegistries().send()) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /nifi-client/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /nifi-client/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/AngularResponse.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import it.agilelab.bigdata.wasp.master.web.openapi.ResultIndicator.ResultIndicator 4 | 5 | case class AngularResponse[T](Result: ResultIndicator, data: T) 6 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/AngularResponseOpenApiComponentSupport.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import it.agilelab.bigdata.wasp.master.web.openapi.ResultIndicator.ResultIndicator 4 | 5 | trait AngularResponseOpenApiComponentSupport 6 | extends ProductOpenApi 7 | with EnumOpenApi { 8 | implicit def angularResponseOpenApi[T: ToOpenApiSchema] 9 | : ToOpenApiSchema[AngularResponse[T]] = product2(AngularResponse.apply[T]) 10 | 11 | implicit val resultEnumOpenApi: ToOpenApiSchema[ResultIndicator] = 12 | enumOpenApi(ResultIndicator) 13 | } 14 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/BatchJobStartResult.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | case class BatchJobStartResult(startResult: String, instance: String) 4 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/BsonDocumentOpenApiDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import org.mongodb.scala.bson.BsonDocument 4 | 5 | trait BsonDocumentOpenApiDefinition extends ProductOpenApi with LangOpenApi with CollectionsOpenApi { 6 | 7 | implicit lazy val bsonDocumentOpenApi: ToOpenApiSchema[BsonDocument] = 8 | objectOpenApi.mapSchema((ctx, schema) => schema.name("BsonDocument")) 9 | } 10 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/DataStoreOpenApiComponentSupport.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import it.agilelab.bigdata.wasp.datastores.DatastoreProduct 4 | 5 | trait DataStoreOpenApiComponentSupport extends OpenApiSchemaSupport with LangOpenApi { 6 | implicit lazy val dataStoreProductOpenApi: ToOpenApiSchema[DatastoreProduct] = stringOpenApi.substituteOf[DatastoreProduct] 7 | } 8 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/ProducerOpenApiComponentSupport.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import it.agilelab.bigdata.wasp.models.ProducerModel 4 | 5 | trait ProducerOpenApiComponentSupport extends ProductOpenApi with LangOpenApi with CollectionsOpenApi { 6 | implicit lazy val producerModel : ToOpenApiSchema[ProducerModel] = product7(ProducerModel) 7 | } 8 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/ResultIndicator.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | object ResultIndicator extends Enumeration { 4 | type ResultIndicator = Value 5 | 6 | val OK, KO = Value 7 | } 8 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/Schemas.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import com.esotericsoftware.kryo.Kryo 4 | import io.swagger.v3.oas.models.media.Schema 5 | 6 | object Schemas { 7 | def copy(schema: Schema[_]): Schema[_] = { 8 | 9 | val kryo = new Kryo 10 | kryo.copy(schema) 11 | 12 | } 13 | 14 | 15 | 16 | } 17 | -------------------------------------------------------------------------------- /openapi/src/main/scala/it/agilelab/bigdata/wasp/master/web/openapi/StrategyModelOpenApiComponentSupport.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.master.web.openapi 2 | 3 | import it.agilelab.bigdata.wasp.models.StrategyModel 4 | 5 | trait StrategyModelOpenApiComponentSupport 6 | extends ProductOpenApi 7 | with CollectionsOpenApi 8 | with LangOpenApi { 9 | implicit lazy val strategyModelOpenApi: ToOpenApiSchema[StrategyModel] = 10 | product2(StrategyModel.apply) 11 | } 12 | -------------------------------------------------------------------------------- /openapi/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /openapi/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /openshift/batch/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WASP_CLASSPATH=`cat /wasp.classpath`:/etc/hbase/conf/:/etc/hadoop/conf/ 4 | 5 | java -cp $WASP_CLASSPATH \ 6 | -Dconfig.file=/docker-environment.conf \ 7 | -Dwasp.process="---batch" \ 8 | -Dwasp.akka.remote.artery.canonical.hostname=${HOSTNAME} \ 9 | -Dwasp.akka.remote.artery.canonical.port=2892 \ 10 | -Dlog4j.configurationFile=file:///log4j2.properties \ 11 | it.agilelab.bigdata.wasp.consumers.spark.launcher.SparkConsumersBatchNodeLauncher -------------------------------------------------------------------------------- /openshift/batch/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://services.example.com:9000 6 | The name of the default file system. A URI whose 7 | scheme and authority determine the FileSystem implementation. The 8 | uri's scheme determines the config property (fs.SCHEME.impl) naming 9 | the FileSystem implementation class. The uri's authority is used to 10 | determine the host, port, etc. for a filesystem. 11 | 12 | -------------------------------------------------------------------------------- /openshift/batch/etc/hbase/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.rootdir 5 | hdfs://services.example.com:9000/hbase 6 | 7 | 8 | hbase.cluster.distributed 9 | true 10 | 11 | 12 | hbase.zookeeper.quorum 13 | services.example.com:2181 14 | 15 | 16 | hbase.client.metrics.enable 17 | true 18 | 19 | -------------------------------------------------------------------------------- /openshift/master/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WASP_CLASSPATH=`cat /wasp.classpath`:/etc/hbase/conf/ 4 | 5 | java -cp $WASP_CLASSPATH \ 6 | -Dconfig.file=/docker-environment.conf \ 7 | -Dwasp.process="---master" \ 8 | -Dwasp.akka.remote.artery.canonical.hostname=${HOSTNAME} \ 9 | -Dwasp.akka.remote.artery.canonical.port=2892 \ 10 | -Dlog4j.configurationFile=file:///log4j2.properties \ 11 | it.agilelab.bigdata.wasp.master.launcher.MasterNodeLauncher -------------------------------------------------------------------------------- /openshift/master/etc/hbase/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.rootdir 5 | hdfs://services.example.com:9000/hbase 6 | 7 | 8 | hbase.cluster.distributed 9 | true 10 | 11 | 12 | hbase.zookeeper.quorum 13 | services.example.com:2181 14 | 15 | 16 | hbase.client.metrics.enable 17 | true 18 | 19 | -------------------------------------------------------------------------------- /openshift/mongodb/mongo.repo: -------------------------------------------------------------------------------- 1 | [mongodb-org-4.2] 2 | name=MongoDB Repository 3 | baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.2/x86_64/ 4 | gpgcheck=1 5 | enabled=1 6 | gpgkey=https://www.mongodb.org/static/pgp/server-4.2.asc -------------------------------------------------------------------------------- /openshift/mongodb/mongodb.txt: -------------------------------------------------------------------------------- 1 | Mongodb is license is regulated by Server Side Public License Version 1 2 | 3 | a full copy of the license is available at this url 4 | 5 | https://www.mongodb.com/licensing/server-side-public-license -------------------------------------------------------------------------------- /openshift/producers/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WASP_CLASSPATH=`cat /wasp.classpath`:/etc/hbase/conf/ 4 | 5 | java -cp $WASP_CLASSPATH \ 6 | -Dconfig.file=/docker-environment.conf \ 7 | -Dwasp.process="---producer" \ 8 | -Dwasp.akka.remote.artery.canonical.hostname=${HOSTNAME} \ 9 | -Dwasp.akka.remote.artery.canonical.port=2892 \ 10 | -Dlog4j.configurationFile=file:///log4j2.properties \ 11 | it.agilelab.bigdata.wasp.producers.launcher.ProducersNodeLauncher -------------------------------------------------------------------------------- /openshift/producers/etc/hbase/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.rootdir 5 | hdfs://services.example.com:9000/hbase 6 | 7 | 8 | hbase.cluster.distributed 9 | true 10 | 11 | 12 | hbase.zookeeper.quorum 13 | services.example.com:2181 14 | 15 | 16 | hbase.client.metrics.enable 17 | true 18 | 19 | -------------------------------------------------------------------------------- /openshift/services/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.gitlab.com/agilefactory/agile.wasp2/cdh-docker:6.3.2-nifi 2 | 3 | COPY docker-entrypoint.sh /docker-entrypoint.sh 4 | 5 | CMD [ "/bin/bash", "/docker-entrypoint.sh" ] -------------------------------------------------------------------------------- /openshift/streaming/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WASP_CLASSPATH=`cat /wasp.classpath`:/etc/hbase/conf/:/etc/hadoop/conf/ 4 | 5 | java -cp $WASP_CLASSPATH \ 6 | -Dconfig.file=/docker-environment.conf \ 7 | -Dwasp.process="---streaming" \ 8 | -Dwasp.akka.remote.artery.canonical.hostname=${HOSTNAME} \ 9 | -Dwasp.akka.remote.artery.canonical.port=2892 \ 10 | -Dlog4j.configurationFile=file:///log4j2.properties \ 11 | it.agilelab.bigdata.wasp.consumers.spark.launcher.SparkConsumersStreamingNodeLauncher -------------------------------------------------------------------------------- /openshift/streaming/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://services.example.com:9000 6 | The name of the default file system. A URI whose 7 | scheme and authority determine the FileSystem implementation. The 8 | uri's scheme determines the config property (fs.SCHEME.impl) naming 9 | the FileSystem implementation class. The uri's authority is used to 10 | determine the host, port, etc. for a filesystem. 11 | 12 | -------------------------------------------------------------------------------- /openshift/streaming/etc/hbase/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.rootdir 5 | hdfs://services.example.com:9000/hbase 6 | 7 | 8 | hbase.cluster.distributed 9 | true 10 | 11 | 12 | hbase.zookeeper.quorum 13 | services.example.com:2181 14 | 15 | 16 | hbase.client.metrics.enable 17 | true 18 | 19 | -------------------------------------------------------------------------------- /plugin-cdc-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.cdc.CdcConsumersSpark -------------------------------------------------------------------------------- /plugin-cdc-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-cdc-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-console-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.console.ConsoleConsumersSpark -------------------------------------------------------------------------------- /plugin-console-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-console-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-elastic-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.elastic.ElasticConsumersSpark -------------------------------------------------------------------------------- /plugin-elastic-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/elastic/ElasticRestClient.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.elastic 2 | 3 | trait ElasticRestClient extends AutoCloseable { 4 | def addAlias(index:String, alias: String) : Boolean 5 | def addIndex(index:String, settings: Option[String] = None): Boolean 6 | def addMapping(index:String, dataType:String, mapping:String): Boolean 7 | def checkIndex(index:String):Boolean 8 | def removeAlias(index: String, alias: String):Boolean 9 | def removeIndex(index: String): Boolean 10 | } 11 | -------------------------------------------------------------------------------- /plugin-elastic-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-elastic-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-hbase-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.hbase.HBaseConsumerSpark -------------------------------------------------------------------------------- /plugin-hbase-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/hbase/HBaseAdminActor.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.hbase 2 | 3 | import akka.actor.Actor 4 | import it.agilelab.bigdata.wasp.core.logging.Logging 5 | 6 | 7 | 8 | object HBaseAdminActor { 9 | 10 | val name = "HBaseAdminActor" 11 | } 12 | 13 | class HBaseAdminActor extends Actor with Logging { 14 | 15 | override def receive: Actor.Receive = { 16 | case message: Any => logger.error("unknown message: " + message) 17 | } 18 | } -------------------------------------------------------------------------------- /plugin-hbase-spark/src/main/scala/org/apache/hadoop/hbase/HBaseInterfaceAudienceSpark.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.hbase; 2 | 3 | /** 4 | * Created by Agile Lab s.r.l. on 04/11/2017. 5 | */ 6 | public class HBaseInterfaceAudienceSpark { 7 | //https://github.com/apache/hbase/blob/e6e52cd80f4ba26b196e2d20cd84ba167b303475/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseInterfaceAudience.java 8 | public static final String SPARK = "Spark"; 9 | } 10 | -------------------------------------------------------------------------------- /plugin-hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DynamicFieldStructure.scala: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.hbase.spark 2 | 3 | object DynamicFieldStructure { 4 | val COLUMN_QUALIFIER = "cq" 5 | val PAYLOAD = "payload" 6 | } 7 | -------------------------------------------------------------------------------- /plugin-hbase-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-hbase-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-http-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.http.HttpConsumersSparkPlugin -------------------------------------------------------------------------------- /plugin-http-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-http-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-jdbc-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.jdbc.JdbcConsumerSpark -------------------------------------------------------------------------------- /plugin-jdbc-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-jdbc-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-kafka-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.kafka.KafkaConsumersSpark -------------------------------------------------------------------------------- /plugin-kafka-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-kafka-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-mailer-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.mailer.MailerConsumerSpark -------------------------------------------------------------------------------- /plugin-mongo-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.mongo.MongoConsumersSpark -------------------------------------------------------------------------------- /plugin-mongo-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-mongo-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.ParallelWriteConsumersSparkPlugin -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | wasp { 2 | plugin { 3 | microservice-catalog { 4 | catalog-class: "it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.catalogservices.MockPlatformCatalogService" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/model/ParallelWriteModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.model 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog.CatalogCoordinates 4 | 5 | 6 | /** 7 | * Model describing a parallel write destination 8 | * @param writerDetails details needed by writer 9 | * @param entityDetails entity coordinates needed to retrieve entity endpoints 10 | */ 11 | case class ParallelWriteModel(writerDetails: WriterDetails, entityDetails: CatalogCoordinates) 12 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/utils/CommitStatus.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.utils 2 | 3 | object CommitStatus { 4 | sealed trait CommitStatus { 5 | val status: String 6 | } 7 | case object Pending extends CommitStatus { 8 | override val status: String = "Pending" 9 | } 10 | case object Success extends CommitStatus { 11 | override val status: String = "Success" 12 | } 13 | case object Failed extends CommitStatus { 14 | override val status: String = "Failed" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/utils/HadoopS3Utils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.utils 2 | 3 | import java.net.URI 4 | 5 | object HadoopS3Utils { 6 | def useS3aScheme(writeUri: URI): URI = { 7 | if (writeUri.getScheme == "s3") 8 | new URI("s3a", writeUri.getUserInfo, writeUri.getHost, writeUri.getPort, 9 | writeUri.getPath, writeUri.getQuery, writeUri.getFragment) 10 | else writeUri 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/MockCredentialProvider.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools 2 | 3 | import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider, AWSSessionCredentials} 4 | 5 | class MockCredentialProvider extends AWSCredentialsProvider { 6 | override def getCredentials: AWSCredentials = new AWSSessionCredentials { 7 | override def getSessionToken: String = "mock" 8 | 9 | override def getAWSAccessKeyId: String = "mock" 10 | 11 | override def getAWSSecretKey: String = "mock" 12 | } 13 | 14 | override def refresh(): Unit = {} 15 | } 16 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/catalogservices/WrongCatalogService.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.catalogservices 2 | 3 | class WrongCatalogService { 4 | } 5 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/MSEntityMockBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog.{EntityCatalogBuilder, EntityCatalogService} 4 | 5 | object MSEntityMockBuilder extends EntityCatalogBuilder { 6 | override def getEntityCatalogService(): EntityCatalogService = { 7 | getEntityCatalogService("plugin.microservice-catalog-msentity.catalog-class") 8 | } 9 | } -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/NotExistingServiceBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | 4 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog._ 5 | 6 | object NotExistingServiceBuilder extends EntityCatalogBuilder { 7 | override def getEntityCatalogService(): EntityCatalogService = { 8 | getEntityCatalogService("plugin.microservice-catalog-notexistingservice.catalog-class") 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/ParameterBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog._ 4 | 5 | object ParameterBuilder extends EntityCatalogBuilder { 6 | override def getEntityCatalogService(): EntityCatalogService = { 7 | getEntityCatalogService("plugin.microservice-catalog-constructorservice.catalog-class") 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/RightMockBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog._ 4 | 5 | object RightMockBuilder extends EntityCatalogBuilder { 6 | override def getEntityCatalogService(): EntityCatalogService = { 7 | getEntityCatalogService("plugin.microservice-catalog.catalog-class") 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/WrongCatalogMockBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog._ 4 | 5 | 6 | object WrongCatalogMockBuilder extends EntityCatalogBuilder { 7 | override def getEntityCatalogService(): EntityCatalogService = { 8 | getEntityCatalogService("plugin.microservice-catalog-wrongcatalog.catalog-class") 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/WrongConfigurationPathBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog._ 4 | 5 | object WrongConfigurationPathBuilder extends EntityCatalogBuilder { 6 | override def getEntityCatalogService(): EntityCatalogService = { 7 | getEntityCatalogService("plugin.microservice-catalog.not-existing-path") 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /plugin-parallel-write-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/parallel/tools/catalog/builders/mockbuilders/WrongMicroserviceMockBuilder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.tools.catalog.builders.mockbuilders 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel.catalog._ 4 | 5 | object WrongMicroserviceMockBuilder extends EntityCatalogBuilder { 6 | override def getEntityCatalogService(): EntityCatalogService = { 7 | getEntityCatalogService("plugin.microservice-catalog-wrongmicroserviceclass.catalog-class") 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /plugin-plain-hbase-writer-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.plain.hbase.PlainHBaseWriterConsumer -------------------------------------------------------------------------------- /plugin-plain-hbase-writer-spark/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.logger.org.apache=ERROR 2 | log4j.logger.Remoting=ERROR 3 | log4j.logger.org.eclipse.jetty=ERROR -------------------------------------------------------------------------------- /plugin-plain-hbase-writer-spark/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline -------------------------------------------------------------------------------- /plugin-plain-hbase-writer-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/plain/hbase/integration/TestFixture.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.plain.hbase.integration 2 | 3 | import org.apache.commons.io.FileUtils 4 | import org.scalatest.{BeforeAndAfter, Matchers, WordSpec} 5 | 6 | import java.io.File 7 | 8 | class TestFixture extends WordSpec with Matchers with BeforeAndAfter { 9 | 10 | val checkpointLocation = "/tmp/checkpoint" 11 | 12 | before { 13 | try { 14 | FileUtils.deleteDirectory(new File(checkpointLocation)) 15 | } catch { 16 | case _: Exception => //do nothing 17 | } 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /plugin-postgresql-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.postgresql.PostgreSQLConsumerSparkPlugin -------------------------------------------------------------------------------- /plugin-postgresql-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/postgresql/JDBCConnectionInfoProvider.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.postgresql 2 | 3 | import java.util.Properties 4 | 5 | trait JDBCConnectionInfoProvider { 6 | def getUrl: String 7 | def getDriver: String 8 | def getProperties: Properties 9 | } 10 | -------------------------------------------------------------------------------- /plugin-postgresql-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/postgresql/TestData.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.postgresql 2 | 3 | import java.sql.Timestamp 4 | 5 | case class TestData(pk1: String, pk2: Int, val1: String, val2: Long, val3: Timestamp) 6 | -------------------------------------------------------------------------------- /plugin-raw-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.raw.RawConsumersSpark -------------------------------------------------------------------------------- /plugin-raw-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/plugins/raw/RawSparkReaderUtils.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.consumers.spark.plugins.raw 2 | 3 | object RawSparkReaderUtils { 4 | def printExtraOptions(modelName: String, extraOptions: Map[String, String]): String = { 5 | if (extraOptions.isEmpty) { 6 | s"Not pushing any extra option for model named ${modelName}" 7 | } else { 8 | s"Pushing extra options for model named ${modelName}:\n" + extraOptions.mkString("* ", "\n*", "") 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /plugin-raw-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-raw-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin-solr-spark/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.consumers.spark.plugins.WaspConsumersSparkPlugin: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.consumers.spark.plugins.solr.SolrConsumersSpark -------------------------------------------------------------------------------- /plugin-solr-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin-solr-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /plugin/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /plugin/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /producers/build.sbt: -------------------------------------------------------------------------------- 1 | Compile / mainClass := Some("it.agilelab.bigdata.wasp.producers.launcher.ProducersNodeLauncher") -------------------------------------------------------------------------------- /producers/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | wasp { 2 | akka { 3 | cluster { 4 | roles = ["producers", "logger"] 5 | } 6 | remote { 7 | artery { 8 | enabled = on 9 | canonical.port = 0 10 | canonical.hostname = "localhost" 11 | } 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /producers/src/main/scala/it/agilelab/bigdata/wasp/producers/metrics/kafka/KafkaOffsetActorAlive.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.producers.metrics.kafka 2 | 3 | case object KafkaOffsetActorAlive 4 | -------------------------------------------------------------------------------- /producers/src/main/scala/it/agilelab/bigdata/wasp/producers/metrics/kafka/KafkaOffsets.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.producers.metrics.kafka 2 | 3 | case class KafkaOffsets(topic: String, offs: Map[Int, Long], ts: Long) 4 | 5 | -------------------------------------------------------------------------------- /producers/src/main/scala/it/agilelab/bigdata/wasp/producers/metrics/kafka/KafkaOffsetsRequest.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.producers.metrics.kafka 2 | 3 | import akka.actor.ActorRef 4 | 5 | case class KafkaOffsetsRequest(replyTo: ActorRef, 6 | topic: String, 7 | ts: Long) 8 | -------------------------------------------------------------------------------- /producers/src/main/scala/it/agilelab/bigdata/wasp/producers/metrics/kafka/WrongKafkaOffsetsRequest.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.producers.metrics.kafka 2 | 3 | case class WrongKafkaOffsetsRequest(message: String) 4 | -------------------------------------------------------------------------------- /producers/src/main/scala/it/agilelab/bigdata/wasp/producers/metrics/kafka/backlog/BacklogInfo.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.producers.metrics.kafka.backlog 2 | 3 | case class BacklogInfo(etlName: String, topicName: String, backlogSize: Long, timestamp: Long) 4 | -------------------------------------------------------------------------------- /producers/src/main/scala/it/agilelab/bigdata/wasp/producers/remote/package.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.producers 2 | 3 | /** 4 | * Misc stuff for remote producers support. 5 | * 6 | * @author Nicolò Bidotti 7 | */ 8 | package object remote { 9 | // name for the (akka) topic for producers stuff 10 | val producersAkkaTopic = "producers" 11 | } 12 | -------------------------------------------------------------------------------- /producers/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /producers/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /project/Utils.scala: -------------------------------------------------------------------------------- 1 | object Utils { 2 | def resolveVariable(varName: String): Option[String] = { 3 | sys.env.get(varName).orElse(Option(System.getProperty(varName))) 4 | } 5 | def printWithBorders(msg: String, separatorChar: String): String = { 6 | (separatorChar * (msg.length + 4)) + "\n" + 7 | (s"${separatorChar} ${msg} ${separatorChar}\n") + 8 | (separatorChar * (msg.length + 4)) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.11.0 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // sbt-buildinfo, for accessing build information in the code - https://github.com/sbt/sbt-buildinfo/ 2 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.13.1") 3 | // sbt-native-packager, used for assembly jars for the start-wasp script 4 | addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.0") 5 | // test coverage 6 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.0") 7 | // sign artifacts 8 | addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") 9 | addDependencyTreePlugin 10 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/BatchSchedulersBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.BatchSchedulerModel 4 | 5 | trait BatchSchedulersBL { 6 | 7 | def getActiveSchedulers(isActive: Boolean = true): Seq[BatchSchedulerModel] 8 | 9 | def persist(schedulerModel: BatchSchedulerModel): Unit 10 | } -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/CdcBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.CdcModel 4 | 5 | trait CdcBL { 6 | 7 | def getByName(name: String): Option[CdcModel] 8 | 9 | def persist(cdcModel: CdcModel): Unit 10 | 11 | def upsert(cdcModel: CdcModel): Unit 12 | 13 | def getAll() : Seq[CdcModel] 14 | 15 | } 16 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/ConfigManagerBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | import scala.reflect.runtime.universe.TypeTag 6 | import scala.reflect.ClassTag 7 | 8 | trait ConfigManagerBL { 9 | 10 | def getByName[T <: Model](name : String)(implicit ct: ClassTag[T], typeTag: TypeTag[T]): Option[T] 11 | 12 | def retrieveConf[T <: Model](default: T, nameConf: String)(implicit ct: ClassTag[T], typeTag: TypeTag[T]): Option[T] 13 | 14 | def retrieveDBConfig(): Seq[String] 15 | 16 | } 17 | 18 | 19 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/DBConfigBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | trait DBConfigBL { 4 | def retrieveDBConfig(): Seq[String] 5 | 6 | } 7 | 8 | 9 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/DocumentBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.DocumentModel 4 | 5 | trait DocumentBL { 6 | 7 | def getByName(name: String): Option[DocumentModel] 8 | def getAll(): Seq[DocumentModel] 9 | def persist(rawModel: DocumentModel): Unit 10 | def upsert(rawModel: DocumentModel): Unit 11 | } 12 | 13 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/FreeCodeBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.FreeCodeModel 4 | 5 | trait FreeCodeBL { 6 | 7 | def getByName(name: String): Option[FreeCodeModel] 8 | 9 | def deleteByName(name : String) : Unit 10 | 11 | def getAll: Seq[FreeCodeModel] 12 | 13 | def insert(freeCodeModel: FreeCodeModel): Unit 14 | 15 | def upsert(freeCodeModel: FreeCodeModel): Unit 16 | } 17 | 18 | 19 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/GenericBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.{GenericModel} 4 | 5 | 6 | trait GenericBL { 7 | 8 | def getByName(name: String): Option[GenericModel] 9 | 10 | def persist(genericModel: GenericModel): Unit 11 | 12 | def upsert(genericModel: GenericModel): Unit 13 | 14 | def getAll() : Seq[GenericModel] 15 | } 16 | 17 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/HttpBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.HttpModel 4 | 5 | trait HttpBL { 6 | def getByName(name: String): Option[HttpModel] 7 | 8 | def persist(HttpModel: HttpModel): Unit 9 | 10 | def getAll(): Seq[HttpModel] 11 | 12 | def upsert(HttpModel: HttpModel): Unit 13 | 14 | def insertIfNotExists(HttpModel: HttpModel): Unit 15 | 16 | } 17 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/IndexBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.IndexModel 4 | 5 | trait IndexBL { 6 | def getByName(name: String): Option[IndexModel] 7 | 8 | def persist(indexModel: IndexModel): Unit 9 | 10 | def getAll(): Seq[IndexModel] 11 | 12 | def upsert(indexModel: IndexModel): Unit 13 | 14 | def insertIfNotExists(indexModel: IndexModel): Unit 15 | 16 | } -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/KeyValueBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.KeyValueModel 4 | 5 | trait KeyValueBL { 6 | 7 | def getByName(name: String): Option[KeyValueModel] 8 | 9 | def getAll(): Seq[KeyValueModel] 10 | 11 | def persist(rawModel: KeyValueModel): Unit 12 | 13 | def upsert(rawModel: KeyValueModel): Unit 14 | } -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/ProcessGroupBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.ProcessGroupModel 4 | 5 | trait ProcessGroupBL { 6 | 7 | def getById(pgId: String): Option[ProcessGroupModel] 8 | 9 | def insert(versionedProcessGroup: ProcessGroupModel): Unit 10 | 11 | def upsert(versionedProcessGroup: ProcessGroupModel): Unit 12 | } 13 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/RawBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.RawModel 4 | 5 | 6 | trait RawBL { 7 | 8 | def getByName(name: String): Option[RawModel] 9 | 10 | def persist(rawModel: RawModel): Unit 11 | 12 | def upsert(rawModel: RawModel): Unit 13 | 14 | def getAll() : Seq[RawModel] 15 | } 16 | 17 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/SQLSinkBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.SQLSinkModel 4 | 5 | trait SQLSinkBL { 6 | def getByName(name: String): Option[SQLSinkModel] 7 | def getAll(): Seq[SQLSinkModel] 8 | def persist(model: SQLSinkModel): Unit 9 | def upsert(model: SQLSinkModel): Unit 10 | def deleteByName(name: String): Unit 11 | } 12 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/SqlSourceBl.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.SqlSourceModel 4 | 5 | trait SqlSourceBl { 6 | 7 | def getByName(name: String): Option[SqlSourceModel] 8 | 9 | def persist(rawModel: SqlSourceModel): Unit 10 | 11 | def upsert(rawModel: SqlSourceModel): Unit 12 | } -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/WebsocketBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.bl 2 | 3 | import it.agilelab.bigdata.wasp.models.WebsocketModel 4 | 5 | trait WebsocketBL { 6 | def getByName(name: String): Option[WebsocketModel] 7 | 8 | def persist(topicModel: WebsocketModel): Unit 9 | } -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/bl/package.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core 2 | 3 | /** 4 | * Type aliases for commonly used structural types built with the BLs. These will also help when migrating away from 5 | * structural typing. 6 | * 7 | * @author Nicolò Bidotti 8 | */ 9 | package object bl { 10 | type DatastoreModelBLs = { 11 | val indexBL: IndexBL 12 | val topicBL: TopicBL 13 | val rawBL: RawBL 14 | val keyValueBL: KeyValueBL 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/db/WaspDB.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.db 2 | 3 | 4 | trait WaspDB { 5 | def close() : Unit 6 | } 7 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/CdcDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.{CdcOptions, Model} 4 | 5 | abstract class CdcDBModel extends Model 6 | 7 | case class CdcDBModelV1(override val name: String, 8 | uri: String, 9 | schema: String, 10 | options: CdcOptions = CdcOptions.default 11 | ) extends CdcDBModel 12 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/DocumentDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | trait DocumentDBModel extends Model 6 | 7 | case class DocumentDBModelV1(override val name: String, 8 | connectionString: String, 9 | schema: String 10 | ) extends DocumentDBModel 11 | 12 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/FreeCodeDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | trait FreeCodeDBModel extends Model 6 | 7 | case class FreeCodeDBModelV1(override val name : String, code : String) extends FreeCodeDBModel 8 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/GenericDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.datastores.GenericProduct 4 | import it.agilelab.bigdata.wasp.models.{GenericOptions, Model} 5 | import org.mongodb.scala.bson.BsonDocument 6 | 7 | trait GenericDBModel extends Model 8 | 9 | case class GenericDBModelV1(override val name: String, 10 | value: BsonDocument, 11 | product: GenericProduct, 12 | options: GenericOptions = GenericOptions.default 13 | ) extends GenericDBModel 14 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/MultiTopicDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | 5 | trait MultiTopicDBModel extends Model 6 | 7 | case class MultiTopicDBModelV1(override val name: String, 8 | topicNameField: String, 9 | topicModelNames: Seq[String] 10 | ) extends MultiTopicDBModel 11 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/ProcessGroupDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | import org.mongodb.scala.bson.BsonDocument 5 | 6 | trait ProcessGroupDBModel extends Model 7 | 8 | case class ProcessGroupDBModelV1(name: String, 9 | content: BsonDocument, 10 | errorPort: String 11 | ) extends ProcessGroupDBModel 12 | -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/RawDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.{Model, RawOptions} 4 | 5 | trait RawDBModel extends Model 6 | 7 | case class RawDBModelV1(override val name: String, 8 | uri: String, 9 | timed: Boolean = true, 10 | schema: String, 11 | options: RawOptions = RawOptions.default 12 | ) extends RawDBModel -------------------------------------------------------------------------------- /repository/core/src/main/scala/it/agilelab/bigdata/wasp/repository/core/dbModels/WebsocketDBModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.core.dbModels 2 | 3 | import it.agilelab.bigdata.wasp.models.Model 4 | import org.mongodb.scala.bson.BsonDocument 5 | 6 | trait WebsocketDBModel extends Model 7 | 8 | case class WebsocketDBModelV1(override val name: String, 9 | host: String, 10 | port: String, 11 | resourceName: String, 12 | options: Option[BsonDocument] = None 13 | ) extends WebsocketDBModel 14 | -------------------------------------------------------------------------------- /repository/core/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /repository/core/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /repository/mongo/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.repository.core.db.RepositoriesFactory: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.repository.mongo.MongoRepositoriesFactory -------------------------------------------------------------------------------- /repository/mongo/src/test/resources/application.conf: -------------------------------------------------------------------------------- 1 | wasp.mongo.authentication-db: admin -------------------------------------------------------------------------------- /repository/mongo/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /repository/mongo/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /repository/mongo/src/test/scala/it/agilelab/bigdata/wasp/repository/mongo/MongoRepositoriesFactoryTest.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.mongo 2 | 3 | import it.agilelab.bigdata.wasp.repository.core.db.RepositoriesFactory 4 | import org.scalatest.{FlatSpec, Matchers} 5 | 6 | class MongoRepositoriesFactoryTest extends FlatSpec with Matchers { 7 | 8 | it should "check that WaspDB use MongoBL" in { 9 | RepositoriesFactory.service.isInstanceOf[MongoRepositoriesFactory] shouldBe true 10 | } 11 | } 12 | 13 | -------------------------------------------------------------------------------- /repository/postgres/src/main/resources/META-INF/services/it.agilelab.bigdata.wasp.repository.core.db.RepositoriesFactory: -------------------------------------------------------------------------------- 1 | it.agilelab.bigdata.wasp.repository.postgres.PostgresRepositoryFactory -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/bl/PostgresBL.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.bl 2 | 3 | import it.agilelab.bigdata.wasp.repository.postgres.WaspPostgresDB 4 | import it.agilelab.bigdata.wasp.repository.postgres.tables.TableDefinition 5 | 6 | trait PostgresBL { 7 | 8 | val waspDB: WaspPostgresDB 9 | 10 | implicit val tableDefinition: TableDefinition[_,_] 11 | 12 | def createTable(): Unit = { 13 | waspDB.createTable() 14 | } 15 | 16 | private[postgres] def dropTable() : Unit = { 17 | waspDB.dropTable() 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/BatchJobTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import it.agilelab.bigdata.wasp.models.BatchJobModel 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object BatchJobTableDefinition extends SimpleModelTableDefinition[BatchJobModel] with JsonSupport{ 8 | 9 | val tableName = "BATCH_JOB" 10 | 11 | override def fromJsonToModel(json: JsValue): BatchJobModel = json.convertTo[BatchJobModel] 12 | 13 | override def fromModelToJson(model: BatchJobModel): JsValue = model.toJson 14 | 15 | 16 | } 17 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/CdcTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import it.agilelab.bigdata.wasp.models.CdcModel 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object CdcTableDefinition extends SimpleModelTableDefinition[CdcModel] with JsonSupport { 8 | 9 | override def tableName: String = "CDC" 10 | 11 | override def fromJsonToModel(json: JsValue): CdcModel = 12 | json.convertTo[CdcModel] 13 | 14 | override def fromModelToJson(model: CdcModel): JsValue = model.toJson 15 | 16 | } 17 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/GenericTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import it.agilelab.bigdata.wasp.models.{GenericModel} 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object GenericTableDefinition extends SimpleModelTableDefinition[GenericModel] with JsonSupport { 8 | 9 | override def tableName: String = "GENERIC" 10 | 11 | override def fromJsonToModel(json: JsValue): GenericModel = 12 | json.convertTo[GenericModel] 13 | 14 | override def fromModelToJson(model: GenericModel): JsValue = model.toJson 15 | 16 | } 17 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/HttpTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables; 2 | 3 | import it.agilelab.bigdata.wasp.models.{HttpModel} 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object HttpTableDefinition extends SimpleModelTableDefinition[HttpModel] with JsonSupport { 8 | override protected def fromModelToJson(model: HttpModel): JsValue = model.toJson 9 | 10 | override protected def fromJsonToModel(json: JsValue): HttpModel = json.convertTo[HttpModel] 11 | 12 | override def tableName: String = "HTTP" 13 | } 14 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/KeyValueTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import it.agilelab.bigdata.wasp.models.KeyValueModel 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object KeyValueTableDefinition extends SimpleModelTableDefinition[KeyValueModel] with JsonSupport{ 8 | 9 | val tableName = "KEY_VALUE" 10 | 11 | override def fromJsonToModel(json: JsValue): KeyValueModel = json.convertTo[KeyValueModel] 12 | 13 | override def fromModelToJson(model: KeyValueModel): JsValue = model.toJson 14 | 15 | } -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/RawTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import it.agilelab.bigdata.wasp.models.RawModel 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object RawTableDefinition extends SimpleModelTableDefinition[RawModel] with JsonSupport { 8 | 9 | override def tableName: String = "RAW" 10 | 11 | override def fromJsonToModel(json: JsValue): RawModel = 12 | json.convertTo[RawModel] 13 | 14 | override def fromModelToJson(model: RawModel): JsValue = model.toJson 15 | 16 | } 17 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/SQLSinkTableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import it.agilelab.bigdata.wasp.models.SQLSinkModel 4 | import it.agilelab.bigdata.wasp.utils.JsonSupport 5 | import spray.json._ 6 | 7 | object SQLSinkTableDefinition extends SimpleModelTableDefinition[SQLSinkModel] with JsonSupport { 8 | 9 | override def tableName: String = "SQLSINK" 10 | 11 | override protected def fromModelToJson(model: SQLSinkModel): JsValue = model.toJson 12 | 13 | override protected def fromJsonToModel(json: JsValue): SQLSinkModel = json.convertTo[SQLSinkModel] 14 | 15 | } 16 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/TableDefinition.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.tables 2 | 3 | import java.sql.ResultSet 4 | 5 | 6 | trait TableDefinition[T,K] { 7 | 8 | def tableName : String 9 | 10 | def columns: List[String] 11 | 12 | def ddl : String 13 | 14 | def from : ResultSet => T 15 | 16 | def to : T=> Array[(String,Any)] 17 | 18 | def conditionPrimaryKey : K => Array[(String,Any)] 19 | 20 | def primaryKeyFromObject : T => K 21 | 22 | 23 | } 24 | 25 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/tables/package.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres 2 | 3 | import java.sql.ResultSet 4 | 5 | 6 | package object tables { 7 | 8 | implicit class ResultSetHelper(resultSet : ResultSet ) { 9 | 10 | def getOption[T](field : String): Option[T] ={ 11 | resultSet.getObject(field) match { 12 | case null => None 13 | case v => Some(v.asInstanceOf[T]) 14 | } 15 | } 16 | 17 | 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /repository/postgres/src/main/scala/it/agilelab/bigdata/wasp/repository/postgres/utils/ConnectionInfoProvider.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.repository.postgres.utils 2 | 3 | trait ConnectionInfoProvider { 4 | protected def getUrl: String 5 | protected def getUser: String 6 | protected def getPassword: String 7 | protected def getDriver: String 8 | protected def getPoolSize : Int 9 | } 10 | -------------------------------------------------------------------------------- /repository/postgres/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /repository/postgres/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /repository/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /repository/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /run-sbt-unprivileged.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # this script is needed because otherwise postgresql embedded does not work 3 | chmod 777 -R . 4 | useradd -m ${UNPRIVILEGED_USER} 5 | # the next two lines are needed because newer git versions will complain 6 | # if the user that invokes git command does not own the .git folder and its 7 | # parent 8 | chown ${UNPRIVILEGED_USER} . 9 | chown ${UNPRIVILEGED_USER} -R .git 10 | # shellcheck disable=SC2068 11 | COMMAND="sbt $@" 12 | su $UNPRIVILEGED_USER -c "${COMMAND}" -------------------------------------------------------------------------------- /spark/nifi-plugin/src/main/scala/it/agilelab/bigdata/wasp/spark/plugins/nifi/CompatibilityNifiPlugin.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.plugins.nifi 2 | 3 | import nifi.NifiPlugin 4 | import org.apache.spark.api.plugin.ExecutorPlugin 5 | 6 | trait CompatibilityNifiPlugin extends ExecutorPlugin { 7 | 8 | self: NifiPlugin => 9 | override def init(ctx: org.apache.spark.api.plugin.PluginContext, extraConf: java.util.Map[String, String]): Unit = { 10 | this.initialization() 11 | } 12 | 13 | } -------------------------------------------------------------------------------- /spark/nifi-plugin/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /spark/nifi-plugin/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/main/scala/it/agilelab/bigdata/wasp/spark/plugins/telemetry/CompatibilityExecutorPlugin.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.plugins.telemetry 2 | 3 | import org.apache.spark.api.plugin.ExecutorPlugin 4 | 5 | trait CompatibilityExecutorPlugin extends ExecutorPlugin{} 6 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/main/scala/it/agilelab/bigdata/wasp/spark/plugins/telemetry/TelemetryMetadataProducerConfig.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.plugins.telemetry 2 | 3 | case class TelemetryMetadataProducerConfig(telemetry: TelemetryPluginTopicConfigModel, global: TelemetryPluginKafkaConfig) 4 | 5 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/main/scala/it/agilelab/bigdata/wasp/spark/plugins/telemetry/TelemetryPluginConfiguration.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.plugins.telemetry 2 | 3 | import scala.concurrent.duration.Duration 4 | 5 | case class TelemetryPluginConfiguration(interval: Duration, producer: TelemetryMetadataProducerConfig) 6 | 7 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/main/scala/it/agilelab/bigdata/wasp/spark/plugins/telemetry/TelemetryPluginConnectionConfig.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.plugins.telemetry 2 | 3 | case class TelemetryPluginConnectionConfig(protocol: String, 4 | host: String, 5 | port: Int = 0, 6 | timeout: Option[Long] = None, 7 | metadata: Option[Map[String, String]]) 8 | 9 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/main/scala/it/agilelab/bigdata/wasp/spark/plugins/telemetry/TelemetryPluginKafkaConfig.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.spark.plugins.telemetry 2 | 3 | case class TelemetryPluginKafkaConfig(connections: Seq[TelemetryPluginConnectionConfig], 4 | batch_send_size: Int, 5 | acks: String, 6 | default_encoder: String, 7 | encoder_fqcn: String, 8 | partitioner_fqcn: String, 9 | others: Seq[(String, String)]) 10 | 11 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /spark/telemetry-plugin/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tools/release-note-generator/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | wasprng.egg-info/ 4 | .vscode/ 5 | -------------------------------------------------------------------------------- /tools/release-note-generator/README.md: -------------------------------------------------------------------------------- 1 | ### Create new Sprint Release Notes 2 | 3 | *Optional: `virtualenv ~/release-note-generator-virtual-env`* 4 | 5 | 1. `source ~/release-note-generator-virtual-env/bin/activate` 6 | 7 | 2. `cd ~/Projects/Agile.Wasp2/tools/release-note-generator` 8 | 9 | 3. `python setup.py install` 10 | 11 | 4. `wasp-release-note-generator -t --sprint "" > ~/Desktop/newRelease.md` 12 | 13 | 5. `deactivate` -------------------------------------------------------------------------------- /tools/release-note-generator/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2018.1.18 2 | chardet==3.0.4 3 | idna==2.6 4 | python-gitlab==1.3.0 5 | requests==2.18.4 6 | six==1.9.0 7 | urllib3==1.22 8 | -------------------------------------------------------------------------------- /tools/release-note-generator/wasprng/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/tools/release-note-generator/wasprng/__init__.py -------------------------------------------------------------------------------- /waspctl/.gitignore: -------------------------------------------------------------------------------- 1 | output -------------------------------------------------------------------------------- /waspctl/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24.2-alpine3.21 2 | 3 | RUN apk add git bash 4 | 5 | RUN go install github.com/danielgtaylor/openapi-cli-generator@latest 6 | 7 | -------------------------------------------------------------------------------- /waspctl/README.md: -------------------------------------------------------------------------------- 1 | # Build locally 2 | 3 | ## Regenerate Api Bindings 4 | 5 | From the root of the wasp repository run 6 | 7 | ```bash 8 | docker run -v $PWD:/code -it registry.gitlab.com/agilefactory/agile.wasp2/waspctl-build:0.3.0 bash -c "cd /code/waspctl/ && openapi-cli-generator generate ../documentation/wasp-openapi.yaml" 9 | ``` 10 | 11 | ## Build executables for all archs 12 | 13 | From the root of the wasp repository run 14 | 15 | ```bash 16 | docker run -v $PWD:/code -eCI_PROJECT_DIR=/code -it registry.gitlab.com/agilefactory/agile.wasp2/waspctl-build:0.3.0 bash /code/waspctl/build-for-all-archs.bash 17 | ``` 18 | 19 | 20 | # Rebuild the docker imageA 21 | -------------------------------------------------------------------------------- /waspctl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/danielgtaylor/openapi-cli-generator/cli" 5 | ) 6 | 7 | func main() { 8 | cli.Init(&cli.Config{ 9 | AppName: "waspctl", 10 | EnvPrefix: "WASPCTL", 11 | Version: "1.0.0", 12 | }) 13 | 14 | waspOpenapiRegister(false) 15 | 16 | cli.Root.Execute() 17 | } 18 | -------------------------------------------------------------------------------- /whitelabel/.gitignore: -------------------------------------------------------------------------------- 1 | #Docker Data 2 | docker/data -------------------------------------------------------------------------------- /whitelabel/consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/consumers/spark/aws/MinioCredentialsProvider.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.consumers.spark.aws 2 | import com.amazonaws.auth.{AWSCredentials, BasicAWSCredentials} 3 | import org.apache.hadoop.conf.Configuration 4 | 5 | import java.net.URI 6 | 7 | class MinioCredentialsProvider(uri: URI, configuration: Configuration) 8 | extends com.amazonaws.auth.AWSCredentialsProvider { 9 | override def getCredentials: AWSCredentials = new BasicAWSCredentials( 10 | sys.env("MINIO_ROOT_USER"), 11 | sys.env("MINIO_ROOT_PASSWORD") 12 | ) 13 | 14 | override def refresh(): Unit = {} 15 | } 16 | -------------------------------------------------------------------------------- /whitelabel/consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/consumers/spark/launcher/SparkConsumersBatchNodeLauncher.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.consumers.spark.launcher 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.launcher.SparkConsumersBatchNodeLauncherTrait 4 | import org.apache.commons.cli.CommandLine 5 | 6 | object SparkConsumersBatchNodeLauncher extends SparkConsumersBatchNodeLauncherTrait { 7 | 8 | override def launch(commandLine: CommandLine): Unit = { 9 | super.launch(commandLine) 10 | } 11 | } -------------------------------------------------------------------------------- /whitelabel/consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/consumers/spark/launcher/SparkConsumersStreamingNodeLauncher.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.consumers.spark.launcher 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.launcher.SparkConsumersStreamingNodeLauncherTrait 4 | import org.apache.commons.cli.CommandLine 5 | 6 | object SparkConsumersStreamingNodeLauncher extends SparkConsumersStreamingNodeLauncherTrait { 7 | 8 | override def launch(commandLine: CommandLine): Unit = { 9 | super.launch(commandLine) 10 | } 11 | } -------------------------------------------------------------------------------- /whitelabel/consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/consumers/spark/strategies/test/TestErrorStrategy.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.consumers.spark.strategies.test 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.strategies.{ReaderKey, Strategy} 4 | import org.apache.spark.sql.DataFrame 5 | 6 | class TestErrorStrategy extends Strategy { 7 | 8 | override def transform(dataFrames: Map[ReaderKey, DataFrame]): DataFrame = { 9 | 10 | println(s"Strategy configuration: $configuration") 11 | 12 | throw new Exception("Fake error to simulate ETL component failure") 13 | } 14 | } -------------------------------------------------------------------------------- /whitelabel/consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/consumers/spark/strategies/test/TestIdentityStrategy.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.consumers.spark.strategies.test 2 | 3 | import it.agilelab.bigdata.wasp.consumers.spark.strategies.{ReaderKey, Strategy} 4 | import org.apache.spark.sql.DataFrame 5 | 6 | class TestIdentityStrategy extends Strategy { 7 | 8 | override def transform(dataFrames: Map[ReaderKey, DataFrame]): DataFrame = { 9 | 10 | println(s"Strategy configuration: $configuration") 11 | 12 | dataFrames.head._2 13 | } 14 | } -------------------------------------------------------------------------------- /whitelabel/consumers-spark/src/test/resources/3.delta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agile-lab-dev/wasp/c16db9f78cd64b87051d7831d9685ac1035579b9/whitelabel/consumers-spark/src/test/resources/3.delta -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build-and-push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ax 4 | 5 | cd build 6 | docker build . -t registry.gitlab.com/agilefactory/agile.wasp2/cdp:717 7 | 8 | docker login registry.gitlab.com/agilefactory/agile.wasp2 9 | docker push registry.gitlab.com/agilefactory/agile.wasp2/cdp:717 10 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/install-scripts/common/bash-defaults.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit # exit when a command fails 4 | set -o nounset # exit when a script tries to use undeclared variables 5 | set -o xtrace # trace what gets executed 6 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/install-scripts/common/yum-clean-caches.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Cleaning yum caches" 4 | 5 | yum clean all -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/install-scripts/install-mongo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | yum update -y 6 | yum install -y mongodb-org 7 | 8 | exec bash ./common/yum-clean-caches.sh 9 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/install-scripts/install-prerequisites.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | yum install epel-release -y 6 | yum update -y 7 | yum install -y java-1.8.0-openjdk-headless 8 | 9 | exec bash ./common/yum-clean-caches.sh 10 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/install-scripts/install-supervisord.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | yum install -y supervisor 6 | 7 | exec bash ./common/yum-clean-caches.sh 8 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/hadoop/workers: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/hbase/regionservers: -------------------------------------------------------------------------------- 1 | ${HOSTNAME} -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/mongo/mongodb.repo: -------------------------------------------------------------------------------- 1 | [MongoDB] 2 | name=MongoDB Repository 3 | baseurl=http://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.2/$basearch/ 4 | gpgcheck=1 5 | enabled=1 6 | gpgkey=https://www.mongodb.org/static/pgp/server-4.2.asc -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-hadoop-templates.sh: -------------------------------------------------------------------------------- 1 | 2 | mkdir -p /env 3 | mkdir -p /etc/hadoop/conf 4 | 5 | echo "HOSTNAME=$HOSTNAME" > /env/hostname.env 6 | 7 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/hadoop --out=/etc/hadoop/conf 8 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-hbase-templates.sh: -------------------------------------------------------------------------------- 1 | 2 | mkdir -p /env 3 | mkdir -p /etc/hbase/conf 4 | 5 | echo "HOSTNAME=$HOSTNAME" > /env/hostname.env 6 | 7 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/hbase --out=/etc/hbase/conf 8 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-kafka-templates.sh: -------------------------------------------------------------------------------- 1 | mkdir -p /env 2 | mkdir -p /etc/kafka/conf 3 | mkdir -p /etc/kafka2/conf 4 | 5 | echo "HOSTNAME=$HOSTNAME" > /env/hostname.env 6 | 7 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/kafka --out=/etc/kafka/conf 8 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/kafka2 --out=/etc/kafka2/conf -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-polynote-templates.sh: -------------------------------------------------------------------------------- 1 | cat ${BUILD_TEMPLATES_DIR}/polynote/config.yml | envsubst \$HOSTNAME > $POLYNOTE_HOME/config.yml 2 | 3 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-solr-templates.sh: -------------------------------------------------------------------------------- 1 | 2 | mkdir -p /env 3 | mkdir -p /etc/solr/conf 4 | 5 | echo "HOSTNAME=$HOSTNAME" > /env/hostname.env 6 | 7 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/solr/conf --out=/etc/solr/conf 8 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/solr/bin --out=/usr/lib/solr/bin 9 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-spark-templates.sh: -------------------------------------------------------------------------------- 1 | mkdir -p /env 2 | mkdir -p /etc/spark/conf 3 | 4 | echo "HOSTNAME=$HOSTNAME" > /env/hostname.env 5 | 6 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/spark --out=/etc/spark/conf 7 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/resolve-zookeeper-templates.sh: -------------------------------------------------------------------------------- 1 | mkdir -p /env 2 | mkdir -p /etc/zookeeper/conf 3 | 4 | echo "HOSTNAME=$HOSTNAME" > /env/hostname.env 5 | 6 | genvsubst --env /env --any --sub $BUILD_TEMPLATES_DIR/zookeeper --out=/etc/zookeeper/conf 7 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/contractions_ca.txt: -------------------------------------------------------------------------------- 1 | # Set of Catalan contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | l 5 | m 6 | n 7 | s 8 | t 9 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/contractions_fr.txt: -------------------------------------------------------------------------------- 1 | # Set of French contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | l 4 | m 5 | t 6 | qu 7 | n 8 | s 9 | j 10 | d 11 | c 12 | jusqu 13 | quoiqu 14 | lorsqu 15 | puisqu 16 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/contractions_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | m 5 | b 6 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/contractions_it.txt: -------------------------------------------------------------------------------- 1 | # Set of Italian contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | c 4 | l 5 | all 6 | dall 7 | dell 8 | nell 9 | sull 10 | coll 11 | pell 12 | gl 13 | agl 14 | dagl 15 | degl 16 | negl 17 | sugl 18 | un 19 | m 20 | t 21 | s 22 | v 23 | d 24 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/hyphenations_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish hyphenations for StopFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | h 4 | n 5 | t 6 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/stemdict_nl.txt: -------------------------------------------------------------------------------- 1 | # Set of overrides for the dutch stemmer 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | fiets fiets 4 | bromfiets bromfiets 5 | ei eier 6 | kind kinder 7 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/_default/conf/lang/stopwords_hy.txt: -------------------------------------------------------------------------------- 1 | # example set of Armenian stopwords. 2 | այդ 3 | այլ 4 | այն 5 | այս 6 | դու 7 | դուք 8 | եմ 9 | են 10 | ենք 11 | ես 12 | եք 13 | է 14 | էի 15 | էին 16 | էինք 17 | էիր 18 | էիք 19 | էր 20 | ըստ 21 | թ 22 | ի 23 | ին 24 | իսկ 25 | իր 26 | կամ 27 | համար 28 | հետ 29 | հետո 30 | մենք 31 | մեջ 32 | մի 33 | ն 34 | նա 35 | նաև 36 | նրա 37 | նրանք 38 | որ 39 | որը 40 | որոնք 41 | որպես 42 | ու 43 | ում 44 | պիտի 45 | վրա 46 | և 47 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/_rest_managed.json: -------------------------------------------------------------------------------- 1 | {"initArgs":{},"managedList":[]} 2 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/_schema_analysis_synonyms_english.json: -------------------------------------------------------------------------------- 1 | { 2 | "initArgs":{ 3 | "ignoreCase":true, 4 | "format":"solr" 5 | }, 6 | "managedMap":{ 7 | "GB":["GiB","Gigabyte"], 8 | "happy":["glad","joyful"], 9 | "TV":["Television"] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/clustering/carrot2/README.txt: -------------------------------------------------------------------------------- 1 | An override location of the clustering algorithm's resources 2 | attribute definitions and lexical resources. 3 | 4 | A directory from which to load algorithm-specific stop words, 5 | stop labels and attribute definition XMLs. 6 | 7 | For an overview of Carrot2 lexical resources, see: 8 | http://download.carrot2.org/head/manual/#chapter.lexical-resources 9 | 10 | For an overview of Lingo3G lexical resources, see: 11 | http://download.carrotsearch.com/lingo3g/manual/#chapter.lexical-resources 12 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/contractions_ca.txt: -------------------------------------------------------------------------------- 1 | # Set of Catalan contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | l 5 | m 6 | n 7 | s 8 | t 9 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/contractions_fr.txt: -------------------------------------------------------------------------------- 1 | # Set of French contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | l 4 | m 5 | t 6 | qu 7 | n 8 | s 9 | j 10 | d 11 | c 12 | jusqu 13 | quoiqu 14 | lorsqu 15 | puisqu 16 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/contractions_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | m 5 | b 6 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/contractions_it.txt: -------------------------------------------------------------------------------- 1 | # Set of Italian contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | c 4 | l 5 | all 6 | dall 7 | dell 8 | nell 9 | sull 10 | coll 11 | pell 12 | gl 13 | agl 14 | dagl 15 | degl 16 | negl 17 | sugl 18 | un 19 | m 20 | t 21 | s 22 | v 23 | d 24 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/hyphenations_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish hyphenations for StopFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | h 4 | n 5 | t 6 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/stemdict_nl.txt: -------------------------------------------------------------------------------- 1 | # Set of overrides for the dutch stemmer 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | fiets fiets 4 | bromfiets bromfiets 5 | ei eier 6 | kind kinder 7 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/lang/stopwords_hy.txt: -------------------------------------------------------------------------------- 1 | # example set of Armenian stopwords. 2 | այդ 3 | այլ 4 | այն 5 | այս 6 | դու 7 | դուք 8 | եմ 9 | են 10 | ենք 11 | ես 12 | եք 13 | է 14 | էի 15 | էին 16 | էինք 17 | էիր 18 | էիք 19 | էր 20 | ըստ 21 | թ 22 | ի 23 | ին 24 | իսկ 25 | իր 26 | կամ 27 | համար 28 | հետ 29 | հետո 30 | մենք 31 | մեջ 32 | մի 33 | ն 34 | նա 35 | նաև 36 | նրա 37 | նրանք 38 | որ 39 | որը 40 | որոնք 41 | որպես 42 | ու 43 | ում 44 | պիտի 45 | վրա 46 | և 47 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/params.json: -------------------------------------------------------------------------------- 1 | { 2 | "params": { 3 | "_UPDATE_JSON_DOCS": { 4 | "srcField": "_src_", 5 | "mapUniqueKeyOnly": true, 6 | "": { 7 | "v": 0 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/spellings.txt: -------------------------------------------------------------------------------- 1 | pizza 2 | history -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/did_you_mean.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Hyperlinked spelling suggestions in results list 3 | *# 4 | 5 | #set($collations = $response.response.spellcheck.collations.getAll("collation")) 6 | #if($collations && $collations.size() > 0) 7 | Did you mean 8 | #foreach($collation in $collations) 9 | $esc.html($collation.collationQuery) ($collation.hits)? 10 | #end 11 | #end 12 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/error.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Show Error Message, if any 3 | *# 4 | 5 | ## Show Error Message, if any 6 | ## Usually rendered inside div class=error 7 | 8 | #if( $response.response.error.code ) 9 |

ERROR $response.response.error.code

10 | $response.response.error.msg 11 | #end 12 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/facet_pivot.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Display Pivot-Based Facets 3 | * e.g.: facets specified by &facet.pivot= 4 | *# 5 | 6 |

7 | Pivot Facets 8 |

9 | 10 | #set($pivot = $response.response.facet_counts.facet_pivot) 11 | 12 | #display_facet_pivot($pivot, "") 13 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/facet_queries.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Display facets based on specific facet queries 3 | * e.g.: facets specified by &facet.query= 4 | *# 5 | 6 | #set($field = $response.response.facet_counts.facet_queries) 7 | 8 |

9 | Query Facets 10 |

11 | 12 | #display_facet_query($field, "", "") 13 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/facets.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Overall Facet display block 3 | * Invokes the 4 facet and 1 cluster template 4 | *# 5 | 6 | #parse('facet_fields.vm') 7 | #parse('facet_queries.vm') 8 | #parse('facet_ranges.vm') 9 | #parse('facet_pivot.vm') 10 | #parse('cluster.vm') 11 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/header.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Render the top section of the page visible to users 3 | *# 4 | 5 | 8 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/join_doc.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Display documents that are joined to other documents 3 | *# 4 | 5 |
6 | #field('compName_s') 7 |
8 | 9 |
10 | Id: #field('id') 11 | (company-details document for 12 | join 13 | ) 14 |
15 | 16 |
17 | Address: #field('address_s') 18 |
19 | 20 | #parse('debug.vm') 21 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/layout.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Overall HTML page layout 3 | *# 4 | 5 | 6 | 7 | #parse("head.vm") 8 | 9 | 10 | 11 | 14 |
15 | #parse("tabs.vm") 16 |
17 |
18 | $content 19 |
20 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/pagination_bottom.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Paging and Statistics at bottom of results 3 | *# 4 | 5 | ## Usually rendered in pagination div tag 6 | 7 | #if($response.response.get('grouped')) 8 | ## pass 9 | #else 10 | 11 | #link_to_previous_page("previous") 12 | 13 | $page.results_found 14 | results found. 15 | 16 | Page $page.current_page_number 17 | of $page.page_count 18 | 19 | #link_to_next_page("next") 20 | 21 | #end 22 |
23 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/results_list.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Render the main Results List 3 | *# 4 | 5 | ## Usually displayed inside
6 | 7 | #if($response.response.get('grouped')) 8 | 9 | #foreach($grouping in $response.response.get('grouped')) 10 | #parse("hit_grouped.vm") 11 | #end 12 | 13 | #else 14 | 15 | #foreach($doc in $response.results) 16 | #parse("hit.vm") 17 | ## Can get an extremely simple view of the doc 18 | ## which might be nicer for debugging 19 | ##parse("hit_plain.vm") 20 | #end 21 | 22 | #end 23 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/build/templates/solr/conf/configsets/sample_techproducts_configs/conf/velocity/suggest.vm: -------------------------------------------------------------------------------- 1 | #** 2 | * Provides cynamic spelling suggestions 3 | * as you type in the search form 4 | *# 5 | 6 | #foreach($t in $response.response.terms.name) 7 | $t.key 8 | #end 9 | -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eax 3 | 4 | $SOLR_HOME/bin/solr zk upconfig -n schemalessTemplate -d $SOLR_HOME/schemalessTemplate -z $HOSTNAME:2181/solr 5 | 6 | exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf -------------------------------------------------------------------------------- /whitelabel/docker/cdp711/get-docker-cmd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Finding whether docker can be run without sudo" 4 | 5 | # attempt to run an innocuous docker command 6 | set +e 7 | docker ps &> /dev/null 8 | EXIT_CODE=$? 9 | set -e 10 | 11 | # check exit code; if 1, assume permission error 12 | if [ ${EXIT_CODE} -eq 0 ]; then 13 | echo "Command \"docker ps\" succeeded, using \"docker\" as command" 14 | DOCKER_CMD="docker" 15 | else 16 | echo "Command \"docker ps\" failed, using \"sudo docker\" as command" 17 | DOCKER_CMD="sudo docker" 18 | fi -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/build-and-tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ax 4 | 5 | cd docker 6 | docker build . -t registry.gitlab.com/agilefactory/agile.wasp2/wasp-hadoop-vanilla:2 7 | 8 | #docker login registry.gitlab.com/agilefactory/agile.wasp2 9 | #docker push registry.gitlab.com/agilefactory/agile.wasp2/cdh7:base 10 | #docker push registry.gitlab.com/agilefactory/agile.wasp2/cdh7:worker 11 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/common/apt-clean-caches.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Cleaning apt caches" 4 | 5 | rm -rf /var/lib/apt/lists/* -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/common/bash-defaults.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit # exit when a command fails 4 | set -o nounset # exit when a script tries to use undeclared variables 5 | set -o xtrace # trace what gets executed 6 | #set -o pipefail # fail pipeline if a command part of a pipeline fails -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/configure-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | echo "Configuring spark classpath" 6 | hadoop classpath --glob | sed 's/:/\n/g' | grep 'jar' | grep -v netty | grep -v jackson | grep -v capacity-scheduler | sed 's|^|cp |g' | sed 's|$| $SPARK_HOME/jars|g' | bash -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/install-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | wget --progress=bar:force:noscroll -O hadoop.tar.gz https://downloads.apache.org/hadoop/common/stable2/hadoop-2.10.1.tar.gz 6 | 7 | mkdir -p $HADOOP_HOME 8 | 9 | tar -xvf hadoop.tar.gz \ 10 | --directory=$HADOOP_HOME\ 11 | --strip 1 \ 12 | --exclude=hadoop-2.10.1/share/doc 13 | 14 | rm hadoop.tar.gz -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/install-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage kafka 2.6.2 5 | KAFKA_VERSION=2.6.2 6 | KAFKA_PKG_NAME=kafka_2.12-$KAFKA_VERSION 7 | wget --progress=bar:force:noscroll \ 8 | --no-check-certificate https://downloads.apache.org/kafka/$KAFKA_VERSION/$KAFKA_PKG_NAME.tgz 9 | tar -xzvf $KAFKA_PKG_NAME.tgz 10 | rm $KAFKA_PKG_NAME.tgz 11 | mv $KAFKA_PKG_NAME $KAFKA_HOME 12 | 13 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/install-prerequisites.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | apt-get update 6 | apt-get install -y apt-transport-https 7 | apt-get install -y ca-certificates wget sudo 8 | apt-get install -y openjdk-8-jdk-headless software-properties-common python-software-properties #hadoop prerequisites 9 | apt-get install -y gettext-base 10 | apt-get install -y multitail 11 | apt-get install -y supervisor 12 | 13 | exec bash ./common/apt-clean-caches.sh -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/install-solr.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage solr 8 5 | SOLR_VERSION=8.5.1 6 | SOLR_PKG_NAME=solr-$SOLR_VERSION 7 | 8 | wget --progress=bar:force:noscroll --no-check-certificate https://archive.apache.org/dist/lucene/solr/$SOLR_VERSION/$SOLR_PKG_NAME.tgz 9 | tar -xzvf $SOLR_PKG_NAME.tgz 10 | rm $SOLR_PKG_NAME.tgz 11 | mv $SOLR_PKG_NAME $SOLR_HOME 12 | 13 | cp $SOLR_HOME/server/solr/solr.xml $SOLR_HOME/ -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/install-scripts/install-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage spark 2.4.7 5 | SPARK_VERSION=2.4.7 6 | SPARK_PKG_NAME=spark-$SPARK_VERSION-bin-without-hadoop 7 | wget --progress=bar:force:noscroll \ 8 | --no-check-certificate https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_PKG_NAME.tgz 9 | 10 | tar -xzvf $SPARK_PKG_NAME.tgz 11 | 12 | rm $SPARK_PKG_NAME.tgz 13 | mv $SPARK_PKG_NAME $SPARK_HOME -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/templates/hadoop/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://$HOSTNAME:9000 6 | The name of the default file system. A URI whose 7 | scheme and authority determine the FileSystem implementation. The 8 | uri's scheme determines the config property (fs.SCHEME.impl) naming 9 | the FileSystem implementation class. The uri's authority is used to 10 | determine the host, port, etc. for a filesystem. 11 | 12 | 13 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/docker/templates/hbase/regionservers: -------------------------------------------------------------------------------- 1 | $HOSTNAME -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/get-docker-cmd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Finding whether docker can be run without sudo" 4 | 5 | # attempt to run an innocuous docker command 6 | set +e 7 | docker ps &> /dev/null 8 | EXIT_CODE=$? 9 | set -e 10 | 11 | # check exit code; if 1, assume permission error 12 | if [ ${EXIT_CODE} -eq 0 ]; then 13 | echo "Command \"docker ps\" succeeded, using \"docker\" as command" 14 | DOCKER_CMD="docker" 15 | else 16 | echo "Command \"docker ps\" failed, using \"sudo docker\" as command" 17 | DOCKER_CMD="sudo docker" 18 | fi -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/log4j-consumer.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=[consumer] %-5p %d %c: %m%n 8 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/log4j-master.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[master] %-5p %d %c: %m%n 7 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/log4j-producer.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | 5 | 6 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=[producer] %-5p %d %c: %m%n 9 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop2/log4j-single.properties: -------------------------------------------------------------------------------- 1 | log4j.logger.it.agilelab=ERROR 2 | log4j.logger.akka.cluster=DEBUG 3 | 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[single] %-5p %d %c: %m%n 7 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/common/apt-clean-caches.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Cleaning apt caches" 4 | 5 | rm -rf /var/lib/apt/lists/* -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/common/bash-defaults.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit # exit when a command fails 4 | set -o nounset # exit when a script tries to use undeclared variables 5 | set -o xtrace # trace what gets executed 6 | #set -o pipefail # fail pipeline if a command part of a pipeline fails -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/configure-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | echo "Configuring spark classpath" 6 | 7 | hadoop classpath --glob | sed 's/:/\n/g' | grep 'jar' | grep -v netty | grep -v jackson | sed 's|^|cp |g' | sed 's|$| $SPARK_HOME/jars|g' | bash -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | wget --progress=bar:force:noscroll -O hadoop.tar.gz http://www-eu.apache.org/dist/hadoop/common/hadoop-3.2.2/hadoop-3.2.2.tar.gz 6 | 7 | mkdir -p $HADOOP_HOME 8 | 9 | tar -xvf hadoop.tar.gz \ 10 | --directory=$HADOOP_HOME\ 11 | --strip 1 \ 12 | --exclude=hadoop-3.2.2/share/doc 13 | 14 | rm hadoop.tar.gz -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage kafka 2.6.3 5 | KAFKA_VERSION=2.8.1 6 | KAFKA_PKG_NAME=kafka_2.12-$KAFKA_VERSION 7 | wget --progress=bar:force:noscroll \ 8 | --no-check-certificate https://downloads.apache.org/kafka/$KAFKA_VERSION/$KAFKA_PKG_NAME.tgz 9 | tar -xzvf $KAFKA_PKG_NAME.tgz 10 | rm $KAFKA_PKG_NAME.tgz 11 | mv $KAFKA_PKG_NAME $KAFKA_HOME -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-minio.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /minio 6 | chmod +x /minio -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-mongo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | export DEBIAN_FRONTEND=noninteractive 4 | rm /etc/mongod.conf 5 | apt-get install gnupg 6 | wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | sudo apt-key add - 7 | echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/5.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-5.0.list 8 | apt-get update 9 | apt-get install -y mongodb-org -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-prerequisites.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | apt-get update 6 | apt-get install -y apt-transport-https 7 | apt-get install -y ca-certificates wget sudo 8 | apt-get install -y openjdk-8-jdk-headless software-properties-common #hadoop prerequisites 9 | apt-get install -y gettext-base 10 | apt-get install -y multitail 11 | apt-get install -y supervisor 12 | apt-get install -y awscli 13 | 14 | apt-get update 15 | apt-get install -y python3 16 | apt-get install -y python3-flask python3-boto3 17 | 18 | exec bash ./common/apt-clean-caches.sh -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-solr.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage solr 8 5 | SOLR_VERSION=8.5.1 6 | SOLR_PKG_NAME=solr-$SOLR_VERSION 7 | 8 | wget --progress=bar:force:noscroll --no-check-certificate https://archive.apache.org/dist/lucene/solr/$SOLR_VERSION/$SOLR_PKG_NAME.tgz 9 | tar -xzvf $SOLR_PKG_NAME.tgz 10 | rm $SOLR_PKG_NAME.tgz 11 | mv $SOLR_PKG_NAME $SOLR_HOME 12 | 13 | cp $SOLR_HOME/server/solr/solr.xml $SOLR_HOME/ -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/install-scripts/install-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage spark 2.4.2 (the only release with scala 2.12) 5 | SPARK_VERSION=2.4.2 6 | SPARK_PKG_NAME=spark-$SPARK_VERSION-bin-without-hadoop 7 | wget --progress=bar:force:noscroll \ 8 | --no-check-certificate https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_PKG_NAME.tgz 9 | 10 | tar -xzvf $SPARK_PKG_NAME.tgz 11 | 12 | rm $SPARK_PKG_NAME.tgz 13 | mv $SPARK_PKG_NAME $SPARK_HOME 14 | 15 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/templates/hadoop/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://$HOSTNAME:9000 6 | The name of the default file system. A URI whose 7 | scheme and authority determine the FileSystem implementation. The 8 | uri's scheme determines the config property (fs.SCHEME.impl) naming 9 | the FileSystem implementation class. The uri's authority is used to 10 | determine the host, port, etc. for a filesystem. 11 | 12 | 13 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/docker/templates/hbase/regionservers: -------------------------------------------------------------------------------- 1 | $HOSTNAME -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/get-docker-cmd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Finding whether docker can be run without sudo" 4 | 5 | # attempt to run an innocuous docker command 6 | set +e 7 | docker ps &> /dev/null 8 | EXIT_CODE=$? 9 | set -e 10 | 11 | # check exit code; if 1, assume permission error 12 | if [ ${EXIT_CODE} -eq 0 ]; then 13 | echo "Command \"docker ps\" succeeded, using \"docker\" as command" 14 | DOCKER_CMD="docker" 15 | else 16 | echo "Command \"docker ps\" failed, using \"sudo docker\" as command" 17 | DOCKER_CMD="sudo docker" 18 | fi -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/log4j-consumer.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=INFO 3 | log4j.logger.akka.cluster=DEBUG 4 | 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=[consumer] %-5p %d %c: %m%n 8 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/log4j-master.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[master] %-5p %d %c: %m%n 7 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla-hadoop3/log4j-producer.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | 5 | 6 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=[producer] %-5p %d %c: %m%n 9 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/build-and-tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ax 4 | 5 | cd docker 6 | docker build . -t registry.gitlab.com/agilefactory/agile.wasp2/wasp-hadoop-vanilla-2.12:2 7 | 8 | #docker login registry.gitlab.com/agilefactory/agile.wasp2 9 | #docker push registry.gitlab.com/agilefactory/agile.wasp2/cdh7:base 10 | #docker push registry.gitlab.com/agilefactory/agile.wasp2/cdh7:worker 11 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/common/apt-clean-caches.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Cleaning apt caches" 4 | 5 | rm -rf /var/lib/apt/lists/* -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/common/bash-defaults.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit # exit when a command fails 4 | set -o nounset # exit when a script tries to use undeclared variables 5 | set -o xtrace # trace what gets executed 6 | #set -o pipefail # fail pipeline if a command part of a pipeline fails -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/configure-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | echo "Configuring spark classpath" 6 | hadoop classpath --glob | sed 's/:/\n/g' | grep 'jar' | grep -v netty | grep -v jackson | grep -v capacity-scheduler | sed 's|^|cp |g' | sed 's|$| $SPARK_HOME/jars|g' | bash -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/install-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | wget --progress=bar:force:noscroll -O hadoop.tar.gz https://downloads.apache.org/hadoop/common/stable2/hadoop-2.10.1.tar.gz 6 | 7 | mkdir -p $HADOOP_HOME 8 | 9 | tar -xvf hadoop.tar.gz \ 10 | --directory=$HADOOP_HOME\ 11 | --strip 1 \ 12 | --exclude=hadoop-2.10.1/share/doc 13 | 14 | rm hadoop.tar.gz -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/install-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage kafka 2.6.2 5 | KAFKA_VERSION=2.6.2 6 | KAFKA_PKG_NAME=kafka_2.12-$KAFKA_VERSION 7 | wget --progress=bar:force:noscroll \ 8 | --no-check-certificate https://downloads.apache.org/kafka/$KAFKA_VERSION/$KAFKA_PKG_NAME.tgz 9 | tar -xzvf $KAFKA_PKG_NAME.tgz 10 | rm $KAFKA_PKG_NAME.tgz 11 | mv $KAFKA_PKG_NAME $KAFKA_HOME 12 | 13 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/install-prerequisites.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./common/bash-defaults.sh 4 | 5 | apt-get update 6 | apt-get install -y apt-transport-https 7 | apt-get install -y ca-certificates wget sudo 8 | apt-get install -y openjdk-8-jdk-headless software-properties-common python-software-properties #hadoop prerequisites 9 | apt-get install -y gettext-base 10 | apt-get install -y multitail 11 | apt-get install -y supervisor 12 | 13 | exec bash ./common/apt-clean-caches.sh -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/install-solr.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage solr 8 5 | SOLR_VERSION=8.5.1 6 | SOLR_PKG_NAME=solr-$SOLR_VERSION 7 | 8 | wget --progress=bar:force:noscroll --no-check-certificate https://archive.apache.org/dist/lucene/solr/$SOLR_VERSION/$SOLR_PKG_NAME.tgz 9 | tar -xzvf $SOLR_PKG_NAME.tgz 10 | rm $SOLR_PKG_NAME.tgz 11 | mv $SOLR_PKG_NAME $SOLR_HOME 12 | 13 | cp $SOLR_HOME/server/solr/solr.xml $SOLR_HOME/ -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/install-scripts/install-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source ./common/bash-defaults.sh 3 | 4 | # Download and unpackage spark 2.4.7 5 | SPARK_VERSION=2.4.7 6 | SPARK_PKG_NAME=spark-$SPARK_VERSION-bin-without-hadoop-scala-2.12 7 | wget --progress=bar:force:noscroll \ 8 | --no-check-certificate https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_PKG_NAME.tgz 9 | 10 | tar -xzvf $SPARK_PKG_NAME.tgz 11 | 12 | rm $SPARK_PKG_NAME.tgz 13 | mv $SPARK_PKG_NAME $SPARK_HOME -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/templates/hadoop/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://$HOSTNAME:9000 6 | The name of the default file system. A URI whose 7 | scheme and authority determine the FileSystem implementation. The 8 | uri's scheme determines the config property (fs.SCHEME.impl) naming 9 | the FileSystem implementation class. The uri's authority is used to 10 | determine the host, port, etc. for a filesystem. 11 | 12 | 13 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/docker/templates/hbase/regionservers: -------------------------------------------------------------------------------- 1 | $HOSTNAME -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/get-docker-cmd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Finding whether docker can be run without sudo" 4 | 5 | # attempt to run an innocuous docker command 6 | set +e 7 | docker ps &> /dev/null 8 | EXIT_CODE=$? 9 | set -e 10 | 11 | # check exit code; if 1, assume permission error 12 | if [ ${EXIT_CODE} -eq 0 ]; then 13 | echo "Command \"docker ps\" succeeded, using \"docker\" as command" 14 | DOCKER_CMD="docker" 15 | else 16 | echo "Command \"docker ps\" failed, using \"sudo docker\" as command" 17 | DOCKER_CMD="sudo docker" 18 | fi -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/log4j-consumer.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=[consumer] %-5p %d %c: %m%n 8 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/log4j-master.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[master] %-5p %d %c: %m%n 7 | -------------------------------------------------------------------------------- /whitelabel/docker/vanilla2.12-hadoop2/log4j-producer.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN,stdout 2 | log4j.logger.it.agilelab=ERROR 3 | log4j.logger.akka.cluster=DEBUG 4 | 5 | 6 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=[producer] %-5p %d %c: %m%n 9 | -------------------------------------------------------------------------------- /whitelabel/master/build.sbt: -------------------------------------------------------------------------------- 1 | /* !!! Custom Node Launcher usage: Add this in standalone applications !!! */ 2 | Compile / mainClass := Some("it.agilelab.bigdata.wasp.whitelabel.master.launcher.MasterNodeLauncher") -------------------------------------------------------------------------------- /whitelabel/models/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/models/test/TestDocumentEncoder.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.models.test 2 | 3 | case class TestDocumentEncoder(id: String, 4 | number: Int, 5 | nested: TestNestedDocument) 6 | -------------------------------------------------------------------------------- /whitelabel/models/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/models/test/TestEnrichmentModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.models.test 2 | 3 | case class TestEnrichmentModel(id: String, exampleAuthor: String, timestamp: Long) 4 | 5 | case class TestEnrichmentResponseModel(id: String, text: String) 6 | -------------------------------------------------------------------------------- /whitelabel/models/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/models/test/TestFreeCodeModels.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.models.test 2 | 3 | import it.agilelab.bigdata.wasp.models.FreeCodeModel 4 | 5 | object TestFreeCodeModels { 6 | 7 | val testFreeCode: FreeCodeModel = FreeCodeModel("test-freecode", 8 | """ 9 | | dataFrames.head._2 10 | |""".stripMargin) 11 | } 12 | -------------------------------------------------------------------------------- /whitelabel/models/src/main/scala/it/agilelab/bigdata/wasp/whitelabel/models/test/TestSqlSourceModel.scala: -------------------------------------------------------------------------------- 1 | package it.agilelab.bigdata.wasp.whitelabel.models.test 2 | 3 | import it.agilelab.bigdata.wasp.models.SqlSourceModel 4 | 5 | private[wasp] object TestSqlSouceModel { 6 | 7 | lazy val mySql = SqlSourceModel( 8 | name = "TestMySqlModel", 9 | connectionName = "mysql", // have to be present in jdbc-subConfig 10 | dbtable = "test_table", 11 | partitioningInfo = None, 12 | numPartitions = None, 13 | fetchSize = None 14 | ) 15 | } -------------------------------------------------------------------------------- /whitelabel/producers/build.sbt: -------------------------------------------------------------------------------- 1 | /* !!! Custom Node Launcher usage: Add this in standalone applications !!! */ 2 | Compile / mainClass := Some("it.agilelab.bigdata.wasp.whitelabel.producers.launcher.ProducersNodeLauncher") --------------------------------------------------------------------------------