├── .editorconfig ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── question.md └── workflows │ ├── add-issue-to-project.yml │ └── ci-check-jacoco.yml ├── .gitignore ├── .sonarcloud.properties ├── CODE_OF_CONDUCT.md ├── LICENSE.md ├── README.md ├── admin ├── Dockerfile ├── entrypoint.sh ├── pom.xml └── src │ ├── main │ ├── resources │ │ ├── logback.xml │ │ └── spline-cli.properties │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ ├── admin │ │ ├── AdminCLI.scala │ │ ├── AppConfig.scala │ │ ├── ArangoDBAuthenticationException.scala │ │ ├── DateTimeUtils.scala │ │ ├── InputConsole.scala │ │ ├── InteractiveArangoManagerFactoryProxy.scala │ │ ├── commands.scala │ │ └── userInteractors.scala │ │ └── arango │ │ ├── ArangoManager.scala │ │ ├── ArangoManagerFactory.scala │ │ ├── AutoClosingArangoManagerProxy.scala │ │ ├── AuxiliaryDBAction.scala │ │ ├── DataRetentionManager.scala │ │ ├── DatabaseCreateOptions.scala │ │ ├── OnDBExistsAction.scala │ │ └── foxx │ │ ├── FoxxManager.scala │ │ ├── FoxxManagerImpl.scala │ │ └── FoxxSourceResolver.scala │ └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ ├── admin │ ├── AdminCLISpec.scala │ ├── ArangoDBAuthenticationExceptionSpec.scala │ ├── ConsoleUserInteractorSpec.scala │ └── DateTimeUtilsSpec.scala │ ├── arango │ ├── ArangoManagerImplSpec.scala │ └── foxx │ │ └── FoxxSourceResolverSpec.scala │ └── package.scala ├── arangodb-foxx-api ├── assembly.xml ├── pom.xml └── src │ └── main │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ ├── persistence │ └── model │ │ ├── Attribute.scala │ │ ├── Schema.scala │ │ ├── entities.scala │ │ ├── expressions.scala │ │ └── operations.scala │ └── producer │ └── service │ └── model │ └── ExecutionPlanPersistentModel.scala ├── arangodb-foxx-services ├── .eslintrc.js ├── .gitignore ├── README.md ├── __mocks__ │ ├── @arangodb.js │ ├── @arangodb │ │ └── locals.ts │ └── README.md ├── assembly.xml ├── jest.config.json ├── package-lock.json ├── package.json ├── pom.xml ├── src │ ├── README.md │ ├── aql │ │ ├── README.txt │ │ └── example.func.ts │ ├── assets │ │ └── spline-logo.300x300.png │ ├── main │ │ ├── index.ts │ │ ├── model.ts │ │ ├── persistence │ │ │ └── model.ts │ │ ├── routes │ │ │ ├── admin-router.ts │ │ │ ├── events-router.ts │ │ │ ├── index.ts │ │ │ └── plans-router.ts │ │ ├── services │ │ │ ├── commons.ts │ │ │ ├── execution-event-store.ts │ │ │ ├── execution-plan-store.ts │ │ │ ├── impact-overview.ts │ │ │ ├── lineage-overview.ts │ │ │ ├── observed-reads-by-write.ts │ │ │ ├── observed-writes-by-read.ts │ │ │ ├── prune-database.ts │ │ │ ├── store.ts │ │ │ └── txm │ │ │ │ ├── index.ts │ │ │ │ ├── subcribable-tx-manager-decorator.ts │ │ │ │ ├── tx-manager-impl.ts │ │ │ │ └── tx-manager.ts │ │ └── utils │ │ │ ├── aql-gen-helper.ts │ │ │ ├── common.ts │ │ │ ├── graph.ts │ │ │ ├── logger.ts │ │ │ └── types.ts │ ├── manifest.json │ ├── modules │ │ └── consumer-index-update │ │ │ ├── data-source.last-write.update.ts │ │ │ └── index.ts │ └── scripts │ │ ├── aql-functions.d.ts │ │ ├── fs.d.ts │ │ ├── setup.ts │ │ └── teardown.ts ├── tests │ ├── aql │ │ └── example.spec.ts │ ├── global.d.ts │ └── main │ │ ├── services │ │ └── txm │ │ │ └── subscribable-tx-manager-decorator.spec.ts │ │ └── utils │ │ ├── aql-gen-helper.spec.ts │ │ └── common.spec.ts ├── tsconfig.json └── webpack.config.js ├── build └── parent-pom │ └── pom.xml ├── commons ├── pom.xml └── src │ ├── main │ ├── resources │ │ └── spline-build.properties │ └── scala │ │ ├── com │ │ ├── arangodb │ │ │ └── entity │ │ │ │ └── ErrorEntityImplicits.scala │ │ └── twitter │ │ │ └── finatra │ │ │ └── jackson │ │ │ └── FinatraInternalModules.scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── common │ │ ├── AsyncCallRetryer.scala │ │ ├── CollectionUtils.scala │ │ ├── ConsoleUtils.scala │ │ ├── JsonPath.scala │ │ ├── SplineBuildInfo.scala │ │ ├── StringEscapeUtils.scala │ │ ├── config │ │ ├── DefaultConfigurationStack.scala │ │ └── HttpConfiguration.scala │ │ ├── graph │ │ └── GraphImplicits.scala │ │ ├── rest │ │ ├── HttpStatusException.scala │ │ ├── RESTClient.scala │ │ └── RESTClientApacheHttpImpl.scala │ │ ├── scala13 │ │ └── Option.scala │ │ ├── security │ │ └── TLSUtils.scala │ │ ├── validation │ │ ├── Constraint.scala │ │ ├── UniqueConstraint.scala │ │ └── ValidationUtils.scala │ │ └── webmvc │ │ ├── AppInitializerUtils.scala │ │ ├── ScalaFutureMethodReturnValueHandler.scala │ │ ├── UnitMethodReturnValueHandler.scala │ │ ├── controller │ │ ├── DefaultRestController.scala │ │ └── package.scala │ │ ├── cors │ │ ├── ConstantCorsConfigurationSource.scala │ │ └── PermissiveCorsFilter.scala │ │ ├── diagnostics │ │ ├── DiagnosticsRESTConfig.scala │ │ ├── RootWebContextConfig.scala │ │ └── controller │ │ │ ├── DiagnosticsController.scala │ │ │ └── package.scala │ │ ├── jackson │ │ ├── NullAcceptingDeserializer.scala │ │ └── ObjectMapperBeanPostProcessor.scala │ │ └── swagger │ │ ├── SwaggerConfig.scala │ │ ├── SwaggerRequiredPropertyBuilderPlugin.scala │ │ ├── SwaggerScalaTypesRules.scala │ │ └── SwaggerUISupport.scala │ └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── common │ ├── ConsoleUtilsSpec.scala │ ├── JsonPathSpec.scala │ ├── StringEscapeUtilsSpec.scala │ ├── graph │ ├── AbstractGraphImplicits_SortedTopologicallySpec.scala │ ├── GraphImplicits_SortedTopologicallyBySpec.scala │ └── GraphImplicits_SortedTopologicallySpec.scala │ ├── validation │ └── UniqueConstraintSpec.scala │ └── webmvc │ ├── ScalaFutureMethodReturnValueHandlerSpec.scala │ └── jackson │ ├── NullAcceptingDeserializerSpec.scala │ └── ObjectMapperBeanPostProcessorSpec.scala ├── consumer-rest-core ├── pom.xml └── src │ ├── main │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── consumer │ │ └── rest │ │ ├── ConsumerRESTConfig.scala │ │ ├── ConsumerTypeResolver.scala │ │ ├── controller │ │ ├── AbstractExecutionEventsController.scala │ │ ├── DataSourcesController.scala │ │ ├── ErrorControllerAdvice.scala │ │ ├── ExecutionEventsController.scala │ │ ├── ExecutionPlansController.scala │ │ ├── ImpactOverviewController.scala │ │ ├── LabelsController.scala │ │ ├── LineageDetailedController.scala │ │ ├── LineageOverviewController.scala │ │ ├── OperationDetailsController.scala │ │ └── package.scala │ │ └── model │ │ ├── LabelQuery.scala │ │ └── WriteModeQuery.scala │ └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── consumer │ └── rest │ └── controller │ ├── ExecutionPlansControllerTest.scala │ └── OperationDetailsControllerTest.scala ├── consumer-services ├── pom.xml └── src │ └── main │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── consumer │ └── service │ ├── ConsumerServicesConfig.scala │ ├── model │ ├── Attribute.scala │ ├── AttributeEdge.scala │ ├── AttributeGraph.scala │ ├── AttributeNode.scala │ ├── DataSourceActionType.scala │ ├── DataSourceInfo.scala │ ├── ExecutionEventInfo.scala │ ├── ExecutionPlanInfo.scala │ ├── ExpressionEdge.scala │ ├── ExpressionGraph.scala │ ├── ExpressionNode.scala │ ├── Graph.scala │ ├── Label.scala │ ├── LineageDetailed.scala │ ├── LineageDetailedGraph.scala │ ├── LineageOverview.scala │ ├── LineageOverviewGraph.scala │ ├── Operation.scala │ ├── OperationDetails.scala │ ├── PageRequest.scala │ ├── Pageable.scala │ ├── PageableExecutionEventsResponse.scala │ ├── PageableExecutionPlansResponse.scala │ ├── Schema.scala │ ├── SortRequest.scala │ ├── Transition.scala │ ├── dataTypes.scala │ └── lineageOverviewNodes.scala │ └── repo │ ├── AbstractExecutionEventRepository.scala │ ├── DataSourceRepository.scala │ ├── DataSourceRepositoryImpl.scala │ ├── ExecutionEventRepository.scala │ ├── ExecutionEventRepositoryImpl.scala │ ├── ExecutionPlanRepository.scala │ ├── ExecutionPlanRepositoryImpl.scala │ ├── ExpressionRepository.scala │ ├── ExpressionRepositoryImpl.scala │ ├── ImpactLineageRepositoryImpl.scala │ ├── ImpactRepository.scala │ ├── LabelRepository.scala │ ├── LabelRepositoryImpl.scala │ ├── LineageRepository.scala │ ├── OperationRepository.scala │ ├── OperationRepositoryImpl.scala │ └── package.scala ├── integration-tests ├── pom.xml └── src │ ├── main │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── test │ │ └── fixture │ │ ├── ArangoDbFixture.scala │ │ ├── ArangoDbFixtureAsync.scala │ │ ├── ArangoDbFixtureLike.scala │ │ ├── TestContainersFixture.scala │ │ └── TestContainersFixtureAsync.scala │ └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── admin │ └── AdminCLISpec.scala ├── kafka-gateway ├── Dockerfile ├── README.md ├── pom.xml └── src │ └── main │ ├── resources │ └── logback.xml │ ├── scala │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── gateway │ │ └── kafka │ │ ├── AppInitializer.scala │ │ ├── KafkaGatewayConfig.scala │ │ └── listener │ │ ├── IngestionListener.scala │ │ ├── handler │ │ ├── HandlerV11.scala │ │ └── HandlerV12.scala │ │ └── package.scala │ └── webapp │ ├── META-INF │ └── context.xml │ ├── favicon.ico │ └── index.html ├── persistence ├── pom.xml └── src │ ├── main │ ├── resources │ │ └── migration-scripts │ │ │ ├── 0.4.0-0.5.0.js │ │ │ ├── 0.5.0-0.5.4.js │ │ │ ├── 0.5.4-0.5.5.js │ │ │ ├── 0.5.5-0.6.1.js │ │ │ ├── 0.6.0-0.6.1.js │ │ │ ├── 0.6.1-0.7.0.js │ │ │ ├── 0.7.0-1.0.0.js │ │ │ └── template.js │ └── scala │ │ ├── com │ │ └── arangodb │ │ │ ├── internal │ │ │ ├── InternalArangoDatabaseOps.scala │ │ │ └── velocystream │ │ │ │ └── VstImplicits.scala │ │ │ └── model │ │ │ └── Implicits.scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── persistence │ │ ├── ArangoCode.scala │ │ ├── ArangoConnectionURL.scala │ │ ├── ArangoDatabaseFacade.scala │ │ ├── ArangoImplicits.scala │ │ ├── ArangoRepoConfig.scala │ │ ├── DatabaseException.scala │ │ ├── DatabaseVersionChecker.scala │ │ ├── DatabaseVersionManager.scala │ │ ├── DefaultJsonSerDe.scala │ │ ├── DryRunnable.scala │ │ ├── LogMessageUtils.scala │ │ ├── MultiDocumentArangoDBException.scala │ │ ├── RetryableExceptionUtils.scala │ │ ├── migration │ │ ├── MigrationScript.scala │ │ ├── MigrationScriptLoader.scala │ │ ├── MigrationScriptRepository.scala │ │ ├── Migrator.scala │ │ └── ResourceMigrationScriptLoader.scala │ │ ├── model │ │ └── persistentDefs.scala │ │ └── tx │ │ ├── AbstractTxBuilder.scala │ │ ├── ArangoTx.scala │ │ ├── FoxxPostTxBuilder.scala │ │ ├── JSTxBuilder.scala │ │ └── queries.scala │ └── test │ └── scala │ ├── com │ └── arangodb │ │ └── model │ │ └── ImplicitsSpec.scala │ └── za │ └── co │ └── absa │ └── spline │ └── persistence │ ├── ArangoConnectionURLSpec.scala │ ├── ArangoRepoConfigSpec.scala │ ├── AsyncCallRetryerSpec.scala │ ├── DryRunnableSpec.scala │ ├── RetryableExceptionUtilsSpec.scala │ ├── migration │ ├── MigrationScriptRepositorySpec.scala │ └── ResourceMigrationScriptLoaderSpec.scala │ ├── model │ └── DataSourceSpec.scala │ └── tx │ └── JSTxBuilderSpec.scala ├── pom.xml ├── producer-model-mapper ├── pom.xml └── src │ └── main │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── producer │ └── modelmapper │ ├── ModelMapper.scala │ ├── v1_0 │ ├── AttributeConverter.scala │ ├── AttributeDependencyResolver.scala │ ├── ExecutionPlanComponentConverterFactory.scala │ ├── ExpressionConverter.scala │ ├── FieldNamesV10.scala │ ├── ModelMapperV10.scala │ ├── ObjectConverter.scala │ ├── OperationConverter.scala │ ├── OperationOutputConverter.scala │ ├── RecursiveSchemaFinder.scala │ ├── TypesV10.scala │ └── spark │ │ ├── AttributeRefConverter.scala │ │ ├── SparkSpline04AttributeDependencyResolver.scala │ │ ├── SparkSplineAttributeConverter.scala │ │ ├── SparkSplineExecutionPlanComponentConverterFactory.scala │ │ ├── SparkSplineExpressionConverter.scala │ │ ├── SparkSplineObjectConverter.scala │ │ └── SparkSplineOperationOutputConverter.scala │ ├── v1_1 │ └── ModelMapperV11.scala │ └── v1_2 │ └── ModelMapperV12.scala ├── producer-model ├── pom.xml └── src │ └── main │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── producer │ └── model │ ├── ExecutionEvent.scala │ ├── ExecutionPlan.scala │ ├── executionPlanModel.scala │ ├── operationModel.scala │ ├── package.scala │ ├── v1_1 │ ├── AttrOrExprRef.scala │ ├── Attribute.scala │ ├── ExecutionEvent.scala │ ├── executionPlan.scala │ ├── expressions.scala │ └── operations.scala │ └── v1_2 │ ├── AttrOrExprRef.scala │ ├── Attribute.scala │ ├── ExecutionEvent.scala │ ├── executionPlan.scala │ ├── expressions.scala │ └── operations.scala ├── producer-rest-core ├── pom.xml └── src │ ├── main │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── producer │ │ └── rest │ │ ├── HttpConstants.scala │ │ ├── ProducerAPI.scala │ │ ├── ProducerRESTConfig.scala │ │ ├── controller │ │ ├── ErrorControllerAdvice.scala │ │ ├── ExecutionPlansControllerMessageLengthCapturingAspect.scala │ │ ├── IngestionController.scala │ │ ├── StatusController.scala │ │ ├── handler │ │ │ ├── HandlerV10.scala │ │ │ ├── HandlerV11.scala │ │ │ └── HandlerV12.scala │ │ └── package.scala │ │ └── filter │ │ ├── GzipFilter.scala │ │ ├── HttpRequestWrapper.scala │ │ ├── MessageLengthCapturingFilter.scala │ │ └── ServletInputStreamAdapter.scala │ └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── producer │ └── modelmapper │ └── v1_0 │ └── spark │ ├── SparkSplineExpressionConverterSpec.scala │ └── SparkSplineObjectConverterSpec.scala ├── producer-services ├── pom.xml └── src │ ├── main │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── producer │ │ └── service │ │ ├── InconsistentEntityException.scala │ │ ├── ProducerServicesConfig.scala │ │ ├── UUIDCollisionDetectedException.scala │ │ ├── model │ │ ├── AbstractNodeKeyCreator.scala │ │ ├── ExecutionEventKeyCreator.scala │ │ ├── ExecutionPlanKeyCreator.scala │ │ └── ExecutionPlanPersistentModelBuilder.scala │ │ └── repo │ │ ├── ExecutionPlanDeserFixAspect.scala │ │ ├── ExecutionProducerRepository.scala │ │ ├── ExecutionProducerRepositoryImpl.scala │ │ └── package.scala │ └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── producer │ └── service │ ├── model │ └── ExecutionPlanPersistentModelBuilderSpec.scala │ └── repo │ ├── ExecutionEventKeyCreatorSpec.scala │ └── ExecutionPlanDeserFixAspectSpec.scala ├── rest-gateway ├── Dockerfile ├── package-lock.json ├── package.json ├── pom.xml └── src │ └── main │ ├── resources │ └── logback.xml │ ├── scala │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── gateway │ │ └── rest │ │ └── AppInitializer.scala │ └── webapp │ ├── META-INF │ └── context.xml │ ├── favicon.ico │ └── index.html └── test-data-generator ├── Dockerfile ├── pom.xml ├── send-plans-and-events-docker.sh ├── send-plans-and-events.sh ├── src ├── main │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── spline │ │ └── testdatagen │ │ ├── FileDispatcher.scala │ │ ├── GenCLI.scala │ │ ├── GraphType.scala │ │ ├── NumericValue.scala │ │ ├── config.scala │ │ └── generators │ │ ├── AttributesGenerator.scala │ │ ├── EventGenerator.scala │ │ ├── ExpressionGenerator.scala │ │ ├── Graph.scala │ │ └── graph │ │ ├── Chain.scala │ │ ├── Diamond.scala │ │ └── Triangle.scala └── test │ └── scala │ └── za │ └── co │ └── absa │ └── spline │ └── testdatagen │ ├── ConfigSpec.scala │ └── generators │ └── graph │ ├── AttributeExpressionReferenceSpec.scala │ ├── ChainSpec.scala │ ├── DiamondSpec.scala │ └── TriangleSpec.scala └── triangle-lineage-3reads-20ops-3attr.json.txt /.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | max_line_length = 140 10 | 11 | [*.scala] 12 | indent_size = 2 13 | ij_scala_indent_first_parameter = false 14 | ij_scala_indent_first_parameter_clause = true 15 | ij_scala_method_parameters_new_line_after_left_paren = true 16 | ij_scala_method_parameters_right_paren_on_new_line = true 17 | 18 | [{*.ts,*.js}] 19 | indent_size = 4 20 | 21 | # quotes 22 | quote_type = single 23 | ij_typescript_force_quote_style = true 24 | ij_typescript_use_double_quotes = false 25 | 26 | # semicolons 27 | ij_typescript_force_semicolon_style = true 28 | ij_typescript_use_semicolon_after_statement = false 29 | 30 | # braces 31 | indent_brace_style = stroustrup 32 | ij_typescript_else_on_new_line = true 33 | ij_typescript_catch_on_new_line = true 34 | ij_typescript_finally_on_new_line = true 35 | 36 | # imports 37 | ij_typescript_blank_lines_after_imports = 2 38 | ij_typescript_import_sort_members = true 39 | 40 | [*.md] 41 | max_line_length = off 42 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @wajda @cerveada 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Describe the bug 11 | 12 | A clear and concise description of what the bug is. 13 | 14 | ## Versions 15 | 16 | Please provide versions of: Spline, Spark and Scala that were in use when the bug happened. 17 | 18 | ## Components State 19 | 20 | Describe the state of different Spline components (Agent, REST or Kafka Gateway, UI, ArangoDB). 21 | 22 | - Where the problem manifests itself? 23 | - Are there any errors in logs? 24 | - Is ArangoDB initialized and contains data? 25 | - Any other observations that you think might be important. 26 | 27 | ## To Reproduce 28 | 29 | Steps to reproduce the behavior OR commands run: 30 | 31 | 1. Go to '...' 32 | 2. Click on '....' 33 | 3. Enter value '...' 34 | 4. See error 35 | 36 | ## Expected behaviour 37 | 38 | A clear and concise description of what you expected to happen. 39 | 40 | ## Screenshots 41 | 42 | If applicable, add screenshots to help explain your problem. 43 | 44 | ## Desktop (please complete the following information): 45 | 46 | - OS: [e.g. iOS] 47 | - Browser [e.g. chrome, safari] 48 | - Version [e.g. 22] 49 | 50 | ## Additional context 51 | 52 | Add any other context about the problem here. 53 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: feature 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Background 11 | 12 | A clear and concise description of where the limitation lies. 13 | 14 | ## Feature 15 | 16 | A description of the requested feature. 17 | 18 | ## Example [Optional] 19 | 20 | A simple example if applicable. 21 | 22 | ## Proposed Solution [Optional] 23 | 24 | Solution Ideas 25 | 26 | 1. 27 | 2. 28 | 3. 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question 4 | title: '' 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Background [Optional] 11 | 12 | A clear explanation of the reason for raising the question. 13 | This gives us a better understanding of your use cases and how we might accommodate them. 14 | 15 | ## Question 16 | 17 | A clear and concise inquiry 18 | -------------------------------------------------------------------------------- /.github/workflows/add-issue-to-project.yml: -------------------------------------------------------------------------------- 1 | name: Add issue to project 2 | on: 3 | issues: 4 | types: [opened, transferred] 5 | 6 | jobs: 7 | add-to-project: 8 | name: Add issue to project 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/add-to-project@v0.4.0 12 | with: 13 | project-url: https://github.com/orgs/AbsaOSS/projects/4 14 | github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} 15 | -------------------------------------------------------------------------------- /.sonarcloud.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline/a7f43d2bc22b6dc8c5bd4e116d0ec45514cd1d46/.sonarcloud.properties -------------------------------------------------------------------------------- /admin/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Run image using: 18 | # $> docker run --rm -it absaoss/spline-admin:latest 19 | 20 | FROM adoptopenjdk:11-jre 21 | 22 | ARG PROJECT_BUILD_FINAL_NAME 23 | ARG IMAGE_NAME 24 | 25 | ENV IMAGE_NAME=$IMAGE_NAME 26 | 27 | LABEL \ 28 | vendor="ABSA" \ 29 | copyright="2021 ABSA Group Limited" \ 30 | license="Apache License, version 2.0" \ 31 | name="Spline Admin Tool" 32 | 33 | # Install Tini 34 | RUN apt-get update \ 35 | && apt-get install -y --no-install-recommends \ 36 | tini=0.18* \ 37 | && rm -rf /var/lib/apt/lists/* 38 | 39 | USER 1001 40 | 41 | COPY target/$PROJECT_BUILD_FINAL_NAME.jar ./admin.jar 42 | COPY entrypoint.sh . 43 | 44 | ENTRYPOINT ["tini", "-g", "--", "sh", "./entrypoint.sh"] 45 | CMD ["--help"] 46 | -------------------------------------------------------------------------------- /admin/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright 2021 ABSA Group Limited 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | CLI_EXECUTABLE="docker run -it --rm $IMAGE_NAME" 20 | 21 | java -Dspline.cli.executable="$CLI_EXECUTABLE" -jar admin.jar "$@" 22 | -------------------------------------------------------------------------------- /admin/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] %-5level [%thread] %-30logger{30} - %msg%n 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /admin/src/main/resources/spline-cli.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | spline.cli.executable=${spline.cli.executable} 17 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/admin/AppConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.admin 18 | 19 | import org.apache.commons.configuration.{CompositeConfiguration, PropertiesConfiguration, SystemConfiguration} 20 | import za.co.absa.commons.config.ConfTyped 21 | import za.co.absa.commons.config.ConfigurationImplicits._ 22 | 23 | import java.util 24 | 25 | object AppConfig 26 | extends CompositeConfiguration(util.Arrays.asList( 27 | new SystemConfiguration, 28 | new PropertiesConfiguration(ClassLoader.getSystemResource("spline-cli.properties")) 29 | )) with ConfTyped { 30 | 31 | private val conf = this 32 | 33 | object Spline extends Conf("spline") { 34 | object CLI extends Conf("cli") { 35 | val Executable: String = conf.getRequiredString(Prop("executable")) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/admin/ArangoDBAuthenticationException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.admin 18 | 19 | import com.arangodb.ArangoDBException 20 | 21 | import java.net.HttpURLConnection.HTTP_UNAUTHORIZED 22 | import scala.annotation.tailrec 23 | import scala.util.control.NonFatal 24 | 25 | object ArangoDBAuthenticationException { 26 | def unapply(t: Throwable): Option[ArangoDBException] = { 27 | @tailrec def loop(ex: Throwable, visited: Set[Throwable]): Option[ArangoDBException] = 28 | if (visited(ex)) None 29 | else ex match { 30 | case null => None 31 | case ae: ArangoDBException if ae.getErrorNum == HTTP_UNAUTHORIZED => Some(ae) 32 | case NonFatal(e) => loop(e.getCause, visited + e) 33 | case _ => None 34 | } 35 | 36 | loop(t, Set.empty) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/admin/InputConsole.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.admin 18 | 19 | import java.io.Console 20 | 21 | trait InputConsole { 22 | def readLine(msg: String): String 23 | def readPassword(msg: String): String 24 | } 25 | 26 | object InputConsole { 27 | 28 | def systemConsoleIfAvailable(): Option[InputConsole] = Option(System.console()).map(new SystemConsole(_)) 29 | 30 | private class SystemConsole(console: Console) extends InputConsole { 31 | 32 | override def readLine(msg: String): String = console.readLine(msg) 33 | 34 | override def readPassword(msg: String): String = new String(console.readPassword(msg)) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/arango/AuxiliaryDBAction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.arango 18 | 19 | sealed trait AuxiliaryDBAction 20 | 21 | object AuxiliaryDBAction { 22 | 23 | case object CheckDBAccess extends AuxiliaryDBAction 24 | 25 | case object FoxxReinstall extends AuxiliaryDBAction 26 | 27 | case object IndicesDelete extends AuxiliaryDBAction 28 | 29 | case object IndicesCreate extends AuxiliaryDBAction 30 | 31 | case object SearchViewsDelete extends AuxiliaryDBAction 32 | 33 | case object SearchViewsCreate extends AuxiliaryDBAction 34 | 35 | case object SearchAnalyzerDelete extends AuxiliaryDBAction 36 | 37 | case object SearchAnalyzerCreate extends AuxiliaryDBAction 38 | 39 | } 40 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/arango/DataRetentionManager.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.arango 18 | 19 | 20 | import com.arangodb.async.ArangoDatabaseAsync 21 | import za.co.absa.spline.persistence.ArangoImplicits._ 22 | import za.co.absa.spline.persistence.DryRunnable 23 | 24 | import scala.concurrent.{ExecutionContext, Future} 25 | 26 | class DataRetentionManager( 27 | db: ArangoDatabaseAsync, 28 | val dryRun: Boolean 29 | )(implicit ec: ExecutionContext) 30 | extends DryRunnable { 31 | 32 | def pruneBefore(timestamp: Long): Future[Unit] = unlessDryRunAsync { 33 | db.restClient.delete(s"spline/admin/data/before/$timestamp") 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/arango/DatabaseCreateOptions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.arango 18 | 19 | import za.co.absa.spline.persistence.model.CollectionDef 20 | 21 | case class DatabaseCreateOptions( 22 | 23 | numShards: Map[CollectionDef, Int] = Map.empty, 24 | numShardsDefault: Option[Int] = None, 25 | 26 | shardKeys: Map[CollectionDef, Seq[String]] = Map.empty, 27 | shardKeysDefault: Option[Seq[String]] = None, 28 | 29 | replFactor: Map[CollectionDef, Int] = Map.empty, 30 | replFactorDefault: Option[Int] = None, 31 | 32 | waitForSync: Boolean = false 33 | ) 34 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/arango/OnDBExistsAction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.arango 18 | 19 | sealed trait OnDBExistsAction 20 | 21 | object OnDBExistsAction { 22 | 23 | case object Fail extends OnDBExistsAction 24 | 25 | case object Drop extends OnDBExistsAction 26 | 27 | case object Skip extends OnDBExistsAction 28 | 29 | } 30 | -------------------------------------------------------------------------------- /admin/src/main/scala/za/co/absa/spline/arango/foxx/FoxxManager.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.arango.foxx 18 | 19 | import scala.concurrent.Future 20 | 21 | trait FoxxManager { 22 | def install(mountPrefix: String, content: Array[Byte]): Future[Unit] 23 | def uninstall(mountPrefix: String): Future[Unit] 24 | def list(): Future[Seq[Map[String, Any]]] 25 | } 26 | -------------------------------------------------------------------------------- /admin/src/test/scala/za/co/absa/spline/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package za.co.absa 17 | 18 | import org.scalatestplus.mockito.MockitoSugar.mock 19 | 20 | import scala.reflect.ClassTag 21 | 22 | package object spline { 23 | /** 24 | * This is just to add another word for `Mockito.mock` method, for better test readability. 25 | * 26 | * @tparam A type of the dummy 27 | * @return a dummy instance of the given type 28 | */ 29 | def dummy[A <: AnyRef : ClassTag]: A = mock[A] 30 | } 31 | -------------------------------------------------------------------------------- /arangodb-foxx-api/assembly.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 20 | ts 21 | 22 | zip 23 | 24 | false 25 | 26 | 27 | ${project.build.directory}/generated-ts 28 | 29 | unix 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Attribute.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.model 18 | 19 | case class Attribute( 20 | override val _key: ArangoDocument.Key, 21 | override val _belongsTo: Option[ArangoDocument.Id], 22 | dataType: Option[Any], 23 | extra: Map[String, Any], 24 | name: String, 25 | ) extends Vertex 26 | -------------------------------------------------------------------------------- /arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/Schema.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.model 18 | 19 | case class Schema( 20 | override val _key: ArangoDocument.Key, 21 | override val _belongsTo: Option[ArangoDocument.Id] 22 | ) extends Vertex 23 | -------------------------------------------------------------------------------- /arangodb-foxx-api/src/main/scala/za/co/absa/spline/persistence/model/expressions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.model 18 | 19 | sealed trait Expression extends Vertex { 20 | def `type`: String 21 | def dataType: Option[Any] 22 | def extra: Map[String, Any] 23 | } 24 | 25 | case class FunctionalExpression( 26 | override val _key: ArangoDocument.Key, 27 | override val _belongsTo: Option[ArangoDocument.Id], 28 | override val dataType: Option[Any], 29 | override val extra: Map[String, Any], 30 | name: String, 31 | arity: Int, 32 | params: Map[String, Any], 33 | ) extends Expression { 34 | val `type`: String = "Func" 35 | } 36 | 37 | case class LiteralExpression( 38 | override val _key: ArangoDocument.Key, 39 | override val _belongsTo: Option[ArangoDocument.Id], 40 | override val dataType: Option[Any], 41 | override val extra: Map[String, Any], 42 | value: Any, 43 | ) extends Expression { 44 | val `type`: String = "Lit" 45 | } 46 | -------------------------------------------------------------------------------- /arangodb-foxx-services/.eslintrc.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | module.exports = { 17 | env: { 18 | browser: false, 19 | commonjs: true, 20 | es6: true 21 | }, 22 | extends: [ 23 | 'eslint:recommended', 24 | 'plugin:@typescript-eslint/recommended' 25 | ], 26 | parser: '@typescript-eslint/parser', 27 | parserOptions: { 28 | ecmaVersion: 'latest' 29 | }, 30 | plugins: [ 31 | '@typescript-eslint' 32 | ], 33 | rules: { 34 | '@typescript-eslint/ban-ts-comment': 'off', 35 | indent: [ 36 | 'error', 37 | 4 38 | ], 39 | 'linebreak-style': [ 40 | process.platform === 'win32' ? 'warn' : 'error', 41 | process.platform === 'win32' ? 'windows' : 'unix' 42 | ], 43 | quotes: [ 44 | 'error', 45 | 'single' 46 | ], 47 | semi: [ 48 | 'error', 49 | 'never' 50 | ] 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /arangodb-foxx-services/.gitignore: -------------------------------------------------------------------------------- 1 | /src/external 2 | -------------------------------------------------------------------------------- /arangodb-foxx-services/README.md: -------------------------------------------------------------------------------- 1 | # Spline Foxx Service 2 | 3 | --- 4 | 5 | ## Build 6 | 7 | ```bash 8 | mvn clean install 9 | ``` 10 | 11 | ## (re-)deploy 12 | 13 | ```shell 14 | # Deploy the "/spline" service to the "foo" database on the given ArangoDB server 15 | # (http://some.server:1234/_db/foo/spline/) 16 | npm run foxx:deploy --database=foo --server=http://some.server:1234 17 | 18 | # If the "--server" option is omitted, the local server is used 19 | # (http://localhost:8529/_db/foo/spline/) 20 | npm run foxx:deploy --database=foo 21 | 22 | # Use "foxx:deploy-dev" to deploy the service in development mode (https://www.arangodb.com/docs/stable/foxx-guides-development-mode.html) 23 | npm run foxx:deploy-dev --database=foo 24 | 25 | # If needed, in addition to the "foxx:deploy" script that executes build, uninstall and install steps, 26 | # you can also use individual "foxx:install" or "foxx:uninstall" scripts with the same arguments. 27 | npm run foxx:uninstall --database=NAME [--server=URL] 28 | npm run foxx:install --database=NAME [--server=URL] 29 | ``` 30 | 31 | --- 32 | 33 | Copyright 2019 ABSA Group Limited 34 | 35 | you may not use this file except in compliance with the License. 36 | You may obtain a copy of the License at 37 | 38 | http://www.apache.org/licenses/LICENSE-2.0 39 | 40 | Unless required by applicable law or agreed to in writing, software 41 | distributed under the License is distributed on an "AS IS" BASIS, 42 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 43 | See the License for the specific language governing permissions and 44 | limitations under the License. 45 | -------------------------------------------------------------------------------- /arangodb-foxx-services/__mocks__/@arangodb/locals.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | exports.context = { 18 | isDevelopment: true, 19 | isProduction: false, 20 | } 21 | -------------------------------------------------------------------------------- /arangodb-foxx-services/assembly.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 20 | dist 21 | 22 | zip 23 | 24 | false 25 | 26 | 27 | ${project.build.directory}/dist 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /arangodb-foxx-services/jest.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "ts-jest", 3 | "testEnvironment": "node", 4 | "verbose": true, 5 | "testMatch": [ 6 | "/tests/**/*.spec.ts" 7 | ], 8 | "setupFilesAfterEnv": [ 9 | "jest-extended/all" 10 | ], 11 | "globals": { 12 | "ts-jest": { 13 | "tsconfig": { 14 | "esModuleInterop": true 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /arangodb-foxx-services/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "spline_foxx_services", 3 | "version": "1.0.0-SNAPSHOT", 4 | "license": "Apache-2.0", 5 | "copyright": "2022 ABSA Group Limited", 6 | "private": true, 7 | "scripts": { 8 | "postinstall": "sync-pom-version", 9 | "build": "webpack", 10 | "test": "jest", 11 | "lint": "eslint src", 12 | "foxx:deploy": "npm run build && npm run foxx:uninstall && npm run foxx:install", 13 | "foxx:deploy-dev": "npm run foxx:deploy --extra_args='--dev'", 14 | "foxx:uninstall": "cross-env-shell foxx uninstall --server $npm_config_server --database $npm_config_database /spline", 15 | "foxx:install": "cross-env-shell foxx install --server $npm_config_server --database $npm_config_database $npm_config_extra_args /spline target/dist" 16 | }, 17 | "devDependencies": { 18 | "@types/arangodb": "^3.5.17", 19 | "@types/jest": "^28.1.5", 20 | "@typescript-eslint/eslint-plugin": "^5.44.0", 21 | "@typescript-eslint/parser": "^5.44.0", 22 | "copy-webpack-plugin": "^11.0.0", 23 | "cross-env": "^7.0.3", 24 | "dedent": "^0.7.0", 25 | "eslint": "^8.20.0", 26 | "foxx-cli": "^2.0.1", 27 | "glob": "^8.0.3", 28 | "jest": "^28.1.2", 29 | "jest-extended": "^3.2.2", 30 | "joi": "^17.6.0", 31 | "sync-pom-version-to-package": "^1.6.1", 32 | "ts-jest": "^28.0.7", 33 | "ts-loader": "^9.4.1", 34 | "typescript": "^4.8.4", 35 | "webpack": "^5.94.0", 36 | "webpack-cli": "^4.10.0" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/README.md: -------------------------------------------------------------------------------- 1 | ## Spline persistence services 2 | 3 | ### Purposes 4 | 5 | - Creates an API layer on top of the Spline database 6 | - Implements a custom application level distributed transaction protocol on top of the ArangoDB guarantees, to provide transaction _atomicity_ and 7 | READ_COMMITTED _isolation_ (A and I in ACID) on the cluster deployment 8 | 9 | ### Build information 10 | 11 | **Version**: @VERSION@\ 12 | **Revision**: @REVISION@\ 13 | **Timestamp**: @BUILD_TIMESTAMP@ 14 | 15 | --- 16 | 17 | Copyright 2022 ABSA Group Limited 18 | 19 | you may not use this file except in compliance with the License. 20 | You may obtain a copy of the License at 21 | 22 | http://www.apache.org/licenses/LICENSE-2.0 23 | 24 | Unless required by applicable law or agreed to in writing, software 25 | distributed under the License is distributed on an "AS IS" BASIS, 26 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | See the License for the specific language governing permissions and 28 | limitations under the License. 29 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/aql/README.txt: -------------------------------------------------------------------------------- 1 | Spline ArangoDB AQL functions are defined in JS files in the following pattern: 2 | 3 | Example: 4 | 5 | a function SPLINE::FOO_BAR_BAZ that takes two arguments `x` and `y` and returns `z` 6 | 7 | File name: 8 | 9 | foo_bar_baz.func.ts 10 | 11 | File content: 12 | 13 | export = function (x: number, y: number): number { 14 | return z 15 | } 16 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/assets/spline-logo.300x300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline/a7f43d2bc22b6dc8c5bd4e116d0ec45514cd1d46/arangodb-foxx-services/src/assets/spline-logo.300x300.png -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import router from './routes' 18 | import { context } from '@arangodb/locals' 19 | 20 | 21 | context.use(router) 22 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/model.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | export type DocumentKey = string 19 | export type DataSource = { 20 | _key: string 21 | name: string 22 | } 23 | 24 | export type ExecutionNode = { 25 | _key: string 26 | name: string 27 | 28 | } 29 | export type LineageGraphNode = ExecutionNode | DataSource 30 | 31 | export type LineageGraphEdge = { 32 | source: string 33 | target: string 34 | } 35 | 36 | export type LineageGraph = { 37 | depth: number 38 | vertices: Array 39 | edges: Array 40 | } 41 | 42 | /** 43 | * Mimics backend's [[za.co.absa.spline.consumer.service.model.LineageOverview]] 44 | */ 45 | export type LineageOverview = { 46 | info: { 47 | timestamp: number 48 | applicationId: string 49 | targetDataSourceId: string 50 | } 51 | graph: { 52 | depthRequested: number 53 | depthComputed: number 54 | nodes: Array 55 | edges: Array 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/routes/admin-router.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createRouter } from '@arangodb/foxx' 18 | import joi from 'joi' 19 | import { pruneBefore } from '../services/prune-database' 20 | 21 | 22 | const adminRouter = createRouter() 23 | 24 | adminRouter 25 | .delete('/data/before/:timestamp', 26 | (req: Foxx.Request, res: Foxx.Response) => { 27 | const timestamp = req.pathParams.timestamp 28 | pruneBefore(timestamp) 29 | res.send('DB Pruning done.') 30 | }) 31 | .pathParam('timestamp', joi.number().integer().min(0).required(), 'Data retention threshold [timestamp in millis]') 32 | .summary('Prune database') 33 | .description('Delete the data older than the given timestamp') 34 | 35 | export default adminRouter 36 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/routes/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createRouter } from '@arangodb/foxx' 18 | import { plansRouter } from './plans-router' 19 | import { eventsRouter } from './events-router' 20 | import adminRouter from './admin-router' 21 | 22 | 23 | const rootRouter: Foxx.Router = createRouter() 24 | rootRouter.use('/admin', adminRouter) 25 | rootRouter.use('/execution-plans', plansRouter) 26 | rootRouter.use('/execution-events', eventsRouter) 27 | 28 | export default rootRouter 29 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/routes/plans-router.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createRouter } from '@arangodb/foxx' 18 | import { ExecutionPlanPersistentModel } from '../../external/api.model' 19 | import { storeExecutionPlan } from '../services/execution-plan-store' 20 | 21 | 22 | export const plansRouter: Foxx.Router = createRouter() 23 | 24 | plansRouter 25 | .post('/', 26 | (req: Foxx.Request, res: Foxx.Response) => { 27 | const execPlanModel: ExecutionPlanPersistentModel = req.body 28 | storeExecutionPlan(execPlanModel) 29 | res.status('created') 30 | }) 31 | .body(['application/json'], 'Execution Plan Persistent Model JSON') 32 | .response(201, 'Plan registered') 33 | .summary('Register a new execution plan') 34 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/services/txm/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | import { TxManagerImpl } from './tx-manager-impl' 19 | import { SubscribableTxManagerDecorator } from './subcribable-tx-manager-decorator' 20 | 21 | 22 | export const TxManager = new SubscribableTxManagerDecorator(new TxManagerImpl()) 23 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/utils/logger.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { context } from '@arangodb/locals' 18 | 19 | 20 | const isDebugEnabled: boolean = context.isDevelopment 21 | 22 | export function error(...args: unknown[]) { 23 | console.error(...args) 24 | } 25 | 26 | export function info(...args: unknown[]) { 27 | console.log(...args) 28 | } 29 | 30 | export function debug(...args: unknown[]) { 31 | if (isDebugEnabled) { 32 | console.log(...args) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/main/utils/types.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* eslint-disable @typescript-eslint/no-explicit-any */ 18 | export type AnyFunction = (...args: any) => any 19 | /* eslint-enable @typescript-eslint/no-explicit-any */ 20 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/foxx-manifest", 3 | "engines": { 4 | "arangodb": "^3.8.0" 5 | }, 6 | "scripts": { 7 | "setup": "scripts/setup.js", 8 | "teardown": "scripts/teardown.js" 9 | }, 10 | "main": "index.js", 11 | "name": "spline-persistence-services", 12 | "version": "@VERSION@", 13 | "description": "Spline persistence REST API", 14 | "author": "ABSA Group Limited", 15 | "license": "Apache-2.0", 16 | "thumbnail": "assets/spline-logo.300x300.png" 17 | } 18 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/modules/consumer-index-update/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import './data-source.last-write.update' 18 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/scripts/setup.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import aqlFunctions from '@arangodb/aql/functions' 18 | import * as fs from 'fs' 19 | import { context } from '@arangodb/locals' 20 | 21 | 22 | const workDir = context.basePath 23 | 24 | // Register AQL functions 25 | 26 | const aqlFuncFileSuffix = '.func.js' 27 | const aqlFuncFileRegexp = RegExp(`(.*)${aqlFuncFileSuffix}`) 28 | 29 | const aqlSrcDir = fs.join(workDir, 'aql') 30 | const aqlFuncFiles = fs.list(aqlSrcDir).filter(fn => fn.endsWith(aqlFuncFileSuffix)) 31 | 32 | for (const aqlFuncFile of aqlFuncFiles) { 33 | const funcRawCode = fs.read(fs.join(aqlSrcDir, aqlFuncFile)) 34 | const funcLocalName = aqlFuncFileRegexp.exec(aqlFuncFile)[1].toUpperCase() 35 | const funcFQN = `SPLINE::${funcLocalName}` 36 | const funcWrappedCode = ` 37 | (function(){ 38 | console.log('Create AQL Function ${funcFQN}') 39 | const module = {} 40 | ;${funcRawCode}; 41 | return module.exports 42 | })() 43 | ` 44 | aqlFunctions.register(funcFQN, funcWrappedCode, true) 45 | } 46 | -------------------------------------------------------------------------------- /arangodb-foxx-services/src/scripts/teardown.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import aqlFunctions from '@arangodb/aql/functions' 18 | 19 | // Unregister AQL functions 20 | const n = aqlFunctions.unregisterGroup('SPLINE') 21 | console.log('Unregistered AQL functions:', n) 22 | -------------------------------------------------------------------------------- /arangodb-foxx-services/tests/aql/example.spec.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import myCoolFunction = require('../../src/aql/example.func') 18 | 19 | 20 | test('myCoolFunction', () => { 21 | expect(myCoolFunction(1, 2)).toBe(3) 22 | }) 23 | -------------------------------------------------------------------------------- /arangodb-foxx-services/tests/global.d.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import 'jest-extended' 18 | -------------------------------------------------------------------------------- /arangodb-foxx-services/tests/main/utils/common.spec.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { memoize } from '../../../src/main/utils/common' 18 | 19 | 20 | test('memoize', () => { 21 | let keyFnCallCnt = 0 22 | let valFnCallCnt = 0 23 | const keySpyFn = (x, y) => { 24 | keyFnCallCnt++ 25 | return x + y 26 | } 27 | const valSpyFn = (x, y) => { 28 | valFnCallCnt++ 29 | return x + y 30 | } 31 | 32 | const memoizedFn = memoize(keySpyFn, valSpyFn) 33 | 34 | expect(memoizedFn(1, 2)).toBe(3) 35 | expect(memoizedFn(2, 3)).toBe(5) 36 | expect(memoizedFn(3, 2)).toBe(5) // valFn should not be called as the key (5) is already in the cache 37 | expect(keyFnCallCnt).toBe(3) 38 | expect(valFnCallCnt).toBe(2) 39 | }) 40 | -------------------------------------------------------------------------------- /arangodb-foxx-services/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es6", 4 | "module": "commonjs", 5 | "esModuleInterop": true, 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /commons/src/main/resources/spline-build.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2017 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | build.version=${project.version} 18 | build.timestamp=${timestamp} 19 | build.revision=${buildNumber} 20 | -------------------------------------------------------------------------------- /commons/src/main/scala/com/arangodb/entity/ErrorEntityImplicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.arangodb.entity 18 | 19 | import com.arangodb.velocypack.{VPack, VPackBuilder, ValueType} 20 | 21 | object ErrorEntityImplicits { 22 | implicit class ErrorEntityOps(val ee: ErrorEntity) { 23 | def copy( 24 | errorMessage: String = ee.getErrorMessage, 25 | exception: String = ee.getException, 26 | code: Int = ee.getCode, 27 | errorNum: Int = ee.getErrorNum 28 | ): ErrorEntity = { 29 | val vpack = new VPack.Builder().build 30 | vpack.deserialize[ErrorEntity]( 31 | new VPackBuilder() 32 | .add(ValueType.OBJECT) 33 | .add("errorMessage", errorMessage) 34 | .add("exception", exception) 35 | .add("code", Int.box(code)) 36 | .add("errorNum", Int.box(errorNum)) 37 | .close() 38 | .slice(), 39 | classOf[ErrorEntity] 40 | ) 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /commons/src/main/scala/com/twitter/finatra/jackson/FinatraInternalModules.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.finatra.jackson 18 | 19 | import com.fasterxml.jackson.module.scala.JacksonModule 20 | import com.twitter.finatra.jackson.caseclass.CaseClassJacksonModule 21 | 22 | /** 23 | * This object serves the only purpose - to make access to some of the com.twitter.finatra Jackson modules, 24 | * that are declared private even though are perfectly reusable. 25 | */ 26 | object FinatraInternalModules { 27 | 28 | def caseClassModule: JacksonModule = new CaseClassJacksonModule( 29 | ScalaObjectMapper.DefaultInjectableTypes, 30 | Some(ScalaObjectMapper.DefaultValidator) 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/CollectionUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common 18 | 19 | object CollectionUtils { 20 | 21 | object Implicits { 22 | 23 | implicit class MapOps[K, V](val map: Map[K, V]) extends AnyVal { 24 | def mapKeys[L](f: K => L): Map[L, V] = map.map { 25 | case (a, b) => (f(a), b) 26 | } 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/SplineBuildInfo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common 18 | 19 | import za.co.absa.commons.buildinfo.{BuildInfo, PropMapping} 20 | 21 | /** 22 | * By using 'spline-build.properties' we should avoid name conflicts with libraries properties. 23 | */ 24 | object SplineBuildInfo extends BuildInfo("/spline-build", PropMapping.Default) { 25 | lazy val Revision: String = BuildProps.getProperty("build.revision") 26 | } 27 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/StringEscapeUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common 18 | 19 | object StringEscapeUtils { 20 | def escapeAQLSearch(str: String): String = str 21 | .replace("_", """\\_""") 22 | .replace("%", """\\%""") 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/config/DefaultConfigurationStack.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.config 18 | 19 | import org.apache.commons.configuration._ 20 | import za.co.absa.commons.config.UpperSnakeCaseEnvironmentConfiguration 21 | import za.co.absa.spline.common.config.DefaultConfigurationStack.jndiConfigurationIfAvailable 22 | 23 | import java.util 24 | import javax.naming.InitialContext 25 | import scala.util.Try 26 | 27 | abstract class DefaultConfigurationStack extends CompositeConfiguration(util.Arrays.asList( 28 | jndiConfigurationIfAvailable.toSeq ++ Seq( 29 | new SystemConfiguration, 30 | new UpperSnakeCaseEnvironmentConfiguration, 31 | new EnvironmentConfiguration 32 | ): _*)) 33 | 34 | object DefaultConfigurationStack { 35 | def jndiConfigurationIfAvailable: Option[JNDIConfiguration] = Try({ 36 | new InitialContext().getEnvironment 37 | new JNDIConfiguration("java:comp/env") 38 | }).toOption 39 | } 40 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/config/HttpConfiguration.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.config 18 | 19 | import org.apache.commons.configuration.Configuration 20 | import za.co.absa.commons.config.ConfTyped 21 | import za.co.absa.commons.config.ConfigurationImplicits._ 22 | import za.co.absa.spline.common.config.HttpConfiguration.{DefaultDefaultTimeout, DefaultMaximumTimeout} 23 | 24 | import scala.concurrent.duration._ 25 | 26 | trait HttpConfiguration { 27 | this: ConfTyped with Configuration => 28 | 29 | val DefaultTimeout: Duration = 30 | this.getOptionalLong(Prop("timeoutDefault")) 31 | .map(_.millis) 32 | .getOrElse(DefaultDefaultTimeout) 33 | 34 | val MaximumTimeout: Duration = 35 | this.getOptionalLong(Prop("timeoutMaximum")) 36 | .map(_.millis) 37 | .getOrElse(DefaultMaximumTimeout) 38 | 39 | } 40 | 41 | object HttpConfiguration { 42 | val DefaultDefaultTimeout: Duration = 2.minutes 43 | val DefaultMaximumTimeout: Duration = 1.hour 44 | } 45 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/rest/HttpStatusException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.rest 18 | 19 | import org.apache.http.HttpException 20 | 21 | class HttpStatusException(val status: Int, message: String) extends HttpException(message) 22 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/rest/RESTClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.rest 18 | 19 | import scala.concurrent.Future 20 | 21 | trait RESTClient { 22 | def get(path: String): Future[String] 23 | def delete(path: String): Future[Unit] 24 | def post(path: String, body: String): Future[Unit] 25 | def post(path: String, body: Array[Byte]): Future[Unit] 26 | } 27 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/security/TLSUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.security 18 | 19 | import java.security.cert.X509Certificate 20 | import javax.net.ssl.{SSLContext, X509TrustManager} 21 | 22 | object TLSUtils { 23 | 24 | lazy val TrustingAllSSLContext: SSLContext = { 25 | val sslContext = SSLContext.getInstance("TLS") 26 | sslContext.init(null, Array(TrustAll), null) 27 | sslContext 28 | } 29 | 30 | // Bypasses both client and server validation. 31 | private object TrustAll extends X509TrustManager { 32 | override val getAcceptedIssuers: Array[X509Certificate] = Array.empty 33 | 34 | override def checkClientTrusted(x509Certificates: Array[X509Certificate], s: String): Unit = { 35 | // do nothing 36 | } 37 | 38 | override def checkServerTrusted(x509Certificates: Array[X509Certificate], s: String): Unit = { 39 | // do nothing 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/validation/Constraint.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.validation 18 | 19 | trait Constraint { 20 | def isValid: Boolean 21 | def message: String 22 | } 23 | 24 | object Constraint { 25 | def unique[A](xs: Seq[A]): UniqueConstraint[A, A] = UniqueConstraint(xs, identity, "item") 26 | } 27 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/validation/ValidationUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.validation 18 | 19 | import jakarta.validation.ConstraintViolationException 20 | 21 | object ValidationUtils { 22 | 23 | @throws[ConstraintViolationException] 24 | def validate(c: Constraint): Unit = { 25 | validate(c.isValid, c.message) 26 | } 27 | 28 | @throws[ConstraintViolationException] 29 | def validate(b: Boolean, msg: => String): Unit = { 30 | if (!b) throw new ConstraintViolationException(msg, null) // NOSONAR 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/controller/DefaultRestController.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.common.webmvc.controller 17 | 18 | import io.swagger.annotations.Api 19 | import org.springframework.http.HttpStatus 20 | import org.springframework.web.bind.annotation._ 21 | 22 | @RestController 23 | @Api(hidden = true) 24 | class DefaultRestController { 25 | 26 | @GetMapping(Array("/**")) 27 | @ResponseStatus(HttpStatus.NOT_FOUND) 28 | def resourceNotFound(): Unit = () 29 | 30 | } 31 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/controller/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc 18 | 19 | package object controller { 20 | 21 | trait _package 22 | 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/cors/ConstantCorsConfigurationSource.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.cors 18 | 19 | import javax.servlet.http.HttpServletRequest 20 | import org.springframework.web.cors.{CorsConfiguration, CorsConfigurationSource} 21 | 22 | class ConstantCorsConfigurationSource(conf: CorsConfiguration) extends CorsConfigurationSource { 23 | override def getCorsConfiguration(request: HttpServletRequest): CorsConfiguration = conf 24 | } 25 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/cors/PermissiveCorsFilter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.cors 18 | 19 | import org.springframework.web.cors.CorsConfiguration 20 | import org.springframework.web.filter.CorsFilter 21 | import za.co.absa.spline.common.webmvc.cors.PermissiveCorsFilter.PermissiveCorsConfiguration 22 | 23 | class PermissiveCorsFilter extends CorsFilter( 24 | new ConstantCorsConfigurationSource(PermissiveCorsConfiguration)) 25 | 26 | object PermissiveCorsFilter { 27 | private val PermissiveCorsConfiguration = new CorsConfiguration().applyPermitDefaultValues() 28 | } 29 | 30 | 31 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/diagnostics/DiagnosticsRESTConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.diagnostics 18 | 19 | import org.springframework.context.annotation.{ComponentScan, Configuration} 20 | import org.springframework.web.servlet.config.annotation.{EnableWebMvc, WebMvcConfigurer} 21 | import za.co.absa.spline.common 22 | 23 | @EnableWebMvc 24 | @Configuration 25 | @ComponentScan(basePackageClasses = Array( 26 | classOf[common.webmvc.controller._package], 27 | classOf[controller._package] 28 | )) 29 | class DiagnosticsRESTConfig extends WebMvcConfigurer 30 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/diagnostics/RootWebContextConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.diagnostics 18 | 19 | import org.springframework.context.annotation.Configuration 20 | import org.springframework.core.Ordered 21 | import org.springframework.web.servlet.config.annotation.{DefaultServletHandlerConfigurer, EnableWebMvc, ViewControllerRegistry, WebMvcConfigurer} 22 | 23 | @EnableWebMvc 24 | @Configuration 25 | class RootWebContextConfig extends WebMvcConfigurer { 26 | 27 | override def configureDefaultServletHandling(configurer: DefaultServletHandlerConfigurer): Unit = { 28 | configurer.enable() 29 | } 30 | 31 | override def addViewControllers(registry: ViewControllerRegistry): Unit = { 32 | registry.addViewController("/").setViewName("redirect:/index.html") 33 | registry.setOrder(Ordered.HIGHEST_PRECEDENCE) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/diagnostics/controller/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.diagnostics 18 | 19 | package object controller { 20 | 21 | trait _package 22 | 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/jackson/NullAcceptingDeserializer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.jackson 18 | 19 | import com.fasterxml.jackson.core.JsonParser 20 | import com.fasterxml.jackson.databind.DeserializationContext 21 | import com.fasterxml.jackson.databind.deser.std.StdDeserializer 22 | 23 | class NullAcceptingDeserializer extends StdDeserializer[Any](classOf[Any]) { 24 | 25 | override def deserialize(jp: JsonParser, ctxt: DeserializationContext): Any = { 26 | ctxt.readValue(jp, classOf[Any]) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/jackson/ObjectMapperBeanPostProcessor.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.webmvc.jackson 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper 20 | import org.springframework.beans.factory.config.BeanPostProcessor 21 | import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter 22 | import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter 23 | 24 | import scala.jdk.CollectionConverters._ 25 | 26 | class ObjectMapperBeanPostProcessor(postProcess: ObjectMapper => Unit) extends BeanPostProcessor { 27 | 28 | override def postProcessBeforeInitialization(bean: AnyRef, beanName: String): AnyRef = { 29 | bean match { 30 | case adapter: RequestMappingHandlerAdapter => 31 | adapter 32 | .getMessageConverters 33 | .asScala 34 | .collect({ case hmc: MappingJackson2HttpMessageConverter => hmc.getObjectMapper }) 35 | .foreach(postProcess(_)) 36 | case _ => 37 | } 38 | bean 39 | } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /commons/src/main/scala/za/co/absa/spline/common/webmvc/swagger/SwaggerUISupport.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package za.co.absa.spline.common.webmvc.swagger 17 | 18 | import org.springframework.web.servlet.config.annotation.{ResourceHandlerRegistry, WebMvcConfigurer} 19 | 20 | trait SwaggerUISupport { 21 | this: WebMvcConfigurer => 22 | 23 | override def addResourceHandlers(registry: ResourceHandlerRegistry): Unit = { 24 | registry. 25 | addResourceHandler("swagger-ui.html"). 26 | addResourceLocations("classpath:/META-INF/resources/") 27 | registry. 28 | addResourceHandler("/webjars/**"). 29 | addResourceLocations("classpath:/META-INF/resources/webjars/") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /commons/src/test/scala/za/co/absa/spline/common/ConsoleUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common 18 | 19 | import org.backuity.ansi.AnsiFormatter.FormattedHelper 20 | import org.scalatest.flatspec.AnyFlatSpec 21 | import org.scalatest.matchers.should.Matchers 22 | 23 | class ConsoleUtilsSpec extends AnyFlatSpec with Matchers { 24 | 25 | behavior of "ConsoleUtils" 26 | 27 | behavior of "stripAnsiEscapeSequences()" 28 | 29 | import za.co.absa.spline.common.ConsoleUtils.stripAnsiEscapeSequences 30 | 31 | it should "de-ANSI-ify the given string" in { 32 | stripAnsiEscapeSequences(ansi"%green{green %bold{bold}} plain %red{red}") should equal("green bold plain red") 33 | } 34 | 35 | it should "remove any ANSI ESC sequences" in { 36 | stripAnsiEscapeSequences("\u001b[s\u001b[0;38;5;232;48;5;255mHello\u001b[u \u001b[2;37;41mWorld") should equal("Hello World") 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /commons/src/test/scala/za/co/absa/spline/common/StringEscapeUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common 18 | 19 | import org.scalatest.flatspec.AnyFlatSpec 20 | import org.scalatest.matchers.should.Matchers 21 | 22 | class StringEscapeUtilsSpec extends AnyFlatSpec with Matchers { 23 | 24 | behavior of "escapeAQLSearch" 25 | 26 | it should "escape _ and % characters" in { 27 | StringEscapeUtils.escapeAQLSearch("foo%bar_baz -i- foo%bar_baz") should equal("""foo\\%bar\\_baz -i- foo\\%bar\\_baz""") 28 | StringEscapeUtils.escapeAQLSearch("foo%%bar__baz -i- foo%%bar__baz") should equal("""foo\\%\\%bar\\_\\_baz -i- foo\\%\\%bar\\_\\_baz""") 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /commons/src/test/scala/za/co/absa/spline/common/graph/GraphImplicits_SortedTopologicallyBySpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.common.graph 18 | 19 | import za.co.absa.spline.common.graph.GraphImplicits._ 20 | 21 | class GraphImplicits_SortedTopologicallyBySpec 22 | extends AbstractGraphImplicits_SortedTopologicallySpec( 23 | "sortedTopologicallyBy", 24 | // `toSeq` is required for Scala 2.13 25 | // noinspection RedundantCollectionConversion 26 | _.toSeq.sortedTopologicallyBy(_._1, _._2) 27 | ) 28 | -------------------------------------------------------------------------------- /consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/ConsumerTypeResolver.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.rest 17 | 18 | import com.fasterxml.jackson.annotation.JsonTypeInfo.{As, Id} 19 | import com.fasterxml.jackson.databind.ObjectMapper.DefaultTypeResolverBuilder 20 | import com.fasterxml.jackson.databind.jsontype.impl.LaissezFaireSubTypeValidator 21 | import com.fasterxml.jackson.databind.{JavaType, ObjectMapper} 22 | import za.co.absa.spline.consumer.service.model.{DataType, LineageOverviewNode} 23 | 24 | class ConsumerTypeResolver extends DefaultTypeResolverBuilder( 25 | ObjectMapper.DefaultTyping.OBJECT_AND_NON_CONCRETE, 26 | LaissezFaireSubTypeValidator.instance) { 27 | 28 | init(Id.NAME, null) 29 | inclusion(As.PROPERTY) 30 | typeProperty("_type") 31 | 32 | override def useForType(t: JavaType): Boolean = { 33 | t.isTypeOrSubTypeOf(classOf[LineageOverviewNode]) || t.isTypeOrSubTypeOf(classOf[DataType]) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/controller/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.rest 18 | 19 | package object controller { 20 | 21 | trait _package 22 | 23 | } 24 | -------------------------------------------------------------------------------- /consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/model/LabelQuery.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.rest.model 18 | 19 | import za.co.absa.spline.consumer.service.model.Label 20 | 21 | object LabelQuery { 22 | private val KeyValuesRegexp = "([^:]+):(.*)".r 23 | 24 | def parse(str: String): Option[Label] = { 25 | KeyValuesRegexp 26 | .findFirstMatchIn(str) 27 | .map(m => 28 | Label( 29 | name = m.group(1), 30 | values = m.group(2).split(',').map(_.trim).filterNot(_.isEmpty) 31 | ) 32 | ) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /consumer-rest-core/src/main/scala/za/co/absa/spline/consumer/rest/model/WriteModeQuery.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.rest.model 18 | 19 | object WriteModeQuery { 20 | def parse(str: String): Array[Option[Boolean]] = { 21 | str.split(",").map(_.trim).map(s => { 22 | if (s.isEmpty) None 23 | else Some(s.toBoolean) 24 | }) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/ConsumerServicesConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service 18 | 19 | import org.springframework.context.annotation.{ComponentScan, Configuration} 20 | 21 | @Configuration 22 | @ComponentScan(basePackageClasses = Array(classOf[repo._package])) 23 | class ConsumerServicesConfig -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Attribute.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | case class Attribute( 20 | id: String, 21 | name: String, 22 | dataTypeId: String) 23 | 24 | object Attribute { 25 | type Id = String 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeEdge.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Link between nodes") 22 | case class AttributeEdge 23 | ( 24 | @ApiModelProperty(value = "Source Node") 25 | source: AttributeNode.Id, 26 | 27 | @ApiModelProperty(value = "Target Node") 28 | target: AttributeNode.Id 29 | ) extends Graph.Edge { 30 | override type JointId = AttributeNode.Id 31 | } 32 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeGraph.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Attribute Dependency Graph") 22 | case class AttributeGraph( 23 | @ApiModelProperty(value = "Array of attribute nodes representing an operation where attribute was created") 24 | nodes: Array[AttributeNode], 25 | @ApiModelProperty(value = "Link representing dependency of one attribute on another") 26 | edges: Array[AttributeEdge] 27 | ) extends Graph { 28 | override type Node = AttributeNode 29 | override type Edge = AttributeEdge 30 | } 31 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/AttributeNode.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Attribute Node") 22 | case class AttributeNode 23 | ( 24 | @ApiModelProperty(value = "Attribute Id") 25 | _id: AttributeNode.Id, 26 | 27 | @ApiModelProperty(value = "Attribute name") 28 | name: String, 29 | 30 | @ApiModelProperty(value = "Operation Id in which the attribute was created") 31 | originOpId: Operation.Id, 32 | 33 | @ApiModelProperty(value = "Operation Ids which the attribute passes through") 34 | transOpIds: Seq[Operation.Id] 35 | 36 | ) extends Graph.Node { 37 | override type Id = AttributeNode.Id 38 | } 39 | 40 | object AttributeNode { 41 | type Id = String 42 | } 43 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceActionType.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import za.co.absa.commons.reflect.EnumerationMacros 20 | 21 | sealed abstract class DataSourceActionType(val name: String) 22 | 23 | object DataSourceActionType { 24 | 25 | case object Read extends DataSourceActionType("read") 26 | 27 | case object Write extends DataSourceActionType("write") 28 | 29 | 30 | val values: Set[DataSourceActionType] = EnumerationMacros.sealedInstancesOf[DataSourceActionType] 31 | 32 | def findValueOf(name: String): Option[DataSourceActionType] = 33 | DataSourceActionType.values.find(_.name.equalsIgnoreCase(name)) 34 | } 35 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/DataSourceInfo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.model 17 | 18 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 19 | 20 | @ApiModel(description = "Information about the DataSource") 21 | case class DataSourceInfo 22 | ( 23 | @ApiModelProperty(value = "Type of the dataSource", example = "csv / parquet") 24 | sourceType: String, 25 | @ApiModelProperty(value = "Uri of the dataSource") 26 | source: String 27 | ) 28 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionGraph.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Expression Graph") 22 | case class ExpressionGraph( 23 | @ApiModelProperty(value = "Array of Expression nodes") 24 | nodes: Array[ExpressionNode], 25 | @ApiModelProperty(value = "`uses` or `takes` edge") 26 | edges: Array[ExpressionEdge] 27 | ) extends Graph { 28 | override type Node = ExpressionNode 29 | override type Edge = ExpressionEdge 30 | } 31 | 32 | 33 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/ExpressionNode.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Expression Node") 22 | case class ExpressionNode 23 | ( 24 | @ApiModelProperty(value = "Expression Id") 25 | _id: ExpressionNode.Id, 26 | 27 | @ApiModelProperty(value = "Expression data type") 28 | dataType: Option[String], 29 | 30 | @ApiModelProperty(value = "Expression name") 31 | name: Option[String], 32 | 33 | @ApiModelProperty(value = "Literal expression value") 34 | value: Option[Any], 35 | 36 | @ApiModelProperty(value = "Expression parameters") 37 | params: Map[String, Any] = Map(), 38 | 39 | @ApiModelProperty(value = "Expression extras") 40 | extra: Map[String, Any] = Map(), 41 | 42 | ) extends Graph.Node { 43 | override type Id = ExpressionNode.Id 44 | } 45 | 46 | object ExpressionNode { 47 | type Id = String 48 | } 49 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Graph.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | trait Graph { 20 | type Node <: Graph.Node 21 | type Edge <: Graph.Edge {type JointId = Node#Id} 22 | val nodes: Array[Node] 23 | val edges: Array[Edge] 24 | } 25 | 26 | object Graph { 27 | 28 | trait Node { 29 | type Id 30 | val _id: Id 31 | } 32 | 33 | trait Edge { 34 | type JointId 35 | val source: JointId 36 | val target: JointId 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Label.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | case class Label(name: Label.Name, values: Array[Label.Value]) 20 | 21 | object Label { 22 | type Name = String 23 | type Value = String 24 | } 25 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailed.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | 22 | @ApiModel(description = "Execution Plan and Lineage Graph") 23 | case class LineageDetailed( 24 | @ApiModelProperty(value = "Information related to the execution plan") 25 | executionPlan: ExecutionPlanInfo, 26 | @ApiModelProperty(value = "Execution plan level lineage") 27 | graph: LineageDetailedGraph 28 | ) 29 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageDetailedGraph.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Lineage Graph") 22 | case class LineageDetailedGraph 23 | ( 24 | @ApiModelProperty(value = "List of graph nodes representing the different operations of the lineage") 25 | nodes: Array[Operation], 26 | @ApiModelProperty(value = "List of graph edges showing the triggered operations order") 27 | edges: Array[Transition] 28 | 29 | ) extends Graph { 30 | override type Node = Operation 31 | override type Edge = Transition 32 | } 33 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverview.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.model 17 | 18 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 19 | 20 | @ApiModel(description = "Lineage/Impact Overview") 21 | case class LineageOverview( 22 | @ApiModelProperty(value = "Lineage/Impact graph") 23 | graph: LineageOverviewGraph, 24 | @ApiModelProperty(value = "Additional information") 25 | info: Map[String, Any] 26 | ) 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/LineageOverviewGraph.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Lineage") 22 | case class LineageOverviewGraph 23 | ( 24 | @ApiModelProperty(value = "Array of Lineage Overview nodes representing an Execution or a DataSource") 25 | nodes: Array[LineageOverviewNode], 26 | @ApiModelProperty(value = "Link between the Executions and the DataSources") 27 | edges: Array[Transition], 28 | @ApiModelProperty(value = "Requested max depth") 29 | depthRequested: Int, 30 | @ApiModelProperty(value = "Computed depth") 31 | depthComputed: Int 32 | 33 | ) extends Graph { 34 | override type Node = LineageOverviewNode 35 | override type Edge = Transition 36 | } 37 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Operation.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Operation") 22 | case class Operation 23 | ( 24 | @ApiModelProperty(value = "Operation Id") 25 | _id: Operation.Id, 26 | @ApiModelProperty(value = "Type of the operation", example = "Read / Transformation / Write") 27 | _type: Operation.Type, 28 | @ApiModelProperty(value = "Name of the operation") 29 | name: Option[String], 30 | @ApiModelProperty(value = "Properties of the operation") 31 | properties: Map[String, Any] 32 | ) extends Graph.Node { 33 | override type Id = Operation.Id 34 | } 35 | 36 | object Operation { 37 | type Id = String 38 | type Type = String // Read / Transformation / Write 39 | } 40 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/OperationDetails.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | 20 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 21 | 22 | @ApiModel(description = "Information Details of an Operation containing the input and output schemas with the description of the dataTypes") 23 | case class OperationDetails 24 | ( 25 | operation: Operation, 26 | @ApiModelProperty(value = "Array of the used DataTypes in the schemas") 27 | dataTypes: Array[DataType], 28 | @ApiModelProperty(value = "Array of all the schemas") 29 | schemas: Array[Array[Attribute]], 30 | @ApiModelProperty(value = "Array of indexes of the schemas Array. The schemas at these indexes represent the input schemas") 31 | inputs: Array[Int], 32 | @ApiModelProperty(value = "Index of the schemas Array. The schema at this index represents the output schemas") 33 | output: Int 34 | ) 35 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageRequest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.model 17 | 18 | case class PageRequest(page: Int, size: Int) 19 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Pageable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | trait Pageable[T] { 20 | val items: Array[T] 21 | val totalCount: Long 22 | val pageNum: Int 23 | val pageSize: Int 24 | } -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageableExecutionEventsResponse.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.ApiModelProperty 20 | 21 | case class PageableExecutionEventsResponse( 22 | @ApiModelProperty(value = "Array of Execution events") 23 | override val items: Array[ExecutionEventInfo], 24 | @ApiModelProperty(value = "Total number of executionEvents in the result set") 25 | override val totalCount: Long, 26 | @ApiModelProperty(value = "Page number") 27 | override val pageNum: Int, 28 | @ApiModelProperty(value = "Page size") 29 | override val pageSize: Int, 30 | @ApiModelProperty(value = "Total date range (min and max timestamp) of the result set") 31 | totalDateRange: Array[Long] 32 | ) extends Pageable[ExecutionEventInfo] 33 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/PageableExecutionPlansResponse.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.ApiModelProperty 20 | 21 | case class PageableExecutionPlansResponse( 22 | @ApiModelProperty(value = "Array of Execution plans") 23 | override val items: Array[ExecutionPlanInfo], 24 | @ApiModelProperty(value = "Total number of executionPlans in the result set") 25 | override val totalCount: Long, 26 | @ApiModelProperty(value = "Page number") 27 | override val pageNum: Int, 28 | @ApiModelProperty(value = "Page size") 29 | override val pageSize: Int 30 | ) extends Pageable[ExecutionPlanInfo] 31 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Schema.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | case class Schema( 20 | attributes: Array[Attribute] 21 | ) 22 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/SortRequest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.model 17 | 18 | case class SortRequest(sortField: String, sortOrder: String) 19 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/model/Transition.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.model 18 | 19 | import io.swagger.annotations.{ApiModel, ApiModelProperty} 20 | 21 | @ApiModel(description = "Link between operations") 22 | case class Transition 23 | ( 24 | @ApiModelProperty(value = "Source Operation") 25 | source: Operation.Id, 26 | @ApiModelProperty(value = "Target Operation") 27 | target: Operation.Id 28 | ) extends Graph.Edge { 29 | override type JointId = Operation.Id 30 | } 31 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/DataSourceRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.repo 17 | 18 | import za.co.absa.spline.consumer.service.model._ 19 | 20 | import scala.concurrent.{ExecutionContext, Future} 21 | 22 | trait DataSourceRepository extends AbstractExecutionEventRepository { 23 | 24 | def findByUsage( 25 | execPlanId: ExecutionPlanInfo.Id, 26 | access: Option[DataSourceActionType]) 27 | (implicit ec: ExecutionContext): Future[Array[String]] 28 | 29 | } 30 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionEventRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.repo 17 | 18 | trait ExecutionEventRepository extends AbstractExecutionEventRepository 19 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExecutionPlanRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.repo 18 | 19 | import za.co.absa.spline.consumer.service.model._ 20 | 21 | import scala.concurrent.{ExecutionContext, Future} 22 | 23 | trait ExecutionPlanRepository { 24 | 25 | def execPlanAttributeLineage(attrId: Attribute.Id)(implicit ec: ExecutionContext): Future[AttributeGraph] 26 | 27 | def execPlanAttributeImpact(attrId: Attribute.Id)(implicit ec: ExecutionContext): Future[AttributeGraph] 28 | 29 | def findById(execId: ExecutionPlanInfo.Id)(implicit ec: ExecutionContext): Future[LineageDetailed] 30 | 31 | def find( 32 | asAtTime: Long, 33 | pageRequest: PageRequest, 34 | sortRequest: SortRequest 35 | )(implicit ec: ExecutionContext): Future[(Seq[ExecutionPlanInfo], Long)] 36 | 37 | def getWriteOperationId(planId: ExecutionPlanInfo.Id)(implicit ec: ExecutionContext): Future[Operation.Id] 38 | } 39 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ExpressionRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.repo 18 | 19 | import za.co.absa.spline.consumer.service.model.{ExpressionGraph, Operation} 20 | 21 | import scala.concurrent.{ExecutionContext, Future} 22 | 23 | trait ExpressionRepository { 24 | 25 | def expressionGraphUsedByOperation(operationId: Operation.Id)(implicit ec: ExecutionContext): Future[ExpressionGraph] 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/ImpactRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.repo 17 | 18 | import za.co.absa.spline.consumer.service.model.{LineageOverview, ExecutionEventInfo} 19 | 20 | import scala.concurrent.{ExecutionContext, Future} 21 | 22 | trait ImpactRepository { 23 | 24 | def impactOverviewForExecutionEvent(eventId: ExecutionEventInfo.Id, maxDepth: Int) 25 | (implicit ec: ExecutionContext): Future[LineageOverview] 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/LabelRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.repo 17 | 18 | import za.co.absa.spline.consumer.service.model._ 19 | 20 | import scala.concurrent.{ExecutionContext, Future} 21 | 22 | trait LabelRepository { 23 | 24 | def findNames(query: Option[String], offset: Int, length: Int)(implicit ec: ExecutionContext): Future[Seq[Label.Name]] 25 | def findValuesFor(labelName: Label.Name, query: Option[String], offset: Int, length: Int)(implicit ec: ExecutionContext): Future[Seq[Label.Value]] 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/LineageRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package za.co.absa.spline.consumer.service.repo 17 | 18 | import za.co.absa.spline.consumer.service.model.{ExecutionEventInfo, LineageOverview} 19 | 20 | import scala.concurrent.{ExecutionContext, Future} 21 | 22 | trait LineageRepository { 23 | 24 | def lineageOverviewForExecutionEvent(eventId: ExecutionEventInfo.Id, maxDepth: Int) 25 | (implicit ec: ExecutionContext): Future[LineageOverview] 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/OperationRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service.repo 18 | 19 | import za.co.absa.spline.consumer.service.model.Operation.Id 20 | import za.co.absa.spline.consumer.service.model.OperationDetails 21 | 22 | import scala.concurrent.{ExecutionContext, Future} 23 | 24 | trait OperationRepository { 25 | def findById(operationId: Id)(implicit ec: ExecutionContext): Future[OperationDetails] 26 | } 27 | -------------------------------------------------------------------------------- /consumer-services/src/main/scala/za/co/absa/spline/consumer/service/repo/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.consumer.service 18 | 19 | 20 | package object repo { 21 | 22 | trait _package 23 | 24 | } 25 | -------------------------------------------------------------------------------- /integration-tests/src/main/scala/za/co/absa/spline/test/fixture/ArangoDbFixture.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.test.fixture 18 | 19 | import com.arangodb.async.ArangoDatabaseAsync 20 | import za.co.absa.spline.persistence.ArangoConnectionURL 21 | 22 | trait ArangoDbFixture extends ArangoDbFixtureLike { 23 | this: TestContainersFixture => 24 | 25 | def withArangoDb[A](testBody: (ArangoDatabaseAsync, ArangoConnectionURL) => A): A = { 26 | withTestContainer(ArangoDbContainer) { 27 | runTestBody(testBody) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /integration-tests/src/main/scala/za/co/absa/spline/test/fixture/ArangoDbFixtureAsync.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.test.fixture 18 | 19 | import com.arangodb.async.ArangoDatabaseAsync 20 | import za.co.absa.spline.persistence.ArangoConnectionURL 21 | 22 | import scala.concurrent.Future 23 | 24 | trait ArangoDbFixtureAsync extends ArangoDbFixtureLike { 25 | this: TestContainersFixtureAsync => 26 | 27 | def withArangoDb[A](testBody: (ArangoDatabaseAsync, ArangoConnectionURL) => Future[A]): Future[A] = { 28 | withTestContainer(ArangoDbContainer) { 29 | runTestBody(testBody) 30 | } 31 | } 32 | } 33 | 34 | 35 | -------------------------------------------------------------------------------- /integration-tests/src/main/scala/za/co/absa/spline/test/fixture/TestContainersFixture.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.test.fixture 18 | 19 | import org.testcontainers.containers.GenericContainer 20 | import za.co.absa.commons.lang.ARM._ 21 | 22 | trait TestContainersFixture { 23 | 24 | def withTestContainer[A <: GenericContainer[A], B](container: A)(testBody: A => B): B = { 25 | using(container) { _ => 26 | container.start() 27 | testBody(container) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /integration-tests/src/main/scala/za/co/absa/spline/test/fixture/TestContainersFixtureAsync.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.test.fixture 18 | 19 | import org.scalatest.AsyncTestSuite 20 | import org.testcontainers.containers.GenericContainer 21 | 22 | import scala.concurrent.Future 23 | 24 | trait TestContainersFixtureAsync { 25 | this: AsyncTestSuite => 26 | 27 | def withTestContainer[A <: GenericContainer[A], B](container: A)(testBody: A => Future[B]): Future[B] = { 28 | container.start() 29 | testBody(container) andThen { 30 | case _ => container.stop() 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /kafka-gateway/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | FROM tomcat:9-jre11-openjdk-slim-buster 18 | 19 | LABEL \ 20 | vendor="ABSA" \ 21 | copyright="2021 ABSA Group Limited" \ 22 | license="Apache License, version 2.0" \ 23 | name="Spline Kafka Gateway Server" 24 | 25 | EXPOSE 8080 26 | EXPOSE 8009 27 | 28 | RUN rm -rf /usr/local/tomcat/webapps/* 29 | 30 | USER 1001 31 | ARG PROJECT_BUILD_FINAL_NAME 32 | COPY --chown=1001 target/$PROJECT_BUILD_FINAL_NAME/ /usr/local/tomcat/webapps/ROOT/ 33 | -------------------------------------------------------------------------------- /kafka-gateway/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] %-5level [%thread] %-30logger{30} - %msg%n 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /kafka-gateway/src/main/scala/za/co/absa/spline/gateway/kafka/listener/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.gateway.kafka 18 | 19 | package object listener { 20 | trait _package 21 | } 22 | -------------------------------------------------------------------------------- /kafka-gateway/src/main/webapp/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline/a7f43d2bc22b6dc8c5bd4e116d0ec45514cd1d46/kafka-gateway/src/main/webapp/favicon.ico -------------------------------------------------------------------------------- /persistence/src/main/resources/migration-scripts/0.5.4-0.5.5.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const VER = "0.5.5" 18 | 19 | const udfs = require("@arangodb/aql/functions"); 20 | 21 | console.log(`[Spline] Start migration to ${VER}`); 22 | 23 | console.log("[Spline] Remove SPLINE AQL UDFs. See https://github.com/AbsaOSS/spline/issues/761"); 24 | 25 | udfs.unregister("SPLINE::OBSERVED_WRITES_BY_READ"); 26 | udfs.unregister("SPLINE::EVENT_LINEAGE_OVERVIEW"); 27 | 28 | console.log(`[Spline] Migration done. Version ${VER}`); 29 | -------------------------------------------------------------------------------- /persistence/src/main/resources/migration-scripts/0.6.0-0.6.1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const VER = "0.6.1" 18 | 19 | const {db} = require("@arangodb"); 20 | 21 | console.log(`[Spline] Start migration to ${VER}`); 22 | 23 | console.log("[Spline] Drop index 'progress.execPlanDetails.executionPlanId'"); 24 | const idxToDrop = db.progress.getIndexes().find(idx => idx.fields.every(fld => fld === "execPlanDetails.executionPlanId")); 25 | db.progress.dropIndex(idxToDrop); 26 | 27 | console.log("[Spline] Create index 'progress.execPlanDetails.executionPlanKey'"); 28 | db.progress.ensureIndex({type: "persistent", fields: ["execPlanDetails.executionPlanKey"]}); 29 | 30 | console.log(`[Spline] Migration done. Version ${VER}`); 31 | -------------------------------------------------------------------------------- /persistence/src/main/resources/migration-scripts/template.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const VER = "0.0.0" 18 | 19 | const {db, aql} = require("@arangodb"); 20 | 21 | console.log(`[Spline] Start migration to ${VER}`); 22 | 23 | // Migration script BEGIN 24 | 25 | console.log("[Spline] ..."); 26 | db._query(aql` 27 | WITH ... 28 | ... 29 | `); 30 | 31 | // Migration script END 32 | 33 | console.log(`[Spline] Migration done. Version ${VER}`); 34 | -------------------------------------------------------------------------------- /persistence/src/main/scala/com/arangodb/internal/velocystream/VstImplicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.arangodb.internal.velocystream 18 | 19 | import com.arangodb.internal.net.AccessType 20 | import com.arangodb.internal.velocystream.internal.VstConnection 21 | 22 | object VstImplicits { 23 | 24 | implicit class InternalVstCommunicationOps(val vstComm: VstCommunication[_, VstConnection[_]]) extends AnyVal { 25 | 26 | def getUser: String = vstComm.user 27 | 28 | def getPassword: String = vstComm.password 29 | 30 | def connect(accessType: AccessType): VstConnection[_] = vstComm.connect(null, accessType) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /persistence/src/main/scala/com/arangodb/model/Implicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.arangodb.model 18 | 19 | import com.arangodb.entity.IndexType 20 | import org.apache.commons.lang3.StringUtils 21 | 22 | object Implicits { 23 | 24 | implicit class IndexOptionsOps(val opts: IndexOptions[_]) extends AnyVal { 25 | def indexType: IndexType = { 26 | val typeName = StringUtils.substringBefore( 27 | opts.getClass.getSimpleName, 28 | classOf[IndexOptions[_]].getSimpleName 29 | ) 30 | IndexType.valueOf(typeName.toLowerCase) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/DatabaseException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence 18 | 19 | class DatabaseException(query: String, cause: Throwable) 20 | extends RuntimeException(s"Query execution failed: $query", cause) 21 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/DefaultJsonSerDe.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence 18 | 19 | import za.co.absa.commons.json.DefaultJacksonJsonSerDe 20 | 21 | object DefaultJsonSerDe extends DefaultJacksonJsonSerDe 22 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/DryRunnable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence 18 | 19 | import scala.concurrent.Future 20 | 21 | trait DryRunnable { 22 | def dryRun: Boolean 23 | 24 | def unlessDryRun[T <: Any](code: => T, default: => T): T = { 25 | if (dryRun) default 26 | else code 27 | } 28 | 29 | def unlessDryRun[T <: AnyRef](code: => T): T = { 30 | unlessDryRun(code, null).asInstanceOf[T] 31 | } 32 | 33 | def unlessDryRunAsync[T](code: => Future[T]): Future[T] = { 34 | unlessDryRun(code, Future.successful(null)).asInstanceOf[Future[T]] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/LogMessageUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence 18 | 19 | import org.apache.commons.lang3.StringUtils 20 | 21 | object LogMessageUtils { 22 | private val separatorLine = "=" * 80 23 | 24 | def createQueryLogMessage(queryString: String): String = { 25 | if (ArangoRepoConfig.Database.LogFullQueryOnError) { 26 | s""" 27 | |$separatorLine 28 | |${queryString.trim} 29 | |$separatorLine""" 30 | .stripMargin 31 | } else { 32 | val fifth = queryString.length / 5 33 | val truncatedQuery = StringUtils.truncate(queryString.trim, fifth * 2, fifth) 34 | s""" 35 | |$separatorLine 36 | |... 37 | |${truncatedQuery.trim} 38 | |... 39 | |$separatorLine""" 40 | .stripMargin 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/MultiDocumentArangoDBException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence 18 | 19 | import com.arangodb.ArangoDBException 20 | import com.arangodb.entity.ErrorEntity 21 | 22 | class MultiDocumentArangoDBException(errorEntities: ErrorEntity*) 23 | extends ArangoDBException(errorEntities.head) { 24 | 25 | private lazy val errorEntitiesByCode: Map[Int, Seq[ErrorEntity]] = errorEntities.groupBy(_.getErrorNum) 26 | 27 | def getErrorNumsWithCounts: Seq[(Int, Int)] = { 28 | errorEntitiesByCode.mapValues(_.length).toSeq 29 | } 30 | 31 | def getErrorNums: Seq[Int] = { 32 | errorEntitiesByCode.keys.toSeq 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/migration/MigrationScriptLoader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.migration 18 | 19 | trait MigrationScriptLoader { 20 | def loadAll(): Seq[MigrationScript] 21 | } 22 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/migration/ResourceMigrationScriptLoader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.migration 18 | 19 | import org.springframework.core.io.support.PathMatchingResourcePatternResolver 20 | import za.co.absa.commons.lang.ARM 21 | import za.co.absa.commons.version.Version 22 | 23 | import scala.io.Source 24 | 25 | class ResourceMigrationScriptLoader(location: String) extends MigrationScriptLoader { 26 | override def loadAll(): Seq[MigrationScript] = { 27 | new PathMatchingResourcePatternResolver(getClass.getClassLoader) 28 | .getResources(s"$location/${MigrationScript.FileNamePattern}").toSeq 29 | .map(res => { 30 | val MigrationScript.NameRegexp(verFrom, verTo) = res.getFilename 31 | val scriptBody = ARM.using(Source.fromURL(res.getURL))(_.getLines().mkString("\n")) 32 | MigrationScript( 33 | Version.asSemVer(verFrom), 34 | Version.asSemVer(verTo), 35 | scriptBody) 36 | }) 37 | } 38 | } 39 | 40 | 41 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/tx/AbstractTxBuilder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.tx 18 | 19 | import com.arangodb.model.TransactionOptions 20 | 21 | import scala.jdk.CollectionConverters._ 22 | 23 | abstract class AbstractTxBuilder { 24 | 25 | protected var queries: Seq[Query] = Vector.empty // NOSONAR 26 | 27 | def buildTx(): ArangoTx 28 | 29 | def addQuery(q: Query): this.type = { 30 | queries :+= q 31 | this 32 | } 33 | 34 | protected[tx] def txOptions: TransactionOptions = { 35 | val params = queries 36 | .map({ 37 | case nq: NativeQuery => nq.params 38 | case iq: InsertQuery => iq.documents.toVector 39 | case uq: UpdateQuery => uq.data 40 | }) 41 | val writeCollections = queries 42 | .flatMap(_.collectionDefs) 43 | .map(_.name) 44 | .distinct 45 | new TransactionOptions() 46 | .params(params.asJava) 47 | .writeCollections(writeCollections: _*) 48 | .allowImplicit(false) 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/tx/ArangoTx.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.tx 18 | 19 | import com.arangodb.async.ArangoDatabaseAsync 20 | 21 | import scala.concurrent.{ExecutionContext, Future} 22 | import scala.reflect.ClassTag 23 | 24 | trait ArangoTx { 25 | def execute[A: ClassTag](db: ArangoDatabaseAsync)(implicit ex: ExecutionContext): Future[A] 26 | } 27 | -------------------------------------------------------------------------------- /persistence/src/main/scala/za/co/absa/spline/persistence/tx/FoxxPostTxBuilder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.tx 18 | 19 | import com.arangodb.async.ArangoDatabaseAsync 20 | import com.arangodb.internal.util.ArangoSerializationFactory.Serializer 21 | 22 | import scala.concurrent.{ExecutionContext, Future} 23 | import scala.jdk.FutureConverters._ 24 | import scala.reflect.ClassTag 25 | 26 | class FoxxPostTxBuilder(endpoint: String, body: AnyRef) extends AbstractTxBuilder { 27 | 28 | override def buildTx(): ArangoTx = 29 | new ArangoTx { 30 | override def execute[A: ClassTag](db: ArangoDatabaseAsync)(implicit ex: ExecutionContext): Future[A] = { 31 | val serialization = db.util(Serializer.CUSTOM) 32 | val serializedBody = serialization.serialize(body) 33 | 34 | db 35 | .route(endpoint) 36 | .withBody(serializedBody) 37 | .post() 38 | .asScala 39 | .asInstanceOf[Future[A]] 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /persistence/src/test/scala/com/arangodb/model/ImplicitsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.arangodb.model 18 | 19 | import com.arangodb.entity.IndexType 20 | import org.scalatest.flatspec.AnyFlatSpec 21 | import org.scalatest.matchers.should.Matchers 22 | 23 | class ImplicitsSpec extends AnyFlatSpec with Matchers { 24 | 25 | import com.arangodb.model.Implicits._ 26 | 27 | behavior of "IndexOptionsOps.indexType" 28 | 29 | it should "resolve index type" in { 30 | (new GeoIndexOptions).indexType should be theSameInstanceAs IndexType.geo 31 | (new InvertedIndexOptions).indexType should be theSameInstanceAs IndexType.inverted 32 | (new PersistentIndexOptions).indexType should be theSameInstanceAs IndexType.persistent 33 | (new TtlIndexOptions).indexType should be theSameInstanceAs IndexType.ttl 34 | (new ZKDIndexOptions).indexType should be theSameInstanceAs IndexType.zkd 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /persistence/src/test/scala/za/co/absa/spline/persistence/ArangoRepoConfigSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence 18 | 19 | import org.scalatest.flatspec.AnyFlatSpec 20 | import org.scalatest.matchers.should.Matchers 21 | import za.co.absa.commons.scalatest.EnvFixture 22 | 23 | class ArangoRepoConfigSpec 24 | extends AnyFlatSpec 25 | with Matchers 26 | with EnvFixture { 27 | 28 | it should "support commas in the database connection string" in { 29 | setEnv("spline.database.connectionUrl", "arangodb://host.a:1,host.b:2/dbname") 30 | ArangoRepoConfig.Database.ConnectionURL.hosts shouldEqual Seq(("host.a", 1), ("host.b", 2)) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /persistence/src/test/scala/za/co/absa/spline/persistence/model/DataSourceSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.persistence.model 18 | 19 | import org.scalatest.flatspec.AnyFlatSpec 20 | import org.scalatest.matchers.should.Matchers 21 | 22 | class DataSourceSpec extends AnyFlatSpec with Matchers { 23 | 24 | behavior of "DataSource.getName" 25 | 26 | it should "return last part of string between slashes" in { 27 | DataSource.getName("foo") shouldEqual "foo" 28 | DataSource.getName("/foo") shouldEqual "foo" 29 | DataSource.getName("foo/") shouldEqual "foo" 30 | DataSource.getName("/foo//") shouldEqual "foo" 31 | DataSource.getName("a://b/c/d/foo") shouldEqual "foo" 32 | } 33 | 34 | it should "return empty string when input string only contains slashes" in { 35 | DataSource.getName("///") shouldEqual "" 36 | } 37 | 38 | it should "return empty string when input string is empty" in { 39 | DataSource.getName("") shouldEqual "" 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/ModelMapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper 18 | 19 | import za.co.absa.spline.producer.model.v1_2.{ExecutionEvent, ExecutionPlan} 20 | 21 | trait ModelMapper[P, E] { 22 | def fromDTO(plan: P): ExecutionPlan 23 | def fromDTO(event: E): ExecutionEvent 24 | } 25 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/AttributeConverter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0 18 | 19 | import za.co.absa.commons.lang.Converter 20 | import za.co.absa.spline.producer.model.v1_2 21 | 22 | trait AttributeConverter extends Converter { 23 | override type From = TypesV10.AttrDef 24 | override type To = v1_2.Attribute 25 | } 26 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/AttributeDependencyResolver.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0 18 | 19 | import za.co.absa.spline.producer.modelmapper.v1_0.TypesV10.AttrId 20 | import za.co.absa.spline.producer.{model => modelV1} 21 | 22 | trait AttributeDependencyResolver { 23 | def resolve( 24 | op: modelV1.OperationLike, 25 | inputSchema: => Seq[AttrId], 26 | outputSchema: => Seq[AttrId] 27 | ): Map[AttrId, Set[AttrId]] 28 | } 29 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/ExpressionConverter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0 18 | 19 | import za.co.absa.commons.lang.Converter 20 | import za.co.absa.spline.producer.model.v1_2 21 | 22 | trait ExpressionConverter extends Converter { 23 | override type From = TypesV10.ExprDef 24 | override type To = v1_2.ExpressionLike 25 | 26 | def isExpression(obj: Any): Boolean 27 | } 28 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/ObjectConverter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0 18 | 19 | import za.co.absa.commons.lang.Converter 20 | 21 | trait ObjectConverter extends Converter { 22 | override type From = Any 23 | override type To = Any 24 | } 25 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/OperationOutputConverter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0 18 | 19 | import za.co.absa.commons.lang.Converter 20 | import za.co.absa.spline.producer.model.v1_2 21 | import za.co.absa.spline.producer.{model => v1} 22 | 23 | trait OperationOutputConverter extends Converter { 24 | override type From = v1.OperationLike 25 | override type To = Option[v1_2.OperationLike.Schema] 26 | } 27 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/TypesV10.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0 18 | 19 | object TypesV10 { 20 | type AttrId = String 21 | type ExprId = String 22 | type AttrDef = Map[String, Any] 23 | type ExprDef = Map[String, Any] 24 | type Schema = Seq[AttrId] 25 | type Params = Map[String, Any] 26 | type Extras = Map[String, Any] 27 | } 28 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_0/spark/SparkSplineObjectConverter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_0.spark 18 | 19 | import za.co.absa.spline.producer.model.v1_2.AttrOrExprRef.exprRef 20 | import za.co.absa.spline.producer.modelmapper.v1_0.{ExpressionConverter, ObjectConverter, TypesV10} 21 | 22 | class SparkSplineObjectConverter( 23 | attrRefConverter: AttributeRefConverter, 24 | exprConverter: ExpressionConverter, 25 | ) extends ObjectConverter { 26 | 27 | override def convert(obj: Any): Any = obj match { 28 | case attrDef: TypesV10.AttrDef@unchecked if attrRefConverter.isAttrRef(attrDef) => attrRefConverter.convert(attrDef) 29 | case exprDef: TypesV10.ExprDef@unchecked if exprConverter.isExpression(exprDef) => exprRef(exprConverter.convert(exprDef).id) 30 | case arr: Seq[_] => arr.map(this.convert) 31 | case m: Map[_, _] => m.mapValues(this.convert).view.force // see: https://github.com/scala/bug/issues/4776 32 | case _ => obj 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_1/ModelMapperV11.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_1 18 | 19 | import io.bfil.automapper._ 20 | import za.co.absa.spline.producer.model.{v1_1, v1_2} 21 | import za.co.absa.spline.producer.modelmapper.ModelMapper 22 | 23 | object ModelMapperV11 extends ModelMapper[v1_1.ExecutionPlan, v1_1.ExecutionEvent] { 24 | 25 | override def fromDTO(plan: v1_1.ExecutionPlan): v1_2.ExecutionPlan = automap(plan).to[v1_2.ExecutionPlan] 26 | 27 | override def fromDTO(event: v1_1.ExecutionEvent): v1_2.ExecutionEvent = automap(event).to[v1_2.ExecutionEvent] 28 | } 29 | -------------------------------------------------------------------------------- /producer-model-mapper/src/main/scala/za/co/absa/spline/producer/modelmapper/v1_2/ModelMapperV12.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.modelmapper.v1_2 18 | 19 | import za.co.absa.spline.producer.model.v1_2 20 | import za.co.absa.spline.producer.modelmapper.ModelMapper 21 | 22 | object ModelMapperV12 extends ModelMapper[v1_2.ExecutionPlan, v1_2.ExecutionEvent] { 23 | 24 | override def fromDTO(plan: v1_2.ExecutionPlan): v1_2.ExecutionPlan = plan 25 | 26 | override def fromDTO(event: v1_2.ExecutionEvent): v1_2.ExecutionEvent = event 27 | } 28 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/ExecutionEvent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.model 18 | 19 | import java.util.UUID 20 | 21 | case class ExecutionEvent( 22 | planId: UUID, 23 | timestamp: Long, 24 | error: Option[Any] = None, 25 | extra: Map[String, Any] = Map.empty 26 | ) 27 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/ExecutionPlan.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.model 18 | 19 | import java.util.UUID 20 | 21 | case class ExecutionPlan( 22 | id: UUID = UUID.randomUUID(), 23 | operations: Operations, 24 | systemInfo: SystemInfo, 25 | agentInfo: Option[AgentInfo] = None, 26 | extraInfo: Map[String, Any] = Map.empty 27 | ) 28 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/executionPlanModel.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package za.co.absa.spline.producer.model 17 | 18 | case class Operations( 19 | write: WriteOperation, 20 | reads: Seq[ReadOperation] = Nil, 21 | other: Seq[DataOperation] = Nil) { 22 | 23 | def all: Seq[OperationLike] = reads ++ other :+ write 24 | } 25 | 26 | /** 27 | * Information about a data framework in use (e.g. Spark, StreamSets etc) 28 | */ 29 | case class SystemInfo(name: String, version: String) 30 | 31 | /** 32 | * Spline agent information 33 | */ 34 | case class AgentInfo(name: String, version: String) 35 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package za.co.absa.spline.producer 17 | 18 | /** 19 | * This package contains classes representing the data model of the Spline Producer. 20 | * 21 | * The entities located directly in this package is considered the model version 22 | * v1_0 (Spline 0.5 and older). 23 | * 24 | * The source code for subsequent versions of the model is located in respective 25 | * sub-packages (e.g. v1_1, v1_2, etc). 26 | */ 27 | package object model 28 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/Attribute.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.model.v1_1 18 | 19 | case class Attribute( 20 | id: Attribute.Id, 21 | dataType: Option[Any] = None, 22 | childRefs: Seq[Attribute.ChildRef] = Nil, 23 | extra: Map[String, Any] = Map.empty, 24 | name: String, 25 | ) 26 | 27 | object Attribute { 28 | type Id = ExpressionLike.Id 29 | type ChildRef = AttrOrExprRef 30 | } 31 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/ExecutionEvent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.model.v1_1 18 | 19 | import za.co.absa.spline.producer.model.v1_1.ExecutionEvent._ 20 | 21 | import scala.language.implicitConversions 22 | 23 | case class ExecutionEvent( 24 | planId: ExecutionPlan.Id, 25 | timestamp: Long, 26 | durationNs: Option[DurationNs], 27 | discriminator: Option[ExecutionPlan.Discriminator] = None, 28 | error: Option[Any] = None, 29 | extra: Map[String, Any] = Map.empty 30 | ) 31 | 32 | object ExecutionEvent { 33 | type DurationNs = java.lang.Long 34 | 35 | implicit def optJavaLong2OptScalaLong(opt: Option[java.lang.Long]): Option[Long] = opt.map(identity(_)) 36 | } 37 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_2/Attribute.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.model.v1_2 18 | 19 | case class Attribute( 20 | id: Attribute.Id, 21 | dataType: Option[Any] = None, 22 | childRefs: Seq[Attribute.ChildRef] = Nil, 23 | extra: Map[String, Any] = Map.empty, 24 | name: String, 25 | ) 26 | 27 | object Attribute { 28 | type Id = ExpressionLike.Id 29 | type ChildRef = AttrOrExprRef 30 | } 31 | -------------------------------------------------------------------------------- /producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_2/ExecutionEvent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.model.v1_2 18 | 19 | import za.co.absa.spline.producer.model.v1_2.ExecutionEvent._ 20 | 21 | import scala.language.implicitConversions 22 | 23 | case class ExecutionEvent( 24 | planId: ExecutionPlan.Id, 25 | labels: Map[String, Seq[String]] = Map.empty, 26 | timestamp: Long, 27 | durationNs: Option[DurationNs], 28 | discriminator: Option[ExecutionPlan.Discriminator] = None, 29 | error: Option[Any] = None, 30 | extra: Map[String, Any] = Map.empty 31 | ) 32 | 33 | object ExecutionEvent { 34 | type DurationNs = java.lang.Long 35 | 36 | implicit def optJavaLong2OptScalaLong(opt: Option[java.lang.Long]): Option[Long] = opt.map(identity(_)) 37 | } 38 | -------------------------------------------------------------------------------- /producer-rest-core/src/main/scala/za/co/absa/spline/producer/rest/HttpConstants.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.rest 18 | 19 | object HttpConstants { 20 | 21 | object SplineHeaders { 22 | private val Prefix = "ABSA-Spline" 23 | 24 | val ApiVersion = s"$Prefix-API-Version" 25 | val ApiLTSVersion = s"$Prefix-API-LTS-Version" 26 | val AcceptRequestEncoding = s"$Prefix-Accept-Request-Encoding" 27 | } 28 | 29 | object Encoding { 30 | val GZIP = "gzip" 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /producer-rest-core/src/main/scala/za/co/absa/spline/producer/rest/ProducerAPI.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.rest 18 | 19 | import za.co.absa.commons.version.Version 20 | import za.co.absa.commons.version.Version._ 21 | 22 | object ProducerAPI { 23 | val CurrentVersion: Version = ver"1.2" 24 | val DeprecatedVersions: Seq[Version] = Seq(ver"1" /*, ...*/) 25 | val LTSVersions: Seq[Version] = Seq(CurrentVersion, ver"1.1" /*, ...*/) 26 | val SupportedVersions: Seq[Version] = LTSVersions ++ DeprecatedVersions 27 | 28 | final val MimeTypeV10 = "application/json" 29 | final val MimeTypeV11 = "application/vnd.absa.spline.producer.v1.1+json" 30 | final val MimeTypeV12 = "application/vnd.absa.spline.producer.v1.2+json" 31 | } 32 | -------------------------------------------------------------------------------- /producer-rest-core/src/main/scala/za/co/absa/spline/producer/rest/controller/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.rest 18 | 19 | package object controller { 20 | 21 | trait _package 22 | 23 | } 24 | -------------------------------------------------------------------------------- /producer-rest-core/src/main/scala/za/co/absa/spline/producer/rest/filter/HttpRequestWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package za.co.absa.spline.producer.rest.filter 17 | 18 | import java.io.{BufferedReader, InputStream, InputStreamReader} 19 | import javax.servlet.ServletInputStream 20 | import javax.servlet.http.{HttpServletRequest, HttpServletRequestWrapper} 21 | 22 | 23 | final class HttpRequestWrapper(request: HttpServletRequest, stream: InputStream) 24 | extends HttpServletRequestWrapper(request) { 25 | 26 | private val reader = new BufferedReader(new InputStreamReader(stream)) 27 | 28 | override def getInputStream: ServletInputStream = new ServletInputStreamAdapter(stream) 29 | 30 | override def getReader: BufferedReader = reader 31 | } 32 | -------------------------------------------------------------------------------- /producer-rest-core/src/main/scala/za/co/absa/spline/producer/rest/filter/ServletInputStreamAdapter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 ABSA Group Limited 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package za.co.absa.spline.producer.rest.filter 17 | 18 | import java.io.InputStream 19 | import javax.servlet.{ReadListener, ServletInputStream} 20 | 21 | final class ServletInputStreamAdapter(val gzipStream: InputStream) extends ServletInputStream { 22 | 23 | override def read: Int = gzipStream.read 24 | 25 | override def read(b: Array[Byte]): Int = gzipStream.read(b) 26 | 27 | override def read(b: Array[Byte], off: Int, len: Int): Int = gzipStream.read(b, off, len) 28 | 29 | override def available: Int = gzipStream.available 30 | 31 | override def close(): Unit = gzipStream.close() 32 | 33 | override def isFinished: Boolean = gzipStream.available() == 0 34 | 35 | override def isReady: Boolean = true 36 | 37 | override def setReadListener(readListener: ReadListener): Unit = throw new UnsupportedOperationException 38 | 39 | } 40 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/InconsistentEntityException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service 18 | 19 | class InconsistentEntityException(msg: String) 20 | extends RuntimeException(s"Inconsistent entity: $msg") 21 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/ProducerServicesConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service 18 | 19 | import org.springframework.context.annotation.{Bean, ComponentScan, Configuration, EnableAspectJAutoProxy} 20 | import za.co.absa.spline.common.AsyncCallRetryer 21 | import za.co.absa.spline.persistence.RetryableExceptionUtils 22 | 23 | @Configuration 24 | @EnableAspectJAutoProxy 25 | @ComponentScan(basePackageClasses = Array(classOf[repo._package])) 26 | class ProducerServicesConfig { 27 | 28 | @Bean def retryer: AsyncCallRetryer = new AsyncCallRetryer(isRetryable = RetryableExceptionUtils.isRetryable, 5) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/UUIDCollisionDetectedException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service 18 | 19 | import java.util.UUID 20 | 21 | class UUIDCollisionDetectedException( 22 | entityName: String, 23 | id: UUID, 24 | discriminator: String 25 | ) extends RuntimeException(s"$entityName UUID collision detected: $id, discriminator: $discriminator") 26 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/model/AbstractNodeKeyCreator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service.model 18 | 19 | import java.util.UUID 20 | 21 | abstract class AbstractNodeKeyCreator(prefix: UUID) { 22 | protected def asCompositeKey(suffix: String) = s"$prefix:$suffix" 23 | } 24 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/model/ExecutionEventKeyCreator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service.model 18 | 19 | import za.co.absa.spline.producer.model.v1_2._ 20 | 21 | class ExecutionEventKeyCreator(ee: ExecutionEvent) 22 | extends AbstractNodeKeyCreator(ee.planId) { 23 | 24 | def executionEventKey: String = 25 | asCompositeKey(java.lang.Long.toString(ee.timestamp, 36)) 26 | 27 | } 28 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/model/ExecutionPlanKeyCreator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service.model 18 | 19 | import za.co.absa.spline.producer.model.v1_2._ 20 | 21 | class ExecutionPlanKeyCreator(ep: ExecutionPlan) extends AbstractNodeKeyCreator(ep.id) { 22 | 23 | def asOperationKey(opId: OperationLike.Id): String = asCompositeKey(opId) 24 | 25 | def asSchemaKey(opId: OperationLike.Id): String = asCompositeKey(opId) 26 | 27 | def asAttributeKey(attrId: Attribute.Id): String = asCompositeKey(attrId) 28 | 29 | def asExpressionKey(exprId: ExpressionLike.Id): String = asCompositeKey(exprId) 30 | } 31 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/repo/ExecutionProducerRepository.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service.repo 18 | 19 | import za.co.absa.spline.producer.model.v1_2.{ExecutionEvent, ExecutionPlan} 20 | 21 | import scala.concurrent.{ExecutionContext, Future} 22 | 23 | trait ExecutionProducerRepository { 24 | def insertExecutionPlan(executionPlan: ExecutionPlan)(implicit ec: ExecutionContext): Future[Unit] 25 | def insertExecutionEvent(executionEvent: ExecutionEvent)(implicit ec: ExecutionContext): Future[Unit] 26 | def isDatabaseOk()(implicit ec: ExecutionContext): Future[Boolean] 27 | } 28 | -------------------------------------------------------------------------------- /producer-services/src/main/scala/za/co/absa/spline/producer/service/repo/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service 18 | 19 | package object repo { 20 | 21 | trait _package 22 | 23 | } 24 | -------------------------------------------------------------------------------- /producer-services/src/test/scala/za/co/absa/spline/producer/service/repo/ExecutionEventKeyCreatorSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.producer.service.repo 18 | 19 | import org.scalatest.flatspec.AnyFlatSpec 20 | import org.scalatest.matchers.should.Matchers 21 | import za.co.absa.spline.producer.model.v1_2.ExecutionEvent 22 | import za.co.absa.spline.producer.service.model.ExecutionEventKeyCreator 23 | 24 | import java.util.UUID 25 | 26 | class ExecutionEventKeyCreatorSpec extends AnyFlatSpec with Matchers { 27 | 28 | "asExecutionEventKey" should "create an event key based on the execution plan ID and the event timestamp" in { 29 | val testEvent = ExecutionEvent( 30 | planId = UUID.fromString("00000000-0000-0000-0000-000000000000"), 31 | labels = Map.empty, 32 | timestamp = 1234567890, 33 | durationNs = None, 34 | error = None, 35 | extra = Map.empty) 36 | 37 | new ExecutionEventKeyCreator(testEvent).executionEventKey should be("00000000-0000-0000-0000-000000000000:kf12oi") 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /rest-gateway/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | FROM tomcat:9-jre11-openjdk-slim-buster 18 | 19 | LABEL \ 20 | vendor="ABSA" \ 21 | copyright="2019 ABSA Group Limited" \ 22 | license="Apache License, version 2.0" \ 23 | name="Spline REST Gateway Server" 24 | 25 | EXPOSE 8080 26 | EXPOSE 8009 27 | 28 | RUN rm -rf /usr/local/tomcat/webapps/* 29 | 30 | USER 1001 31 | ARG PROJECT_BUILD_FINAL_NAME 32 | COPY --chown=1001 target/$PROJECT_BUILD_FINAL_NAME/ /usr/local/tomcat/webapps/ROOT/ 33 | -------------------------------------------------------------------------------- /rest-gateway/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "Apache-2.0", 3 | "scripts": { 4 | "gen-api-doc:all": "npm run gen-api-doc --doc_name=consumer && npm run gen-api-doc --doc_name=producer", 5 | "gen-api-doc": "cross-env-shell node ./node_modules/@redocly/cli/bin/cli.js build-docs -o $npm_config_docs_dir/$npm_config_doc_name.html $npm_config_docs_dir/$npm_config_doc_name.swagger.json", 6 | "build": "echo unused stage", 7 | "test": "echo unused stage", 8 | "lint": "echo unused stage" 9 | }, 10 | "devDependencies": { 11 | "@redocly/cli": "^1.34.1", 12 | "cross-env": "^7.0.3" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /rest-gateway/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] %-5level [%thread] %-30logger{30} - %msg%n 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /rest-gateway/src/main/webapp/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/spline/a7f43d2bc22b6dc8c5bd4e116d0ec45514cd1d46/rest-gateway/src/main/webapp/favicon.ico -------------------------------------------------------------------------------- /test-data-generator/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | FROM fabric8/java-alpine-openjdk11-jre 19 | 20 | LABEL \ 21 | vendor="ABSA" \ 22 | copyright="2019 ABSA Group Limited" \ 23 | license="Apache License, version 2.0" \ 24 | name="Spline Performance Testing" 25 | 26 | ARG JAR_FILE 27 | 28 | ENV SPLINE_URL=http://localhost:8080 29 | ENV GRAPH_TYPE=triangle 30 | ENV READS=3 31 | ENV OPERATIONS=30-100@10 32 | ENV ATTRIBUTES=3 33 | 34 | RUN rm -rf /usr/local/tomcat/webapps/* && apk --no-cache add curl 35 | ADD ./target/${JAR_FILE} /opt/test-data-generator.jar 36 | ADD send-plans-and-events-docker.sh /opt/send-plans-and-events-docker.sh 37 | 38 | ENTRYPOINT ["sh", "/opt/send-plans-and-events-docker.sh"] 39 | CMD ["--help"] 40 | -------------------------------------------------------------------------------- /test-data-generator/send-plans-and-events.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | : ' 3 | Copyright 2019 ABSA Group Limited 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ' 17 | 18 | FILENAME=$1 19 | SPLINE_URL=$2 20 | 21 | while read line; do 22 | sleep 5 23 | if [[ ${line:0:1} = '{' ]] 24 | then 25 | echo "Sending plan" 26 | curl -w "@curl-format.txt" -o /dev/null -H "Content-Type: application/vnd.absa.spline.producer.v1.1+json" -X POST --data "${line}" ${SPLINE_URL}/producer/execution-plans 27 | else 28 | echo "Sending event" 29 | curl -w "@curl-format.txt" -o /dev/null -H "Content-Type: application/vnd.absa.spline.producer.v1.1+json" -X POST --data "${line}" ${SPLINE_URL}/producer/execution-events 30 | fi 31 | done < $FILENAME 32 | -------------------------------------------------------------------------------- /test-data-generator/src/main/scala/za/co/absa/spline/testdatagen/FileDispatcher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.testdatagen 18 | 19 | import java.io.File 20 | 21 | import org.apache.commons.io.FileUtils 22 | import org.json4s.JsonAST.JValue 23 | import za.co.absa.spline.producer.model.v1_2.{ExecutionEvent, ExecutionPlan} 24 | import org.json4s.jackson.JsonMethods 25 | import za.co.absa.commons.json.AbstractJsonSerDe 26 | import za.co.absa.commons.json.format.{DefaultFormatsBuilder, JavaTypesSupport} 27 | import scala.jdk.CollectionConverters._ 28 | 29 | class FileDispatcher(fileName: String) extends AbstractJsonSerDe[JValue] 30 | with JsonMethods 31 | with DefaultFormatsBuilder 32 | with JavaTypesSupport { 33 | 34 | private val outputFile = new File(fileName) 35 | 36 | def send(event: ExecutionEvent, plan: ExecutionPlan): Unit = { 37 | 38 | val strings = Seq(plan.toJson, Seq(event).toJson) 39 | 40 | FileUtils.writeLines( 41 | outputFile, 42 | strings.asJava, 43 | true 44 | ) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /test-data-generator/src/main/scala/za/co/absa/spline/testdatagen/GraphType.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.testdatagen 18 | import za.co.absa.commons.reflect.EnumerationMacros 19 | 20 | sealed abstract class GraphType(val name: String) 21 | 22 | object GraphType { 23 | 24 | case object ChainType extends GraphType("chain") 25 | 26 | case object DiamondType extends GraphType("diamond") 27 | 28 | case object TriangleType extends GraphType("triangle") 29 | 30 | private val values: Set[GraphType] = EnumerationMacros.sealedInstancesOf[GraphType] 31 | val stringValues: Set[String] = values.map(_.name) 32 | 33 | def fromString(value: String): Option[GraphType] = { 34 | values.find(_.name.equalsIgnoreCase(value)) 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /test-data-generator/src/main/scala/za/co/absa/spline/testdatagen/generators/AttributesGenerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.testdatagen.generators 18 | 19 | import java.util.UUID 20 | 21 | import za.co.absa.spline.producer.model.v1_2.Attribute.Id 22 | import za.co.absa.spline.producer.model.v1_2.{AttrOrExprRef, Attribute} 23 | 24 | object AttributesGenerator { 25 | 26 | def generateSchema(nr: Int): Seq[Attribute] = { 27 | 1.to(nr).map(id => Attribute(id = UUID.randomUUID().toString, name = s"dummy_attr_$id")) 28 | } 29 | 30 | def generateAttributeFromExpressionParent(parentId: Id): Attribute = { 31 | val attrId = UUID.randomUUID().toString 32 | Attribute(id = attrId, name = s"dummy_attr_$attrId", childRefs = 33 | Seq(AttrOrExprRef( 34 | __attrId = None, 35 | __exprId = Some(parentId)) 36 | ) 37 | ) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test-data-generator/src/main/scala/za/co/absa/spline/testdatagen/generators/EventGenerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.testdatagen.generators 18 | 19 | import za.co.absa.spline.producer.model.v1_2.{ExecutionEvent, ExecutionPlan} 20 | 21 | object EventGenerator { 22 | 23 | def generate(executionPlan: ExecutionPlan): ExecutionEvent = { 24 | ExecutionEvent( 25 | planId = executionPlan.id, 26 | timestamp = System.currentTimeMillis(), 27 | durationNs = None, 28 | error = None, 29 | extra = Map.empty 30 | ) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /test-data-generator/src/main/scala/za/co/absa/spline/testdatagen/generators/ExpressionGenerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.spline.testdatagen.generators 18 | 19 | import java.util.UUID 20 | 21 | import za.co.absa.spline.producer.model.v1_2.{AttrOrExprRef, Attribute, FunctionalExpression, Literal} 22 | 23 | object ExpressionGenerator { 24 | 25 | //1 func expire with 1 lit per attribute 26 | def generateExpressionAndLiteralForAttribute(attribute: Attribute): (FunctionalExpression, Literal) = { 27 | val literal = genLiteral() 28 | 29 | val literalChildRef = AttrOrExprRef(None, Some(literal.id)) 30 | val attributeChildRef = AttrOrExprRef(Some(attribute.id), None) 31 | 32 | val expression = FunctionalExpression(UUID.randomUUID().toString, name = s"dummy_funcexpr", childRefs = Seq(literalChildRef, attributeChildRef)) 33 | (expression, literal) 34 | } 35 | 36 | private def genLiteral(): Literal = { 37 | val id = UUID.randomUUID().toString 38 | Literal(id, value = s"val_$id") 39 | } 40 | 41 | } 42 | --------------------------------------------------------------------------------