├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── question.md └── workflows │ ├── jacoco.yml │ ├── python.yml │ ├── release.yml │ └── scala.yml ├── .gitignore ├── LICENSES ├── LICENSE └── headers │ └── code_header ├── README.md ├── pramen-py ├── .env.ci ├── .env.example ├── .env.local ├── .env.pypi ├── .env.test ├── .flake8 ├── .gitignore ├── .pre-commit-config.yaml ├── Makefile ├── README.md ├── poetry.lock ├── pyproject.toml ├── src │ └── pramen_py │ │ ├── __init__.py │ │ ├── app │ │ ├── __init__.py │ │ ├── cli.py │ │ └── logger.py │ │ ├── metastore │ │ ├── __init__.py │ │ ├── reader.py │ │ ├── reader_base.py │ │ ├── writer.py │ │ └── writer_base.py │ │ ├── models │ │ ├── __init__.py │ │ └── utils.py │ │ ├── py.typed │ │ ├── runner │ │ ├── __init__.py │ │ ├── runner_base.py │ │ └── runner_transformation.py │ │ ├── test_utils │ │ ├── __init__.py │ │ ├── fixtures.py │ │ ├── plugin.py │ │ ├── spark_utils.py │ │ └── when.py │ │ ├── transformation │ │ ├── __init__.py │ │ └── transformation_base.py │ │ └── utils │ │ ├── __init__.py │ │ └── file_system.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── resources │ │ ├── real_config.yaml │ │ ├── test_common.conf │ │ └── test_metastore.conf │ ├── test_app │ │ ├── __init__.py │ │ ├── test_cli.py │ │ └── test_config_serialization.py │ ├── test_identity_transformer.py │ ├── test_metastore │ │ ├── __init__.py │ │ ├── test_reader │ │ │ ├── __init__.py │ │ │ ├── test_extract_info_dates.py │ │ │ ├── test_get_latest.py │ │ │ ├── test_get_latest_available_date.py │ │ │ ├── test_get_table.py │ │ │ ├── test_is_data_available.py │ │ │ └── test_reader.py │ │ └── test_writer │ │ │ ├── __init__.py │ │ │ └── test_writer.py │ ├── test_runner │ │ ├── __init__.py │ │ └── test_runner_transformation.py │ ├── test_test_utils │ │ ├── __init__.py │ │ ├── test_spark_utils.py │ │ └── test_when │ │ │ ├── __init__.py │ │ │ ├── test_create_call_key.py │ │ │ └── test_integration.py │ └── test_utils │ │ ├── __init__.py │ │ ├── test_file_system.py │ │ └── test_utils.py └── transformations │ ├── __init__.py │ ├── example_trasformation_one │ ├── __init__.py │ └── some_transformation.py │ ├── example_trasformation_two │ ├── __init__.py │ └── some_transformation.py │ └── identity_transformer │ ├── __init__.py │ └── example_config.yaml ├── pramen ├── api │ ├── pom.xml │ └── src │ │ ├── main │ │ └── scala │ │ │ └── za │ │ │ └── co │ │ │ └── absa │ │ │ └── pramen │ │ │ └── api │ │ │ ├── CachePolicy.scala │ │ │ ├── CatalogTable.scala │ │ │ ├── DataFormat.scala │ │ │ ├── ExternalChannel.scala │ │ │ ├── ExternalChannelFactory.scala │ │ │ ├── ExternalChannelFactoryV2.scala │ │ │ ├── FieldChange.scala │ │ │ ├── MetaTableDef.scala │ │ │ ├── MetaTableRunInfo.scala │ │ │ ├── MetadataManager.scala │ │ │ ├── MetadataValue.scala │ │ │ ├── MetastoreReader.scala │ │ │ ├── NotificationBuilder.scala │ │ │ ├── NotificationTarget.scala │ │ │ ├── PartitionInfo.scala │ │ │ ├── PartitionScheme.scala │ │ │ ├── PipelineInfo.scala │ │ │ ├── PipelineNotificationTarget.scala │ │ │ ├── Pramen.scala │ │ │ ├── Query.scala │ │ │ ├── Reason.scala │ │ │ ├── RunMode.scala │ │ │ ├── SchemaDifference.scala │ │ │ ├── Sink.scala │ │ │ ├── SinkResult.scala │ │ │ ├── Source.scala │ │ │ ├── SourceResult.scala │ │ │ ├── TableReader.scala │ │ │ ├── Transformer.scala │ │ │ ├── app │ │ │ └── PramenFactory.scala │ │ │ ├── common │ │ │ ├── BuildPropertiesRetriever.scala │ │ │ └── FactoryLoader.scala │ │ │ ├── jobdef │ │ │ ├── Schedule.scala │ │ │ ├── SinkTable.scala │ │ │ ├── SourceTable.scala │ │ │ ├── TransferTable.scala │ │ │ └── TransformExpression.scala │ │ │ ├── notification │ │ │ ├── Align.scala │ │ │ ├── NotificationEntry.scala │ │ │ ├── Style.scala │ │ │ ├── TableHeader.scala │ │ │ └── TextElement.scala │ │ │ ├── offset │ │ │ ├── DataOffset.scala │ │ │ ├── OffsetInfo.scala │ │ │ ├── OffsetType.scala │ │ │ └── OffsetValue.scala │ │ │ ├── sql │ │ │ ├── QuotingPolicy.scala │ │ │ ├── SqlColumnType.scala │ │ │ ├── SqlConfig.scala │ │ │ ├── SqlGenerator.scala │ │ │ └── SqlGeneratorBase.scala │ │ │ └── status │ │ │ ├── CustomNotification.scala │ │ │ ├── DependencyFailure.scala │ │ │ ├── DependencyWarning.scala │ │ │ ├── JobType.scala │ │ │ ├── MetastoreDependency.scala │ │ │ ├── NotificationFailure.scala │ │ │ ├── PipelineNotificationFailure.scala │ │ │ ├── PipelineStateSnapshot.scala │ │ │ ├── RunInfo.scala │ │ │ ├── RunStatus.scala │ │ │ ├── RuntimeInfo.scala │ │ │ ├── TaskDef.scala │ │ │ ├── TaskResult.scala │ │ │ ├── TaskRunReason.scala │ │ │ └── TaskStatus.scala │ │ └── test │ │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── pramen │ │ └── api │ │ ├── CatalogTableSuite.scala │ │ ├── PramenSuite.scala │ │ ├── common │ │ └── FactoryLoaderSuite.scala │ │ ├── mocks │ │ ├── DummyEmptySingletonClass.scala │ │ ├── DummyPramen.scala │ │ ├── DummySingletonClass.scala │ │ ├── DummySingletonFactory.scala │ │ └── DummySingletonTrait.scala │ │ ├── offset │ │ ├── OffsetTypeSuite.scala │ │ └── OffsetValueSuite.scala │ │ └── sql │ │ └── QuotingPolicySuite.scala ├── build.sbt ├── core │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── resources │ │ │ ├── email_template │ │ │ │ └── style.css │ │ │ ├── pramen_banner.txt │ │ │ ├── pramen_build.properties │ │ │ └── reference.conf │ │ ├── scala │ │ │ └── za │ │ │ │ └── co │ │ │ │ └── absa │ │ │ │ └── pramen │ │ │ │ ├── core │ │ │ │ ├── ExternalChannelFactoryReflect.scala │ │ │ │ ├── PramenImpl.scala │ │ │ │ ├── RunnerCommons.scala │ │ │ │ ├── app │ │ │ │ │ ├── AppConfig.scala │ │ │ │ │ ├── AppContext.scala │ │ │ │ │ ├── AppContextImpl.scala │ │ │ │ │ └── config │ │ │ │ │ │ ├── BookkeeperConfig.scala │ │ │ │ │ │ ├── GeneralConfig.scala │ │ │ │ │ │ ├── HadoopFormat.scala │ │ │ │ │ │ ├── HookConfig.scala │ │ │ │ │ │ ├── InfoDateConfig.scala │ │ │ │ │ │ └── RuntimeConfig.scala │ │ │ │ ├── bookkeeper │ │ │ │ │ ├── Bookkeeper.scala │ │ │ │ │ ├── BookkeeperBase.scala │ │ │ │ │ ├── BookkeeperDeltaBase.scala │ │ │ │ │ ├── BookkeeperDeltaPath.scala │ │ │ │ │ ├── BookkeeperDeltaTable.scala │ │ │ │ │ ├── BookkeeperHadoop.scala │ │ │ │ │ ├── BookkeeperJdbc.scala │ │ │ │ │ ├── BookkeeperMongoDb.scala │ │ │ │ │ ├── BookkeeperNull.scala │ │ │ │ │ ├── BookkeeperText.scala │ │ │ │ │ ├── OffsetManager.scala │ │ │ │ │ ├── OffsetManagerCached.scala │ │ │ │ │ ├── OffsetManagerJdbc.scala │ │ │ │ │ ├── OffsetManagerUtils.scala │ │ │ │ │ └── model │ │ │ │ │ │ ├── BookkeepingRecord.scala │ │ │ │ │ │ ├── BookkeepingRecords.scala │ │ │ │ │ │ ├── DataOffsetAggregated.scala │ │ │ │ │ │ ├── DataOffsetRequest.scala │ │ │ │ │ │ ├── MetadataRecord.scala │ │ │ │ │ │ ├── MetadataRecords.scala │ │ │ │ │ │ ├── OffsetCommitRequest.scala │ │ │ │ │ │ ├── OffsetRecord.scala │ │ │ │ │ │ ├── OffsetRecordConverter.scala │ │ │ │ │ │ ├── OffsetRecords.scala │ │ │ │ │ │ ├── SchemaRecord.scala │ │ │ │ │ │ ├── SchemaRecords.scala │ │ │ │ │ │ └── TableSchemaJson.scala │ │ │ │ ├── cmd │ │ │ │ │ └── CmdLineConfig.scala │ │ │ │ ├── config │ │ │ │ │ ├── InfoDateExpressions.scala │ │ │ │ │ ├── InfoDateOverride.scala │ │ │ │ │ ├── Keys.scala │ │ │ │ │ └── PartitionSchemeParser.scala │ │ │ │ ├── dao │ │ │ │ │ ├── DocumentDb.scala │ │ │ │ │ ├── MongoDb.scala │ │ │ │ │ ├── ScalaMongoImplicits.scala │ │ │ │ │ └── model │ │ │ │ │ │ ├── DbVersion.scala │ │ │ │ │ │ └── Index.scala │ │ │ │ ├── databricks │ │ │ │ │ ├── DatabricksClient.scala │ │ │ │ │ ├── DatabricksClientImpl.scala │ │ │ │ │ ├── PramenPyJobTemplate.scala │ │ │ │ │ └── Responses.scala │ │ │ │ ├── exceptions │ │ │ │ │ ├── CmdFailedException.scala │ │ │ │ │ ├── FatalErrorWrapper.scala │ │ │ │ │ ├── OsSignalException.scala │ │ │ │ │ ├── ProcessFailedException.scala │ │ │ │ │ ├── ReasonException.scala │ │ │ │ │ ├── ThreadStackTrace.scala │ │ │ │ │ ├── TimeoutException.scala │ │ │ │ │ └── ValidationException.scala │ │ │ │ ├── expr │ │ │ │ │ ├── DateExprEvaluator.scala │ │ │ │ │ ├── exceptions │ │ │ │ │ │ └── SyntaxErrorException.scala │ │ │ │ │ ├── lexer │ │ │ │ │ │ ├── Lexer.scala │ │ │ │ │ │ └── Token.scala │ │ │ │ │ └── parser │ │ │ │ │ │ ├── DateExprBuilder.scala │ │ │ │ │ │ ├── DateExpressionEvaluator.scala │ │ │ │ │ │ └── Parser.scala │ │ │ │ ├── journal │ │ │ │ │ ├── Journal.scala │ │ │ │ │ ├── JournalHadoopCsv.scala │ │ │ │ │ ├── JournalHadoopDeltaPath.scala │ │ │ │ │ ├── JournalHadoopDeltaTable.scala │ │ │ │ │ ├── JournalJdbc.scala │ │ │ │ │ ├── JournalMongoDb.scala │ │ │ │ │ ├── JournalNull.scala │ │ │ │ │ └── model │ │ │ │ │ │ ├── JournalTask.scala │ │ │ │ │ │ ├── JournalTasks.scala │ │ │ │ │ │ ├── TaskCompleted.scala │ │ │ │ │ │ └── TaskCompletedCsv.scala │ │ │ │ ├── lock │ │ │ │ │ ├── TokenLock.scala │ │ │ │ │ ├── TokenLockAllow.scala │ │ │ │ │ ├── TokenLockBase.scala │ │ │ │ │ ├── TokenLockFactory.scala │ │ │ │ │ ├── TokenLockFactoryAllow.scala │ │ │ │ │ ├── TokenLockFactoryHadoopPath.scala │ │ │ │ │ ├── TokenLockFactoryJdbc.scala │ │ │ │ │ ├── TokenLockFactoryMongoDb.scala │ │ │ │ │ ├── TokenLockHadoopPath.scala │ │ │ │ │ ├── TokenLockJdbc.scala │ │ │ │ │ ├── TokenLockMongoDb.scala │ │ │ │ │ └── model │ │ │ │ │ │ ├── LockTicket.scala │ │ │ │ │ │ └── LockTickets.scala │ │ │ │ ├── metadata │ │ │ │ │ ├── MetadataManagerBase.scala │ │ │ │ │ ├── MetadataManagerJdbc.scala │ │ │ │ │ ├── MetadataManagerNull.scala │ │ │ │ │ └── MetadataTableKey.scala │ │ │ │ ├── metastore │ │ │ │ │ ├── MetaTableStats.scala │ │ │ │ │ ├── Metastore.scala │ │ │ │ │ ├── MetastoreImpl.scala │ │ │ │ │ ├── MetastoreReaderBase.scala │ │ │ │ │ ├── MetastoreReaderBatchImpl.scala │ │ │ │ │ ├── MetastoreReaderIncremental.scala │ │ │ │ │ ├── MetastoreReaderIncrementalImpl.scala │ │ │ │ │ ├── NoDataInTable.scala │ │ │ │ │ ├── NoSuchTable.scala │ │ │ │ │ ├── TableNotConfigured.scala │ │ │ │ │ ├── model │ │ │ │ │ │ ├── DataFormatParser.scala │ │ │ │ │ │ ├── HiveApi.scala │ │ │ │ │ │ ├── HiveConfig.scala │ │ │ │ │ │ ├── HiveDefaultConfig.scala │ │ │ │ │ │ ├── MetaTable.scala │ │ │ │ │ │ ├── MetastoreDependencyFactory.scala │ │ │ │ │ │ ├── ReaderMode.scala │ │ │ │ │ │ └── TrackingTable.scala │ │ │ │ │ └── peristence │ │ │ │ │ │ ├── MetastorePartition.scala │ │ │ │ │ │ ├── MetastorePersistence.scala │ │ │ │ │ │ ├── MetastorePersistenceDelta.scala │ │ │ │ │ │ ├── MetastorePersistenceIceberg.scala │ │ │ │ │ │ ├── MetastorePersistenceNull.scala │ │ │ │ │ │ ├── MetastorePersistenceParquet.scala │ │ │ │ │ │ ├── MetastorePersistenceRaw.scala │ │ │ │ │ │ ├── MetastorePersistenceTransient.scala │ │ │ │ │ │ ├── MetastorePersistenceTransientEager.scala │ │ │ │ │ │ ├── TransientJobManager.scala │ │ │ │ │ │ └── TransientTableManager.scala │ │ │ │ ├── model │ │ │ │ │ ├── DataChunk.scala │ │ │ │ │ ├── QueryBuilder.scala │ │ │ │ │ ├── SyncStatus.scala │ │ │ │ │ ├── TableChunk.scala │ │ │ │ │ ├── TableSchema.scala │ │ │ │ │ ├── TableStatus.scala │ │ │ │ │ └── TableUpdates.scala │ │ │ │ ├── mongo │ │ │ │ │ └── MongoDbConnection.scala │ │ │ │ ├── notify │ │ │ │ │ ├── NotificationTargetManager.scala │ │ │ │ │ ├── PipelineNotificationTargetFactory.scala │ │ │ │ │ ├── Sendable.scala │ │ │ │ │ ├── message │ │ │ │ │ │ ├── MessageBuilder.scala │ │ │ │ │ │ ├── MessageBuilderHtml.scala │ │ │ │ │ │ ├── ParagraphBuilder.scala │ │ │ │ │ │ ├── TableBuilder.scala │ │ │ │ │ │ └── TableBuilderHtml.scala │ │ │ │ │ └── pipeline │ │ │ │ │ │ ├── PipelineNotification.scala │ │ │ │ │ │ ├── PipelineNotificationBuilder.scala │ │ │ │ │ │ ├── PipelineNotificationBuilderHtml.scala │ │ │ │ │ │ ├── PipelineNotificationDirector.scala │ │ │ │ │ │ ├── PipelineNotificationEmail.scala │ │ │ │ │ │ ├── PipelineStatus.scala │ │ │ │ │ │ └── ValidatedEmails.scala │ │ │ │ ├── pipeline │ │ │ │ │ ├── IncrementalIngestionJob.scala │ │ │ │ │ ├── IngestionJob.scala │ │ │ │ │ ├── Job.scala │ │ │ │ │ ├── JobBase.scala │ │ │ │ │ ├── JobDependency.scala │ │ │ │ │ ├── JobNotificationTarget.scala │ │ │ │ │ ├── JobPreRunResult.scala │ │ │ │ │ ├── JobPreRunStatus.scala │ │ │ │ │ ├── OperationDef.scala │ │ │ │ │ ├── OperationSplitter.scala │ │ │ │ │ ├── OperationType.scala │ │ │ │ │ ├── PipelineDef.scala │ │ │ │ │ ├── PramenPyCmdConfig.scala │ │ │ │ │ ├── PythonTransformationJob.scala │ │ │ │ │ ├── RunResult.scala │ │ │ │ │ ├── SaveResult.scala │ │ │ │ │ ├── SinkJob.scala │ │ │ │ │ ├── SinkTableParser.scala │ │ │ │ │ ├── SourceTableParser.scala │ │ │ │ │ ├── Task.scala │ │ │ │ │ ├── TaskPreDef.scala │ │ │ │ │ ├── TransferJob.scala │ │ │ │ │ ├── TransferTableParser.scala │ │ │ │ │ ├── TransformExpressionParser.scala │ │ │ │ │ └── TransformationJob.scala │ │ │ │ ├── process │ │ │ │ │ ├── ProcessRunner.scala │ │ │ │ │ └── ProcessRunnerImpl.scala │ │ │ │ ├── rdb │ │ │ │ │ ├── DbVersion.scala │ │ │ │ │ ├── PramenDb.scala │ │ │ │ │ ├── Rdb.scala │ │ │ │ │ └── RdbJdbc.scala │ │ │ │ ├── reader │ │ │ │ │ ├── JdbcUrlSelector.scala │ │ │ │ │ ├── JdbcUrlSelectorImpl.scala │ │ │ │ │ ├── TableReaderJdbc.scala │ │ │ │ │ ├── TableReaderJdbcBase.scala │ │ │ │ │ ├── TableReaderJdbcNative.scala │ │ │ │ │ ├── TableReaderSpark.scala │ │ │ │ │ └── model │ │ │ │ │ │ ├── JdbcConfig.scala │ │ │ │ │ │ ├── OffsetInfoParser.scala │ │ │ │ │ │ └── TableReaderJdbcConfig.scala │ │ │ │ ├── runner │ │ │ │ │ ├── AppRunner.scala │ │ │ │ │ ├── InfoDateCalculator.scala │ │ │ │ │ ├── PipelineSparkSessionBuilder.scala │ │ │ │ │ ├── jobrunner │ │ │ │ │ │ ├── ConcurrentJobRunner.scala │ │ │ │ │ │ └── ConcurrentJobRunnerImpl.scala │ │ │ │ │ ├── orchestrator │ │ │ │ │ │ ├── DependencyResolver.scala │ │ │ │ │ │ ├── DependencyResolverImpl.scala │ │ │ │ │ │ ├── Orchestrator.scala │ │ │ │ │ │ └── OrchestratorImpl.scala │ │ │ │ │ ├── splitter │ │ │ │ │ │ ├── ScheduleParams.scala │ │ │ │ │ │ ├── ScheduleStrategy.scala │ │ │ │ │ │ ├── ScheduleStrategyIncremental.scala │ │ │ │ │ │ ├── ScheduleStrategySourcing.scala │ │ │ │ │ │ ├── ScheduleStrategyTransformation.scala │ │ │ │ │ │ └── ScheduleStrategyUtils.scala │ │ │ │ │ └── task │ │ │ │ │ │ ├── TaskRunner.scala │ │ │ │ │ │ ├── TaskRunnerBase.scala │ │ │ │ │ │ └── TaskRunnerMultithreaded.scala │ │ │ │ ├── schedule │ │ │ │ │ └── ScheduleParser.scala │ │ │ │ ├── sink │ │ │ │ │ ├── CmdLineSink.scala │ │ │ │ │ ├── ColumnNameTransform.scala │ │ │ │ │ ├── CsvConversionParams.scala │ │ │ │ │ ├── LocalCsvSink.scala │ │ │ │ │ ├── SinkManager.scala │ │ │ │ │ ├── SparkSink.scala │ │ │ │ │ └── SparkSinkFormat.scala │ │ │ │ ├── source │ │ │ │ │ ├── JdbcSource.scala │ │ │ │ │ ├── LocalSparkSource.scala │ │ │ │ │ ├── RawFileSource.scala │ │ │ │ │ ├── SourceManager.scala │ │ │ │ │ └── SparkSource.scala │ │ │ │ ├── sql │ │ │ │ │ ├── SqlGeneratorDatabricks.scala │ │ │ │ │ ├── SqlGeneratorDb2.scala │ │ │ │ │ ├── SqlGeneratorDenodo.scala │ │ │ │ │ ├── SqlGeneratorGeneric.scala │ │ │ │ │ ├── SqlGeneratorHive.scala │ │ │ │ │ ├── SqlGeneratorHsqlDb.scala │ │ │ │ │ ├── SqlGeneratorLoader.scala │ │ │ │ │ ├── SqlGeneratorMicrosoft.scala │ │ │ │ │ ├── SqlGeneratorMySQL.scala │ │ │ │ │ ├── SqlGeneratorOracle.scala │ │ │ │ │ ├── SqlGeneratorPostgreSQL.scala │ │ │ │ │ ├── SqlGeneratorSas.scala │ │ │ │ │ └── dialects │ │ │ │ │ │ ├── DatabricksDialect.scala │ │ │ │ │ │ ├── DenodoDialect.scala │ │ │ │ │ │ ├── HiveDialect.scala │ │ │ │ │ │ └── SasDialect.scala │ │ │ │ ├── state │ │ │ │ │ ├── NotificationBuilderImpl.scala │ │ │ │ │ ├── PipelineState.scala │ │ │ │ │ ├── PipelineStateImpl.scala │ │ │ │ │ ├── PramenSignalHandler.scala │ │ │ │ │ └── SystemExitCatcherSecurityManager.scala │ │ │ │ ├── transformers │ │ │ │ │ ├── ConversionTransformer.scala │ │ │ │ │ └── IdentityTransformer.scala │ │ │ │ ├── utils │ │ │ │ │ ├── AlgorithmUtils.scala │ │ │ │ │ ├── BuildPropertyUtils.scala │ │ │ │ │ ├── CircularBuffer.scala │ │ │ │ │ ├── ClassLoaderUtils.scala │ │ │ │ │ ├── ConfigUtils.scala │ │ │ │ │ ├── CsvUtils.scala │ │ │ │ │ ├── DateUtils.scala │ │ │ │ │ ├── Emoji.scala │ │ │ │ │ ├── FsUtils.scala │ │ │ │ │ ├── FutureImplicits.scala │ │ │ │ │ ├── JavaXConfig.scala │ │ │ │ │ ├── JdbcNativeUtils.scala │ │ │ │ │ ├── JdbcSparkUtils.scala │ │ │ │ │ ├── JsonUtils.scala │ │ │ │ │ ├── JvmUtils.scala │ │ │ │ │ ├── LocalFsUtils.scala │ │ │ │ │ ├── MutableStack.scala │ │ │ │ │ ├── ResourceUtils.scala │ │ │ │ │ ├── ScheduleUtils.scala │ │ │ │ │ ├── SlickUtils.scala │ │ │ │ │ ├── SparkUtils.scala │ │ │ │ │ ├── StringUtils.scala │ │ │ │ │ ├── ThreadUtils.scala │ │ │ │ │ ├── TimeUtils.scala │ │ │ │ │ ├── hive │ │ │ │ │ │ ├── HiveFormat.scala │ │ │ │ │ │ ├── HiveHelper.scala │ │ │ │ │ │ ├── HiveHelperSparkCatalog.scala │ │ │ │ │ │ ├── HiveHelperSql.scala │ │ │ │ │ │ ├── HiveQueryTemplates.scala │ │ │ │ │ │ ├── QueryExecutor.scala │ │ │ │ │ │ ├── QueryExecutorJdbc.scala │ │ │ │ │ │ └── QueryExecutorSpark.scala │ │ │ │ │ ├── impl │ │ │ │ │ │ ├── JdbcFieldMetadata.scala │ │ │ │ │ │ ├── ResultSetToRowIterator.scala │ │ │ │ │ │ └── ThreadWithException.scala │ │ │ │ │ └── traverser │ │ │ │ │ │ ├── FileStatus.scala │ │ │ │ │ │ ├── FsTraverser.scala │ │ │ │ │ │ └── FsTraverserLocal.scala │ │ │ │ └── validator │ │ │ │ │ ├── RunDecision.scala │ │ │ │ │ ├── RunDecisionSummary.scala │ │ │ │ │ ├── SyncJobValidator.scala │ │ │ │ │ └── ValidationCheck.scala │ │ │ │ └── runner │ │ │ │ └── PipelineRunner.scala │ │ ├── scala_2.11 │ │ │ └── za │ │ │ │ └── co │ │ │ │ └── absa │ │ │ │ └── pramen │ │ │ │ └── core │ │ │ │ └── metastore │ │ │ │ └── peristence │ │ │ │ └── MetastorePersistenceIcebergOps.scala │ │ ├── scala_2.12 │ │ │ └── za │ │ │ │ └── co │ │ │ │ └── absa │ │ │ │ └── pramen │ │ │ │ └── core │ │ │ │ └── metastore │ │ │ │ └── peristence │ │ │ │ └── MetastorePersistenceIcebergOps.scala │ │ └── scala_2.13 │ │ │ └── za │ │ │ └── co │ │ │ └── absa │ │ │ └── pramen │ │ │ └── core │ │ │ └── metastore │ │ │ └── peristence │ │ │ └── MetastorePersistenceIcebergOps.scala │ │ └── test │ │ ├── resources │ │ ├── log4j.properties │ │ ├── log4j2.properties │ │ ├── reference.conf │ │ └── test │ │ │ ├── config │ │ │ ├── app_context.conf │ │ │ ├── disable_count_query.conf │ │ │ ├── incremental_pipeline.conf │ │ │ ├── incremental_pipeline_jdbc.conf │ │ │ ├── incremental_pipeline_ts.conf │ │ │ ├── integration_file_based_source.conf │ │ │ ├── integration_file_source_sink.conf │ │ │ ├── integration_ingestion_native.conf │ │ │ ├── integration_inner_source.conf │ │ │ ├── integration_multi_table_transformers.conf │ │ │ ├── integration_notification_targets.conf │ │ │ ├── integration_parallel_execution.conf │ │ │ ├── integration_transient_eager_transformer.conf │ │ │ ├── integration_transient_transformer.conf │ │ │ ├── metastore.conf │ │ │ ├── pipeline_v2.conf │ │ │ ├── pipeline_v2_empty.conf │ │ │ ├── pipeline_v3.conf │ │ │ ├── test_javax.conf │ │ │ └── testconfig.conf │ │ │ ├── notify │ │ │ ├── expectedMessage.dat │ │ │ ├── test_pipeline_email_body_complex.txt │ │ │ └── test_pipeline_email_body_default.txt │ │ │ └── testResource.txt │ │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── pramen │ │ └── core │ │ ├── AppConfigFactory.scala │ │ ├── BookkeepingConfigFactory.scala │ │ ├── ExternalChannelFactorySuite.scala │ │ ├── GeneralConfigFactory.scala │ │ ├── InfoDateConfigFactory.scala │ │ ├── MetaTableDefFactory.scala │ │ ├── NestedDataFrameFactory.scala │ │ ├── OperationDefFactory.scala │ │ ├── PipelineRunnerSuite.scala │ │ ├── PramenHookSpy.scala │ │ ├── PramenImplSuite.scala │ │ ├── RunnerCommonsSuite.scala │ │ ├── RuntimeConfigFactory.scala │ │ ├── TaskDefFactory.scala │ │ ├── TaskNotificationFactory.scala │ │ ├── app │ │ ├── AppContextSuite.scala │ │ └── config │ │ │ ├── BookkeeperConfigSuite.scala │ │ │ ├── GeneralConfigSuite.scala │ │ │ ├── HookConfigSuite.scala │ │ │ ├── InfoDateConfigSuite.scala │ │ │ ├── RunModeSuite.scala │ │ │ └── RuntimeConfigSuite.scala │ │ ├── base │ │ ├── SparkTestBase.scala │ │ └── SparkTestIcebergBase.scala │ │ ├── cmd │ │ └── CmdLineConfigSuite.scala │ │ ├── config │ │ ├── InfoDateOverrideSuite.scala │ │ └── PartitionSchemeParserSuite.scala │ │ ├── databricks │ │ ├── DatabricksClientImplSuite.scala │ │ ├── DatabricksClientSuite.scala │ │ ├── PramenPyJobTemplateSuite.scala │ │ └── ResponsesSuite.scala │ │ ├── fixtures │ │ ├── AppContextFixture.scala │ │ ├── MongoDbFixture.scala │ │ ├── RelationalDbFixture.scala │ │ ├── ScriptProcessRunnerFixture.scala │ │ ├── TempDirFixture.scala │ │ └── TextComparisonFixture.scala │ │ ├── integration │ │ ├── DisableCountQueryLongSuite.scala │ │ ├── FileSourceSinkSuite.scala │ │ ├── FileSourcingSuite.scala │ │ ├── IncrementalPipelineDeltaLongSuite.scala │ │ ├── IncrementalPipelineJdbcLongSuite.scala │ │ ├── IncrementalPipelineLongFixture.scala │ │ ├── IncrementalPipelineParquetLongSuite.scala │ │ ├── InnerSourceLongSuite.scala │ │ ├── JdbcNativeTableLongSuite.scala │ │ ├── MultiJobTransformersLongSuite.scala │ │ ├── NotificationTargetSuite.scala │ │ ├── ParallelExecutionLongSuite.scala │ │ ├── TransientEagerTablesSuite.scala │ │ └── TransientTablesSuite.scala │ │ ├── metadata │ │ ├── MetadataManagerBaseSuite.scala │ │ ├── MetadataManagerJdbcSuite.scala │ │ └── MetadataManagerNullSuite.scala │ │ ├── metastore │ │ ├── MetastoreSuite.scala │ │ ├── model │ │ │ ├── DataFormatSuite.scala │ │ │ ├── HiveConfigSuite.scala │ │ │ ├── HiveDefaultConfigSuite.scala │ │ │ └── MetaTableSuite.scala │ │ └── persistence │ │ │ ├── MetastorePartitionSchemeLongSuite.scala │ │ │ ├── MetastorePersistenceIcebergLongSuite.scala │ │ │ ├── MetastorePersistenceParquetSuite.scala │ │ │ ├── MetastorePersistenceRawSuite.scala │ │ │ ├── MetastorePersistenceSuite.scala │ │ │ ├── MetastorePersistenceTransientEagerSuiteEager.scala │ │ │ ├── TransientJobManagerSuite.scala │ │ │ └── TransientTableManagerSuite.scala │ │ ├── mocks │ │ ├── DataChunkFactory.scala │ │ ├── DummyEmptySingletonClass.scala │ │ ├── DummyProcessRunner.scala │ │ ├── DummySingletonClass.scala │ │ ├── DummySingletonFactory.scala │ │ ├── DummySingletonTrait.scala │ │ ├── DummySqlConfigFactory.scala │ │ ├── ExitException.scala │ │ ├── ExternalChannelMock.scala │ │ ├── ExternalChannelV2Mock.scala │ │ ├── FieldChangeFactory.scala │ │ ├── MetaTableFactory.scala │ │ ├── NoExitSecurityManager.scala │ │ ├── PipelineInfoFactory.scala │ │ ├── PipelineNotificationFactory.scala │ │ ├── PipelineStateSnapshotFactory.scala │ │ ├── RunStatusFactory.scala │ │ ├── RunnableSpy.scala │ │ ├── SchemaDifferenceFactory.scala │ │ ├── SinkTableFactory.scala │ │ ├── SourceTableFactory.scala │ │ ├── SqlGeneratorDummy.scala │ │ ├── TaskCompletedFactory.scala │ │ ├── TaskResultFactory.scala │ │ ├── TestPrototypes.scala │ │ ├── TransferTableFactory.scala │ │ ├── bookkeeper │ │ │ └── SyncBookkeeperMock.scala │ │ ├── dao │ │ │ ├── DocumentDbMock.scala │ │ │ └── MongoDbSingleton.scala │ │ ├── databricks │ │ │ └── DatabricksClientSpy.scala │ │ ├── job │ │ │ ├── JobBaseDummy.scala │ │ │ ├── JobSpy.scala │ │ │ ├── SinkSpy.scala │ │ │ └── SourceSpy.scala │ │ ├── journal │ │ │ └── JournalMock.scala │ │ ├── lock │ │ │ ├── TokenLockFactoryMock.scala │ │ │ └── TokenLockMock.scala │ │ ├── metadata │ │ │ └── MetadataManagerSpy.scala │ │ ├── metastore │ │ │ ├── MetastoreReaderMock.scala │ │ │ └── MetastoreSpy.scala │ │ ├── notify │ │ │ ├── NotificationTargetMock.scala │ │ │ ├── NotificationTargetSpy.scala │ │ │ ├── PipelineNotificationBuilderSpy.scala │ │ │ └── PipelineNotificationTargetMock.scala │ │ ├── process │ │ │ └── ProcessRunnerSpy.scala │ │ ├── reader │ │ │ ├── ReaderSpy.scala │ │ │ ├── ReaderStub.scala │ │ │ └── TableReaderSparkFactory.scala │ │ ├── runner │ │ │ └── ConcurrentJobRunnerSpy.scala │ │ ├── sink │ │ │ ├── CmdLineSinkSuite.scala │ │ │ ├── CsvConversionParamsFactory.scala │ │ │ └── PostProcessingSink.scala │ │ ├── source │ │ │ └── SourceMock.scala │ │ ├── state │ │ │ ├── PipelineStateSpy.scala │ │ │ ├── ShutdownHookFailureMock.scala │ │ │ └── ShutdownHookSuccessMock.scala │ │ ├── transformer │ │ │ ├── DummyTransformer1.scala │ │ │ ├── DummyTransformer2.scala │ │ │ ├── DummyTransformer3.scala │ │ │ ├── FailingTransformer.scala │ │ │ ├── GeneratingTransformer.scala │ │ │ ├── InnerSourceTransformer.scala │ │ │ └── RangedTransformer.scala │ │ ├── utils │ │ │ └── hive │ │ │ │ └── QueryExecutorMock.scala │ │ └── validator │ │ │ └── DummySyncJobValidator.scala │ │ ├── model │ │ └── QueryBuilderSuite.scala │ │ ├── pipeline │ │ ├── IngestionJobSuite.scala │ │ ├── JobBaseSuite.scala │ │ ├── OperationDefSuite.scala │ │ ├── OperationSplitterSuite.scala │ │ ├── OperationTypeSuite.scala │ │ ├── PipelineDefSuite.scala │ │ ├── PythonTransformationJobSuite.scala │ │ ├── SinkJobSuite.scala │ │ ├── SinkTableParserSuite.scala │ │ ├── SourceTableParserSuite.scala │ │ ├── TaskRunReasonSuite.scala │ │ ├── TransferJobSuite.scala │ │ ├── TransferTableSuite.scala │ │ ├── TransformationJobSuite.scala │ │ └── TransformerSpy.scala │ │ ├── process │ │ └── ProcessRunnerSuite.scala │ │ ├── samples │ │ ├── RdbExampleTable.scala │ │ ├── SampleCaseClass.scala │ │ └── SampleCaseClass2.scala │ │ ├── schedule │ │ └── ScheduleSuite.scala │ │ ├── sink │ │ ├── ColumnNameTransformSuite.scala │ │ ├── CsvConversionParamsSuite.scala │ │ ├── LocalCsvSinkSuite.scala │ │ ├── SparkSinkFormatSuite.scala │ │ └── SparkSinkSuite.scala │ │ ├── source │ │ ├── JdbcSourceSuite.scala │ │ ├── LocalSparkSourceSuite.scala │ │ ├── RawFileSourceSuite.scala │ │ ├── SourceValidationSuite.scala │ │ └── SparkSourceSuite.scala │ │ ├── state │ │ ├── NotificationBuilderImplSuite.scala │ │ ├── PipelineStateSuite.scala │ │ └── PramenSignalHandlerSuite.scala │ │ ├── tests │ │ ├── bookkeeper │ │ │ ├── BookkeeperCommonSuite.scala │ │ │ ├── BookkeeperDeltaPathLongSuite.scala │ │ │ ├── BookkeeperDeltaTableLongSuite.scala │ │ │ ├── BookkeeperJdbcSuite.scala │ │ │ ├── BookkeeperMemSuite.scala │ │ │ ├── BookkeeperMongoDbLongSuite.scala │ │ │ ├── BookkeeperNullSuite.scala │ │ │ ├── BookkeeperSuite.scala │ │ │ ├── BookkeeperTextLongSuite.scala │ │ │ ├── BookkeeperTransientSuiteEager.scala │ │ │ ├── OffsetManagerJdbcSuite.scala │ │ │ └── OffsetManagerUtilsSuite.scala │ │ ├── expr │ │ │ ├── DateExprEvaluatorSuite.scala │ │ │ └── parser │ │ │ │ └── LexerSuite.scala │ │ ├── journal │ │ │ ├── JournalHadoopCsvSuite.scala │ │ │ ├── JournalHadoopDeltaPathLongSuite.scala │ │ │ ├── JournalHadoopDeltaTableSuite.scala │ │ │ ├── JournalJdbcSuite.scala │ │ │ ├── JournalMongoDbSuite.scala │ │ │ ├── JournalNullSuite.scala │ │ │ ├── TaskCompletedSuite.scala │ │ │ └── TestCases.scala │ │ ├── lock │ │ │ ├── TokenLockHadoopPathSuite.scala │ │ │ ├── TokenLockJdbcSuite.scala │ │ │ └── TokenLockMongoDbSuite.scala │ │ ├── notify │ │ │ ├── NotificationTargetManagerSuite.scala │ │ │ ├── PipelineNotificationEmailSuite.scala │ │ │ ├── message │ │ │ │ ├── CustomException.scala │ │ │ │ ├── MessageBuilderHtmlSuite.scala │ │ │ │ └── TableBuilderHtmlSuite.scala │ │ │ └── pipeline │ │ │ │ ├── PipelineNotificationBuilderHtmlSuite.scala │ │ │ │ └── PipelineNotificationDirectorSuite.scala │ │ ├── reader │ │ │ ├── JdbcUrlSelectorImplSuite.scala │ │ │ ├── TableReaderJdbcNativeSuite.scala │ │ │ ├── TableReaderJdbcSuite.scala │ │ │ └── TableReaderSparkSuite.scala │ │ ├── runner │ │ │ ├── AppRunnerSuite.scala │ │ │ ├── InfoDateCalculatorSuite.scala │ │ │ ├── PipelineSparkSessionBuilderSuite.scala │ │ │ ├── jobrunner │ │ │ │ └── TaskRunnerMultithreadedSuite.scala │ │ │ ├── orchestrator │ │ │ │ ├── DependencyResolverSuite.scala │ │ │ │ └── OrchestratorSuite.scala │ │ │ ├── splitter │ │ │ │ ├── ScheduleStrategyIncrementalSuite.scala │ │ │ │ ├── ScheduleStrategySuite.scala │ │ │ │ └── ScheduleStrategyUtilsSuite.scala │ │ │ └── task │ │ │ │ ├── RunStatusSuite.scala │ │ │ │ ├── TaskRunnerBaseSuite.scala │ │ │ │ └── TaskRunnerMultithreadedSuite.scala │ │ ├── schedule │ │ │ └── ScheduleSuite.scala │ │ ├── sql │ │ │ ├── SqlColumnTypeSuite.scala │ │ │ ├── SqlGeneratorDatabricksSuite.scala │ │ │ ├── SqlGeneratorDb2Suite.scala │ │ │ ├── SqlGeneratorDenodoSuite.scala │ │ │ ├── SqlGeneratorGenericSuite.scala │ │ │ ├── SqlGeneratorHiveSuite.scala │ │ │ ├── SqlGeneratorHsqlSuite.scala │ │ │ ├── SqlGeneratorLoaderSuite.scala │ │ │ ├── SqlGeneratorMicrosoftSuite.scala │ │ │ ├── SqlGeneratorMySqlSuite.scala │ │ │ ├── SqlGeneratorOracleSuite.scala │ │ │ ├── SqlGeneratorPostgreSqlSuite.scala │ │ │ ├── SqlGeneratorSasSuite.scala │ │ │ └── dilects │ │ │ │ └── DenodoDialectSuite.scala │ │ └── utils │ │ │ ├── AlgorithmUtilsSuite.scala │ │ │ ├── BuildPropertyUtilsSuite.scala │ │ │ ├── CircularBufferSuite.scala │ │ │ ├── ClassLoaderUtilsSuite.scala │ │ │ ├── ConfigUtilsSuite.scala │ │ │ ├── CsvUtilsSuite.scala │ │ │ ├── DateUtilsSuite.scala │ │ │ ├── FsUtilsSuite.scala │ │ │ ├── JavaXConfigSuite.scala │ │ │ ├── JdbcNativeUtilsSuite.scala │ │ │ ├── JdbcSparkUtilsSuite.scala │ │ │ ├── JsonUtilsSuite.scala │ │ │ ├── JvmUtilsSuite.scala │ │ │ ├── LocalFsUtilsSuite.scala │ │ │ ├── MutableStackSuite.scala │ │ │ ├── ResourceUtilsSuite.scala │ │ │ ├── ScheduleUtilsSuite.scala │ │ │ ├── SparkUtilsSuite.scala │ │ │ ├── StringUtilsSuite.scala │ │ │ ├── ThreadUtilsSuite.scala │ │ │ ├── TimeUtilsSuite.scala │ │ │ ├── hive │ │ │ ├── HiveApiSuite.scala │ │ │ ├── HiveFormatSuite.scala │ │ │ ├── HiveHelperSparkCatalogSuite.scala │ │ │ ├── HiveHelperSqlSuite.scala │ │ │ ├── HiveHelperSuite.scala │ │ │ ├── QueryExecutorJdbcSuite.scala │ │ │ └── QueryExecutorSparkSuite.scala │ │ │ └── traverser │ │ │ └── FsTraverserLocalSuite.scala │ │ └── transformers │ │ ├── ConversionTransformerSuite.scala │ │ └── IdentityTransformerSuite.scala ├── examples │ ├── combined_example.sh │ ├── enceladus_single_config │ │ ├── daily_ingestion.conf │ │ ├── daily_ingestion.sh │ │ ├── daily_mainframe_ingestion.conf │ │ ├── weekly_ingestion.conf │ │ └── weekly_ingestion.sh │ ├── enceladus_sourcing │ │ ├── common.conf │ │ ├── daily_ingestion.conf │ │ ├── daily_ingestion.sh │ │ ├── daily_snapshot.conf │ │ ├── daily_snapshot.sh │ │ ├── metastore.conf │ │ ├── sinks.conf │ │ └── sources.conf │ ├── ingestion_pipeline │ │ └── ingestion_pipeline.conf │ └── jdbc_sourcing │ │ ├── common.conf │ │ ├── daily_ingestion.conf │ │ ├── daily_ingestion.sh │ │ ├── metastore.conf │ │ └── sources.conf ├── extras │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── reference.conf │ │ └── scala │ │ │ └── za │ │ │ └── co │ │ │ └── absa │ │ │ └── pramen │ │ │ └── extras │ │ │ ├── avro │ │ │ └── AvroUtils.scala │ │ │ ├── infofile │ │ │ └── InfoFileGeneration.scala │ │ │ ├── model │ │ │ └── InfoDateColumn.scala │ │ │ ├── notification │ │ │ ├── EcsNotificationTarget.scala │ │ │ ├── EcsPipelineNotificationTarget.scala │ │ │ ├── HyperdriveNotificationTarget.scala │ │ │ └── mq │ │ │ │ ├── SingleMessageProducer.scala │ │ │ │ └── SingleMessageProducerKafka.scala │ │ │ ├── query │ │ │ ├── QueryExecutor.scala │ │ │ └── QueryExecutorSpark.scala │ │ │ ├── sink │ │ │ ├── EnceladusConfig.scala │ │ │ ├── EnceladusSink.scala │ │ │ ├── EnceladusUtils.scala │ │ │ ├── InfoVersionStatus.scala │ │ │ ├── KafkaAvroSink.scala │ │ │ ├── StandardizationConfig.scala │ │ │ └── StandardizationSink.scala │ │ │ ├── utils │ │ │ ├── ConfigUtils.scala │ │ │ ├── FsUtils.scala │ │ │ ├── JsonUtils.scala │ │ │ ├── MainRunner.scala │ │ │ ├── PartitionUtils.scala │ │ │ ├── ResourceUtils.scala │ │ │ └── httpclient │ │ │ │ ├── HttpMethod.scala │ │ │ │ ├── SimpleHttpClient.scala │ │ │ │ ├── SimpleHttpRequest.scala │ │ │ │ ├── SimpleHttpResponse.scala │ │ │ │ └── impl │ │ │ │ ├── BasicHttpClient.scala │ │ │ │ ├── HttpDeleteWithBody.scala │ │ │ │ ├── HttpGetWithBody.scala │ │ │ │ └── RetryableHttpClient.scala │ │ │ └── writer │ │ │ ├── TableWriter.scala │ │ │ ├── TableWriterKafka.scala │ │ │ └── model │ │ │ ├── KafkaWriterConfig.scala │ │ │ └── NamingStrategy.scala │ │ └── test │ │ ├── resources │ │ ├── log4j.properties │ │ ├── log4j2.properties │ │ └── test │ │ │ ├── app_context.conf │ │ │ ├── config │ │ │ └── testconfig.conf │ │ │ ├── nestedDf1.json │ │ │ ├── nestedDf1_fixed_avro.json │ │ │ ├── nestedDf1_origin_avro.json │ │ │ ├── nestedMap_fixed_avro.json │ │ │ ├── nestedMap_origin_avro.json │ │ │ └── testResource.txt │ │ └── scala │ │ └── za │ │ └── co │ │ └── absa │ │ └── pramen │ │ └── extras │ │ ├── MetaTableDefFactory.scala │ │ ├── NestedDataFrameFactory.scala │ │ ├── TaskDefFactory.scala │ │ ├── TaskNotificationFactory.scala │ │ ├── base │ │ └── SparkTestBase.scala │ │ ├── fixtures │ │ ├── TempDirFixture.scala │ │ └── TextComparisonFixture.scala │ │ ├── mocks │ │ ├── AppMainMock.scala │ │ ├── AppMainSilentMock.scala │ │ ├── CloseableHttpResponseAdapterMock.scala │ │ ├── DummyKafkaConfigFactory.scala │ │ ├── DummyNamingStrategyFactory.scala │ │ ├── HttpClientSpy.scala │ │ ├── QueryExecutorMock.scala │ │ ├── QueryExecutorSpy.scala │ │ ├── ReaderSpy.scala │ │ ├── SimpleHttpClientSpy.scala │ │ ├── TestPrototypes.scala │ │ └── WriterSpy.scala │ │ ├── notification │ │ ├── EcsNotificationTargetSuite.scala │ │ ├── EcsPipelineNotificationTargetSuite.scala │ │ ├── HyperdriveNotificationTargetSuite.scala │ │ └── SingleMessageProducerSpy.scala │ │ ├── tests │ │ ├── avro │ │ │ └── AvroUtilsSuite.scala │ │ ├── builtin │ │ │ └── infofile │ │ │ │ └── InfoFileGenerationSuite.scala │ │ ├── sink │ │ │ ├── EnceladusConfigSuite.scala │ │ │ ├── EnceladusSinkSuite.scala │ │ │ ├── EnceladusUtilsSuite.scala │ │ │ ├── StandardizationConfigSuite.scala │ │ │ └── StandardizationSinkSuite.scala │ │ └── writer │ │ │ ├── TableWriterKafkaSuite.scala │ │ │ └── model │ │ │ ├── KafkaWriterConfigSuite.scala │ │ │ └── NamingStrategySuite.scala │ │ └── utils │ │ ├── ConfigUtilsSuite.scala │ │ ├── FsUtilsSuite.scala │ │ ├── JsonUtilsSuite.scala │ │ ├── MainRunnerSuite.scala │ │ ├── PartitionUtilsSuite.scala │ │ ├── ResourceUtilsSuite.scala │ │ └── httpclient │ │ ├── BasicHttpClientSuite.scala │ │ ├── HttpClientSpecialMethodsSuite.scala │ │ ├── HttpMethodSuite.scala │ │ ├── RetryableHttpClientSuite.scala │ │ └── SimpleHttpClientSuite.scala ├── pom.xml ├── project │ ├── BuildInfoTemplateSettings.scala │ ├── Dependencies.scala │ ├── Versions.scala │ ├── build.properties │ └── plugins.sbt ├── publish.sbt └── version.sbt └── resources ├── concepts.png ├── date_inference.png ├── file_based_sourcing.png ├── ops_jobs_tasks.png ├── pipeline_example.png └── run_diagram.png /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | /pramen/ @yruslan 2 | /pramen-py/ @yruslan 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Describe the bug 11 | A clear and concise description of what the bug is. 12 | 13 | ## Code and/or configuration snippet that caused the issue 14 | 15 | ```hocon 16 | pramen { 17 | } 18 | ``` 19 | 20 | ## Expected behavior 21 | A clear and concise description of what you expected to happen. 22 | 23 | ## Context 24 | - Pramen/pramen-py version: 25 | - Spark version: 26 | - Scala/Python version: 27 | - Operating system: 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Background 11 | A clear and concise description of where the limitation lies. 12 | 13 | ## Feature 14 | A description of the requested feature. 15 | 16 | ## Example [Optional] 17 | A simple example if applicable. 18 | 19 | ## Proposed Solution [Optional] 20 | Solution Ideas 21 | 1. 22 | 2. 23 | 3. 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question 4 | title: '' 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Background [Optional] 11 | A clear explanation of the reason for raising the question. 12 | This gives us a better understanding of your use cases and how we might accommodate them. 13 | 14 | ## Question 15 | A clear and concise inquiry 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # use glob syntax. 2 | syntax: glob 3 | *.ser 4 | *.class 5 | *~ 6 | *.bak 7 | *.old 8 | 9 | # eclipse conf file 10 | .settings 11 | .classpath 12 | .project 13 | .manager 14 | .scala_dependencies 15 | 16 | # idea 17 | .idea 18 | *.iml 19 | 20 | # building 21 | target 22 | build 23 | null 24 | tmp* 25 | temp* 26 | dist 27 | test-output 28 | build.log 29 | sbt.json 30 | 31 | # output not in target directory 32 | .cache* 33 | dependency-reduced-pom.xml 34 | spark-warehouse 35 | 36 | # Environment-specific configuration 37 | application.conf 38 | .gitsecret/keys/random_seed 39 | !*.secret 40 | -------------------------------------------------------------------------------- /LICENSES/headers/code_header: -------------------------------------------------------------------------------- 1 | Copyright 2022 ABSA Group Limited 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/.env.pypi: -------------------------------------------------------------------------------- 1 | # enable debug 2 | export PRAMENPY_DEBUG=True 3 | # app settings (needed to run the app) 4 | ## default metastore filesystem (available options are: hdfs, s3, local) 5 | ## default = "hdfs" 6 | export PRAMENPY_DEFAULT_FS="local" 7 | ## in case of Py4JJavaError execution will be retried this number of times 8 | ## default is 1 9 | export PRAMENPY_MAX_RETRIES_EXECUTE_TRANSFORMATION=1 10 | ## sparks related 11 | ### if not present, will be skipped 12 | # export PRAMENPY_SPARK_JAVA_HOME= 13 | ### if not present, will be skipped 14 | export PRAMENPY_SPARK_CONFIG=spark.driver.host=127.0.0.1 15 | -------------------------------------------------------------------------------- /pramen-py/.env.test: -------------------------------------------------------------------------------- 1 | # enable debug 2 | export PRAMENPY_DEBUG=False 3 | # app settings (needed to run the app) 4 | ## default metastore filesystem (available options are: hdfs, s3, local) 5 | ## default = "hdfs" 6 | export PRAMENPY_DEFAULT_FS="local" 7 | ## in case of Py4JJavaError execution will be retried this number of times 8 | ## default is 1 9 | export PRAMENPY_MAX_RETRIES_EXECUTE_TRANSFORMATION=1 10 | ## sparks related 11 | ### if not present, will be skipped 12 | #export PRAMENPY_SPARK_JAVA_HOME=/usr 13 | ### if not present, will be skipped 14 | export PRAMENPY_SPARK_CONFIG=spark.master=local,spark.driver.host=127.0.0.1,spark.driver.bindAddress=127.0.0.1,spark.executor.instances=1,spark.executor.cores=1,spark.ui.enabled=false 15 | -------------------------------------------------------------------------------- /pramen-py/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ban-relative-imports = true 3 | extend-ignore = 4 | E501, 5 | E203, 6 | E231, 7 | A003, 8 | N812, 9 | N802, 10 | N803, 11 | N806, 12 | inline-quotes = double 13 | max-complexity = 15 14 | multiline-quotes = double 15 | -------------------------------------------------------------------------------- /pramen-py/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ENV ?= local 16 | 17 | ifneq (,$(wildcard ./.env.$(ENV))) 18 | include .env.$(ENV) 19 | endif 20 | 21 | install: .poetry_installed 22 | 23 | .poetry_installed: 24 | poetry install 25 | touch $@ 26 | 27 | .env: 28 | ln -sfv .env.$(ENV) .env 29 | 30 | .pypi_token: 31 | poetry config pypi-token.pypi $(PRAMENPY_PYPI_TOKEN) 32 | 33 | build: install 34 | rm -frv dist 35 | poetry build 36 | 37 | test: install .env 38 | poetry run pytest --cov -n 1 39 | 40 | pre-commit: install 41 | poetry run pre-commit run --all-files 42 | 43 | publish: .env .pypi_token build 44 | poetry publish 45 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pramen_py.metastore.reader import MetastoreReader 16 | from pramen_py.metastore.writer import MetastoreWriter 17 | from pramen_py.transformation.transformation_base import ( 18 | T_EXTRA_OPTIONS, 19 | Transformation, 20 | ) 21 | 22 | 23 | __all__ = ( 24 | "MetastoreWriter", 25 | "MetastoreReader", 26 | "Transformation", 27 | "T_EXTRA_OPTIONS", 28 | ) 29 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/app/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from environs import Env 16 | 17 | from pramen_py.app.logger import setup_logger 18 | 19 | 20 | env = Env(expand_vars=True) 21 | try: 22 | env.read_env() 23 | except OSError: 24 | env.read_env(recurse=False) 25 | 26 | setup_logger(env=env) 27 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/app/logger.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | from typing import Optional 18 | 19 | from environs import Env 20 | from loguru import logger 21 | 22 | 23 | def setup_logger(level: str = "INFO", env: Optional[Env] = None) -> int: 24 | force_debug = env.bool("PRAMENPY_DEBUG", False) if env else False 25 | level = "DEBUG" if force_debug else level 26 | logger.remove() 27 | if level == "INFO": 28 | logger.disable("py4j") 29 | logger_id = logger.add( 30 | sink=sys.stderr, 31 | level=level, 32 | format="{time} | {level} | {name} | {extra} | {message}", 33 | ) 34 | logger.debug(f"Logger setup complete. Logger level is {level}.") 35 | return logger_id 36 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/metastore/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/pramen-py/src/pramen_py/py.typed -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/runner/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/test_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Any, cast 16 | from unittest import mock 17 | 18 | 19 | class AsyncMock: 20 | """Backport of AsyncMock to python3.6.""" 21 | 22 | def __init__(self, *args: Any, **kwargs: Any) -> None: 23 | self.mock = mock.Mock(*args, **kwargs) 24 | 25 | async def __call__(self, *args: Any, **kwargs: Any) -> mock.Mock: 26 | return cast(mock.Mock, self.mock(*args, **kwargs)) 27 | 28 | def __getattr__(self, item: Any) -> Any: 29 | return getattr(self.mock, item) 30 | 31 | 32 | mock.AsyncMock = cast(mock.Mock, AsyncMock) # type: ignore 33 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/test_utils/plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | pytest_plugins = [ 16 | "pramen_py.test_utils.fixtures", 17 | "pramen_py.test_utils.when", 18 | ] 19 | -------------------------------------------------------------------------------- /pramen-py/src/pramen_py/transformation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/resources/test_common.conf: -------------------------------------------------------------------------------- 1 | pramen { 2 | environment.name = "TestEnv" 3 | bookkeeping.enabled = true 4 | bookkeeping.jdbc { 5 | driver = "org.postgresql.Driver" 6 | url = "jdbc:postgresql://127.0.0.1:8080/newpipeline" 7 | user = "user" 8 | } 9 | warn.throughput.rps = 2000 10 | good.throughput.rps = 40000 11 | } 12 | mail { 13 | smtp.host = "127.0.0.1" 14 | send.from = "Pramen " 15 | send.to = "user@example.com" 16 | } 17 | sourcing.base.path = "/projects/batch/source" 18 | -------------------------------------------------------------------------------- /pramen-py/tests/test_app/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_metastore/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_metastore/test_reader/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_metastore/test_writer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_runner/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_runner/test_runner_transformation.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_test_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/tests/test_test_utils/test_when/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def some_normal_function( 17 | arg1: str, 18 | arg2: int, 19 | *, 20 | kwarg1: str, 21 | kwarg2: str, 22 | ) -> str: 23 | return "Not mocked" 24 | -------------------------------------------------------------------------------- /pramen-py/tests/test_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/transformations/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/transformations/example_trasformation_one/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/transformations/example_trasformation_two/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pramen-py/transformations/identity_transformer/example_config.yaml: -------------------------------------------------------------------------------- 1 | run_transformers: 2 | - info_date: 2022-02-14 3 | output_table: table_out1 4 | name: IdentityTransformer 5 | options: 6 | table: "cif" 7 | metastore_tables: 8 | - description: Table 1 description 9 | format: parquet 10 | info_date_settings: 11 | column: "enceladus_info_date" 12 | format: yyyy-MM-dd 13 | start: 2017-01-31 14 | name: "cif" 15 | path: "/bigdatahdfs/datalake/publish/cif/CISA521" 16 | records_per_partition: 1000000 17 | - description: Output table 18 | format: parquet 19 | info_date_settings: 20 | column: SYNCWATCHER_INFO_DATE 21 | format: yyyy-MM-dd 22 | start: 2017-01-29 23 | name: table_out1 24 | path: /tmp/syncwatcher-py/identity-transformer/ 25 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/CachePolicy.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | sealed trait CachePolicy { 20 | val name: String 21 | } 22 | 23 | object CachePolicy { 24 | case object Cache extends CachePolicy { 25 | override val name: String = "cache" 26 | } 27 | 28 | case object NoCache extends CachePolicy { 29 | override val name: String = "no_cache" 30 | } 31 | 32 | case object Persist extends CachePolicy { 33 | override val name: String = "persist" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/ExternalChannelFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import com.typesafe.config.Config 20 | import org.apache.spark.sql.SparkSession 21 | 22 | /** 23 | * Base interface for all Pramen source and sink factories. 24 | */ 25 | trait ExternalChannelFactory[+A <: ExternalChannel] { 26 | def apply(conf: Config, parentPath: String, spark: SparkSession): A 27 | } 28 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/ExternalChannelFactoryV2.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import com.typesafe.config.Config 20 | import org.apache.spark.sql.SparkSession 21 | 22 | /** 23 | * Base interface for all Pramen source and sink factories. 24 | */ 25 | trait ExternalChannelFactoryV2[+A <: ExternalChannel] { 26 | def apply(conf: Config, workflowConfig: Config, parentPath: String, spark: SparkSession): A 27 | } 28 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/FieldChange.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | sealed trait FieldChange 20 | 21 | object FieldChange { 22 | case class NewField(columnName: String, dataType: String) extends FieldChange 23 | 24 | case class DeletedField(columnName: String, dataType: String) extends FieldChange 25 | 26 | case class ChangedType(columnName: String, oldType: String, newType: String) extends FieldChange 27 | } 28 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/MetaTableRunInfo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import java.time.{Instant, LocalDate} 20 | 21 | case class MetaTableRunInfo(tableName: String, 22 | infoDate: LocalDate, 23 | inputRecordCount: Long, 24 | outputRecordCount: Long, 25 | jobStarted: Instant, 26 | jobFinished: Instant) 27 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/MetadataValue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import java.time.Instant 20 | 21 | /** 22 | * An entry of metastore metadata that can be set for a partition partition (table + info date). 23 | */ 24 | case class MetadataValue(value: String, lastUpdated: Instant) 25 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/NotificationTarget.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import za.co.absa.pramen.api.status.TaskResult 20 | 21 | trait NotificationTarget extends ExternalChannel { 22 | /** Sends a notification after completion of a task. */ 23 | def sendNotification(pipelineInfo: PipelineInfo, notification: TaskResult): Unit 24 | } 25 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/PartitionInfo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | trait PartitionInfo 20 | 21 | object PartitionInfo { 22 | case object Default extends PartitionInfo 23 | 24 | case class Explicit(numberOfPartitions: Int) extends PartitionInfo 25 | 26 | case class PerRecordCount(recordsPerPartition: Long) extends PartitionInfo 27 | 28 | } 29 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/PartitionScheme.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | sealed trait PartitionScheme 20 | 21 | object PartitionScheme { 22 | case object PartitionByDay extends PartitionScheme 23 | 24 | case class PartitionByMonth(generatedMonthColumn: String, generatedYearColumn: String) extends PartitionScheme 25 | 26 | case class PartitionByYearMonth(generatedMonthYearColumn: String) extends PartitionScheme 27 | 28 | case class PartitionByYear(generatedYearColumn: String) extends PartitionScheme 29 | 30 | case object NotPartitioned extends PartitionScheme 31 | } 32 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/PipelineNotificationTarget.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import za.co.absa.pramen.api.status.{CustomNotification, TaskResult} 20 | 21 | trait PipelineNotificationTarget extends ExternalChannel { 22 | /** Sends a notification after completion of the pipeline. */ 23 | def sendNotification(pipelineInfo: PipelineInfo, 24 | tasksCompleted: Seq[TaskResult], 25 | customNotification: CustomNotification): Unit 26 | } 27 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/SchemaDifference.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import java.time.LocalDate 20 | 21 | case class SchemaDifference( 22 | tableName: String, 23 | infoDateOld: LocalDate, 24 | infoDateNew: LocalDate, 25 | changes: List[FieldChange] 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/SinkResult.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | case class SinkResult( 20 | recordsSent: Long, 21 | filesSent: Seq[String] = Seq.empty[String], // The list of files written. Empty list for non-file based sources. 22 | hiveTables: Seq[String] = Seq.empty[String], // The list of Hive/Athena/Trino tables updated if any 23 | warnings: Seq[String] = Seq.empty[String] 24 | ) 25 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/SourceResult.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import org.apache.spark.sql.DataFrame 20 | 21 | case class SourceResult( 22 | data: DataFrame, 23 | filesRead: Seq[String] = Seq.empty[String], // The list of files to read. Empty list for non-file based sources. 24 | warnings: Seq[String] = Seq.empty[String] 25 | ) 26 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/TableReader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import org.apache.spark.sql.DataFrame 20 | import za.co.absa.pramen.api.offset.OffsetValue 21 | 22 | import java.time.LocalDate 23 | 24 | trait TableReader { 25 | def getRecordCount(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate): Long 26 | 27 | def getData(query: Query, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String]): DataFrame 28 | 29 | def getIncrementalData(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): DataFrame 30 | } 31 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/app/PramenFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.app 18 | 19 | import za.co.absa.pramen.api.Pramen 20 | 21 | /** 22 | * This is the factory used to get the globally accessible Pramen client. 23 | * 24 | * The instance is available in runtime only, and only when 'pramen-core' is in the classpath. 25 | */ 26 | trait PramenFactory { 27 | def instance: Pramen 28 | } 29 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/common/BuildPropertiesRetriever.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.common 18 | 19 | trait BuildPropertiesRetriever { 20 | def buildVersion: String 21 | 22 | def buildTimestamp: String 23 | 24 | def getFullVersion: String 25 | } 26 | 27 | object BuildPropertiesRetriever { 28 | // This call assumes 'pramen-core' is in the class path, but not a compile time dependency, but either 29 | // provided or runtime dependency. 30 | def apply(): BuildPropertiesRetriever = 31 | Class.forName("za.co.absa.pramen.core.utils.BuildPropertyUtils") 32 | .getConstructor() 33 | .newInstance() 34 | .asInstanceOf[BuildPropertiesRetriever] 35 | } 36 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/jobdef/TransformExpression.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.jobdef 18 | 19 | case class TransformExpression( 20 | column: String, 21 | expression: Option[String], 22 | comment: Option[String] 23 | ) 24 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/notification/Align.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.notification 18 | 19 | sealed trait Align 20 | 21 | object Align { 22 | case object Left extends Align 23 | case object Center extends Align 24 | case object Right extends Align 25 | } 26 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/notification/Style.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.notification 18 | 19 | sealed trait Style 20 | 21 | object Style { 22 | case object Normal extends Style 23 | 24 | case object Bold extends Style 25 | 26 | case object Italic extends Style 27 | 28 | case object Success extends Style 29 | 30 | case object Warning extends Style 31 | 32 | case object Exception extends Style 33 | 34 | case object Error extends Style 35 | } 36 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/notification/TableHeader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.notification 18 | 19 | case class TableHeader(textElement: TextElement, align: Align = Align.Left) 20 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/notification/TextElement.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.notification 18 | 19 | case class TextElement(text: String, style: Style = Style.Normal) 20 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/offset/OffsetInfo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.offset 18 | 19 | case class OffsetInfo(offsetColumn: String, offsetType: OffsetType) 20 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.sql 18 | 19 | import com.typesafe.config.Config 20 | import za.co.absa.pramen.api.offset.OffsetInfo 21 | 22 | import java.time.ZoneId 23 | 24 | case class SqlConfig( 25 | infoDateColumn: String, 26 | infoDateType: SqlColumnType, 27 | dateFormatApp: String, 28 | offsetInfo: Option[OffsetInfo], 29 | serverTimeZone: ZoneId, 30 | identifierQuotingPolicy: QuotingPolicy, 31 | sqlGeneratorClass: Option[String], 32 | extraConfig: Config 33 | ) 34 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/CustomNotification.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | import za.co.absa.pramen.api.notification.{NotificationEntry, TextElement} 20 | 21 | case class CustomNotification( 22 | customEntries: Seq[NotificationEntry], 23 | customSignature: Seq[TextElement] 24 | ) 25 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/DependencyWarning.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | case class DependencyWarning( 20 | table: String 21 | ) 22 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/MetastoreDependency.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | case class MetastoreDependency( 20 | tables: Seq[String], 21 | dateFromExpr: String, 22 | dateUntilExpr: Option[String], 23 | triggerUpdates: Boolean, 24 | isOptional: Boolean, 25 | isPassive: Boolean 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/NotificationFailure.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | import java.time.LocalDate 20 | 21 | case class NotificationFailure( 22 | table: String, 23 | notificationTarget: String, 24 | infoDate: LocalDate, 25 | ex: Throwable 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/PipelineNotificationFailure.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | case class PipelineNotificationFailure( 20 | notificationTarget: String, 21 | ex: Throwable 22 | ) 23 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/RunInfo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | import java.time.{Instant, LocalDate} 20 | 21 | case class RunInfo( 22 | infoDate: LocalDate, 23 | started: Instant, 24 | finished: Instant 25 | ) 26 | -------------------------------------------------------------------------------- /pramen/api/src/main/scala/za/co/absa/pramen/api/status/TaskDef.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.status 18 | 19 | import com.typesafe.config.Config 20 | import za.co.absa.pramen.api.MetaTableDef 21 | import za.co.absa.pramen.api.jobdef.Schedule 22 | 23 | case class TaskDef( 24 | name: String, 25 | jobType: JobType, 26 | outputTable: MetaTableDef, 27 | schedule: Schedule, 28 | operationConf: Config 29 | ) 30 | -------------------------------------------------------------------------------- /pramen/api/src/test/scala/za/co/absa/pramen/api/PramenSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api 18 | 19 | import org.scalatest.wordspec.AnyWordSpec 20 | 21 | import scala.util.Try 22 | 23 | class PramenSuite extends AnyWordSpec { 24 | "Pramen.instance" should { 25 | "handle the situation when pramen-core is not in the classpath" in { 26 | val pramenOpt = Try { 27 | Pramen.instance 28 | }.toOption 29 | 30 | assert(pramenOpt.isEmpty) 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /pramen/api/src/test/scala/za/co/absa/pramen/api/mocks/DummyEmptySingletonClass.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.mocks 18 | 19 | object DummyEmptySingletonClass extends DummySingletonTrait 20 | -------------------------------------------------------------------------------- /pramen/api/src/test/scala/za/co/absa/pramen/api/mocks/DummySingletonClass.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.mocks 18 | 19 | class DummySingletonClass(val dummyParam: String) extends DummySingletonTrait 20 | 21 | object DummySingletonClass extends DummySingletonFactory[DummySingletonTrait] { 22 | override def apply(dummyParam: String): DummySingletonTrait = new DummySingletonClass(dummyParam) 23 | } 24 | -------------------------------------------------------------------------------- /pramen/api/src/test/scala/za/co/absa/pramen/api/mocks/DummySingletonFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.mocks 18 | 19 | trait DummySingletonFactory[+A <: DummySingletonTrait] { 20 | def apply(dummyParam: String): A 21 | } 22 | -------------------------------------------------------------------------------- /pramen/api/src/test/scala/za/co/absa/pramen/api/mocks/DummySingletonTrait.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.api.mocks 18 | 19 | trait DummySingletonTrait -------------------------------------------------------------------------------- /pramen/core/src/main/resources/pramen_banner.txt: -------------------------------------------------------------------------------- 1 | _____ 2 | | __ \ 3 | | |__) | __ __ _ _ __ ___ ___ _ __ 4 | | ___/ '__/ _` | '_ ` _ \ / _ \ '_ \ 5 | | | | | | (_| | | | | | | __/ | | | 6 | |_| |_| \__,_|_| |_| |_|\___|_| |_| 7 | 8 | version project_version 9 | -------------------------------------------------------------------------------- /pramen/core/src/main/resources/pramen_build.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | build.version=${project.version} 15 | build.timestamp=${build.timestamp} 16 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/app/AppContext.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.app 18 | 19 | import za.co.absa.pramen.core.bookkeeper.Bookkeeper 20 | import za.co.absa.pramen.core.journal.Journal 21 | import za.co.absa.pramen.core.lock.TokenLockFactory 22 | import za.co.absa.pramen.core.metastore.Metastore 23 | 24 | trait AppContext { 25 | val appConfig: AppConfig 26 | 27 | def bookkeeper: Bookkeeper 28 | 29 | def tokenLockFactory: TokenLockFactory 30 | 31 | def journal: Journal 32 | 33 | def metastore: Metastore 34 | 35 | def close(): Unit 36 | } 37 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/app/config/HadoopFormat.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.app.config 18 | 19 | trait HadoopFormat 20 | 21 | case object HadoopFormat { 22 | case object Text extends HadoopFormat 23 | case object Delta extends HadoopFormat 24 | 25 | def apply(format: String): HadoopFormat = format.toLowerCase match { 26 | case "text" => Text 27 | case "delta" => Delta 28 | case _ => throw new IllegalArgumentException(s"Unknown Hadoop format: $format") 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/BookkeepingRecord.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | case class BookkeepingRecord(pramenTableName: String, 20 | infoDate: String, /* Use String to workaround serialization issues */ 21 | infoDateBegin: String, 22 | infoDateEnd: String, 23 | inputRecordCount: Long, 24 | outputRecordCount: Long, 25 | jobStarted: Long, 26 | jobFinished: Long) 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetAggregated.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | import za.co.absa.pramen.api.offset.{DataOffset, OffsetValue} 20 | 21 | import java.time.LocalDate 22 | 23 | case class DataOffsetAggregated( 24 | tableName: String, 25 | maximumInfoDate: LocalDate, 26 | minimumOffset: OffsetValue, 27 | maximumOffset: OffsetValue, 28 | offsetsForTheDay: Array[DataOffset] 29 | ) 30 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/DataOffsetRequest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | import java.time.{Instant, LocalDate} 20 | 21 | case class DataOffsetRequest( 22 | tableName: String, 23 | infoDate: LocalDate, 24 | batchId: Long, 25 | createdAt: Instant 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/MetadataRecord.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | case class MetadataRecord(pramenTableName: String, 20 | infoDate: String, /* Use String to workaround serialization issues */ 21 | key: String, 22 | value: String, 23 | lastUpdated: Long /* Use epoch seconds */ 24 | ) 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/OffsetCommitRequest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | import za.co.absa.pramen.api.offset.OffsetValue 20 | 21 | import java.time.{Instant, LocalDate} 22 | 23 | case class OffsetCommitRequest( 24 | table: String, 25 | infoDate: LocalDate, 26 | minOffset: OffsetValue, 27 | maxOffset: OffsetValue, 28 | createdAt: Instant 29 | ) 30 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/SchemaRecord.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | case class SchemaRecord(pramenTableName: String, 20 | infoDate: String, /* Use String to workaround serialization issues */ 21 | schemaJson: String) 22 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/bookkeeper/model/TableSchemaJson.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.bookkeeper.model 18 | 19 | /* This class is needed since Json format used for storing schema changes is immutable so it needs the timestamp 20 | * to apply last-write-wins policy*/ 21 | case class TableSchemaJson(tableName: String, 22 | infoDate: String, /* Use String to workaround serialization issues */ 23 | schemaJson: String, 24 | updatedTimestamp: Long 25 | ) 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/dao/model/DbVersion.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.dao.model 18 | 19 | case class DbVersion (version: Int) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/dao/model/Index.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.dao.model 18 | 19 | case class Index(collection: String, key: Seq[IndexField], unique: Boolean = false, sparse: Boolean = false) 20 | 21 | case class IndexField(field: String, sort: Sort) { 22 | 23 | def toPair: (String, Int) = field -> sort.order 24 | 25 | override def toString: String = s"($field: $sort)" 26 | 27 | } 28 | 29 | sealed abstract class Sort(val order: Int) 30 | 31 | case object ASC extends Sort(1) { 32 | override def toString: String = "ASC" 33 | } 34 | 35 | case object DESC extends Sort(-1) { 36 | override def toString: String = "DESC" 37 | } 38 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/databricks/Responses.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.databricks 18 | 19 | object Responses { 20 | case class RunJobResponse(run_id: Long) 21 | 22 | case class RunState(life_cycle_state: String) 23 | 24 | case class RunStatusResponse(run_id: String, run_name: String, run_page_url: String, state: RunState) { 25 | def isJobPending: Boolean = { 26 | state.life_cycle_state != "TERMINATED" && state.life_cycle_state != "INTERNAL_ERROR" 27 | } 28 | 29 | def isFailure: Boolean = { 30 | state.life_cycle_state == "INTERNAL_ERROR" 31 | } 32 | 33 | def pretty = s"Job '$run_name' - ($run_page_url). State: ${state.life_cycle_state}" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/CmdFailedException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | case class CmdFailedException(msg: String, log: Array[String]) extends RuntimeException(msg) 20 | 21 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/FatalErrorWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | /** 20 | * This is the wrapper of fatal errors occurring in a thread so that the error doesn't kill the thread. 21 | * This exception is returned from the thread feature, propagated to the bottom ot the stack trace, and 22 | * is handled properly there. 23 | */ 24 | class FatalErrorWrapper(val msg: String, val cause: Throwable = null) extends RuntimeException(msg, cause) 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/OsSignalException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | case class OsSignalException(signalName: String, 20 | threadStackTraces: Seq[ThreadStackTrace]) extends RuntimeException(s"The process was interrupted by $signalName.") 21 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/ProcessFailedException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | case class ProcessFailedException(msg: String, stdout: Array[String], stderr: Array[String]) extends RuntimeException(msg) 20 | 21 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/ReasonException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | import za.co.absa.pramen.api.Reason 20 | 21 | class ReasonException(val reason: Reason, reasonMessage: String) extends RuntimeException(reasonMessage) 22 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/ThreadStackTrace.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | case class ThreadStackTrace(threadName: String, stackTrace: Array[StackTraceElement]) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/TimeoutException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | class TimeoutException(val msg: String, val cause: Throwable = null) extends RuntimeException(msg, cause) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/exceptions/ValidationException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.exceptions 18 | 19 | class ValidationException(msg: String, cause: Throwable = null) extends Exception(msg, cause) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/expr/exceptions/SyntaxErrorException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.expr.exceptions 18 | 19 | class SyntaxErrorException(message: String, cause: Throwable = null) 20 | extends RuntimeException(message, cause) { 21 | } 22 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/expr/parser/DateExprBuilder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.expr.parser 18 | 19 | import java.time.LocalDate 20 | 21 | trait DateExprBuilder { 22 | def openParen(pos: Int): Unit 23 | def closeParen(pos: Int): Unit 24 | def addOperationPlus(pos: Int): Unit 25 | def addOperationMinus(pos: Int): Unit 26 | 27 | def addVariable(name: String, pos: Int): Unit 28 | def addFunction(name: String, pos: Int): Unit 29 | def addNumLiteral(num: Int, pos: Int): Unit 30 | def addDateLiteral(date: LocalDate, pos: Int): Unit 31 | } 32 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/journal/Journal.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.journal 18 | 19 | import za.co.absa.pramen.core.journal.model.TaskCompleted 20 | 21 | import java.time.Instant 22 | 23 | /** 24 | * A journal is responsible of keeping track of all completed tasks. 25 | */ 26 | trait Journal { 27 | 28 | def addEntry(entry: TaskCompleted): Unit 29 | 30 | def getEntries(from: Instant, to: Instant): Seq[TaskCompleted] 31 | 32 | } 33 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/journal/JournalNull.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.journal 18 | 19 | import za.co.absa.pramen.core.journal.model.TaskCompleted 20 | import java.time.Instant 21 | 22 | /** 23 | * The implementation of journal that does nothing. The idea is as follows. When bookkeeping is disabled, 24 | * this journal implementation is used. It just does nothing. 25 | */ 26 | class JournalNull extends Journal { 27 | override def addEntry(entry: TaskCompleted): Unit = {} 28 | 29 | override def getEntries(from: Instant, to: Instant): Seq[TaskCompleted] = Nil 30 | } 31 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/TokenLockAllow.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock 18 | 19 | class TokenLockAllow extends TokenLock { 20 | override def tryAcquire(): Boolean = true 21 | 22 | override def release(): Unit = {} 23 | 24 | override def close(): Unit = {} 25 | } 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/TokenLockFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock 18 | 19 | trait TokenLockFactory { 20 | def getLock(token: String): TokenLock 21 | } 22 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/TokenLockFactoryAllow.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock 18 | 19 | class TokenLockFactoryAllow extends TokenLockFactory { 20 | override def getLock(token: String): TokenLock = { 21 | new TokenLockAllow 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/TokenLockFactoryHadoopPath.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock 18 | 19 | import org.apache.hadoop.conf.Configuration 20 | 21 | class TokenLockFactoryHadoopPath(hadoopConf: Configuration, lockLocation: String) extends TokenLockFactory { 22 | override def getLock(token: String): TokenLock = { 23 | new TokenLockHadoopPath(token, hadoopConf, lockLocation) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/TokenLockFactoryJdbc.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock 18 | 19 | import slick.jdbc.H2Profile.api._ 20 | 21 | class TokenLockFactoryJdbc(db: Database) extends TokenLockFactory { 22 | 23 | override def getLock(token: String): TokenLock = { 24 | new TokenLockJdbc(token, db) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/TokenLockFactoryMongoDb.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock 18 | 19 | import za.co.absa.pramen.core.mongo.MongoDbConnection 20 | 21 | class TokenLockFactoryMongoDb(mongoDbConnection: MongoDbConnection) extends TokenLockFactory { 22 | override def getLock(token: String): TokenLock = { 23 | new TokenLockMongoDb(token, mongoDbConnection) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/model/LockTicket.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock.model 18 | 19 | case class LockTicket(token: String, 20 | owner: String, 21 | expires: Long) 22 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/lock/model/LockTickets.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.lock.model 18 | 19 | import slick.jdbc.H2Profile.api._ 20 | import slick.lifted.TableQuery 21 | 22 | class LockTickets(tag: Tag) extends Table[LockTicket](tag, "lock_tickets") { 23 | def token = column[String]("token", O.PrimaryKey, O.Length(255)) 24 | def owner = column[String]("owner", O.Length(255)) 25 | def expires = column[Long]("expires") 26 | def * = (token, owner, expires) <> (LockTicket.tupled, LockTicket.unapply) 27 | } 28 | 29 | object LockTickets { 30 | lazy val lockTickets = TableQuery[LockTickets] 31 | } 32 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metadata/MetadataTableKey.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metadata 18 | 19 | import java.time.LocalDate 20 | 21 | case class MetadataTableKey(tableName: String, infoDate: String) 22 | 23 | object MetadataTableKey { 24 | def apply(tableName: String, infoDate: LocalDate): MetadataTableKey = MetadataTableKey(tableName.toLowerCase, infoDate.toString) 25 | } -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetaTableStats.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore 18 | 19 | case class MetaTableStats( 20 | recordCount: Option[Long], 21 | recordCountAppended: Option[Long], 22 | dataSizeBytes: Option[Long] 23 | ) 24 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/MetastoreReaderIncremental.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore 18 | 19 | import za.co.absa.pramen.api.MetastoreReader 20 | 21 | trait MetastoreReaderIncremental extends MetastoreReader { 22 | def commitIncrementalOutputTable(tableName: String, trackingName: String): Unit 23 | 24 | def commitIncrementalStage(): Unit 25 | } 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/NoDataInTable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore 18 | 19 | class NoDataInTable(msg: String, cause: Throwable = null) extends RuntimeException(msg, cause) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/NoSuchTable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore 18 | 19 | class NoSuchTable(msg: String, cause: Throwable = null) extends RuntimeException(msg, cause) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/TableNotConfigured.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore 18 | 19 | class TableNotConfigured(msg: String, cause: Throwable = null) extends RuntimeException(msg, cause) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/model/HiveApi.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore.model 18 | 19 | sealed trait HiveApi 20 | 21 | object HiveApi { 22 | case object Sql extends HiveApi 23 | case object SparkCatalog extends HiveApi 24 | 25 | def fromString(s: String): HiveApi = s.toLowerCase() match { 26 | case "sql" => Sql 27 | case "spark_catalog" => SparkCatalog 28 | case _ => throw new IllegalArgumentException(s"Unknown Hive API config: '$s'. Only 'sql' and 'spark_catalog' are supported.") 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/model/ReaderMode.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore.model 18 | 19 | trait ReaderMode 20 | 21 | object ReaderMode { 22 | case object Batch extends ReaderMode 23 | 24 | case object IncrementalValidation extends ReaderMode 25 | 26 | case object IncrementalRun extends ReaderMode 27 | 28 | case object IncrementalPostProcessing extends ReaderMode 29 | } 30 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/metastore/peristence/MetastorePartition.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.metastore.peristence 18 | 19 | case class MetastorePartition(tableName: String, infoDateStr: String) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/model/SyncStatus.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.model 18 | 19 | case class SyncStatus( 20 | ingestionStatus: Map[String, TableStatus] = Map() 21 | ) 22 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/model/TableChunk.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.model 18 | 19 | case class TableChunk( 20 | tableName: String, 21 | dataChunk: DataChunk 22 | ) 23 | 24 | 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/model/TableStatus.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.model 18 | 19 | case class TableStatus(processedChunks: List[DataChunk]) 20 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/model/TableUpdates.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.model 18 | 19 | import java.time.ZonedDateTime 20 | 21 | case class TableUpdates ( 22 | outputTableLanded: ZonedDateTime, 23 | inputTablesLanded: Seq[(String, ZonedDateTime)] 24 | ) 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/notify/message/TableBuilder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.notify.message 18 | 19 | import za.co.absa.pramen.api.notification.{TableHeader, TextElement} 20 | 21 | trait TableBuilder { 22 | def withHeaders(headers: Seq[TableHeader]): TableBuilder 23 | 24 | def withRow(row: Iterable[TextElement]): TableBuilder 25 | 26 | def renderTable: String 27 | } 28 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineStatus.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.notify.pipeline 18 | 19 | sealed trait PipelineStatus 20 | 21 | object PipelineStatus { 22 | case object Success extends PipelineStatus 23 | case object Warning extends PipelineStatus 24 | case object PartialSuccess extends PipelineStatus 25 | case object Failure extends PipelineStatus 26 | } 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/notify/pipeline/ValidatedEmails.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.notify.pipeline 18 | 19 | case class ValidatedEmails( 20 | validEmails: Seq[String], 21 | invalidFormatEmails: Seq[String], 22 | invalidDomainEmails: Seq[String] 23 | ) 24 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobDependency.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | /** 20 | * Each task is a transformation of one or more input tables into one output table. 21 | */ 22 | case class JobDependency( 23 | inputTables: Seq[String], 24 | outputTable: String 25 | ) { 26 | lazy val nonRecursiveInputTables: Seq[String] = inputTables.filterNot(inputTable => inputTable == outputTable) 27 | } 28 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobNotificationTarget.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | import za.co.absa.pramen.api.NotificationTarget 20 | 21 | case class JobNotificationTarget( 22 | name: String, 23 | options: Map[String, String], 24 | target: NotificationTarget 25 | ) 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/JobPreRunResult.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | import za.co.absa.pramen.api.status.DependencyWarning 20 | 21 | case class JobPreRunResult( 22 | status: JobPreRunStatus, 23 | inputRecordsCount: Option[Long], 24 | dependencyWarnings: Seq[DependencyWarning], 25 | warnings: Seq[String] 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/PramenPyCmdConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | case class PramenPyCmdConfig( 20 | location: String, 21 | executable: String, 22 | cmdLineTemplate: String 23 | ) 24 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/RunResult.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | import org.apache.spark.sql.DataFrame 20 | 21 | case class RunResult( 22 | data: DataFrame, 23 | filesRead: Seq[String] = Seq.empty[String], // The list of files read. Empty list for non-file based sources. 24 | warnings: Seq[String] = Seq.empty[String] 25 | ) 26 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/SaveResult.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | import za.co.absa.pramen.core.metastore.MetaTableStats 20 | 21 | case class SaveResult( 22 | stats: MetaTableStats, 23 | filesSent: Seq[String] = Seq.empty[String], // The list of files written. Empty list for non-file based sources. 24 | hiveTablesUpdates: Seq[String] = Seq.empty, // The list of Hive/Athena/Trino tables updated if any 25 | warnings: Seq[String] = Seq.empty[String] 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/Task.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | import za.co.absa.pramen.api.status.TaskRunReason 20 | 21 | import java.time.LocalDate 22 | 23 | case class Task( 24 | job: Job, 25 | infoDate: LocalDate, 26 | reason: TaskRunReason 27 | ) 28 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/pipeline/TaskPreDef.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.pipeline 18 | 19 | import za.co.absa.pramen.api.status.TaskRunReason 20 | 21 | import java.time.LocalDate 22 | 23 | case class TaskPreDef( 24 | infoDate: LocalDate, 25 | reason: TaskRunReason 26 | ) 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/rdb/DbVersion.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.rdb 18 | 19 | case class DbVersion (version: Int) -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/runner/jobrunner/ConcurrentJobRunner.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.runner.jobrunner 18 | 19 | import com.github.yruslan.channel.ReadChannel 20 | import za.co.absa.pramen.api.status.TaskResult 21 | import za.co.absa.pramen.core.pipeline.Job 22 | 23 | trait ConcurrentJobRunner { 24 | def startWorkerLoop(incomingJobs: ReadChannel[Job]): Unit 25 | 26 | def shutdown(): Unit 27 | 28 | def getCompletedJobsChannel: ReadChannel[ConcurrentJobRunner.JobRunResults] 29 | } 30 | 31 | object ConcurrentJobRunner { 32 | type JobRunResults = (Job, Seq[TaskResult], Boolean) 33 | } 34 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/runner/task/TaskRunner.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.runner.task 18 | 19 | import za.co.absa.pramen.api.status.RunStatus 20 | import za.co.absa.pramen.core.pipeline.{Job, TaskPreDef} 21 | 22 | import java.time.LocalDate 23 | import scala.concurrent.Future 24 | 25 | trait TaskRunner extends AutoCloseable { 26 | /** Run a job for specified information dates as a part of pipeline execution. */ 27 | def runJobTasks(job: Job, infoDates: Seq[TaskPreDef]): Future[Seq[RunStatus]] 28 | 29 | /** Run a job for specified information date when requested from another job. */ 30 | def runLazyTask(job: Job, infoDate: LocalDate): RunStatus 31 | } 32 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/sink/SparkSinkFormat.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.sink 18 | 19 | import org.apache.hadoop.fs.Path 20 | 21 | sealed trait SparkSinkFormat 22 | 23 | object SparkSinkFormat { 24 | case class PathFormat(path: Path) extends SparkSinkFormat { 25 | override def toString = s"path: $path" 26 | } 27 | case class TableFormat(table: String) extends SparkSinkFormat { 28 | override def toString = s"table: $table" 29 | } 30 | 31 | case class ConnectionFormat(connectionType: String) extends SparkSinkFormat { 32 | override def toString = s"the $connectionType connection" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/state/PipelineState.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.state 18 | 19 | import za.co.absa.pramen.api.status.{PipelineStateSnapshot, TaskResult} 20 | 21 | trait PipelineState extends AutoCloseable { 22 | def getState: PipelineStateSnapshot 23 | 24 | def getBatchId: Long 25 | 26 | def setShutdownHookCanRun(): Unit 27 | 28 | def setSuccess(): Unit 29 | 30 | def setWarningFlag(): Unit 31 | 32 | def setFailure(stage: String, exception: Throwable): Unit 33 | 34 | def setSparkAppId(sparkAppId: String): Unit 35 | 36 | def addTaskCompletion(statuses: Seq[TaskResult]): Unit 37 | 38 | def getExitCode: Int 39 | } 40 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/Emoji.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils 18 | 19 | object Emoji { 20 | val SUCCESS = "\u2705" 21 | val WARNING = "\u26A0\uFE0F" 22 | val FAILURE = "\u274C" 23 | val EXCLAMATION = s"\u2757" 24 | val WRENCH = "\uD83D\uDD27" 25 | val PARALLEL = "\u29B7" 26 | 27 | val ROCKET = "\uD83D\uDE80" 28 | val EMAIL1 = "\uD83D\uDCE7" 29 | val EMAIL2 = "\uD83D\uDCE9" 30 | val EMAIL3 = "\u2709" 31 | val LIGHT_BULB = "\uD83D\uDCA1" 32 | val STAR = "\u2B50" 33 | val VOLTAGE = "\u26A1" 34 | } 35 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/FutureImplicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils 18 | 19 | import java.util.concurrent.TimeUnit 20 | 21 | import scala.concurrent.duration.Duration 22 | import scala.concurrent.{Await, Future} 23 | 24 | object FutureImplicits { 25 | private val executionTimeout = Duration(300, TimeUnit.SECONDS) 26 | 27 | implicit class FutureExecutor[T](future: Future[T]) { 28 | def execute(): T = Await.result(future, executionTimeout) 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/ResourceUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils 18 | 19 | import org.apache.commons.io.IOUtils 20 | 21 | object ResourceUtils { 22 | def getResourceString(name: String): String = 23 | IOUtils.toString(getClass.getResourceAsStream(name), "UTF-8") 24 | } 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/hive/QueryExecutor.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils.hive 18 | 19 | trait QueryExecutor extends AutoCloseable { 20 | def doesTableExist(dbName: Option[String], tableName: String): Boolean 21 | 22 | def execute(query: String): Unit 23 | } 24 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/impl/JdbcFieldMetadata.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils.impl 18 | 19 | case class JdbcFieldMetadata( 20 | name: String, 21 | label: String, 22 | sqlType: Int, 23 | sqlTypeName: String, 24 | displaySize: Int, 25 | precision: Int, 26 | scale: Int, 27 | nullable: Boolean 28 | ) 29 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/impl/ThreadWithException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils.impl 18 | 19 | class ThreadWithException extends Thread { 20 | private var threadException: Option[Throwable] = None 21 | 22 | def setException(ex: Throwable): Unit = synchronized { threadException = Option(ex) } 23 | 24 | def getException: Option[Throwable] = synchronized { 25 | val ex = threadException 26 | ex 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/traverser/FileStatus.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils.traverser 18 | 19 | import java.io.File 20 | 21 | case class FileStatus( 22 | path: File, 23 | size: Long 24 | ) 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/utils/traverser/FsTraverser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.utils.traverser 18 | 19 | trait FsTraverser { 20 | def traverse(path: String, 21 | fileMask: String, 22 | isRecursive: Boolean, 23 | includeHiddenFiles: Boolean)(action: FileStatus => Unit): Unit 24 | } 25 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/validator/RunDecision.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.validator 18 | 19 | sealed trait RunDecision 20 | 21 | object RunDecision { 22 | case object SkipUpToDate extends RunDecision 23 | case object SkipNoData extends RunDecision 24 | case object RunNew extends RunDecision 25 | case object RunUpdates extends RunDecision 26 | } 27 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/validator/RunDecisionSummary.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.validator 18 | 19 | import java.time.ZonedDateTime 20 | 21 | case class RunDecisionSummary( 22 | decision: RunDecision, 23 | inputRecordCount: Long, 24 | inputRecordCountOld: Option[Long], 25 | outputRecordCountOld: Option[Long], 26 | outputTableLastUpdated: Option[ZonedDateTime], 27 | updatedTables: Seq[(String, ZonedDateTime)], 28 | noDataTables: Seq[String] 29 | ) 30 | -------------------------------------------------------------------------------- /pramen/core/src/main/scala/za/co/absa/pramen/core/validator/SyncJobValidator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.validator 18 | 19 | import java.time.LocalDate 20 | 21 | trait SyncJobValidator { 22 | def isEmpty: Boolean 23 | 24 | def validateTask(infoDateBegin: LocalDate, 25 | infoDateEnd: LocalDate, 26 | infoDateOutput: LocalDate): Unit 27 | 28 | def decideRunTask(infoDateBegin: LocalDate, 29 | infoDateEnd: LocalDate, 30 | infoDateOutput: LocalDate, 31 | outputTableName: String, 32 | forceRun: Boolean): RunDecisionSummary 33 | 34 | } 35 | -------------------------------------------------------------------------------- /pramen/core/src/test/resources/reference.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | # Disable Hive support in tests to avoid adding Hive dependencies to the project if not necessary. 16 | enable.hive = false 17 | } 18 | -------------------------------------------------------------------------------- /pramen/core/src/test/resources/test/config/app_context.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | bookkeeping.enabled = false 16 | 17 | pipeline { 18 | name = "dummy_pipeline" 19 | } 20 | 21 | stop.spark.session = false 22 | } 23 | -------------------------------------------------------------------------------- /pramen/core/src/test/resources/test/config/pipeline_v2_empty.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | bookkeeping.enabled = false 16 | allow.empty.pipeline = true 17 | } 18 | 19 | pramen.metastore { 20 | tables = [ ] 21 | } 22 | 23 | pramen.pipeline { 24 | name = "pipeline_v2" 25 | } 26 | 27 | pramen.operations = [ ] 28 | 29 | pramen.hook = { 30 | startup.class = "za.co.absa.pramen.core.PramenHookSpy" 31 | shutdown.class = "za.co.absa.pramen.core.PramenHookSpy" 32 | } 33 | -------------------------------------------------------------------------------- /pramen/core/src/test/resources/test/config/test_javax.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | javax.net.ssl.trustStore = "test2.tmp" 15 | javax.net.ssl.trustStorePassword = "TestPass1" 16 | javax.net.ssl.keyStore = "test2.tmp" 17 | javax.net.ssl.keyStorePassword = "TestPass2" 18 | javax.net.ssl.password = "TestPass3" 19 | java.security.auth.login.config = "test2.tmp" 20 | java.security.krb5.conf = "TestKrbConf" 21 | javax.net.debug = "TestDebug" 22 | -------------------------------------------------------------------------------- /pramen/core/src/test/resources/test/testResource.txt: -------------------------------------------------------------------------------- 1 | Hello -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/GeneralConfigFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core 18 | 19 | import za.co.absa.pramen.core.app.config.GeneralConfig 20 | 21 | import java.time.ZoneId 22 | 23 | object GeneralConfigFactory { 24 | def getDummyGeneralConfig(timezoneId: ZoneId = ZoneId.of("Africa/Johannesburg"), 25 | environmentName: String = "TEST", 26 | temporaryDirectory: Option[String] = None, 27 | enableMultipleJobsPerTable: Boolean = false): GeneralConfig = { 28 | GeneralConfig( 29 | timezoneId, 30 | environmentName, 31 | temporaryDirectory, 32 | enableMultipleJobsPerTable 33 | ) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/PramenHookSpy.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core 18 | 19 | import com.typesafe.config.Config 20 | 21 | class PramenHookSpy(val conf: Config) extends Runnable { 22 | var runCallCount = 0 23 | 24 | override def run(): Unit = { 25 | runCallCount += 1 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummyEmptySingletonClass.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | object DummyEmptySingletonClass extends DummySingletonTrait 20 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySingletonClass.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | class DummySingletonClass(val dummyParam: String) extends DummySingletonTrait 20 | 21 | object DummySingletonClass extends DummySingletonFactory[DummySingletonTrait] { 22 | override def apply(dummyParam: String): DummySingletonTrait = new DummySingletonClass(dummyParam) 23 | } 24 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySingletonFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | trait DummySingletonFactory[+A <: DummySingletonTrait] { 20 | def apply(dummyParam: String): A 21 | } 22 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySingletonTrait.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | trait DummySingletonTrait 20 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/ExitException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | case class ExitException(status: Int) extends SecurityException("System.exit() is not allowed") 20 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/NoExitSecurityManager.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | import java.security.Permission 20 | 21 | sealed class NoExitSecurityManager extends SecurityManager { 22 | override def checkPermission(perm: Permission): Unit = {} 23 | 24 | override def checkPermission(perm: Permission, context: Object): Unit = {} 25 | 26 | override def checkExit(status: Int): Unit = { 27 | super.checkExit(status) 28 | throw ExitException(status) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/RunnableSpy.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks 18 | 19 | class RunnableSpy(exOpt: Option[Throwable] = None) extends Runnable { 20 | var runCount = 0 21 | 22 | override def run(): Unit = this.synchronized{ 23 | runCount += 1 24 | exOpt.foreach(ex => throw ex) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/journal/JournalMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.journal 18 | 19 | import za.co.absa.pramen.core.journal.Journal 20 | import za.co.absa.pramen.core.journal.model.TaskCompleted 21 | 22 | import java.time.Instant 23 | import scala.collection.mutable.ListBuffer 24 | 25 | class JournalMock extends Journal { 26 | val entries: ListBuffer[TaskCompleted] = new ListBuffer[TaskCompleted] 27 | 28 | override def addEntry(entry: TaskCompleted): Unit = entries += entry 29 | 30 | override def getEntries(from: Instant, to: Instant): Seq[TaskCompleted] = 31 | entries.filter(e => e.finishedAt >= from.getEpochSecond && e.finishedAt <= to.getEpochSecond).toSeq 32 | } 33 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/lock/TokenLockFactoryMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.lock 18 | 19 | import za.co.absa.pramen.core.lock.{TokenLock, TokenLockFactory} 20 | 21 | class TokenLockFactoryMock extends TokenLockFactory { 22 | override def getLock(token: String): TokenLock = { 23 | new TokenLockMock 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/lock/TokenLockMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.lock 18 | 19 | import za.co.absa.pramen.core.lock.TokenLock 20 | 21 | class TokenLockMock extends TokenLock { 22 | var acquired = false 23 | 24 | override def tryAcquire(): Boolean = this.synchronized { 25 | if (acquired) { 26 | false 27 | } else { 28 | acquired = true 29 | true 30 | } 31 | } 32 | 33 | override def release(): Unit = this.synchronized { 34 | acquired = false 35 | } 36 | 37 | override def close(): Unit = { 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/state/ShutdownHookFailureMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.state 18 | 19 | import com.typesafe.config.Config 20 | 21 | class ShutdownHookFailureMock(conf: Config) extends Runnable { 22 | override def run(): Unit = this.synchronized { 23 | ShutdownHookFailureMock.ranTimes += 1 24 | 25 | throw new LinkageError("Test fatal error") 26 | } 27 | } 28 | 29 | object ShutdownHookFailureMock { 30 | var ranTimes: Int = 0 31 | } 32 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/state/ShutdownHookSuccessMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.state 18 | 19 | import com.typesafe.config.Config 20 | 21 | class ShutdownHookSuccessMock(conf: Config) extends Runnable { 22 | override def run(): Unit = this.synchronized { 23 | ShutdownHookSuccessMock.ranTimes += 1 24 | } 25 | } 26 | 27 | object ShutdownHookSuccessMock { 28 | var ranTimes: Int = 0 29 | } 30 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/transformer/DummyTransformer1.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.transformer 18 | 19 | import org.apache.spark.sql.DataFrame 20 | import za.co.absa.pramen.api.{MetastoreReader, Reason, Transformer} 21 | 22 | import java.time.LocalDate 23 | 24 | class DummyTransformer1 extends Transformer { 25 | override def validate(metastore: MetastoreReader, infoDate: LocalDate, options: Map[String, String]): Reason = 26 | throw new NotImplementedError 27 | override def run(metastore: MetastoreReader, infoDate: LocalDate, options: Map[String, String]): DataFrame = 28 | throw new NotImplementedError 29 | } 30 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/transformer/DummyTransformer2.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.transformer 18 | 19 | import com.typesafe.config.Config 20 | import org.apache.spark.sql.DataFrame 21 | import za.co.absa.pramen.api.{MetastoreReader, Reason, Transformer} 22 | 23 | import java.time.LocalDate 24 | 25 | class DummyTransformer2(val transformerConfig: Config) extends Transformer { 26 | override def validate(metastore: MetastoreReader, infoDate: LocalDate, options: Map[String, String]): Reason = 27 | throw new NotImplementedError 28 | override def run(metastore: MetastoreReader, infoDate: LocalDate, options: Map[String, String]): DataFrame = 29 | throw new NotImplementedError 30 | } 31 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/utils/hive/QueryExecutorMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.mocks.utils.hive 18 | 19 | import za.co.absa.pramen.core.utils.hive.QueryExecutor 20 | 21 | import scala.collection.mutable.ListBuffer 22 | 23 | class QueryExecutorMock(tableExists: Boolean) extends QueryExecutor { 24 | val queries = new ListBuffer[String] 25 | var closeCalled = 0 26 | 27 | override def doesTableExist(dbName: Option[String], tableName: String): Boolean = tableExists 28 | 29 | override def execute(query: String): Unit = { 30 | queries += query 31 | } 32 | 33 | override def close(): Unit = closeCalled += 1 34 | } 35 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/samples/SampleCaseClass.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.samples 18 | 19 | import java.time.LocalDate 20 | 21 | case class SampleCaseClass( 22 | strValue: String, 23 | intValue: Int, 24 | longValue: Long, 25 | dateValue: LocalDate, 26 | listStr: List[String] 27 | ) 28 | 29 | object SampleCaseClass { 30 | def getDummy: SampleCaseClass = { 31 | SampleCaseClass("String1", 100000, 10000000000L, LocalDate.of(2020,8,10), "Str1" :: "Str2" :: Nil) 32 | } 33 | } -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/samples/SampleCaseClass2.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.samples 18 | 19 | case class SampleCaseClass2( 20 | strValue: String, 21 | intValue: Int, 22 | longValue: Long 23 | ) 24 | 25 | object SampleCaseClass2 { 26 | def getDummy: SampleCaseClass2 = { 27 | SampleCaseClass2("String1", 100000, 10000000000L) 28 | } 29 | } -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/tests/notify/message/CustomException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.tests.notify.message 18 | 19 | class CustomException(msg: String, cause: Throwable = null) extends RuntimeException(msg, cause) { 20 | override def fillInStackTrace(): Throwable = { 21 | this 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/tests/utils/ResourceUtilsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.tests.utils 18 | 19 | import org.scalatest.wordspec.AnyWordSpec 20 | import za.co.absa.pramen.core.utils.ResourceUtils 21 | 22 | class ResourceUtilsSuite extends AnyWordSpec { 23 | "getResourceString" should { 24 | "return the content of the resource" in { 25 | val str = ResourceUtils.getResourceString("/test/testResource.txt") 26 | 27 | assert(str == "Hello") 28 | } 29 | 30 | "thrown an exception if the resource is not found" in { 31 | intercept[NullPointerException] { 32 | ResourceUtils.getResourceString("test/NoSuchFile.txt") 33 | } 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /pramen/core/src/test/scala/za/co/absa/pramen/core/tests/utils/hive/HiveApiSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.core.tests.utils.hive 18 | 19 | import org.scalatest.wordspec.AnyWordSpec 20 | import za.co.absa.pramen.core.metastore.model.HiveApi 21 | 22 | class HiveApiSuite extends AnyWordSpec { 23 | "fromString()" should { 24 | "properly parse 'sql'" in { 25 | assert(HiveApi.fromString("SqL") == HiveApi.Sql) 26 | } 27 | 28 | "properly parse 'spark_catalog'" in { 29 | assert(HiveApi.fromString("spark_catalog") == HiveApi.SparkCatalog) 30 | } 31 | 32 | "throw on an invalid value" in { 33 | assertThrows[IllegalArgumentException](HiveApi.fromString("dummy")) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pramen/examples/enceladus_single_config/daily_ingestion.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | #!/bin/bash 15 | ME=`basename "$0" .sh` 16 | 17 | cd $(dirname $(readlink -f $0)) 18 | 19 | SCALA_VERSION="2.11" 20 | 21 | EXTRAS_JAR="pramen-extras_${SCALA_VERSION}-1.0.0.jar" 22 | RUNNER_JAR="pramen-runner_${SCALA_VERSION}-1.0.0.jar" 23 | 24 | set -euxo pipefail 25 | 26 | $SPARK_HOME/bin/spark-submit \ 27 | --master yarn \ 28 | --deploy-mode client \ 29 | --jars $EXTRAS_JAR \ 30 | --class za.co.absa.pramen.runner.PipelineRunner \ 31 | $RUNNER_JAR --workflow ${ME}.conf $@ 32 | 33 | cd - 34 | -------------------------------------------------------------------------------- /pramen/examples/enceladus_single_config/weekly_ingestion.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | #!/bin/bash 15 | ME=`basename "$0" .sh` 16 | 17 | cd $(dirname $(readlink -f $0)) 18 | 19 | SCALA_VERSION="2.11" 20 | 21 | EXTRAS_JAR="pramen-extras_${SCALA_VERSION}-1.0.0.jar" 22 | RUNNER_JAR="pramen-runner_${SCALA_VERSION}-1.0.0.jar" 23 | 24 | set -euxo pipefail 25 | 26 | $SPARK_HOME/bin/spark-submit \ 27 | --master yarn \ 28 | --deploy-mode client \ 29 | --jars $EXTRAS_JAR \ 30 | --class za.co.absa.pramen.runner.PipelineRunner \ 31 | $RUNNER_JAR --workflow ${ME}.conf $@ 32 | 33 | cd - 34 | -------------------------------------------------------------------------------- /pramen/examples/enceladus_sourcing/common.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | run.type = "(Prod)" 16 | environment.name = "MyEnv "${pramen.run.type} 17 | 18 | bookkeeping.enabled = false 19 | 20 | warn.if.no.data = true 21 | email.if.no.changes = false 22 | 23 | temporary.directory = "/tmp/pramen" 24 | 25 | warn.throughput.rps = 2000 26 | good.throughput.rps = 50000 27 | 28 | dry.run = false 29 | } 30 | 31 | mail { 32 | smtp.host = "" 33 | smtp.port = "25" 34 | smtp.auth = "false" 35 | smtp.starttls.enable = "false" 36 | smtp.EnableSSL.enable = "false" 37 | debug = "false" 38 | 39 | send.from = "Pramen " 40 | send.to = "" 41 | } 42 | -------------------------------------------------------------------------------- /pramen/examples/enceladus_sourcing/daily_ingestion.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022 ABSA Group Limited 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | ME=`basename "$0" .sh` 17 | 18 | cd $(dirname $(readlink -f $0)) 19 | 20 | SCALA_VERSION="2.11" 21 | 22 | EXTRAS_JAR="pramen-extras_${SCALA_VERSION}-1.0.0.jar" 23 | RUNNER_JAR="pramen-runner_${SCALA_VERSION}-1.0.0.jar" 24 | 25 | set -euxo pipefail 26 | 27 | $SPARK_HOME/bin/spark-submit \ 28 | --master yarn \ 29 | --deploy-mode client \ 30 | --jars $EXTRAS_JAR \ 31 | --class za.co.absa.pramen.runner.PipelineRunner \ 32 | $RUNNER_JAR --workflow ${ME}.conf $@ 33 | 34 | cd - 35 | -------------------------------------------------------------------------------- /pramen/examples/enceladus_sourcing/daily_snapshot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022 ABSA Group Limited 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | ME=`basename "$0" .sh` 17 | 18 | cd $(dirname $(readlink -f $0)) 19 | 20 | SCALA_VERSION="2.11" 21 | 22 | EXTRAS_JAR="pramen-extras_${SCALA_VERSION}-1.0.0.jar" 23 | RUNNER_JAR="pramen-runner_${SCALA_VERSION}-1.0.0.jar" 24 | 25 | set -euxo pipefail 26 | 27 | export INFO_VERSION="1" 28 | 29 | $SPARK_HOME/bin/spark-submit \ 30 | --master yarn \ 31 | --deploy-mode client \ 32 | --jars $EXTRAS_JAR \ 33 | --class za.co.absa.pramen.runner.PipelineRunner \ 34 | $RUNNER_JAR --workflow ${ME}.conf $@ 35 | 36 | cd - 37 | -------------------------------------------------------------------------------- /pramen/examples/enceladus_sourcing/metastore.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen.metastore { 15 | tables = [ 16 | # No table needs to be configured if the pipeline contains only transfer jobs 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /pramen/examples/jdbc_sourcing/common.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | run.type = "(Prod)" 16 | environment.name = "MyEnv "${pramen.run.type} 17 | 18 | bookkeeping.enabled = false 19 | 20 | warn.if.no.data = true 21 | email.if.no.changes = false 22 | 23 | # Jobs are T+1 24 | expected.delay.days = 1 25 | 26 | temporary.directory = "/tmp/pramen" 27 | 28 | warn.throughput.rps = 2000 29 | good.throughput.rps = 50000 30 | 31 | dry.run = false 32 | } 33 | 34 | mail { 35 | smtp.host = "" 36 | smtp.port = "25" 37 | smtp.auth = "false" 38 | smtp.starttls.enable = "false" 39 | smtp.EnableSSL.enable = "false" 40 | debug = "false" 41 | 42 | send.from = "Pramen " 43 | send.to = "" 44 | } 45 | -------------------------------------------------------------------------------- /pramen/examples/jdbc_sourcing/daily_ingestion.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022 ABSA Group Limited 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | ME=`basename "$0" .sh` 17 | 18 | cd $(dirname $(readlink -f $0)) 19 | 20 | SCALA_VERSION="2.11" 21 | 22 | EXTRAS_JAR="pramen-extras_${SCALA_VERSION}-1.0.0.jar" 23 | RUNNER_JAR="pramen-runner_${SCALA_VERSION}-1.0.0.jar" 24 | 25 | set -euxo pipefail 26 | 27 | $SPARK_HOME/bin/spark-submit \ 28 | --master yarn \ 29 | --deploy-mode client \ 30 | --jars $EXTRAS_JAR \ 31 | --class za.co.absa.pramen.runner.PipelineRunner \ 32 | $RUNNER_JAR --check-new-only --workflow ${ME}.conf $@ 33 | 34 | cd - 35 | -------------------------------------------------------------------------------- /pramen/examples/jdbc_sourcing/metastore.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen.metastore { 15 | tables = [ 16 | { 17 | name = "my_table1" 18 | description = "My source table 1" 19 | format = "parquet" 20 | path = "/bigdata/datalake/my_table1/landing" 21 | }, 22 | { 23 | name = "my_table2" 24 | description = "My source table 2" 25 | format = "delta" 26 | path = "/bigdata/datalake/my_table2/landing" 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /pramen/extras/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | # Define jobs as factory classes: 16 | # jobs = [ za.co.absa.pramen.framework.EdwToParquetSyncJob ] 17 | 18 | # Generate INFO file for Enceladus 19 | plugins { 20 | info.file { 21 | generate = false 22 | 23 | source.application = "Unspecified" 24 | country = "Africa" 25 | history.type = "Snapshot" 26 | 27 | timestamp.format = "dd-MM-yyyy HH:mm:ss Z" 28 | date.format = "yyyy-MM-dd" 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/model/InfoDateColumn.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.model 18 | 19 | case class InfoDateColumn( 20 | name: String, 21 | format: String 22 | ) 23 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/notification/mq/SingleMessageProducer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.notification.mq 18 | 19 | trait SingleMessageProducer { 20 | def send(topic: String, message: String, numberOrRetries: Int = 3): Unit 21 | 22 | def connect(): Unit 23 | 24 | def close(): Unit 25 | } 26 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/query/QueryExecutor.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.query 18 | 19 | import org.apache.spark.sql.DataFrame 20 | 21 | trait QueryExecutor { 22 | def execute(sql: String): Unit 23 | 24 | def query(sql: String): DataFrame 25 | } 26 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/query/QueryExecutorSpark.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.query 18 | 19 | import org.apache.spark.sql.{DataFrame, SparkSession} 20 | import org.slf4j.LoggerFactory 21 | 22 | class QueryExecutorSpark(spark: SparkSession) extends QueryExecutor { 23 | private val log = LoggerFactory.getLogger(this.getClass) 24 | 25 | override def execute(sql: String): Unit = { 26 | log.info(s"Executing SQL: $sql") 27 | spark.sql(sql).take(100) 28 | } 29 | override def query(sql: String): DataFrame = { 30 | log.info(s"Executing SQL: $sql") 31 | spark.sql(sql) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/sink/InfoVersionStatus.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.sink 18 | 19 | trait InfoVersionStatus 20 | 21 | object InfoVersionStatus { 22 | case class Detected(maxVersion: Int) extends InfoVersionStatus 23 | 24 | case object NotPresent extends InfoVersionStatus 25 | 26 | case class DetectionFailure(ex: Throwable) extends InfoVersionStatus 27 | } 28 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/utils/JsonUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper 20 | 21 | object JsonUtils { 22 | def prettyJSON(jsonIn: String): String = { 23 | val mapper = new ObjectMapper() 24 | 25 | val jsonUnindented = mapper.readValue(jsonIn, classOf[Any]) 26 | val indented = mapper.writerWithDefaultPrettyPrinter.writeValueAsString(jsonUnindented) 27 | indented.replace("\r\n", "\n") 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/utils/ResourceUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils 18 | 19 | import org.apache.commons.io.IOUtils 20 | 21 | object ResourceUtils { 22 | def getResourceString(name: String): String = 23 | IOUtils.toString(getClass.getResourceAsStream(name), "UTF-8") 24 | } 25 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/utils/httpclient/HttpMethod.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils.httpclient 18 | 19 | sealed trait HttpMethod { 20 | def name: String 21 | } 22 | 23 | object HttpMethod { 24 | case object GET extends HttpMethod { 25 | def name = "GET" 26 | } 27 | 28 | case object POST extends HttpMethod { 29 | def name = "POST" 30 | } 31 | 32 | case object PUT extends HttpMethod { 33 | def name = "PUT" 34 | } 35 | 36 | case object DELETE extends HttpMethod { 37 | def name = "DELETE" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/utils/httpclient/SimpleHttpRequest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils.httpclient 18 | 19 | case class SimpleHttpRequest(url: String, 20 | method: HttpMethod, 21 | headers: Map[String, String] = Map.empty, 22 | body: Option[String] = None, 23 | cacheKey: Option[String] = None, // You can specify a custom cache key so that cache sill work if the URL changes 24 | allowStaleData: Boolean = 25 | false) // If yes, expired data form cache can be used if the primary endpoint is not available 26 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/utils/httpclient/SimpleHttpResponse.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils.httpclient 18 | 19 | case class SimpleHttpResponse(statusCode: Int, 20 | body: Option[String], 21 | warnings: Seq[String]) 22 | -------------------------------------------------------------------------------- /pramen/extras/src/main/scala/za/co/absa/pramen/extras/writer/TableWriter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.writer 18 | 19 | import org.apache.spark.sql.DataFrame 20 | 21 | import java.time.LocalDate 22 | 23 | trait TableWriter { 24 | def write(df: DataFrame, infoDate: LocalDate, numOfRecordsEstimate: Option[Long]): Long 25 | } 26 | -------------------------------------------------------------------------------- /pramen/extras/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | log4j.rootCategory=INFO, console 16 | log4j.appender.console=org.apache.log4j.ConsoleAppender 17 | log4j.appender.console.target=System.err 18 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 20 | log4j.appender.console.Threshold=ERROR 21 | 22 | log4j.logger.cmd=ERROR 23 | log4j.logger.za.co.absa.pramen.DummyProcessRunner=OFF 24 | log4j.logger.za.co.absa.pramen.extras.sink.EnceladusSink=OFF 25 | log4j.logger.za.co.absa.pramen.extras.sink.StandardizationSink=OFF 26 | log4j.logger.za.co.absa.pramen.extras.utils.httpclient.impl.RetryableHttpClient=OFF 27 | -------------------------------------------------------------------------------- /pramen/extras/src/test/resources/test/app_context.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 ABSA Group Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | pramen { 15 | bookkeeping.enabled = false 16 | 17 | pipeline { 18 | name = "dummy_pipeline" 19 | } 20 | } 21 | 22 | pramen.metastore { 23 | tables = [] 24 | } 25 | -------------------------------------------------------------------------------- /pramen/extras/src/test/resources/test/nestedMap_fixed_avro.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "record", 3 | "name" : "topLevelRecord", 4 | "fields" : [ { 5 | "name" : "name", 6 | "type" : [ "null", "string" ], 7 | "default" : null 8 | }, { 9 | "name" : "addresses", 10 | "type" : [ "null", { 11 | "type" : "array", 12 | "items" : [ { 13 | "type" : "record", 14 | "name" : "addresses", 15 | "namespace" : "topLevelRecord", 16 | "fields" : [ { 17 | "name" : "city", 18 | "type" : [ "null", "string" ], 19 | "default" : null 20 | }, { 21 | "name" : "state", 22 | "type" : [ "null", "string" ], 23 | "default" : null 24 | } ] 25 | }, "null" ] 26 | } ], 27 | "default" : null 28 | }, { 29 | "name" : "properties", 30 | "type" : [ "null", { 31 | "type" : "map", 32 | "values" : [ "string", "null" ] 33 | } ], 34 | "default" : null 35 | } ] 36 | } 37 | -------------------------------------------------------------------------------- /pramen/extras/src/test/resources/test/nestedMap_origin_avro.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "record", 3 | "name" : "topLevelRecord", 4 | "fields" : [ { 5 | "name" : "name", 6 | "type" : [ "string", "null" ] 7 | }, { 8 | "name" : "addresses", 9 | "type" : [ { 10 | "type" : "array", 11 | "items" : [ { 12 | "type" : "record", 13 | "name" : "addresses", 14 | "namespace" : "topLevelRecord", 15 | "fields" : [ { 16 | "name" : "city", 17 | "type" : [ "string", "null" ] 18 | }, { 19 | "name" : "state", 20 | "type" : [ "string", "null" ] 21 | } ] 22 | }, "null" ] 23 | }, "null" ] 24 | }, { 25 | "name" : "properties", 26 | "type" : [ { 27 | "type" : "map", 28 | "values" : [ "string", "null" ] 29 | }, "null" ] 30 | } ] 31 | } 32 | -------------------------------------------------------------------------------- /pramen/extras/src/test/resources/test/testResource.txt: -------------------------------------------------------------------------------- 1 | Hello -------------------------------------------------------------------------------- /pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/AppMainMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.mocks 18 | 19 | object AppMainMock { 20 | def main(args: Array[String]): Unit = { 21 | throw new RuntimeException(s"Main reached: ${args.mkString(" ")}") 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/AppMainSilentMock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.mocks 18 | 19 | object AppMainSilentMock { 20 | def main(args: Array[String]): Unit = {} 21 | } 22 | -------------------------------------------------------------------------------- /pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/DummyNamingStrategyFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.mocks 18 | 19 | import za.co.absa.pramen.extras.writer.model.NamingStrategy 20 | 21 | object DummyNamingStrategyFactory { 22 | def getDummyNamingStrategy(namingStrategy: String = "topic.name", 23 | recordName: Option[String] = None, 24 | recordNamespace: Option[String] = None 25 | ): NamingStrategy = { 26 | NamingStrategy(namingStrategy = namingStrategy, 27 | recordName = recordName, 28 | recordNamespace = recordNamespace) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pramen/extras/src/test/scala/za/co/absa/pramen/extras/mocks/WriterSpy.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.mocks 18 | 19 | import org.apache.spark.sql.{DataFrame, SparkSession} 20 | import za.co.absa.pramen.extras.writer.TableWriter 21 | 22 | import java.time.LocalDate 23 | import scala.collection.mutable.ListBuffer 24 | 25 | class WriterSpy(numRecords: Long = 0L)(implicit spark: SparkSession) extends TableWriter { 26 | 27 | val calls = new ListBuffer[(LocalDate, Option[Long])] 28 | 29 | override def write(df: DataFrame, infoDate: LocalDate, numOfRecordsEstimate: Option[Long]): Long = { 30 | calls += infoDate -> numOfRecordsEstimate 31 | numRecords 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pramen/extras/src/test/scala/za/co/absa/pramen/extras/utils/JsonUtilsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils 18 | 19 | import org.scalatest.wordspec.AnyWordSpec 20 | 21 | class JsonUtilsSuite extends AnyWordSpec { 22 | 23 | "prettyJSON" should { 24 | "prettify a JSON" in { 25 | val json = """{"a":1,"b":2}""" 26 | val expected = """{ 27 | | "a" : 1, 28 | | "b" : 2 29 | |}""".stripMargin 30 | assert(JsonUtils.prettyJSON(json) == expected) 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /pramen/extras/src/test/scala/za/co/absa/pramen/extras/utils/ResourceUtilsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 ABSA Group Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package za.co.absa.pramen.extras.utils 18 | 19 | import org.scalatest.wordspec.AnyWordSpec 20 | 21 | class ResourceUtilsSuite extends AnyWordSpec { 22 | "getResourceString" should { 23 | "return the content of the resource" in { 24 | val str = ResourceUtils.getResourceString("/test/testResource.txt") 25 | 26 | assert(str == "Hello") 27 | } 28 | 29 | "thrown an exception if the resource is not found" in { 30 | intercept[NullPointerException] { 31 | ResourceUtils.getResourceString("test/NoSuchFile.txt") 32 | } 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /pramen/project/build.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022 ABSA Group Limited 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | sbt.version=1.9.9 17 | -------------------------------------------------------------------------------- /pramen/version.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / version := "1.11.7-SNAPSHOT" 2 | -------------------------------------------------------------------------------- /resources/concepts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/resources/concepts.png -------------------------------------------------------------------------------- /resources/date_inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/resources/date_inference.png -------------------------------------------------------------------------------- /resources/file_based_sourcing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/resources/file_based_sourcing.png -------------------------------------------------------------------------------- /resources/ops_jobs_tasks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/resources/ops_jobs_tasks.png -------------------------------------------------------------------------------- /resources/pipeline_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/resources/pipeline_example.png -------------------------------------------------------------------------------- /resources/run_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbsaOSS/pramen/9c41556a7bf9659619e6cd6ad28c3650680889bb/resources/run_diagram.png --------------------------------------------------------------------------------