├── .gitignore ├── .java-version ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DISCLAIMER.md ├── LICENSE ├── LICENSES └── Apache-2.0.txt ├── README.md ├── REUSE.toml ├── commercedbsync ├── .classpath ├── buildcallbacks.xml ├── extensioninfo.xml ├── external-dependencies.xml ├── project.properties ├── resources │ ├── anonymizer │ │ ├── address.json │ │ └── config.yml │ ├── commercedbsync-beans.xml │ ├── commercedbsync-items.xml │ ├── commercedbsync-spring.xml │ ├── commercedbsync │ │ └── sap-hybris-platform.png │ ├── groovy │ │ ├── MigrationSummaryScript.groovy │ │ └── ddlaltercreate.groovy │ ├── impex │ │ └── projectdata-commercemigration-jobs.impex │ ├── localization │ │ ├── commercedbsync-locales_de.properties │ │ ├── commercedbsync-locales_en.properties │ │ ├── commercedbsync-locales_es.properties │ │ ├── commercedbsync-locales_fr.properties │ │ ├── commercedbsync-locales_it.properties │ │ ├── commercedbsync-locales_ja.properties │ │ ├── commercedbsync-locales_ko.properties │ │ ├── commercedbsync-locales_pt.properties │ │ ├── commercedbsync-locales_ru.properties │ │ └── commercedbsync-locales_zh.properties │ └── sql │ │ ├── createSchedulerTablesHANA.sql │ │ ├── createSchedulerTablesMSSQL.sql │ │ ├── createSchedulerTablesMYSQL.sql │ │ ├── createSchedulerTablesORACLE.sql │ │ ├── createSchedulerTablesPOSTGRESQL.sql │ │ ├── createSchemaSchedulerTablesHANA.sql │ │ ├── createSchemaSchedulerTablesMSSQL.sql │ │ ├── createSchemaSchedulerTablesMYSQL.sql │ │ ├── createSchemaSchedulerTablesORACLE.sql │ │ ├── createSchemaSchedulerTablesPOSTGRESQL.sql │ │ └── transformationFunctions │ │ ├── mssql-general.sql │ │ └── mssql-typeinfotable.sql ├── src │ ├── com │ │ └── sap │ │ │ └── cx │ │ │ └── boosters │ │ │ └── commercedbsync │ │ │ ├── CommercedbsyncStandalone.java │ │ │ ├── adapter │ │ │ ├── DataRepositoryAdapter.java │ │ │ └── impl │ │ │ │ └── ContextualDataRepositoryAdapter.java │ │ │ ├── anonymizer │ │ │ ├── AnonymizeFunctions.java │ │ │ ├── AnonymizerConfigurator.java │ │ │ ├── TextEvaluator.java │ │ │ ├── TextTokenizer.java │ │ │ ├── exception │ │ │ │ └── ValidationException.java │ │ │ ├── functions │ │ │ │ ├── FunctionEvaluator.java │ │ │ │ └── impl │ │ │ │ │ ├── GuidFunctionEvaluator.java │ │ │ │ │ ├── RandomAddressGenerator.java │ │ │ │ │ └── RandomFunctionEvaluator.java │ │ │ └── model │ │ │ │ ├── AnonymizerConfiguration.java │ │ │ │ ├── Column.java │ │ │ │ ├── Operation.java │ │ │ │ └── Table.java │ │ │ ├── concurrent │ │ │ ├── DataCopyMethod.java │ │ │ ├── DataPipe.java │ │ │ ├── DataPipeFactory.java │ │ │ ├── DataThreadPoolConfigBuilder.java │ │ │ ├── DataThreadPoolFactory.java │ │ │ ├── DataThreadPoolMonitor.java │ │ │ ├── DataWorkerExecutor.java │ │ │ ├── DataWorkerPoolFactory.java │ │ │ ├── MDCTaskDecorator.java │ │ │ ├── MaybeFinished.java │ │ │ ├── PipeAbortedException.java │ │ │ └── impl │ │ │ │ ├── DefaultDataPipe.java │ │ │ │ ├── DefaultDataPipeFactory.java │ │ │ │ ├── DefaultDataThreadPoolFactory.java │ │ │ │ ├── DefaultDataThreadPoolMonitor.java │ │ │ │ ├── DefaultDataWorkerExecutor.java │ │ │ │ └── task │ │ │ │ ├── BatchMarkerDataReaderTask.java │ │ │ │ ├── BatchOffsetDataReaderTask.java │ │ │ │ ├── ChunkDataReaderTask.java │ │ │ │ ├── DataReaderTask.java │ │ │ │ ├── DefaultDataReaderTask.java │ │ │ │ ├── PartitionedBatchMarkerDataReaderTask.java │ │ │ │ ├── PipeTaskContext.java │ │ │ │ └── RetriableTask.java │ │ │ ├── constants │ │ │ └── CommercedbsyncConstants.java │ │ │ ├── context │ │ │ ├── CopyContext.java │ │ │ ├── IncrementalMigrationContext.java │ │ │ ├── LaunchOptions.java │ │ │ ├── MigrationContext.java │ │ │ ├── MigrationContextFactory.java │ │ │ ├── SchemaDifferenceContext.java │ │ │ ├── impl │ │ │ │ ├── DefaultIncrementalMigrationContext.java │ │ │ │ └── DefaultMigrationContext.java │ │ │ └── validation │ │ │ │ ├── MigrationContextValidator.java │ │ │ │ └── impl │ │ │ │ └── DefaultMigrationContextValidator.java │ │ │ ├── cron │ │ │ ├── FullMigrationCronJob.java │ │ │ ├── IncrementalMigrationCronJob.java │ │ │ └── MigrationCronJob.java │ │ │ ├── dataset │ │ │ ├── DataColumn.java │ │ │ ├── DataSet.java │ │ │ └── impl │ │ │ │ ├── DefaultDataColumn.java │ │ │ │ └── DefaultDataSet.java │ │ │ ├── datasource │ │ │ ├── MigrationDataSourceFactory.java │ │ │ └── impl │ │ │ │ ├── AbstractMigrationDataSourceFactory.java │ │ │ │ └── DefaultMigrationDataSourceFactory.java │ │ │ ├── events │ │ │ ├── CopyCompleteEvent.java │ │ │ ├── CopyDatabaseTableEvent.java │ │ │ ├── OperationEvent.java │ │ │ ├── SchemaDifferenceEvent.java │ │ │ └── handlers │ │ │ │ ├── CopyCompleteEventListener.java │ │ │ │ └── CopyDatabaseTableEventListener.java │ │ │ ├── filter │ │ │ ├── DataCopyTableFilter.java │ │ │ └── impl │ │ │ │ ├── CompositeDataCopyTableFilter.java │ │ │ │ ├── ExclusionDataCopyTableFilter.java │ │ │ │ ├── InclusionDataCopyTableFilter.java │ │ │ │ └── IncrementalDataCopyTableFilter.java │ │ │ ├── interceptors │ │ │ └── DefaultCMTRemoveInterceptor.java │ │ │ ├── jalo │ │ │ └── ItemDeletionMarker.java │ │ │ ├── jobs │ │ │ ├── AbstractMigrationJobPerformable.java │ │ │ ├── FullMigrationJob.java │ │ │ ├── IncrementalMigrationJob.java │ │ │ └── MigrationPrepJob.java │ │ │ ├── listeners │ │ │ └── DefaultCMTAfterSaveListener.java │ │ │ ├── logging │ │ │ ├── JDBCQueriesStore.java │ │ │ ├── JdbcQueryLog.java │ │ │ ├── LoggingConnectionWrapper.java │ │ │ ├── LoggingPreparedStatementWrapper.java │ │ │ └── LoggingStatementWrapper.java │ │ │ ├── performance │ │ │ ├── PerformanceCategory.java │ │ │ ├── PerformanceProfiler.java │ │ │ ├── PerformanceRecorder.java │ │ │ ├── PerformanceUnit.java │ │ │ └── impl │ │ │ │ └── DefaultPerformanceProfiler.java │ │ │ ├── processors │ │ │ ├── MigrationPostProcessor.java │ │ │ ├── MigrationPreProcessor.java │ │ │ └── impl │ │ │ │ ├── AdjustActiveTypeSystemPostProcessor.java │ │ │ │ ├── DefaultMigrationPostProcessor.java │ │ │ │ ├── IndexAlignerPostProcessor.java │ │ │ │ ├── JdbcQueriesPostProcessor.java │ │ │ │ ├── MSSQLUpdateStatisticsPostProcessor.java │ │ │ │ ├── ReportMigrationPostProcessor.java │ │ │ │ ├── TransformFunctionGeneratorPreProcessor.java │ │ │ │ ├── TruncateNotMigratedTablesPreProcessor.java │ │ │ │ ├── TypeInfoTableGeneratorPreProcessor.java │ │ │ │ ├── UpdateYDeploymentsPostProcessor.java │ │ │ │ ├── ViewDropPostProcessor.java │ │ │ │ └── ViewGeneratorPreProcessor.java │ │ │ ├── profile │ │ │ ├── DataSourceConfiguration.java │ │ │ ├── DataSourceConfigurationFactory.java │ │ │ └── impl │ │ │ │ ├── DefaultDataSourceConfiguration.java │ │ │ │ ├── DefaultDataSourceConfigurationFactory.java │ │ │ │ └── InvalidDataSourceConfigurationException.java │ │ │ ├── provider │ │ │ ├── CopyItemProvider.java │ │ │ └── impl │ │ │ │ └── DefaultDataCopyItemProvider.java │ │ │ ├── repository │ │ │ ├── DataRepository.java │ │ │ ├── impl │ │ │ │ ├── AbstractDataRepository.java │ │ │ │ ├── AzureDataRepository.java │ │ │ │ ├── AzureIncrementalDataRepository.java │ │ │ │ ├── DataRepositoryFactory.java │ │ │ │ ├── HanaDataRepository.java │ │ │ │ ├── HsqlRepository.java │ │ │ │ ├── MySQLDataRepository.java │ │ │ │ ├── MySQLIncrementalDataRepository.java │ │ │ │ ├── NullRepository.java │ │ │ │ ├── OracleDataRepository.java │ │ │ │ └── PostGresDataRepository.java │ │ │ └── platform │ │ │ │ ├── MigrationHybrisHANABuilder.java │ │ │ │ ├── MigrationHybrisHANAPlatform.java │ │ │ │ ├── MigrationHybrisMSSqlBuilder.java │ │ │ │ ├── MigrationHybrisMSSqlPlatform.java │ │ │ │ ├── MigrationHybrisMySqlBuilder.java │ │ │ │ ├── MigrationHybrisMySqlPlatform.java │ │ │ │ ├── MigrationHybrisPostGresBuilder.java │ │ │ │ └── MigrationHybrisPostGresPlatform.java │ │ │ ├── scheduler │ │ │ ├── ClusterTableSplittingStrategy.java │ │ │ ├── DatabaseCopyScheduler.java │ │ │ ├── DatabaseOperationSchedulerAlgorithm.java │ │ │ ├── DatabaseSchemaDifferenceScheduler.java │ │ │ └── impl │ │ │ │ ├── CustomClusterDatabaseCopyScheduler.java │ │ │ │ ├── DefaultClusterTableSplittingStrategy.java │ │ │ │ ├── DefaultDatabaseSchemaDifferenceScheduler.java │ │ │ │ └── RoundRobinClusterSchedulerAlgorithm.java │ │ │ ├── service │ │ │ ├── DatabaseCopyTaskRepository.java │ │ │ ├── DatabaseMigrationCopyService.java │ │ │ ├── DatabaseMigrationDataTypeMapperService.java │ │ │ ├── DatabaseMigrationReportService.java │ │ │ ├── DatabaseMigrationReportStorageService.java │ │ │ ├── DatabaseMigrationService.java │ │ │ ├── DatabaseSchemaDifferenceService.java │ │ │ ├── DatabaseSchemaDifferenceTaskRepository.java │ │ │ └── impl │ │ │ │ ├── BlobDatabaseMigrationReportStorageService.java │ │ │ │ ├── DefaultDatabaseCopyTaskRepository.java │ │ │ │ ├── DefaultDatabaseMigrationDataTypeMapperService.java │ │ │ │ ├── DefaultDatabaseMigrationReportService.java │ │ │ │ ├── DefaultDatabaseMigrationService.java │ │ │ │ ├── DefaultDatabaseSchemaDifferenceService.java │ │ │ │ ├── DefaultDatabaseSchemaDifferenceTaskRepository.java │ │ │ │ └── PipeDatabaseMigrationCopyService.java │ │ │ ├── setup │ │ │ └── InitUpdateProcessTrigger.java │ │ │ ├── strategy │ │ │ ├── PipeWriterStrategy.java │ │ │ └── impl │ │ │ │ ├── CopyPipeWriterContext.java │ │ │ │ ├── CopyPipeWriterStrategy.java │ │ │ │ ├── CopyPipeWriterTask.java │ │ │ │ └── DataDeleteWriterTask.java │ │ │ ├── utils │ │ │ ├── FileUtils.java │ │ │ ├── LocalDateTypeAdapter.java │ │ │ └── MaskUtil.java │ │ │ └── views │ │ │ ├── TableViewGenerator.java │ │ │ └── ViewConfigurationContext.java │ └── de │ │ └── hybris │ │ └── platform │ │ ├── azure │ │ └── media │ │ │ └── AzureCloudUtils.java │ │ └── core │ │ └── TenantPropertiesLoader.java └── testsrc │ └── com │ └── sap │ └── cx │ └── boosters │ └── commercedbsync │ └── views │ └── TableViewGeneratorTest.java ├── commercedbsynchac ├── .classpath ├── .externalToolBuilders │ └── HybrisCodeGeneration.launch ├── buildcallbacks.xml ├── extensioninfo.xml ├── external-dependencies.xml ├── hac │ ├── resources │ │ ├── jsp │ │ │ ├── dataCopy.jsp │ │ │ ├── dataSource.jsp │ │ │ ├── migrationReports.jsp │ │ │ └── schemaCopy.jsp │ │ └── static │ │ │ ├── audio │ │ │ └── pling.mp3 │ │ │ ├── css │ │ │ ├── dataCopy.css │ │ │ ├── database.css │ │ │ ├── schemaCopy.css │ │ │ └── table.css │ │ │ └── js │ │ │ ├── configPanel.js │ │ │ ├── customStatistics.js │ │ │ ├── dataCopy.js │ │ │ ├── dataSource.js │ │ │ ├── migrationMetrics.js │ │ │ ├── migrationReports.js │ │ │ └── schemaCopy.js │ ├── src │ │ └── de │ │ │ └── hybris │ │ │ └── platform │ │ │ └── hac │ │ │ └── controller │ │ │ └── CommercemigrationhacController.java │ └── testsrc │ │ └── de │ │ └── hybris │ │ └── platform │ │ └── hac │ │ └── controller │ │ └── CommercemigrationhacControllerTest.java ├── project.properties ├── resources │ ├── com │ │ └── sap │ │ │ └── cx │ │ │ └── boosters │ │ │ └── commercedbsynchac │ │ │ └── dummy.txt │ ├── commercedbsynchac-beans.xml │ ├── commercedbsynchac-items.xml │ ├── commercedbsynchac-spring.xml │ ├── commercedbsynchac-tab-config.json │ ├── commercedbsynchac-without-migration-tab-config.json │ └── localization │ │ ├── i2ihac-locales_de.properties │ │ ├── i2ihac-locales_en.properties │ │ ├── i2ihac-locales_es.properties │ │ ├── i2ihac-locales_fr.properties │ │ ├── i2ihac-locales_it.properties │ │ ├── i2ihac-locales_ja.properties │ │ ├── i2ihac-locales_ko.properties │ │ ├── i2ihac-locales_pt.properties │ │ ├── i2ihac-locales_ru.properties │ │ └── i2ihac-locales_zh.properties └── src │ └── com │ └── sap │ └── cx │ └── boosters │ └── commercedbsynchac │ ├── CommercedbsynchacStandalone.java │ ├── constants │ └── YhacextConstants.java │ └── metric │ ├── MetricService.java │ ├── impl │ └── DefaultMetricService.java │ └── populator │ ├── MetricPopulator.java │ └── impl │ ├── CpuMetricPopulator.java │ ├── DTUMetricPopulator.java │ ├── HikariConnectionMetricPopulator.java │ ├── HikariSourceConnectionMetricPopulator.java │ ├── HikariTargetConnectionMetricPopulator.java │ ├── IOMetricPopulator.java │ ├── MemoryMetricPopulator.java │ ├── TaskExecutorMetricPopulator.java │ └── ThreadPoolMetricPopulator.java └── docs ├── anonymizer └── README.md ├── commercedbsync ├── after_save_listener_1.png └── after_save_listener_2.png ├── configuration ├── CONFIGURATION-GUIDE.md └── CONFIGURATION-REFERENCE.md ├── developer └── DEVELOPER-GUIDE.md ├── performance ├── PERFORMANCE-GUIDE.md ├── performance_architecture.png └── template_for_scheduled_operational_activity.docx ├── security └── SECURITY-GUIDE.md ├── support ├── SUPPORT-GUIDE.md └── support_groovy_preview.png ├── troubleshooting └── TROUBLESHOOTING-GUIDE.md └── user ├── SUPPORT-DELETE-GUIDE.md ├── USER-GUIDE-DATA-MIGRATION.md ├── USER-GUIDE-DATA-REPLICATION.md ├── commerce-db-sync-demo.png ├── data_migration_architecture.drawio ├── data_migration_architecture.png ├── data_replication_architecture.drawio ├── data_replication_architecture.png ├── db-sync-flows.svg ├── hac_migrate_data.png ├── hac_report.png ├── hac_schema_diff_exec.png ├── hac_schema_diff_prev.png ├── hac_validate_ds.png └── proxy_timeout.png /.gitignore: -------------------------------------------------------------------------------- 1 | build.xml 2 | extensioninfo.xsd 3 | beans.xsd 4 | items.xsd 5 | *testclasses.xml 6 | platformhome.properties 7 | 8 | classes/ 9 | gensrc/ 10 | lib/ -------------------------------------------------------------------------------- /.java-version: -------------------------------------------------------------------------------- 1 | 17.0 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Code of Conduct 4 | 5 | All members of the project community must abide by the [Contributor Covenant, version 2.1](CODE_OF_CONDUCT.md). 6 | Only by respecting each other we can develop a productive, collaborative community. 7 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting [a project maintainer](.reuse/dep5). 8 | 9 | ## Engaging in Our Project 10 | 11 | We use GitHub to manage reviews of pull requests. 12 | 13 | * If you are a new contributor, see: [Steps to Contribute](#steps-to-contribute) 14 | 15 | * Before implementing your change, create an issue that describes the problem you would like to solve or the code that should be enhanced. Please note that you are willing to work on that issue. 16 | 17 | * The team will review the issue and decide whether it should be implemented as a pull request. In that case, they will assign the issue to you. If the team decides against picking up the issue, the team will post a comment with an explanation. 18 | 19 | ## Steps to Contribute 20 | 21 | Should you wish to work on an issue, please claim it first by commenting on the GitHub issue that you want to work on. This is to prevent duplicated efforts from other contributors on the same issue. 22 | 23 | If you have questions about one of the issues, please comment on them, and one of the maintainers will clarify. 24 | 25 | ## Contributing Code or Documentation 26 | 27 | You are welcome to contribute code in order to fix a bug or to implement a new feature that is logged as an issue. 28 | 29 | The following rule governs code contributions: 30 | 31 | * Contributions must be licensed under the [Apache 2.0 License](./LICENSE) 32 | * Due to legal reasons, contributors will be asked to accept a Developer Certificate of Origin (DCO) when they create the first pull request to this project. This happens in an automated fashion during the submission process. SAP uses [the standard DCO text of the Linux Foundation](https://developercertificate.org/). 33 | 34 | ## Issues and Planning 35 | 36 | * We use GitHub issues to track bugs and enhancement requests. 37 | 38 | * Please provide as much context as possible when you open an issue. The information you provide must be comprehensive enough to reproduce that issue for the assignee. 39 | -------------------------------------------------------------------------------- /DISCLAIMER.md: -------------------------------------------------------------------------------- 1 | These Project development objects are not managed or delivered or intended for future inclusion as a standard component of the SAP Software. Therefore, at Project closure, these Project development objects will not include any further support services, defect resolution, maintenance, or upgrades or in any way be within scope of SAP support obligations for licensed SAP Software. Licensee is solely responsible for supporting such objects. SAP does not assure the compatibility of such objects with future releases of SAP Software or other SAP solutions. 2 | -------------------------------------------------------------------------------- /REUSE.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | SPDX-PackageName = "SAP Commerce DB Sync" 3 | SPDX-PackageSupplier = "yannick.robin@sap.com" 4 | SPDX-PackageDownloadLocation = "https://github.com/SAP/sap-commerce-db-sync" 5 | SPDX-PackageComment = "The code in this project may include calls to APIs (\"API Calls\") of\n SAP or third-party products or services developed outside of this project\n (\"External Products\").\n \"APIs\" means application programming interfaces, as well as their respective\n specifications and implementing code that allows software to communicate with\n other software.\n API Calls to External Products are not licensed under the open source license\n that governs this project. The use of such API Calls and related External\n Products are subject to applicable additional agreements with the relevant\n provider of the External Products. In no event shall the open source license\n that governs this project grant any rights in or to any External Products,or\n alter, expand or supersede any terms of the applicable additional agreements.\n If you have a valid license agreement with SAP for the use of a particular SAP\n External Product, then you may make use of any API Calls included in this\n project's code for that SAP External Product, subject to the terms of such\n license agreement. If you do not have a valid license agreement for the use of\n a particular SAP External Product, then you may only make use of any API Calls\n in this project for that SAP External Product for your internal, non-productive\n and non-commercial test and evaluation of such API Calls. Nothing herein grants\n you any rights to use or access any SAP External Product, or provide any third\n parties the right to use of access any SAP External Product, through API Calls." 6 | 7 | [[annotations]] 8 | path = "**" 9 | precedence = "aggregate" 10 | SPDX-FileCopyrightText = "2025 SAP SE or an SAP affiliate company and SAP Commerce DB Sync contributors" 11 | SPDX-License-Identifier = "Apache-2.0" 12 | -------------------------------------------------------------------------------- /commercedbsync/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /commercedbsync/extensioninfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /commercedbsync/external-dependencies.xml: -------------------------------------------------------------------------------- 1 | 6 | 8 | 4.0.0 9 | de.hybris.platform 10 | commercedbsync 11 | 6.7.0.0-RC19 12 | 13 | jar 14 | 15 | 16 | 17 | com.google.code.gson 18 | gson 19 | 2.8.6 20 | 21 | 22 | com.google.guava 23 | guava 24 | 28.0-jre 25 | 26 | 27 | org.apache.commons 28 | commons-dbcp2 29 | 2.7.0 30 | 31 | 32 | com.zaxxer 33 | HikariCP 34 | 5.0.1 35 | 36 | 37 | com.github.freva 38 | ascii-table 39 | 1.1.0 40 | 41 | 42 | com.fasterxml.jackson.datatype 43 | jackson-datatype-jsr310 44 | 2.13.3 45 | 46 | 47 | org.openjdk.jol 48 | jol-core 49 | 0.10 50 | 51 | 52 | com.fasterxml.jackson.dataformat 53 | jackson-dataformat-yaml 54 | 2.16.0 55 | 56 | 57 | org.yaml 58 | snakeyaml 59 | 2.2 60 | 61 | 62 | com.googlecode.json-simple 63 | json-simple 64 | 1.1.1 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /commercedbsync/resources/commercedbsync/sap-hybris-platform.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/commercedbsync/resources/commercedbsync/sap-hybris-platform.png -------------------------------------------------------------------------------- /commercedbsync/resources/groovy/MigrationSummaryScript.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package groovy 8 | 9 | import de.hybris.platform.util.Config 10 | import org.apache.commons.lang.StringUtils 11 | 12 | import java.util.stream.Collectors 13 | 14 | def result = generateMigrationSummary(migrationContext) 15 | println result 16 | return result 17 | 18 | def generateMigrationSummary(context) { 19 | StringBuilder sb = new StringBuilder(); 20 | try { 21 | final String sourcePrefix = context.getDataSourceRepository().getDataSourceConfiguration().getTablePrefix(); 22 | final String targetPrefix = context.getDataTargetRepository().getDataSourceConfiguration().getTablePrefix(); 23 | final String dbPrefix = Config.getString("db.tableprefix", ""); 24 | final Set sourceSet = migrationContext.getDataSourceRepository().getAllTableNames() 25 | .stream() 26 | .map({ tableName -> tableName.replace(sourcePrefix, "") }) 27 | .collect(Collectors.toSet()); 28 | 29 | final Set targetSet = migrationContext.getDataTargetRepository().getAllTableNames() 30 | sb.append("------------").append("\n") 31 | sb.append("All tables: ").append(sourceSet.size() + targetSet.size()).append("\n") 32 | sb.append("Source tables: ").append(sourceSet.size()).append("\n") 33 | sb.append("Target tables: ").append(targetSet.size()).append("\n") 34 | sb.append("------------").append("\n") 35 | sb.append("Source prefix: ").append(sourcePrefix).append("\n") 36 | sb.append("Target prefix: ").append(targetPrefix).append("\n") 37 | sb.append("DB prefix: ").append(dbPrefix).append("\n") 38 | sb.append("------------").append("\n") 39 | sb.append("Migration Type: ").append("\n") 40 | sb.append(StringUtils.isNotEmpty(dbPrefix) && 41 | StringUtils.isNotEmpty(targetPrefix) && !StringUtils.equalsIgnoreCase(dbPrefix, targetPrefix) ? "STAGED" : "DIRECT").append("\n") 42 | sb.append("------------").append("\n") 43 | sb.append("Found prefixes:").append("\n") 44 | 45 | Map prefixes = new HashMap<>() 46 | targetSet.forEach({ tableName -> 47 | String srcTable = schemaDifferenceService.findCorrespondingSrcTable(sourceSet, tableName); 48 | String prefix = tableName.replace(srcTable, ""); 49 | prefixes.put(prefix, targetSet.stream().filter({ e -> e.startsWith(prefix) }).count()); 50 | }); 51 | prefixes.forEach({ k, v -> sb.append("Prefix: ").append(k).append(" number of tables: ").append(v).append("\n") }); 52 | sb.append("------------").append("\n"); 53 | 54 | } catch (Exception e) { 55 | e.printStackTrace(); 56 | } 57 | return sb.toString(); 58 | } 59 | 60 | -------------------------------------------------------------------------------- /commercedbsync/resources/groovy/ddlaltercreate.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | // Parameters 8 | queryToRun = 'select p_streetname from addresses where p_streetname like \'%Test%\'' 9 | indexToCreateaddresses = 'CREATE INDEX addresses_ownerpkstring_ddl ON ADDRESSES(typepkstring, ownerpkstring) ONLINE;' 10 | indexToDropAddresses = 'drop index addresses_ownerpkstring_ddl ONLINE;' 11 | AlterAddColumnAddressesQuery = 'ALTER TABLE ADDRESSES ADD (TEST%s BIGINT) ONLINE;' 12 | AlterRemoveColumnAddresses = 'ALTER TABLE ADDRESSES DROP (TEST%s) ONLINE;' 13 | 14 | queryLoopCount = 600 15 | threadSize = 1 16 | threadPoolSize = 1 // Long Running Query Test 17 | 18 | import java.util.concurrent.Callable 19 | import java.util.concurrent.TimeUnit 20 | import java.util.concurrent.Executors 21 | import groovy.time.TimeCategory 22 | import de.hybris.platform.core.Registry 23 | import de.hybris.platform.jalo.JaloSession 24 | import de.hybris.platform.core.Tenant 25 | 26 | queryTasks = [] 27 | Tenant currentTenant = Registry.getCurrentTenant(); 28 | // Create Callable Threads 29 | 1.upto(threadSize) { index -> 30 | // Create a database connection within each Thread 31 | queryTasks << { 32 | def totalTime1 = 0 33 | def totalTime2 = 0 34 | def indexCreation = true; 35 | // Each thread runs a thread/loop specific template query X times 36 | 1.upto(queryLoopCount) { loopIndex -> 37 | 38 | try { 39 | Registry.setCurrentTenant(currentTenant); 40 | JaloSession.getCurrentSession().activate(); 41 | 42 | start = new Date() 43 | totalTime2 = 0 44 | String AlterAddColumnAddresses = String.format(AlterRemoveColumnAddresses ,loopIndex); 45 | println(AlterAddColumnAddresses) 46 | jdbcTemplate.execute(AlterAddColumnAddresses) 47 | stop = new Date() 48 | totalTime2 += TimeCategory.minus(stop, start).toMilliseconds() 49 | println "Table Column Creation AlterAddColumnAddresses loop ${loopIndex} totalTime(ms) ${totalTime2}" 50 | // Thread.sleep(5000); 51 | 52 | } finally { 53 | JaloSession.getCurrentSession().close(); 54 | Registry.unsetCurrentTenant(); 55 | } 56 | } 57 | // Return average as the result of the Callable 58 | totalTime1 + totalTime2 / queryLoopCount 59 | } as Callable 60 | } 61 | 62 | executorService = Executors.newFixedThreadPool(threadPoolSize) 63 | println "Test started at ${new Date()}" 64 | results = executorService.invokeAll(queryTasks) 65 | totalAverage = 0 66 | results.eachWithIndex { it, index -> 67 | totalAverage += it.get() 68 | println "$index --> ${it.get()}" 69 | } 70 | println "Total Average --> ${totalAverage / threadSize}" 71 | println "Test finished at ${new Date()}" 72 | executorService.shutdown() 73 | executorService.awaitTermination(200, TimeUnit.SECONDS) -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_de.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_en.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | type.MigrationCronJob.truncateEnabled.name=Truncate Enabled 8 | type.MigrationCronJob.truncateEnabled.description= 9 | 10 | type.MigrationCronJob.schemaAutotrigger.name=Schema Auto Trigger Enabled 11 | type.MigrationCronJob.schemaAutotrigger.description= 12 | 13 | type.MigrationCronJob.lastStartTime.name=Last Start time For Incremental Job 14 | type.MigrationCronJob.lastStartTime.description= 15 | 16 | type.MigrationCronJob.migrationItems.name=Migration Tables 17 | type.MigrationCronJob.migrationItems.description= 18 | 19 | type.FullMigrationCronJob.fullDatabaseMigration.name=Full Database Migration 20 | type.FullMigrationCronJob.fullDatabaseMigration.description= 21 | 22 | type.MigrationCronJob.maxParallelTableCopy.name=Parallel Tables 23 | type.MigrationCronJob.maxParallelTableCopy.description=Number of tables to be copied in parallel 24 | 25 | type.MigrationCronJob.maxReaderWorkers.name=Reader Workers 26 | type.MigrationCronJob.maxReaderWorkers.description=Number of reader workers to be used for each table 27 | 28 | type.MigrationCronJob.maxWriterWorkers.name=Writer Workers 29 | type.MigrationCronJob.maxWriterWorkers.description=Number of writer workers to be used for each table 30 | 31 | type.MigrationCronJob.batchSize.name=Batch Size 32 | type.MigrationCronJob.batchSize.description=Batch size used to query data 33 | 34 | type.FullMigrationCronJob.resumeMigration.name=Resume Migration 35 | type.FullMigrationCronJob.resumeMigration.description= 36 | 37 | type.FullMigrationCronJob.migrationId.name=Migration ID 38 | type.FullMigrationCronJob.migrationId.description= 39 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_es.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_fr.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_it.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_ja.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_ko.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_pt.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_ru.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/localization/commercedbsync-locales_zh.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsync/resources/sql/transformationFunctions/mssql-general.sql: -------------------------------------------------------------------------------- 1 | CREATE OR ALTER FUNCTION mask_str (@originialvalue nvarchar(255), @maskedvalue nvarchar(255)) 2 | RETURNS nvarchar(255) 3 | AS 4 | BEGIN 5 | IF @originialvalue IS NULL 6 | RETURN NULL 7 | 8 | RETURN @maskedvalue 9 | END; 10 | 11 | 12 | CREATE OR ALTER FUNCTION mask_unique_str (@pk bigint, @suffix nvarchar(255)) 13 | RETURNS nvarchar(255) 14 | AS 15 | BEGIN 16 | RETURN CONCAT(@pk, @suffix) 17 | END; 18 | 19 | CREATE OR ALTER FUNCTION mask_password (@uid nvarchar(255)) 20 | RETURNS nvarchar(255) 21 | AS 22 | BEGIN 23 | IF @uid = 'admin' 24 | RETURN 'nimda' 25 | 26 | RETURN NULL 27 | END; 28 | 29 | CREATE OR ALTER FUNCTION mask_passwordencoding (@uid nvarchar(255)) 30 | RETURNS nvarchar(255) 31 | AS 32 | BEGIN 33 | IF @uid = 'admin' 34 | RETURN 'plain' 35 | 36 | RETURN '*' 37 | END; -------------------------------------------------------------------------------- /commercedbsync/resources/sql/transformationFunctions/mssql-typeinfotable.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS MIGRATIONTOOLKIT_TF_TYPEINFO; 2 | 3 | CREATE TABLE MIGRATIONTOOLKIT_TF_TYPEINFO ( 4 | InternalCode NVARCHAR(255) NOT NULL, 5 | PK BIGINT NOT NULL 6 | PRIMARY KEY (InternalCode) 7 | ); -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/CommercedbsyncStandalone.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync; 8 | 9 | import de.hybris.platform.core.Registry; 10 | import de.hybris.platform.jalo.JaloSession; 11 | import de.hybris.platform.util.RedeployUtilities; 12 | import de.hybris.platform.util.Utilities; 13 | 14 | /** 15 | * Demonstration of how to write a standalone application that can be run 16 | * directly from within eclipse or from the commandline.
17 | * To run this from commandline, just use the following command:
18 | * 19 | * java -jar bootstrap/bin/ybootstrap.jar "new CommercedbsyncStandalone().run();" 20 | * From eclipse, just run as Java Application. Note that you maybe need 21 | * to add all other projects like ext-commerce, ext-pim to the Launch 22 | * configuration classpath. 23 | */ 24 | public class CommercedbsyncStandalone { 25 | /** 26 | * Main class to be able to run it directly as a java program. 27 | * 28 | * @param args 29 | * the arguments from commandline 30 | */ 31 | public static void main(final String[] args) { 32 | new CommercedbsyncStandalone().run(); 33 | } 34 | 35 | public void run() { 36 | Registry.activateStandaloneMode(); 37 | Registry.activateMasterTenant(); 38 | 39 | final JaloSession jaloSession = JaloSession.getCurrentSession(); 40 | System.out.println("Session ID: " + jaloSession.getSessionID()); // NOPMD 41 | System.out.println("User: " + jaloSession.getUser()); // NOPMD 42 | Utilities.printAppInfo(); 43 | 44 | RedeployUtilities.shutdown(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/adapter/DataRepositoryAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.adapter; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 10 | import com.sap.cx.boosters.commercedbsync.dataset.DataSet; 11 | import com.sap.cx.boosters.commercedbsync.MarkersQueryDefinition; 12 | import com.sap.cx.boosters.commercedbsync.OffsetQueryDefinition; 13 | import com.sap.cx.boosters.commercedbsync.SeekQueryDefinition; 14 | import java.util.List; 15 | 16 | public interface DataRepositoryAdapter { 17 | long getRowCount(MigrationContext context, String table) throws Exception; 18 | 19 | DataSet getAll(MigrationContext context, String table) throws Exception; 20 | 21 | DataSet getBatchWithoutIdentifier(MigrationContext context, OffsetQueryDefinition queryDefinition) throws Exception; 22 | 23 | DataSet getBatchOrderedByColumn(MigrationContext context, SeekQueryDefinition queryDefinition) throws Exception; 24 | 25 | DataSet getBatchMarkersOrderedByColumn(MigrationContext context, MarkersQueryDefinition queryDefinition) 26 | throws Exception; 27 | 28 | List getPartitions(String table) throws Exception; 29 | } 30 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/AnonymizeFunctions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer; 8 | 9 | import com.sap.cx.boosters.commercedbsync.anonymizer.functions.FunctionEvaluator; 10 | import com.sap.cx.boosters.commercedbsync.anonymizer.functions.impl.GuidFunctionEvaluator; 11 | import com.sap.cx.boosters.commercedbsync.anonymizer.functions.impl.RandomFunctionEvaluator; 12 | 13 | import java.util.List; 14 | 15 | public class AnonymizeFunctions { 16 | private static final List evaluators = List.of(new GuidFunctionEvaluator(), 17 | new RandomFunctionEvaluator()); 18 | 19 | public String getForFunction(final String function) { 20 | final String thisFunction = function.replaceAll("[{}]", "").trim(); 21 | return evaluators.stream().filter(evaluator1 -> evaluator1.canApply(thisFunction)).findAny() 22 | .map(evaluator -> evaluator.getForFunction(thisFunction)).orElse(""); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/AnonymizerConfigurator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer; 8 | 9 | import com.fasterxml.jackson.databind.ObjectMapper; 10 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; 11 | import com.sap.cx.boosters.commercedbsync.anonymizer.model.AnonymizerConfiguration; 12 | import org.springframework.core.io.ClassPathResource; 13 | 14 | import java.io.IOException; 15 | 16 | public class AnonymizerConfigurator { 17 | private final AnonymizerConfiguration anonymizerConfiguration; 18 | 19 | public AnonymizerConfigurator() { 20 | final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); 21 | try { 22 | anonymizerConfiguration = objectMapper.readValue(new ClassPathResource("./anonymizer/config.yml").getFile(), 23 | AnonymizerConfiguration.class); 24 | } catch (IOException e) { 25 | throw new RuntimeException("Error reading anonymizer configuration", e); 26 | } 27 | } 28 | 29 | public AnonymizerConfiguration getConfiguration() { 30 | return anonymizerConfiguration; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/TextEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer; 8 | 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | public class TextEvaluator { 13 | private final AnonymizeFunctions anonymizeFunctions; 14 | 15 | public TextEvaluator() { 16 | this.anonymizeFunctions = new AnonymizeFunctions(); 17 | } 18 | 19 | public TextEvaluator(AnonymizeFunctions anonymizeFunctions) { 20 | this.anonymizeFunctions = anonymizeFunctions; 21 | } 22 | 23 | public String getForTokens(final List tokens) { 24 | return tokens.stream().map(token -> isTokenFunction(token) ? anonymizeFunctions.getForFunction(token) : token) 25 | .collect(Collectors.joining()); 26 | } 27 | 28 | private boolean isTokenFunction(String token) { 29 | return token.startsWith("{") && token.endsWith("}"); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/TextTokenizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer; 8 | 9 | import com.sap.cx.boosters.commercedbsync.anonymizer.exception.ValidationException; 10 | import org.apache.commons.lang3.StringUtils; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.regex.Matcher; 15 | import java.util.regex.Pattern; 16 | 17 | public class TextTokenizer { 18 | private static final Pattern TOKEN_PATTERN = Pattern.compile("\\{[^}]*}|[^{}]+"); 19 | 20 | public List tokenizeText(final String text) { 21 | checkForErrors(text); 22 | if (StringUtils.isBlank(text)) { 23 | return List.of(); 24 | } 25 | 26 | if (!text.contains("{")) { 27 | return List.of(text.trim()); 28 | } 29 | 30 | final Matcher matcher = TOKEN_PATTERN.matcher(text); 31 | final List tokens = new ArrayList<>(); 32 | while (matcher.find()) { 33 | tokens.add(matcher.group()); 34 | } 35 | return tokens; 36 | } 37 | 38 | private void checkForErrors(String text) { 39 | if (StringUtils.countMatches(text, "{") != StringUtils.countMatches(text, "}") 40 | || StringUtils.countMatches(text, "(") != StringUtils.countMatches(text, ")")) { 41 | throw new ValidationException( 42 | "Text " + text + " has invalid defined functions (not closing/opening functions)"); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/exception/ValidationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.exception; 8 | 9 | public class ValidationException extends RuntimeException { 10 | public ValidationException(String text) { 11 | super(text); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/functions/FunctionEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.functions; 8 | 9 | public interface FunctionEvaluator { 10 | boolean canApply(String function); 11 | String getForFunction(String function); 12 | } 13 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/functions/impl/GuidFunctionEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.functions.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.anonymizer.functions.FunctionEvaluator; 10 | 11 | import java.util.UUID; 12 | 13 | public class GuidFunctionEvaluator implements FunctionEvaluator { 14 | @Override 15 | public boolean canApply(String function) { 16 | return function.equals("GUID"); 17 | } 18 | 19 | @Override 20 | public String getForFunction(String function) { 21 | return UUID.randomUUID().toString(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/functions/impl/RandomAddressGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.functions.impl; 8 | 9 | import org.apache.logging.log4j.LogManager; 10 | import org.apache.logging.log4j.Logger; 11 | import org.json.simple.JSONObject; 12 | import org.json.simple.parser.JSONParser; 13 | import org.json.simple.parser.ParseException; 14 | 15 | import java.io.IOException; 16 | import java.io.InputStream; 17 | import java.io.InputStreamReader; 18 | import java.security.SecureRandom; 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | public class RandomAddressGenerator { 25 | private static final String CITY = "city"; 26 | private static final String STREET = "street"; 27 | 28 | private final JSONParser parser; 29 | private final Map> map; 30 | private final SecureRandom random; 31 | private static final Logger LOGGER = LogManager.getLogger(RandomAddressGenerator.class); 32 | 33 | public RandomAddressGenerator() { 34 | this.parser = new JSONParser(); 35 | this.map = getAddressMap(); 36 | this.random = new SecureRandom(); 37 | } 38 | 39 | public String generateRandomCity() { 40 | final List city = map.get(CITY); 41 | return city.get(random.nextInt(city.size())); 42 | } 43 | 44 | public String generateRandomStreet() { 45 | final List streets = map.get(STREET); 46 | return streets.get(random.nextInt(streets.size())); 47 | } 48 | 49 | @SuppressWarnings("unchecked") 50 | private Map> getAddressMap() { 51 | Map> addressMap = new HashMap<>(); 52 | try (final InputStream inputStream = RandomAddressGenerator.class.getClassLoader() 53 | .getResourceAsStream("./anonymizer/address.json")) { 54 | if (inputStream == null) { 55 | return Collections.emptyMap(); 56 | } 57 | JSONObject jsonObject = (JSONObject) parser.parse(new InputStreamReader(inputStream)); 58 | List cityList = (List) jsonObject.get(CITY); 59 | List streetList = (List) jsonObject.get(STREET); 60 | addressMap.put(CITY, cityList); 61 | addressMap.put(STREET, streetList); 62 | } catch (IOException e) { 63 | LOGGER.error("IOException: {}", e.getMessage()); 64 | } catch (ParseException e) { 65 | LOGGER.error("ParseException: {} ", e.getMessage()); 66 | } 67 | return addressMap; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/model/AnonymizerConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.model; 8 | 9 | import org.apache.commons.collections4.CollectionUtils; 10 | 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | public class AnonymizerConfiguration { 16 | private List tables; 17 | private final Map tableMap = new HashMap<>(); 18 | 19 | public List
getTables() { 20 | return tables; 21 | } 22 | 23 | public void setTables(List
tables) { 24 | CollectionUtils.emptyIfNull(tables).forEach(table -> tableMap.put(table.getName(), table)); 25 | this.tables = tables; 26 | } 27 | 28 | public Table getTable(String tableName) { 29 | return tableMap.get(tableName); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/model/Column.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.model; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Objects; 12 | 13 | public class Column { 14 | private String name; 15 | private Operation operation; 16 | private String text; 17 | private List exclude = new ArrayList<>(); 18 | private List excludeRow = new ArrayList<>(); 19 | 20 | public Column() { 21 | } 22 | 23 | public Column(final String name) { 24 | this.name = name; 25 | } 26 | 27 | public String getName() { 28 | return name; 29 | } 30 | public void setName(String name) { 31 | this.name = name; 32 | } 33 | public Operation getOperation() { 34 | return operation; 35 | } 36 | public void setOperation(Operation operation) { 37 | this.operation = operation; 38 | } 39 | 40 | public String getText() { 41 | return text; 42 | } 43 | public void setText(String text) { 44 | this.text = text; 45 | } 46 | 47 | public List getExclude() { 48 | return exclude; 49 | } 50 | public void setExclude(List exclude) { 51 | this.exclude = exclude; 52 | } 53 | 54 | public List getExcludeRow() { 55 | return excludeRow; 56 | } 57 | public void setExcludeRow(List excludeRow) { 58 | this.excludeRow = excludeRow; 59 | } 60 | 61 | @Override 62 | public boolean equals(Object o) { 63 | if (this == o) { 64 | return true; 65 | } 66 | if (o == null || getClass() != o.getClass()) { 67 | return false; 68 | } 69 | Column column = (Column) o; 70 | return Objects.equals(name, column.name); 71 | } 72 | 73 | @Override 74 | public int hashCode() { 75 | return Objects.hashCode(name); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/model/Operation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.model; 8 | 9 | public enum Operation { 10 | REPLACE, APPEND, DELETE 11 | } 12 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/anonymizer/model/Table.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.anonymizer.model; 8 | 9 | import org.apache.commons.collections4.CollectionUtils; 10 | 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.Objects; 15 | 16 | public class Table { 17 | private String name; 18 | private List columns; 19 | private final Map columnMap = new HashMap<>(); 20 | 21 | public Table() { 22 | } 23 | 24 | public Table(final String name) { 25 | this.name = name; 26 | } 27 | 28 | public String getName() { 29 | return name; 30 | } 31 | 32 | public void setName(String name) { 33 | this.name = name; 34 | } 35 | 36 | public List getColumns() { 37 | return columns; 38 | } 39 | 40 | public void setColumns(List columns) { 41 | CollectionUtils.emptyIfNull(columns).forEach(column -> columnMap.put(column.getName(), column)); 42 | this.columns = columns; 43 | } 44 | 45 | public Column getColumn(final String columnName) { 46 | return columnMap.get(columnName); 47 | } 48 | 49 | @Override 50 | public boolean equals(Object o) { 51 | if (this == o) { 52 | return true; 53 | } 54 | if (o == null || getClass() != o.getClass()) { 55 | return false; 56 | } 57 | Table table = (Table) o; 58 | return Objects.equals(name, table.name); 59 | } 60 | 61 | @Override 62 | public int hashCode() { 63 | return Objects.hashCode(name); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataCopyMethod.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | public enum DataCopyMethod { 10 | SEEK, OFFSET, DEFAULT 11 | } 12 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataPipe.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import javax.annotation.concurrent.ThreadSafe; 10 | 11 | /** 12 | * Used to separate database reading and writing operations, after reading data 13 | * from the DB, the result is put to the pipe and can be used by the database 14 | * writer later on -> asynchronously 15 | * 16 | * @param 17 | */ 18 | @ThreadSafe 19 | public interface DataPipe { 20 | void requestAbort(Exception e); 21 | 22 | void put(MaybeFinished value) throws Exception; 23 | 24 | MaybeFinished get() throws Exception; 25 | 26 | int size(); 27 | 28 | int getWaitersCount(); 29 | } 30 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataPipeFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | 11 | import javax.annotation.concurrent.ThreadSafe; 12 | 13 | @ThreadSafe 14 | public interface DataPipeFactory { 15 | DataPipe create(CopyContext context, CopyContext.DataCopyItem item) throws Exception; 16 | } 17 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataThreadPoolConfigBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import com.sap.cx.boosters.commercedbsync.DataThreadPoolConfig; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | 12 | public class DataThreadPoolConfigBuilder { 13 | 14 | private final DataThreadPoolConfig config; 15 | 16 | public DataThreadPoolConfigBuilder(MigrationContext context) { 17 | config = new DataThreadPoolConfig(); 18 | } 19 | 20 | public DataThreadPoolConfigBuilder withPoolSize(int poolSize) { 21 | config.setPoolSize(poolSize); 22 | return this; 23 | } 24 | 25 | public DataThreadPoolConfig build() { 26 | return config; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataThreadPoolFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | import com.sap.cx.boosters.commercedbsync.DataThreadPoolConfig; 11 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 12 | 13 | public interface DataThreadPoolFactory { 14 | ThreadPoolTaskExecutor create(CopyContext context, DataThreadPoolConfig config); 15 | 16 | void destroy(ThreadPoolTaskExecutor executor); 17 | 18 | DataThreadPoolMonitor getMonitor(); 19 | } 20 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataThreadPoolMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 10 | 11 | public interface DataThreadPoolMonitor { 12 | void subscribe(ThreadPoolTaskExecutor executor); 13 | 14 | void unsubscribe(ThreadPoolTaskExecutor executor); 15 | 16 | int getActiveCount(); 17 | 18 | int getMaxPoolSize(); 19 | } 20 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataWorkerExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import java.util.concurrent.Callable; 10 | import java.util.concurrent.ExecutionException; 11 | import java.util.concurrent.Future; 12 | 13 | public interface DataWorkerExecutor { 14 | Future safelyExecute(Callable callable) throws InterruptedException; 15 | 16 | void waitAndRethrowUncaughtExceptions() throws ExecutionException, InterruptedException; 17 | } 18 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/DataWorkerPoolFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 11 | 12 | public interface DataWorkerPoolFactory { 13 | ThreadPoolTaskExecutor create(CopyContext context); 14 | } 15 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/MDCTaskDecorator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | import org.slf4j.MDC; 10 | import org.springframework.core.task.TaskDecorator; 11 | 12 | import java.util.Map; 13 | 14 | public class MDCTaskDecorator implements TaskDecorator { 15 | @Override 16 | public Runnable decorate(Runnable runnable) { 17 | Map contextMap = MDC.getCopyOfContextMap(); 18 | return () -> { 19 | try { 20 | MDC.setContextMap(contextMap); 21 | runnable.run(); 22 | } finally { 23 | MDC.clear(); 24 | } 25 | }; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/MaybeFinished.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | /** 10 | * MaybeFinished keeps track status of the data set that is currently being 11 | * processed -> if all is ok, then status will be done, if theres an exception, 12 | * it will be poison 13 | * 14 | * @param 15 | */ 16 | public final class MaybeFinished { 17 | private final T value; 18 | private final boolean done; 19 | private final boolean poison; 20 | 21 | private MaybeFinished(T value, boolean done, boolean poison) { 22 | this.value = value; 23 | this.done = done; 24 | this.poison = poison; 25 | } 26 | 27 | public static MaybeFinished of(T value) { 28 | return new MaybeFinished<>(value, false, false); 29 | } 30 | 31 | public static MaybeFinished finished(T value) { 32 | return new MaybeFinished<>(value, true, false); 33 | } 34 | 35 | public static MaybeFinished poison() { 36 | return new MaybeFinished<>(null, true, true); 37 | } 38 | 39 | public T getValue() { 40 | return value; 41 | } 42 | 43 | public boolean isDone() { 44 | return done; 45 | } 46 | 47 | public boolean isPoison() { 48 | return poison; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/PipeAbortedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent; 8 | 9 | public class PipeAbortedException extends Exception { 10 | public PipeAbortedException(String message) { 11 | super(message); 12 | } 13 | 14 | public PipeAbortedException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/DefaultDataThreadPoolMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.DataThreadPoolMonitor; 10 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 11 | 12 | import java.util.ArrayList; 13 | import java.util.Collections; 14 | import java.util.List; 15 | 16 | /** 17 | * 18 | */ 19 | public class DefaultDataThreadPoolMonitor implements DataThreadPoolMonitor { 20 | 21 | private final List executors; 22 | 23 | public DefaultDataThreadPoolMonitor() { 24 | this.executors = Collections.synchronizedList(new ArrayList<>()); 25 | } 26 | 27 | @Override 28 | public void subscribe(ThreadPoolTaskExecutor executor) { 29 | executors.add(executor); 30 | } 31 | 32 | @Override 33 | public void unsubscribe(ThreadPoolTaskExecutor executor) { 34 | executors.remove(executor); 35 | } 36 | 37 | @Override 38 | public int getActiveCount() { 39 | return executors.stream().mapToInt(ThreadPoolTaskExecutor::getActiveCount).sum(); 40 | } 41 | 42 | @Override 43 | public int getMaxPoolSize() { 44 | return executors.stream().mapToInt(ThreadPoolTaskExecutor::getMaxPoolSize).sum(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/DefaultDataWorkerExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.DataWorkerExecutor; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.core.task.AsyncTaskExecutor; 13 | import org.springframework.core.task.TaskRejectedException; 14 | import org.springframework.util.backoff.BackOffExecution; 15 | import org.springframework.util.backoff.ExponentialBackOff; 16 | 17 | import java.util.ArrayDeque; 18 | import java.util.Queue; 19 | import java.util.concurrent.Callable; 20 | import java.util.concurrent.ExecutionException; 21 | import java.util.concurrent.Future; 22 | 23 | public class DefaultDataWorkerExecutor implements DataWorkerExecutor { 24 | 25 | private static final Logger LOG = LoggerFactory.getLogger(DefaultDataWorkerExecutor.class); 26 | 27 | private final AsyncTaskExecutor executor; 28 | private final Queue> futures = new ArrayDeque<>(); 29 | 30 | public DefaultDataWorkerExecutor(AsyncTaskExecutor executor) { 31 | this.executor = executor; 32 | } 33 | 34 | @Override 35 | public Future safelyExecute(Callable callable) throws InterruptedException { 36 | Future future = internalSafelyExecute(callable, 0); 37 | futures.add(future); 38 | return future; 39 | } 40 | 41 | private Future internalSafelyExecute(Callable callable, int rejections) throws InterruptedException { 42 | try { 43 | return executor.submit(callable); 44 | } catch (TaskRejectedException e) { 45 | BackOffExecution backOff = new ExponentialBackOff().start(); 46 | long waitInterval = backOff.nextBackOff(); 47 | for (int i = 0; i < rejections; i++) { 48 | waitInterval = backOff.nextBackOff(); 49 | } 50 | LOG.trace("Could not fetch new worker, because all are busy. Retrying again in {}ms...", waitInterval); 51 | Thread.sleep(waitInterval); 52 | return internalSafelyExecute(callable, rejections + 1); 53 | } 54 | } 55 | 56 | @Override 57 | public void waitAndRethrowUncaughtExceptions() throws ExecutionException, InterruptedException { 58 | Future future; 59 | while ((future = futures.poll()) != null) { 60 | future.get(); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/task/BatchOffsetDataReaderTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl.task; 8 | 9 | import com.sap.cx.boosters.commercedbsync.OffsetQueryDefinition; 10 | import com.sap.cx.boosters.commercedbsync.adapter.DataRepositoryAdapter; 11 | import com.sap.cx.boosters.commercedbsync.concurrent.MaybeFinished; 12 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 13 | import com.sap.cx.boosters.commercedbsync.dataset.DataSet; 14 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceUnit; 15 | import java.util.Set; 16 | import java.util.stream.Collectors; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | public class BatchOffsetDataReaderTask extends DataReaderTask { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(BatchOffsetDataReaderTask.class); 23 | 24 | private final long offset; 25 | private final Set batchColumns; 26 | private final int batchId; 27 | 28 | public BatchOffsetDataReaderTask(PipeTaskContext pipeTaskContext, int batchId, long offset, 29 | Set batchColumns) { 30 | super(pipeTaskContext); 31 | this.batchId = batchId; 32 | this.offset = offset; 33 | this.batchColumns = batchColumns; 34 | } 35 | 36 | @Override 37 | protected Boolean internalRun() throws Exception { 38 | waitForFreeMemory(); 39 | process(); 40 | return Boolean.TRUE; 41 | } 42 | 43 | private void process() throws Exception { 44 | DataRepositoryAdapter adapter = getPipeTaskContext().getDataRepositoryAdapter(); 45 | CopyContext context = getPipeTaskContext().getContext(); 46 | String table = getPipeTaskContext().getTable(); 47 | long pageSize = getPipeTaskContext().getPageSize(); 48 | OffsetQueryDefinition queryDefinition = new OffsetQueryDefinition(); 49 | queryDefinition.setBatchId(batchId); 50 | queryDefinition.setTable(table); 51 | queryDefinition.setOrderByColumns(batchColumns.stream().collect(Collectors.joining(","))); 52 | queryDefinition.setBatchSize(pageSize); 53 | queryDefinition.setOffset(offset); 54 | queryDefinition.setDeletionEnabled(context.getMigrationContext().isDeletionEnabled()); 55 | queryDefinition.setLpTableEnabled(context.getMigrationContext().isLpTableMigrationEnabled()); 56 | DataSet result = adapter.getBatchWithoutIdentifier(context.getMigrationContext(), queryDefinition); 57 | profileData(context, batchId, table, pageSize, result); 58 | getPipeTaskContext().getRecorder().record(PerformanceUnit.ROWS, result.getAllResults().size()); 59 | getPipeTaskContext().getPipe().put(MaybeFinished.of(result)); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/task/DataReaderTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl.task; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.PipeAbortedException; 10 | import com.sap.cx.boosters.commercedbsync.constants.CommercedbsyncConstants; 11 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 12 | import com.sap.cx.boosters.commercedbsync.dataset.DataSet; 13 | import de.hybris.platform.core.MasterTenant; 14 | import org.openjdk.jol.info.GraphLayout; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | public abstract class DataReaderTask extends RetriableTask { 19 | 20 | private static final Logger LOG = LoggerFactory.getLogger(DataReaderTask.class); 21 | 22 | private final PipeTaskContext pipeTaskContext; 23 | 24 | protected DataReaderTask(PipeTaskContext pipeTaskContext) { 25 | super(pipeTaskContext.getContext(), pipeTaskContext.getTable()); 26 | this.pipeTaskContext = pipeTaskContext; 27 | } 28 | 29 | public PipeTaskContext getPipeTaskContext() { 30 | return pipeTaskContext; 31 | } 32 | 33 | protected void waitForFreeMemory() throws Exception { 34 | CopyContext context = getPipeTaskContext().getContext(); 35 | final long minMem = context.getMigrationContext().getMemoryMin(); 36 | long freeMem = Runtime.getRuntime().freeMemory(); 37 | 38 | int cnt = 0; 39 | while (freeMem < minMem) { 40 | LOG.trace("Waiting for freeMem {} / {} Attempts={}", freeMem, minMem, cnt); 41 | Thread.sleep(context.getMigrationContext().getMemoryWait()); 42 | cnt++; 43 | if (cnt >= context.getMigrationContext().getMemoryMaxAttempts()) { 44 | throw new PipeAbortedException("Maximum wait time exceeded. See property " 45 | + CommercedbsyncConstants.MIGRATION_PROFILING_MEMORY_ATTEMPTS + " for more details."); 46 | } 47 | freeMem = Runtime.getRuntime().freeMemory(); 48 | } 49 | } 50 | 51 | protected void profileData(final CopyContext context, final int batchId, final String table, final long pageSize, 52 | final DataSet result) { 53 | if (context.getMigrationContext().isProfiling() && result != null) { 54 | final long objSize = GraphLayout.parseInstance(result.getAllResults()).totalSize(); 55 | final long freeMem = Runtime.getRuntime().freeMemory(); 56 | final int clusterID = MasterTenant.getInstance().getClusterID(); 57 | LOG.trace( 58 | "Memory usage: [{}], Table = {}, BatchId = {}, Page Size = {}, Batch Memory Size = {}, Free System Memory = {}", 59 | clusterID, table, batchId, pageSize, objSize, freeMem); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/task/DefaultDataReaderTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl.task; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.MaybeFinished; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.dataset.DataSet; 12 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceUnit; 13 | 14 | public class DefaultDataReaderTask extends DataReaderTask { 15 | 16 | public DefaultDataReaderTask(PipeTaskContext pipeTaskContext) { 17 | super(pipeTaskContext); 18 | } 19 | 20 | @Override 21 | protected Boolean internalRun() throws Exception { 22 | waitForFreeMemory(); 23 | process(); 24 | return Boolean.TRUE; 25 | } 26 | 27 | private void process() throws Exception { 28 | MigrationContext migrationContext = getPipeTaskContext().getContext().getMigrationContext(); 29 | DataSet all = getPipeTaskContext().getDataRepositoryAdapter().getAll(migrationContext, 30 | getPipeTaskContext().getTable()); 31 | profileData(getPipeTaskContext().getContext(), -1, getPipeTaskContext().getTable(), 32 | getPipeTaskContext().getPageSize(), all); 33 | getPipeTaskContext().getRecorder().record(PerformanceUnit.ROWS, all.getAllResults().size()); 34 | getPipeTaskContext().getPipe().put(MaybeFinished.of(all)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/task/PartitionedBatchMarkerDataReaderTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl.task; 8 | 9 | import com.sap.cx.boosters.commercedbsync.SeekQueryDefinition; 10 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 11 | import org.apache.commons.lang3.tuple.Pair; 12 | 13 | public class PartitionedBatchMarkerDataReaderTask extends BatchMarkerDataReaderTask { 14 | private final String partition; 15 | 16 | public PartitionedBatchMarkerDataReaderTask(PipeTaskContext pipeTaskContext, int batchId, String batchColumn, 17 | Pair batchMarkersPair, boolean upperBoundInclusive, String partition) { 18 | super(pipeTaskContext, batchId, batchColumn, batchMarkersPair, upperBoundInclusive); 19 | this.partition = partition; 20 | } 21 | 22 | @Override 23 | protected SeekQueryDefinition createSeekQueryDefinition(final Object lastValue, final Object nextValue, 24 | final String table, final long pageSize, final CopyContext ctx) { 25 | final var seekQueryDefinition = super.createSeekQueryDefinition(lastValue, nextValue, table, pageSize, ctx); 26 | seekQueryDefinition.setPartition(getPartition()); 27 | return seekQueryDefinition; 28 | } 29 | 30 | public String getPartition() { 31 | return partition; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/task/PipeTaskContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl.task; 8 | 9 | import com.sap.cx.boosters.commercedbsync.adapter.DataRepositoryAdapter; 10 | import com.sap.cx.boosters.commercedbsync.concurrent.DataPipe; 11 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 12 | import com.sap.cx.boosters.commercedbsync.dataset.DataSet; 13 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceRecorder; 14 | import com.sap.cx.boosters.commercedbsync.service.DatabaseCopyTaskRepository; 15 | 16 | public class PipeTaskContext { 17 | private final CopyContext context; 18 | private final DataPipe pipe; 19 | private final String table; 20 | private final DataRepositoryAdapter dataRepositoryAdapter; 21 | private final long pageSize; 22 | private final PerformanceRecorder recorder; 23 | private final DatabaseCopyTaskRepository taskRepository; 24 | 25 | public PipeTaskContext(CopyContext context, DataPipe pipe, String table, 26 | DataRepositoryAdapter dataRepositoryAdapter, long pageSize, PerformanceRecorder recorder, 27 | DatabaseCopyTaskRepository taskRepository) { 28 | this.context = context; 29 | this.pipe = pipe; 30 | this.table = table; 31 | this.dataRepositoryAdapter = dataRepositoryAdapter; 32 | this.pageSize = pageSize; 33 | this.recorder = recorder; 34 | this.taskRepository = taskRepository; 35 | } 36 | 37 | public CopyContext getContext() { 38 | return context; 39 | } 40 | 41 | public DataPipe getPipe() { 42 | return pipe; 43 | } 44 | 45 | public String getTable() { 46 | return table; 47 | } 48 | 49 | public DataRepositoryAdapter getDataRepositoryAdapter() { 50 | return dataRepositoryAdapter; 51 | } 52 | 53 | public long getPageSize() { 54 | return pageSize; 55 | } 56 | 57 | public PerformanceRecorder getRecorder() { 58 | return recorder; 59 | } 60 | 61 | public DatabaseCopyTaskRepository getTaskRepository() { 62 | return taskRepository; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/concurrent/impl/task/RetriableTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.concurrent.impl.task; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.PipeAbortedException; 10 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 11 | import org.apache.commons.lang3.exception.ExceptionUtils; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.concurrent.Callable; 16 | 17 | public abstract class RetriableTask implements Callable { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(RetriableTask.class); 20 | 21 | private final CopyContext context; 22 | private final String table; 23 | private int retryCount = 0; 24 | 25 | protected RetriableTask(CopyContext context, String table) { 26 | this.context = context; 27 | this.table = table; 28 | } 29 | 30 | @Override 31 | public Boolean call() { 32 | try { 33 | return internalRun(); 34 | } catch (PipeAbortedException e) { 35 | throw new RuntimeException("Ignore retries", e); 36 | } catch (Exception e) { 37 | if (retryCount < context.getMigrationContext().getMaxWorkerRetryAttempts()) { 38 | LOG.error("Retrying failed task {} for table {}. Retry count: {}. Cause: {}", getClass().getName(), 39 | table, retryCount, e); 40 | retryCount++; 41 | return call(); 42 | } else { 43 | handleFailure(e); 44 | return Boolean.FALSE; 45 | } 46 | } 47 | } 48 | 49 | protected void handleFailure(Exception e) { 50 | throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); 51 | } 52 | 53 | protected abstract Boolean internalRun() throws Exception; 54 | 55 | } 56 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/context/IncrementalMigrationContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.context; 8 | 9 | import java.time.Instant; 10 | import java.util.Set; 11 | 12 | /** 13 | * The MigrationContext contains all information needed to perform a Source -> 14 | * Target Migration 15 | */ 16 | public interface IncrementalMigrationContext extends MigrationContext { 17 | 18 | Instant getIncrementalMigrationTimestamp(); 19 | 20 | void setSchemaMigrationAutoTriggerEnabled(final boolean autoTriggerEnabled); 21 | 22 | void setTruncateEnabled(final boolean truncateEnabled); 23 | 24 | void setIncrementalMigrationTimestamp(final Instant timeStampInstant); 25 | 26 | Set setIncrementalTables(final Set incrementalTables); 27 | 28 | void setIncrementalModeEnabled(final boolean incrementalModeEnabled); 29 | 30 | void setIncludedTables(final Set includedTables); 31 | 32 | void setDeletionEnabled(boolean deletionEnabled); 33 | 34 | void setLpTableMigrationEnabled(boolean lpTableMigrationEnabled); 35 | } 36 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/context/LaunchOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.context; 8 | 9 | import java.io.Serializable; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class LaunchOptions { 14 | 15 | public static final LaunchOptions NONE = new LaunchOptions(); 16 | 17 | private final Map propertyOverrideMap; 18 | 19 | public LaunchOptions() { 20 | this.propertyOverrideMap = new HashMap<>(); 21 | } 22 | 23 | public Map getPropertyOverrideMap() { 24 | return propertyOverrideMap; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/context/MigrationContextFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.context; 8 | 9 | import com.sap.cx.boosters.commercedbsync.constants.CommercedbsyncConstants; 10 | import com.sap.cx.boosters.commercedbsync.context.impl.DefaultIncrementalMigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.context.impl.DefaultMigrationContext; 12 | import com.sap.cx.boosters.commercedbsync.profile.DataSourceConfigurationFactory; 13 | import com.sap.cx.boosters.commercedbsync.repository.impl.DataRepositoryFactory; 14 | import org.apache.commons.configuration.Configuration; 15 | 16 | public class MigrationContextFactory { 17 | final DataRepositoryFactory dataRepositoryFactory; 18 | final DataSourceConfigurationFactory dataSourceConfigurationFactory; 19 | final Configuration configuration; 20 | final boolean reversed; 21 | 22 | public MigrationContextFactory(final DataRepositoryFactory dataRepositoryFactory, 23 | final DataSourceConfigurationFactory dataSourceConfigurationFactory, final Configuration configuration, 24 | final boolean reversed) { 25 | this.dataRepositoryFactory = dataRepositoryFactory; 26 | this.dataSourceConfigurationFactory = dataSourceConfigurationFactory; 27 | this.configuration = configuration; 28 | this.reversed = reversed; 29 | } 30 | 31 | public MigrationContext create() throws Exception { 32 | if (configuration.getBoolean(CommercedbsyncConstants.MIGRATION_DATA_SYNCHRONIZATION_ENABLED, false)) { 33 | return new DefaultIncrementalMigrationContext(dataRepositoryFactory, dataSourceConfigurationFactory, 34 | configuration, reversed); 35 | } 36 | 37 | return new DefaultMigrationContext(dataRepositoryFactory, dataSourceConfigurationFactory, configuration, false); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/context/SchemaDifferenceContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.context; 8 | 9 | import java.util.UUID; 10 | 11 | /** 12 | * Contains the Information needed to perform Schema Differences Check 13 | */ 14 | public class SchemaDifferenceContext { 15 | 16 | private final String schemaDifferenceId; 17 | 18 | private final MigrationContext migrationContext; 19 | 20 | public SchemaDifferenceContext(MigrationContext migrationContext) { 21 | this.schemaDifferenceId = UUID.randomUUID().toString(); 22 | this.migrationContext = migrationContext; 23 | } 24 | 25 | public SchemaDifferenceContext(String schemaDifferenceId, MigrationContext migrationContext) { 26 | this.schemaDifferenceId = schemaDifferenceId; 27 | this.migrationContext = migrationContext; 28 | } 29 | 30 | public String getSchemaDifferenceId() { 31 | return schemaDifferenceId; 32 | } 33 | 34 | public MigrationContext getMigrationContext() { 35 | return migrationContext; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/context/validation/MigrationContextValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.context.validation; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 10 | 11 | public interface MigrationContextValidator { 12 | 13 | void validateContext(MigrationContext context); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/cron/FullMigrationCronJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.cron; 8 | 9 | import de.hybris.platform.jalo.Item; 10 | import de.hybris.platform.jalo.JaloBusinessException; 11 | import de.hybris.platform.jalo.SessionContext; 12 | import de.hybris.platform.jalo.type.ComposedType; 13 | import org.apache.log4j.Logger; 14 | 15 | public class FullMigrationCronJob extends GeneratedFullMigrationCronJob { 16 | @SuppressWarnings("unused") 17 | private static final Logger LOG = Logger.getLogger(FullMigrationCronJob.class.getName()); 18 | 19 | @Override 20 | protected Item createItem(final SessionContext ctx, final ComposedType type, 21 | final Item.ItemAttributeMap allAttributes) throws JaloBusinessException { 22 | // business code placed here will be executed before the item is created 23 | // then create the item 24 | final Item item = super.createItem(ctx, type, allAttributes); 25 | // business code placed here will be executed after the item was created 26 | // and return the item 27 | return item; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/cron/IncrementalMigrationCronJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.cron; 8 | 9 | import de.hybris.platform.jalo.Item; 10 | import de.hybris.platform.jalo.JaloBusinessException; 11 | import de.hybris.platform.jalo.SessionContext; 12 | import de.hybris.platform.jalo.type.ComposedType; 13 | import org.apache.log4j.Logger; 14 | 15 | public class IncrementalMigrationCronJob extends GeneratedIncrementalMigrationCronJob { 16 | @SuppressWarnings("unused") 17 | private static final Logger LOG = Logger.getLogger(IncrementalMigrationCronJob.class.getName()); 18 | 19 | @Override 20 | protected Item createItem(final SessionContext ctx, final ComposedType type, final ItemAttributeMap allAttributes) 21 | throws JaloBusinessException { 22 | // business code placed here will be executed before the item is created 23 | // then create the item 24 | final Item item = super.createItem(ctx, type, allAttributes); 25 | // business code placed here will be executed after the item was created 26 | // and return the item 27 | return item; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/cron/MigrationCronJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.cron; 8 | 9 | import de.hybris.platform.jalo.Item; 10 | import de.hybris.platform.jalo.JaloBusinessException; 11 | import de.hybris.platform.jalo.SessionContext; 12 | import de.hybris.platform.jalo.type.ComposedType; 13 | import org.apache.log4j.Logger; 14 | 15 | public class MigrationCronJob extends GeneratedMigrationCronJob { 16 | @SuppressWarnings("unused") 17 | private static final Logger LOG = Logger.getLogger(MigrationCronJob.class.getName()); 18 | 19 | @Override 20 | protected Item createItem(final SessionContext ctx, final ComposedType type, final ItemAttributeMap allAttributes) 21 | throws JaloBusinessException { 22 | // business code placed here will be executed before the item is created 23 | // then create the item 24 | final Item item = super.createItem(ctx, type, allAttributes); 25 | // business code placed here will be executed after the item was created 26 | // and return the item 27 | return item; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/dataset/DataColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.dataset; 8 | 9 | public interface DataColumn { 10 | 11 | String getColumnName(); 12 | 13 | int getColumnType(); 14 | 15 | int getPrecision(); 16 | 17 | int getScale(); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/dataset/DataSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.dataset; 8 | 9 | import com.sap.cx.boosters.commercedbsync.dataset.impl.DefaultDataSet; 10 | 11 | import java.util.Collections; 12 | import java.util.List; 13 | import java.util.Objects; 14 | 15 | public interface DataSet { 16 | 17 | DataSet EMPTY = new DefaultDataSet(0, 0, Collections.emptyList(), Collections.emptyList(), null); 18 | 19 | int getBatchId(); 20 | 21 | int getColumnCount(); 22 | 23 | List> getAllResults(); 24 | 25 | Object getColumnValue(String column, List row, DataColumn sourceColumn, int targetColumnType); 26 | 27 | default Object getColumnValue(String column, List row) { 28 | var dataColumn = getColumn(column); 29 | 30 | Objects.requireNonNull(dataColumn); 31 | 32 | return getColumnValue(column, row, dataColumn, dataColumn.getColumnType()); 33 | } 34 | 35 | boolean isNotEmpty(); 36 | 37 | boolean hasColumn(String column); 38 | 39 | DataColumn getColumn(int columnIndex); 40 | 41 | DataColumn getColumn(String columnName); 42 | 43 | String getPartition(); 44 | } 45 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/dataset/impl/DefaultDataColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.dataset.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.dataset.DataColumn; 10 | 11 | public class DefaultDataColumn implements DataColumn { 12 | 13 | private final String name; 14 | private final int type; 15 | private final int precision; 16 | private final int scale; 17 | 18 | public DefaultDataColumn(String name, int type, int precision, int scale) { 19 | this.name = name; 20 | this.type = type; 21 | this.precision = precision; 22 | this.scale = scale; 23 | } 24 | 25 | @Override 26 | public String getColumnName() { 27 | return name; 28 | } 29 | 30 | @Override 31 | public int getColumnType() { 32 | return type; 33 | } 34 | 35 | @Override 36 | public int getPrecision() { 37 | return precision; 38 | } 39 | 40 | @Override 41 | public int getScale() { 42 | return scale; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/datasource/MigrationDataSourceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.datasource; 8 | 9 | import com.sap.cx.boosters.commercedbsync.profile.DataSourceConfiguration; 10 | 11 | import java.util.Map; 12 | 13 | import javax.sql.DataSource; 14 | 15 | /** 16 | * Factory to create the DataSources used for Migration 17 | */ 18 | public interface MigrationDataSourceFactory { 19 | DataSource create(DataSourceConfiguration dataSourceConfiguration); 20 | 21 | /** 22 | * Generates DataSource configuration with a configuration map 23 | * 24 | * @param dataSourceConfigurationMap 25 | * @return 26 | */ 27 | DataSource create(Map dataSourceConfigurationMap); 28 | } 29 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/datasource/impl/AbstractMigrationDataSourceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.datasource.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.datasource.MigrationDataSourceFactory; 10 | 11 | /** 12 | * 13 | */ 14 | public abstract class AbstractMigrationDataSourceFactory implements MigrationDataSourceFactory { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/datasource/impl/DefaultMigrationDataSourceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.datasource.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.profile.DataSourceConfiguration; 10 | import com.zaxxer.hikari.HikariConfig; 11 | import com.zaxxer.hikari.HikariDataSource; 12 | 13 | import java.util.Map; 14 | 15 | import javax.sql.DataSource; 16 | 17 | public class DefaultMigrationDataSourceFactory extends AbstractMigrationDataSourceFactory { 18 | 19 | @Override 20 | public DataSource create(final DataSourceConfiguration dataSourceConfiguration) { 21 | HikariConfig config = new HikariConfig(); 22 | config.setJdbcUrl(dataSourceConfiguration.getConnectionString()); 23 | config.setDriverClassName(dataSourceConfiguration.getDriver()); 24 | config.setUsername(dataSourceConfiguration.getUserName()); 25 | config.setPassword(dataSourceConfiguration.getPassword()); 26 | config.setMaximumPoolSize(dataSourceConfiguration.getMaxActive()); 27 | config.setMinimumIdle(dataSourceConfiguration.getMinIdle()); 28 | config.setRegisterMbeans(true); 29 | config.setMaxLifetime(dataSourceConfiguration.getMaxLifetime()); 30 | config.setSchema(dataSourceConfiguration.getSchema()); 31 | return new HikariDataSource(config); 32 | } 33 | 34 | @Override 35 | public DataSource create(final Map dataSourceConfigurationMap) { 36 | HikariConfig config = new HikariConfig(); 37 | config.setJdbcUrl((String) dataSourceConfigurationMap.get("connection.url")); 38 | config.setDriverClassName((String) dataSourceConfigurationMap.get("driver")); 39 | config.setUsername((String) dataSourceConfigurationMap.get("username")); 40 | config.setPassword((String) dataSourceConfigurationMap.get("password")); 41 | config.setMaximumPoolSize((Integer) dataSourceConfigurationMap.get("pool.size.max")); 42 | config.setMinimumIdle((Integer) dataSourceConfigurationMap.get("pool.size.idle.min")); 43 | config.setRegisterMbeans((Boolean) dataSourceConfigurationMap.get("registerMbeans")); 44 | return new HikariDataSource(config); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/events/CopyCompleteEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.events; 8 | 9 | /** 10 | * * ClusterAwareEvent to signal completion of the assigned copy ta 11 | */ 12 | public class CopyCompleteEvent extends OperationEvent { 13 | private final Boolean copyResult = false; 14 | 15 | public CopyCompleteEvent(final Integer sourceNodeId, final String migrationId, final boolean reversed) { 16 | super(sourceNodeId, migrationId, reversed); 17 | } 18 | 19 | public Boolean getCopyResult() { 20 | return copyResult; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/events/CopyDatabaseTableEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.events; 8 | 9 | import java.io.Serializable; 10 | import java.util.Map; 11 | 12 | /** 13 | * Cluster Event to notify a Cluster to start the copy process 14 | */ 15 | public class CopyDatabaseTableEvent extends OperationEvent { 16 | 17 | /** 18 | * contains property value updates that should be populated in the cluster 19 | */ 20 | private final Map propertyOverrideMap; 21 | 22 | public CopyDatabaseTableEvent(final Integer sourceNodeId, final String migrationId, 23 | final Map propertyOverrideMap, final boolean reversed) { 24 | super(sourceNodeId, migrationId, reversed); 25 | this.propertyOverrideMap = propertyOverrideMap; 26 | } 27 | 28 | public Map getPropertyOverrideMap() { 29 | return propertyOverrideMap; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/events/OperationEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.events; 8 | 9 | import de.hybris.platform.servicelayer.event.ClusterAwareEvent; 10 | import de.hybris.platform.servicelayer.event.PublishEventContext; 11 | import de.hybris.platform.servicelayer.event.events.AbstractEvent; 12 | 13 | /** 14 | * ClusterAwareEvent to notify other Nodes to start the operation 15 | */ 16 | public abstract class OperationEvent extends AbstractEvent implements ClusterAwareEvent { 17 | private final int sourceNodeId; 18 | private final String operationId; 19 | private final boolean reversed; 20 | 21 | public OperationEvent(final int sourceNodeId, final String operationId, final boolean reversed) { 22 | super(); 23 | this.sourceNodeId = sourceNodeId; 24 | this.operationId = operationId; 25 | this.reversed = reversed; 26 | } 27 | 28 | @Override 29 | public boolean canPublish(PublishEventContext publishEventContext) { 30 | return true; 31 | } 32 | 33 | /** 34 | * @return the masterNodeId 35 | */ 36 | public int getSourceNodeId() { 37 | return sourceNodeId; 38 | } 39 | 40 | /** 41 | * @return the operationId 42 | */ 43 | public String getOperationId() { 44 | return operationId; 45 | } 46 | 47 | /** 48 | * @return true if reversed 49 | */ 50 | public boolean isReversed() { 51 | return reversed; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/events/SchemaDifferenceEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.events; 8 | 9 | public class SchemaDifferenceEvent extends OperationEvent { 10 | 11 | public SchemaDifferenceEvent(final int sourceNodeId, final String migrationId, final boolean reversed) { 12 | super(sourceNodeId, migrationId, reversed); 13 | } 14 | 15 | public String getSchemaDifferenceId() { 16 | return getOperationId(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/filter/DataCopyTableFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.filter; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 10 | 11 | import java.util.function.Predicate; 12 | 13 | public interface DataCopyTableFilter { 14 | Predicate filter(MigrationContext context); 15 | } 16 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/filter/impl/CompositeDataCopyTableFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.filter.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 10 | import com.sap.cx.boosters.commercedbsync.filter.DataCopyTableFilter; 11 | 12 | import java.util.List; 13 | import java.util.function.Predicate; 14 | 15 | public class CompositeDataCopyTableFilter implements DataCopyTableFilter { 16 | 17 | private List filters; 18 | 19 | @Override 20 | public Predicate filter(MigrationContext context) { 21 | return p -> filters.stream().allMatch(f -> f.filter(context).test(p)); 22 | } 23 | 24 | public void setFilters(List filters) { 25 | this.filters = filters; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/filter/impl/ExclusionDataCopyTableFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.filter.impl; 8 | 9 | import com.google.common.base.Predicates; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.filter.DataCopyTableFilter; 12 | import org.apache.commons.lang.StringUtils; 13 | 14 | import java.util.Set; 15 | import java.util.function.Predicate; 16 | 17 | public class ExclusionDataCopyTableFilter implements DataCopyTableFilter { 18 | 19 | @Override 20 | public Predicate filter(MigrationContext context) { 21 | Set excludedTables = context.getExcludedTables(); 22 | if (excludedTables == null || excludedTables.isEmpty()) { 23 | return Predicates.alwaysTrue(); 24 | } 25 | return p -> excludedTables.stream().noneMatch(e -> StringUtils.equalsIgnoreCase(e, p)); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/filter/impl/InclusionDataCopyTableFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.filter.impl; 8 | 9 | import com.google.common.base.Predicates; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.filter.DataCopyTableFilter; 12 | import org.apache.commons.lang.StringUtils; 13 | 14 | import java.util.Set; 15 | import java.util.function.Predicate; 16 | 17 | public class InclusionDataCopyTableFilter implements DataCopyTableFilter { 18 | 19 | @Override 20 | public Predicate filter(MigrationContext context) { 21 | Set includedTables = context.getIncludedTables(); 22 | if (includedTables == null || includedTables.isEmpty()) { 23 | return Predicates.alwaysTrue(); 24 | } 25 | return p -> includedTables.stream().anyMatch(e -> StringUtils.equalsIgnoreCase(e, p)); 26 | 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/filter/impl/IncrementalDataCopyTableFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.filter.impl; 8 | 9 | import com.google.common.base.Predicates; 10 | import com.sap.cx.boosters.commercedbsync.constants.CommercedbsyncConstants; 11 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 12 | import com.sap.cx.boosters.commercedbsync.filter.DataCopyTableFilter; 13 | import org.apache.commons.lang.StringUtils; 14 | 15 | import java.util.Set; 16 | import java.util.function.Predicate; 17 | 18 | public class IncrementalDataCopyTableFilter implements DataCopyTableFilter { 19 | 20 | @Override 21 | public Predicate filter(MigrationContext context) { 22 | if (!context.isIncrementalModeEnabled()) { 23 | return Predicates.alwaysTrue(); 24 | } 25 | Set incrementalTables = context.getIncrementalTables(); 26 | if (incrementalTables == null || incrementalTables.isEmpty()) { 27 | throw new IllegalStateException("At least one table for incremental copy must be specified. Check property " 28 | + CommercedbsyncConstants.MIGRATION_DATA_INCREMENTAL_TABLES); 29 | } 30 | return p -> incrementalTables.stream().anyMatch(e -> StringUtils.equalsIgnoreCase(e, p)); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/jalo/ItemDeletionMarker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.jalo; 8 | 9 | import de.hybris.platform.directpersistence.annotation.SLDSafe; 10 | import de.hybris.platform.jalo.Item; 11 | import de.hybris.platform.jalo.JaloBusinessException; 12 | import de.hybris.platform.jalo.SessionContext; 13 | import de.hybris.platform.jalo.type.ComposedType; 14 | import org.apache.log4j.Logger; 15 | 16 | @SLDSafe 17 | public class ItemDeletionMarker extends GeneratedItemDeletionMarker { 18 | @SuppressWarnings("unused") 19 | private static final Logger LOG = Logger.getLogger(ItemDeletionMarker.class.getName()); 20 | 21 | @Override 22 | protected Item createItem(final SessionContext ctx, final ComposedType type, final ItemAttributeMap allAttributes) 23 | throws JaloBusinessException { 24 | // business code placed here will be executed before the item is created 25 | // then create the item 26 | final Item item = super.createItem(ctx, type, allAttributes); 27 | // business code placed here will be executed after the item was created 28 | // and return the item 29 | return item; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/jobs/MigrationPrepJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.jobs; 8 | 9 | import com.sap.cx.boosters.commercedbsync.service.impl.DefaultDatabaseMigrationService; 10 | import de.hybris.platform.cronjob.enums.CronJobResult; 11 | import de.hybris.platform.cronjob.enums.CronJobStatus; 12 | import de.hybris.platform.cronjob.model.CronJobModel; 13 | import de.hybris.platform.servicelayer.cronjob.PerformResult; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | public class MigrationPrepJob extends AbstractMigrationJobPerformable { 18 | private static final Logger LOG = LoggerFactory.getLogger(DefaultDatabaseMigrationService.class); 19 | 20 | @Override 21 | public PerformResult perform(CronJobModel cronJobModel) { 22 | boolean caughtExeption = false; 23 | try { 24 | databaseMigrationService.prepareMigration(migrationContext); 25 | } catch (final Exception e) { 26 | caughtExeption = true; 27 | LOG.error(" Exception caught: message= " + e.getMessage(), e); 28 | } 29 | return new PerformResult(caughtExeption ? CronJobResult.FAILURE : CronJobResult.SUCCESS, 30 | CronJobStatus.FINISHED); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/logging/JdbcQueryLog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.logging; 8 | 9 | import com.sap.cx.boosters.commercedbsync.repository.DataRepository; 10 | 11 | import java.time.ZonedDateTime; 12 | import java.util.Collections; 13 | import java.util.Map; 14 | 15 | /** 16 | * Immutable value-based classes representing a JDBC query ran by the migration 17 | * tool against a {@link DataRepository} 18 | */ 19 | public class JdbcQueryLog { 20 | 21 | private final ZonedDateTime jdbcQueryTimestamp; 22 | private final String jdbcQuery; 23 | private final Map parameters; 24 | 25 | public JdbcQueryLog(final String jdbcQuery) { 26 | this(jdbcQuery, null); 27 | } 28 | 29 | public JdbcQueryLog(final String jdbcQuery, final Map parameters) { 30 | this.jdbcQueryTimestamp = ZonedDateTime.now(); 31 | this.jdbcQuery = jdbcQuery; 32 | if (parameters == null || parameters.isEmpty()) { 33 | this.parameters = null; 34 | } else { 35 | this.parameters = Collections.unmodifiableMap(parameters); 36 | } 37 | } 38 | 39 | /** 40 | * Timestamp when the JDBC query was executed 41 | * 42 | * @return timestamp of the JDBC query 43 | */ 44 | public ZonedDateTime getJdbcQueryTimestamp() { 45 | return jdbcQueryTimestamp; 46 | } 47 | 48 | /** 49 | * String representation of the JDBC query 50 | * 51 | * @return string representation of the JDBC query 52 | */ 53 | public String getJdbcQuery() { 54 | return jdbcQuery; 55 | } 56 | 57 | /** 58 | * If the JDBC query has parameters, this will return the map of the parameters 59 | * with the parameter index as key and the parameter value as value 60 | * 61 | * @return the JDBC query parameters, if it has any; null otherwise. 62 | */ 63 | public Map getParameters() { 64 | return parameters; 65 | } 66 | 67 | @Override 68 | public String toString() { 69 | StringBuilder sb = new StringBuilder("{timestamp=").append(jdbcQueryTimestamp).append(", query='") 70 | .append(jdbcQuery).append("'"); 71 | if (parameters != null) { 72 | sb.append(", parameters=").append(parameters); 73 | } 74 | sb.append("}"); 75 | return sb.toString(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/performance/PerformanceCategory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.performance; 8 | 9 | public enum PerformanceCategory { 10 | DB_READ, DB_WRITE 11 | } 12 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/performance/PerformanceProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.performance; 8 | 9 | import java.util.Collection; 10 | import java.util.concurrent.ConcurrentMap; 11 | 12 | public interface PerformanceProfiler { 13 | PerformanceRecorder createRecorder(PerformanceCategory category, String name); 14 | 15 | void muteRecorder(PerformanceCategory category, String name); 16 | 17 | ConcurrentMap getRecorders(); 18 | 19 | Collection getRecordersByCategory(PerformanceCategory category); 20 | 21 | double getAverageByCategoryAndUnit(PerformanceCategory category, PerformanceUnit unit); 22 | 23 | PerformanceRecorder getRecorder(PerformanceCategory category, String name); 24 | 25 | void reset(); 26 | } 27 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/performance/PerformanceUnit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.performance; 8 | 9 | public enum PerformanceUnit { 10 | ROWS, MB 11 | } 12 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/performance/impl/DefaultPerformanceProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.performance.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceCategory; 10 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceProfiler; 11 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceRecorder; 12 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceUnit; 13 | 14 | import java.util.Collection; 15 | import java.util.concurrent.ConcurrentHashMap; 16 | import java.util.stream.Collectors; 17 | 18 | public class DefaultPerformanceProfiler implements PerformanceProfiler { 19 | 20 | private final ConcurrentHashMap recorders = new ConcurrentHashMap<>(); 21 | 22 | @Override 23 | public PerformanceRecorder createRecorder(PerformanceCategory category, String name) { 24 | String recorderName = createRecorderName(category, name); 25 | return recorders.computeIfAbsent(recorderName, key -> new PerformanceRecorder(category, recorderName)); 26 | } 27 | 28 | @Override 29 | public void muteRecorder(PerformanceCategory category, String name) { 30 | String recorderName = createRecorderName(category, name); 31 | this.recorders.remove(recorderName); 32 | } 33 | 34 | @Override 35 | public ConcurrentHashMap getRecorders() { 36 | return recorders; 37 | } 38 | 39 | @Override 40 | public Collection getRecordersByCategory(PerformanceCategory category) { 41 | return recorders.values().stream().filter(r -> category == r.getCategory()).collect(Collectors.toList()); 42 | } 43 | 44 | @Override 45 | public double getAverageByCategoryAndUnit(PerformanceCategory category, PerformanceUnit unit) { 46 | Collection recordersByCategory = getRecordersByCategory(category); 47 | return recordersByCategory.stream().filter(r -> r.getRecords().get(unit) != null) 48 | .mapToDouble(r -> r.getRecords().get(unit).getAvgThroughput().get()).average().orElse(0); 49 | } 50 | 51 | @Override 52 | public PerformanceRecorder getRecorder(PerformanceCategory category, String name) { 53 | return recorders.get(createRecorderName(category, name)); 54 | } 55 | 56 | @Override 57 | public void reset() { 58 | getRecorders().clear(); 59 | } 60 | 61 | protected String createRecorderName(PerformanceCategory category, String name) { 62 | return category + "->" + name; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/MigrationPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | 11 | /** 12 | * Postprocessor activated after a migration has terminated 13 | */ 14 | public interface MigrationPostProcessor { 15 | 16 | void process(CopyContext context); 17 | 18 | default boolean shouldExecute(CopyContext context) { 19 | return true; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/MigrationPreProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | 11 | /** 12 | * Preproscessor activated before a migration starts 13 | */ 14 | public interface MigrationPreProcessor { 15 | 16 | void process(CopyContext context); 17 | 18 | default boolean shouldExecute(CopyContext context) { 19 | return true; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/DefaultMigrationPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPostProcessor; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | /** 15 | * Implements the {@link MigrationPostProcessor} 16 | */ 17 | public class DefaultMigrationPostProcessor implements MigrationPostProcessor { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(DefaultMigrationPostProcessor.class.getName()); 20 | 21 | @Override 22 | public void process(CopyContext context) { 23 | LOG.info("DefaultMigrationPostProcessor Finished"); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/JdbcQueriesPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPostProcessor; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | /** 15 | * Post-processor producing and storing reports on the JDBC queries that were 16 | * executed during a migration against the source and target data repositories. 17 | */ 18 | public class JdbcQueriesPostProcessor implements MigrationPostProcessor { 19 | 20 | private static final Logger LOG = LoggerFactory.getLogger(JdbcQueriesPostProcessor.class.getName()); 21 | 22 | @Override 23 | public void process(CopyContext context) { 24 | try { 25 | context.getMigrationContext().getDataSourceRepository().getJdbcQueriesStore() 26 | .writeToLogFileAndCompress(context.getMigrationId()); 27 | context.getMigrationContext().getDataTargetRepository().getJdbcQueriesStore() 28 | .writeToLogFileAndCompress(context.getMigrationId()); 29 | LOG.info("Finished writing jdbc entries report"); 30 | } catch (Exception e) { 31 | LOG.error("Error executing post processor", e); 32 | } 33 | } 34 | 35 | @Override 36 | public boolean shouldExecute(CopyContext context) { 37 | return context.getMigrationContext().isLogSql(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/ReportMigrationPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import com.google.gson.Gson; 10 | import com.google.gson.GsonBuilder; 11 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 12 | import com.sap.cx.boosters.commercedbsync.service.DatabaseMigrationReportService; 13 | import com.sap.cx.boosters.commercedbsync.service.DatabaseMigrationReportStorageService; 14 | import com.sap.cx.boosters.commercedbsync.utils.LocalDateTypeAdapter; 15 | import com.sap.cx.boosters.commercedbsync.MigrationReport; 16 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPostProcessor; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | import java.io.ByteArrayInputStream; 21 | import java.io.InputStream; 22 | import java.nio.charset.StandardCharsets; 23 | import java.time.LocalDateTime; 24 | 25 | public class ReportMigrationPostProcessor implements MigrationPostProcessor { 26 | 27 | private static final Logger LOG = LoggerFactory.getLogger(ReportMigrationPostProcessor.class.getName()); 28 | 29 | private DatabaseMigrationReportService databaseMigrationReportService; 30 | private DatabaseMigrationReportStorageService databaseMigrationReportStorageService; 31 | 32 | @Override 33 | public void process(CopyContext context) { 34 | if (!databaseMigrationReportStorageService.validateConnection()) { 35 | LOG.warn("Could not establish connection to report storage. Migration report will not be stored"); 36 | return; 37 | } 38 | 39 | try { 40 | final GsonBuilder gsonBuilder = new GsonBuilder(); 41 | gsonBuilder.setPrettyPrinting(); 42 | gsonBuilder.registerTypeAdapter(LocalDateTime.class, new LocalDateTypeAdapter()); 43 | gsonBuilder.disableHtmlEscaping(); 44 | Gson gson = gsonBuilder.create(); 45 | MigrationReport migrationReport = databaseMigrationReportService.getMigrationReport(context); 46 | InputStream is = new ByteArrayInputStream(gson.toJson(migrationReport).getBytes(StandardCharsets.UTF_8)); 47 | databaseMigrationReportStorageService.store(context.getMigrationId() + ".json", is); 48 | LOG.info("Finished writing database migration report"); 49 | } catch (Exception e) { 50 | LOG.error("Error executing post processor", e); 51 | } 52 | } 53 | 54 | public void setDatabaseMigrationReportService(DatabaseMigrationReportService databaseMigrationReportService) { 55 | this.databaseMigrationReportService = databaseMigrationReportService; 56 | } 57 | 58 | public void setDatabaseMigrationReportStorageService( 59 | DatabaseMigrationReportStorageService databaseMigrationReportStorageService) { 60 | this.databaseMigrationReportStorageService = databaseMigrationReportStorageService; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/TransformFunctionGeneratorPreProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import org.apache.commons.lang.StringUtils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.core.io.ClassPathResource; 13 | 14 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 15 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 16 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPreProcessor; 17 | import com.sap.cx.boosters.commercedbsync.repository.DataRepository; 18 | 19 | import de.hybris.bootstrap.ddl.DataBaseProvider; 20 | 21 | public class TransformFunctionGeneratorPreProcessor implements MigrationPreProcessor { 22 | 23 | private static final Logger LOG = LoggerFactory.getLogger(TransformFunctionGeneratorPreProcessor.class); 24 | 25 | @Override 26 | public void process(final CopyContext context) { 27 | final MigrationContext migrationContext = context.getMigrationContext(); 28 | final DataRepository dataSourceRepository = migrationContext.getDataSourceRepository(); 29 | final String platformSpecificSQL = getPlatformSpecificSQL(dataSourceRepository.getDatabaseProvider()); 30 | if (StringUtils.isNotBlank(platformSpecificSQL)) { 31 | dataSourceRepository.runSqlScriptOnPrimary( 32 | new ClassPathResource("/sql/transformationFunctions/" + platformSpecificSQL)); 33 | } 34 | } 35 | 36 | private String getPlatformSpecificSQL(final DataBaseProvider databaseProvider) { 37 | String platformSpecificSQL = null; 38 | 39 | if (databaseProvider.isMssqlUsed()) { 40 | platformSpecificSQL = "mssql-general.sql"; 41 | } 42 | 43 | LOG.info("Identified platform specific transformation function SQL: {}", 44 | StringUtils.defaultIfEmpty(platformSpecificSQL, "")); 45 | 46 | return platformSpecificSQL; 47 | } 48 | 49 | @Override 50 | public boolean shouldExecute(CopyContext context) { 51 | return context.getMigrationContext().isDataSynchronizationEnabled() 52 | && context.getMigrationContext().getDataSourceRepository().getDatabaseProvider().isMssqlUsed(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/TruncateNotMigratedTablesPreProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import java.util.Set; 10 | 11 | import org.apache.commons.collections4.CollectionUtils; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 16 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 17 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPreProcessor; 18 | import com.sap.cx.boosters.commercedbsync.repository.DataRepository; 19 | 20 | public class TruncateNotMigratedTablesPreProcessor implements MigrationPreProcessor { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(TruncateNotMigratedTablesPreProcessor.class); 23 | 24 | @Override 25 | public void process(final CopyContext context) { 26 | final MigrationContext migrationContext = context.getMigrationContext(); 27 | final Set migrationItems = migrationContext.getIncludedTables(); 28 | final DataRepository dataTargetRepository = migrationContext.getDataTargetRepository(); 29 | try { 30 | dataTargetRepository.getAllTableNames().stream().filter(table -> !migrationItems.contains(table)) 31 | .forEach(notMigratedTable -> { 32 | try { 33 | dataTargetRepository.truncateTable(notMigratedTable); 34 | LOG.debug("Not-migrated {} table is truncated", notMigratedTable); 35 | } catch (final Exception e) { 36 | LOG.error("Cannot truncate not-migrated table", e); 37 | } 38 | }); 39 | } catch (Exception e) { 40 | LOG.error("TruncateNotMigratedTablesPreprocessor is failed", e); 41 | } 42 | } 43 | 44 | @Override 45 | public boolean shouldExecute(CopyContext context) { 46 | return context.getMigrationContext().isDataSynchronizationEnabled() 47 | && context.getMigrationContext().isFullDatabaseMigration() 48 | && CollectionUtils.isNotEmpty(context.getMigrationContext().getIncludedTables()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/ViewDropPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import java.util.Set; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 15 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 16 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPostProcessor; 17 | import com.sap.cx.boosters.commercedbsync.repository.DataRepository; 18 | 19 | public class ViewDropPostProcessor implements MigrationPostProcessor { 20 | private static final Logger LOG = LoggerFactory.getLogger(ViewDropPostProcessor.class); 21 | 22 | private static final String DROP_VIEW = "DROP VIEW %s;"; 23 | 24 | @Override 25 | public void process(final CopyContext context) { 26 | final MigrationContext migrationContext = context.getMigrationContext(); 27 | final DataRepository dataSourceRepository = migrationContext.getDataSourceRepository(); 28 | final Set tables = migrationContext.getTablesForViews(); 29 | 30 | tables.stream().forEach(table -> { 31 | try { 32 | final String viewName = migrationContext.getItemTypeViewNameByTable(table, dataSourceRepository); 33 | dataSourceRepository.executeUpdateAndCommitOnPrimary(String.format(DROP_VIEW, viewName)); 34 | LOG.info("View {} is dropped", viewName); 35 | } catch (Exception e) { 36 | LOG.error("View dropped failed", e); 37 | } 38 | }); 39 | } 40 | 41 | @Override 42 | public boolean shouldExecute(CopyContext context) { 43 | return context.getMigrationContext().isDataSynchronizationEnabled(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/processors/impl/ViewGeneratorPreProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.processors.impl; 8 | 9 | import java.sql.SQLException; 10 | import java.util.Objects; 11 | import java.util.Set; 12 | 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 17 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 18 | import com.sap.cx.boosters.commercedbsync.processors.MigrationPreProcessor; 19 | import com.sap.cx.boosters.commercedbsync.views.TableViewGenerator; 20 | 21 | public class ViewGeneratorPreProcessor implements MigrationPreProcessor { 22 | 23 | private static final Logger LOG = LoggerFactory.getLogger(ViewGeneratorPreProcessor.class); 24 | 25 | @Override 26 | public void process(final CopyContext context) { 27 | final MigrationContext ctx = context.getMigrationContext(); 28 | final Set tables = ctx.getTablesForViews(); 29 | final TableViewGenerator generator = new TableViewGenerator(); 30 | tables.stream().map(t -> { 31 | try { 32 | return generator.generateForTable(t, ctx); 33 | } catch (Exception e) { 34 | LOG.error(String.format("couldn't generate ctx for table %s", t), e); 35 | return null; 36 | } 37 | }).filter(Objects::nonNull).map(generator::generateViewDefinition).forEach(t -> { 38 | try { 39 | context.getMigrationContext().getDataSourceRepository().executeUpdateAndCommitOnPrimary(t); 40 | } catch (Exception e) { 41 | LOG.error(String.format("couldn't execute view creation %s", t), e); 42 | } 43 | }); 44 | // Override setting if view has not been existing before 45 | context.getCopyItems().stream().forEach(ci -> { 46 | try { 47 | final String sTableName = context.getMigrationContext().getItemTypeViewNameByTable(ci.getSourceItem(), 48 | context.getMigrationContext().getDataSourceRepository()); 49 | ci.setSourceItem(sTableName); 50 | } catch (SQLException e) { 51 | LOG.error(String.format("could not check view mapping for table: %s", ci.getSourceItem()), e); 52 | } 53 | }); 54 | } 55 | 56 | @Override 57 | public boolean shouldExecute(CopyContext context) { 58 | return context.getMigrationContext().isDataSynchronizationEnabled(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/profile/DataSourceConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.profile; 8 | 9 | /** 10 | * Contains a DataSource Configuration 11 | */ 12 | public interface DataSourceConfiguration { 13 | String getProfile(); 14 | 15 | String getDriver(); 16 | 17 | String getConnectionString(); 18 | 19 | String getConnectionStringPrimary(); 20 | 21 | String getUserName(); 22 | 23 | String getPassword(); 24 | 25 | String getSchema(); 26 | 27 | String getTypeSystemName(); 28 | 29 | String getTypeSystemSuffix(); 30 | 31 | String getCatalog(); 32 | 33 | String getTablePrefix(); 34 | 35 | int getMaxActive(); 36 | 37 | int getMaxIdle(); 38 | 39 | int getMinIdle(); 40 | 41 | boolean isRemoveAbandoned(); 42 | 43 | long getMaxLifetime(); 44 | } 45 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/profile/DataSourceConfigurationFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.profile; 8 | 9 | /** 10 | * Factory to create datasource configurations based on profiles 11 | */ 12 | public interface DataSourceConfigurationFactory { 13 | DataSourceConfiguration create(String profile); 14 | } 15 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/profile/impl/DefaultDataSourceConfigurationFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.profile.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.profile.DataSourceConfiguration; 10 | import com.sap.cx.boosters.commercedbsync.profile.DataSourceConfigurationFactory; 11 | import de.hybris.platform.servicelayer.config.ConfigurationService; 12 | 13 | public class DefaultDataSourceConfigurationFactory implements DataSourceConfigurationFactory { 14 | 15 | private final ConfigurationService configurationService; 16 | 17 | public DefaultDataSourceConfigurationFactory(ConfigurationService configurationService) { 18 | this.configurationService = configurationService; 19 | } 20 | 21 | @Override 22 | public DataSourceConfiguration create(String profile) { 23 | return new DefaultDataSourceConfiguration(configurationService.getConfiguration(), profile); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/profile/impl/InvalidDataSourceConfigurationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.profile.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.profile.DataSourceConfiguration; 10 | 11 | public class InvalidDataSourceConfigurationException extends RuntimeException { 12 | public InvalidDataSourceConfigurationException(String message, DataSourceConfiguration dataSourceConfiguration) { 13 | super(message + ": " + dataSourceConfiguration); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/provider/CopyItemProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.provider; 8 | 9 | import com.sap.cx.boosters.commercedbsync.TableCandidate; 10 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 11 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 12 | 13 | import java.util.Set; 14 | 15 | /** 16 | * Provides the means to copy an Item fro Source to Target 17 | */ 18 | public interface CopyItemProvider { 19 | String TYPE_SYSTEM_PROPS_TABLE = "typesystemprops"; 20 | String[] TYPE_SYSTEM_RELATED_TYPES = new String[]{"atomictypes", "attributedescriptors", "collectiontypes", 21 | "composedtypes", "enumerationvalues", "maptypes", TYPE_SYSTEM_PROPS_TABLE}; 22 | Set get(MigrationContext context) throws Exception; 23 | 24 | Set getSourceTableCandidates(MigrationContext context) throws Exception; 25 | 26 | Set getTargetTableCandidates(MigrationContext context) throws Exception; 27 | } 28 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/repository/platform/MigrationHybrisMySqlBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.repository.platform; 8 | 9 | import de.hybris.bootstrap.ddl.DatabaseSettings; 10 | import de.hybris.bootstrap.ddl.sql.HybrisMySqlBuilder; 11 | import org.apache.commons.lang3.StringUtils; 12 | import org.apache.ddlutils.Platform; 13 | import org.apache.ddlutils.model.Column; 14 | import org.apache.ddlutils.model.TypeMap; 15 | 16 | import java.sql.Types; 17 | 18 | public class MigrationHybrisMySqlBuilder extends HybrisMySqlBuilder { 19 | 20 | public MigrationHybrisMySqlBuilder(Platform platform, DatabaseSettings databaseSettings) { 21 | super(platform, databaseSettings); 22 | } 23 | 24 | @Override 25 | protected String getSqlType(Column column) { 26 | if (column.getTypeCode() == Types.NVARCHAR && Integer.parseInt(column.getSize()) > 5000) { 27 | return "TEXT"; 28 | } 29 | 30 | if (column.getTypeCode() == Types.VARBINARY && Integer.parseInt(column.getSize()) > 65535) { 31 | return "LONGBLOB"; 32 | } 33 | 34 | if (column.getTypeCode() == Types.TIMESTAMP) { 35 | final StringBuilder nativeType = new StringBuilder(getPlatformInfo().getNativeType(column.getTypeCode())); 36 | 37 | if (getPlatformInfo().hasSize(column.getTypeCode())) { 38 | nativeType.append('(').append(getPlatformInfo().getDefaultSize(column.getTypeCode())).append(')'); 39 | } 40 | 41 | return nativeType.toString(); 42 | } 43 | 44 | return super.getSqlType(column); 45 | } 46 | 47 | @Override 48 | public boolean isValidDefaultValue(String defaultSpec, int typeCode) { 49 | return StringUtils.isNumeric(defaultSpec) 50 | && (defaultSpec.length() > 0 || !TypeMap.isNumericType(typeCode) && !TypeMap.isDateTimeType(typeCode)); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/scheduler/ClusterTableSplittingStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.scheduler; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | import org.apache.commons.lang3.tuple.Pair; 11 | 12 | import java.util.List; 13 | 14 | public interface ClusterTableSplittingStrategy { 15 | List> split(CopyContext.DataCopyItem item, Long rowCount, int numNodes); 16 | } 17 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/scheduler/DatabaseCopyScheduler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.scheduler; 8 | 9 | import com.sap.cx.boosters.commercedbsync.MigrationStatus; 10 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 11 | 12 | import java.time.OffsetDateTime; 13 | 14 | /** 15 | * Scheduler for Cluster Migration 16 | */ 17 | public interface DatabaseCopyScheduler { 18 | void schedule(CopyContext context) throws Exception; 19 | 20 | void resumeUnfinishedItems(CopyContext copyContext) throws Exception; 21 | 22 | MigrationStatus getCurrentState(CopyContext context, OffsetDateTime since) throws Exception; 23 | 24 | boolean isAborted(CopyContext context) throws Exception; 25 | 26 | void abort(CopyContext context) throws Exception; 27 | } 28 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/scheduler/DatabaseOperationSchedulerAlgorithm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.scheduler; 8 | 9 | import java.util.List; 10 | 11 | public interface DatabaseOperationSchedulerAlgorithm { 12 | int getOwnNodeId(); 13 | 14 | List getNodeIds(); 15 | 16 | int next(); 17 | 18 | void reset(); 19 | } 20 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/scheduler/DatabaseSchemaDifferenceScheduler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.scheduler; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.SchemaDifferenceContext; 10 | 11 | /** 12 | * Scheduler for Schema Difference logic execution 13 | */ 14 | public interface DatabaseSchemaDifferenceScheduler { 15 | 16 | void schedule(SchemaDifferenceContext context) throws Exception; 17 | 18 | void abort(SchemaDifferenceContext context) throws Exception; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/scheduler/impl/RoundRobinClusterSchedulerAlgorithm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.scheduler.impl; 8 | 9 | import com.google.common.collect.ImmutableList; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.scheduler.DatabaseOperationSchedulerAlgorithm; 12 | import de.hybris.platform.cluster.PingBroadcastHandler; 13 | import de.hybris.platform.servicelayer.cluster.ClusterService; 14 | import org.apache.commons.collections4.CollectionUtils; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import java.util.ArrayList; 19 | import java.util.Collections; 20 | import java.util.List; 21 | 22 | public class RoundRobinClusterSchedulerAlgorithm implements DatabaseOperationSchedulerAlgorithm { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(RoundRobinClusterSchedulerAlgorithm.class); 25 | 26 | private final MigrationContext migrationContext; 27 | 28 | private final ClusterService clusterService; 29 | 30 | private List nodeIds = null; 31 | 32 | private int nodeIndex = 0; 33 | 34 | public RoundRobinClusterSchedulerAlgorithm(MigrationContext migrationContext, ClusterService clusterService) { 35 | this.migrationContext = migrationContext; 36 | this.clusterService = clusterService; 37 | } 38 | 39 | @Override 40 | public int getOwnNodeId() { 41 | return clusterService.getClusterId(); 42 | } 43 | 44 | @Override 45 | public List getNodeIds() { 46 | if (nodeIds == null) { 47 | nodeIds = ImmutableList.copyOf(detectClusterNodes()); 48 | } 49 | return nodeIds; 50 | } 51 | 52 | @Override 53 | public int next() { 54 | if (nodeIndex >= (getNodeIds().size())) { 55 | nodeIndex = 0; 56 | } 57 | return getNodeIds().get(nodeIndex++); 58 | } 59 | 60 | public void reset() { 61 | nodeIds = null; 62 | nodeIndex = 0; 63 | } 64 | 65 | private List detectClusterNodes() { 66 | if (!migrationContext.isClusterMode()) { 67 | return Collections.singletonList(clusterService.getClusterId()); 68 | } 69 | final List nodeIdList = new ArrayList<>(); 70 | try { 71 | // Same code as the hac cluster overview page 72 | PingBroadcastHandler pingBroadcastHandler = PingBroadcastHandler.getInstance(); 73 | pingBroadcastHandler.getNodes().forEach(i -> nodeIdList.add(i.getNodeID())); 74 | } catch (final Exception e) { 75 | LOG.warn( 76 | "Using single cluster node because an error was encountered while fetching cluster nodes information: {{}}", 77 | e.getMessage(), e); 78 | } 79 | if (CollectionUtils.isEmpty(nodeIdList)) { 80 | nodeIdList.add(clusterService.getClusterId()); 81 | } 82 | return nodeIdList; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/service/DatabaseMigrationCopyService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.service; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | 11 | /** 12 | * Actual Service to perform the Migration 13 | */ 14 | public interface DatabaseMigrationCopyService { 15 | 16 | void copyAllAsync(CopyContext context); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/service/DatabaseMigrationDataTypeMapperService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.service; 8 | 9 | import java.io.IOException; 10 | import java.sql.SQLException; 11 | 12 | /** 13 | * Service to deal with Mapping different types between Databases 14 | */ 15 | public interface DatabaseMigrationDataTypeMapperService { 16 | 17 | /** 18 | * Converts BLOB, CLOB and NCLOB Data 19 | */ 20 | Object dataTypeMapper(final Object sourceColumnValue, final int jdbcType) throws IOException, SQLException; 21 | } 22 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/service/DatabaseMigrationReportService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.service; 8 | 9 | import com.sap.cx.boosters.commercedbsync.MigrationReport; 10 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 11 | 12 | public interface DatabaseMigrationReportService { 13 | 14 | MigrationReport getMigrationReport(CopyContext copyContext) throws Exception; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/service/DatabaseMigrationReportStorageService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.service; 8 | 9 | import java.io.InputStream; 10 | 11 | public interface DatabaseMigrationReportStorageService { 12 | void store(String fileName, InputStream inputStream) throws Exception; 13 | 14 | boolean validateConnection(); 15 | } 16 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/service/DatabaseSchemaDifferenceService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.service; 8 | 9 | import com.sap.cx.boosters.commercedbsync.SchemaDifferenceStatus; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.context.SchemaDifferenceContext; 12 | import com.sap.cx.boosters.commercedbsync.service.impl.DefaultDatabaseSchemaDifferenceService; 13 | 14 | import java.util.concurrent.CompletableFuture; 15 | 16 | /** 17 | * Calculates and applies Schema Differences between two Databases 18 | */ 19 | public interface DatabaseSchemaDifferenceService { 20 | 21 | String generateSchemaDifferencesSql(MigrationContext context, 22 | DefaultDatabaseSchemaDifferenceService.SchemaDifferenceResult differenceResult) throws Exception; 23 | 24 | void executeSchemaDifferencesSql(MigrationContext context, String sql) throws Exception; 25 | 26 | void executeSchemaDifferences(MigrationContext context) throws Exception; 27 | 28 | /** 29 | * Calculates the differences between two schemas 30 | * 31 | * @param migrationContext 32 | * @return 33 | */ 34 | DefaultDatabaseSchemaDifferenceService.SchemaDifferenceResult getSchemaDifferenceFromStatus( 35 | MigrationContext migrationContext, SchemaDifferenceStatus schemaDifferenceStatus) throws Exception; 36 | 37 | DefaultDatabaseSchemaDifferenceService.SchemaDifferenceResult createSchemaDifferenceResult( 38 | MigrationContext migrationContext) throws Exception; 39 | 40 | SchemaDifferenceStatus getSchemaDifferenceStatusById(String schemaDifferenceId, MigrationContext migrationContext) 41 | throws Exception; 42 | 43 | SchemaDifferenceStatus getMostRecentSchemaDifference(MigrationContext migrationContext) throws Exception; 44 | 45 | CompletableFuture checkSchemaDifferenceAsync(SchemaDifferenceContext context); 46 | 47 | void abortRunningSchemaDifference(MigrationContext migrationContext) throws Exception; 48 | 49 | String startSchemaDifferenceCheck(MigrationContext context) throws Exception; 50 | 51 | SchemaDifferenceStatus startSchemaDifferenceCheckAndWaitForFinish(MigrationContext context) throws Exception; 52 | } 53 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/service/impl/DefaultDatabaseMigrationDataTypeMapperService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.service.impl; 8 | 9 | import com.google.common.io.ByteStreams; 10 | import org.apache.commons.io.IOUtils; 11 | import com.sap.cx.boosters.commercedbsync.service.DatabaseMigrationDataTypeMapperService; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.io.ByteArrayInputStream; 16 | import java.io.IOException; 17 | import java.io.Reader; 18 | import java.io.StringWriter; 19 | import java.sql.Blob; 20 | import java.sql.Clob; 21 | import java.sql.NClob; 22 | import java.sql.SQLException; 23 | import java.sql.Types; 24 | 25 | /** 26 | * 27 | */ 28 | public class DefaultDatabaseMigrationDataTypeMapperService implements DatabaseMigrationDataTypeMapperService { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(DefaultDatabaseMigrationDataTypeMapperService.class); 31 | 32 | @Override 33 | public Object dataTypeMapper(final Object sourceColumnValue, final int jdbcType) throws IOException, SQLException { 34 | Object targetColumnValue = sourceColumnValue; 35 | if (sourceColumnValue == null) { 36 | // do nothing 37 | } else if (jdbcType == Types.BLOB) { 38 | targetColumnValue = new ByteArrayInputStream( 39 | ByteStreams.toByteArray(((Blob) sourceColumnValue).getBinaryStream())); 40 | } else if (jdbcType == Types.NCLOB) { 41 | targetColumnValue = getValue((NClob) sourceColumnValue); 42 | } else if (jdbcType == Types.CLOB) { 43 | targetColumnValue = getValue((Clob) sourceColumnValue); 44 | } 45 | return targetColumnValue; 46 | } 47 | 48 | private String getValue(final NClob nClob) throws SQLException, IOException { 49 | return getValue(nClob.getCharacterStream()); 50 | } 51 | 52 | private String getValue(final Clob clob) throws SQLException, IOException { 53 | return getValue(clob.getCharacterStream()); 54 | } 55 | 56 | private String getValue(final Reader in) throws SQLException, IOException { 57 | final StringWriter w = new StringWriter(); 58 | IOUtils.copy(in, w); 59 | String value = w.toString(); 60 | w.close(); 61 | in.close(); 62 | return value; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/setup/InitUpdateProcessTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.setup; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.LaunchOptions; 10 | import de.hybris.platform.media.services.MediaStorageInitializer; 11 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 12 | import com.sap.cx.boosters.commercedbsync.service.DatabaseMigrationService; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | public class InitUpdateProcessTrigger implements MediaStorageInitializer { 17 | 18 | private static final Logger LOG = LoggerFactory.getLogger(InitUpdateProcessTrigger.class); 19 | 20 | private final MigrationContext migrationContext; 21 | private final DatabaseMigrationService databaseMigrationService; 22 | private boolean failOnError = false; 23 | 24 | public InitUpdateProcessTrigger(MigrationContext migrationContext, 25 | DatabaseMigrationService databaseMigrationService) { 26 | this.migrationContext = migrationContext; 27 | this.databaseMigrationService = databaseMigrationService; 28 | } 29 | 30 | @Override 31 | public void onInitialize() { 32 | // Do nothing 33 | } 34 | 35 | @Override 36 | public void onUpdate() { 37 | try { 38 | if (migrationContext.isMigrationTriggeredByUpdateProcess()) { 39 | LOG.info("Starting data migration ..."); 40 | databaseMigrationService.prepareMigration(migrationContext); 41 | String migrationId = databaseMigrationService.startMigration(migrationContext, LaunchOptions.NONE); 42 | databaseMigrationService.waitForFinish(migrationContext, migrationId); 43 | // note: further update activities not stopped here -> should we? 44 | } 45 | } catch (Exception e) { 46 | failOnError = migrationContext.isFailOnErrorEnabled(); 47 | if (failOnError) { 48 | throw new Error(e); 49 | } 50 | } 51 | } 52 | 53 | @Override 54 | public boolean failOnInitUpdateError() { 55 | return failOnError; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/strategy/PipeWriterStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.strategy; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.DataPipe; 10 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 11 | 12 | import javax.annotation.concurrent.ThreadSafe; 13 | 14 | /** 15 | * Main Strategy to Write Data to a target Database 16 | * 17 | * @param 18 | */ 19 | @ThreadSafe 20 | public interface PipeWriterStrategy { 21 | /** 22 | * Performs the actual copying of Data Items 23 | * 24 | * @param context 25 | * @param pipe 26 | * @param item 27 | * @throws Exception 28 | */ 29 | void write(CopyContext context, DataPipe pipe, CopyContext.DataCopyItem item) throws Exception; 30 | } 31 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/strategy/impl/CopyPipeWriterContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.strategy.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.CopyContext; 10 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceRecorder; 11 | import com.sap.cx.boosters.commercedbsync.service.DatabaseCopyTaskRepository; 12 | 13 | import java.util.List; 14 | import java.util.Set; 15 | import java.util.concurrent.atomic.AtomicLong; 16 | 17 | class CopyPipeWriterContext { 18 | private final CopyContext context; 19 | private final CopyContext.DataCopyItem copyItem; 20 | private final List columnsToCopy; 21 | private final Set nullifyColumns; 22 | private final PerformanceRecorder performanceRecorder; 23 | private final AtomicLong totalCount; 24 | private final List upsertIds; 25 | private final boolean requiresIdentityInsert; 26 | private final DatabaseCopyTaskRepository databaseCopyTaskRepository; 27 | 28 | public CopyPipeWriterContext(CopyContext context, CopyContext.DataCopyItem copyItem, List columnsToCopy, 29 | Set nullifyColumns, PerformanceRecorder performanceRecorder, AtomicLong totalCount, 30 | List upsertIds, boolean requiresIdentityInsert, 31 | DatabaseCopyTaskRepository databaseCopyTaskRepository) { 32 | this.context = context; 33 | this.copyItem = copyItem; 34 | this.columnsToCopy = columnsToCopy; 35 | this.nullifyColumns = nullifyColumns; 36 | this.performanceRecorder = performanceRecorder; 37 | this.totalCount = totalCount; 38 | this.upsertIds = upsertIds; 39 | this.requiresIdentityInsert = requiresIdentityInsert; 40 | this.databaseCopyTaskRepository = databaseCopyTaskRepository; 41 | } 42 | 43 | public CopyContext getContext() { 44 | return context; 45 | } 46 | 47 | public CopyContext.DataCopyItem getCopyItem() { 48 | return copyItem; 49 | } 50 | 51 | public List getColumnsToCopy() { 52 | return columnsToCopy; 53 | } 54 | 55 | public Set getNullifyColumns() { 56 | return nullifyColumns; 57 | } 58 | 59 | public PerformanceRecorder getPerformanceRecorder() { 60 | return performanceRecorder; 61 | } 62 | 63 | public AtomicLong getTotalCount() { 64 | return totalCount; 65 | } 66 | 67 | public List getUpsertIds() { 68 | return upsertIds; 69 | } 70 | 71 | public boolean isRequiresIdentityInsert() { 72 | return requiresIdentityInsert; 73 | } 74 | 75 | public DatabaseCopyTaskRepository getDatabaseCopyTaskRepository() { 76 | return databaseCopyTaskRepository; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/utils/FileUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.utils; 8 | 9 | import java.io.ByteArrayOutputStream; 10 | import java.io.IOException; 11 | import java.util.zip.ZipEntry; 12 | import java.util.zip.ZipOutputStream; 13 | 14 | public class FileUtils { 15 | 16 | public static byte[] zipBytes(String filename, byte[] input) throws IOException { 17 | ByteArrayOutputStream baos = new ByteArrayOutputStream(); 18 | ZipOutputStream zos = new ZipOutputStream(baos); 19 | ZipEntry entry = new ZipEntry(filename); 20 | entry.setSize(input.length); 21 | zos.putNextEntry(entry); 22 | zos.write(input); 23 | zos.closeEntry(); 24 | zos.close(); 25 | return baos.toByteArray(); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/utils/LocalDateTypeAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.utils; 8 | 9 | import java.io.IOException; 10 | import java.time.LocalDateTime; 11 | import java.time.ZonedDateTime; 12 | 13 | import com.google.gson.TypeAdapter; 14 | import com.google.gson.stream.JsonReader; 15 | import com.google.gson.stream.JsonToken; 16 | import com.google.gson.stream.JsonWriter; 17 | 18 | public class LocalDateTypeAdapter extends TypeAdapter { 19 | 20 | @Override 21 | public LocalDateTime read(JsonReader jsonReader) throws IOException { 22 | if (jsonReader.peek() == JsonToken.NULL) { 23 | jsonReader.nextNull(); 24 | return null; 25 | } 26 | return ZonedDateTime.parse(jsonReader.nextString()).toLocalDateTime(); 27 | } 28 | 29 | @Override 30 | public void write(final JsonWriter jsonWriter, final LocalDateTime localDate) throws IOException { 31 | if (localDate == null) { 32 | jsonWriter.nullValue(); 33 | return; 34 | } 35 | jsonWriter.value(localDate.toString()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/utils/MaskUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.utils; 8 | 9 | public class MaskUtil { 10 | 11 | public static String stripJdbcPassword(final String jdbcConnectionString) { 12 | return jdbcConnectionString.replaceFirst("password=.*?;", "password=***;"); 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /commercedbsync/src/com/sap/cx/boosters/commercedbsync/views/ViewConfigurationContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.views; 8 | 9 | import java.util.Map; 10 | import java.util.Set; 11 | 12 | /** 13 | * Context holder for one table configuration. Object generated by 14 | * {@link TableViewGenerator}. 15 | * 16 | * @author i303764 17 | */ 18 | public class ViewConfigurationContext { 19 | 20 | private final String additionalWhereClause; 21 | private final Map columnReplacements; 22 | private final Set originalColumns; 23 | private final String table; 24 | private final String view; 25 | private final String viewColumnPrefix; 26 | 27 | public ViewConfigurationContext(String table, String view, Set originalColumns, 28 | Map columnReplacements, String additionalWhereClause, String viewColumnPrefix) { 29 | super(); 30 | this.table = table; 31 | this.view = view; 32 | this.originalColumns = originalColumns; 33 | this.columnReplacements = columnReplacements; 34 | this.additionalWhereClause = additionalWhereClause; 35 | this.viewColumnPrefix = viewColumnPrefix; 36 | } 37 | 38 | public String getAdditionalWhereClause() { 39 | return additionalWhereClause; 40 | } 41 | 42 | public Map getColumnReplacements() { 43 | return columnReplacements; 44 | } 45 | 46 | public Set getOriginalColumns() { 47 | return originalColumns; 48 | } 49 | 50 | public String getTable() { 51 | return table; 52 | } 53 | 54 | public String getView() { 55 | return view; 56 | } 57 | 58 | public String getViewColumnPrefix() { 59 | return viewColumnPrefix; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /commercedbsync/src/de/hybris/platform/core/TenantPropertiesLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package de.hybris.platform.core; 8 | 9 | import de.hybris.bootstrap.ddl.PropertiesLoader; 10 | 11 | import java.util.Objects; 12 | 13 | public class TenantPropertiesLoader implements PropertiesLoader { 14 | private final Tenant tenant; 15 | 16 | public TenantPropertiesLoader(final Tenant tenant) { 17 | Objects.requireNonNull(tenant); 18 | this.tenant = tenant; 19 | } 20 | 21 | @Override 22 | public String getProperty(final String key) { 23 | return tenant.getConfig().getParameter(key); 24 | } 25 | 26 | @Override 27 | public String getProperty(final String key, final String defaultValue) { 28 | return tenant.getConfig().getString(key, defaultValue); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /commercedbsync/testsrc/com/sap/cx/boosters/commercedbsync/views/TableViewGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsync.views; 8 | 9 | import java.util.HashSet; 10 | import java.util.Map; 11 | import java.util.stream.Collectors; 12 | import java.util.stream.Stream; 13 | 14 | import org.junit.Assert; 15 | import org.junit.Before; 16 | import org.junit.Test; 17 | import org.mockito.Mockito; 18 | 19 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 20 | import com.sap.cx.boosters.commercedbsync.repository.DataRepository; 21 | 22 | class TableViewGeneratorTest { 23 | 24 | TableViewGenerator testObj; 25 | MigrationContext ctx; 26 | 27 | @Before 28 | void setUp() throws Exception { 29 | ctx = Mockito.mock(MigrationContext.class); 30 | DataRepository dr = Mockito.mock(DataRepository.class); 31 | Mockito.when(ctx.getDataSourceRepository()).thenReturn(dr); 32 | Mockito.when(dr.getAllColumnNames(Mockito.anyString())) 33 | .thenReturn(Stream.of("hjmpTS", "createdTS", "modifiedTS", "TypePkString", "OwnerPkString", "PK", 34 | "sealed", "p_mime", "p_size", "p_datapk", "p_location", "p_locationhash", "p_realfilename", 35 | "p_code", "p_internalurl", "p_description", "p_alttext", "p_removable", "p_mediaformat", 36 | "p_folder", "p_subfolderpath", "p_mediacontainer", "p_catalog", "p_catalogversion", "aCLTS", 37 | "propTS", "p_outputmimetype", "p_inputmimetype", "p_itemtimestamp", "p_format", "p_sourceitem", 38 | "p_fieldseparator", "p_quotecharacter", "p_commentcharacter", "p_encoding", "p_linestoskip", 39 | "p_removeonsuccess", "p_zipentry", "p_extractionid", "p_auditrootitem", "p_auditreportconfig", 40 | "p_scheduledcount", "p_cronjobpos", "p_cronjob") 41 | .collect(Collectors.toCollection(HashSet::new))); 42 | Mockito.when(ctx.getItemTypeViewNamePattern()).thenReturn("v_%s"); 43 | Mockito.when(ctx.getViewWhereClause(Mockito.matches("medias"))).thenReturn(""); 44 | Mockito.when(ctx.getCustomColumnsForView(Mockito.matches("medias"))).thenReturn(Map.of()); 45 | testObj = new TableViewGenerator(); 46 | } 47 | 48 | @Test 49 | void testSimplestMedia() throws Exception { 50 | ViewConfigurationContext result = testObj.generateForTable("medias", ctx); 51 | Assert.assertNotNull(result); 52 | 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /commercedbsynchac/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /commercedbsynchac/.externalToolBuilders/HybrisCodeGeneration.launch: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /commercedbsynchac/buildcallbacks.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /commercedbsynchac/extensioninfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /commercedbsynchac/external-dependencies.xml: -------------------------------------------------------------------------------- 1 | 6 | 8 | 4.0.0 9 | de.hybris.platform 10 | commercedbsynchac 11 | 6.7.0.0-RC19 12 | 13 | jar 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/jsp/dataSource.jsp: -------------------------------------------------------------------------------- 1 | <%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %> 2 | <%@ taglib prefix="hac" uri="/WEB-INF/custom.tld" %> 3 | <%-- 4 | ~ Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 5 | ~ License: Apache-2.0 6 | ~ 7 | --%> 8 | 9 | 10 | 11 | Migrate Data To SAP Commerce Cloud 12 | " type="text/css" media="screen, projection" /> 13 | " type="text/css" media="screen, projection" /> 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 22 |
23 |

Data Migration

24 |
25 | 29 | 30 |
31 |
32 |
"> 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
PropertyValue
43 | 44 | 45 |
46 | 47 |
48 |
49 | "> 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 |
PropertyValue
60 |
61 | 62 |
63 |
64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/jsp/migrationReports.jsp: -------------------------------------------------------------------------------- 1 | <%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %> 2 | <%@ taglib prefix="hac" uri="/WEB-INF/custom.tld" %> 3 | <%@ taglib prefix="form" uri="http://www.springframework.org/tags/form" %> 4 | <%-- 5 | ~ Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 6 | ~ License: Apache-2.0 7 | ~ 8 | --%> 9 | 10 | 11 | 12 | Copy Schema To SAP Commerce Cloud 13 | " type="text/css" media="screen, projection"/> 14 | " type="text/css" 15 | media="screen, projection"/> 16 | " type="text/css" 17 | media="screen, projection"/> 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 |
26 | 27 |
28 |

Migration Reports

29 |
" data-downloadUrl=""> 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 |
Report idTimestampDownload
42 |
43 |
44 |
45 | 46 | 47 | -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/static/audio/pling.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/commercedbsynchac/hac/resources/static/audio/pling.mp3 -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/static/css/dataCopy.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | .placeholder { 8 | color: dimgrey; 9 | } 10 | 11 | .status dd { 12 | font-size: 1rem; 13 | } 14 | 15 | .completed { 16 | color: green; 17 | } 18 | 19 | .failed { 20 | color: red; 21 | } 22 | 23 | #copySummary .total, #copySummary .failed, #copySummary .completed { 24 | font-weight: bold; 25 | } 26 | 27 | #copyStatus .completed { 28 | color: green; 29 | text-transform: uppercase; 30 | font-weight: bolder; 31 | } 32 | 33 | #copyStatus .failed { 34 | text-transform: uppercase; 35 | font-weight: bolder; 36 | } 37 | 38 | #copyLogContainer { 39 | height: 600px; 40 | overflow: auto; 41 | font-family: monospace; 42 | font-size: 1rem; 43 | background-color: #FAFAFF; 44 | padding: 1rem; 45 | margin: 1rem 1rem 1rem 0; 46 | border: 1px grey dashed; 47 | border-radius: 3px; 48 | } 49 | 50 | #copyLogContainer p + p { 51 | text-indent: 0; 52 | } 53 | 54 | #copyLogContainer .failed { 55 | font-weight: bold; 56 | font-size: 1.02em; 57 | } 58 | 59 | #copyLogContainer .completed { 60 | font-weight: bold; 61 | font-size: 1.02em; 62 | } 63 | 64 | button[disabled] { 65 | cursor: default; 66 | opacity: 0.5; 67 | } 68 | 69 | button.control-button { 70 | float:left; 71 | } 72 | -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/static/css/database.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | .nobox { 8 | border: 0; 9 | width: 100%; 10 | } 11 | 12 | .textarea { 13 | height: auto; 14 | } 15 | 16 | #spinner { 17 | margin: 100px auto; 18 | opacity: 0.5; 19 | } 20 | 21 | .spinner { 22 | opacity: 0.5; 23 | } 24 | 25 | #spinnerWrapper,#loggingSpinnerWrapper { 26 | text-align: center; 27 | } 28 | 29 | #tableWrapper { 30 | display: none; 31 | padding-bottom: 2em; 32 | } 33 | 34 | #tableCopySchemaWrapper { 35 | display: none; 36 | padding-bottom: 2em; 37 | } 38 | 39 | #tableCopyDataWrapper { 40 | display: none; 41 | padding-bottom: 2em; 42 | } 43 | 44 | #loggingContentWrapper,#downloadLog,#slider-size,#downloadForm,#analyzeResults { 45 | display: none; 46 | } 47 | 48 | #loggingContentWrapper { 49 | margin-bottom: 3em; 50 | } 51 | 52 | #dataSourceInfos legend { 53 | color: #005BBC; 54 | font-size: 16px; 55 | } 56 | 57 | .floatLeft { 58 | float: left; 59 | } 60 | #copyStatusContainer { 61 | 62 | } 63 | 64 | #copyStatusContainer dd { 65 | font-size: 1.2em; 66 | } 67 | 68 | .progress { 69 | font-weight: bolder; 70 | } 71 | -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/static/css/schemaCopy.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | .CodeMirror { 8 | height: 100%; 9 | } 10 | 11 | .CodeMirror-line-numbers { 12 | background-color: lightgray; 13 | border-right: 1px solid #eee; 14 | min-width: 2em; 15 | height: 100%; 16 | color: gray; 17 | text-align: right; 18 | padding: .4em .2em .4em .4em; 19 | font-family: "Consolas", "Monaco", "Bitstream Vera Sans Mono", "Courier New", Courier, monospace !important; 20 | } 21 | 22 | .border { 23 | background-color: #FAFAFF; 24 | border: 1px solid darkgray; 25 | } 26 | 27 | .textarea-container { 28 | position: relative; 29 | border: 1px dashed #666 !important; 30 | min-height: 300px; 31 | } 32 | 33 | textarea { 34 | width: 100%; 35 | } 36 | 37 | #spinnerWrapper { 38 | display: none; 39 | margin-top: 100px; 40 | width: 100%; 41 | position: absolute; 42 | z-index: 1000; 43 | text-align: center; 44 | } 45 | 46 | textarea { 47 | width: 100%; 48 | } 49 | 50 | button[disabled] { 51 | cursor: default; 52 | opacity: 0.5; 53 | } 54 | 55 | button.control-button { 56 | float:left; 57 | } 58 | 59 | .placeholder { 60 | color: dimgrey; 61 | } 62 | 63 | .status dd { 64 | font-size: 1rem; 65 | } 66 | 67 | .completed { 68 | color: green; 69 | } 70 | 71 | .failed { 72 | color: red; 73 | } 74 | 75 | #schemaPreviewSummary .total, #schemaPreviewSummary .failed, #schemaPreviewSummary .completed { 76 | font-weight: bold; 77 | } 78 | 79 | #schemaPreviewStatus .completed { 80 | color: green; 81 | text-transform: uppercase; 82 | font-weight: bolder; 83 | } 84 | 85 | #schemaPreviewStatus .failed { 86 | text-transform: uppercase; 87 | font-weight: bolder; 88 | } -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/static/js/customStatistics.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | $(function() { 8 | $('#statistics').dataTable({ 9 | "iDisplayLength" : 50 10 | }); 11 | }) -------------------------------------------------------------------------------- /commercedbsynchac/hac/resources/static/js/migrationReports.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | var reportsTable; 8 | 9 | $(document).ready(function () { 10 | reportsTable = $('#reportsTable').dataTable({ 11 | "bStateSave": true, 12 | "bAutoWidth": false, 13 | "aLengthMenu": [[10, 25, 50, 100, -1], [10, 25, 50, 100, 'all']] 14 | }); 15 | loadMigrationReports(); 16 | }); 17 | 18 | function loadMigrationReports() { 19 | $('#logsWrapper').fadeOut(); 20 | reportsTable.fnClearTable(); 21 | const token = $("meta[name='_csrf']").attr("content"); 22 | const url = $('#reportsWrapper').attr('data-url'); 23 | $.ajax({ 24 | url: url, 25 | type: 'GET', 26 | headers: { 27 | 'Accept': 'application/json', 28 | 'X-CSRF-TOKEN': token 29 | }, 30 | success: function (data) { 31 | if (data.length > 0) { 32 | data.forEach((report) => { 33 | let strippedMigrationId = report.reportId; 34 | reportsTable.fnAddData([ 35 | strippedMigrationId, 36 | report.modifiedTimestamp, 37 | '' 38 | ]) 39 | }); 40 | } 41 | }, 42 | error: hac.global.err 43 | }); 44 | } 45 | 46 | function downloadReport(migrationId) { 47 | const token = $("meta[name='_csrf']").attr("content"); 48 | const url = $('#reportsWrapper').attr('data-downloadUrl') + '?migrationId=' + migrationId; 49 | $.ajax({ 50 | url: url, 51 | type: 'GET', 52 | headers: { 53 | 'X-CSRF-TOKEN': token 54 | }, 55 | success: function (data) { 56 | debug.log(data); 57 | var blob = new Blob([data], {type: "text/plain"}); 58 | var link = document.createElement("a"); 59 | link.href = window.URL.createObjectURL(blob); 60 | link.download = migrationId; 61 | link.click(); 62 | }, 63 | error: hac.global.err 64 | }); 65 | } 66 | 67 | -------------------------------------------------------------------------------- /commercedbsynchac/hac/testsrc/de/hybris/platform/hac/controller/CommercemigrationhacControllerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package de.hybris.platform.hac.controller; 8 | 9 | import de.hybris.bootstrap.annotations.IntegrationTest; 10 | import org.junit.After; 11 | import org.junit.Before; 12 | import org.junit.Test; 13 | 14 | /** 15 | * Test for {@link CommercemigrationhacController}. 16 | */ 17 | @IntegrationTest 18 | public class CommercemigrationhacControllerTest { 19 | 20 | /** 21 | * Code under test. 22 | */ 23 | protected CommercemigrationhacController cut; 24 | 25 | /** 26 | * Set up the code under test. 27 | */ 28 | @Before 29 | public void setup() { 30 | cut = new CommercemigrationhacController(); 31 | } 32 | 33 | /** 34 | * Clean up the code under test. 35 | */ 36 | @After 37 | public void teardown() { 38 | cut = null; 39 | } 40 | 41 | @Test 42 | public void testSayHello() { 43 | /* 44 | * final String helloText = cut.sayHello(); 45 | * 46 | * assertNotNull(helloText); assertNotEquals(0, helloText.length()); 47 | */ 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /commercedbsynchac/project.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | commercedbsynchac.key=value 7 | # Specifies the location of the spring context file putted automatically to the global platform application context. 8 | commercedbsynchac.application-context=commercedbsynchac-spring.xml 9 | migration.from.hac.enabled=true 10 | configuration.view.blacklist.migration=${migration.properties.masked} 11 | 12 | ## fix for Config Panel rendering error due to: "EvalError: Refused to evaluate a string as JavaScript because 'unsafe-eval' is not an allowed" at static/js/configPanel.js:169 13 | hac.xss.filter.header.Content-Security-Policy=default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; script-src 'self' 'unsafe-inline' 'unsafe-eval' -------------------------------------------------------------------------------- /commercedbsynchac/resources/com/sap/cx/boosters/commercedbsynchac/dummy.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/commercedbsynchac/resources/com/sap/cx/boosters/commercedbsynchac/dummy.txt -------------------------------------------------------------------------------- /commercedbsynchac/resources/commercedbsynchac-items.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 11 | 13 | 14 | 15 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/commercedbsynchac-tab-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "basePath": "/commercedbsynchac", 4 | "mainTabLabel": "Migration", 5 | "subTabs": [ 6 | { 7 | "path": "/migrationDataSource", 8 | "label": "Data Sources", 9 | "skipPrefix": false 10 | }, 11 | { 12 | "path": "/migrationSchema", 13 | "label": "Schema Migration", 14 | "skipPrefix": false 15 | }, 16 | { 17 | "path": "/migrationData", 18 | "label": "Data Migration", 19 | "skipPrefix": false 20 | }, 21 | { 22 | "path": "/migrationReports", 23 | "label": "Reports", 24 | "skipPrefix": false 25 | } 26 | ] 27 | } 28 | ] 29 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/commercedbsynchac-without-migration-tab-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | ] 3 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_de.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_en.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_es.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_fr.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_it.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_ja.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_ko.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_pt.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_ru.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/resources/localization/i2ihac-locales_zh.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright: 2022 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | # License: Apache-2.0 4 | # 5 | # 6 | 7 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/CommercedbsynchacStandalone.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac; 8 | 9 | import de.hybris.platform.core.Registry; 10 | import de.hybris.platform.jalo.JaloSession; 11 | import de.hybris.platform.util.RedeployUtilities; 12 | import de.hybris.platform.util.Utilities; 13 | 14 | /** 15 | * Demonstration of how to write a standalone application that can be run 16 | * directly from within eclipse or from the commandline.
17 | * To run this from commandline, just use the following command:
18 | * 19 | * java -jar bootstrap/bin/ybootstrap.jar "new commercedbsynchac.CommercedbsynchacStandalone().run();" 20 | * From eclipse, just run as Java Application. Note that you maybe need 21 | * to add all other projects like ext-commerce, ext-pim to the Launch 22 | * configuration classpath. 23 | */ 24 | public class CommercedbsynchacStandalone { 25 | /** 26 | * Main class to be able to run it directly as a java program. 27 | * 28 | * @param args 29 | * the arguments from commandline 30 | */ 31 | public static void main(final String[] args) { 32 | new CommercedbsynchacStandalone().run(); 33 | } 34 | 35 | public void run() { 36 | Registry.activateStandaloneMode(); 37 | Registry.activateMasterTenant(); 38 | 39 | final JaloSession jaloSession = JaloSession.getCurrentSession(); 40 | System.out.println("Session ID: " + jaloSession.getSessionID()); // NOPMD 41 | System.out.println("User: " + jaloSession.getUser()); // NOPMD 42 | Utilities.printAppInfo(); 43 | 44 | RedeployUtilities.shutdown(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/constants/YhacextConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.constants; 8 | 9 | /** 10 | * Global class for all Commercedbsynchac constants. You can add global 11 | * constants for your extension into this class. 12 | */ 13 | public final class YhacextConstants extends GeneratedYhacextConstants { 14 | public static final String EXTENSIONNAME = "commercedbsynchac"; 15 | 16 | private YhacextConstants() { 17 | // empty to avoid instantiating this constant class 18 | } 19 | 20 | // implement here constants used by this extension 21 | } 22 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/MetricService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric; 8 | 9 | import de.hybris.platform.commercedbsynchac.data.MetricData; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | 12 | import java.util.List; 13 | 14 | public interface MetricService { 15 | List getMetrics(MigrationContext context); 16 | } 17 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/impl/DefaultMetricService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.impl; 8 | 9 | import de.hybris.platform.commercedbsynchac.data.MetricData; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsynchac.metric.MetricService; 12 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | public class DefaultMetricService implements MetricService { 20 | 21 | private static final Logger LOG = LoggerFactory.getLogger(DefaultMetricService.class); 22 | 23 | private final List populators; 24 | 25 | public DefaultMetricService(List populators) { 26 | this.populators = populators; 27 | } 28 | 29 | @Override 30 | public List getMetrics(MigrationContext context) { 31 | List dataList = new ArrayList<>(); 32 | for (MetricPopulator populator : populators) { 33 | if (populator.canHandle(context)) { 34 | try { 35 | dataList.add(populator.populate(context)); 36 | } catch (Exception e) { 37 | LOG.error("Error while populating metric. Populator: " + e.getMessage()); 38 | } 39 | } 40 | } 41 | return dataList; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/MetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator; 8 | 9 | import de.hybris.platform.commercedbsynchac.data.MetricData; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | 12 | public interface MetricPopulator { 13 | String PRIMARY_STANDARD_COLOR = "#92cae4"; 14 | String PRIMARY_CRITICAL_COLOR = "#de5d70"; 15 | String SECONDARY_STANDARD_COLOR = "#d5edf8"; 16 | String SECONDARY_CRITICAL_COLOR = "#e8acb5"; 17 | 18 | MetricData populate(MigrationContext context) throws Exception; 19 | 20 | default void populateColors(MetricData data) { 21 | data.setPrimaryValueStandardColor(PRIMARY_STANDARD_COLOR); 22 | data.setPrimaryValueCriticalColor(PRIMARY_CRITICAL_COLOR); 23 | data.setSecondaryValueStandardColor(SECONDARY_STANDARD_COLOR); 24 | data.setSecondaryValueCriticalColor(SECONDARY_CRITICAL_COLOR); 25 | } 26 | 27 | default boolean canHandle(MigrationContext context) { 28 | return true; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/CpuMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 10 | import de.hybris.platform.commercedbsynchac.data.MetricData; 11 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 12 | import org.springframework.beans.factory.annotation.Value; 13 | 14 | import java.lang.management.OperatingSystemMXBean; 15 | 16 | public class CpuMetricPopulator implements MetricPopulator { 17 | 18 | @Value("#{T(java.lang.management.ManagementFactory).getOperatingSystemMXBean()}") 19 | private OperatingSystemMXBean operatingSystemMXBean; 20 | 21 | @Override 22 | public MetricData populate(MigrationContext context) throws Exception { 23 | MetricData data = new MetricData(); 24 | double systemLoadAverage = operatingSystemMXBean.getSystemLoadAverage(); 25 | int availableProcessors = operatingSystemMXBean.getAvailableProcessors(); 26 | int loadAverage = (int) (systemLoadAverage * 100 / availableProcessors); 27 | if (loadAverage > 100) { 28 | loadAverage = 100; 29 | } 30 | data.setMetricId("cpu"); 31 | data.setName("CPU"); 32 | data.setDescription("The system load in percent"); 33 | data.setPrimaryValue((double) loadAverage); 34 | data.setPrimaryValueLabel("Load"); 35 | data.setPrimaryValueUnit("%"); 36 | data.setPrimaryValueThreshold(90d); 37 | data.setSecondaryValue((double) 100 - loadAverage); 38 | data.setSecondaryValueLabel("Idle"); 39 | data.setSecondaryValueUnit("%"); 40 | data.setSecondaryValueThreshold(0d); 41 | populateColors(data); 42 | return data; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/DTUMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.repository.DataRepository; 10 | import com.sap.cx.boosters.commercedbsync.repository.impl.AzureDataRepository; 11 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 12 | import de.hybris.platform.commercedbsynchac.data.MetricData; 13 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 14 | 15 | import java.util.Optional; 16 | 17 | public class DTUMetricPopulator implements MetricPopulator { 18 | @Override 19 | public MetricData populate(MigrationContext context) throws Exception { 20 | int primaryValue = getAzureDataRepository(context).map(DataRepository::getDatabaseUtilization).orElse(-1f) 21 | .intValue(); 22 | if (primaryValue > 100) { 23 | primaryValue = 100; 24 | } 25 | int secondaryValue = 100 - primaryValue; 26 | if (primaryValue < 0) { 27 | primaryValue = -1; 28 | secondaryValue = -1; 29 | } 30 | 31 | MetricData data = new MetricData(); 32 | 33 | data.setMetricId("dtu"); 34 | data.setName("DTU"); 35 | data.setDescription("The current DTU utilization of the azure database"); 36 | data.setPrimaryValue((double) primaryValue); 37 | data.setPrimaryValueLabel("Used"); 38 | data.setPrimaryValueUnit("%"); 39 | data.setPrimaryValueThreshold(90d); 40 | data.setSecondaryValue((double) secondaryValue); 41 | data.setSecondaryValueLabel("Idle"); 42 | data.setSecondaryValueUnit("%"); 43 | data.setSecondaryValueThreshold(0d); 44 | populateColors(data); 45 | 46 | return data; 47 | } 48 | 49 | private Optional getAzureDataRepository(MigrationContext context) { 50 | if (context.getDataTargetRepository() instanceof AzureDataRepository) { 51 | return Optional.of(context.getDataTargetRepository()); 52 | } else if (context.getDataSourceRepository() instanceof AzureDataRepository) { 53 | return Optional.of(context.getDataSourceRepository()); 54 | } 55 | 56 | return Optional.empty(); 57 | } 58 | 59 | @Override 60 | public boolean canHandle(MigrationContext context) { 61 | return getAzureDataRepository(context).isPresent(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/HikariConnectionMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 10 | import com.zaxxer.hikari.HikariDataSource; 11 | import de.hybris.platform.commercedbsynchac.data.MetricData; 12 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 13 | 14 | import javax.sql.DataSource; 15 | 16 | public abstract class HikariConnectionMetricPopulator implements MetricPopulator { 17 | 18 | @Override 19 | public MetricData populate(MigrationContext context) throws Exception { 20 | if (!(getDataSource(context) instanceof HikariDataSource)) { 21 | throw new RuntimeException("Populator cannot be used for non-hikari datasources"); 22 | } 23 | MetricData data = new MetricData(); 24 | HikariDataSource hikariDS = (HikariDataSource) getDataSource(context); 25 | double activeConnections = hikariDS.getHikariPoolMXBean().getActiveConnections(); 26 | double maxConnections = hikariDS.getHikariConfigMXBean().getMaximumPoolSize(); 27 | data.setMetricId(getMetricId(context)); 28 | data.setName(getName(context)); 29 | data.setDescription("The proportion of active and idle hikari connections"); 30 | data.setPrimaryValue(activeConnections); 31 | data.setPrimaryValueLabel("Active"); 32 | data.setPrimaryValueUnit("#"); 33 | data.setPrimaryValueThreshold(maxConnections); 34 | data.setSecondaryValue(maxConnections - activeConnections); 35 | data.setSecondaryValueLabel("Idle"); 36 | data.setSecondaryValueUnit("#"); 37 | data.setSecondaryValueThreshold(0d); 38 | populateColors(data); 39 | return data; 40 | } 41 | 42 | protected abstract String getMetricId(MigrationContext context); 43 | 44 | protected abstract String getName(MigrationContext context); 45 | 46 | protected abstract DataSource getDataSource(MigrationContext context); 47 | } 48 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/HikariSourceConnectionMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 10 | 11 | import javax.sql.DataSource; 12 | 13 | public class HikariSourceConnectionMetricPopulator extends HikariConnectionMetricPopulator { 14 | 15 | @Override 16 | protected String getMetricId(MigrationContext context) { 17 | return "hikari-source-pool"; 18 | } 19 | 20 | @Override 21 | protected String getName(MigrationContext context) { 22 | return "Source DB Pool"; 23 | } 24 | 25 | @Override 26 | protected DataSource getDataSource(MigrationContext context) { 27 | return context.getDataSourceRepository().getDataSource(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/HikariTargetConnectionMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 10 | 11 | import javax.sql.DataSource; 12 | 13 | public class HikariTargetConnectionMetricPopulator extends HikariConnectionMetricPopulator { 14 | 15 | @Override 16 | protected String getMetricId(MigrationContext context) { 17 | return "hikari-target-pool"; 18 | } 19 | 20 | @Override 21 | protected String getName(MigrationContext context) { 22 | return "Target DB Pool"; 23 | } 24 | 25 | @Override 26 | protected DataSource getDataSource(MigrationContext context) { 27 | return context.getDataTargetRepository().getDataSource(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/IOMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import de.hybris.platform.commercedbsynchac.data.MetricData; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceCategory; 12 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceProfiler; 13 | import com.sap.cx.boosters.commercedbsync.performance.PerformanceUnit; 14 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 15 | 16 | public class IOMetricPopulator implements MetricPopulator { 17 | 18 | private final PerformanceProfiler performanceProfiler; 19 | 20 | public IOMetricPopulator(PerformanceProfiler performanceProfiler) { 21 | this.performanceProfiler = performanceProfiler; 22 | } 23 | 24 | @Override 25 | public MetricData populate(MigrationContext context) throws Exception { 26 | MetricData data = new MetricData(); 27 | int avgRowReading = (int) performanceProfiler.getAverageByCategoryAndUnit(PerformanceCategory.DB_READ, 28 | PerformanceUnit.ROWS); 29 | int avgRowWriting = (int) performanceProfiler.getAverageByCategoryAndUnit(PerformanceCategory.DB_WRITE, 30 | PerformanceUnit.ROWS); 31 | int totalIO = avgRowReading + avgRowWriting; 32 | if (avgRowReading < 1 && avgRowWriting < 1) { 33 | avgRowReading = -1; 34 | avgRowWriting = -1; 35 | } 36 | data.setMetricId("db-io"); 37 | data.setName("Database I/O"); 38 | data.setDescription("The proportion of items read from source compared to items written to target"); 39 | data.setPrimaryValue((double) avgRowReading); 40 | data.setPrimaryValueLabel("Read"); 41 | data.setPrimaryValueUnit("rows/s"); 42 | data.setPrimaryValueThreshold(totalIO * 0.75); 43 | data.setSecondaryValue((double) avgRowWriting); 44 | data.setSecondaryValueLabel("Write"); 45 | data.setSecondaryValueUnit("rows/s"); 46 | data.setSecondaryValueThreshold(totalIO * 0.75); 47 | populateColors(data); 48 | return data; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/MemoryMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 10 | import de.hybris.platform.commercedbsynchac.data.MetricData; 11 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 12 | 13 | public class MemoryMetricPopulator implements MetricPopulator { 14 | @Override 15 | public MetricData populate(MigrationContext context) throws Exception { 16 | MetricData data = new MetricData(); 17 | Runtime runtime = Runtime.getRuntime(); 18 | double freeMemory = runtime.freeMemory() / 1048576L; 19 | double totalMemory = runtime.totalMemory() / 1048576L; 20 | double usedMemory = totalMemory - freeMemory; 21 | data.setMetricId("memory"); 22 | data.setName("Memory"); 23 | data.setDescription("The proportion of free and used memory"); 24 | data.setPrimaryValue(usedMemory); 25 | data.setPrimaryValueLabel("Used"); 26 | data.setPrimaryValueUnit("MB"); 27 | data.setPrimaryValueThreshold(totalMemory * 0.9); 28 | data.setSecondaryValue(freeMemory); 29 | data.setSecondaryValueLabel("Free"); 30 | data.setSecondaryValueUnit("MB"); 31 | data.setSecondaryValueThreshold(0d); 32 | populateColors(data); 33 | return data; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/TaskExecutorMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 10 | import de.hybris.platform.commercedbsynchac.data.MetricData; 11 | import org.apache.commons.lang.StringUtils; 12 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 13 | import org.springframework.core.task.AsyncTaskExecutor; 14 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 15 | 16 | public class TaskExecutorMetricPopulator implements MetricPopulator { 17 | 18 | private final AsyncTaskExecutor executor; 19 | private final String name; 20 | 21 | public TaskExecutorMetricPopulator(AsyncTaskExecutor executor, String name) { 22 | this.executor = executor; 23 | this.name = name; 24 | } 25 | 26 | @Override 27 | public MetricData populate(MigrationContext context) throws Exception { 28 | if (!(executor instanceof ThreadPoolTaskExecutor)) { 29 | throw new RuntimeException("Populator can only be used for " + ThreadPoolTaskExecutor.class.getName()); 30 | } 31 | ThreadPoolTaskExecutor taskExecutor = (ThreadPoolTaskExecutor) executor; 32 | MetricData data = new MetricData(); 33 | int activeCount = taskExecutor.getActiveCount(); 34 | int maxPoolSize = taskExecutor.getMaxPoolSize(); 35 | data.setMetricId(name + "-executor"); 36 | data.setName(StringUtils.capitalize(name) + " Tasks"); 37 | data.setDescription("The tasks running in parallel in the task executor"); 38 | data.setPrimaryValue((double) activeCount); 39 | data.setPrimaryValueLabel("Running"); 40 | data.setPrimaryValueUnit("#"); 41 | data.setPrimaryValueThreshold(-1d); 42 | data.setSecondaryValue((double) maxPoolSize - activeCount); 43 | data.setSecondaryValueLabel("Free"); 44 | data.setSecondaryValueUnit("#"); 45 | data.setSecondaryValueThreshold(-1d); 46 | populateColors(data); 47 | return data; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /commercedbsynchac/src/com/sap/cx/boosters/commercedbsynchac/metric/populator/impl/ThreadPoolMetricPopulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright: 2023 SAP SE or an SAP affiliate company and commerce-db-synccontributors. 3 | * License: Apache-2.0 4 | * 5 | */ 6 | 7 | package com.sap.cx.boosters.commercedbsynchac.metric.populator.impl; 8 | 9 | import com.sap.cx.boosters.commercedbsync.concurrent.DataThreadPoolFactory; 10 | import com.sap.cx.boosters.commercedbsync.context.MigrationContext; 11 | import com.sap.cx.boosters.commercedbsynchac.metric.populator.MetricPopulator; 12 | import de.hybris.platform.commercedbsynchac.data.MetricData; 13 | import org.apache.commons.lang.StringUtils; 14 | 15 | public class ThreadPoolMetricPopulator implements MetricPopulator { 16 | 17 | private final DataThreadPoolFactory factory; 18 | private final String name; 19 | 20 | public ThreadPoolMetricPopulator(DataThreadPoolFactory factory, String name) { 21 | this.factory = factory; 22 | this.name = name; 23 | } 24 | 25 | @Override 26 | public MetricData populate(MigrationContext context) throws Exception { 27 | MetricData data = new MetricData(); 28 | double activeCount = factory.getMonitor().getActiveCount(); 29 | double maxPoolSize = factory.getMonitor().getMaxPoolSize(); 30 | if (maxPoolSize < 1) { 31 | // make primary and secondary value negative to indicate inactive widget 32 | activeCount = -1; 33 | maxPoolSize = -2; 34 | } 35 | data.setMetricId(name + "-executor"); 36 | data.setName(StringUtils.capitalize(name) + " Tasks"); 37 | data.setDescription("The workers running in parallel in the task executor"); 38 | data.setPrimaryValue(activeCount); 39 | data.setPrimaryValueLabel("Running"); 40 | data.setPrimaryValueUnit("#"); 41 | data.setPrimaryValueThreshold(-1d); 42 | data.setSecondaryValue(maxPoolSize - activeCount); 43 | data.setSecondaryValueLabel("Free"); 44 | data.setSecondaryValueUnit("#"); 45 | data.setSecondaryValueThreshold(-1d); 46 | populateColors(data); 47 | return data; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /docs/anonymizer/README.md: -------------------------------------------------------------------------------- 1 | ## Anonymizer 2 | 3 | The config.yaml file holds the tables and columns used for anonymization along with the new values. 4 | You can utilize the tool by using the pre-set config file found in the git repository, or by creating your own. 5 | The location where it is found is commercedbsync/resources/anonymizer/config.yml. 6 | During the migration process, if the table and the column that is handled is part of the anonymizer configuration and the value is not part of the exclude list, then the configuration defined config.yml is going to be applied. 7 | 8 | ```` 9 | tables: 10 | - name: addresses -> name of the table that will be included in the process 11 | columns: 12 | - name: p_cellphone -> column name 13 | operation: REPLACE -> operation that will be performed 14 | text: "{RANDOM(number,10)}" -> the new value 15 | exclude: ["some_string"] -> values to be excluded (i.e. not changed) 16 | excludeRow: ["some_string"] -> optional, if value is matched for configured column anonymization for entire row is skipped and row is copied unaltered 17 | ```` 18 | 19 | The 'exclude' clause is optional. You can use it if you wish to exclude certain values from anonymization. For instance, if there are test users for which you don't want to alter the values. 20 | `name`, `operation` and `text` are mandatory. 21 | 22 | Operations that can be performed on the data: 23 | - REPLACE -> will replace the entire value 24 | - APPEND -> will append a text to the current value 25 | - DELETE -> will delete the value (set it to null) 26 | 27 | Functions available: 28 | - GUID -> generate a guid 29 | - RANDOM -> generate a random value 30 | 31 | The RANDOM function can have two parameters: 32 | 1. the type of random value: "number", "string", "city", "street" 33 | 2. length of the random value (this parameter is needed only for "number" and "string" types) 34 | 35 | 36 | Text that can be used to replace/append to the value: 37 | - simple string
38 | text: "My description" 39 | 40 | 41 | - random number with n digits "{RANDOM(number,n)}"
42 | text: "{RANDOM(number,10)}" 43 | 44 | 45 | - random string with n characters "{RANDOM(string,n)}"
46 | text: "{RANDOM(string,3)}" 47 | 48 | 49 | - GUID value
50 | text: "{GUID}" 51 | 52 | 53 | - any combination 54 | text: "{GUID}-{RANDOM(string,3)}-SOME_TEXT" 55 | 56 | 57 | The absence of text will result in a blank field. If the configuration file is not correct an error will be displayed in the console. 58 | 59 | -------------------------------------------------------------------------------- /docs/commercedbsync/after_save_listener_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/commercedbsync/after_save_listener_1.png -------------------------------------------------------------------------------- /docs/commercedbsync/after_save_listener_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/commercedbsync/after_save_listener_2.png -------------------------------------------------------------------------------- /docs/performance/performance_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/performance/performance_architecture.png -------------------------------------------------------------------------------- /docs/performance/template_for_scheduled_operational_activity.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/performance/template_for_scheduled_operational_activity.docx -------------------------------------------------------------------------------- /docs/support/support_groovy_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/support/support_groovy_preview.png -------------------------------------------------------------------------------- /docs/user/commerce-db-sync-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/commerce-db-sync-demo.png -------------------------------------------------------------------------------- /docs/user/data_migration_architecture.drawio: -------------------------------------------------------------------------------- 1 | 7Vxdc5s4FP01eYwHEGD8GMdJOrtpm1nvZtunjAwyZgPIFXJj99evhCUbkHDMxuBklnTawkV8nXt079GVyAW4TtZ3BC4Xn3GA4gvLCNYXYHJhWaZtuew/btlsLZ7lbA0hiQLRaG+YRr+QMBrCuooClJUaUoxjGi3LRh+nKfJpyQYJwS/lZnMcl++6hCFSDFMfxqr17yigC/kWw739E4rChbyz6Y62RxIoG4s3yRYwwC8FE7i5ANcEY7rdStbXKObgSVy2593WHN09GEEpPeaEr/i3z9/h1ebHt98z9GPy5a/lp5tLcZWfMF6JFxYPSzcSAXYVBjbbGbM3WHKjH+MVu+j4ZRFRNF1CnxtfmP+ZbUGTmO2ZbHMexfE1jjHJrwMCiLy5zy9DCX5GhSOu76HZnB1RX0k+HyIUrQsm8Yp3CCeIkg1rIo46QwG34Jstdl/2znOlbVFwnDsSRigIE+4uvceUbQhYm0B8BMYhwSuOXsbAjNLwHs35YxmNINn1ATiTlzUOQjWsQAVMFSsANFiZZmtYWQpW0z/vFLgYWmmAAgHRKzSEcRSmbDvOQa2ycu7wP1pW5j/8DJzSgn37I+xT8VD8RjGcofgBZxGNML+hz1yG2Elj7qqIhZT7SoMkCgJ+9q7BlXhUivk7hDHMMvGO2TOi/kLuyEhiKJRxtJQ5zMvX+5YgiAzdZ+QHUPhxjZMEEeZ7znY35i6eEbYV8q2r5TJmwOZ4V0mUu+t4+ijelM6aYUpxou28wjIWLSY2s0UJSxlX1Yvm1kctDfJDbC9KQoaZz4IxJJRt3SFInkzLW7O/g2UaVhhpnJ4G4iiwy2FDZYWnIUV78dU+xIkJ6w7GdJP6vf9P5X/TGb4vAjivJ9iddCGYx9SG0sUwHAPNdUnCMMzJ+FpNEvP8p5VgLI9a5eztql6wPNULoDUvuIoXJuNaPwSQwoxigl73RWsIWm4ZQUcjFXVKsTUEh8cKxe40oa3Rz13nfE+B5eGPSa8JW9GEwxoi1WvC8/Nj1GvCU2uC42lQowlVVnQqCSQre03Ykf8VTXhuAhxRdHn3mrC5F6qaUPVCp5pQgn42TdgYwaomtFVV3akmtNSCR+eacFdDFZiYI03OH3oDR8XFNtoCBjQY9KE1z/sBXs1EFATjRmqxpK1qFVhFUwYwW+Q3MHcxQk4ggOMZvPP/0Qy+rKTiS41C28Xdkq+s1nw1UlyDghDJrIQJXeAQpzC+2VsLHuIA7tvcY54Bc+M/iNKNmDKCK4rLXmPAks034Zx85zvfGQBX7k/WxaOTTXHvAZGIvT1PxKqrGzovwyvio0MNhcsoJCE6dEUgBtkcvoNkIChmCvdneRJL59j81CtC4KbQYImjlGaFKz9wQ31EcJwiR15tbo5AhVPbB9gzbPcmb4icallwcvOoELEfNp5g2NggShWjkNXpONFSM0Y/TnzbOKF5dqokJ82MY7fDBLW21KlAbQ6gKYVP/ZxcpwIVqGXfx4cvCoR4ReMoZTFRrtIwihLJOBDZdl1EH9mULlbWbbnfknXIF6cM4EsGBj+X6ZMQwPkNqlH81vEcwDsbOyeI0D5epzgt6xIlftZH2e7k324SX/awoasQxNFrda8lisjs/5Hln+ov81htN/pY2s6urgrxSmKtafuWxJ1a8+3Xibxd3DWfQqsoe6AbbHYr82Qxq5d5/0XmNWfAsFL5P3flF6iV37YkXfOVM8OyeAPnLi8CtUDbz62+gzhqa9YidB1H+6V254yjtmbuptM4aqv19P+RagcfrCJre+UA4hiHS7LV9p2odqApyb7X1GwPz52a1dVgX1lkNR4ISqIMKbAV4KjLfzUVkzw6j6H/HObdVzdjXR+IS5UNS5fXDcMzbpU0Wqr9VAs1Ec6Gg8jHaTZYMDiVqkttpq6VBCdgSHWga2pytGR4aXbdkdWW03NELV9e/VoRnqCnq1nmk2ipzceHCaJhkuJ3hURlt4tambJkYnxza+kKc/yht/8+ZcUHP7rcVp5t7YoSrlulhBo0HE3QAG0RwrbfnLRL4J0wgzuvJPA35Gr3yFwtV5KeLle/zVlO76wDzrLel7Pc3lkH6rHvyldqUaOf+T9DUaP6LYFutrfjteJ9TaPLmoZnlGsauo8FOq1pfKwPxZrjPap0OF0Vscv1v/2XmW11rZFVXm9/9q7V3dd/zb+UNEClX2i+WD1RbYft7n+jx7Zutv+9KODmXw== 2 | -------------------------------------------------------------------------------- /docs/user/data_migration_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/data_migration_architecture.png -------------------------------------------------------------------------------- /docs/user/data_replication_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/data_replication_architecture.png -------------------------------------------------------------------------------- /docs/user/hac_migrate_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/hac_migrate_data.png -------------------------------------------------------------------------------- /docs/user/hac_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/hac_report.png -------------------------------------------------------------------------------- /docs/user/hac_schema_diff_exec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/hac_schema_diff_exec.png -------------------------------------------------------------------------------- /docs/user/hac_schema_diff_prev.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/hac_schema_diff_prev.png -------------------------------------------------------------------------------- /docs/user/hac_validate_ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/hac_validate_ds.png -------------------------------------------------------------------------------- /docs/user/proxy_timeout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SAP/sap-commerce-db-sync/00d8f18148102c6f4db3d05910d118a0f2dc892e/docs/user/proxy_timeout.png --------------------------------------------------------------------------------