├── .gitattributes ├── resources ├── impex │ ├── essentialdata-jobs.impex │ ├── sanecleanup │ │ ├── solrfacetsearch │ │ │ ├── 001-disable-logtodatabase.impex │ │ │ └── 002-cleanup-solrindexoperation.impex │ │ ├── ruleengine │ │ │ ├── 001-enable-rules-cleanup.impex │ │ │ ├── 003-rule-engine-orphans.impex.draft │ │ │ └── 002-delete-expired-rules.impex │ │ ├── cms2 │ │ │ └── 001-optimized-versiongc.impex │ │ ├── impex │ │ │ ├── 001-cleanup-impex.impex │ │ │ └── 002-cleanup-distributed-impex.impex │ │ ├── core │ │ │ ├── 001-cleanup-httpsession.impex │ │ │ └── 002-cleanup-savedvalueentries.impex │ │ ├── platformservices │ │ │ └── 001-enable-cronjoblogs-cleanup.impex │ │ ├── processing │ │ │ ├── 002-cleanup-cronjobs.impex │ │ │ ├── 001-cleanup-cronjobhistory.impex │ │ │ ├── 004-cleanup-processtasklog.impex │ │ │ └── 003-cleanup-businessprocess.impex │ │ ├── commerceservices │ │ │ └── 001-cleanup-oldcarts.impex │ │ └── acceleratorservices │ │ │ └── 001-cleanup-emails.impex │ └── bulkdelete-cronjoblogs.impex ├── sanecleanup.build.number ├── sanecleanup-spring.xml ├── retentionrule-to-impex.groovy └── sanecleanup-items.xml ├── .settings ├── org.springframework.ide.eclipse.beans.core.prefs ├── org.springframework.ide.eclipse.core.prefs ├── org.eclipse.jdt.ui.prefs └── org.eclipse.jdt.core.prefs ├── src └── mpern │ └── sap │ ├── cleanup │ ├── constants │ │ └── SanecleanupConstants.java │ ├── VersionRangeEvaluator.java │ ├── CleanupAfterInitListener.java │ └── cms2 │ │ └── CMSVersionGCPerformable.java │ └── commerce │ └── build │ └── util │ └── Version.java ├── .gitignore ├── .classpath ├── extensioninfo.xml ├── testsrc └── mpern │ └── sap │ ├── cleanup │ └── VersionRangeEvaluatorTest.groovy │ └── commerce │ └── build │ └── util │ └── VersionTest.groovy ├── .project ├── project.properties ├── check-audit.groovy ├── CHANGELOG.md ├── LICENSE.txt ├── README.md └── excessive-platform-types.groovy /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /resources/impex/essentialdata-jobs.impex: -------------------------------------------------------------------------------- 1 | 2 | INSERT_UPDATE ServicelayerJob;code[unique=true];springId; 3 | ;jdbcVersionGCJob;jdbcVersionGCPerformable; 4 | -------------------------------------------------------------------------------- /.settings/org.springframework.ide.eclipse.beans.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.springframework.ide.eclipse.beans.core.ignoreMissingNamespaceHandler=false 3 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/solrfacetsearch/001-disable-logtodatabase.impex: -------------------------------------------------------------------------------- 1 | UPDATE CronJob[batchmode = true]; itemtype(code)[unique = true]; logToDatabase; 2 | ; SolrIndexerCronJob ; false ; 3 | -------------------------------------------------------------------------------- /src/mpern/sap/cleanup/constants/SanecleanupConstants.java: -------------------------------------------------------------------------------- 1 | package mpern.sap.cleanup.constants; 2 | 3 | public final class SanecleanupConstants extends GeneratedSanecleanupConstants { 4 | public static final String EXTENSIONNAME = "sanecleanup"; 5 | 6 | private SanecleanupConstants() { 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /resources/sanecleanup.build.number: -------------------------------------------------------------------------------- 1 | #Ant properties 2 | #Thu Nov 05 12:00:45 CET 2020 3 | version.api=2005 4 | vendor=hybris 5 | group.id=de.hybris.platform 6 | name=sanecleanup 7 | description=sanecleanup 8 | builddate=20201105 1200 9 | releasedate=20200513 1957 10 | version=2005.5 11 | module.name=platform-module 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #generated by platform build artifacts 2 | *Generated*Constants.java 3 | *Generated*Manager.java 4 | items.xsd 5 | beans.xsd 6 | extensioninfo.xsd 7 | ruleset.xml 8 | platformhome.properties 9 | *testclasses.xml 10 | build.xml 11 | gensrc 12 | eclipsebin 13 | velocity.log 14 | 15 | *_bof.jar 16 | 17 | classes 18 | testclasses -------------------------------------------------------------------------------- /resources/impex/sanecleanup/ruleengine/001-enable-rules-cleanup.impex: -------------------------------------------------------------------------------- 1 | # https://help.sap.com/viewer/9d346683b0084da2938be8a285c0c27a/LATEST/en-US/7482a16e3d7c4848a5a7bec7185132c5.html 2 | 3 | UPDATE MaintenanceCleanupJob;code[unique=true];active; 4 | ;droolsRulesMaintenanceCleanupPerformable;true 5 | 6 | UPDATE Trigger;cronJob(code)[unique=true];active 7 | ;droolsRulesMaintenanceCleanupJob;true 8 | -------------------------------------------------------------------------------- /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /extensioninfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /testsrc/mpern/sap/cleanup/VersionRangeEvaluatorTest.groovy: -------------------------------------------------------------------------------- 1 | package mpern.sap.cleanup 2 | 3 | import mpern.sap.commerce.build.util.Version 4 | import org.junit.Test 5 | import spock.lang.Specification 6 | 7 | class VersionRangeEvaluatorTest extends Specification { 8 | 9 | @Test 10 | def "version range expressions work as expected"(expression, result) { 11 | given: 12 | VersionRangeEvaluator evaluator = new VersionRangeEvaluator(Version.parseVersion("3012.3")); 13 | 14 | expect: 15 | evaluator.evaluate(expression) == result 16 | 17 | where: 18 | expression | result 19 | 'between("3012.0", "3012.2")' | false 20 | 'between("3012.0", "3012.1") or between("3012.3", "3012.5") ' | true 21 | "between('2900.0','3100.0')" | true 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /testsrc/mpern/sap/commerce/build/util/VersionTest.groovy: -------------------------------------------------------------------------------- 1 | package mpern.sap.commerce.build.util 2 | 3 | import spock.lang.Specification 4 | 5 | class VersionTest extends Specification { 6 | 7 | def "version parsing works as expected"(String input, Version result) { 8 | given: 9 | def version = Version.parseVersion(input) 10 | 11 | expect: 12 | version == result 13 | 14 | where: 15 | input | result 16 | "2105.12" | new Version(21, 5, 0, 12, "2105.12") 17 | "2105" | new Version(21, 5, 0, Version.UNDEFINED_PART, "2105") 18 | "2105.10-2105-2202.26-20220510.1-cbe2a6a-develop" | new Version(21, 5, 0, 10, "2105.10-2105-2202.26-20220510.1-cbe2a6a-develop") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | sanecleanup 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.springframework.ide.eclipse.core.springbuilder 15 | 16 | 17 | 18 | 19 | 20 | org.springframework.ide.eclipse.core.springnature 21 | org.eclipse.jdt.core.javanature 22 | 23 | 24 | 25 | 1606724818477 26 | 27 | 30 28 | 29 | org.eclipse.core.resources.regexFilterMatcher 30 | node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /resources/sanecleanup-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/ruleengine/003-rule-engine-orphans.impex.draft: -------------------------------------------------------------------------------- 1 | # Import config properties into impex macros 2 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 3 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 4 | 5 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 6 | ; orphanedPromotionActionParameterRule ; " 7 | SELECT {ap:pk}, {ap:itemtype} 8 | FROM { 9 | PromotionActionParameter AS ap 10 | LEFT JOIN RuleBasedPotentialPromotionMessageAction AS a ON {a:parameters} LIKE 11 | } 12 | WHERE {ar:status} = {rs:pk} 13 | AND {rs:code} = 'PUBLISHED' 14 | AND {ar:enddate} IS NOT NULL 15 | AND {ar:enddate} < ?JAVA_CURRENT_TIME" ; 0 ; basicRemoveCleanupAction ; 16 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 17 | ; abstractRuleCleanupJob ; abstractRuleRule ; 1000 18 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 19 | ; abstractRuleCleanupCronJob ; abstractRuleCleanupJob ; 20 | INSERT_UPDATE Trigger; cronJob(code)[unique = true] ; cronExpression 21 | # every day at midnight 22 | ; abstractRuleCleanupCronJob ; 0 0 0 * * ? 23 | 24 | -------------------------------------------------------------------------------- /project.properties: -------------------------------------------------------------------------------- 1 | sanecleanup.application-context=sanecleanup-spring.xml 2 | 3 | # disable change history entries / "Last Changes" 4 | # hmc.storing.modifiedvalues.size=0 5 | 6 | # replaced with mpern.sap.cleanup.cms2.CMSVersionGCPerformable 7 | version.gc.enabled=false 8 | 9 | sanecleanup.jobs.sessionlanguage=en 10 | 11 | # 7200 = 2 hours 12 | # 86400 = 1 day 13 | # 1209600 = 2 weeks 14 | # 2419200 = 1 month / 4 weeks 15 | # 15778476 = 6 months 16 | sanecleanup.retentiontimeseconds.emailmessage=1209600 17 | sanecleanup.retentiontimeseconds.cart.regular=2419200 18 | sanecleanup.retentiontimeseconds.cart.anonymous=1209600 19 | sanecleanup.retentiontimeseconds.storedhttpsession=7200 20 | sanecleanup.retentiontimeseconds.savedvalues.delete=1209600 21 | sanecleanup.retentiontimeseconds.impexmedia.generated=1209600 22 | sanecleanup.retentiontimeseconds.impeximportcronjob.distributed=1209600 23 | sanecleanup.retentiontimeseconds.cronjob.generated=1209600 24 | sanecleanup.retentiontimeseconds.solrindexerjob.generated=86400 25 | sanecleanup.retentiontimeseconds.businessprocess.succeeded=2419200 26 | sanecleanup.retentiontimeseconds.businessprocess.failed=15778476 27 | sanecleanup.retentiontimeseconds.businessprocess.running=15778476 28 | sanecleanup.retentiontimeseconds.taskcondition.premature=15778476 29 | sanecleanup.retentiontimeseconds.solrindexoperation=86400 30 | -------------------------------------------------------------------------------- /src/mpern/sap/cleanup/VersionRangeEvaluator.java: -------------------------------------------------------------------------------- 1 | package mpern.sap.cleanup; 2 | 3 | import mpern.sap.commerce.build.util.Version; 4 | import org.springframework.expression.Expression; 5 | import org.springframework.expression.ExpressionParser; 6 | import org.springframework.expression.spel.standard.SpelExpressionParser; 7 | 8 | public class VersionRangeEvaluator { 9 | 10 | private final ExpressionParser parser = new SpelExpressionParser(); 11 | private final VersionCompare compare; 12 | 13 | 14 | public VersionRangeEvaluator(Version currentVersion) { 15 | this.compare = new VersionCompare(currentVersion); 16 | } 17 | 18 | public boolean evaluate(String expression) { 19 | final Expression exp = parser.parseExpression(expression); 20 | Boolean result = (Boolean) exp.getValue(compare); 21 | if (result == null) { 22 | throw new IllegalArgumentException(String.format("Illegal version expression %s", expression)); 23 | } 24 | return result; 25 | } 26 | 27 | public static class VersionCompare { 28 | private final Version currentVersion; 29 | 30 | public VersionCompare(Version currentVersion) { 31 | this.currentVersion = currentVersion; 32 | } 33 | 34 | public boolean between(String from, String to) { 35 | return currentVersion.compareTo(Version.parseVersion(from)) >= 0 && currentVersion.compareTo(Version.parseVersion(to)) <= 0; 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/cms2/001-optimized-versiongc.impex: -------------------------------------------------------------------------------- 1 | #:versions: between("1905.0", "1905.29") or between("2005.0", "2005.13") or between("2011.0", "2011.8") 2 | # Import config properties into impex macros 3 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 4 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 5 | 6 | INSERT_UPDATE CronJob;code[unique=true];job(code);queryCount;sessionLanguage(isoCode)[default = $sessionLanguage] 7 | ;jdbcVersionGCCronJob;jdbcVersionGCJob;1000; 8 | 9 | INSERT Trigger;cronjob(code)[unique=true];cronExpression 10 | ;jdbcVersionGCCronJob; 0 0 0 * * ? 11 | 12 | # delete cms version gc business processes 13 | $oneDay=86400 14 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 15 | ; cmsVersionGCProcessRule ; " 16 | SELECT {p:pk}, {p:itemtype} 17 | FROM {BusinessProcess AS p } 18 | WHERE {p:code} LIKE 'cmsVersionGCProcess%' 19 | AND {p:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $oneDay ; basicRemoveCleanupAction ; 20 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 21 | ; cmsVersionGCProcessCleanupJob ; cmsVersionGCProcessRule ; 1000 22 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 23 | ; cmsVersionGCProcessCleanupCronJob ; cmsVersionGCProcessCleanupJob ; 24 | INSERT Trigger; cronJob(code)[unique = true] ; cronExpression 25 | # every day at 03:00 26 | ; cmsVersionGCProcessCleanupCronJob ; 0 0 3 * * ? 27 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/impex/001-cleanup-impex.impex: -------------------------------------------------------------------------------- 1 | # Import config properties into impex macros 2 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 3 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 4 | $retentionTime = $config-sanecleanup.retentiontimeseconds.impexmedia.generated 5 | 6 | # @readme ImpexMedia 7 | # Are there more than a handful (>100) of generated impex medias? 8 | # SELECT 9 | # COUNT(*) 10 | # FROM 11 | # {ImpexMedia AS i} 12 | # WHERE 13 | # ( 14 | # {i:code} LIKE '0_______' 15 | # OR {i:code} LIKE 'generated impex media - %' 16 | # ) 17 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 18 | ; impexMediaRule ; " 19 | SELECT {i:pk}, {i:itemtype} 20 | FROM {ImpexMedia AS i} 21 | WHERE ( {i:code} LIKE '0_______' OR {i:code} LIKE 'generated impex media - %' ) 22 | AND {i:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ; 23 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 24 | ; impexMediaCleanupJob ; impexMediaRule ; 1000 25 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 26 | ; impexMediaCleanupCronJob ; impexMediaCleanupJob ; 27 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 28 | # every day at 05:00 29 | ; impexMediaCleanupCronJob ; 0 0 5 * * ? 30 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/solrfacetsearch/002-cleanup-solrindexoperation.impex: -------------------------------------------------------------------------------- 1 | 2 | # Import config properties into impex macros 3 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 4 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 5 | $retentionTime=$config-sanecleanup.retentiontimeseconds.solrindexoperation 6 | 7 | # @readme SolrIndexOperation 8 | # Too many solr operations (more than ~100 per index)? 9 | # SELECT {i:qualifier}, 10 | # COUNT({o:pk}) AS "total", 11 | # MIN({o:modifiedTime}) AS "oldest", 12 | # MAX({o:modifiedTime}) AS "newest" 13 | # FROM {SolrIndexOperation AS o 14 | # LEFT JOIN SolrIndex AS i 15 | # ON {o:index} = {i:pk} } 16 | # GROUP BY {i:qualifier} 17 | # ORDER BY "total" DESC 18 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 19 | ; solrIndexOperationRule ; " 20 | SELECT {o:pk}, {o:itemtype} 21 | FROM {SolrIndexOperation AS o} 22 | WHERE {o:endTime} IS NOT NULL 23 | AND {o:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ; 24 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 25 | ; solrIndexOperationCleanupJob ; solrIndexOperationRule ; 1000 26 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 27 | ; solrIndexOperationCleanupCronJob ; solrIndexOperationCleanupJob ; 28 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 29 | # every day at 04:00 30 | ; solrIndexOperationCleanupCronJob ; 0 0 4 * * ? 31 | -------------------------------------------------------------------------------- /resources/retentionrule-to-impex.groovy: -------------------------------------------------------------------------------- 1 | import de.hybris.platform.retention.RetentionRequestParams 2 | import de.hybris.platform.servicelayer.search.FlexibleSearchQuery 3 | import de.hybris.platform.servicelayer.impex.impl.StreamBasedImpExResource 4 | 5 | // close transaction to avoid errors when creating the impex job in the hac scripting console 6 | de.hybris.platform.tx.Transaction.current().commit() 7 | 8 | def RETENTIION_RULE = 'impexMediaRule' 9 | def ruleQuery = new FlexibleSearchQuery("SELECT {pk} FROM {FlexibleSearchRetentionRule} WHERE {code} = ?rule") 10 | ruleQuery.addQueryParameter("rule", RETENTIION_RULE) 11 | 12 | def rule = flexibleSearchService.searchUnique(ruleQuery) 13 | 14 | // rule.retentionTimeSeconds = 1 15 | def retentionParams = RetentionRequestParams.builder().withRuleModel(rule).withBatchSize(1000).build(); 16 | 17 | def itemProvider = retentionItemsProviderFactory.create(retentionParams) 18 | def typeToPK = [:].withDefault { [] as Set } 19 | 20 | def items = itemProvider.nextItemsForCleanup() 21 | while (items) 22 | { 23 | items.forEach { 24 | typeToPK[it.itemType].add(it.pk) 25 | } 26 | items = itemProvider.nextItemsForCleanup() 27 | } 28 | 29 | typeToPK.forEach { type, pks -> 30 | def impex = "REMOVE ${type};pk[unique=true]\n" 31 | impex += ';' + pks.join(";\n;") + ";" 32 | 33 | def impexResource = new StreamBasedImpExResource(new ByteArrayInputStream(impex.getBytes('UTF-8')), 'UTF-8') 34 | 35 | def importConfig = spring.getBean('importConfig') 36 | importConfig.removeOnSuccess = true 37 | importConfig.script = impexResource 38 | importConfig.synchronous = false 39 | 40 | def importResult = importService.importData(importConfig) 41 | 42 | println("Bulk-delete ${type}: ${importResult.cronJob.code}") 43 | } -------------------------------------------------------------------------------- /resources/impex/sanecleanup/core/001-cleanup-httpsession.impex: -------------------------------------------------------------------------------- 1 | # delete all stored sessions one day after there last update 2 | 3 | # Import config properties into impex macros 4 | UPDATE GenericItem[processor = de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor]; pk[unique = true] 5 | $sessionLanguage = $config-sanecleanup.jobs.sessionlanguage 6 | $retentionTime = $config-sanecleanup.retentiontimeseconds.storedhttpsession 7 | 8 | # @readme StoredHttpSession 9 | # Excessive amount of session? This is hard to generalize as it highly depends on your site's traffic, but if you are near or over 5 digits, it's probably too much. 10 | # 11 | # Simarly, stale sessions (e.g older than a day) don't need to be retained. 12 | # SELECT 13 | # COUNT({s:pk}) AS "total", 14 | # MIN({s:modifiedtime}) AS "oldest", 15 | # MAX({s:modifiedtime}) AS "newest" 16 | # FROM {StoredHttpSession AS s} 17 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 18 | ; storedHttpSessionRule ; "select {s:pk}, {s:itemtype} 19 | from {StoredHttpSession as s} 20 | where {s:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ; 21 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 22 | ; storedHttpSessionCleanupJob ; storedHttpSessionRule ; 1000 23 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 24 | ; storedHttpSessionCleanupCronJob ; storedHttpSessionCleanupJob ; 25 | INSERT Trigger; cronJob(code)[unique = true] ; cronExpression 26 | # every 30 minutes 27 | ; storedHttpSessionCleanupCronJob ; 0 0/30 * * * ? 28 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/ruleengine/002-delete-expired-rules.impex: -------------------------------------------------------------------------------- 1 | # https://www.sap.com/cxworks/article/538808299/top_10_recommendations_for_improving_the_performance_of_your_commerce_cloud_promotion_engine 2 | # Tip #7 3 | 4 | # Import config properties into impex macros 5 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 6 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 7 | 8 | # @readme AbstractRule 9 | # Are there any outdated rules? i.e rules that aren't valid anymore because their enddate is in the past. 10 | # 11 | # Warning: change `getutcdate()` to your DBMS (for HANA/MySQL: `now()` ) 12 | # SELECT COUNT({ar:pk}), 13 | # MIN({ar:modifiedtime}) AS "oldest", 14 | # MAX({ar:modifiedtime}) AS "newest" 15 | # FROM {AbstractRule AS ar}, {RuleStatus AS rs} 16 | # WHERE {ar:status} = {rs:pk} 17 | # AND {rs:code} = 'PUBLISHED' 18 | # AND {ar:enddate} IS NOT NULL 19 | # AND {ar:enddate} < getutcdate() 20 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 21 | ; abstractRuleRule ; " 22 | SELECT {ar:pk}, {ar:itemtype} 23 | FROM {AbstractRule AS ar}, {RuleStatus AS rs} 24 | WHERE {ar:status} = {rs:pk} 25 | AND {rs:code} = 'PUBLISHED' 26 | AND {ar:enddate} IS NOT NULL 27 | AND {ar:enddate} < ?JAVA_CURRENT_TIME" ; 0 ; basicRemoveCleanupAction ; 28 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 29 | ; abstractRuleCleanupJob ; abstractRuleRule ; 1000 30 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 31 | ; abstractRuleCleanupCronJob ; abstractRuleCleanupJob ; 32 | INSERT Trigger; cronJob(code)[unique = true] ; cronExpression 33 | # every day at midnight 34 | ; abstractRuleCleanupCronJob ; 0 0 0 * * ? 35 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/platformservices/001-enable-cronjoblogs-cleanup.impex: -------------------------------------------------------------------------------- 1 | # enable ootb cronjob logs cleanup 2 | # if you have never enabled this before, have a look at bulkdelete-cronjoblogs.impex for a cronjob that you can use to 3 | # delete a large number of stale job logs 4 | 5 | # Import config properties into impex macros 6 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 7 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 8 | 9 | # @readme LogFile 10 | # Are there are cronjob with more than ~10 logs and/or logs older than 14 days? 11 | # (those are default values for log file retention) 12 | # SELECT 13 | # COALESCE({cj:code}, ''), 14 | # COUNT({l:pk}) AS "total", 15 | # MIN({l:modifiedtime}) AS "oldest", 16 | # MAX({l:modifiedtime}) AS "newest" 17 | # FROM 18 | # {LogFile AS l 19 | # LEFT JOIN 20 | # CronJob AS cj 21 | # ON {l:owner} = {cj:pk} } 22 | # GROUP BY 23 | # {cj:code} 24 | # ORDER BY 25 | # "total" DESC 26 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ;queryCount; sessionLanguage(isoCode)[default = $sessionLanguage] 27 | ; cronJobLogCleanupCronJob ; cleanUpLogsJobPerformable ; 2147483647 ; 28 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 29 | # every hour 30 | ; cronJobLogCleanupCronJob ; 0 0 0/1 * * ? 31 | 32 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 33 | ; orphanedLogsRule ; " 34 | SELECT {l:pk}, {l:itemtype} 35 | FROM {LogFile AS l LEFT JOIN CronJob AS cj ON {l:owner} = {cj:pk} } 36 | WHERE {cj:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 37 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 38 | ; orphanedLogsCleanupJob ; orphanedLogsRule ; 1000 39 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 40 | ; orphanedLogsCleanupCronJob ; orphanedLogsCleanupJob ; 41 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 42 | # every day at midnight 43 | ; orphanedLogsCleanupCronJob ; 0 0 0 * * ? 44 | -------------------------------------------------------------------------------- /check-audit.groovy: -------------------------------------------------------------------------------- 1 | import de.hybris.platform.persistence.audit.gateway.AuditStorageUtils 2 | 3 | def auditCount(type) { 4 | try { 5 | def query = "SELECT COUNT(1) FROM ${AuditStorageUtils.getAuditTableName(type.code)}" 6 | def result = jdbcTemplate.queryForObject(query, Long.class) 7 | return result ? result : 0L 8 | } catch (ex) { 9 | println(ex.message) 10 | } 11 | return 0L 12 | } 13 | 14 | def superWithDeployment(type) { 15 | def walk = type.superType 16 | def ancestor = null 17 | while (walk.table == type.table) { 18 | ancestor = walk 19 | walk = walk.superType 20 | } 21 | return ancestor ?: type 22 | } 23 | 24 | def ALLOWED_AUDIT = [ 25 | "user", 26 | "address" 27 | ].collect{ it.toLowerCase() } as Set 28 | 29 | def typeService = spring.getBean("typeService") 30 | 31 | def auditEnabled = auditEnablementService.isAuditEnabledGlobally() 32 | 33 | def genericItem = typeService.getComposedTypeForCode("GenericItem") 34 | 35 | def itemTypes = genericItem.allSubTypes.sort{ it.code }.groupBy { superWithDeployment(it) } 36 | itemTypes = itemTypes.sort{ AuditStorageUtils.getAuditTableName(it.key.code) } 37 | itemTypes.each{ k, v -> 38 | def numAudit = auditCount(k) 39 | if (numAudit > 0 && (!auditEnabled || !(k.code.toLowerCase() in ALLOWED_AUDIT ))) { 40 | println("Audit table ${AuditStorageUtils.getAuditTableName(k.code)} | Entries: ${numAudit}") 41 | println("Affected Types:") 42 | v.each{ 43 | println("\t${it.code}${auditEnablementService.isAuditEnabledForType(it.code) ? " - Audit Enabled" : ""}") 44 | } 45 | println() 46 | } 47 | } 48 | 49 | println "**************************************************" 50 | println "* :Disclaimer: *" 51 | println "* Please align any changes to the audit settings *" 52 | println "* with your data protection requirements. *" 53 | println "**************************************************" 54 | println "If unnecessary audit tables detected:" 55 | println "- check if audit is enabled for any of the affected types" 56 | println " (property audit..enabled=true) and disable as necessary" 57 | println "- delete audit data from the audit table(s)" 58 | 59 | 60 | return "" -------------------------------------------------------------------------------- /resources/impex/bulkdelete-cronjoblogs.impex: -------------------------------------------------------------------------------- 1 | INSERT_UPDATE Script; code[unique=true];active[unique=true];content 2 | ;removeCronjobLogs;true;" 3 | import de.hybris.platform.servicelayer.impex.ImportConfig 4 | import de.hybris.platform.servicelayer.impex.impl.StreamBasedImpExResource 5 | import de.hybris.platform.servicelayer.cronjob.PerformResult 6 | import de.hybris.platform.cronjob.enums.CronJobStatus 7 | import de.hybris.platform.cronjob.enums.CronJobResult 8 | 9 | // all log files EXCEPT the five most recent logs per cronjob 10 | // warning: query uses DBMS-specific dialact for partioning the logs per cronjob 11 | // query was tested on HANA and MS SQL 12 | def QUERY = ''' 13 | SELECT t.pk 14 | FROM 15 | (SELECT m.pk, 16 | m.OwnerPKString, 17 | row_number() OVER (PARTITION BY m.OwnerPKString 18 | ORDER BY m.createdTS DESC) AS rn 19 | FROM medias m 20 | JOIN composedtypes t ON m.typepkstring = t.pk 21 | AND t.internalcode = 'LogFile' ) t 22 | WHERE t.rn > 5 OR t.OwnerPKString IS NULL 23 | ''' 24 | 25 | def IMPEX_HEADER = 'REMOVE LogFile;pk[unique=true]\n' 26 | 27 | def logPKs = jdbcTemplate.queryForList(QUERY) 28 | 29 | if (logPKs) { 30 | def impexScript = IMPEX_HEADER 31 | log.info ""Number of logs to delete: ${logPKs.size}"" 32 | logPKs.each { 33 | impexScript += "";${it.pk}\n"" 34 | } 35 | 36 | def impexResource = new StreamBasedImpExResource(new ByteArrayInputStream(impexScript.getBytes('UTF-8')), 'UTF-8') 37 | 38 | def importConfig = new ImportConfig() 39 | importConfig.synchronous = true 40 | importConfig.sldForData = true 41 | importConfig.removeOnSuccess = true 42 | importConfig.script = impexResource 43 | 44 | def importResult = importService.importData(importConfig) 45 | log.info ""Deleted cronjob log files. ${importResult.successful ? 'SUCCESS' : 'FAILED'}"" 46 | if (!importResult.successful) { 47 | return new PerformResult(CronJobResult.ERROR, CronJobStatus.FINISHED) 48 | } 49 | } else { 50 | log.info 'Nothing to delete' 51 | } 52 | return new PerformResult(CronJobResult.SUCCESS, CronJobStatus.FINISHED) 53 | " 54 | 55 | 56 | INSERT_UPDATE ScriptingJob; code[unique=true];scriptURI 57 | ;logFileBulkCleanupJob;model://removeCronjobLogs 58 | 59 | INSERT_UPDATE CronJob; code[unique=true];job(code);sessionLanguage(isocode) 60 | ;logFileBulkCleanupCronjob;logFileBulkCleanupJob;en 61 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/processing/002-cleanup-cronjobs.impex: -------------------------------------------------------------------------------- 1 | # Delete all generated jobs without a trigger 2 | 3 | # Import config properties into impex macros 4 | UPDATE GenericItem[processor = de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor]; pk[unique = true] 5 | $sessionLanguage = $config-sanecleanup.jobs.sessionlanguage 6 | $retentionTimeCronjobs=$config-sanecleanup.retentiontimeseconds.cronjob.generated 7 | $retentionTimeSolrIndexerJob =$config-sanecleanup.retentiontimeseconds.solrindexerjob.generated 8 | 9 | # @readme CronJob (auto-generated) 10 | # Are there too many (>10) outdated, auto-geneated jobs in your system? 11 | # SELECT 12 | # {t:code} AS "CronJob Type", 13 | # COUNT({c:pk}) AS "total", 14 | # MIN({c:modifiedtime}) AS "oldest", 15 | # MAX({c:modifiedtime}) AS "newest" 16 | # FROM 17 | # {CronJob AS c 18 | # JOIN 19 | # ComposedType AS t 20 | # ON {c:itemtype} = {t:pk} 21 | # LEFT JOIN 22 | # TRIGGER AS trg 23 | # ON {trg:cronjob} = {c:pk} } 24 | # WHERE 25 | # {trg:pk} IS NULL 26 | # AND {c:code} LIKE '00%' 27 | # AND {t:code} IN 28 | # ( 29 | # 'ImpExImportCronJob', 30 | # 'CatalogVersionSyncCronJob', 31 | # 'SolrIndexerCronJob' 32 | # ) 33 | # GROUP BY 34 | # {t:code} 35 | # ORDER BY 36 | # "total" DESC 37 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 38 | ; cronJobRule ; " 39 | SELECT {c:pk}, {c:itemType} 40 | FROM {CronJob AS c JOIN ComposedType AS t ON {c:itemtype} = {t:pk} LEFT JOIN Trigger AS trg ON {trg:cronjob} = {c:pk} } 41 | WHERE {trg:pk} IS NULL 42 | AND {c:code} LIKE '00%' 43 | AND {t:code} IN ( 'ImpExImportCronJob', 'CatalogVersionSyncCronJob', 'SolrIndexerCronJob' ) 44 | AND {c:endTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeCronjobs ; basicRemoveCleanupAction ; 45 | ; solrJobRule ; " 46 | SELECT {j:pk},{j:itemType} 47 | FROM {ServicelayerJob AS j LEFT JOIN Trigger AS trg on {trg:job} = {j:pk} } 48 | WHERE {trg:pk} IS NULL 49 | AND ({j:code} LIKE 'solrIndexerJob_full_%' OR {j:code} LIKE 'solrIndexerJob_update_%') 50 | AND {j:modifiedtime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeSolrIndexerJob ; basicRemoveCleanupAction ; 51 | INSERT_UPDATE RetentionJob; code[unique = true]; retentionRule(code); batchSize 52 | ; cronJobCleanupJob ; cronJobRule ; 1000 53 | ; solrJobCleanupJob ; solrJobRule ; 1000 54 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 55 | ; cronJobCleanupCronJob ; cronJobCleanupJob ; 56 | ; solrJobCleanupCronJob ; solrJobCleanupJob ; 57 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 58 | # every day at 04:00 59 | ; cronJobCleanupCronJob ; 0 0 4 * * ? 60 | ; solrJobCleanupCronJob ; 0 0 4 * * ? 61 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/core/002-cleanup-savedvalueentries.impex: -------------------------------------------------------------------------------- 1 | # delete all stored sessions one day after there last update 2 | 3 | # Import config properties into impex macros 4 | UPDATE GenericItem[processor = de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor]; pk[unique = true] 5 | $sessionLanguage = $config-sanecleanup.jobs.sessionlanguage 6 | $retentionTime=$config-sanecleanup.retentiontimeseconds.savedvalues.delete 7 | 8 | # @readme SavedValues, SavedValueEntry 9 | # A lot of those items accumulated over the project lifetime. 10 | # If possible, disable storing saved values. (`hmc.storing.modifiedvalues.size=0`) 11 | # -- total SavedValue / SavedValueEntry 12 | # SELECT 13 | # * 14 | # FROM 15 | # ( 16 | # {{ 17 | # SELECT 18 | # 'SavedValues' AS "type", 19 | # COUNT({s:pk}) AS "total" 20 | # FROM 21 | # {savedvalues AS s} }} 22 | # UNION ALL 23 | # {{ 24 | # SELECT 25 | # 'SavedValueEntry' AS "type", 26 | # COUNT({e:pk}) AS "total" 27 | # FROM 28 | # {savedvalueentry AS e} }} 29 | # ) 30 | # summary 31 | # -- SavedValues per item 32 | # SELECT 33 | # {s:modifiedItem} AS "item", 34 | # COUNT({s:pk}) AS "total", 35 | # MIN({s:modifiedtime}) AS "oldest", 36 | # MAX({s:modifiedtime}) AS "newest" 37 | # FROM 38 | # {SavedValues AS s } 39 | # GROUP BY 40 | # {s:modifiedItem} 41 | # ORDER BY 42 | # "total" DESC 43 | # -- orphaned SavedValueEntry 44 | # -- (there shouldn't be any) 45 | # SELECT 46 | # COUNT({e:pk}) AS "total", 47 | # MIN({e:modifiedtime}) AS "oldest", 48 | # MAX({e:modifiedtime}) AS "newest" 49 | # FROM {SavedValueEntry as e LEFT JOIN SavedValues AS s ON {e:parent} = {s:pk} } 50 | # WHERE {s:pk} IS NULL 51 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 52 | ; savedValuesDeleteRule ; "select {s:pk}, {s:itemtype} 53 | from {SavedValues as s } 54 | where {s.modifiedItem} IS NULL AND {s.modifiedtime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ; 55 | ; savedValueEntryRule ; "select {e:pk}, {e:itemtype} 56 | from {SavedValueEntry as e LEFT JOIN SavedValues AS s ON {e:parent} = {s:pk} } 57 | where {s:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 58 | 59 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 60 | ; savedValuesDeleteCleanupJob ; savedValuesDeleteRule ; 1000 61 | ; savedValueEntryCleanupJob ; savedValueEntryRule ; 1000 62 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 63 | ; savedValuesDeleteCleanupCronJob ; savedValuesDeleteCleanupJob ; 64 | ; savedValueEntryCleanupCronJob ; savedValueEntryCleanupJob ; 65 | 66 | 67 | INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage] 68 | ;savedValuesCleanupCompositeCronJob;compositeJobPerformable; 69 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 70 | # at midnight 71 | ; savedValuesCleanupCompositeCronJob ; 0 0 0 * * ? 72 | 73 | INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='savedValuesCleanupCompositeCronJob'] 74 | ;savedValuesDeleteCleanupCronJobEntry;savedValuesDeleteCleanupCronJob; 75 | ;savedValueEntryCleanupCronJobEntry;savedValueEntryCleanupCronJob; 76 | 77 | UPDATE CompositeCronJob;code[unique=true];compositeEntries(code) 78 | ;savedValuesCleanupCompositeCronJob;" 79 | savedValuesDeleteCleanupCronJobEntry, 80 | savedValueEntryCleanupCronJobEntry 81 | " 82 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/commerceservices/001-cleanup-oldcarts.impex: -------------------------------------------------------------------------------- 1 | # based on de.hybris.platform.commercewebservices.core.cronjob.OldCartRemovalJob 2 | # and de.hybris.platform.commerceservices.order.dao.impl.DefaultCommerceCartDao.getCartsForRemovalForSiteAndUser 3 | # 4 | # - cleanup anonymous carts after two weeks for *every* site (excluding saved carts) 5 | # - cleanup carts of registered users after four weeks for *every* site (excluding saved carts) 6 | 7 | # Import config properties into impex macros 8 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 9 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 10 | $retentionTimeRegular=$config-sanecleanup.retentiontimeseconds.cart.regular 11 | $retentionTimeAnonymous =$config-sanecleanup.retentiontimeseconds.cart.anonymous 12 | 13 | # @readme Cart 14 | # - Are there excessive amount of carts per site or per user? 15 | # - Too many saved carts? 16 | # - Stale (= old) carts? 17 | # SELECT 18 | # {b:uid} AS "BaseSite", 19 | # {u:uid} AS "User", 20 | # CASE 21 | # WHEN 22 | # {c:saveTime} IS NULL 23 | # THEN 24 | # 'regular' 25 | # ELSE 26 | # 'saved' 27 | # END 28 | # AS "cart type", 29 | # COUNT({c:pk}) AS "total", 30 | # MIN({c:modifiedtime}) AS "oldest", 31 | # MAX({c:modifiedtime}) AS "newest" 32 | # FROM 33 | # { Cart AS c 34 | # LEFT JOIN 35 | # USER AS u 36 | # ON {c:user} = {u:pk} 37 | # LEFT JOIN 38 | # BaseSite AS b 39 | # ON {c:site} = {b:pk} } 40 | # GROUP BY 41 | # {b:uid}, {u:uid}, 42 | # CASE 43 | # WHEN 44 | # {c:saveTime} IS NULL 45 | # THEN 46 | # 'regular' 47 | # ELSE 48 | # 'saved' 49 | # END 50 | # ORDER BY 51 | # "total" DESC 52 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 53 | ; cartRule ; " 54 | SELECT {c:pk}, {c:itemtype} 55 | FROM { Cart AS c LEFT JOIN User AS u ON {c:user} = {u:pk} } 56 | WHERE {c:saveTime} IS NULL 57 | AND {u:uid} <> 'anonymous' 58 | AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeRegular ; basicRemoveCleanupAction ; 59 | ; anonymousCartRule ; " 60 | SELECT {c:pk}, {c:itemtype} 61 | FROM {Cart AS c LEFT JOIN User AS u ON {c:user} = {u:pk}} 62 | WHERE {c:saveTime} IS NULL 63 | AND ( {u:uid} = 'anonymous' OR {u:uid} IS NULL ) 64 | AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeAnonymous ; basicRemoveCleanupAction ; 65 | 66 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 67 | ; cartCleanupJob ; cartRule ; 1000 68 | ; anonymousCartCleanupJob ; anonymousCartRule ; 1000 69 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 70 | ; cartCleanupCronJob ; cartCleanupJob ; 71 | ; anonymousCartCleanupCronJob ; anonymousCartCleanupJob ; 72 | 73 | ### 74 | 75 | INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage] 76 | ;cartCleanupCompositeCronJob;compositeJobPerformable; 77 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 78 | # every day at 03:00 79 | ; cartCleanupCompositeCronJob ; 0 0 3 * * ? 80 | 81 | INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='cartCleanupCompositeCronJob'] 82 | ;cartCleanupCronJobEntry;cartCleanupCronJob; 83 | ;anonymousCartCleanupCronJobEntry;anonymousCartCleanupCronJob; 84 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/processing/001-cleanup-cronjobhistory.impex: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. 3 | # --------------------------------------------------------------------------- 4 | # ref. https://launchpad.support.sap.com/#/notes/2848601 5 | 6 | # Import config properties into impex macros 7 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 8 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 9 | 10 | # @readme CronJobHistory 11 | # Is there any job with > 50 histories and/or histories older than an hour? 12 | # 13 | # This cleanup is enabled by default in recent SAP Commerce patch releases, so this query shouldn't find anything. 14 | # SELECT 15 | # {cj:code}, 16 | # COUNT({h:pk}) AS "total", 17 | # MIN({h:modifiedtime}) AS "oldest", 18 | # MAX({h:modifiedtime}) AS "newest" 19 | # FROM 20 | # {cronjobhistory AS h 21 | # JOIN 22 | # cronjob AS cj 23 | # ON {h:cronjob} = {cj:pk} } 24 | # GROUP BY 25 | # {cj:code} 26 | # ORDER BY 27 | # "total" DESC 28 | INSERT_UPDATE FlexibleSearchRetentionRule;code[unique=true];searchQuery;actionReference; 29 | "#% beforeEach: 30 | import de.hybris.platform.core.Registry; 31 | import de.hybris.platform.cronjob.model.CronJobModel; 32 | CronJobModel cronJob; 33 | try 34 | { 35 | cronJob = Registry.getApplicationContext().getBean(""cronJobService"").getCronJob(""cronJobHistoryRetentionCronJob""); 36 | } 37 | catch (Exception e) 38 | { 39 | cronJob = null; 40 | } 41 | if (cronJob != null) 42 | { 43 | line.clear(); 44 | }" 45 | ; cronJobHistoryRetentionRule; SELECT {h1:PK}, {h1:itemtype} FROM {CronJobHistory as h1} WHERE {h1:creationtime} < (SELECT max FROM ({{SELECT max({h2:creationtime}) as max, {h2:cronjob} as cronjob FROM {CronJobHistory as h2} GROUP BY {h2:cronjob} }}) temptable where cronjob = {h1:cronjob}); basicRemoveCleanupAction; 46 | 47 | # JOB 48 | INSERT_UPDATE RetentionJob; code[unique=true]; retentionRule(code); batchSize 49 | "#% beforeEach: 50 | import de.hybris.platform.core.Registry; 51 | import de.hybris.platform.cronjob.model.CronJobModel; 52 | CronJobModel cronJob; 53 | try 54 | { 55 | cronJob = Registry.getApplicationContext().getBean(""cronJobService"").getCronJob(""cronJobHistoryRetentionCronJob""); 56 | } 57 | catch (Exception e) 58 | { 59 | cronJob = null; 60 | } 61 | if (cronJob != null) 62 | { 63 | line.clear(); 64 | }" 65 | ; cronJobHistoryRetentionJob; cronJobHistoryRetentionRule; 1000 66 | 67 | # CRON JOB 68 | INSERT_UPDATE CronJob;code[unique=true]; job(code); sessionLanguage(isoCode)[default=$sessionLanguage] 69 | "#% beforeEach: 70 | import de.hybris.platform.core.Registry; 71 | import de.hybris.platform.cronjob.model.CronJobModel; 72 | CronJobModel cronJob; 73 | try 74 | { 75 | cronJob = Registry.getApplicationContext().getBean(""cronJobService"").getCronJob(""cronJobHistoryRetentionCronJob""); 76 | } 77 | catch (Exception e) 78 | { 79 | cronJob = null; 80 | } 81 | if (cronJob != null) 82 | { 83 | line.clear(); 84 | }" 85 | ; cronJobHistoryRetentionCronJob; cronJobHistoryRetentionJob; 86 | 87 | INSERT Trigger; cronJob(code)[unique=true]; cronExpression 88 | # every hour 89 | "#% beforeEach: 90 | import de.hybris.platform.core.Registry; 91 | import de.hybris.platform.cronjob.model.CronJobModel; 92 | CronJobModel cronJob; 93 | try 94 | { 95 | cronJob = Registry.getApplicationContext().getBean(""cronJobService"").getCronJob(""cronJobHistoryRetentionCronJob""); 96 | } 97 | catch (Exception e) 98 | { 99 | cronJob = null; 100 | } 101 | if (cronJob != null) 102 | { 103 | line.clear(); 104 | }" 105 | ;cronJobHistoryRetentionCronJob; 0 0 0/1 * * ? ; 106 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/processing/004-cleanup-processtasklog.impex: -------------------------------------------------------------------------------- 1 | # Import config properties into impex macros 2 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 3 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 4 | 5 | # @readme ProcessTaskLog 6 | # We recommend customer to BusinessProcess cleanup, which will eventually take care of TaskLogs cleanup. 7 | # There might be the few scenarios for ProcessTaskLog cleanup: 8 | # 1. The customer wants to keep the BusinessProcess for reporting, although we don't recommend it. 9 | # 1. The customer might be using the custom task without any business process. 10 | # -- Query tested with MS SQL 11 | # -- Adjust the date calculation for 12 | # -- other database types 13 | # SELECT 14 | # COUNT({l:pk}) AS "total", 15 | # MIN({l:modifiedtime}) AS "oldest", 16 | # MAX({l:modifiedtime}) AS "newest" 17 | # FROM 18 | # {ProcessTaskLog AS l} 19 | # WHERE 20 | # {l:creationTime} < DATEADD( 21 | # month, 22 | # -2, 23 | # GETUTCDATE() 24 | # ) 25 | INSERT_UPDATE Script; code[unique=true]; description[default = $sessionLanguage]; content; active[default=true] 26 | ;directCleanupProcessTaskLogScript;"Delete ProcessTaskLog" ;" 27 | import de.hybris.platform.servicelayer.search.FlexibleSearchQuery 28 | import de.hybris.platform.util.persistence.PersistenceUtils 29 | 30 | import java.time.ZonedDateTime 31 | import java.time.Instant 32 | import java.time.ZoneId 33 | import java.time.temporal.ChronoUnit 34 | 35 | def fss = spring.getBean('flexibleSearchService') 36 | def ms = spring.getBean('modelService') 37 | 38 | PersistenceUtils.doWithSLDPersistence({ -> 39 | // TODO: adjust retention period 40 | def retention = Date.from(ZonedDateTime.now().minus(3, ChronoUnit.MONTHS).toInstant()) 41 | def processTaskLogQuery = new FlexibleSearchQuery('SELECT {pk} FROM {ProcessTaskLog} WHERE {modifiedTime} < ?CALC_RETENTION_TIME'); 42 | processTaskLogQuery.addQueryParameter('CALC_RETENTION_TIME', retention) 43 | processTaskLogQuery.setCount(10000); // TODO: adjust total count 44 | def processTaskLogs = fss.search(processTaskLogQuery).result 45 | // out.println('ProcessTaskLog count: ' + processTaskLogs.size()) 46 | if (processTaskLogs.size() > 0) { 47 | // out.println('Deleting ProcessTaskLog: ' + processTaskLogs.size()) 48 | def counter 49 | try { // try bulk remove 50 | ms.removeAll(processTaskLogs) 51 | counter = processTaskLogs.size() 52 | processTaskLogs.each { 53 | ms.detach(it) 54 | } 55 | 56 | } catch (Exception bulkEx) { // if exception do individual remove 57 | // out.println('Exception: ' + bulkEx.getMessage()) 58 | counter = 0 59 | for (taskLog in processTaskLogs) { 60 | try { 61 | ms.remove(taskLog) 62 | ms.detach(taskLog) 63 | counter++ 64 | // out.println(counter + ' | Deleted - ProcessTaskLog: ' + taskLog?.pk + ' - ' + taskLog?.actionId + ' - ' + taskLog?.process) 65 | } catch (Exception indEx) { 66 | // out.println('Exception: ' + indEx.getMessage()) 67 | } 68 | } 69 | } 70 | // out.println('Deleted ProcessTaskLog: ' + counter) 71 | // processTaskLogQuery.setCount(-1); 72 | // processTaskLogs = fss.search(processTaskLogQuery).result 73 | // return [deleted: counter, remaining: processTaskLogs.size()] 74 | return [deleted: counter] 75 | 76 | } else { 77 | // out.println('NO ProcessTaskLog to delete.') 78 | return [deleted: 0, remaining: 0] 79 | } 80 | }); 81 | "; 82 | 83 | INSERT_UPDATE ScriptingJob;code[unique = true];scriptURI 84 | ;directCleanupProcessTaskLogScriptJob;"model://directCleanupProcessTaskLogScript" 85 | 86 | INSERT_UPDATE CronJob;code[unique=true]; job(code); sessionLanguage(isoCode)[default = $sessionLanguage]; 87 | ;directCleanupProcessTaskLogCronjob;directCleanupProcessTaskLogScriptJob ; 88 | 89 | # every 40 minutes 90 | INSERT Trigger; cronjob(code)[unique = true]; cronExpression; active 91 | ;directCleanupProcessTaskLogCronjob ;0 0/40 * ? * * * ;true -------------------------------------------------------------------------------- /resources/impex/sanecleanup/acceleratorservices/001-cleanup-emails.impex: -------------------------------------------------------------------------------- 1 | # Import config properties into impex macros 2 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 3 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 4 | $retentionTime = $config-sanecleanup.retentiontimeseconds.emailmessage 5 | 6 | # @readme EmailMessage 7 | # - Are there more than a handful sent/unsent messages? 8 | # - Are there messages that do not belong to any process? 9 | # SELECT 10 | # {bp:processDefinitionName} AS "source", 11 | # {m:sent}, 12 | # COUNT({m:pk}) AS "total", 13 | # MIN({m:modifiedtime}) AS "oldest", 14 | # MAX({m:modifiedtime}) AS "newest" 15 | # FROM 16 | # {EmailMessage AS m 17 | # LEFT JOIN 18 | # BusinessProcess AS bp 19 | # ON {m:process} = {bp:pk} } 20 | # GROUP BY 21 | # {bp:processDefinitionName}, {m:sent} 22 | # ORDER BY 23 | # "total" DESC 24 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 25 | ; emailMessageRule ; " 26 | SELECT {m:pk}, {m:itemtype} 27 | FROM {EmailMessage AS m LEFT JOIN BusinessProcess AS bp ON {m:process} = {bp:pk}} 28 | WHERE {bp:pk} IS NULL 29 | AND {m:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ; 30 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 31 | ; emailMessageCleanupJob ; emailMessageRule ; 1000 32 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 33 | ; emailMessageCleanupCronJob ; emailMessageCleanupJob ; 34 | 35 | 36 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 37 | ; emailAddressRule ; " 38 | SELECT DISTINCT {a:pk}, {a:itemType} from { 39 | EmailAddress AS a 40 | LEFT JOIN EmailMessage2ToAddressesRel AS to ON {to:target} = {a:pk} 41 | LEFT JOIN EmailMessage2CcAddressesRel AS cc ON {cc:target} = {a:pk} 42 | LEFT JOIN EmailMessage2BccAddressesRel AS bcc ON {bcc:target} = {a:pk} 43 | LEFT JOIN EmailMessage AS m ON {m:fromAddress} = {a:pk} 44 | } 45 | WHERE {to:source} IS NULL 46 | AND {cc:source} IS NULL 47 | AND {bcc:source} IS NULL 48 | AND {m:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 49 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 50 | ; emailAddressCleanupJob ; emailAddressRule ; 1000 51 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 52 | ; emailAddressCleanupCronJob ; emailAddressCleanupJob ; 53 | 54 | 55 | 56 | # EmailAttachment 57 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 58 | ; emailAttachmentRule ; " 59 | SELECT {a:pk}, {a:itemType} from { 60 | EmailAttachment AS a 61 | LEFT JOIN EmailMessage AS m ON {a:message} = {m:pk} 62 | } 63 | WHERE {m:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 64 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 65 | ; emailAttachmentCleanupJob ; emailAttachmentRule ; 1000 66 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 67 | ; emailAttachmentCleanupCronJob ; emailAttachmentCleanupJob ; 68 | 69 | 70 | ### 71 | 72 | INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage] 73 | ;emailRetentionCompositeCronJob;compositeJobPerformable; 74 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 75 | # every day at 02:00 76 | ; emailRetentionCompositeCronJob ; 0 0 2 * * ? 77 | 78 | INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='emailRetentionCompositeCronJob'] 79 | ;emailMessageCleanupCronJobEntry;emailMessageCleanupCronJob; 80 | ;emailAddressCleanupJobEntry;emailAddressCleanupCronJob; 81 | ;emailAttachmentCleanupCronJobEntry;emailAttachmentCleanupCronJob; 82 | 83 | # ensure correct order of execution 84 | UPDATE CompositeCronJob;code[unique=true];compositeEntries(code) 85 | ;emailRetentionCompositeCronJob;"emailMessageCleanupCronJobEntry, 86 | emailAddressCleanupJobEntry, 87 | emailAttachmentCleanupCronJobEntry 88 | " -------------------------------------------------------------------------------- /src/mpern/sap/commerce/build/util/Version.java: -------------------------------------------------------------------------------- 1 | package mpern.sap.commerce.build.util; 2 | 3 | import java.util.Comparator; 4 | import java.util.Objects; 5 | import java.util.regex.Matcher; 6 | import java.util.regex.Pattern; 7 | 8 | public class Version implements Comparable { 9 | private static final Pattern NEW_VERSION = Pattern.compile("(\\d\\d)(\\d\\d)(\\.([1-9]?\\d))?.*"); 10 | private static final Pattern OLD_VERSION = Pattern.compile("(\\d)\\.(\\d)\\.(\\d)(\\.([1-9]?\\d))?"); 11 | public static final int UNDEFINED_PART = Integer.MAX_VALUE; 12 | public static final Version UNDEFINED = new Version(UNDEFINED_PART, UNDEFINED_PART, UNDEFINED_PART, UNDEFINED_PART, 13 | ""); 14 | public static final Comparator VERSION_COMPARATOR = Comparator.comparingInt(Version::getMajor) 15 | .thenComparingInt(Version::getMinor).thenComparingInt(Version::getRelease) 16 | .thenComparingInt(Version::getPatch); 17 | 18 | private final int major; 19 | private final int minor; 20 | private final int release; 21 | private final int patch; 22 | private final String original; 23 | 24 | Version(int major, int minor, int release, int patch, String original) { 25 | this.major = major; 26 | this.minor = minor; 27 | this.release = release; 28 | this.patch = patch; 29 | this.original = original; 30 | } 31 | 32 | public static Version parseVersion(String v) { 33 | Objects.requireNonNull(v); 34 | 35 | Matcher oldV = OLD_VERSION.matcher(v); 36 | Matcher newV = NEW_VERSION.matcher(v); 37 | 38 | if (newV.matches()) { 39 | int patch = UNDEFINED_PART; 40 | 41 | if (newV.groupCount() > 3 && newV.group(4) != null) { 42 | patch = Integer.parseInt(newV.group(4)); 43 | } 44 | return new Version(Integer.parseInt(newV.group(1)), Integer.parseInt(newV.group(2)), 0, patch, v); 45 | } else if (oldV.matches()) { 46 | int patch = UNDEFINED_PART; 47 | if (oldV.groupCount() > 4 && oldV.group(5) != null) { 48 | patch = Integer.parseInt(oldV.group(5)); 49 | } 50 | return new Version(Integer.parseInt(oldV.group(1)), Integer.parseInt(oldV.group(2)), 51 | Integer.parseInt(oldV.group(3)), patch, v); 52 | } 53 | String[] split = v.split("\\."); 54 | int major = UNDEFINED_PART, minor = UNDEFINED_PART, release = UNDEFINED_PART, patch = UNDEFINED_PART; 55 | switch (split.length) { 56 | case 4: 57 | patch = Integer.parseInt(split[3]); 58 | case 3: 59 | release = Integer.parseInt(split[2]); 60 | case 2: 61 | minor = Integer.parseInt(split[1]); 62 | case 1: 63 | major = Integer.parseInt(split[0]); 64 | break; 65 | default: 66 | throw new IllegalArgumentException("Could not parse " + v); 67 | } 68 | return new Version(major, minor, release, patch, v); 69 | } 70 | 71 | public Version withoutPatch() { 72 | return new Version(major, minor, release, UNDEFINED_PART, original); 73 | } 74 | 75 | @Override 76 | public int compareTo(Version o) { 77 | return VERSION_COMPARATOR.compare(this, o); 78 | } 79 | 80 | public boolean equalsIgnorePatch(Version o) { 81 | if (this == o) 82 | return true; 83 | if (o == null || getClass() != o.getClass()) 84 | return false; 85 | Version version = (Version) o; 86 | return major == version.major && minor == version.minor && release == version.release; 87 | } 88 | 89 | @Override 90 | public boolean equals(Object o) { 91 | if (this == o) 92 | return true; 93 | if (o == null || getClass() != o.getClass()) 94 | return false; 95 | Version version = (Version) o; 96 | return major == version.major && minor == version.minor && release == version.release && patch == version.patch; 97 | } 98 | 99 | @Override 100 | public int hashCode() { 101 | return Objects.hash(major, minor, release, patch); 102 | } 103 | 104 | @Override 105 | public String toString() { 106 | return original; 107 | } 108 | 109 | public int getMajor() { 110 | return major; 111 | } 112 | 113 | public int getMinor() { 114 | return minor; 115 | } 116 | 117 | public int getRelease() { 118 | return release; 119 | } 120 | 121 | public int getPatch() { 122 | return patch; 123 | } 124 | 125 | public String getDependencyVersion() { 126 | String v = this.original; 127 | if (this.getPatch() == UNDEFINED_PART) { 128 | if (!v.endsWith(".")) { 129 | v += "."; 130 | } 131 | v += "+"; 132 | } 133 | return v; 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /.settings/org.springframework.ide.eclipse.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.springframework.ide.eclipse.core.builders.enable.aopreferencemodelbuilder=true 3 | org.springframework.ide.eclipse.core.builders.enable.beanmetadatabuilder=true 4 | org.springframework.ide.eclipse.core.enable.project.preferences=false 5 | org.springframework.ide.eclipse.core.validator.enable.org.springframework.ide.eclipse.beans.core.beansvalidator=true 6 | org.springframework.ide.eclipse.core.validator.enable.org.springframework.ide.eclipse.core.springvalidator=false 7 | org.springframework.ide.eclipse.core.validator.enable.org.springframework.ide.eclipse.webflow.core.validator=true 8 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanAlias-org.springframework.ide.eclipse.beans.core.beansvalidator=true 9 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanClass-org.springframework.ide.eclipse.beans.core.beansvalidator=true 10 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanConstructorArgument-org.springframework.ide.eclipse.beans.core.beansvalidator=true 11 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanDefinition-org.springframework.ide.eclipse.beans.core.beansvalidator=true 12 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanDefinitionHolder-org.springframework.ide.eclipse.beans.core.beansvalidator=true 13 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanFactory-org.springframework.ide.eclipse.beans.core.beansvalidator=true 14 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanInitDestroyMethod-org.springframework.ide.eclipse.beans.core.beansvalidator=true 15 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanProperty-org.springframework.ide.eclipse.beans.core.beansvalidator=true 16 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.beanReference-org.springframework.ide.eclipse.beans.core.beansvalidator=true 17 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.methodOverride-org.springframework.ide.eclipse.beans.core.beansvalidator=true 18 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.parsingProblems-org.springframework.ide.eclipse.beans.core.beansvalidator=true 19 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.beans.core.requiredProperty-org.springframework.ide.eclipse.beans.core.beansvalidator=false 20 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.core.springClasspath-org.springframework.ide.eclipse.core.springvalidator=false 21 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.action-org.springframework.ide.eclipse.webflow.core.validator=true 22 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.actionstate-org.springframework.ide.eclipse.webflow.core.validator=true 23 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.attribute-org.springframework.ide.eclipse.webflow.core.validator=true 24 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.attributemapper-org.springframework.ide.eclipse.webflow.core.validator=true 25 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.beanaction-org.springframework.ide.eclipse.webflow.core.validator=true 26 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.evaluationaction-org.springframework.ide.eclipse.webflow.core.validator=true 27 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.evaluationresult-org.springframework.ide.eclipse.webflow.core.validator=true 28 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.exceptionhandler-org.springframework.ide.eclipse.webflow.core.validator=true 29 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.import-org.springframework.ide.eclipse.webflow.core.validator=true 30 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.inputattribute-org.springframework.ide.eclipse.webflow.core.validator=true 31 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.mapping-org.springframework.ide.eclipse.webflow.core.validator=true 32 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.outputattribute-org.springframework.ide.eclipse.webflow.core.validator=true 33 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.set-org.springframework.ide.eclipse.webflow.core.validator=true 34 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.state-org.springframework.ide.eclipse.webflow.core.validator=true 35 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.subflowstate-org.springframework.ide.eclipse.webflow.core.validator=true 36 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.transition-org.springframework.ide.eclipse.webflow.core.validator=true 37 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.variable-org.springframework.ide.eclipse.webflow.core.validator=true 38 | org.springframework.ide.eclipse.core.validator.rule.enable.org.springframework.ide.eclipse.webflow.core.validation.webflowstate-org.springframework.ide.eclipse.webflow.core.validator=true 39 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/impex/002-cleanup-distributed-impex.impex: -------------------------------------------------------------------------------- 1 | # Import config properties into impex macros 2 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 3 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 4 | $retentionTime = $config-sanecleanup.retentiontimeseconds.impeximportcronjob.distributed 5 | 6 | # @readme ImpExImportCronJob (distributed impex) 7 | # - More than ~10 `FINISHED` distributed impex jobs? 8 | # - More than a few `PAUSED` jobs? You may have a faulty distributed impex script. 9 | # SELECT 10 | # {s:code} AS "status", 11 | # COUNT({i:pk}) AS "total", 12 | # MIN({i:modifiedtime}) AS "oldest", 13 | # MAX({i:modifiedtime}) AS "newest" 14 | # FROM 15 | # {ImpExImportCronJob AS i 16 | # LEFT JOIN 17 | # CronJobStatus AS s 18 | # ON {i:status} = {s:pk} } 19 | # WHERE 20 | # {i:code} LIKE 'distributed-impex-%' 21 | # GROUP BY 22 | # {s:code} 23 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 24 | ; distributedImpexCronJobRule ; " 25 | SELECT {i:pk}, {i:itemType} 26 | FROM {ImpExImportCronJob AS i} 27 | WHERE {i:code} LIKE 'distributed-impex-%' 28 | AND {i:modifiedtime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ; 29 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 30 | ; distributedImpexCronJobCleanupJob ; distributedImpexCronJobRule ; 1000 31 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 32 | ; distributedImpexCronJobCleanupCronJob ; distributedImpexCronJobCleanupJob ; 33 | 34 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 35 | ; distributedImpexJobRule ; " 36 | SELECT {j:pk}, {j:itemtype} 37 | FROM {ImpExImportJob AS j LEFT JOIN ImpExImportCronJob as cj on {cj:job} = {j:pk} } 38 | WHERE {j:code} LIKE 'distributed-impex-%' 39 | AND {cj:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 40 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 41 | ; distributedImpexJobCleanupJob ; distributedImpexJobRule ; 1000 42 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 43 | ; distributedImpexJobCleanupCronJob ; distributedImpexJobCleanupJob ; 44 | 45 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 46 | ; distributedImportProcessRule ; " 47 | SELECT {p:pk}, {p:itemtype} 48 | FROM {DistributedImportProcess AS p LEFT JOIN ImpExImportCronJob as cj on {p:impExImportCronJob} = {cj:pk} } 49 | WHERE {cj:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 50 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 51 | ; distributedImportProcessCleanupJob ; distributedImportProcessRule ; 1000 52 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 53 | ; distributedImportProcessCleanupCronJob ; distributedImportProcessCleanupJob ; 54 | 55 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 56 | ; importBatchRule ; " 57 | SELECT {b:pk}, {b:itemtype} 58 | FROM {ImportBatch AS b LEFT JOIN DistributedImportProcess as p on {b:process} = {p:pk} } 59 | WHERE {p:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 60 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 61 | ; importBatchCleanupJob ; importBatchRule ; 1000 62 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 63 | ; importBatchCleanupCronJob ; importBatchCleanupJob ; 64 | 65 | # @readme ImportBatchContent 66 | # Are there any left-over distributed import batches? 67 | # SELECT 68 | # COUNT({c:pk}) AS "total", 69 | # MIN({c:modifiedTime}) AS "oldest", 70 | # MAX({c:modifiedTime}) AS "newest" 71 | # FROM 72 | # {ImportBatchContent AS c 73 | # LEFT JOIN 74 | # ImportBatch AS b 75 | # ON {b:importContentCode} = {c:code} } 76 | # WHERE 77 | # {b:pk} IS NULL 78 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ; 79 | ; importBatchContentRule ; " 80 | SELECT {c:pk}, {c:itemtype} 81 | FROM {ImportBatchContent AS c LEFT JOIN ImportBatch as b on {b:importContentCode} = {c:code} } 82 | WHERE {b:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 83 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize 84 | ; importBatchContentCleanupJob ; importBatchContentRule ; 1000 85 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 86 | ; importBatchContentCleanupCronJob ; importBatchContentCleanupJob ; 87 | 88 | ### 89 | 90 | INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage] 91 | ;distributedImpexRetentionCompositeCronJob;compositeJobPerformable; 92 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 93 | # every day at 04:30 94 | ; distributedImpexRetentionCompositeCronJob ; 0 30 4 * * ? 95 | 96 | INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='distributedImpexRetentionCompositeCronJob'] 97 | ;distributedImpexCronJobCleanupCronJobEntry;distributedImpexCronJobCleanupCronJob; 98 | ;distributedImpexJobCleanupCronJobEntry;distributedImpexJobCleanupCronJob; 99 | ;distributedImportProcessCleanupCronJobEntry;distributedImportProcessCleanupCronJob; 100 | ;importBatchCleanupCronJobEntry;importBatchCleanupCronJob; 101 | ;importBatchContentCleanupCronJobEntry;importBatchContentCleanupCronJob; 102 | 103 | UPDATE CompositeCronJob;code[unique=true];compositeEntries(code) 104 | ;distributedImpexRetentionCompositeCronJob;" 105 | distributedImpexCronJobCleanupCronJobEntry, 106 | distributedImpexJobCleanupCronJobEntry, 107 | distributedImportProcessCleanupCronJobEntry, 108 | importBatchCleanupCronJobEntry, 109 | importBatchContentCleanupCronJobEntry 110 | " 111 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | TBD 11 | 12 | ## [3.4.1] - 2022-05-13 13 | 14 | ### Fixed 15 | 16 | - Fix startup error, caused by version parser failing to parse extended `build.number` generated by 17 | CCv2 build process ([#6]) 18 | 19 | Many thanks to [@ths-sybit] for fixing the issue! 20 | 21 | [#6]: https://github.com/sap-commerce-tools/sanecleanup/pull/6 22 | [@ths-sybit]: https://github.com/ths-sybit 23 | 24 | ## [3.4.0] 2022-03-28 25 | 26 | ### Added 27 | 28 | - Cleanup `SavedValue`s for delete operations after one month 29 | - *Experimental:* Define DB indices to accelerate cleanup queries in CCv2\ 30 | **The minimum required versions for this feature are: [1905.33], [2005.17], [2011.12], [2105.2]**\ 31 | In case of problems 32 | - Report an [issue] in this repository 33 | - Comment-out the offending type / index in the `items.xml` 34 | 35 | ### Changed 36 | 37 | - Composite cronjob to cleanup `SavedValue` / `SavedValueEntry` 38 | - `ProcessTaskLog` cleanup now checks `modifiedTime` like all the other cleanup jobs 39 | 40 | 41 | ### Fixed 42 | 43 | - Custom CMS Version GC job (`jdbcVersionGCJob`) is now backwards compatible with 2105 and recent patch releases ([#3]).\ 44 | If you are on a more recent Commerce version, the job will do nothing. Please use the improved 45 | ootb cleanup instead (introduced with [1905.30], [2005.14][2005.14], [2011.9][2011.9]). 46 | 47 | ### Upgrade Guide 48 | 49 | - Remove triggers of cronjobs that are now part of a composite job. 50 | - Remove custom CMS version GC job 51 | 52 | ```impex 53 | REMOVE Trigger; cronJob(code)[unique = true] 54 | ;savedValueEntryCleanupCronJob; 55 | 56 | # remove custom job 57 | REMOVE Cronjob;code[unique = true] 58 | ;jdbcVersionGCCronJob; 59 | ``` 60 | 61 | [#3]: https://github.com/sap-commerce-tools/sanecleanup/issues/3 62 | [2011.9]:https://help.sap.com/docs/SAP_COMMERCE/eed845124da0491e875df8139c4e6e8c/f18f6a711d07462b80137df6ed533eee.html?version=2011#patch-2011.9 63 | [2005.14]:https://help.sap.com/docs/SAP_COMMERCE/eed845124da0491e875df8139c4e6e8c/f18f6a711d07462b80137df6ed533eee.html?version=2005#patch-2005.14 64 | [1905.30]:https://help.sap.com/docs/SAP_COMMERCE/eed845124da0491e875df8139c4e6e8c/f18f6a711d07462b80137df6ed533eee.html?version=1905#patch-1905.30 65 | 66 | [1905.33]: https://help.sap.com/docs/SAP_COMMERCE_CLOUD_PUBLIC_CLOUD/75d4c3895cb346008545900bffe851ce/f18f6a711d07462b80137df6ed533eee.html?version=v1905#patch-1905.33 67 | [2005.17]: https://help.sap.com/docs/SAP_COMMERCE_CLOUD_PUBLIC_CLOUD/75d4c3895cb346008545900bffe851ce/f18f6a711d07462b80137df6ed533eee.html?version=v2005#patch-2005.17 68 | [2011.12]: https://help.sap.com/docs/SAP_COMMERCE_CLOUD_PUBLIC_CLOUD/75d4c3895cb346008545900bffe851ce/f18f6a711d07462b80137df6ed533eee.html?version=v2011#patch-2011.12 69 | [2105.2]: https://help.sap.com/docs/SAP_COMMERCE_CLOUD_PUBLIC_CLOUD/75d4c3895cb346008545900bffe851ce/f18f6a711d07462b80137df6ed533eee.html#patch-2105.2 70 | 71 | ## [3.3.0] - 2021-06-09 72 | 73 | ### Added 74 | 75 | - Retention periods are now configurable via properties 76 | (check `project.properties` for details) 77 | 78 | ### Changed 79 | 80 | - Retention rules and cron jobs that depend on each other are now grouped and 81 | executed in the correct order via composite cronjobs 82 | 83 | ### Upgrade Guide 84 | 85 | Remove all triggers of cronjobs that are now part of a composite job 86 | 87 | ```impex 88 | REMOVE Trigger; cronJob(code)[unique = true] 89 | ; emailMessageCleanupCronJob 90 | ; emailAddressCleanupCronJob 91 | ; emailAttachmentCleanupCronJob 92 | ; cartCleanupCronJob 93 | ; anonymousCartCleanupCronJob 94 | ; distributedImpexCronJobCleanupCronJob 95 | ; distributedImpexJobCleanupCronJob 96 | ; distributedImportProcessCleanupCronJob 97 | ; importBatchCleanupCronJob 98 | ; importBatchContentCleanupCronJob 99 | ; businessProcessCleanupCronJob 100 | ; failedBusinessProcessCleanupCronJob 101 | ; progressBusinessProcessCleanupCronJob 102 | ; orphanedTaskConditionCleanupCronJob 103 | ; orphanedProcessTaskCleanupCronJob 104 | ; orphanedBusinessProcessParameterCleanupCronJob 105 | ; orphanedProcessTaskLogCleanupCronJob 106 | ``` 107 | 108 | 109 | ## [3.2.0] - 2021-05-06 110 | 111 | ### Added 112 | 113 | - Cleanup `ProcessTaskLog` (thanks to [@ashwinineha] :tada:) 114 | 115 | [@ashwinineha]: https://github.com/ashwinineha 116 | 117 | ## [3.1.0] - 2021-04-30 118 | 119 | ### Added 120 | 121 | - Cleanup `SavedValueEntries` 122 | 123 | ## [3.0.0] - 2021-04-30 124 | 125 | ### Added 126 | 127 | - Cleanup all `BusinessProcess` - all `BusinessProcess`es, regardless of their state, 128 | are deleted after 6 months at the latest. **Make sure to adjust this to your project 129 | requirements!** 130 | - Cleanup potentially orphaned items related to `BusinessProcess` 131 | - Aggressive cleanup for `cmsVersionGCProcess` 132 | - Cleanup additional generated impex media 133 | - Cleanup `EmailMessage` and `EmailAddress` 134 | - Cleanup `SolrIndexOperation` 135 | - Cleanup all types related to [Distributed ImpEx](https://help.sap.com/viewer/d0224eca81e249cb821f2cdf45a82ace/LATEST/en-US/3e0138c9bfc642349cad227cfcd72d9f.html) 136 | - `retentionrule-to-impex.groovy` - helper script that takes the results of a `FlexibleSearchRetentionRule` and delete 137 | the outdated items via impex. Useful for bulk cleanup. 138 | - README now documents queries to analyze outdated/stale data 139 | 140 | ### Changed 141 | 142 | - CMS Version Garbage Collection Job 143 | 144 | - renamed to `jdbcVersionGCCronJob` / `jdbcVersionGCJob` 145 | - optimized cleanup logic 146 | - dynamically determine correct DB table names using the type system 147 | 148 | - Simplify cronjob retention rule (`cronJobRule`) 149 | - Cleanup CronJobs now execute between 00:00 - 06:00 150 | - Longer retention period (4 weeks) for successfully finished `BusinessProcess` 151 | 152 | ### Fixed 153 | 154 | - CMS Version Garbage Collection Job - job is now abortable for real 155 | 156 | ### Upgrade Guide 157 | 158 | - Delete old CMS Version GC Job definition 159 | 160 | ```impex 161 | REMOVE CronJob;code[unique=true];job(code)[unique=true] 162 | ;cmsVersionGCCronJob;cmsVersionGCJob; 163 | 164 | REMOVE ServicelayerJob;code[unique=true];springId[unique=true]; 165 | ;cmsVersionGCJob;cmsVersionGCPerformable; 166 | ``` 167 | 168 | ## [2.0.0] - 2021-03-23 169 | 170 | ### Changed 171 | 172 | - Cleanup jobs / retention rules are now imported on-demand based on the extensions the project uses. 173 | 174 | ### Added 175 | 176 | - Custom retention cronjob to replace CMS Version [Garbage Collection][versiongc].\ 177 | The ootb garbage collection mechanism is over-engineered and should be a regular cronjob. 178 | 179 | - Retention rules and jobs for the promotion engine, based on 180 | [Top 10 Recommendations for Improving the Performance of your Commerce Cloud Promotion Engine][top10] 181 | 182 | [versiongc]: https://help.sap.com/viewer/9d346683b0084da2938be8a285c0c27a/2011/en-US/9089116335ac4f4d8708e0c5516531e3.html 183 | [top10]: https://www.sap.com/cxworks/article/538808299/top_10_recommendations_for_improving_the_performance_of_your_commerce_cloud_promotion_engine 184 | 185 | ## [1.0.1] - 2020-12-09 186 | 187 | ### Added 188 | 189 | - Bulk cleanup cronjob for log files - useful for a one-time cleanup before the retention 190 | job for job logs is enabled 191 | 192 | ## [1.0.0] - 2020-11-26 193 | 194 | Initial release 195 | 196 | ### Added 197 | 198 | - Cleanup for: 199 | 200 | - CronJobs 201 | - CronJob Histories 202 | - Lob Logs / Log Files 203 | - Impex Media 204 | - HTTP Sessions 205 | - Business Processes 206 | - Carts 207 | 208 | 209 | [Unreleased]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.4.1...HEAD 210 | [3.4.1]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.4.0...v3.4.1 211 | [3.4.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.3.0...v3.4.0 212 | [3.3.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.2.0...v3.3.0 213 | [3.2.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.1.0...v.3.2.0 214 | [3.1.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.0.0...v3.1.0 215 | [3.0.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v2.0.0...v3.0.0 216 | [2.0.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v1.0.1...v2.0.0 217 | [1.0.1]: https://github.com/sap-commerce-tools/sanecleanup/compare/v1.0.0...v1.0.1 218 | [1.0.0]: https://github.com/sap-commerce-tools/sanecleanup/releases/tag/v1.0.0 219 | 220 | [issue]: https://github.com/sap-commerce-tools/sanecleanup/issues -------------------------------------------------------------------------------- /src/mpern/sap/cleanup/CleanupAfterInitListener.java: -------------------------------------------------------------------------------- 1 | package mpern.sap.cleanup; 2 | 3 | import de.hybris.bootstrap.config.ConfigUtil; 4 | import de.hybris.bootstrap.config.ExtensionInfo; 5 | import de.hybris.platform.core.Registry; 6 | import de.hybris.platform.core.model.initialization.SystemSetupAuditModel; 7 | import de.hybris.platform.servicelayer.event.events.AfterInitializationEndEvent; 8 | import de.hybris.platform.servicelayer.event.impl.AbstractEventListener; 9 | import de.hybris.platform.servicelayer.impex.ImportConfig; 10 | import de.hybris.platform.servicelayer.impex.ImportResult; 11 | import de.hybris.platform.servicelayer.impex.ImportService; 12 | import de.hybris.platform.servicelayer.impex.impl.StreamBasedImpExResource; 13 | import de.hybris.platform.servicelayer.model.ModelService; 14 | import de.hybris.platform.servicelayer.search.FlexibleSearchQuery; 15 | import de.hybris.platform.servicelayer.search.FlexibleSearchService; 16 | import de.hybris.platform.servicelayer.search.SearchResult; 17 | import de.hybris.platform.servicelayer.user.UserService; 18 | import de.hybris.platform.util.persistence.PersistenceUtils; 19 | import mpern.sap.cleanup.constants.SanecleanupConstants; 20 | import org.apache.commons.codec.digest.DigestUtils; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | import org.springframework.core.io.Resource; 24 | import org.springframework.core.io.support.PathMatchingResourcePatternResolver; 25 | 26 | import java.io.BufferedReader; 27 | import java.io.IOException; 28 | import java.io.InputStreamReader; 29 | import java.nio.charset.StandardCharsets; 30 | import java.util.*; 31 | import java.util.stream.Collectors; 32 | 33 | import static org.springframework.core.io.support.ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX; 34 | 35 | public class CleanupAfterInitListener extends AbstractEventListener { 36 | 37 | private static final Logger LOG = LoggerFactory.getLogger(CleanupAfterInitListener.class); 38 | public static final String VERSIONS_PREFIX = "#:versions:"; 39 | 40 | private final ImportService importService; 41 | private final FlexibleSearchService flexibleSearchService; 42 | private final ModelService modelService; 43 | private final UserService userService; 44 | private final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); 45 | private final VersionRangeEvaluator versionRangeEvaluator; 46 | 47 | 48 | public CleanupAfterInitListener(ImportService importService, 49 | FlexibleSearchService flexibleSearchService, 50 | ModelService modelService, 51 | UserService userService, 52 | VersionRangeEvaluator versionRangeEvaluator 53 | ) { 54 | this.importService = importService; 55 | this.flexibleSearchService = flexibleSearchService; 56 | this.modelService = modelService; 57 | this.userService = userService; 58 | this.versionRangeEvaluator = versionRangeEvaluator; 59 | } 60 | 61 | @Override 62 | protected void onEvent(AfterInitializationEndEvent afterInitializationEndEvent) { 63 | try { 64 | final List applicableImpex = getApplicableImpex(); 65 | Map hashToResource = calculateHashes(applicableImpex); 66 | Map filtered = filterAlreadyImported(hashToResource); 67 | final List auditModels = importImpexes(filtered); 68 | saveAudit(auditModels); 69 | } catch (Exception e) { 70 | LOG.error("sanecleanup - failed", e); 71 | } 72 | } 73 | 74 | private List getApplicableImpex() throws IOException { 75 | final List extensions = ConfigUtil.getPlatformConfig(Registry.class).getExtensionInfosInBuildOrder(); 76 | final List applicableImpex = new ArrayList<>(); 77 | for (ExtensionInfo extension : extensions) { 78 | Resource[] resources = resolver.getResources(CLASSPATH_ALL_URL_PREFIX + "/impex/sanecleanup/" + extension.getName() + "/*.impex"); 79 | List resourceList = Arrays.asList(resources); 80 | resourceList.sort(Comparator.comparing(Resource::getFilename)); 81 | applicableImpex.addAll(resourceList); 82 | } 83 | return applicableImpex.stream() 84 | .filter(this::applicableForCurrentVersion) 85 | .collect(Collectors.toList()); 86 | } 87 | 88 | private boolean applicableForCurrentVersion(Resource impex) { 89 | String versionExpression = ""; 90 | try (BufferedReader br = new BufferedReader(new InputStreamReader(impex.getInputStream(), StandardCharsets.UTF_8))) { 91 | String line = br.readLine(); 92 | if (line != null && line.startsWith(VERSIONS_PREFIX)) { 93 | versionExpression = line.substring(VERSIONS_PREFIX.length()); 94 | } 95 | } catch (IOException e) { 96 | // ignore 97 | } 98 | boolean result = true; 99 | if (!versionExpression.isEmpty()) { 100 | result = versionRangeEvaluator.evaluate(versionExpression); 101 | } 102 | LOG.debug("{}: applicable for current Commerce version? {} ({})", impex.getFilename(), result, versionExpression); 103 | return result; 104 | } 105 | 106 | private Map calculateHashes(List applicableImpex) throws Exception { 107 | if (applicableImpex.isEmpty()) { 108 | return Collections.emptyMap(); 109 | } 110 | Map hashToResource = new LinkedHashMap<>(); 111 | for (Resource impex : applicableImpex) { 112 | // use different hash function to avoid collision with hash calculation for @SystemSetup classes 113 | // ref. de.hybris.platform.core.initialization.SystemSetupCollectorResult#computePatchHash 114 | String hash = DigestUtils.sha1Hex(impex.getInputStream()); 115 | hashToResource.put(hash, impex); 116 | } 117 | return hashToResource; 118 | } 119 | 120 | private Map filterAlreadyImported(Map hashResource) { 121 | if (hashResource.isEmpty()) { 122 | return Collections.emptyMap(); 123 | } 124 | Map filtered = new HashMap<>(hashResource); 125 | FlexibleSearchQuery fsq = new FlexibleSearchQuery("select {hash} from {SystemSetupAudit} where {hash} IN (?maybeNew)"); 126 | fsq.setResultClassList(Collections.singletonList(String.class)); 127 | fsq.addQueryParameter("maybeNew", hashResource.keySet()); 128 | final SearchResult search = flexibleSearchService.search(fsq); 129 | filtered.keySet().removeAll(search.getResult()); 130 | return filtered; 131 | } 132 | 133 | private List importImpexes(Map filtered) throws IOException { 134 | List auditModels = new ArrayList<>(); 135 | for (Map.Entry entry : filtered.entrySet()) { 136 | Resource resource = entry.getValue(); 137 | LOG.info("sanecleanup: Importing {}", resource.getFilename()); 138 | ImportConfig cfg = new ImportConfig(); 139 | cfg.setEnableCodeExecution(true); 140 | cfg.setScript(new StreamBasedImpExResource(resource.getInputStream(), "UTF-8")); 141 | ImportResult importResult = importService.importData(cfg); 142 | if (importResult.isError()) { 143 | LOG.error("sanecleanup: Importing {} FAILED", resource.getFilename()); 144 | } 145 | auditModels.add(generateAuditEntry(entry)); 146 | } 147 | return auditModels; 148 | } 149 | 150 | private SystemSetupAuditModel generateAuditEntry(Map.Entry entry) { 151 | final SystemSetupAuditModel audit = modelService.create(SystemSetupAuditModel.class); 152 | audit.setHash(entry.getKey()); 153 | audit.setName(entry.getValue().getFilename()); 154 | audit.setClassName(CleanupAfterInitListener.class.getCanonicalName()); 155 | audit.setMethodName("onEvent"); 156 | audit.setRequired(false); 157 | audit.setExtensionName(SanecleanupConstants.EXTENSIONNAME); 158 | audit.setUser(userService.getCurrentUser()); 159 | return audit; 160 | } 161 | 162 | private void saveAudit(final List auditModels) { 163 | PersistenceUtils.doWithSLDPersistence(() -> { 164 | modelService.saveAll(auditModels); 165 | return true; 166 | }); 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /resources/sanecleanup-items.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | -------------------------------------------------------------------------------- /resources/impex/sanecleanup/processing/003-cleanup-businessprocess.impex: -------------------------------------------------------------------------------- 1 | # Import config properties into impex macros 2 | UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true] 3 | $sessionLanguage=$config-sanecleanup.jobs.sessionlanguage 4 | $retentionTimeProcessSuccess=$config-sanecleanup.retentiontimeseconds.businessprocess.succeeded 5 | $retentionTimeProcessFailed=$config-sanecleanup.retentiontimeseconds.businessprocess.failed 6 | $retentionTimeProcessRunning=$config-sanecleanup.retentiontimeseconds.businessprocess.running 7 | $retentionTimeTaskConditionPremature=$config-sanecleanup.retentiontimeseconds.taskcondition.premature 8 | 9 | 10 | # @readme BusinessProcess 11 | # Are there too many (let's say > 1000) or very old BusinessProcess in your system? 12 | # 13 | # Also, if a lot of processes are stuck in "RUNNING" / "WAITING", you have to investigate what's wrong. 14 | # (What is causing your processes to be stuck?) 15 | # SELECT {p:processDefinitionName}, 16 | # {s:code} AS "status", 17 | # COUNT({p:pk}) AS "total", 18 | # MIN({p:modifiedTime}) AS "oldest", 19 | # MAX({p:modifiedTime}) AS "newest" 20 | # FROM {BusinessProcess AS p LEFT JOIN ProcessState AS s ON {p:state} = {s:pk} } 21 | # GROUP BY {p:processDefinitionName}, {s:code} 22 | # ORDER BY "total" DESC 23 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 24 | ; businessProcessRule ; " 25 | SELECT {p:pk}, {p:itemtype} 26 | FROM {BusinessProcess AS p JOIN ProcessState AS s ON {p:state} = {s:pk} } 27 | WHERE {s:code} IN ('SUCCEEDED') 28 | AND {p:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeProcessSuccess ; basicRemoveCleanupAction ; 29 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 30 | ; businessProcessCleanupJob ; businessProcessRule ; 1000 31 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 32 | ; businessProcessCleanupCronJob ; businessProcessCleanupJob ; 33 | 34 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 35 | ; failedBusinessProcessRule ; " 36 | SELECT {p:pk}, {p:itemtype} 37 | FROM {BusinessProcess AS p JOIN ProcessState AS s ON {p:state} = {s:pk} } 38 | WHERE {s:code} IN ('FAILED', 'ERROR') 39 | AND {p:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeProcessFailed ; basicRemoveCleanupAction ; 40 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 41 | ; failedBusinessProcessCleanupJob ; failedBusinessProcessRule ; 1000 42 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 43 | ; failedBusinessProcessCleanupCronJob ; failedBusinessProcessCleanupJob ; 44 | 45 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 46 | ; progressBusinessProcessRule ; " 47 | SELECT {p:pk}, {p:itemtype} 48 | FROM {BusinessProcess AS p JOIN ProcessState AS s ON {p:state} = {s:pk} } 49 | WHERE {s:code} IN ('CREATED', 'RUNNING', 'WAITING') 50 | AND {p:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeProcessRunning ; basicRemoveCleanupAction ; 51 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 52 | ; progressBusinessProcessCleanupJob ; progressBusinessProcessRule ; 1000 53 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 54 | ; progressBusinessProcessCleanupCronJob ; progressBusinessProcessCleanupJob ; 55 | 56 | #### delete orphans if BusinessProcess / Task is gone. this shouldn't happen, but you never know 57 | 58 | # TaskCondition - usually assigned to a task. 59 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 60 | ; orphanedTaskConditionRule ; " 61 | SELECT {tc:pk}, {tc:itemtype} 62 | FROM {TaskCondition AS tc LEFT JOIN Task AS t ON {tc:task} = {t:pk} } 63 | WHERE {tc:task} IS NOT NULL 64 | AND {t:pk} IS NULL" ; 0 ; basicRemoveCleanupAction ; 65 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 66 | ; orphanedTaskConditionCleanupJob ; orphanedTaskConditionRule ; 1000 67 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 68 | ; orphanedTaskConditionCleanupCronJob ; orphanedTaskConditionCleanupJob ; 69 | 70 | # ProcessTask 71 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 72 | ; orphanedProcessTaskRule ; " 73 | SELECT {pt:pk}, {pt:itemtype} 74 | FROM {ProcessTask AS pt LEFT JOIN BusinessProcess AS bp ON {pt:process} = {bp:pk} } 75 | WHERE ( 76 | ( {pt:process} IS NOT NULL AND {bp:pk} IS NULL ) 77 | OR 78 | {pt:process} IS NULL 79 | )" ; 0 ; basicRemoveCleanupAction ; 80 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 81 | ; orphanedProcessTaskCleanupJob ; orphanedProcessTaskRule ; 1000 82 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 83 | ; orphanedProcessTaskCleanupCronJob ; orphanedProcessTaskCleanupJob ; 84 | 85 | # BusinessProcessParameter 86 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 87 | ; orphanedBusinessProcessParameterRule ; " 88 | SELECT {p:pk}, {p:itemtype} 89 | FROM {BusinessProcessParameter AS p LEFT JOIN BusinessProcess AS bp ON {p:process} = {bp:pk} } 90 | WHERE ( 91 | ( {p:process} IS NOT NULL AND {bp:pk} IS NULL ) 92 | OR 93 | {p:process} IS NULL 94 | )" ; 0 ; basicRemoveCleanupAction ; 95 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 96 | ; orphanedBusinessProcessParameterCleanupJob ; orphanedBusinessProcessParameterRule ; 1000 97 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 98 | ; orphanedBusinessProcessParameterCleanupCronJob ; orphanedBusinessProcessParameterCleanupJob ; 99 | 100 | # ProcessTaskLog 101 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 102 | ; orphanedProcessTaskLogRule ; " 103 | SELECT {l:pk}, {l:itemtype} 104 | FROM {ProcessTaskLog AS l LEFT JOIN BusinessProcess AS bp ON {l:process} = {bp:pk} } 105 | WHERE ( 106 | ( {l:process} IS NOT NULL AND {bp:pk} IS NULL ) 107 | OR 108 | {l:process} IS NULL 109 | )" ; 0 ; basicRemoveCleanupAction ; 110 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 111 | ; orphanedProcessTaskLogCleanupJob ; orphanedProcessTaskLogRule ; 1000 112 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 113 | ; orphanedProcessTaskLogCleanupCronJob ; orphanedProcessTaskLogCleanupJob ; 114 | 115 | ### 116 | 117 | INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage] 118 | ;businessProcessRetentionCompositeCronJob;compositeJobPerformable; 119 | INSERT Trigger; cronJob(code)[unique = true]; cronExpression 120 | # every day at midnight 121 | ; businessProcessRetentionCompositeCronJob ; 0 0 0 * * ? 122 | 123 | INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='businessProcessRetentionCompositeCronJob'] 124 | ;businessProcessCleanupCronJobEntry;businessProcessCleanupCronJob; 125 | ;failedBusinessProcessCleanupCronJobEntry;failedBusinessProcessCleanupCronJob; 126 | ;progressBusinessProcessCleanupCronJobEntry;progressBusinessProcessCleanupCronJob; 127 | ;orphanedTaskConditionCleanupCronJobEntry;orphanedTaskConditionCleanupCronJob; 128 | ;orphanedProcessTaskCleanupCronJobEntry;orphanedProcessTaskCleanupCronJob; 129 | ;orphanedBusinessProcessParameterCleanupCronJobEntry;orphanedBusinessProcessParameterCleanupCronJob; 130 | ;orphanedProcessTaskLogCleanupCronJobEntry;orphanedProcessTaskLogCleanupCronJob; 131 | 132 | # 133 | UPDATE CompositeCronJob;code[unique=true];compositeEntries(code) 134 | ;businessProcessRetentionCompositeCronJob;"businessProcessCleanupCronJobEntry, 135 | failedBusinessProcessCleanupCronJobEntry, 136 | progressBusinessProcessCleanupCronJobEntry, 137 | orphanedTaskConditionCleanupCronJobEntry, 138 | orphanedProcessTaskCleanupCronJobEntry, 139 | orphanedBusinessProcessParameterCleanupCronJobEntry, 140 | orphanedProcessTaskLogCleanupCronJobEntry" 141 | 142 | # @readme TaskCondition 143 | # Is there an excessive amount of "premature events"? Or very old (older than a a few weeks) events? 144 | # 145 | # https://help.sap.com/viewer/d0224eca81e249cb821f2cdf45a82ace/2011/en-US/7e8ff9d7653f43e8890bc8eb395d52a7.html 146 | # SELECT COUNT({tc:pk}), 147 | # MIN({tc:modifiedtime}) AS "oldest", 148 | # MAX({tc:modifiedtime}) AS "newest" 149 | # FROM {TaskCondition AS tc } 150 | # WHERE {tc:task} IS NULL 151 | INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ; 152 | ; prematureTaskConditionRule ; " 153 | SELECT {tc:pk}, {tc:itemtype} 154 | FROM {TaskCondition AS tc } 155 | WHERE {tc:task} IS NULL 156 | AND {tc:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeTaskConditionPremature ; basicRemoveCleanupAction ; 157 | INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize 158 | ; prematureTaskConditionCleanupJob ; prematureTaskConditionRule ; 1000 159 | INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage] 160 | ; prematureTaskConditionCleanupCronJob ; prematureTaskConditionCleanupJob ; 161 | INSERT Trigger; cronJob(code)[unique = true] ; cronExpression 162 | # every day at 02:00 163 | ; prematureTaskConditionCleanupCronJob ; 0 0 2 * * ? 164 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /src/mpern/sap/cleanup/cms2/CMSVersionGCPerformable.java: -------------------------------------------------------------------------------- 1 | package mpern.sap.cleanup.cms2; 2 | 3 | import de.hybris.platform.cms2.model.CMSVersionModel; 4 | import de.hybris.platform.cms2.version.service.CMSVersionGCService; 5 | import de.hybris.platform.core.PK; 6 | import de.hybris.platform.core.model.type.ComposedTypeModel; 7 | import de.hybris.platform.cronjob.enums.CronJobResult; 8 | import de.hybris.platform.cronjob.enums.CronJobStatus; 9 | import de.hybris.platform.cronjob.model.CronJobModel; 10 | import de.hybris.platform.servicelayer.config.ConfigurationService; 11 | import de.hybris.platform.servicelayer.cronjob.AbstractJobPerformable; 12 | import de.hybris.platform.servicelayer.cronjob.PerformResult; 13 | import de.hybris.platform.servicelayer.search.FlexibleSearchQuery; 14 | import de.hybris.platform.servicelayer.search.SearchResult; 15 | import de.hybris.platform.servicelayer.type.TypeService; 16 | import de.hybris.platform.tx.Transaction; 17 | import de.hybris.platform.util.Utilities; 18 | import org.apache.commons.collections4.CollectionUtils; 19 | import org.apache.commons.lang3.time.StopWatch; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import org.springframework.beans.BeansException; 23 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 24 | import org.springframework.context.ApplicationContext; 25 | import org.springframework.context.ApplicationContextAware; 26 | import org.springframework.expression.spel.support.StandardEvaluationContext; 27 | import org.springframework.jdbc.core.BatchPreparedStatementSetter; 28 | import org.springframework.jdbc.core.JdbcTemplate; 29 | 30 | import java.lang.reflect.Field; 31 | import java.sql.PreparedStatement; 32 | import java.sql.SQLException; 33 | import java.util.*; 34 | 35 | //Sane implementation of Content Version GC process. 36 | //Same logic, without creating Business Process for every run and fast delete using JDBC batching 37 | public class CMSVersionGCPerformable extends AbstractJobPerformable implements ApplicationContextAware { 38 | 39 | private static final Logger LOG = LoggerFactory.getLogger(CMSVersionGCPerformable.class); 40 | 41 | private final ConfigurationService configurationService; 42 | private final CMSVersionGCService cmsVersionGCService; 43 | private final JdbcTemplate jdbcTemplate; 44 | private final TypeService typeService; 45 | 46 | private static final String CMSVERSIONGCPROCESS_RELATION; 47 | 48 | static { 49 | String value = ""; 50 | try { 51 | Field field = CMSVersionModel.class.getField("_CMSVERSIONGCPROCESS2CMSVERSION"); 52 | value = (String) field.get(null); 53 | } catch (NoSuchFieldException | IllegalAccessException e) { 54 | // ignore; relation not present anymore 55 | } 56 | CMSVERSIONGCPROCESS_RELATION = value; 57 | } 58 | 59 | private ApplicationContext applicationContext; 60 | 61 | public CMSVersionGCPerformable(ConfigurationService configurationService, CMSVersionGCService cmsVersionGCService, JdbcTemplate jdbcTemplate, TypeService typeService) { 62 | this.configurationService = configurationService; 63 | this.cmsVersionGCService = cmsVersionGCService; 64 | this.jdbcTemplate = jdbcTemplate; 65 | this.typeService = typeService; 66 | 67 | } 68 | 69 | @Override 70 | public boolean isAbortable() { 71 | return true; 72 | } 73 | 74 | @Override 75 | public PerformResult perform(CronJobModel cronJobModel) { 76 | if (CMSVERSIONGCPROCESS_RELATION.isEmpty() || improvedGCVersionCleanupJobExists()) { 77 | LOG.debug("Please use the improved, ootb cmsVersionGCJob. Exiting."); 78 | return new PerformResult(CronJobResult.SUCCESS, CronJobStatus.FINISHED); 79 | } 80 | try { 81 | final List retainableVersions = getRetainableVersions(); 82 | Set retainablePKs = collectAllRetainableVersionPKs(retainableVersions); 83 | 84 | if (clearAbortRequestedIfNeeded(cronJobModel)) { 85 | return new PerformResult(CronJobResult.UNKNOWN, CronJobStatus.ABORTED); 86 | } 87 | final Optional performResult = deleteObsoleteVersionsInBatches(cronJobModel, retainablePKs); 88 | if (performResult.isPresent()) { 89 | return performResult.get(); 90 | } 91 | } catch (Exception e) { 92 | LOG.error("Processing failed.", e); 93 | return new PerformResult(CronJobResult.ERROR, CronJobStatus.UNKNOWN); 94 | } 95 | return new PerformResult(CronJobResult.SUCCESS, CronJobStatus.FINISHED); 96 | } 97 | 98 | private boolean improvedGCVersionCleanupJobExists() { 99 | try { 100 | applicationContext.getBean("cmsVersionGCPerformable"); 101 | return true; 102 | } catch (BeansException ignored) { 103 | } 104 | return false; 105 | } 106 | 107 | // de.hybris.platform.cms2.version.processengine.action.impl.CollectRetainableCMSVersionsGCProcessAction 108 | private List getRetainableVersions() { 109 | int maxAgeInDays = configurationService.getConfiguration().getInt("version.gc.maxAgeDays", 0); 110 | int maxNumberVersions = configurationService.getConfiguration().getInt("version.gc.maxNumberVersions", 0); 111 | return cmsVersionGCService.getRetainableVersions(maxAgeInDays, maxNumberVersions); 112 | } 113 | 114 | // de.hybris.platform.cms2.version.processengine.action.impl.CollectRelatedCMSVersionsGCProcessAction 115 | private Set collectAllRetainableVersionPKs(List retainableVersions) { 116 | Set retainablePKs = new HashSet<>(); 117 | for (CMSVersionModel retainableVersion : retainableVersions) { 118 | if (retainableVersion == null) { 119 | continue; 120 | } 121 | retainablePKs.add(retainableVersion.getPk()); 122 | if (CollectionUtils.isNotEmpty(retainableVersion.getRelatedChildren())) { 123 | retainableVersion.getRelatedChildren().stream() 124 | .filter(Objects::nonNull) 125 | .forEach(v -> { 126 | retainablePKs.add(v.getPk()); 127 | //detach models to avoid memory leaks 128 | modelService.detach(v); 129 | }); 130 | } 131 | modelService.detach(retainableVersion); 132 | } 133 | return retainablePKs; 134 | } 135 | 136 | // de.hybris.platform.cms2.version.processengine.action.impl.RemoveCMSVersionsGCProcessAction 137 | private Optional deleteObsoleteVersionsInBatches(CronJobModel cronJobModel, Set retainablePKs) { 138 | if (retainablePKs.isEmpty()) { 139 | retainablePKs = Collections.singleton(PK.NULL_PK); 140 | } 141 | FlexibleSearchQuery versionsToDelete = new FlexibleSearchQuery("select {v:pk} from {cmsversion as v} order by {v:pk} desc"); 142 | versionsToDelete.setResultClassList(Collections.singletonList(PK.class)); 143 | SearchResult result = flexibleSearchService.search(versionsToDelete); 144 | 145 | if (clearAbortRequestedIfNeeded(cronJobModel)) { 146 | return Optional.of(new PerformResult(CronJobResult.UNKNOWN, CronJobStatus.ABORTED)); 147 | } 148 | // this is actually faster than sending a massive value list to the DB, i.e. `where {v:pk} NOT IN (?retainable)` 149 | // especially for a non-trivial amount (> 1000) of retainable versions 150 | // query and logic only use PKs -> memory usage is minimal, one PK = 24 bytes of heap on a 64bit JVM 151 | List toDelete = new ArrayList<>(result.getResult()); 152 | final int totalSize = toDelete.size(); 153 | toDelete.removeAll(retainablePKs); 154 | final int deleteCount = toDelete.size(); 155 | StopWatch sw = StopWatch.createStarted(); 156 | int pageSize = cronJobModel.getQueryCount() > 0 ? cronJobModel.getQueryCount() : 1000; 157 | 158 | List statements = prepareDeleteStatements(); 159 | 160 | for (int i = 0; i < deleteCount; i += pageSize) { 161 | int endIdx = Math.min(i + pageSize, deleteCount); 162 | final List batchToDelete = toDelete.subList(i, endIdx); 163 | boolean success = false; 164 | try { 165 | Transaction.current().begin(); 166 | deleteBatchWithJDBC(batchToDelete, statements); 167 | success = true; 168 | LOG.debug("Deleted {} / {}...", i + batchToDelete.size(), deleteCount); 169 | } finally { 170 | if (success) { 171 | Transaction.current().commit(); 172 | } else { 173 | Transaction.current().rollback(); 174 | } 175 | } 176 | if (clearAbortRequestedIfNeeded(cronJobModel)) { 177 | return Optional.of(new PerformResult(CronJobResult.UNKNOWN, CronJobStatus.ABORTED)); 178 | } 179 | } 180 | sw.stop(); 181 | LOG.info("Total versions: {}; Retainable versions: {}; {} versions deleted in {}", totalSize, retainablePKs.size(), deleteCount, sw.toString()); 182 | return Optional.empty(); 183 | } 184 | 185 | private List prepareDeleteStatements() { 186 | ComposedTypeModel versionType = typeService.getComposedTypeForClass(CMSVersionModel.class); 187 | ComposedTypeModel relationType = typeService.getComposedTypeForCode(CMSVERSIONGCPROCESS_RELATION); 188 | 189 | List statements = new ArrayList<>(); 190 | 191 | statements.add(String.format("DELETE FROM %s WHERE %s = ?", versionType.getTable(), 192 | typeService.getAttributeDescriptor(versionType, CMSVersionModel.PK).getDatabaseColumn())); 193 | statements.add(String.format("DELETE FROM %s WHERE %s = ?", relationType.getTable(), 194 | typeService.getAttributeDescriptor(relationType, "source").getDatabaseColumn())); 195 | statements.add(String.format("DELETE FROM %s WHERE %s = ?", relationType.getTable(), 196 | typeService.getAttributeDescriptor(relationType, "target").getDatabaseColumn())); 197 | 198 | return statements; 199 | } 200 | 201 | private void deleteBatchWithJDBC(final List batchToDelete, List deletes) { 202 | 203 | for (String delete : deletes) { 204 | jdbcTemplate.batchUpdate(delete, new BatchPreparedStatementSetter() { 205 | @Override 206 | public void setValues(PreparedStatement preparedStatement, int i) throws SQLException { 207 | preparedStatement.setLong(1, batchToDelete.get(i).getLong()); 208 | } 209 | 210 | @Override 211 | public int getBatchSize() { 212 | return batchToDelete.size(); 213 | } 214 | }); 215 | } 216 | invalidateCache(batchToDelete); 217 | } 218 | 219 | private void invalidateCache(List batchToDelete) { 220 | batchToDelete.forEach(Utilities::invalidateCache); 221 | } 222 | 223 | @Override 224 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 225 | this.applicationContext = applicationContext; 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | comment_clear_blank_lines=true 2 | comment_format_comments=true 3 | comment_format_header=false 4 | comment_format_html=true 5 | comment_format_source_code=true 6 | comment_indent_parameter_description=true 7 | comment_indent_root_tags=true 8 | comment_line_length=160 9 | comment_new_line_for_parameter=false 10 | comment_separate_root_tags=true 11 | eclipse.preferences.version=1 12 | editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true 13 | formatter_settings_version=11 14 | org.eclipse.jdt.ui.exception.name=e 15 | org.eclipse.jdt.ui.gettersetter.use.is=true 16 | org.eclipse.jdt.ui.ignorelowercasenames=true 17 | org.eclipse.jdt.ui.importorder=de.hybris;java;javax;org;com;de; 18 | org.eclipse.jdt.ui.javadoc=true 19 | org.eclipse.jdt.ui.keywordthis=false 20 | org.eclipse.jdt.ui.ondemandthreshold=50 21 | org.eclipse.jdt.ui.overrideannotation=true 22 | org.eclipse.jdt.ui.staticondemandthreshold=50 23 | org.eclipse.jdt.ui.text.custom_code_templates=