├── .gitattributes ├── .github └── ISSUE_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.adoc ├── assets ├── grabbitConnection.png ├── jobKickedOff.png ├── jobStatus.png ├── monitor.png └── newJob.png ├── build.gradle ├── buildSrc ├── build.gradle └── src │ ├── main │ └── groovy │ │ └── com │ │ └── twcable │ │ └── grabbit │ │ └── Version.groovy │ └── test │ └── groovy │ └── com │ └── twcable │ └── grabbit │ └── VersionSpec.groovy ├── docs ├── AEMSupport.adoc ├── Building.adoc ├── Cleaning.adoc ├── GeneralLayout.adoc ├── GettingStarted.adoc ├── LibraryAttribution.adoc ├── LicenseInfo.adoc ├── Monitoring.adoc ├── RELEASE_NOTES.md ├── RELEASING.adoc ├── Running.adoc └── jcr-spec.pdf ├── grabbit.sh ├── gradle.properties ├── gradle ├── bundle.gradle ├── dependencies.gradle ├── idea.gradle ├── packageExclusions.gradle ├── utils.gradle ├── verifyComponentConfig.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── install_protoc_from_source.sh ├── sample-config.json ├── sample_config.yaml ├── settings.gradle ├── src ├── integrationTest │ └── groovy │ │ └── com │ │ └── twcable │ │ └── grabbit │ │ ├── server │ │ └── services │ │ │ └── ServerServiceSpec.groovy │ │ └── testutils │ │ └── MockServletOutputStream.groovy ├── main │ ├── content │ │ ├── META-INF │ │ │ └── vault │ │ │ │ ├── config.xml │ │ │ │ ├── definition │ │ │ │ └── .content.xml │ │ │ │ ├── filter.xml │ │ │ │ ├── properties.xml │ │ │ │ └── settings.xml │ │ └── SLING-INF │ │ │ └── content │ │ │ └── apps │ │ │ └── grabbit │ │ │ └── config │ │ │ ├── com.twcable.grabbit.resources.GrabbitResourceProvider.xml │ │ │ ├── org.apache.sling.commons.log.LogManager.factory.config-com.twcable.grabbit.receive.xml │ │ │ ├── org.apache.sling.commons.log.LogManager.factory.config-com.twcable.grabbit.send.xml │ │ │ └── org.apache.sling.commons.log.LogManager.factory.config-org.apache.jackrabbit.core.SessionImpl.xml │ ├── groovy │ │ └── com │ │ │ └── twcable │ │ │ └── grabbit │ │ │ ├── ClientJobStatus.groovy │ │ │ ├── DateUtil.groovy │ │ │ ├── GrabbitConfiguration.groovy │ │ │ ├── NonExistentJobException.groovy │ │ │ ├── client │ │ │ ├── batch │ │ │ │ ├── ClientBatchJob.groovy │ │ │ │ ├── ClientBatchJobContext.groovy │ │ │ │ ├── ClientBatchJobExecutionListener.groovy │ │ │ │ ├── steps │ │ │ │ │ ├── http │ │ │ │ │ │ └── CreateHttpConnectionTasklet.groovy │ │ │ │ │ ├── jcrnodes │ │ │ │ │ │ ├── JcrNodesReader.groovy │ │ │ │ │ │ ├── JcrNodesWriter.groovy │ │ │ │ │ │ └── LoggingStepExecutionListener.groovy │ │ │ │ │ ├── namespace │ │ │ │ │ │ └── NamespaceSyncTasklet.groovy │ │ │ │ │ ├── validation │ │ │ │ │ │ └── ValidJobDecider.groovy │ │ │ │ │ ├── workflows │ │ │ │ │ │ ├── WorkflowOffTasklet.groovy │ │ │ │ │ │ └── WorkflowOnTasklet.groovy │ │ │ │ │ └── workspace │ │ │ │ │ │ ├── DeleteBeforeWriteDecider.groovy │ │ │ │ │ │ └── DeleteBeforeWriteTasklet.groovy │ │ │ │ └── workflows │ │ │ │ │ ├── WorkflowManager.groovy │ │ │ │ │ └── impl │ │ │ │ │ └── DefaultWorkFlowManager.groovy │ │ │ ├── services │ │ │ │ ├── ClientService.groovy │ │ │ │ └── impl │ │ │ │ │ └── DefaultClientService.groovy │ │ │ └── servlets │ │ │ │ ├── GrabbitJobServlet.groovy │ │ │ │ ├── GrabbitRootServlet.groovy │ │ │ │ └── GrabbitTransactionServlet.groovy │ │ │ ├── jcr │ │ │ ├── ACLProtoNodeDecorator.groovy │ │ │ ├── AuthorizableProtoNodeDecorator.groovy │ │ │ ├── DefaultProtoNodeDecorator.groovy │ │ │ ├── JCRNodeDecorator.groovy │ │ │ ├── JcrPropertyDecorator.groovy │ │ │ ├── JcrUtil.groovy │ │ │ ├── ProtoNodeDecorator.groovy │ │ │ └── ProtoPropertyDecorator.groovy │ │ │ ├── resources │ │ │ ├── CleanJobRepositoryResource.groovy │ │ │ ├── ContentResource.groovy │ │ │ ├── GrabbitResourceProvider.groovy │ │ │ ├── JobResource.groovy │ │ │ ├── RootResource.groovy │ │ │ └── TransactionResource.groovy │ │ │ ├── security │ │ │ ├── AuthorizablePrincipal.groovy │ │ │ └── InsufficientGrabbitPrivilegeException.groovy │ │ │ ├── server │ │ │ ├── GrabbitContentPullServlet.groovy │ │ │ ├── batch │ │ │ │ ├── ServerBatchJob.groovy │ │ │ │ ├── ServerBatchJobContext.groovy │ │ │ │ ├── ServerBatchJobExecutionListener.groovy │ │ │ │ └── steps │ │ │ │ │ ├── jcrnodes │ │ │ │ │ ├── JcrNodesProcessor.groovy │ │ │ │ │ ├── JcrNodesReader.groovy │ │ │ │ │ └── JcrNodesWriter.groovy │ │ │ │ │ └── namespace │ │ │ │ │ ├── NamespaceProcessor.groovy │ │ │ │ │ ├── NamespaceReader.groovy │ │ │ │ │ └── NamespaceWriter.groovy │ │ │ └── services │ │ │ │ ├── ExcludePathNodeIterator.groovy │ │ │ │ ├── RootNodeWithMandatoryIterator.groovy │ │ │ │ ├── ServerService.groovy │ │ │ │ └── impl │ │ │ │ └── DefaultServerService.groovy │ │ │ ├── spring │ │ │ └── batch │ │ │ │ └── repository │ │ │ │ ├── AbstractJcrDao.groovy │ │ │ │ ├── GrabbitExecutionContextDao.groovy │ │ │ │ ├── GrabbitJobExecution.groovy │ │ │ │ ├── GrabbitJobExecutionDao.groovy │ │ │ │ ├── GrabbitJobInstanceDao.groovy │ │ │ │ ├── GrabbitStepExecutionDao.groovy │ │ │ │ ├── JcrGrabbitExecutionContextDao.groovy │ │ │ │ ├── JcrGrabbitJobExecutionDao.groovy │ │ │ │ ├── JcrGrabbitJobInstanceDao.groovy │ │ │ │ ├── JcrGrabbitStepExecutionDao.groovy │ │ │ │ ├── JcrJobExplorerFactoryBean.groovy │ │ │ │ ├── JcrJobRepositoryFactoryBean.groovy │ │ │ │ ├── services │ │ │ │ ├── CleanJobRepository.groovy │ │ │ │ └── impl │ │ │ │ │ └── DefaultCleanJobRepository.groovy │ │ │ │ └── servlets │ │ │ │ └── GrabbitCleanJobRepositoryServlet.groovy │ │ │ └── util │ │ │ └── CryptoUtil.groovy │ ├── proto │ │ ├── namespace.proto │ │ └── node.proto │ └── resources │ │ ├── META-INF │ │ └── spring │ │ │ ├── client-batch-http.xml │ │ │ ├── client-batch-job-context.xml │ │ │ ├── client-batch-job.xml │ │ │ ├── client-deleteBeforeWrite-steps.xml │ │ │ ├── client-jcrNodes-step.xml │ │ │ ├── client-namespace-step.xml │ │ │ ├── client-osgi-config.xml │ │ │ ├── client-validateJob.xml │ │ │ ├── client-workflow-off-step.xml │ │ │ ├── client-workflow-on-step.xml │ │ │ ├── server-batch-job-context.xml │ │ │ ├── server-batch-job.xml │ │ │ ├── server-jcrNodes-step.xml │ │ │ └── server-namespace-step.xml │ │ └── com │ │ └── twcable │ │ └── grabbit │ │ └── client │ │ └── servlets │ │ ├── GrabbitTransactionResource-BadRequest.txt │ │ ├── GrabbitTransactionResource-all.txt │ │ └── RootResource.txt └── test │ ├── groovy │ └── com │ │ └── twcable │ │ └── grabbit │ │ ├── ClientJobStatusSpec.groovy │ │ ├── client │ │ ├── GrabbitConfigurationSpec.groovy │ │ ├── GrabbitJobServletSpec.groovy │ │ ├── batch │ │ │ ├── ClientBatchJobContextSpec.groovy │ │ │ ├── ClientBatchJobSpec.groovy │ │ │ ├── steps │ │ │ │ ├── http │ │ │ │ │ └── CreateHttpConnectionTaskletSpec.groovy │ │ │ │ ├── jcrnodes │ │ │ │ │ └── JcrNodesWriterSpec.groovy │ │ │ │ ├── namespace │ │ │ │ │ └── NamespaceSyncTaskletSpec.groovy │ │ │ │ ├── validation │ │ │ │ │ └── ValidJobDeciderSpec.groovy │ │ │ │ └── workspace │ │ │ │ │ ├── DeleteBeforeWriteDeciderSpec.groovy │ │ │ │ │ └── DeleteBeforeWriteTaskletSpec.groovy │ │ │ └── workflows │ │ │ │ ├── StubWorkflowLauncher.groovy │ │ │ │ └── WorkflowManagerSpec.groovy │ │ └── servlets │ │ │ ├── GrabbitRootServletSpec.groovy │ │ │ └── GrabbitTransactionServletSpec.groovy │ │ ├── jcr │ │ ├── ACLProtoNodeDecoratorSpec.groovy │ │ ├── AuthorizableProtoNodeDecoratorSpec.groovy │ │ ├── DefaultProtoNodeDecoratorSpec.groovy │ │ ├── JCRNodeDecoratorSpec.groovy │ │ ├── JcrPropertyDecoratorSpec.groovy │ │ ├── ProtoMock.groovy │ │ ├── ProtoNodeDecoratorSpec.groovy │ │ └── ProtoPropertyDecoratorSpec.groovy │ │ ├── resources │ │ ├── ContentResourceSpec.groovy │ │ ├── GrabbitResourceProviderSpec.groovy │ │ ├── JobResourceSpec.groovy │ │ └── TransactionResourceSpec.groovy │ │ ├── security │ │ └── AuthorizablePrincipalSpec.groovy │ │ ├── server │ │ ├── GrabbitContentPullServletSpec.groovy │ │ ├── batch │ │ │ └── steps │ │ │ │ └── jcrnodes │ │ │ │ └── JcrNodesProcessorSpec.groovy │ │ └── services │ │ │ ├── ExcludePathNodeIteratorSpec.groovy │ │ │ └── RootNodeWithMandatoryIteratorSpec.groovy │ │ ├── spring │ │ └── batch │ │ │ └── repository │ │ │ ├── JcrGrabbitExecutionContextDaoSpec.groovy │ │ │ ├── JcrGrabbitJobExecutionDaoSpec.groovy │ │ │ ├── JcrGrabbitJobInstanceDaoSpec.groovy │ │ │ ├── JcrGrabbitStepExecutionDaoSpec.groovy │ │ │ └── servlets │ │ │ └── GrabbitCleanJobRepositoryServletSpec.groovy │ │ └── testutil │ │ └── StubInputStream.groovy │ └── resources │ └── com │ └── twcable │ └── grabbit │ └── client │ └── test_config.yaml └── testutils ├── .gitignore ├── build.gradle └── gradle └── testing.gradle /.gitattributes: -------------------------------------------------------------------------------- 1 | gradlew -crlf 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Thanks for taking the time to fill out an issue! To help the community solve your issue, please provide answers to the following questions, along with supporting log and configuration files. 2 | 3 | 4 | 5 | **What version of AEM are you running?** 6 | 7 | 8 | 9 | **How do you produce the issue?** 10 | 11 | 12 | 13 | **Does the issue occur consistently?** 14 | 15 | 16 | 17 | **Any additional details?** 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.vlt 2 | *.tmp 3 | .DS_Store 4 | *.iml 5 | *.ipr 6 | *.iws 7 | *.jar 8 | .gradle 9 | .gradletasknamecache 10 | .project 11 | .settings 12 | .idea 13 | .classpath 14 | .metadata/ 15 | dist/ 16 | atlassian-ide-plugin.xml 17 | install/ 18 | out/ 19 | build/ 20 | bin/ 21 | .cache 22 | test-output/ 23 | classes/ 24 | generated/ 25 | .vagrant/ 26 | puppet/r10k 27 | target/ 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: groovy 2 | 3 | #Yeah, we are going to want to cache our build dependencies so our builds aren't held up downloading for every container 4 | cache: 5 | directories: 6 | - $HOME/.gradle 7 | - $HOME/protobuf 8 | 9 | dist: trusty 10 | 11 | jdk: 12 | - oraclejdk8 13 | - openjdk8 14 | 15 | # Allow running on a container instead of a traditional VM 16 | sudo: false 17 | 18 | #There is currently an issue in Gradle where a forked JVM will ignore any no-daemon flag (--no-daemon, org.gradle.daemon) for the forked process if jvm args are supplied. 19 | #We remove them to get around this 20 | env: 21 | global: 22 | - GRADLE_OPTS="-Dorg.gradle.daemon=false -Dorg.gradle.jvmargs=''" 23 | - PATH="$PATH:$HOME/protobuf/bin" 24 | 25 | before_install: 26 | - bash install_protoc_from_source.sh 27 | install: ./gradlew assemble 28 | script: ./gradlew check 29 | 30 | 31 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Grabbit 2 | 3 | If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request. 4 | 5 | When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. 6 | 7 | ## License 8 | 9 | By contributing your code, you agree to license your contribution under the terms of the APLv2: [LICENSE](LICENSE.txt) 10 | 11 | All files are released with the Apache 2.0 license. 12 | 13 | If you are adding a new file it should have a header like this: 14 | 15 | ``` 16 | /** 17 | * Copyright 2015 Time Warner Cable, Inc. 18 | * 19 | * Licensed under the Apache License, Version 2.0 (the "License"); 20 | * you may not use this file except in compliance with the License. 21 | * You may obtain a copy of the License at 22 | * 23 | * http://www.apache.org/licenses/LICENSE-2.0 24 | * 25 | * Unless required by applicable law or agreed to in writing, software 26 | * distributed under the License is distributed on an "AS IS" BASIS, 27 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 28 | * See the License for the specific language governing permissions and 29 | * limitations under the License. 30 | */ 31 | ``` 32 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | = Grabbit 2 | :docsDir: docs 3 | 4 | image:https://travis-ci.org/TWCable/grabbit.svg?branch=master[title = "Build Status", link = "https://travis-ci.org/TWCable/grabbit"] image:https://badge.waffle.io/TWCable/grabbit.png?label=ready&title=Ready[title = "Stories in Ready", link = "https://waffle.io/TWCable/grabbit"] 5 | 6 | image:https://api.bintray.com/packages/twcable/aem/Grabbit/images/download.svg[title = "Download", link = "https://bintray.com/twcable/aem/Grabbit/_latestVersion"] 7 | 8 | 9 | == Project Purpose 10 | 11 | The purpose of Grabbit is to provide a fast and reliable way of copying content from one Sling (specifically Adobe CQ/AEM) instance to another. 12 | 13 | Existing solutions have been tried and found insufficient for very large data sets (GB-TB), especially over a network. CQ's .zip packages are extremely space inefficient, causing a lot of extra I/O. http://jackrabbit.apache.org/filevault/usage.html[`vlt rcp`] and Mark Adamcin's http://adamcin.net/net.adamcin.recap/[`recap`] use essentially the same mechanism: WebDAV using XML, doing an HTTP handshake for every node and many sets of properties, which means that any latency whatsoever on the network hurts performance enormously. 14 | 15 | Grabbit creates a stream of data using https://developers.google.com/protocol-buffers/[Google's Protocol Buffers] aka "ProtoBuf". Protocol Buffers are an extremely efficient (in terms of CPU, memory and wire size) binary protocol that includes compression. 16 | 17 | Moreover, by doing a continuous stream, we avoid the latency issues. Depending on the size and nature of the data, as well as network latency, we have so far seen speed improvements ranging from 2 to 10 times that of Recap/vlt. 18 | 19 | NOTE: "Grabbit" obviously refers to this "grabbing" content from one CQ/AEM instance and copying it to another. However it also refers to "Jackrabbit," the reference JCR implementation that the content is being copied to and from. 20 | 21 | image:https://api.bintray.com/packages/twcable/aem/Grabbit/images/download.svg[title = "Download", link = "https://bintray.com/twcable/aem/Grabbit/_latestVersion"] 22 | 23 | 24 | == Table of Contents 25 | 26 | * General Information 27 | 28 | ** link:{docsDir}/RELEASE_NOTES.md[Releases] 29 | ** link:{docsDir}/AEMSupport.adoc[Installation & Version Support] 30 | 31 | * Using Grabbit 32 | 33 | ** link:{docsDir}/GeneralLayout.adoc[General Layout] 34 | ** link:{docsDir}/Running.adoc[Running] 35 | ** link:{docsDir}/Monitoring.adoc[Monitoring / Validating the Content Sync] 36 | ** link:{docsDir}/Cleaning.adoc[Cleaning Grabbit Job Repository] 37 | 38 | * Grabbit Development 39 | 40 | ** link:{docsDir}/GettingStarted.adoc[Getting Started] 41 | ** link:{docsDir}/Building.adoc[Building from Source] 42 | ** link:{docsDir}/RELEASING.adoc[Releasing A New Version] 43 | 44 | * link:{docsDir}/LibraryAttribution.adoc[Library Attribution] 45 | 46 | * link:{docsDir}/LicenseInfo.adoc[License] 47 | -------------------------------------------------------------------------------- /assets/grabbitConnection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/assets/grabbitConnection.png -------------------------------------------------------------------------------- /assets/jobKickedOff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/assets/jobKickedOff.png -------------------------------------------------------------------------------- /assets/jobStatus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/assets/jobStatus.png -------------------------------------------------------------------------------- /assets/monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/assets/monitor.png -------------------------------------------------------------------------------- /assets/newJob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/assets/newJob.png -------------------------------------------------------------------------------- /buildSrc/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'groovy' 2 | 3 | dependencies { 4 | testCompile('org.spockframework:spock-core:1.0-groovy-2.4') { 5 | exclude module: 'groovy-all' 6 | } 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | -------------------------------------------------------------------------------- /buildSrc/src/main/groovy/com/twcable/grabbit/Version.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit 18 | 19 | // ************************************************************************** 20 | // 21 | // VERSION CLASS 22 | // 23 | // ************************************************************************** 24 | 25 | 26 | class Version { 27 | String originalVersion 28 | String thisVersion 29 | String status 30 | Date buildTime 31 | 32 | 33 | Version(String versionValue) { 34 | buildTime = new Date() 35 | originalVersion = versionValue 36 | if (originalVersion.endsWith('-SNAPSHOT')) { 37 | status = 'integration' 38 | thisVersion = originalVersion - 'SNAPSHOT' + getTimestamp() 39 | } 40 | else { 41 | status = 'release' 42 | thisVersion = versionValue 43 | } 44 | } 45 | 46 | 47 | @SuppressWarnings("UnnecessaryQualifiedReference") 48 | String getTimestamp() { 49 | // Convert local file timestamp to UTC 50 | def format = new java.text.SimpleDateFormat('yyyyMMddHHmmss') 51 | format.setCalendar(Calendar.getInstance(TimeZone.getTimeZone('UTC'))); 52 | return format.format(buildTime) 53 | } 54 | 55 | 56 | String toString() { 57 | originalVersion 58 | } 59 | 60 | 61 | String getBintrayVersion() { 62 | thisVersion 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /buildSrc/src/test/groovy/com/twcable/grabbit/VersionSpec.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit 18 | 19 | import spock.lang.Specification 20 | 21 | class VersionSpec extends Specification { 22 | 23 | def "version When Snapshot"(){ 24 | given: 25 | final String ver = "Test-SNAPSHOT" 26 | when: 27 | Version version = new Version(ver) 28 | then: 29 | version.thisVersion == ver - "SNAPSHOT" + version.getTimestamp() 30 | } 31 | 32 | def "default Version"(){ 33 | given: 34 | final String ver = "1.0.5" 35 | when: 36 | Version version = new Version(ver) 37 | then: 38 | version.thisVersion == ver 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /docs/AEMSupport.adoc: -------------------------------------------------------------------------------- 1 | == What you need to run Grabbit. 2 | 3 | * Supported AEM/CQ version per below 4 | * One-time installation of a system Fragment Bundle is needed. It can be found link:https://bintray.com/artifact/download/twcable/aem/dependencies/Sun-Misc-Fragment-Bundle-1.0.0.zip[here] 5 | * If running AEM 6.2+ with the AEM Deserialization Firewall enabled, link:https://bintray.com/twcable/aem/download_file?file_path=dependencies%2FGrabbit-Deserialization-Firewall-Configuration-1.0.zip[this package] is required in order 6 | for Grabbit to work. If running the deserialization firewall with custom blacklist/whitelist rules, check out the package description to gather which additional paths will need to be considered. 7 | * In Addition to the AEM Deserialization Firewall package, you would need to install link:https://bintray.com/artifact/download/twcable/aem/dependencies/Grabbit-Apache-Sling-Login-Whitelist-1.0.zip[this] to have the grabbit bundle be whitelisted with Apache Sling Login Admin. 8 | 9 | == Below details AEM version support for the various releases of Grabbit. 10 | ``` 11 | v7.x - AEM 6.1 and AEM 6.2 and AEM 6.3 12 | v5.x - AEM 6.1 13 | v4.x - AEM 6.1 14 | v3.x - CQ 5.6 and AEM 6.0 15 | v2.x - CQ 5.6 16 | ``` 17 | We follow link:http://semver.org/[semantic versioning]. Installation of different major versions between machines are unsupported, and likely incompatible. 18 | 19 | Active development is on the "master" branch. Security patches and the like are sometimes back-ported to prior versions. 20 | 21 | Of course pull-requests are happily accepted for those that would like to submit things like back-porting features for AEM 5.6, etc. 22 | 23 | image:https://api.bintray.com/packages/twcable/aem/Grabbit/images/download.svg[title = "Download", link = "https://bintray.com/twcable/aem/Grabbit/_latestVersion"] 24 | -------------------------------------------------------------------------------- /docs/Building.adoc: -------------------------------------------------------------------------------- 1 | == Building From Source 2 | 3 | Once you have cloned the project, run following command(from project root directory) to build and upload the Grabbit to your AEM instance 4 | `gradlew clean build install refreshAllBundles` -------------------------------------------------------------------------------- /docs/Cleaning.adoc: -------------------------------------------------------------------------------- 1 | == Cleaning Grabbit Job Repository 2 | 3 | Over time, the link:grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository[Grabbit Job Repository] which is stored under `/var/grabbit` grows and without pruning the data that it is storing, the performance of the Grabbit Status API degrades slowly. 4 | In order to manage that, there is a `Clean Repository` API available that allows you to delete any Grabbit job history that is older than `X` hours from the time the API was hit. 5 | 6 | The usage is : 7 | 8 | ``` 9 | POST /grabbit/jobrepository/clean 10 | Parameter: hours=X 11 | 12 | where X is the number of hours. 13 | ``` 14 | 15 | This API will delete all the link:grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobInstanceDao.groovy[JobInstances], link:grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobExecutionDao.groovy[JobExecutions], link:grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitStepExecutionDao.groovy[StepExecutions] and link:grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitExecutionContextDao.groovy[ExecutionContexts] that were either `COMPLETED` or `FAILED` X hours ago from `now`. 16 | -------------------------------------------------------------------------------- /docs/GeneralLayout.adoc: -------------------------------------------------------------------------------- 1 | == General Layout 2 | 3 | Any server with Grabbit installed acts as a Grabbit peer that can send, or receive content to another Grabbit peer. 4 | 5 | To pull content into a server, a new job needs to be created on the receiving server. To do this, using the RESTful API exposed by Grabbit, PUT /grabbit/job with configuration specifying server to pull from, paths to pull, etc. This is outlined in more detail at link:Running.adoc[Running Grabbit] 6 | 7 | A recommended systems layout style is to have all content from a production publisher copied down to a staging "data warehouse" (DW) server to which all lower environments (beta, continuous integration, developer workstations, etc.) will connect. This way minimal load is placed on Production, and additional DW machines can be added to scale out if needed, each of which can grab from the "main" DW. 8 | 9 | -------------------------------------------------------------------------------- /docs/GettingStarted.adoc: -------------------------------------------------------------------------------- 1 | == Getting Started 2 | 3 | The JCR 2.0 Specification (JSR-283) is included under docs for quick reference 4 | 5 | == Prerequisites 6 | 7 | === Installing Protocol Buffers Compiler 8 | 9 | ==== For Windows 10 | The easiest way to install the compiler is to install the binary from https://github.com/google/protobuf/releases/download/v2.6.1/protoc-2.6.1-win32.zip and then set the `/path/to/protoc/parent` on your PATH variable. 11 | 12 | ==== For Macs 13 | 14 | Run ./install_protoc_from_source.sh located at the root of this project. This will compile the protoc compiler 15 | 16 | _For both Windows and Mac : To verify that installation was successful, `protoc --version` should display `2.6.1`_ 17 | -------------------------------------------------------------------------------- /docs/LibraryAttribution.adoc: -------------------------------------------------------------------------------- 1 | == Library Attribution 2 | 3 | * http://groovy.codehaus.org/Download[Groovy v2.3.6] 4 | * https://code.google.com/p/protobuf/downloads/list[Google Protocol Buffers v2.4.1] - The compiler and runtime library is used for Serialization and De-serialization of Data 5 | * http://docs.spring.io/spring-batch/2.2.x/downloads.html[Spring Batch v2.2.7.RELEASE] - It is used on the server and client to read/write, marshal/unmarshall and send/receive the data to client in a controlled manner. 6 | * https://bintray.com/twcable/aem/jackalope/2.0.0/view[] - Jackalope is used for testing 7 | * https://bintray.com/twcable/aem/cq-gradle-plugins/2.0.1/view[CQ Gradle Plugins v2.0.1] : They provide Gradle build support. 8 | * http://search.maven.org/#artifactdetails%7Cws.antonov.gradle.plugins%7Cgradle-plugin-protobuf%7C0.9.1%7Cjar[Gradle Protocol Buffers Plugin v0.9.1] - It provides easy integration of the ProtoBuf compiler with Gradle. 9 | -------------------------------------------------------------------------------- /docs/LicenseInfo.adoc: -------------------------------------------------------------------------------- 1 | == License 2 | 3 | Copyright 2015 Time Warner Cable, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance 6 | with the License. You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 11 | an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for 12 | the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- /docs/Monitoring.adoc: -------------------------------------------------------------------------------- 1 | == Monitoring/Validation Content Sync 2 | 3 | You may choose to use grabbit.sh to monitor your sync, or you can validate/monitor your sync by going to the following URIs: 4 | 5 | `/grabbit/job/` 6 | `/grabbit/transaction/` 7 | 8 | A job status has the following format : 9 | 10 | ```json 11 | { 12 | "endTime": "Timestamp", 13 | "exitStatus": { 14 | "exitCode": "Code", 15 | "exitDescription": "", 16 | "running": "true/false" 17 | }, 18 | "jcrNodesWritten": "#OfNodes", 19 | "jobExecutionId": "JobExecutionId", 20 | "path": "currentPath", 21 | "startTime": "TimeStamp", 22 | "timeTaken": "TimeInMilliSeconds", 23 | "transactionId": "transactionId" 24 | } 25 | ``` 26 | 27 | Couple of points worth noting here: 28 | `"exitCode"` can have 4 states - `UNKNOWN`, `COMPLETED`, `FAILED`, or `VALIDATION_FAILED`. `UNKNOWN` means the job is still running. `COMPLETED` means that the job was completed successfully. `FAILED` means the job failed. `VALIDATION_FAILED` means the job was aborted due to configuration; This could mean that although the configuration was valid, Grabbit refused to sync a path - for e.g, a non-existing parent path. Grabbit will not implicitly write parent nodes. 29 | `"jcrNodesWritten"` : This indicates how many nodes are currently written (increments by 1000) 30 | `"timeTaken"` : This will indicate the total time taken to complete content grab for `currentPath` 31 | 32 | If `exitCode` returns as `UNKNOWN`, that means the job is still running and you should check for its status again. 33 | 34 | 35 | __Sample of a real Grabbit Job status__ 36 | 37 | image::../assets/jobStatus.png[Job Status] 38 | 39 | Two loggers are predefined for Grabbit. One detailing content receive operations, another for content push operations. 40 | They are link:../src/main/content/SLING-INF/content/apps/grabbit/config/org.apache.sling.commons.log.LogManager.factory.config-com.twcable.grabbit.send.xml[grabbit-send.log] and link:../src/main/content/SLING-INF/content/apps/grabbit/config/org.apache.sling.commons.log.LogManager.factory.config-com.twcable.grabbit.receive.xml[grabbit-receive.log] respectively. 41 | These log files are for anything logged in **com.twcable.grabbit.server** and **com.twcable.grabbit.client** packages. 42 | 43 | If you want to see what nodes are being written to a receiving server, change the logging for `grabbit-receive.log` above to `DEBUG` or `TRACE`. 44 | 45 | -------------------------------------------------------------------------------- /docs/RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | # RELEASE NOTES 2 | 3 | [ ![Download](https://api.bintray.com/packages/twcable/aem/Grabbit/images/download.svg) ](https://bintray.com/twcable/aem/Grabbit/_latestVersion) 4 | 5 | ## 7.1.5 6 | 7 | * Bug fixes (Fixes #2) 8 | 9 | ## 7.1.4 10 | 11 | * Bug fixes 12 | 13 | ## 7.1.3 14 | 15 | * Bug fixes (Fixes #192) 16 | 17 | ## 7.1.2 18 | 19 | * Bug fixes 20 | 21 | ## 7.1.1 22 | * Bug fixes 23 | * Documentation changes 24 | * Log names are now changed from batch-client.log to grabbit-receive.log; and batch-server.log to grabbit-send.log 25 | 26 | ## 7.1.0 27 | * Support for writing rep:policy nodes 28 | * Bug fixes 29 | * Performance improvements 30 | 31 | ## 7.0.2 32 | * Bug fixes 33 | 34 | ## 7.0.1 35 | * Add https sync support (fix #149) 36 | 37 | ## 7.0.0 38 | 39 | * Refactor JcrNodesProcessor to use JcrPropertyDecorator 40 | * Provide Root ResourceProvider for Grabbit 41 | * Upgrade joda-time dependency to 2.7 and update MANIFEST.MF to accept [2,3) for AEM 6.1 & AEM 6.2 compatibility 42 | * Added Content-Type header in curl request in grabbit.sh for it to work in AEM 6.2 43 | * Transfer current property's state whether it's multiple or not from Server to Client 44 | 45 | ## 5.0.1 46 | 47 | * Adds feature that provides a way to delete JcrJobRepository that is older than X hours from "now" 48 | * The API is : POST /grabbit/jobrepository/clean --data hours=X 49 | 50 | ## 5.0.0 51 | 52 | * Added transaction support 53 | * New resource modeling, with new RESTful endpoints 54 | * Some misc refactoring 55 | * Additional specifications for missing code coverage 56 | 57 | ## 4.1.1 58 | 59 | * Fixed concurrency issues with the Workflow Manager. Miscellaneous Workflow Manager refactoring 60 | * Trivial README updates 61 | 62 | ## 4.1.0 63 | 64 | * Job paths are now validated as to not write 'dirty data' 65 | * Small performance improvements 66 | * Improved shell 67 | * Small fixes 68 | 69 | ## 4.0.7 70 | 71 | * Added YAML support. Backwards compatible to existing JSON format 72 | * Use AuthCache for Apache HttpClient to do preemptive authentication 73 | 74 | ## 4.0.6 75 | 76 | * Update Grabbit 4.0.x build so that OOTB/Provided dependencies are properly excluded 77 | 78 | ## 4.0.5 79 | 80 | * Updated README to remove instructions around workflow bundle 81 | * Use earlier version of AEM workflow so that CI jobs can complete. Issue #63 82 | * Removed some unnecessary files : karama.gradle, reporting.gradle 83 | * Clarified versioning and compatibility in README 84 | * Small changes to README 85 | 86 | ## 4.0.4 87 | 88 | * Finished implementing "delta copy" functionality 89 | 90 | ## 4.0.3 91 | 92 | * Adding DEBUG logging for nodes being saved on Client 93 | 94 | ## 4.0.2 95 | 96 | * Bug fixes 97 | 98 | ## 4.0.1 99 | 100 | * Updated Client and Server to use request credentials as login credentials when creating a JCR session (GH-56) 101 | 102 | ## 4.0.0 103 | 104 | * Updated for AEM 6.1 105 | 106 | ## 3.0.0 107 | 108 | * Added "excludePath" feature ( GH-23 ) 109 | * Added "Delete before write" functionality ( GH-20 ) 110 | * Fixed issue with not writing mandatory child nodes with their parent ( GH-34 ) 111 | * A number of smaller bug fixes and refactoring 112 | 113 | ## 2.0.1 114 | 115 | * Removed an unnecessary bundle dependency on SCR and downgraded guava to 15 to make AEM6 happy 116 | * Removed the broken client UI code (#13) 117 | 118 | ## 2.0.0 119 | 120 | Initial public release 121 | -------------------------------------------------------------------------------- /docs/RELEASING.adoc: -------------------------------------------------------------------------------- 1 | == Releasing A New Version 2 | 3 | === Update Files 4 | 5 | . Update the `version` property in link:../gradle.properties[`gradle.properties`]. We follow http://semver.org/spec/v2.0.0.html[Semantic Versioning 2.0.0], so increment accordingly 6 | 7 | . Update link:./RELEASE_NOTES.md[RELEASE_NOTES.md] with what has changed. Assuming the commit messages have been written well, it will generally be the subject lines of the commits since the last release, but edit (including adding) as appropriate for a human to easily understand what has changed 8 | 9 | === Create A Clean Build 10 | 11 | [source,bash] 12 | -- 13 | $ ./gradlew clean check integrationTestReport createPackage 14 | -- 15 | 16 | === Commit and Push 17 | 18 | [source,bash] 19 | -- 20 | $ git add gradle.properties docs/RELEASE_NOTES.md 21 | $ git commit -m v9.9.9 # "v" followed by the version number 22 | $ git tag v9.9.9 23 | $ git push origin HEAD:master 24 | $ git push origin v9.9.9 25 | -- 26 | 27 | Create a https://github.com/TWCable/grabbit/releases/new[new GitHub release], selecting the just-uploaded tag from the list. There's no need for a "`Release title`", though copy & paste the latest version information from RELEASE_NOTES.md into the "`Describe this release`" box. If you want you can attach the built artifact to the GitHub release, but the "`official`" location is https://bintray.com/twcable/aem/Grabbit/view[BinTray]. 28 | 29 | *NOTE*: If you are back-porting a release or the like, GitHub's "`release`" feature is extremely simplistic and always labels most recently create "release" record as being the "`Latest release`". 30 | 31 | 32 | === Upload to BinTray 33 | 34 | [source,bash] 35 | -- 36 | $ ./gradlew bintrayUpload 37 | -- 38 | 39 | If you have not yet set up your credentials, the task will tell you what to do. 40 | -------------------------------------------------------------------------------- /docs/jcr-spec.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/docs/jcr-spec.pdf -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # suppress inspection "UnusedProperty" for whole file 2 | org.gradle.daemon = true 3 | org.gradle.jvmargs=-XX:MaxPermSize=512m -XX:+CMSClassUnloadingEnabled -XX:+CMSPermGenSweepingEnabled -XX:+HeapDumpOnOutOfMemoryError -Xmx1024m -Dfile.encoding=utf-8 4 | 5 | bundleInstallRoot = /apps/grabbit/install 6 | 7 | group = com.twcable.grabbit 8 | version = 7.1.5 9 | 10 | # Please keep alphabetical 11 | cglib_nodep_version = 2.2.2 12 | commons_collections_version = 3.2.1 13 | commons_io_version = 1.4 14 | commons_lang_version = 3.3.2 15 | commons_text_version = 1.1.8 16 | 17 | # Currently using an "older" version of cq_workflow_console to compile locally since this is the latest version that is available publicly. 18 | # Fortunately, the later version used on 6.1 does not have any API incompatibilities with this version 19 | cq_workflow_console_version = 5.6.6 20 | 21 | felix_osgi_version = 1.4.0 22 | gradle_plugins_version = 3.0.1 23 | granite_version = 5.12.2 24 | groovy_version = 2.3.6 25 | guava_version = 15.0 26 | jackalope_version = 2.0.0 27 | jackrabbit_ocm_version = 1.5.3 28 | jackrabbit_version = 2.10.0 29 | jcr_version = 2.0 30 | jms_version = 3.1.1 31 | jsr305_version = 2.0.0 32 | logback_version = 1.0.4 33 | oak_version = 1.2.2 34 | objenesis_version = 2.1 35 | okhttp_version = 3.5.0_1 36 | okio_version = 1.11.0_1 37 | protobuf_gradle_plugin_version = 0.9.1 38 | protobuf_version = 2.6.1 39 | scr_annotations_version = 1.7.0 40 | servlet_api_version = 2.5 41 | slf4j_version = 1.7.6 42 | sling_api_version = 2.9.0 43 | sling_base_version = 2.2.2 44 | sling_commons_testing_version = 2.0.12 45 | sling_commons_version = 2.2.0 46 | sling_event_version = 3.1.4 47 | sling_jcr_api_version = 2.5.0 48 | sling_jcr_resource_version = 2.5.0 49 | sling_resourceresolver_version = 1.0.6 50 | sling_rewriter_version = 1.0.4 51 | spock_version = 0.7-groovy-2.0 52 | spring_batch_version = 2.2.7.RELEASE 53 | spring_osgi_version = 2.0.0.M1 54 | woodstox_version = 4.2.0 55 | snakeyaml_version = 1.16 56 | -------------------------------------------------------------------------------- /gradle/bundle.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'cqpackage' 2 | 3 | ext.packageName = project.name 4 | ext.bundleName = project.name 5 | ext.bundleDescription = 'Grabbit' 6 | ext.symbolicName = "com.twcable.grabbit" 7 | jar.baseName = 'grabbit' 8 | 9 | createPackage.bundleInstallRoot = '/apps/grabbit/install' 10 | 11 | bundle.installPath = '/apps/grabbit/install' 12 | 13 | jar { 14 | from('src/main/content/SLING-INF/content', { 15 | into 'SLING-INF/content' 16 | }) 17 | } 18 | 19 | verifyBundles.dependsOn 'jar' 20 | 21 | jar.manifest { 22 | 23 | attributes 'Sling-Initial-Content': project.configFolders.collect { 24 | "SLING-INF/content/apps/grabbit/${it}; overwrite:=true; uninstall:=true; path:=/apps/grabbit/${it}" 25 | }.join(', ') 26 | 27 | attributes 'Bundle-Name': project.bundleName 28 | attributes 'Bundle-SymbolicName': project.symbolicName 29 | attributes 'Bundle-Description': project.bundleDescription 30 | instruction 'Import-Package', 'groovy.json; version="[2.3,3.0)"' 31 | instruction 'Import-Package', 'groovy.json.internal; version="[2.3,3.0)"' 32 | instruction 'Import-Package', 'org.springframework.batch.core.scope; version="[2.2,3.0)"' 33 | instruction 'Import-Package', 'org.springframework.batch.item.file.transform; version="[2.2,3.0)"' 34 | instruction 'Import-Package', 'org.springframework.batch.item.file; version="[2.2,3.0)"' 35 | instruction 'Import-Package', 'org.springframework.batch.item.support; version="[2.2,3.0)"' 36 | instruction 'Import-Package', 'org.springframework.batch.core.configuration.xml; version="[2.2,3.0)"' 37 | instruction 'Import-Package', 'org.springframework.batch.core.configuration.support; version="[2.2,3.0)"' 38 | instruction 'Import-Package', 'org.springframework.batch.core.explore.support; version="[2.2,3.0)"' 39 | instruction 'Import-Package', 'org.springframework.batch.core.repository.support; version="[2.2,3.0)"' 40 | instruction 'Import-Package', 'org.springframework.batch.core.repository; version="[2.2,3.0)"' 41 | instruction 'Import-Package', 'org.springframework.batch.core.launch.support; version="[2.2,3.0)"' 42 | instruction 'Import-Package', 'org.springframework.batch.core.job.flow.support; version="[2.2,3.0)"' 43 | instruction 'Import-Package', 'org.springframework.batch.core.job.flow.support.state; version="[2.2,3.0)"' 44 | instruction 'Import-Package', 'org.springframework.batch.core.repository.dao; version="[2.2,3.0)"' 45 | instruction 'Import-Package', 'org.springframework.batch.support.transaction; version="[2.2,3.0)"' 46 | instruction 'Import-Package', 'org.springframework.beans.factory.config; version="[3.1,4.0)"' 47 | instruction 'Import-Package', 'org.springframework.core.io; version="[3.1,4.0)"' 48 | instruction 'Import-Package', 'org.springframework.core.task; version="[3.1,4.0)"' 49 | instruction 'Import-Package', 'org.springframework.aop; version="[3.1,4.0)"' 50 | instruction 'Import-Package', 'org.springframework.aop.framework; version="[3.1,4.0)"' 51 | instruction 'Import-Package', 'org.springframework.aop.scope; version="[3.1,4.0)"' 52 | instruction 'Import-Package', 'org.springframework.scheduling.concurrent; version="[3.1,4.0)"' 53 | instruction 'Import-Package', "org.aopalliance.aop;version=1.0.0" 54 | instruction 'Import-Package', "sun.misc" 55 | instruction 'Import-Package', '*' 56 | 57 | // export everything to OSGi except *.impl packages 58 | instruction 'Export-Package', "!*.impl, *;-noimport:=false;version=${version}" 59 | } 60 | 61 | bundle.installPath = '/apps/grabbit/install/${nativeStartLevel}' 62 | -------------------------------------------------------------------------------- /gradle/idea.gradle: -------------------------------------------------------------------------------- 1 | idea { 2 | module { 3 | // The whole build dir is excluded by default, but we need build/generated-sources, 4 | // which contains the generated proto classes. 5 | excludeDirs = [ 6 | file("$buildDir/classes"), 7 | file("$buildDir/docs"), 8 | file("$buildDir/dependency-cache"), 9 | file("$buildDir/libs"), 10 | file("$buildDir/reports"), 11 | file("$buildDir/resources"), 12 | file("$buildDir/test-results"), 13 | file("$buildDir/tmp"), 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /gradle/packageExclusions.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'cqpackage' 2 | // exclude bundles provided by CQ 3 | configurations.cq_package { 4 | exclude group: 'javax.servlet', module: 'servlet-api' 5 | exclude group: 'commons-logging', module: 'commons-logging' 6 | exclude group: 'com.sun.xml.bind' 7 | exclude group: 'aopalliance', module: 'aopalliance' 8 | exclude group: 'javax.xml.stream' 9 | exclude group: 'javax.jcr', module: 'jcr' 10 | 11 | // use AEM's logging, not logback 12 | exclude group: 'ch.qos.logback', module: 'logback-classic' 13 | exclude group: 'ch.qos.logback', module: 'logback-core' 14 | 15 | // Slf4j is provided by AEM 16 | exclude group: 'org.slf4j', module: 'slf4j-api' 17 | exclude group: 'org.slf4j', module: 'jcl-over-slf4j' 18 | 19 | // excluding Joda Time too as AEM provides its own in com.day.commons.osgi.wrapper.joda-time 20 | exclude group: 'joda-time', module: 'joda-time' 21 | 22 | exclude group: 'com.google.code.findbugs', module: 'jsr305' 23 | 24 | exclude group: 'com.day.cq.workflow', module: 'cq-workflow-console' 25 | 26 | exclude group: 'org.apache.felix', module: 'org.apache.felix.scr.annotations' 27 | exclude group: 'org.apache.felix', module: 'org.osgi.core' 28 | exclude group: 'org.apache.felix', module: 'org.osgi.compendium' 29 | 30 | exclude group: 'org.apache.commons', module: 'commons-lang3' 31 | exclude group: 'org.apache.jackrabbit', module: 'jackrabbit-jcr-commons' 32 | exclude group: 'org.apache.jackrabbit', module: 'jackrabbit-api' 33 | exclude group: 'org.apache.jackrabbit', module: 'oak-core' 34 | exclude group: 'commons-io', module: 'commons-io' 35 | 36 | //Exclude Apache Sling Libraries 37 | exclude group: 'org.apache.sling', module: 'org.apache.sling.api' 38 | exclude group: 'org.apache.sling', module: 'org.apache.sling.jcr.base' 39 | exclude group: 'org.apache.sling', module:'org.apache.sling.jcr.resource' 40 | exclude group: 'org.apache.sling', module: 'org.apache.sling.jcr.api' 41 | 42 | // 6.x exclude bundles 43 | exclude group: 'com.google.guava', module: 'guava' 44 | exclude module: 'cq-workflow-console' 45 | } 46 | -------------------------------------------------------------------------------- /gradle/utils.gradle: -------------------------------------------------------------------------------- 1 | task getConfigFolders { 2 | def configBaseDir = new File(project.projectDir, 'src/main/content/SLING-INF/content/apps/grabbit/') 3 | project.ext.configFolders = [] 4 | configBaseDir.eachDir { File dir -> 5 | if (dir.name ==~ /config\.?.*/) { 6 | configFolders.add(dir.name) 7 | } 8 | } 9 | } 10 | 11 | jar.dependsOn += getConfigFolders 12 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TWCable/grabbit/48a5d8f022ccd9ce3108afac16ef32356bbde317/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Jul 13 15:49:06 IST 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-3.0-all.zip 7 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /install_protoc_from_source.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | PROTOC_VERSION="2.6.1" 4 | 5 | if [[ ! -d "$HOME/protobuf/lib" || "$(protoc --version)" != *${PROTOC_VERSION}* ]]; then 6 | pushd . 7 | cd /tmp 8 | wget --no-check-certificate https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protobuf-${PROTOC_VERSION}.tar.gz 9 | tar -xzvf protobuf-${PROTOC_VERSION}.tar.gz 10 | cd protobuf-${PROTOC_VERSION} 11 | ./configure --prefix=$HOME/protobuf 12 | make 13 | make check 14 | make install 15 | cd .. 16 | rm -rf protobuf-${PROTOC_VERSION} 17 | rm -f protobuf-${PROTOC_VERSION}.zip 18 | popd 19 | echo "Add $HOME/protobuf/bin to your path!" 20 | echo "On *nix:" 21 | echo "echo 'PATH=\$PATH:~/protobuf/bin' >> ~/.bash_profile" 22 | else 23 | echo "$(protoc --version) already installed" 24 | fi 25 | -------------------------------------------------------------------------------- /sample-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "serverUsername" : "", 3 | "serverPassword" : "", 4 | "serverScheme" : "http", 5 | "serverHost" : "some.other.server", 6 | "serverPort" : "4502", 7 | "batchSize" : 150, 8 | "deltaContent" : true, 9 | "pathConfigurations" : [ 10 | { 11 | "path" : "/content/someContent", 12 | "batchSize" : 50 13 | }, 14 | { 15 | "path" : "/content/someContent", 16 | "batchSize" : 200, 17 | "excludePaths" : 18 | [ 19 | "someOtherContent/someExcludeContent" 20 | ] 21 | }, 22 | { 23 | "path" : "/content/dam/someDamContent", 24 | "excludePaths": 25 | [ 26 | "someContent/someExcludeContent", 27 | "someContent/someOtherExcludeContent" 28 | ], 29 | "workflowConfigIds" : 30 | [ 31 | "/etc/workflow/launcher/config/update_asset_mod", 32 | "/etc/workflow/launcher/config/update_asset_create", 33 | "/etc/workflow/launcher/config/dam_xmp_nested_writeback", 34 | "/etc/workflow/launcher/config/dam_xmp_writeback" 35 | ], 36 | "deltaContent" : false 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /sample_config.yaml: -------------------------------------------------------------------------------- 1 | # Client Type: author 2 | 3 | # Information for connecting to the source content 4 | serverUsername : '' 5 | serverPassword : '' 6 | serverScheme : http 7 | serverHost : some.other.server 8 | serverPort : 4502 9 | 10 | deltaContent : true # default for all the paths 11 | 12 | # A reference to the standard set of workflow configuration ids that 13 | # we want to turn off when working with DAM assets. 14 | damWorkflows: &ourDamWorkflows 15 | - /etc/workflow/launcher/config/update_asset_mod 16 | - /etc/workflow/launcher/config/update_asset_create 17 | - /etc/workflow/launcher/config/dam_xmp_nested_writeback 18 | - /etc/workflow/launcher/config/dam_xmp_writeback 19 | 20 | 21 | # Each of the paths to include in the copy 22 | pathConfigurations : 23 | - 24 | path : /content/someContent 25 | - 26 | path : /content/someOtherContent 27 | excludePaths: [ someExcludeContent ] 28 | - 29 | path : /content/dam/someDamContent 30 | excludePaths : 31 | - someContent/someExcludeContent 32 | - someContent/someOtherExcludeContent 33 | workflowConfigIds : *ourDamWorkflows 34 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "Grabbit" 2 | -------------------------------------------------------------------------------- /src/integrationTest/groovy/com/twcable/grabbit/server/services/ServerServiceSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.services 18 | 19 | import com.twcable.grabbit.server.services.impl.DefaultServerService 20 | import com.twcable.grabbit.testutils.MockServletOutputStream 21 | import com.twcable.jackalope.NodeBuilder as FakeNodeBuilder 22 | import org.apache.sling.jcr.api.SlingRepository 23 | import org.springframework.context.ConfigurableApplicationContext 24 | import org.springframework.context.support.ClassPathXmlApplicationContext 25 | import spock.lang.Shared 26 | import spock.lang.Specification 27 | import spock.lang.Subject 28 | 29 | import static com.twcable.jackalope.JCRBuilder.node 30 | import static com.twcable.jackalope.JCRBuilder.property 31 | import static com.twcable.jackalope.JCRBuilder.repository 32 | import static com.twcable.jackalope.JcrConstants.NT_FILE 33 | 34 | @Subject(ServerService) 35 | class ServerServiceSpec extends Specification { 36 | 37 | @Shared 38 | SlingRepository slingRepository 39 | 40 | @Shared 41 | ConfigurableApplicationContext configurableApplicationContext 42 | 43 | @Shared 44 | ServerService syncServerService 45 | 46 | 47 | def setupSpec() { 48 | configurableApplicationContext = new ClassPathXmlApplicationContext("META-INF/spring/server-batch-job.xml") 49 | 50 | FakeNodeBuilder fakeNodeBuilder = 51 | node("default.groovy", 52 | node("jcr:content", 53 | property("jcr:data", "foo") 54 | ), 55 | property("jcr:primaryType", NT_FILE), 56 | property("jcr:lastModified", "Date"), 57 | property("multiValueLong", [1L, 2L, 4L] as Object[]), 58 | property("multiValueString", ["a", "b", "c"] as Object[]), 59 | ) 60 | 61 | 62 | slingRepository = repository(fakeNodeBuilder).build() 63 | 64 | syncServerService = new DefaultServerService(slingRepository: slingRepository, 65 | configurableApplicationContext: configurableApplicationContext) 66 | } 67 | 68 | 69 | def "Service should write data to provided outputStream"() { 70 | given: 71 | MockServletOutputStream mockServletOutputStream = new MockServletOutputStream() 72 | 73 | when: 74 | //This will also actually execute the Batch Job internally 75 | syncServerService.getContentForRootPath("admin", "/default.groovy", (Collection)Collections.EMPTY_LIST, "", mockServletOutputStream) 76 | 77 | then: 78 | mockServletOutputStream != null 79 | mockServletOutputStream.toString().contains("default.groovy") 80 | } 81 | 82 | def "Service excludes data in excludePaths and writes rest of the data to provided outputStream"() { 83 | given: 84 | MockServletOutputStream mockServletOutputStream = new MockServletOutputStream() 85 | 86 | when: 87 | //This will also actually execute the Batch Job internally 88 | syncServerService.getContentForRootPath("admin", "/default.groovy", ["/default.groovy/jcr:content"] as Collection, "", mockServletOutputStream) 89 | 90 | then: 91 | mockServletOutputStream != null 92 | !mockServletOutputStream.toString().contains("jcr:content") 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/integrationTest/groovy/com/twcable/grabbit/testutils/MockServletOutputStream.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.testutils 18 | 19 | import javax.servlet.ServletOutputStream 20 | 21 | class MockServletOutputStream extends ServletOutputStream { 22 | protected ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 23 | 24 | 25 | public void write(int b) throws IOException { 26 | byteArrayOutputStream.write(b); 27 | } 28 | 29 | 30 | public String toString() { 31 | try { 32 | return byteArrayOutputStream.toString("UTF-8"); 33 | } 34 | catch (UnsupportedEncodingException e) { 35 | throw new RuntimeException(e); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/content/META-INF/vault/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 10 | 11 | 12 | 17 | 18 | 19 | 23 | 24 | 25 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /src/main/content/META-INF/vault/definition/.content.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/main/content/META-INF/vault/filter.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/main/content/META-INF/vault/properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | FileVault Package Properties 5 | ${project.packageName} 6 | 1 7 | ${version} 8 | 9 | 2 10 | ${project.bundleDescription} 11 | twc 12 | 13 | -------------------------------------------------------------------------------- /src/main/content/META-INF/vault/settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/main/content/SLING-INF/content/apps/grabbit/config/com.twcable.grabbit.resources.GrabbitResourceProvider.xml: -------------------------------------------------------------------------------- 1 | 2 | sling:OsgiConfig 3 | 4 | provider.roots 5 | /grabbit 6 | String 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/main/content/SLING-INF/content/apps/grabbit/config/org.apache.sling.commons.log.LogManager.factory.config-com.twcable.grabbit.receive.xml: -------------------------------------------------------------------------------- 1 | 2 | sling:OsgiConfig 3 | 4 | org.apache.sling.commons.log.file 5 | logs/grabbit-receive.log 6 | String 7 | 8 | 9 | 10 | org.apache.sling.commons.log.level 11 | info 12 | String 13 | 14 | 15 | 16 | org.apache.sling.commons.log.names 17 | com.twcable.grabbit.client 18 | String 19 | 20 | 21 | 22 | org.apache.sling.commons.log.pattern 23 | {0,date,dd.MM.yyyy HH:mm:ss.SSS} [{2}] {3} *{4}* {5} 24 | String 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/content/SLING-INF/content/apps/grabbit/config/org.apache.sling.commons.log.LogManager.factory.config-com.twcable.grabbit.send.xml: -------------------------------------------------------------------------------- 1 | 2 | sling:OsgiConfig 3 | 4 | org.apache.sling.commons.log.file 5 | logs/grabbit-send.log 6 | String 7 | 8 | 9 | 10 | org.apache.sling.commons.log.level 11 | info 12 | String 13 | 14 | 15 | 16 | org.apache.sling.commons.log.names 17 | com.twcable.grabbit.server 18 | String 19 | 20 | 21 | 22 | org.apache.sling.commons.log.pattern 23 | {0,date,dd.MM.yyyy HH:mm:ss.SSS} [{2}] {3} *{4}* {5} 24 | String 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/content/SLING-INF/content/apps/grabbit/config/org.apache.sling.commons.log.LogManager.factory.config-org.apache.jackrabbit.core.SessionImpl.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | sling:OsgiConfig 8 | 9 | org.apache.sling.commons.log.file 10 | logs/jackrabbit-sessionImpl.log 11 | String 12 | 13 | 14 | 15 | org.apache.sling.commons.log.level 16 | info 17 | String 18 | 19 | 20 | 21 | org.apache.sling.commons.log.names 22 | org.apache.jackrabbit.core.SessionImpl 23 | String 24 | 25 | 26 | 27 | org.apache.sling.commons.log.pattern 28 | {0,date,dd.MM.yyyy HH:mm:ss.SSS} [{2}] {3} *{4}* {5} 29 | String 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/DateUtil.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit 18 | 19 | import groovy.transform.CompileStatic 20 | import groovy.util.logging.Slf4j 21 | import org.joda.time.DateTime 22 | import org.joda.time.format.ISODateTimeFormat 23 | 24 | @CompileStatic 25 | @Slf4j 26 | class DateUtil { 27 | 28 | public static Calendar getCalendarFromISOString(String calendarAsISO8601) { 29 | DateTime dateTime = new DateTime(calendarAsISO8601) 30 | dateTime.toGregorianCalendar() 31 | } 32 | 33 | 34 | public static String getISOStringFromCalendar(Calendar calendar) { 35 | DateTime dateTime = new DateTime(calendar) 36 | final String calendarAsISO8601 = ISODateTimeFormat.dateTime().print(dateTime) 37 | return calendarAsISO8601 38 | } 39 | 40 | 41 | public static Date getDateFromISOString(String dateAsISO8601) { 42 | DateTime dateTime = new DateTime(dateAsISO8601) 43 | dateTime.toDate() 44 | } 45 | 46 | 47 | public static String getISOStringFromDate(Date date) { 48 | DateTime dateTime = new DateTime(date) 49 | final String dateAsISO8601 = ISODateTimeFormat.dateTime().print(dateTime) 50 | return dateAsISO8601 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/NonExistentJobException.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit 17 | 18 | import groovy.transform.CompileStatic 19 | import groovy.transform.InheritConstructors 20 | 21 | /** 22 | * Exception representing a fault in retrieving a non-existent job from a job explorer. 23 | * See {@link ClientJobStatus} for usage. 24 | */ 25 | @CompileStatic 26 | @InheritConstructors 27 | class NonExistentJobException extends RuntimeException {} 28 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/ClientBatchJobExecutionListener.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch 18 | 19 | import com.twcable.grabbit.jcr.JcrUtil 20 | import groovy.transform.CompileStatic 21 | import groovy.util.logging.Slf4j 22 | import org.apache.sling.jcr.api.SlingRepository 23 | import org.springframework.batch.core.JobExecution 24 | import org.springframework.batch.core.JobExecutionListener 25 | 26 | import javax.jcr.Session 27 | 28 | /** 29 | * A JobExecutionListener that hooks into {@link JobExecutionListener#beforeJob(JobExecution)} to setup() the job and 30 | * {@link JobExecutionListener#afterJob(JobExecution)} to cleanup() after the job 31 | */ 32 | @Slf4j 33 | @CompileStatic 34 | @SuppressWarnings("GrMethodMayBeStatic") 35 | class ClientBatchJobExecutionListener implements JobExecutionListener { 36 | 37 | /** 38 | * {@link SlingRepository} is managed by Spring-OSGi 39 | */ 40 | private SlingRepository slingRepository 41 | 42 | 43 | void setSlingRepository(SlingRepository slingRepository) { 44 | this.slingRepository = slingRepository 45 | } 46 | 47 | // ********************************************************************** 48 | // METHODS BELOW ARE USED TO SETUP THE CLIENT JOB. 49 | // THE METHOD beforeJob() IS CALLED AFTER THE CLIENT JOB STARTS AND BEFORE 50 | // ANY OF THE STEPS ARE EXECUTED 51 | // ********************************************************************** 52 | 53 | /** 54 | * Callback before a job executes. 55 | * @param jobExecution the current {@link JobExecution} 56 | */ 57 | @Override 58 | void beforeJob(JobExecution jobExecution) { 59 | log.debug "SlingRepository : ${slingRepository}" 60 | final clientUsername = jobExecution.jobParameters.getString(ClientBatchJob.CLIENT_USERNAME) 61 | final Session session = JcrUtil.getSession(slingRepository, clientUsername) 62 | 63 | ClientBatchJobContext.setSession(session) 64 | log.info "Starting job : ${jobExecution}\n\n" 65 | } 66 | 67 | // ********************************************************************** 68 | // METHODS BELOW ARE USED TO CLEANUP AFTER CLIENT JOB IS COMPLETE. 69 | // THE METHOD afterJob() IS CALLED AFTER THE CLIENT JOB STEPS ARE COMPLETE 70 | // AND BEFORE THE JOB ACTUALLY TERMINATES 71 | // ********************************************************************** 72 | 73 | /** 74 | * Callback after completion of a job. 75 | * @param jobExecution the current {@link JobExecution} 76 | */ 77 | @Override 78 | void afterJob(JobExecution jobExecution) { 79 | log.info "Cleanup : ${jobExecution} . Job Complete. Releasing session, and input stream" 80 | ClientBatchJobContext.cleanup() 81 | final long timeTaken = jobExecution.endTime.time - jobExecution.startTime.time 82 | log.info "Grab from ${jobExecution.jobParameters.getString(ClientBatchJob.HOST)} " + 83 | "for Current Path ${jobExecution.jobParameters.getString(ClientBatchJob.PATH)} took : ${timeTaken} milliseconds\n\n" 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/jcrnodes/JcrNodesReader.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.steps.jcrnodes 18 | 19 | import com.twcable.grabbit.client.batch.ClientBatchJobContext 20 | import com.twcable.grabbit.proto.NodeProtos 21 | import com.twcable.grabbit.proto.NodeProtos.Node as ProtoNode 22 | import groovy.transform.CompileStatic 23 | import groovy.util.logging.Slf4j 24 | import org.springframework.batch.item.ItemReader 25 | import org.springframework.batch.item.NonTransientResourceException 26 | import org.springframework.batch.item.ParseException 27 | import org.springframework.batch.item.UnexpectedInputException 28 | 29 | /** 30 | * A Custom ItemReader that provides the "next" {@link NodeProtos.Node} from the {@link ClientBatchJobContext#inputStream}. 31 | * Returns null to indicate that all Items have been read. 32 | */ 33 | @Slf4j 34 | @CompileStatic 35 | @SuppressWarnings("GrMethodMayBeStatic") 36 | class JcrNodesReader implements ItemReader { 37 | 38 | @Override 39 | NodeProtos.Node read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { 40 | ProtoNode nodeProto = ProtoNode.parseDelimitedFrom(theInputStream()) 41 | if (!nodeProto) { 42 | log.info "Received all data from Server" 43 | return null 44 | } 45 | return nodeProto 46 | } 47 | 48 | private InputStream theInputStream() { 49 | ClientBatchJobContext.inputStream 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/jcrnodes/JcrNodesWriter.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.steps.jcrnodes 18 | 19 | import com.twcable.grabbit.client.batch.ClientBatchJobContext 20 | import com.twcable.grabbit.jcr.JCRNodeDecorator 21 | import com.twcable.grabbit.jcr.ProtoNodeDecorator 22 | import com.twcable.grabbit.proto.NodeProtos.Node as ProtoNode 23 | import groovy.transform.CompileStatic 24 | import groovy.util.logging.Slf4j 25 | import org.springframework.batch.core.ItemWriteListener 26 | import org.springframework.batch.item.ItemWriter 27 | import org.springframework.util.StopWatch 28 | 29 | import javax.jcr.Session 30 | 31 | /** 32 | * A Custom ItemWriter that will write the provided Jcr Nodes to the {@link JcrNodesWriter#theSession()} 33 | * Will save() the {@link JcrNodesWriter#theSession()} after writing provided Jcr Nodes 34 | * @see ItemWriteListener 35 | */ 36 | @Slf4j 37 | @CompileStatic 38 | @SuppressWarnings('GrMethodMayBeStatic') 39 | class JcrNodesWriter implements ItemWriter, ItemWriteListener { 40 | 41 | @Override 42 | void beforeWrite(List nodeProtos) { 43 | //no-op 44 | } 45 | 46 | 47 | @Override 48 | void afterWrite(List nodeProtos) { 49 | log.info "Saving ${nodeProtos.size()} nodes" 50 | log.debug """Saving Nodes : ${(nodeProtos as List).collectMany { ProtoNode pNode -> 51 | [ pNode.name , pNode.mandatoryChildNodeList.collect { it.name - pNode.name }] 52 | }.flatten()}""" 53 | theSession().save() 54 | withStopWatch("Refreshing session: ${theSession()}") { 55 | theSession().refresh(false) 56 | } 57 | } 58 | 59 | 60 | @Override 61 | void onWriteError(Exception exception, List nodeProtos) { 62 | log.error "Exception writing JCR Nodes to current JCR Session : ${theSession()}. ", exception 63 | } 64 | 65 | 66 | /** 67 | * The JcrNodesReader that funnels proto nodes into here 68 | * will return null when the stream is finished to indicate completion, but Spring will pass null to this point 69 | */ 70 | @Override 71 | void write(List nodeProtos) throws Exception { 72 | Session session = theSession() 73 | for (ProtoNode nodeProto : nodeProtos) { 74 | writeToJcr(nodeProto, session) 75 | } 76 | } 77 | 78 | private static T withStopWatch(String stopWatchId, Closure cl) { 79 | StopWatch stopWatch = new StopWatch(stopWatchId) 80 | stopWatch.start() 81 | 82 | T retVal = cl.call() 83 | 84 | stopWatch.stop() 85 | log.info stopWatch.shortSummary() 86 | 87 | return retVal 88 | } 89 | 90 | private static void writeToJcr(ProtoNode nodeProto, Session session) { 91 | ProtoNodeDecorator.createFrom(nodeProto).writeToJcr(session) 92 | } 93 | 94 | private Session theSession() { 95 | ClientBatchJobContext.session 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/jcrnodes/LoggingStepExecutionListener.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.steps.jcrnodes 18 | 19 | import com.twcable.grabbit.client.batch.ClientBatchJob 20 | import groovy.transform.CompileStatic 21 | import groovy.util.logging.Slf4j 22 | import org.springframework.batch.core.ExitStatus 23 | import org.springframework.batch.core.JobParameters 24 | import org.springframework.batch.core.StepExecution 25 | import org.springframework.batch.core.StepExecutionListener 26 | 27 | /** 28 | * A Custom {@link StepExecutionListener} 29 | */ 30 | @Slf4j 31 | @CompileStatic 32 | class LoggingStepExecutionListener implements StepExecutionListener { 33 | 34 | @Override 35 | void beforeStep(StepExecution stepExecution) { 36 | log.info "Starting JcrNodes Step ${stepExecution}\n" 37 | } 38 | 39 | /** 40 | * Assumes that Step Completed successfully 41 | * Logs information about the Step like the currentPath and the {@link StepExecution#writeCount} 42 | * @param stepExecution current StepExecution 43 | * @return ExitStatus.COMPLETED (Assumes Step Completed and there were no errors) 44 | */ 45 | @Override 46 | ExitStatus afterStep(StepExecution stepExecution) { 47 | JobParameters jobParameters = stepExecution.jobParameters 48 | final String currentPath = jobParameters.getString(ClientBatchJob.PATH) 49 | log.info "JcrNodes Step Complete. Current Path : ${currentPath} . Total nodes written : ${stepExecution.writeCount}\n\n" 50 | return ExitStatus.COMPLETED 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/validation/ValidJobDecider.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.client.batch.steps.validation 17 | 18 | import com.twcable.grabbit.client.batch.ClientBatchJob 19 | import com.twcable.grabbit.client.batch.ClientBatchJobContext 20 | import groovy.transform.CompileStatic 21 | import groovy.util.logging.Slf4j 22 | import org.springframework.batch.core.JobExecution 23 | import org.springframework.batch.core.StepExecution 24 | import org.springframework.batch.core.job.flow.FlowExecutionStatus 25 | import org.springframework.batch.core.job.flow.JobExecutionDecider 26 | 27 | import javax.jcr.PathNotFoundException 28 | import javax.jcr.RepositoryException 29 | import javax.jcr.Session 30 | 31 | /** 32 | * This class serves as a validation gate for jobs in-flight. It should be the first step on the client when running a job to determine 33 | * if the job is job that is safe, and valid to execute. 34 | */ 35 | @CompileStatic 36 | @Slf4j 37 | class ValidJobDecider implements JobExecutionDecider { 38 | 39 | final static FlowExecutionStatus JOB_VALID = new FlowExecutionStatus("VALID") 40 | final static FlowExecutionStatus JOB_INVALID = new FlowExecutionStatus("INVALID") 41 | 42 | 43 | private Session theSession() { 44 | ClientBatchJobContext.session 45 | } 46 | 47 | /** 48 | * Determines if the job to be executed is a valid job 49 | * 50 | * For example, are we being asked to sync a path for which a path has no existing parents? 51 | * If so, we should label this scenario as an invalid job; and not attempt to write "dirty data" 52 | * 53 | * @param jobExecution a job execution 54 | * @param stepExecution the latest step execution (may be null) 55 | * @return the exit status code. Is it a valid or invalid job? 56 | */ 57 | @Override 58 | FlowExecutionStatus decide(JobExecution jobExecution, StepExecution stepExecution) { 59 | String jobPath = jobExecution.jobParameters.getString(ClientBatchJob.PATH) 60 | //For processing, remove trailing / 61 | jobPath = jobPath.replaceFirst(/\/$/, '') 62 | //Get the parent's path (if applicable) and determine if it exists already 63 | final parts = jobPath.split('/') 64 | //No parent, so nothing to worry about 65 | if(parts.length <= 2) return JOB_VALID 66 | 67 | final parentPath = parts[0..-2].join('/') 68 | final Session session = theSession() 69 | try { 70 | session.getNode(parentPath) 71 | } catch(PathNotFoundException pathException) { 72 | log.warn "${jobPath} is not a valid job path. Make sure a parent is synched or created before this job is run" 73 | log.debug pathException.toString() 74 | return JOB_INVALID 75 | } 76 | catch(RepositoryException repoException) { 77 | log.error "${RepositoryException.class.canonicalName} Something went wrong when accessing the repository at ${this.class.canonicalName} for job path ${jobPath}!" 78 | log.error repoException.toString() 79 | return JOB_INVALID 80 | } 81 | log.debug "${ValidJobDecider.class.canonicalName} Job determined to be valid for job path ${jobPath}" 82 | return JOB_VALID 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/workflows/WorkflowOffTasklet.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.steps.workflows 18 | 19 | import com.twcable.grabbit.client.batch.workflows.WorkflowManager 20 | import groovy.transform.CompileStatic 21 | import groovy.util.logging.Slf4j 22 | import org.springframework.batch.core.StepContribution 23 | import org.springframework.batch.core.scope.context.ChunkContext 24 | import org.springframework.batch.core.step.tasklet.Tasklet 25 | import org.springframework.batch.repeat.RepeatStatus 26 | 27 | @CompileStatic 28 | @Slf4j 29 | class WorkflowOffTasklet implements Tasklet { 30 | 31 | private WorkflowManager workflowManager 32 | 33 | private String workflowConfigs 34 | 35 | 36 | void setWorkflowManager(WorkflowManager workflowManager) { 37 | this.workflowManager = workflowManager 38 | } 39 | 40 | 41 | void setWorkflowConfigs(String workflowConfigs) { 42 | log.info("WorkflowConfig : ${workflowConfigs}") 43 | this.workflowConfigs = workflowConfigs 44 | } 45 | 46 | 47 | @Override 48 | RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { 49 | if (!workflowConfigs || !workflowConfigs.contains("/etc/workflow") /* temporary for testing */) { 50 | //nothing to process as there are no workflow configs for the current path 51 | log.info "Nothing to process..." 52 | return RepeatStatus.FINISHED 53 | } 54 | 55 | Collection configIds = workflowConfigs.split("\\|") as Collection 56 | 57 | workflowManager.turnOff(configIds) 58 | return RepeatStatus.FINISHED 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/workflows/WorkflowOnTasklet.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.steps.workflows 18 | 19 | import com.twcable.grabbit.client.batch.workflows.WorkflowManager 20 | import groovy.transform.CompileStatic 21 | import groovy.util.logging.Slf4j 22 | import org.springframework.batch.core.StepContribution 23 | import org.springframework.batch.core.scope.context.ChunkContext 24 | import org.springframework.batch.core.step.tasklet.Tasklet 25 | import org.springframework.batch.repeat.RepeatStatus 26 | 27 | @CompileStatic 28 | @Slf4j 29 | class WorkflowOnTasklet implements Tasklet { 30 | 31 | private WorkflowManager workflowManager 32 | 33 | private String workflowConfigs 34 | 35 | 36 | void setWorkflowManager(WorkflowManager workflowManager) { 37 | this.workflowManager = workflowManager 38 | } 39 | 40 | 41 | void setWorkflowConfigs(String workflowConfigs) { 42 | log.info("WorkflowConfig : ${workflowConfigs}") 43 | this.workflowConfigs = workflowConfigs 44 | } 45 | 46 | 47 | @Override 48 | RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { 49 | if (!workflowConfigs || !workflowConfigs.contains("/etc/workflow") /* temporary for testing */) { 50 | //nothing to process as there are no workflow configs for the current path 51 | log.info "Nothing to process..." 52 | return RepeatStatus.FINISHED 53 | } 54 | 55 | Collection configIds = workflowConfigs.split("\\|") as Collection 56 | 57 | workflowManager.turnOn(configIds) 58 | return RepeatStatus.FINISHED 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/steps/workspace/DeleteBeforeWriteDecider.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.client.batch.steps.workspace 17 | 18 | import com.twcable.grabbit.client.batch.ClientBatchJob 19 | import groovy.transform.CompileStatic 20 | import groovy.util.logging.Slf4j 21 | import org.springframework.batch.core.JobExecution 22 | import org.springframework.batch.core.StepExecution 23 | import org.springframework.batch.core.job.flow.FlowExecutionStatus 24 | import org.springframework.batch.core.job.flow.JobExecutionDecider 25 | 26 | /** 27 | * This class decides whether to execute the @see{@link DeleteBeforeWriteTasklet} based on the 28 | * deleteBeforeWrite parameter in the job parameters 29 | */ 30 | @CompileStatic 31 | @Slf4j 32 | class DeleteBeforeWriteDecider implements JobExecutionDecider { 33 | 34 | /** 35 | * Strategy for branching an execution based on the state of an ongoing 36 | * {@link JobExecution}. The return value will be used as a status to 37 | * determine the next step in the job. 38 | * 39 | * @param jobExecution a job execution 40 | * @param stepExecution the latest step execution (may be null) 41 | * @return the exit status code 42 | */ 43 | @Override 44 | FlowExecutionStatus decide(JobExecution jobExecution, StepExecution stepExecution) { 45 | final boolean shouldDeleteBeforeWrite = jobExecution.jobParameters.getString(ClientBatchJob.DELETE_BEFORE_WRITE).toBoolean() 46 | log.info shouldDeleteBeforeWrite ? "Will delete nodes under job path before writing for ${jobExecution.id}..." : 47 | "Will retain nodes under job path for ${jobExecution.id}..." 48 | return shouldDeleteBeforeWrite ? new FlowExecutionStatus("YES") : new FlowExecutionStatus("NO") 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/batch/workflows/WorkflowManager.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.workflows 18 | 19 | interface WorkflowManager { 20 | 21 | void turnOff(Collection wfConfigIds) 22 | 23 | void turnOn(Collection wfConfigIds) 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/services/ClientService.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.services 18 | 19 | import com.twcable.grabbit.GrabbitConfiguration 20 | 21 | interface ClientService { 22 | 23 | /** 24 | * This API will perform Content Grab for the given configuration 25 | * @param configuration : the {@link GrabbitConfiguration} 26 | * @param clientUsername : the user that will be used by Grabbit Client 27 | * @return Collection of Job's Execution Ids 28 | */ 29 | Collection initiateGrab(GrabbitConfiguration configuration, String clientUsername) 30 | } 31 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/services/impl/DefaultClientService.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.services.impl 18 | 19 | import com.twcable.grabbit.GrabbitConfiguration 20 | import com.twcable.grabbit.GrabbitConfiguration.PathConfiguration 21 | import com.twcable.grabbit.client.batch.ClientBatchJob 22 | import com.twcable.grabbit.client.services.ClientService 23 | import groovy.transform.CompileStatic 24 | import groovy.util.logging.Slf4j 25 | import org.apache.felix.scr.annotations.Activate 26 | import org.apache.felix.scr.annotations.Component 27 | import org.apache.felix.scr.annotations.Reference 28 | import org.apache.felix.scr.annotations.Service 29 | import org.apache.sling.jcr.api.SlingRepository 30 | import org.springframework.batch.core.JobExecution 31 | import org.springframework.batch.core.JobInstance 32 | import org.springframework.batch.core.explore.JobExplorer 33 | import org.springframework.context.ConfigurableApplicationContext 34 | 35 | @Slf4j 36 | @CompileStatic 37 | @Component(label = "Grabbit Client Service", description = "Grabbit Client Service", immediate = true, metatype = true, enabled = true) 38 | @Service(ClientService) 39 | @SuppressWarnings(['GroovyUnusedDeclaration', 'GrMethodMayBeStatic']) 40 | class DefaultClientService implements ClientService { 41 | 42 | @Reference(bind = 'setSlingRepository') 43 | SlingRepository slingRepository 44 | 45 | @Reference(bind = 'setConfigurableApplicationContext') 46 | ConfigurableApplicationContext configurableApplicationContext 47 | 48 | 49 | @Activate 50 | void activate() { 51 | log.info "Activate\n\n" 52 | } 53 | 54 | 55 | @Override 56 | Collection initiateGrab(GrabbitConfiguration configuration, String clientUsername) { 57 | 58 | Collection jobExecutionIds = [] 59 | 60 | for (PathConfiguration pathConfig : configuration.pathConfigurations) { 61 | try { 62 | final clientBatchJob = new ClientBatchJob.ServerBuilder(configurableApplicationContext) 63 | .andServer(configuration.serverScheme, configuration.serverHost, configuration.serverPort) 64 | .andCredentials(clientUsername, configuration.serverUsername, configuration.serverPassword) 65 | .andClientJobExecutions(fetchAllClientJobExecutions()) 66 | .withTransactionID(configuration.transactionID) 67 | .andConfiguration(pathConfig) 68 | .build() 69 | final Long currentJobExecutionId = clientBatchJob.start() 70 | jobExecutionIds << currentJobExecutionId 71 | } 72 | catch (Exception e) { 73 | log.error "Error while requesting a content sync for current Path: ${[pathConfig.path]}", e 74 | throw new IllegalStateException("Failed to initiate job for path: ${pathConfig.path}") 75 | } 76 | } 77 | return jobExecutionIds 78 | 79 | } 80 | 81 | 82 | private List fetchAllClientJobExecutions() { 83 | final explorer = configurableApplicationContext.getBean("clientJobExplorer", JobExplorer) 84 | final instances = explorer.getJobInstances("clientJob", 0, Integer.MAX_VALUE - 1) ?: [] as List 85 | final executions = instances.collect { explorer.getJobExecutions(it) }.flatten() as List 86 | executions 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/client/servlets/GrabbitRootServlet.groovy: -------------------------------------------------------------------------------- 1 | package com.twcable.grabbit.client.servlets 2 | 3 | import groovy.transform.CompileStatic 4 | import groovy.util.logging.Slf4j 5 | import org.apache.felix.scr.annotations.sling.SlingServlet 6 | import org.apache.sling.api.SlingHttpServletRequest 7 | import org.apache.sling.api.SlingHttpServletResponse 8 | import org.apache.sling.api.servlets.SlingAllMethodsServlet 9 | import static javax.servlet.http.HttpServletResponse.SC_OK 10 | 11 | /** 12 | * Copyright 2015 Time Warner Cable, Inc. 13 | * 14 | * Licensed under the Apache License, Version 2.0 (the "License"); 15 | * you may not use this file except in compliance with the License. 16 | * You may obtain a copy of the License at 17 | * 18 | * http://www.apache.org/licenses/LICENSE-2.0 19 | * 20 | * Unless required by applicable law or agreed to in writing, software 21 | * distributed under the License is distributed on an "AS IS" BASIS, 22 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | * See the License for the specific language governing permissions and 24 | * limitations under the License. 25 | * 26 | * Root resource for /grabbit requests. 27 | * Acts as a handler for {@link com.twcable.grabbit.resources.RootResource} resource. 28 | */ 29 | @Slf4j 30 | @CompileStatic 31 | @SlingServlet(methods = ['GET'], resourceTypes = ['twcable:grabbit']) 32 | class GrabbitRootServlet extends SlingAllMethodsServlet { 33 | 34 | /** 35 | * This method gets called when a request is made to either Grabbit Root resource directly (/grabbit) or any invalid 36 | * URL under it. 37 | * Response will contain links to valid resources' urls under Grabbit Root url, in accordance with HATEOAS. 38 | * A hypermedia-driven response provides information to navigate Grabbit's resources dynamically, by including 39 | * valid hypermedia links along with the response. 40 | * For reference, {@see https://spring.io/understanding/HATEOAS} 41 | * 42 | * From Grabbit Root resource, Valid resources are 43 | * - Grabbit's Job Resource (/grabbit/job) 44 | * - Grabbit's Content Resource (/grabbit/content) 45 | * - Grabbit's Transaction Resource (/grabbit/transaction) 46 | */ 47 | @Override 48 | void doGet(SlingHttpServletRequest request, SlingHttpServletResponse response) { 49 | response.setContentType("text/html") 50 | response.setStatus(SC_OK) 51 | response.writer.write this.class.getResourceAsStream("RootResource.txt").text 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/jcr/JcrPropertyDecorator.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.jcr 17 | 18 | import com.google.protobuf.ByteString 19 | import com.twcable.grabbit.proto.NodeProtos.Property as ProtoProperty 20 | import com.twcable.grabbit.proto.NodeProtos.Property.Builder as ProtoPropertyBuilder 21 | import com.twcable.grabbit.proto.NodeProtos.Value as ProtoValue 22 | import com.twcable.grabbit.proto.NodeProtos.Value.Builder as ProtoValueBuilder 23 | import groovy.transform.CompileStatic 24 | import groovy.util.logging.Slf4j 25 | 26 | import javax.annotation.Nonnull 27 | import javax.jcr.Property as JCRProperty 28 | import javax.jcr.Value 29 | 30 | import static javax.jcr.PropertyType.BINARY 31 | import static org.apache.jackrabbit.JcrConstants.* 32 | 33 | @CompileStatic 34 | @Slf4j 35 | class JcrPropertyDecorator { 36 | 37 | @Delegate 38 | JCRProperty innerProperty 39 | 40 | private final JCRNodeDecorator nodeOwner 41 | 42 | JcrPropertyDecorator(JCRProperty property, JCRNodeDecorator nodeOwner) { 43 | this.innerProperty = property 44 | this.nodeOwner = nodeOwner 45 | } 46 | 47 | /** 48 | * Determines if JCR Property object can be "rewritten" to the JCR. For example, we can not rewrite a node's 49 | * primary type; That is forbidden by the JCR spec. 50 | */ 51 | boolean isTransferable() { 52 | //If property is "jcr:lastModified", we don't want to send this property to the client. If we send it, and 53 | //the client writes it to JCR, then we can have lastModified date for a node that is older than the creation 54 | //date itself 55 | if (name == JCR_LASTMODIFIED) { 56 | return false 57 | } 58 | 59 | if ([JCR_PRIMARYTYPE, JCR_MIXINTYPES].contains(name)) { 60 | return true 61 | } 62 | 63 | if(nodeOwner.isAuthorizableType() || nodeOwner.isACType()) { 64 | return true 65 | } 66 | 67 | return !definition.isProtected() 68 | } 69 | 70 | /** 71 | * Marshalls current Jcr Property to a ProtoProperty 72 | */ 73 | @Nonnull 74 | ProtoProperty toProtoProperty() { 75 | ProtoPropertyBuilder propertyBuilder = ProtoProperty.newBuilder() 76 | ProtoValueBuilder valueBuilder = ProtoValue.newBuilder() 77 | propertyBuilder.setName(name) 78 | 79 | if(type == BINARY) { 80 | propertyBuilder.addValues(valueBuilder.setBytesValue(ByteString.readFrom(value.binary.stream))) 81 | } 82 | else { 83 | //Other property types can potentially have multiple values 84 | final Value[] values = multiple ? values : [value] as Value[] 85 | values.each { Value value -> 86 | propertyBuilder.addValues(valueBuilder.setStringValue(value.string)) 87 | } 88 | } 89 | propertyBuilder.setMultiple(multiple) 90 | propertyBuilder.setType(type) 91 | propertyBuilder.build() 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/jcr/ProtoNodeDecorator.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.jcr 18 | 19 | import com.twcable.grabbit.proto.NodeProtos.Node as ProtoNode 20 | import groovy.transform.CompileStatic 21 | 22 | import javax.annotation.Nonnull 23 | import javax.jcr.Session 24 | 25 | @CompileStatic 26 | abstract class ProtoNodeDecorator { 27 | 28 | @Delegate 29 | protected ProtoNode innerProtoNode 30 | 31 | protected Collection protoProperties 32 | 33 | protected String nameOverride 34 | 35 | protected abstract JCRNodeDecorator writeNode(@Nonnull Session session) 36 | 37 | static ProtoNodeDecorator createFrom(@Nonnull ProtoNode node, String nameOverride = null) { 38 | if(!node) throw new IllegalArgumentException("node must not be null!") 39 | final protoProperties = node.propertiesList.collect { new ProtoPropertyDecorator(it) } 40 | final primaryType = protoProperties.find { it.primaryType } 41 | if(primaryType.isUserType() || primaryType.isGroupType()) { 42 | return new AuthorizableProtoNodeDecorator(node, protoProperties) 43 | } 44 | else if(primaryType.isRepAclType()) { 45 | return new ACLProtoNodeDecorator(node, protoProperties, nameOverride) 46 | } 47 | return new DefaultProtoNodeDecorator(node, protoProperties, nameOverride) 48 | } 49 | 50 | 51 | JCRNodeDecorator writeToJcr(@Nonnull Session session) { 52 | final JCRNodeDecorator writtenNode = writeNode(session) 53 | writtenNode.setLastModified() 54 | return writtenNode 55 | } 56 | 57 | 58 | boolean hasProperty(String propertyName) { 59 | propertiesList.any{ it.name == propertyName } 60 | } 61 | 62 | 63 | protected ProtoPropertyDecorator getPrimaryType() { 64 | protoProperties.find { it.isPrimaryType() } 65 | } 66 | 67 | 68 | protected String getStringValueFrom(String propertyName) { 69 | protoProperties.find { it.name == propertyName }.stringValue 70 | } 71 | 72 | protected String getParentPath() { 73 | final pathTokens = getName().tokenize('/') 74 | //remove last index, as this is the Authorizable node name 75 | pathTokens.remove(pathTokens.size() - 1) 76 | return "/${pathTokens.join('/')}" 77 | } 78 | 79 | @Override 80 | String getName() { 81 | nameOverride ?: innerProtoNode.getName() 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/resources/CleanJobRepositoryResource.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.resources 18 | 19 | import groovy.transform.CompileStatic 20 | import org.apache.sling.api.resource.ResourceResolver 21 | 22 | import javax.annotation.Nonnull 23 | 24 | /** 25 | * A resource representing what needs to be deleted from Grabbit's JobRepository. 26 | * provided by {@link GrabbitResourceProvider}. 27 | * Queried from {@link com.twcable.grabbit.spring.batch.repository.servlets.GrabbitCleanJobRepositoryServlet}. 28 | */ 29 | @CompileStatic 30 | class CleanJobRepositoryResource extends RootResource { 31 | 32 | public static final String CLEAN_JOBREPOSITORY_RESOURCE_TYPE = "${ROOT_RESOURCE_TYPE}/jobrepository/clean" 33 | 34 | CleanJobRepositoryResource(@Nonnull final ResourceResolver resourceResolver, @Nonnull final String resolutionPath) { 35 | super(resourceResolver, resolutionPath, CLEAN_JOBREPOSITORY_RESOURCE_TYPE) 36 | } 37 | 38 | 39 | @Override 40 | String getResourceType() { 41 | return CLEAN_JOBREPOSITORY_RESOURCE_TYPE 42 | } 43 | 44 | @Override 45 | String getResourceSuperType(){ 46 | return ROOT_RESOURCE_TYPE 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/resources/ContentResource.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.resources 18 | 19 | import groovy.transform.CompileStatic 20 | import org.apache.sling.api.resource.ResourceResolver 21 | 22 | import javax.annotation.Nonnull 23 | 24 | /** 25 | * A resource representing some content to be streamed from a Grabbit instance. 26 | * provided by {@link GrabbitResourceProvider}. 27 | * Queried from {@link com.twcable.grabbit.server.GrabbitContentPullServlet}. 28 | */ 29 | @CompileStatic 30 | class ContentResource extends RootResource { 31 | 32 | public static final String CONTENT_RESOURCE_TYPE = "${ROOT_RESOURCE_TYPE}/content" 33 | 34 | ContentResource(@Nonnull final ResourceResolver resourceResolver, @Nonnull final String resolutionPath) { 35 | super(resourceResolver, resolutionPath, CONTENT_RESOURCE_TYPE) 36 | } 37 | 38 | 39 | @Override 40 | String getResourceType() { 41 | return CONTENT_RESOURCE_TYPE 42 | } 43 | 44 | @Override 45 | String getResourceSuperType(){ 46 | return ROOT_RESOURCE_TYPE 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/resources/JobResource.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.resources 18 | 19 | import groovy.transform.CompileStatic 20 | import org.apache.sling.api.resource.ResourceResolver 21 | 22 | import javax.annotation.Nonnull 23 | import java.util.regex.Matcher 24 | 25 | /** 26 | * Simple representation of a Grabbit's Job Resource provided by {@link GrabbitResourceProvider}. 27 | * Queried from {@link com.twcable.grabbit.client.servlets.GrabbitJobServlet}. 28 | */ 29 | @CompileStatic 30 | class JobResource extends RootResource { 31 | 32 | public static final String JOB_RESOURCE_TYPE = "${ROOT_RESOURCE_TYPE}/job" 33 | public static final String JOB_EXECUTION_ID_KEY = "${ROOT_RESOURCE_TYPE}:jobresource.jobExecutionId" 34 | 35 | JobResource(@Nonnull final ResourceResolver resourceResolver, @Nonnull final String resolutionPath) { 36 | super(resourceResolver, resolutionPath, JOB_RESOURCE_TYPE) 37 | super.resourceMetadata.put(JOB_EXECUTION_ID_KEY, getJobIdFromPath(resolutionPath)) 38 | } 39 | 40 | 41 | private static String getJobIdFromPath(String path) { 42 | Matcher matcher = path =~ /\/grabbit\/job\/(.+)$/ 43 | if (matcher.matches()) { 44 | final matchedList = matcher[0] as Collection 45 | return matchedList[1] - ~/\..+/ 46 | } 47 | else { 48 | return "" 49 | } 50 | } 51 | 52 | 53 | @Override 54 | String getResourceType() { 55 | return JOB_RESOURCE_TYPE 56 | } 57 | 58 | @Override 59 | String getResourceSuperType(){ 60 | return ROOT_RESOURCE_TYPE 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/resources/RootResource.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Representation of a Grabbit's Root Resource provided by {@link GrabbitResourceProvider}. 17 | * Queried from {@link com.twcable.grabbit.client.servlets.GrabbitRootServlet} 18 | */ 19 | 20 | package com.twcable.grabbit.resources 21 | 22 | import groovy.transform.CompileStatic 23 | import org.apache.sling.api.resource.ResourceResolver 24 | import org.apache.sling.api.resource.SyntheticResource 25 | 26 | import javax.annotation.Nonnull 27 | 28 | @CompileStatic 29 | class RootResource extends SyntheticResource { 30 | 31 | static final String ROOT_RESOURCE_TYPE = "twcable:grabbit" 32 | 33 | RootResource(@Nonnull final ResourceResolver resourceResolver,@Nonnull final String resolutionPath, 34 | String resourceType = ROOT_RESOURCE_TYPE) { 35 | super(resourceResolver, resolutionPath, resourceType) 36 | } 37 | 38 | @Override 39 | String getResourceType() { 40 | return ROOT_RESOURCE_TYPE 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/resources/TransactionResource.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.resources 18 | 19 | import groovy.transform.CompileStatic 20 | import org.apache.sling.api.resource.ResourceResolver 21 | 22 | import javax.annotation.Nonnull 23 | import java.util.regex.Matcher 24 | 25 | /** 26 | * {@link TransactionResource} represents a a logical group of jobs by configuration run. 27 | * 28 | * Provided by {@link com.twcable.grabbit.resources.GrabbitResourceProvider}. 29 | * Queried from {@link com.twcable.grabbit.client.servlets.GrabbitTransactionServlet}. 30 | */ 31 | @CompileStatic 32 | class TransactionResource extends RootResource { 33 | 34 | public static final String TRANSACTION_RESOURCE_TYPE = "${ROOT_RESOURCE_TYPE}/transaction" 35 | public static final String TRANSACTION_ID_KEY = "${ROOT_RESOURCE_TYPE}:transactionresource.transactionID" 36 | 37 | TransactionResource(@Nonnull final ResourceResolver resourceResolver, @Nonnull final String resolutionPath) { 38 | super(resourceResolver, resolutionPath, TRANSACTION_RESOURCE_TYPE) 39 | super.resourceMetadata.put(TRANSACTION_ID_KEY, getTransactionIdFromPath(resolutionPath)) 40 | } 41 | 42 | 43 | private static String getTransactionIdFromPath(String path) { 44 | Matcher matcher = path =~ /\/grabbit\/transaction\/(.+)$/ 45 | if (matcher.matches()) { 46 | final matchedList = matcher[0] as Collection 47 | return matchedList[1] - ~/\..+/ 48 | } 49 | else { 50 | return "" 51 | } 52 | } 53 | 54 | 55 | @Override 56 | String getResourceType() { 57 | return TRANSACTION_RESOURCE_TYPE 58 | } 59 | 60 | @Override 61 | String getResourceSuperType(){ 62 | return ROOT_RESOURCE_TYPE 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/security/AuthorizablePrincipal.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.security 18 | 19 | import groovy.transform.CompileStatic 20 | 21 | import javax.annotation.Nonnull 22 | import java.security.Principal 23 | 24 | 25 | @CompileStatic 26 | class AuthorizablePrincipal implements Principal { 27 | 28 | final private String principalName 29 | 30 | AuthorizablePrincipal(@Nonnull final String principalName) { 31 | this.principalName = principalName 32 | } 33 | 34 | @Override 35 | boolean equals(Object other) { 36 | if(other == null) return false 37 | return this.hashCode() == other.hashCode() 38 | } 39 | 40 | @Override 41 | String getName() { 42 | return principalName 43 | } 44 | 45 | @Override 46 | int hashCode() { 47 | return principalName.hashCode() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/security/InsufficientGrabbitPrivilegeException.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.security 17 | 18 | import groovy.transform.CompileStatic 19 | import groovy.transform.InheritConstructors 20 | 21 | @InheritConstructors 22 | @CompileStatic 23 | class InsufficientGrabbitPrivilegeException extends RuntimeException {} 24 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/ServerBatchJobContext.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch 18 | 19 | import groovy.transform.CompileStatic 20 | 21 | import javax.jcr.Node as JcrNode 22 | import javax.servlet.ServletOutputStream 23 | 24 | /** 25 | * Helper class that wraps a {@link ThreadLocal < ClientBatchJobContext >} variable used to store {@link ServletOutputStream} 26 | * , {@link ServerBatchJobContext#namespacesIterator} and {@link ServerBatchJobContext#nodeIterator} in ThreadLocal. 27 | */ 28 | @CompileStatic 29 | class ServerBatchJobContext { 30 | 31 | static final ThreadLocal THREAD_LOCAL = new ThreadLocal() 32 | 33 | final ServletOutputStream servletOutputStream 34 | final Iterator> namespacesIterator 35 | final Iterator nodeIterator 36 | 37 | 38 | ServerBatchJobContext(ServletOutputStream servletOutputStream, Iterator> namespacesIterator, 39 | Iterator nodeIterator) { 40 | this.servletOutputStream = servletOutputStream 41 | this.namespacesIterator = namespacesIterator 42 | this.nodeIterator = nodeIterator 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/ServerBatchJobExecutionListener.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch 18 | 19 | import groovy.transform.CompileStatic 20 | import groovy.util.logging.Slf4j 21 | import org.springframework.batch.core.JobExecution 22 | import org.springframework.batch.core.JobExecutionListener 23 | 24 | /** 25 | * A JobExecutionListener that hooks into {@link JobExecutionListener#beforeJob(JobExecution)} and 26 | * {@link JobExecutionListener#afterJob(JobExecution)} 27 | */ 28 | @Slf4j 29 | @CompileStatic 30 | class ServerBatchJobExecutionListener implements JobExecutionListener { 31 | 32 | /** 33 | * Callback before a job executes. 34 | * @param jobExecution the current {@link JobExecution} 35 | */ 36 | @Override 37 | void beforeJob(JobExecution jobExecution) { 38 | log.info "Starting job : ${jobExecution}\n\n" 39 | } 40 | 41 | /** 42 | * Callback after completion of a job. 43 | * Cleans up current Thread's ThreadLocal from {@link ServerBatchJobContext#THREAD_LOCAL} 44 | * @param jobExecution the current {@link JobExecution} 45 | */ 46 | @Override 47 | void afterJob(JobExecution jobExecution) { 48 | log.info "Clearing ThreadLocal for current job: ${jobExecution} . Job Complete" 49 | ServerBatchJobContext serverBatchJobContext = ServerBatchJobContext.THREAD_LOCAL.get() 50 | try { 51 | serverBatchJobContext.servletOutputStream.close() 52 | } 53 | catch (Exception ignore) { /* just doing cleanup */ 54 | } 55 | ServerBatchJobContext.THREAD_LOCAL.remove() 56 | final long timeTaken = jobExecution.endTime.time - jobExecution.startTime.time 57 | log.info "Content sent for ${jobExecution.jobParameters.getString(ServerBatchJob.PATH)} took : ${timeTaken} milliseconds\n\n" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/steps/jcrnodes/JcrNodesProcessor.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch.steps.jcrnodes 18 | 19 | import com.twcable.grabbit.DateUtil 20 | import com.twcable.grabbit.jcr.JCRNodeDecorator 21 | import com.twcable.grabbit.proto.NodeProtos.Node as ProtoNode 22 | import groovy.transform.CompileStatic 23 | import groovy.util.logging.Slf4j 24 | import org.springframework.batch.item.ItemProcessor 25 | 26 | import javax.annotation.Nullable 27 | import javax.jcr.Node as JcrNode 28 | 29 | /** 30 | * This ItemProcessor takes javax.jcr.Node references from JcrNodesReader, and converts them into ProtoNode objects 31 | */ 32 | @Slf4j 33 | @CompileStatic 34 | class JcrNodesProcessor implements ItemProcessor { 35 | 36 | private String contentAfterDate 37 | 38 | void setContentAfterDate(String contentAfterDate) { 39 | this.contentAfterDate = contentAfterDate 40 | } 41 | 42 | /** 43 | * Converts a JCR Node to a {@link ProtoNode} object. 44 | * Returns null if current node does not need to be processed 45 | */ 46 | @Override 47 | @Nullable 48 | ProtoNode process(JcrNode jcrNode) throws Exception { 49 | 50 | JCRNodeDecorator decoratedNode = new JCRNodeDecorator(jcrNode) 51 | 52 | if (contentAfterDate) { 53 | final Date afterDate = DateUtil.getDateFromISOString(contentAfterDate) 54 | log.debug "ContentAfterDate received : ${afterDate}. Will ignore content created or modified before the afterDate" 55 | final date = decoratedNode.getModifiedOrCreatedDate() 56 | if (date && date.before(afterDate)) { //if there are no date properties, we treat nodes as new 57 | log.debug "Not sending any data older than ${afterDate}" 58 | return null 59 | } 60 | } 61 | 62 | // Skip some nodes because they have already been processed by their parent 63 | if(decoratedNode.isMandatoryNode() || decoratedNode.isAuthorizablePart() || decoratedNode.isACPart()) { 64 | return null 65 | } else { 66 | // Build parent node 67 | return decoratedNode.toProtoNode() 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/steps/jcrnodes/JcrNodesReader.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch.steps.jcrnodes 18 | 19 | import com.twcable.grabbit.server.batch.ServerBatchJobContext 20 | import groovy.transform.CompileStatic 21 | import org.springframework.batch.item.ItemReader 22 | import org.springframework.batch.item.NonTransientResourceException 23 | import org.springframework.batch.item.ParseException 24 | import org.springframework.batch.item.UnexpectedInputException 25 | 26 | import javax.jcr.Node as JcrNode 27 | 28 | /** 29 | * A Custom ItemReader that provides the "next" Node from the {@link ServerBatchJobContext#nodeIterator}. 30 | * Returns null to indicate that all Items have been read. 31 | */ 32 | @CompileStatic 33 | @SuppressWarnings("GrMethodMayBeStatic") 34 | class JcrNodesReader implements ItemReader { 35 | 36 | @Override 37 | JcrNode read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { 38 | Iterator nodeIterator = theNodeIterator() 39 | if (nodeIterator == null) throw new IllegalStateException("nodeIterator must be set.") 40 | if (nodeIterator.hasNext()) { 41 | nodeIterator.next() 42 | } 43 | else { 44 | null 45 | } 46 | } 47 | 48 | private Iterator theNodeIterator() { 49 | ServerBatchJobContext serverBatchJobContext = ServerBatchJobContext.THREAD_LOCAL.get() 50 | serverBatchJobContext.nodeIterator 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/steps/jcrnodes/JcrNodesWriter.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch.steps.jcrnodes 18 | 19 | import com.twcable.grabbit.proto.NodeProtos 20 | import com.twcable.grabbit.server.batch.ServerBatchJobContext 21 | import groovy.transform.CompileStatic 22 | import groovy.util.logging.Slf4j 23 | import org.springframework.batch.core.ItemWriteListener 24 | import org.springframework.batch.item.ItemWriter 25 | 26 | import javax.servlet.ServletOutputStream 27 | 28 | /** 29 | * A Custom ItemWriter that will write the provided Protocol Buffer Nodes to the {@link JcrNodesWriter#theServletOutputStream()} 30 | * Will flush the {@link JcrNodesWriter#theServletOutputStream()} after writing provided Protocol Buffer Nodes 31 | * @see ItemWriteListener 32 | */ 33 | @Slf4j 34 | @CompileStatic 35 | @SuppressWarnings('GrMethodMayBeStatic') 36 | class JcrNodesWriter implements ItemWriter, ItemWriteListener { 37 | 38 | @Override 39 | void write(List nodeProtos) throws Exception { 40 | ServletOutputStream servletOutputStream = theServletOutputStream() 41 | if (servletOutputStream == null) throw new IllegalStateException("servletOutputStream must be set.") 42 | 43 | nodeProtos.each { NodeProtos.Node node -> 44 | log.debug "Sending NodeProto : ${node.name}" 45 | node.writeDelimitedTo(servletOutputStream) 46 | } 47 | } 48 | 49 | 50 | @Override 51 | void beforeWrite(List items) { 52 | } 53 | 54 | 55 | @Override 56 | void afterWrite(List items) { 57 | theServletOutputStream().flush() 58 | } 59 | 60 | 61 | @Override 62 | void onWriteError(Exception exception, List items) { 63 | log.error "Exception occurred while writing the current chunk", exception 64 | } 65 | 66 | 67 | private ServletOutputStream theServletOutputStream() { 68 | ServerBatchJobContext serverBatchJobContext = ServerBatchJobContext.THREAD_LOCAL.get() 69 | serverBatchJobContext.servletOutputStream 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/steps/namespace/NamespaceProcessor.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch.steps.namespace 18 | 19 | import com.twcable.grabbit.proto.NamespaceProtos.NamespaceEntry 20 | import groovy.transform.CompileStatic 21 | import org.springframework.batch.item.ItemProcessor 22 | 23 | /** 24 | * A Custom ItemProcessor that effectively acts as a Marshaller for Namespace {prefix->url} mapping. 25 | */ 26 | @CompileStatic 27 | class NamespaceProcessor implements ItemProcessor, NamespaceEntry> { 28 | 29 | /** 30 | * Converts a Namespace {prefix->url} Entry to a Protocol Buffer Message {@link NamespaceEntry object} 31 | */ 32 | @Override 33 | NamespaceEntry process(Map.Entry entry) throws Exception { 34 | NamespaceEntry.Builder namespaceEntryBuilder = NamespaceEntry.newBuilder() 35 | namespaceEntryBuilder 36 | .setPrefix(entry.key) 37 | .setUri(entry.value) 38 | .build() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/steps/namespace/NamespaceReader.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch.steps.namespace 18 | 19 | import com.twcable.grabbit.server.batch.ServerBatchJobContext 20 | import groovy.transform.CompileStatic 21 | import org.springframework.batch.item.ItemReader 22 | import org.springframework.batch.item.NonTransientResourceException 23 | import org.springframework.batch.item.ParseException 24 | import org.springframework.batch.item.UnexpectedInputException 25 | 26 | import static java.util.Map.Entry 27 | 28 | /** 29 | * A Custom ItemReader that provides the "next" Namespace Entry from the {@link ServerBatchJobContext#namespacesIterator}. 30 | * Returns null to indicate that all Items have been read. 31 | */ 32 | @CompileStatic 33 | @SuppressWarnings("GrMethodMayBeStatic") 34 | class NamespaceReader implements ItemReader> { 35 | 36 | @Override 37 | Entry read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { 38 | Iterator> namespaces = theNamespaces() 39 | if (namespaces == null) throw new IllegalStateException("namespaces must be set.") 40 | 41 | if (namespaces.hasNext()) { 42 | namespaces.next() 43 | } 44 | else { 45 | null 46 | } 47 | } 48 | 49 | 50 | private Iterator> theNamespaces() { 51 | ServerBatchJobContext serverBatchJobContext = ServerBatchJobContext.THREAD_LOCAL.get() 52 | serverBatchJobContext.namespacesIterator 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/batch/steps/namespace/NamespaceWriter.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.batch.steps.namespace 18 | 19 | import com.twcable.grabbit.proto.NamespaceProtos.NamespaceEntry 20 | import com.twcable.grabbit.proto.NamespaceProtos.NamespaceRegistry 21 | import com.twcable.grabbit.proto.NamespaceProtos.NamespaceRegistry.Builder as NameSpaceRegistryBuilder 22 | import com.twcable.grabbit.server.batch.ServerBatchJobContext 23 | import groovy.transform.CompileStatic 24 | import groovy.util.logging.Slf4j 25 | import org.springframework.batch.core.ItemWriteListener 26 | import org.springframework.batch.item.ItemWriter 27 | 28 | import javax.servlet.ServletOutputStream 29 | 30 | /** 31 | * A Custom ItemWriter that will write the provided Protocol Buffer NamespacesEntries to the {@link NamespaceWriter#theServletOutputStream()} 32 | * Will flush the {@link NamespaceWriter#theServletOutputStream()} after writing provided Protocol Buffer NamespaceEntries 33 | * @see ItemWriteListener 34 | */ 35 | @Slf4j 36 | @CompileStatic 37 | @SuppressWarnings('GrMethodMayBeStatic') 38 | class NamespaceWriter implements ItemWriter, ItemWriteListener { 39 | 40 | @Override 41 | void write(List namespaceEntries) throws Exception { 42 | ServletOutputStream servletOutputStream = theServletOutputStream() 43 | if (servletOutputStream == null) throw new IllegalStateException("servletOutputStream must be set.") 44 | 45 | NameSpaceRegistryBuilder namespaceRegistryBuilder = NamespaceRegistry.newBuilder() 46 | NamespaceRegistry namespaceRegistry = namespaceRegistryBuilder.addAllEntry(namespaceEntries).build() 47 | 48 | log.debug "Writing Namespace Registry : ${NamespaceRegistry}" 49 | namespaceRegistry.writeDelimitedTo(servletOutputStream) 50 | } 51 | 52 | 53 | @Override 54 | void beforeWrite(List items) { 55 | //no-op 56 | } 57 | 58 | 59 | @Override 60 | void afterWrite(List items) { 61 | theServletOutputStream().flush() 62 | } 63 | 64 | 65 | @Override 66 | void onWriteError(Exception exception, List items) { 67 | log.error "Exception occurred while writing the current chunk", exception 68 | } 69 | 70 | 71 | private ServletOutputStream theServletOutputStream() { 72 | ServerBatchJobContext serverBatchJobContext = ServerBatchJobContext.THREAD_LOCAL.get() 73 | serverBatchJobContext.servletOutputStream 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/services/ExcludePathNodeIterator.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.services 18 | 19 | import groovy.transform.CompileStatic 20 | import javax.jcr.Node as JcrNode 21 | 22 | /** 23 | * Custom Wrapper for Node Iterator that will iterate through a list of Nodes containing of the root node and its children 24 | * Accounts for cases where certain paths(i.e nodes) needs to be excluded. 25 | */ 26 | @CompileStatic 27 | final class ExcludePathNodeIterator implements Iterator { 28 | 29 | private Iterator nodeIterator 30 | private Collection excludePathList 31 | 32 | public ExcludePathNodeIterator(Iterator nodeIterator, Collection excludePaths) { 33 | this.nodeIterator = nodeIterator 34 | this.excludePathList = (excludePaths == null) ? (Collection)Collections.EMPTY_LIST : excludePaths 35 | } 36 | 37 | @Override 38 | boolean hasNext() { 39 | nodeIterator.hasNext() 40 | } 41 | 42 | @Override 43 | JcrNode next() { 44 | if(!excludePathList.isEmpty()) 45 | (JcrNode)nodeIterator.find { JcrNode node -> !isPathInExcludedList(node.path) } 46 | else 47 | nodeIterator.next() 48 | } 49 | 50 | @Override 51 | void remove() { 52 | nodeIterator.remove() 53 | } 54 | 55 | private boolean isPathInExcludedList(String path) { 56 | return excludePathList.any { path.equals(it) || path.startsWith("${it}/") } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/services/RootNodeWithMandatoryIterator.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.services 18 | 19 | import com.twcable.grabbit.jcr.JCRNodeDecorator 20 | import groovy.transform.CompileStatic 21 | import groovy.transform.TailRecursive 22 | 23 | import javax.jcr.Node as JcrNode 24 | 25 | /** 26 | * Custom Node Iterator that will iterate over a rootNode node and its immediate children 27 | * taking into account cases where a node (rootNode node or any of the children) has any number of 28 | * mandatory subnodes 29 | */ 30 | @CompileStatic 31 | final class RootNodeWithMandatoryIterator implements Iterator { 32 | 33 | private boolean doneRoot 34 | private JCRNodeDecorator rootNode 35 | private Iterator immediateChildren 36 | private Iterator mandatoryWriteNodes 37 | 38 | 39 | public RootNodeWithMandatoryIterator(JcrNode root) { 40 | this.rootNode = new JCRNodeDecorator(root) 41 | this.doneRoot = false 42 | //Get all immediate children that are not mandatory write nodes. We will handle those by iterating over mandatoryWriteNodes 43 | immediateChildren = getNonMandatoryChildren(this.rootNode).iterator() 44 | //Calls a tail recursive method, gathering all required nodes 45 | mandatoryWriteNodes = getMandatoryChildren(rootNode, []).iterator() 46 | } 47 | 48 | 49 | @Override 50 | boolean hasNext() { 51 | !doneRoot || immediateChildren.hasNext() || mandatoryWriteNodes.hasNext() 52 | } 53 | 54 | 55 | @Override 56 | JcrNode next() { 57 | if(!hasNext()) throw new NoSuchElementException("No more elements in Iterator") 58 | 59 | //always process rootNode first 60 | if (!doneRoot) { 61 | doneRoot = true 62 | return rootNode 63 | } 64 | 65 | if(immediateChildren.hasNext()) { 66 | return immediateChildren.next() 67 | } 68 | 69 | return mandatoryWriteNodes.next() 70 | } 71 | 72 | 73 | @Override 74 | void remove() { 75 | throw new UnsupportedOperationException("Not supported.") 76 | } 77 | 78 | 79 | private static Collection getNonMandatoryChildren(final JCRNodeDecorator node) { 80 | node.getImmediateChildNodes().findAll { !it.isMandatoryNode() } 81 | } 82 | 83 | 84 | @TailRecursive 85 | private static Collection getMandatoryChildren(final JCRNodeDecorator currentNode, final Collection nodesToAdd) { 86 | if(!currentNode.hasMandatoryChildNodes()) { 87 | return nodesToAdd 88 | } 89 | 90 | final mandatoryNodes = currentNode.getRequiredChildNodes() 91 | 92 | return mandatoryNodes.collectMany { JCRNodeDecorator mandatoryNode -> 93 | return getMandatoryChildren(mandatoryNode, (nodesToAdd << mandatoryNode)) 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/server/services/ServerService.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.server.services 18 | 19 | import javax.servlet.ServletOutputStream 20 | 21 | interface ServerService { 22 | 23 | /** 24 | * Accepts a rootPath, retrieves content and writes it to the {@param servletOutputStream} 25 | * @param path 26 | * @param servletOutputStream 27 | */ 28 | void getContentForRootPath(String serverUsername, String path, Collection excludePaths, String afterDateString, ServletOutputStream servletOutputStream) 29 | } 30 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/AbstractJcrDao.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import groovy.transform.CompileStatic 20 | import groovy.util.logging.Slf4j 21 | 22 | /** 23 | * A simple base class that is extended by all the DAOs 24 | */ 25 | @CompileStatic 26 | @Slf4j 27 | abstract class AbstractJcrDao { 28 | 29 | public static final String ROOT_RESOURCE_NAME = "/var/grabbit/job/repository" 30 | 31 | protected abstract void ensureRootResource() 32 | } 33 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitExecutionContextDao.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import org.springframework.batch.core.repository.dao.ExecutionContextDao 20 | import org.springframework.batch.item.ExecutionContext; 21 | 22 | 23 | /** 24 | * Modified DAO Interface for persisting and retrieving {@link ExecutionContext} 25 | * @see ExecutionContextDao for more details 26 | */ 27 | interface GrabbitExecutionContextDao extends ExecutionContextDao { 28 | 29 | /** 30 | * Returns job execution context paths by comparing "executionId" property on "executionContext/job/" with 31 | * "executionId" property on JobExecutions for the @param jobExecutionResourcePaths 32 | */ 33 | public Collection getJobExecutionContextPaths(Collection jobExecutionResourcePaths) 34 | 35 | /** 36 | * Returns step execution context paths by comparing "executionId" property on "executionContext/job/" with 37 | * "id" property on StepExecutions for the @param stepExecutionResourcePaths 38 | */ 39 | public Collection getStepExecutionContextPaths(Collection stepExecutionResourcePaths) 40 | } 41 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobExecution.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import groovy.transform.CompileStatic 20 | import groovy.transform.InheritConstructors 21 | import org.springframework.batch.core.JobExecution 22 | 23 | /** 24 | * GrabbitJobExecution is a simple extension of {@Link JobExecution}. 25 | * 26 | *

27 | * It simply provides a new property "transactionID." This transactionID allows us to associate a group 28 | * of job executions together by their common run transaction (configuration run). For example, if I 29 | * give Grabbit a configuration with two paths, two job executions will be started, and they could both then 30 | * be grouped/referenced by their common transactionID. 31 | *

32 | * 33 | * @see {@link JcrGrabbitJobExecutionDao} for a good handle on how we 34 | * serve up this JobExecution during the Spring Batch lifecycle. 35 | */ 36 | @CompileStatic 37 | @InheritConstructors 38 | class GrabbitJobExecution extends JobExecution { 39 | long transactionID 40 | } 41 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobExecutionDao.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import org.springframework.batch.core.BatchStatus 20 | import org.springframework.batch.core.JobExecution 21 | import org.springframework.batch.core.repository.dao.JobExecutionDao 22 | 23 | /** 24 | * Modified DAO Interface for persisting and retrieving {@link JobExecution} 25 | * @see JobExecutionDao for more details 26 | */ 27 | interface GrabbitJobExecutionDao extends JobExecutionDao{ 28 | 29 | /** 30 | * Returns job execution paths for given BatchStatuses 31 | */ 32 | public Collection getJobExecutions(Collection batchStatuses) 33 | 34 | /** 35 | * Returns job execution paths which ended @param hours ago from "Now" 36 | */ 37 | public Collection getJobExecutions(int hours, Collection jobExecutionPaths) 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobInstanceDao.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import org.springframework.batch.core.JobInstance 20 | import org.springframework.batch.core.repository.dao.JobInstanceDao 21 | 22 | /** 23 | * Modified DAO Interface for persisting and retrieving {@link JobInstance} 24 | * @see JobInstanceDao for more details 25 | */ 26 | interface GrabbitJobInstanceDao extends JobInstanceDao { 27 | 28 | /** 29 | * Returns job instance paths by comparing "id" property on "jobInstances" with 30 | * "instanceId" property on JobExecutions for the @param jobExecutionResourcePaths 31 | */ 32 | public Collection getJobInstancePaths(Collection jobExecutionResourcePaths) 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitStepExecutionDao.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import org.springframework.batch.core.StepExecution 20 | import org.springframework.batch.core.repository.dao.StepExecutionDao 21 | 22 | /** 23 | * Modified DAO Interface for persisting and retrieving {@link StepExecution} 24 | * @see StepExecutionDao for more details 25 | */ 26 | interface GrabbitStepExecutionDao extends StepExecutionDao { 27 | 28 | /** 29 | * Returns step execution paths by comparing "jobExecutionId" property on "stepExecutions with 30 | * "executionId" property on JobExecutions for the @param jobExecutionResourcePaths 31 | */ 32 | public Collection getStepExecutionPaths(Collection jobExecutionResourcePaths) 33 | } 34 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/JcrJobExplorerFactoryBean.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository 18 | 19 | import groovy.transform.CompileStatic 20 | import org.springframework.batch.core.explore.JobExplorer 21 | import org.springframework.batch.core.explore.support.AbstractJobExplorerFactoryBean 22 | import org.springframework.batch.core.explore.support.SimpleJobExplorer 23 | import org.springframework.batch.core.repository.dao.ExecutionContextDao 24 | import org.springframework.batch.core.repository.dao.JobExecutionDao 25 | import org.springframework.batch.core.repository.dao.JobInstanceDao 26 | import org.springframework.batch.core.repository.dao.StepExecutionDao 27 | import org.springframework.beans.factory.InitializingBean 28 | import org.springframework.util.Assert 29 | 30 | /** 31 | * A {@link org.springframework.beans.factory.FactoryBean} that automates the creation of a 32 | * {@link SimpleJobExplorer} using JCR DAO implementations. 33 | * 34 | * @see JcrJobRepositoryFactoryBean 35 | */ 36 | @CompileStatic 37 | public class JcrJobExplorerFactoryBean extends AbstractJobExplorerFactoryBean implements InitializingBean { 38 | 39 | private JcrJobRepositoryFactoryBean repositoryFactory 40 | 41 | 42 | public void setRepositoryFactory(JcrJobRepositoryFactoryBean repositoryFactory) { 43 | this.repositoryFactory = repositoryFactory; 44 | } 45 | 46 | 47 | JcrJobExplorerFactoryBean(JcrJobRepositoryFactoryBean repositoryFactory) { 48 | this.repositoryFactory = repositoryFactory 49 | } 50 | 51 | 52 | JcrJobExplorerFactoryBean() {} 53 | 54 | 55 | @Override 56 | protected JobInstanceDao createJobInstanceDao() throws Exception { 57 | repositoryFactory.jobInstanceDao 58 | } 59 | 60 | 61 | @Override 62 | protected JobExecutionDao createJobExecutionDao() throws Exception { 63 | repositoryFactory.jobExecutionDao 64 | } 65 | 66 | 67 | @Override 68 | protected StepExecutionDao createStepExecutionDao() throws Exception { 69 | repositoryFactory.stepExecutionDao 70 | } 71 | 72 | 73 | @Override 74 | protected ExecutionContextDao createExecutionContextDao() throws Exception { 75 | repositoryFactory.executionContextDao 76 | } 77 | 78 | 79 | @Override 80 | public JobExplorer getObject() throws Exception { 81 | return new SimpleJobExplorer(createJobInstanceDao(), createJobExecutionDao(), createStepExecutionDao(), 82 | createExecutionContextDao()); 83 | 84 | } 85 | 86 | 87 | @Override 88 | void afterPropertiesSet() throws Exception { 89 | Assert.state(repositoryFactory != null, "A JcrJobRepositoryFactoryBean must be provided") 90 | repositoryFactory.afterPropertiesSet() 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/services/CleanJobRepository.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository.services 18 | 19 | interface CleanJobRepository { 20 | 21 | /** 22 | * This API is used to clean Grabbit's JCR Job Repository. It removes all job executions and all associated 23 | * job instances etc. that are @param hours older than NOW (time when this API is called) 24 | * @return Collection of JobExecutionIds that were removed 25 | */ 26 | public Collection cleanJobRepository(int hours) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/spring/batch/repository/servlets/GrabbitCleanJobRepositoryServlet.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository.servlets 18 | 19 | import com.twcable.grabbit.spring.batch.repository.services.CleanJobRepository 20 | import groovy.transform.CompileStatic 21 | import groovy.util.logging.Slf4j 22 | import org.apache.felix.scr.annotations.Reference 23 | import org.apache.felix.scr.annotations.sling.SlingServlet 24 | import org.apache.sling.api.SlingHttpServletRequest 25 | import org.apache.sling.api.SlingHttpServletResponse 26 | import org.apache.sling.api.servlets.SlingAllMethodsServlet 27 | import org.apache.sling.api.servlets.SlingSafeMethodsServlet 28 | 29 | import javax.annotation.Nonnull 30 | import javax.servlet.http.HttpServletResponse 31 | 32 | /** 33 | * This servlet is used for cleanup of the Grabbit's JobRepository stored under /var/grabbit/job. 34 | * 35 | * It is a handler for the {@link com.twcable.grabbit.resources.CleanJobRepositoryResource} resource. 36 | */ 37 | @Slf4j 38 | @CompileStatic 39 | @SlingServlet(methods = ['POST'], resourceTypes = ['twcable:grabbit/jobrepository/clean']) 40 | class GrabbitCleanJobRepositoryServlet extends SlingAllMethodsServlet { 41 | 42 | @Reference 43 | CleanJobRepository cleanJobRepository 44 | 45 | @Override 46 | protected void doPost( @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) { 47 | String hoursParam = request.getParameter("hours") ?: "" 48 | if(!hoursParam.isInteger()) { 49 | log.warn "Parameter 'hours' must be an integer" 50 | response.status = HttpServletResponse.SC_BAD_REQUEST 51 | response.writer.write("Parameter 'hours' must be an integer") 52 | return 53 | } 54 | int hours = hoursParam.toInteger() 55 | Collection removedJobExecutions = cleanJobRepository.cleanJobRepository(hours) 56 | response.status = HttpServletResponse.SC_OK 57 | response.writer.write ("JobExecutions and the corresponding JobInstances, StepExecutions and ExecutionContexts " + 58 | "were removed. JobExecutionsIds that were removed: ${removedJobExecutions}") 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/groovy/com/twcable/grabbit/util/CryptoUtil.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.util 17 | 18 | import groovy.transform.CompileStatic 19 | 20 | @CompileStatic 21 | class CryptoUtil { 22 | 23 | /** 24 | * @return a unique positive UUID (Long) 25 | */ 26 | public static Long generateNextId() { 27 | final uuid = UUID.randomUUID() 28 | long nextId = uuid.mostSignificantBits ^ uuid.leastSignificantBits 29 | //Making sure that the nextId is not negative. 30 | nextId >>>= 1 31 | return Long.valueOf(nextId) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/proto/namespace.proto: -------------------------------------------------------------------------------- 1 | package com.twcable.grabbit.proto; 2 | 3 | option java_package = "com.twcable.grabbit.proto"; 4 | option java_outer_classname = "NamespaceProtos"; 5 | 6 | message NamespaceRegistry { 7 | repeated NamespaceEntry entry = 1; 8 | } 9 | 10 | message NamespaceEntry { 11 | required string prefix = 1; 12 | required string uri = 2; 13 | } 14 | -------------------------------------------------------------------------------- /src/main/proto/node.proto: -------------------------------------------------------------------------------- 1 | package com.twcable.grabbit.proto; 2 | 3 | option java_package = "com.twcable.grabbit.proto"; 4 | option java_outer_classname = "NodeProtos"; 5 | 6 | message Node { 7 | required string name = 1; 8 | repeated Property properties = 2; 9 | repeated Node mandatoryChildNode = 3; 10 | } 11 | 12 | message Property { 13 | required string name = 1; 14 | required uint32 type = 2; 15 | required bool multiple = 3; 16 | repeated Value values = 4; 17 | } 18 | 19 | message Value { 20 | //Essentially a C-Style union 21 | oneof type { 22 | string stringValue = 1; 23 | bytes bytesValue = 2; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-batch-http.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-deleteBeforeWrite-steps.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-jcrNodes-step.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-namespace-step.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-osgi-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-validateJob.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-workflow-off-step.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/client-workflow-on-step.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/server-batch-job-context.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 24 | 25 | 26 | 29 | 30 | 31 | 34 | 35 | 36 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/server-batch-job.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 37 | 38 | 39 | 40 | 41 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/server-jcrNodes-step.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 23 | 24 | 25 | Steps for serverJcrNodes 26 | - "serverJcrNodesReader" - Returns the 'next()' Node from the passed in Lazy Node Iterator 27 | - "serverJcrNodesProcessor" - Effectively a Marshaller, converts a JCR Node to Protobuf Message Node object 28 | - "serverProtobufNodesWriter" - Writes a "chunk" of Protobuf Message Nodes to the provided Output Stream 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring/server-namespace-step.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 22 | 23 | 24 | Steps for serverNamespaceSync 25 | - "serverNamespaceReader" - Returns the "next()" "namespace prefix -> url" Entry from the provided Namespaces Iterator 26 | - "serverNamespaceProcessor" - Effectively a Marshaller, converts a "namespace prefix -> url" Entry to Protobuf Message NamespaceEntry object 27 | - "serverNamespaceWriter" - Writes a "chunk" of Protobuf Message NamespaceEntries to the provided Output Stream 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/main/resources/com/twcable/grabbit/client/servlets/GrabbitTransactionResource-BadRequest.txt: -------------------------------------------------------------------------------- 1 | Request was made to get status of a transaction, but a transactionID was not provided for /grabbit/transaction.
2 | Click to view a list of all transactions. -------------------------------------------------------------------------------- /src/main/resources/com/twcable/grabbit/client/servlets/GrabbitTransactionResource-all.txt: -------------------------------------------------------------------------------- 1 | Click on one of the following Transaction ids for Job Details
-------------------------------------------------------------------------------- /src/main/resources/com/twcable/grabbit/client/servlets/RootResource.txt: -------------------------------------------------------------------------------- 1 | Grabbit Does not support direct requests on this url.
2 | 3 | Visit one of the following links for more info!
4 | 5 | 6 | Job Requests 7 |
8 | 9 | Content Requests 10 |
11 | 12 | Transaction Requests 13 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/client/batch/ClientBatchJobContextSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.client.batch 17 | 18 | import spock.lang.Specification 19 | import spock.lang.Subject 20 | 21 | import javax.jcr.Session 22 | 23 | import java.util.concurrent.Callable as Job 24 | import java.util.concurrent.Executor 25 | import java.util.concurrent.ExecutorService 26 | import java.util.concurrent.Executors 27 | 28 | @Subject(ClientBatchJobContext) 29 | class ClientBatchJobContextSpec extends Specification { 30 | 31 | Job setSessionAndInputStreamOnJob(final Session session, final InputStream inputStream) { 32 | return new Job() { 33 | @Override 34 | List call() throws Exception { 35 | ClientBatchJobContext.setSession(session) 36 | ClientBatchJobContext.setInputStream(inputStream) 37 | return [ClientBatchJobContext.session, ClientBatchJobContext.inputStream] as List 38 | } 39 | } 40 | } 41 | 42 | def "Jobs running on separate threads can set their own ClientBatchJobContext values"() { 43 | given: 44 | final Session jobOneSession = Mock(Session) 45 | final InputStream jobOneInputStream = Mock(InputStream) 46 | 47 | final Session jobTwoSession = Mock(Session) 48 | final InputStream jobTwoInputStream = Mock(InputStream) 49 | 50 | //This will set unique objects on each job's ClientBatchJobContext 51 | final Job jobOne = setSessionAndInputStreamOnJob(jobOneSession, jobOneInputStream) 52 | final Job jobTwo = setSessionAndInputStreamOnJob(jobTwoSession, jobTwoInputStream) 53 | 54 | //We should get a list of the same values back, unchanged by the other job's thread 55 | ExecutorService executorService = Executors.newFixedThreadPool(2) 56 | final List jobOneObjects = executorService.submit(jobOne).get() 57 | final List jobTwoObjects = executorService.submit(jobTwo).get() 58 | executorService.shutdown() 59 | 60 | expect: 61 | jobOneObjects.get(0).is jobOneSession 62 | jobOneObjects.get(1).is jobOneInputStream 63 | 64 | jobTwoObjects.get(0).is jobTwoSession 65 | jobTwoObjects.get(1).is jobTwoInputStream 66 | } 67 | 68 | def "Session, and InputStream are cleaned up correctly on cleanup()"() { 69 | given: 70 | final Session session = Mock(Session) { 71 | isLive() >> true 72 | } 73 | final InputStream inputStream = Mock(InputStream) 74 | 75 | when: 76 | ClientBatchJobContext.setSession(session) 77 | ClientBatchJobContext.setInputStream(inputStream) 78 | 79 | ClientBatchJobContext.cleanup() 80 | 81 | then: 82 | 1 * session.logout() 83 | 1 * inputStream.close() 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/client/batch/ClientBatchJobSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch 18 | 19 | import com.twcable.grabbit.DateUtil 20 | import com.twcable.grabbit.GrabbitConfiguration 21 | import org.springframework.batch.core.BatchStatus 22 | import org.springframework.batch.core.JobExecution 23 | import org.springframework.batch.core.JobParametersBuilder 24 | import org.springframework.batch.core.launch.JobOperator 25 | import org.springframework.context.ConfigurableApplicationContext 26 | import spock.lang.Shared 27 | import spock.lang.Specification 28 | import spock.lang.Subject 29 | import spock.lang.Unroll 30 | 31 | @Subject(ClientBatchJob) 32 | class ClientBatchJobSpec extends Specification { 33 | 34 | @Shared 35 | def dateNow 36 | 37 | 38 | def setupSpec() { 39 | dateNow = new Date() 40 | } 41 | 42 | 43 | @Unroll 44 | def "Make sure ClientBatch job gets configured correctly"() { 45 | when: 46 | final appContext = Mock(ConfigurableApplicationContext) 47 | appContext.getBean(_ as String, JobOperator) >> Mock(JobOperator) 48 | final job = new ClientBatchJob.ServerBuilder(appContext) 49 | .andServer("scheme", "host", "port") 50 | .andCredentials("clientUser", "serverUser", "serverPass") 51 | .andClientJobExecutions(jobExecutions) 52 | .andConfiguration(new GrabbitConfiguration.PathConfiguration(path, [], [], deleteBeforeWrite, pathDeltaContent, 100)) 53 | .build() 54 | 55 | then: 56 | job != null 57 | job.jobParameters != null 58 | job.jobParameters.get(ClientBatchJob.PATH) == path 59 | job.jobParameters.get(ClientBatchJob.CONTENT_AFTER_DATE) == contentAfterDate 60 | job.jobParameters.get(ClientBatchJob.DELETE_BEFORE_WRITE).toBoolean() == deleteBeforeWrite 61 | job.jobParameters.get(ClientBatchJob.PATH_DELTA_CONTENT).toBoolean() == pathDeltaContent 62 | 63 | where: 64 | pathDeltaContent | path | contentAfterDate | deleteBeforeWrite 65 | true | "/path1" | DateUtil.getISOStringFromDate(dateNow) | true 66 | false | "/path1" | null | false 67 | true | "/path2" | null | true 68 | false | "/path2" | null | false 69 | } 70 | 71 | 72 | def getJobExecutions() { 73 | def ex1 = new JobExecution(1, new JobParametersBuilder().addString("path", "/path1").toJobParameters()) 74 | ex1.endTime = dateNow 75 | ex1.status = BatchStatus.COMPLETED 76 | def ex2 = new JobExecution(2, new JobParametersBuilder().addString("path", "/path2").toJobParameters()) 77 | ex2.endTime = dateNow 78 | ex2.status = BatchStatus.FAILED 79 | 80 | [ex1, ex2] 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/client/batch/steps/workspace/DeleteBeforeWriteDeciderSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.client.batch.steps.workspace 17 | 18 | import com.twcable.grabbit.client.batch.ClientBatchJob 19 | import org.springframework.batch.core.JobExecution 20 | import org.springframework.batch.core.JobParameters 21 | import org.springframework.batch.core.StepExecution 22 | import org.springframework.batch.core.job.flow.FlowExecutionStatus 23 | import spock.lang.Specification 24 | 25 | class DeleteBeforeWriteDeciderSpec extends Specification { 26 | 27 | def "Decides when to delete before writing nodes correctly"() { 28 | when: 29 | final jobExecution = Mock(JobExecution) { 30 | getJobParameters() >> Mock(JobParameters){ 31 | getString(ClientBatchJob.DELETE_BEFORE_WRITE) >> deleteBeforeWriteParameter 32 | } 33 | } 34 | 35 | final deleteBeforeWriteDecider = new DeleteBeforeWriteDecider() 36 | final flowExecutionStatus = deleteBeforeWriteDecider.decide(jobExecution, Mock(StepExecution)) 37 | then: 38 | flowExecutionStatus == expectedFlowExecutionStatus 39 | 40 | where: 41 | deleteBeforeWriteParameter | expectedFlowExecutionStatus 42 | "true" | new FlowExecutionStatus("YES") 43 | "false" | new FlowExecutionStatus("NO") 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/client/batch/workflows/StubWorkflowLauncher.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.workflows 18 | 19 | import com.day.cq.workflow.launcher.ConfigEntry 20 | import com.day.cq.workflow.launcher.WorkflowLauncher 21 | import groovy.transform.CompileStatic 22 | import groovy.util.logging.Slf4j 23 | 24 | import javax.jcr.RepositoryException 25 | 26 | /** 27 | * A simple Stubbed {@link WorkflowLauncher} implementation to simplify 28 | * tracking and verification when testing {@link WorkflowManager} 29 | */ 30 | @Slf4j 31 | @CompileStatic 32 | class StubWorkflowLauncher implements WorkflowLauncher { 33 | 34 | private int editConfigCallCount 35 | 36 | private List configEntries 37 | 38 | 39 | public StubWorkflowLauncher(int noOfEntries) { 40 | editConfigCallCount = 0 41 | configEntries = (1..noOfEntries).collect { i -> 42 | new ConfigEntry(0, "", "", "", "", "id${i}", "", true, [], []) 43 | } 44 | } 45 | 46 | 47 | @Override 48 | void addConfigEntry(ConfigEntry configEntry) throws RepositoryException { 49 | log.info "Adding configEntry : ${configEntry}" 50 | } 51 | 52 | 53 | @Override 54 | void removeConfigEntry(String s) throws RepositoryException { 55 | log.info "Removing configEntry with ID : ${s}" 56 | } 57 | 58 | 59 | @Override 60 | List getConfigEntries() { 61 | configEntries 62 | } 63 | 64 | 65 | @Override 66 | void editConfigEntry(String s, ConfigEntry configEntry) throws RepositoryException { 67 | log.info "Editing configEntry with ID : ${s} with new entry: ${configEntry}" 68 | configEntries.find { entry -> entry.id == configEntry.id }.enabled = configEntry.enabled 69 | editConfigCallCount++ 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/client/batch/workflows/WorkflowManagerSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.client.batch.workflows 18 | 19 | import com.day.cq.workflow.launcher.WorkflowLauncher 20 | import com.twcable.grabbit.client.batch.workflows.impl.DefaultWorkFlowManager 21 | import spock.lang.Specification 22 | import spock.lang.Subject 23 | 24 | import java.util.concurrent.Callable 25 | import java.util.concurrent.ExecutorService 26 | import java.util.concurrent.Executors 27 | import java.util.concurrent.Future 28 | import java.util.concurrent.TimeUnit 29 | 30 | @Subject(WorkflowManager) 31 | class WorkflowManagerSpec extends Specification { 32 | 33 | def "Should manage multithreading correctly"() { 34 | setup: 35 | ExecutorService threadPool = Executors.newCachedThreadPool() 36 | WorkflowLauncher workflowLauncher = new StubWorkflowLauncher(5) 37 | WorkflowManager workflowManager = new DefaultWorkFlowManager(workflowLauncher: workflowLauncher) 38 | Collection workflowIds = ['id1', 'id2', 'id3'] 39 | 40 | //Simulates a WhiteList with 10 paths. 41 | //In this case (worst case for the most part), all of the 10 paths 42 | //Have requested the same 3 workflows to be turned off 43 | def callables = (1..10).collect { i -> 44 | new Callable() { 45 | @Override 46 | Void call() throws Exception { 47 | workflowManager.turnOff(workflowIds) 48 | Thread.sleep(new Random().nextInt(200)) 49 | workflowManager.turnOn(workflowIds) 50 | } 51 | } 52 | } 53 | 54 | when: 55 | workflowManager.activate() 56 | Collection futures = threadPool.invokeAll(callables) 57 | 58 | then: 59 | futures.each { Future future -> future.get(20, TimeUnit.SECONDS) } 60 | 61 | //Number of times editConfig is called must be equal to twice 62 | //the number of unique workflows that were requested to be turned off 63 | workflowLauncher.editConfigCallCount == workflowIds.size() * 2 64 | 65 | cleanup: 66 | workflowManager.deactivate() 67 | threadPool.shutdown() 68 | threadPool.awaitTermination(10, TimeUnit.SECONDS) 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/client/servlets/GrabbitRootServletSpec.groovy: -------------------------------------------------------------------------------- 1 | package twcable.grabbit.client.servlets 2 | 3 | import com.twcable.grabbit.client.servlets.GrabbitRootServlet 4 | import org.apache.sling.api.SlingHttpServletRequest 5 | import org.apache.sling.api.SlingHttpServletResponse 6 | import org.apache.sling.api.resource.Resource 7 | import static javax.servlet.http.HttpServletResponse.SC_OK 8 | import spock.lang.Specification 9 | import spock.lang.Unroll 10 | 11 | /* 12 | * Copyright 2015 Time Warner Cable, Inc. 13 | * 14 | * Licensed under the Apache License, Version 2.0 (the "License"); 15 | * you may not use this file except in compliance with the License. 16 | * You may obtain a copy of the License at 17 | * 18 | * http://www.apache.org/licenses/LICENSE-2.0 19 | * 20 | * Unless required by applicable law or agreed to in writing, software 21 | * distributed under the License is distributed on an "AS IS" BASIS, 22 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | * See the License for the specific language governing permissions and 24 | * limitations under the License. 25 | */ 26 | class GrabbitRootServletSpec extends Specification { 27 | 28 | @Unroll 29 | def "Request to #path will result in OK response"() { 30 | given: 31 | final request = Mock(SlingHttpServletRequest) { 32 | getResource() >> Mock(Resource) { 33 | } 34 | getPathInfo() >> "/grabbit/${path}" 35 | } 36 | final response = Mock(SlingHttpServletResponse) { 37 | 1 * setStatus(SC_OK) 38 | getWriter() >> Mock(PrintWriter) 39 | } 40 | 41 | final grabbitRootServlet = new GrabbitRootServlet() 42 | 43 | expect: 44 | grabbitRootServlet.doGet(request, response) 45 | 46 | where: 47 | path << ["", null, "invalid", 1] 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/jcr/JcrPropertyDecoratorSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.jcr 18 | 19 | import javax.jcr.Property 20 | import javax.jcr.nodetype.PropertyDefinition 21 | import spock.lang.Specification 22 | import spock.lang.Unroll 23 | 24 | 25 | import static org.apache.jackrabbit.JcrConstants.JCR_LASTMODIFIED 26 | import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES 27 | import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE 28 | 29 | @SuppressWarnings("GroovyAssignabilityCheck") 30 | class JcrPropertyDecoratorSpec extends Specification { 31 | 32 | @Unroll 33 | def "check if property is transferable"() { 34 | given: 35 | Property property = Mock(Property) { 36 | getName() >> propertyName 37 | getDefinition() >> Mock(PropertyDefinition) { 38 | isProtected() >> protectedFlag 39 | } 40 | } 41 | final nodeOwner = Mock(JCRNodeDecorator) { 42 | isAuthorizableType() >> authorizableType 43 | isACType() >> acType 44 | } 45 | 46 | when: 47 | final propertyDecorator = new JcrPropertyDecorator(property, nodeOwner) 48 | 49 | then: 50 | expectedOutput == propertyDecorator.isTransferable() 51 | 52 | where: 53 | propertyName | protectedFlag | expectedOutput | authorizableType | acType 54 | JCR_LASTMODIFIED | true | false | false | false 55 | JCR_PRIMARYTYPE | false | true | false | false 56 | JCR_MIXINTYPES | false | true | false | false 57 | 'otherProperty' | true | false | false | false 58 | 'otherProperty' | false | true | false | false 59 | 'protectedProperty' | true | true | true | false 60 | 'protectedProperty' | true | true | true | false 61 | 'rep:privileges' | true | true | false | true 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/jcr/ProtoMock.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.jcr 18 | 19 | import com.google.protobuf.ByteString 20 | import com.twcable.grabbit.proto.NodeProtos 21 | 22 | import javax.jcr.PropertyType 23 | 24 | class ProtoMock { 25 | 26 | static getSingleValueProperty(final int type = PropertyType.STRING, final String value = "value") { 27 | return NodeProtos.Property.newBuilder(). 28 | setName("name"). 29 | setType(type). 30 | setValue(NodeProtos.Value.newBuilder(). 31 | setStringValue(value). 32 | setBytesValue(ByteString.copyFrom(value.bytes)) 33 | ).build() 34 | } 35 | 36 | 37 | static getMultiValuedProperty(final int type = PropertyType.STRING, final String... values = ["value1", "value2"]) { 38 | 39 | final theValues = values.collect { String value -> 40 | NodeProtos.Value.newBuilder(). 41 | setStringValue(value). 42 | setBytesValue(ByteString.copyFrom(value.bytes)).build() 43 | } 44 | return NodeProtos.Property.newBuilder(). 45 | setName("name"). 46 | setType(type). 47 | setValues(NodeProtos.Values.newBuilder(). 48 | addAllValue(theValues) 49 | ).build() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/resources/ContentResourceSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.resources 17 | 18 | import org.apache.sling.api.resource.ResourceResolver 19 | import spock.lang.Specification 20 | 21 | import static com.twcable.grabbit.resources.ContentResource.CONTENT_RESOURCE_TYPE 22 | 23 | class ContentResourceSpec extends Specification { 24 | 25 | def "ContentResource is created as expected"() { 26 | given: 27 | final ContentResource contentResource = new ContentResource(Mock(ResourceResolver), "/resolution/path") 28 | 29 | expect: 30 | contentResource.getPath() == "/resolution/path" 31 | contentResource.getResourceType() == CONTENT_RESOURCE_TYPE 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/resources/GrabbitResourceProviderSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.resources 18 | 19 | import org.apache.sling.api.resource.ResourceResolver 20 | import spock.lang.Specification 21 | import spock.lang.Subject 22 | import spock.lang.Unroll 23 | 24 | @Subject(GrabbitResourceProvider) 25 | class GrabbitResourceProviderSpec extends Specification { 26 | 27 | @Unroll 28 | def "Can provide a job resource from path #path"() { 29 | given: 30 | GrabbitResourceProvider provider = new GrabbitResourceProvider() 31 | 32 | when: 33 | final resource = provider.getResource(Mock(ResourceResolver), path) 34 | 35 | then: 36 | resource instanceof JobResource 37 | 38 | where: 39 | path << [ 40 | '/grabbit/job/all.json', 41 | '/grabbit/job/1.json', 42 | '/grabbit/job/1.html', 43 | '/grabbit/job/1', 44 | '/grabbit/job', 45 | '/grabbit/job/' 46 | ] 47 | } 48 | 49 | 50 | @Unroll 51 | def "Can provide a transaction resource from path #path"() { 52 | given: 53 | GrabbitResourceProvider provider = new GrabbitResourceProvider() 54 | 55 | when: 56 | final resource = provider.getResource(Mock(ResourceResolver), path) 57 | 58 | then: 59 | resource instanceof TransactionResource 60 | 61 | where: 62 | path << [ 63 | '/grabbit/transaction/123.json', 64 | '/grabbit/transaction/123.html', 65 | '/grabbit/transaction/123', 66 | '/grabbit/transaction', 67 | '/grabbit/transaction/' 68 | ] 69 | } 70 | 71 | 72 | def "Can provide a content resource from path #path"() { 73 | given: 74 | GrabbitResourceProvider provider = new GrabbitResourceProvider() 75 | 76 | when: 77 | final resource = provider.getResource(Mock(ResourceResolver), path) 78 | 79 | then: 80 | resource instanceof ContentResource 81 | 82 | where: 83 | path << [ 84 | '/grabbit/content', 85 | '/grabbit/content/' 86 | ] 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/resources/JobResourceSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.resources 17 | 18 | import org.apache.sling.api.resource.ResourceResolver 19 | import spock.lang.Specification 20 | import spock.lang.Unroll 21 | 22 | import static com.twcable.grabbit.resources.JobResource.* 23 | 24 | class JobResourceSpec extends Specification { 25 | 26 | @Unroll 27 | def "JobResource creation works as we expect with valid input path: #path"() { 28 | given: 29 | final jobResource = new JobResource(Mock(ResourceResolver), path) 30 | 31 | expect: 32 | jobResource.resourceMetadata[JOB_EXECUTION_ID_KEY] == "jobExecutionId" 33 | jobResource.getPath() == path 34 | jobResource.getResourceType() == JOB_RESOURCE_TYPE 35 | 36 | where: 37 | path << [ 38 | "/grabbit/job/jobExecutionId.json", 39 | "/grabbit/job/jobExecutionId.html", 40 | "/grabbit/job/jobExecutionId.doesntmatter", 41 | "/grabbit/job/jobExecutionId" 42 | ] 43 | } 44 | 45 | def "JobResource handles resource creation gracefully if no jobID was provided"() { 46 | given: 47 | final jobResource = new JobResource(Mock(ResourceResolver), path) 48 | 49 | expect: 50 | jobResource.resourceMetadata[JOB_EXECUTION_ID_KEY] == "" 51 | jobResource.getPath() == path 52 | jobResource.getResourceType() == JOB_RESOURCE_TYPE 53 | 54 | where: 55 | path << ["/grabbit/job/", "/grabbit/job"] 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/resources/TransactionResourceSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.resources 17 | 18 | import org.apache.sling.api.resource.ResourceResolver 19 | import spock.lang.Specification 20 | import spock.lang.Unroll 21 | 22 | import static com.twcable.grabbit.resources.TransactionResource.TRANSACTION_ID_KEY 23 | import static com.twcable.grabbit.resources.TransactionResource.TRANSACTION_RESOURCE_TYPE 24 | 25 | class TransactionResourceSpec extends Specification { 26 | 27 | @Unroll 28 | def "TransactionResource creation works as we expect with valid input path: #path"() { 29 | given: 30 | final transactionResource = new TransactionResource(Mock(ResourceResolver), path) 31 | 32 | expect: 33 | transactionResource.resourceMetadata[TRANSACTION_ID_KEY] == "transactionID" 34 | transactionResource.getPath() == path 35 | transactionResource.getResourceType() == TRANSACTION_RESOURCE_TYPE 36 | 37 | where: 38 | path << [ 39 | "/grabbit/transaction/transactionID.json", 40 | "/grabbit/transaction/transactionID.html", 41 | "/grabbit/transaction/transactionID.doesntmatter", 42 | "/grabbit/transaction/transactionID" 43 | ] 44 | } 45 | 46 | def "TransactionResource handles resource creation gracefully if no transactionID was provided"() { 47 | given: 48 | final transactionResource = new TransactionResource(Mock(ResourceResolver), path) 49 | 50 | expect: 51 | transactionResource.resourceMetadata[TRANSACTION_ID_KEY] == "" 52 | transactionResource.getPath() == path 53 | transactionResource.getResourceType() == TRANSACTION_RESOURCE_TYPE 54 | 55 | where: 56 | path << ["/grabbit/transaction/", "/grabbit/transaction"] 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/security/AuthorizablePrincipalSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.security 17 | 18 | import spock.lang.Specification 19 | 20 | class AuthorizablePrincipalSpec extends Specification { 21 | 22 | def "Can get the name of an AuthorizablePrincipal"() { 23 | given: 24 | final authorizablePrincipal = new AuthorizablePrincipal('principalName') 25 | 26 | expect: 27 | authorizablePrincipal.getName() == 'principalName' 28 | } 29 | 30 | def "One AuthorizablePrincipal is equal to another"() { 31 | expect: 32 | assert new AuthorizablePrincipal('one').equals(new AuthorizablePrincipal('one')) 33 | assert !(new AuthorizablePrincipal('two').equals(new AuthorizablePrincipal('three'))) 34 | assert !(new AuthorizablePrincipal('two').equals(null)) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/server/GrabbitContentPullServletSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.server 17 | 18 | import com.twcable.grabbit.server.services.ServerService 19 | import org.apache.sling.api.SlingHttpServletRequest 20 | import org.apache.sling.api.SlingHttpServletResponse 21 | import spock.lang.Specification 22 | import spock.lang.Subject 23 | 24 | import javax.servlet.ServletOutputStream 25 | 26 | import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST 27 | import static javax.servlet.http.HttpServletResponse.SC_OK 28 | 29 | @Subject(GrabbitContentPullServlet) 30 | class GrabbitContentPullServletSpec extends Specification { 31 | 32 | 33 | def "Can pull a content stream from a path correctly"() { 34 | given: 35 | final request = Mock(SlingHttpServletRequest) { 36 | getParameter("path") >> "%2Fsome%2Fpath" 37 | getParameterValues("excludePath") >> ["%2Fexclude%2Fme", "%2Fexclude%2Fme%2Ftoo"] 38 | getParameter("after") >> "2016-01-29T13%3A53%3A58.831-05%3A00" 39 | getRemoteUser() >> "user" 40 | } 41 | final outputStream = Mock(ServletOutputStream) 42 | final response = Mock(SlingHttpServletResponse) { 43 | 1 * setContentType("application/octet-stream") 44 | 1 * setStatus(SC_OK) 45 | getOutputStream() >> outputStream 46 | } 47 | final serverService = Mock(ServerService) { 48 | 1 * getContentForRootPath("user", "/some/path", ["/exclude/me", "/exclude/me/too"], "2016-01-29T13:53:58.831-05:00", outputStream) 49 | } 50 | 51 | when: 52 | final contentPullServlet = new GrabbitContentPullServlet() 53 | contentPullServlet.setServerService(serverService) 54 | 55 | then: 56 | contentPullServlet.doGet(request, response) 57 | } 58 | 59 | def "Request for a content stream with no path results in a bad request response"() { 60 | given: 61 | final request = Mock(SlingHttpServletRequest) { 62 | getParameter("path") >> null 63 | } 64 | final response = Mock(SlingHttpServletResponse) { 65 | 1 * setStatus(SC_BAD_REQUEST) 66 | getWriter() >> Mock(PrintWriter) { 67 | 1 * write(_) 68 | } 69 | } 70 | 71 | expect: 72 | new GrabbitContentPullServlet().doGet(request, response) 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/spring/batch/repository/servlets/GrabbitCleanJobRepositoryServletSpec.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twcable.grabbit.spring.batch.repository.servlets 18 | 19 | import com.twcable.grabbit.spring.batch.repository.services.CleanJobRepository 20 | import org.apache.sling.api.SlingHttpServletRequest 21 | import org.apache.sling.api.SlingHttpServletResponse 22 | import spock.lang.Specification 23 | import spock.lang.Subject 24 | 25 | @Subject(GrabbitCleanJobRepositoryServlet) 26 | class GrabbitCleanJobRepositoryServletSpec extends Specification { 27 | 28 | def "Servlet handles the case when hours parameter is not passed"() { 29 | given: 30 | def servlet = new GrabbitCleanJobRepositoryServlet(cleanJobRepository: Mock(CleanJobRepository)) 31 | def request = Mock(SlingHttpServletRequest) 32 | request.getParameter("hours") >> null 33 | def response = Mock(SlingHttpServletResponse) 34 | def writer = new StringWriter() 35 | response.getWriter() >> new PrintWriter(writer) 36 | when: 37 | servlet.doPost(request, response) 38 | 39 | then: 40 | writer != null 41 | writer.toString() == "Parameter 'hours' must be an integer" 42 | } 43 | 44 | def "Servlet handles the case when hours parameter is correctly passed"() { 45 | given: 46 | def clientJobRepository = Mock(CleanJobRepository) 47 | clientJobRepository.cleanJobRepository(_) >> (["id1","id2","id3"] as List) 48 | def servlet = new GrabbitCleanJobRepositoryServlet(cleanJobRepository: clientJobRepository) 49 | def request = Mock(SlingHttpServletRequest) 50 | request.getParameter("hours") >> 5 51 | def response = Mock(SlingHttpServletResponse) 52 | def writer = new StringWriter() 53 | response.getWriter() >> new PrintWriter(writer) 54 | when: 55 | servlet.doPost(request, response) 56 | 57 | then: 58 | writer != null 59 | writer.toString().contains("JobExecutionsIds that were removed") 60 | writer.toString().contains("[id1, id2, id3]") 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/groovy/com/twcable/grabbit/testutil/StubInputStream.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Time Warner Cable, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twcable.grabbit.testutil 17 | 18 | import javax.annotation.Nonnull 19 | import javax.servlet.ServletInputStream 20 | 21 | class StubInputStream extends ServletInputStream { 22 | 23 | 24 | private final int byte_length 25 | private final byte[] bytes 26 | private int byte_index = 0 27 | 28 | private StubInputStream(@Nonnull final String data) { 29 | bytes = data as byte[] 30 | byte_length = bytes.length 31 | } 32 | 33 | private StubInputStream(@Nonnull final File fromFile) { 34 | bytes = fromFile.readBytes() 35 | byte_length = bytes.length 36 | } 37 | 38 | static InputStream inputStream(@Nonnull final String data) { 39 | return new StubInputStream(data) 40 | } 41 | 42 | static InputStream inputStream(@Nonnull final File fromFile) { 43 | return new StubInputStream(fromFile) 44 | } 45 | 46 | @Override 47 | int read() throws IOException { 48 | if (byte_index <= byte_length - 1) { 49 | final thisByte = bytes[byte_index] as int 50 | byte_index++ 51 | return thisByte 52 | } 53 | return -1 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/resources/com/twcable/grabbit/client/test_config.yaml: -------------------------------------------------------------------------------- 1 | serverUsername: "username" 2 | serverPassword: "password" 3 | serverHost: "host" 4 | serverPort: 4502 5 | deltaContent: true 6 | pathConfigurations: 7 | - path: "/content/testpath" 8 | workflowConfigIds: 9 | - '/etc/workflow/launcher/config/update_asset_mod' 10 | -------------------------------------------------------------------------------- /testutils/.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | -------------------------------------------------------------------------------- /testutils/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'osgi' 3 | apply plugin: 'groovy' 4 | 5 | group = 'com.twcable.grabbit' 6 | description = 'Test Utilities' 7 | 8 | dependencies { 9 | 10 | compile project(":commons-utils") 11 | // Library for Groovy 12 | compile "org.codehaus.groovy:groovy-all:${groovy_version}" 13 | compile "org.spockframework:spock-core:${spock_version}" 14 | 15 | compile "javax.servlet:servlet-api:${servlet_api_version}" 16 | 17 | compile "org.apache.sling:org.apache.sling.commons.testing:${sling_commons_testing_version}", { 18 | exclude group: 'org.apache.tika', module: 'tika-parsers' 19 | exclude group: 'org.apache.pdfbox', module: 'pdfbox' 20 | exclude group: 'org.slf4j', module: 'slf4j-simple' 21 | exclude group: 'junit', module: 'junit' 22 | exclude group: 'org.jmock', module: 'jmock-junit4' 23 | exclude group: 'rhino', module: 'js' 24 | } 25 | 26 | compile ":cq-workflow-console:${cq_workflow_console_version}" 27 | 28 | compile "org.slf4j:slf4j-api:${slf4j_version}" 29 | compile "org.mockito:mockito-all:${mockito_version}" 30 | compile "org.apache.commons:commons-lang3:${commons_lang_version}" 31 | compile "javax.servlet:servlet-api:${servlet_api_version}" 32 | compile "org.apache.felix:org.osgi.core:${felix_osgi_version}" 33 | compile "org.apache.felix:org.osgi.compendium:${felix_osgi_version}" 34 | 35 | compile "com.google.guava:guava:${guava_version}" 36 | } 37 | -------------------------------------------------------------------------------- /testutils/gradle/testing.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | jcenter() 4 | 5 | //Provides cq-gradle-plugins and jackalope 6 | maven { 7 | url "http://dl.bintray.com/twcable/aem" 8 | } 9 | mavenLocal() 10 | } 11 | 12 | dependencies { 13 | classpath "com.twcable.gradle:cq-gradle-plugins:${gradle_plugins_version}" 14 | 15 | classpath "commons-io:commons-io:2.4" 16 | } 17 | } 18 | 19 | apply plugin: 'groovy' 20 | apply plugin: 'osgi' 21 | 22 | dependencies { 23 | testCompile "org.spockframework:spock-core:${spock_version}" 24 | testCompile "org.spockframework:spock-spring:${spock_version}" 25 | testCompile "org.springframework.batch:spring-batch-test:${spring_batch_version}" 26 | testCompile "org.springframework:spring-test:3.2.9.RELEASE" 27 | 28 | 29 | testCompile "com.twcable.jackalope:jackalope:${jackalope_version}" 30 | 31 | testRuntime "cglib:cglib-nodep:${cglib_nodep_version}" 32 | } 33 | // ******************************************************************** 34 | // 35 | // Testing 36 | // 37 | // ******************************************************************** 38 | task testReport(type: TestReport, dependsOn: test) { 39 | destinationDir = file("$buildDir/reports/tests/") 40 | reportOn file("$buildDir/test-results/binary/test/") 41 | } 42 | 43 | check.dependsOn testReport 44 | 45 | // 46 | // Add integration testing support 47 | // 48 | sourceSets { 49 | integrationTest { 50 | compileClasspath = sourceSets.main.output + sourceSets.test.output + configurations.testRuntime 51 | runtimeClasspath = output + sourceSets.main.output + sourceSets.test.output + configurations.testRuntime 52 | groovy.srcDir file('src/integrationTest/groovy') 53 | java.srcDir file('src/integrationTest/java') 54 | } 55 | 56 | functionalTest { 57 | compileClasspath = sourceSets.main.output + sourceSets.test.output + configurations.testRuntime 58 | runtimeClasspath = output + sourceSets.main.output + sourceSets.test.output + configurations.testRuntime 59 | groovy.srcDir file('src/functionalTest/groovy') 60 | java.srcDir file('src/functionalTest/java') 61 | } 62 | } 63 | 64 | task integrationTest(type: Test) { 65 | testClassesDir = sourceSets.integrationTest.output.classesDir 66 | classpath = sourceSets.integrationTest.runtimeClasspath 67 | } 68 | 69 | task integrationTestReport(type: TestReport, dependsOn: integrationTest) { 70 | destinationDir = file("$buildDir/reports/integrationTests/") 71 | reportOn file("$buildDir/test-results/binary/integrationTest/") 72 | } 73 | 74 | task functionalTest(type: Test) { 75 | testClassesDir = sourceSets.functionalTest.output.classesDir 76 | maxParallelForks = 2 77 | System.properties.each { String key, String value -> 78 | if (key.startsWith('test.')) { 79 | systemProperty((key - 'test.'), value) 80 | } 81 | } 82 | classpath = sourceSets.functionalTest.runtimeClasspath 83 | } 84 | 85 | task functionalTestReport(type: TestReport, dependsOn: functionalTest) { 86 | destinationDir = file("$buildDir/reports/functionalTests/") 87 | reportOn file("$buildDir/test-results/binary/functionalTest/") 88 | } 89 | 90 | 91 | [test, integrationTest]*.systemProperty 'java.awt.headless', true 92 | [compileGroovy, compileTestGroovy, compileIntegrationTestGroovy]*.options*.forkOptions*.memoryMaximumSize = '2048m' 93 | 94 | idea.module.testSourceDirs += sourceSets.integrationTest.groovy.srcDirs 95 | idea.module.testSourceDirs += sourceSets.integrationTest.java.srcDirs 96 | idea.module.testSourceDirs += sourceSets.functionalTest.groovy.srcDirs 97 | idea.module.testSourceDirs += sourceSets.functionalTest.java.srcDirs 98 | 99 | // workaround for http://issues.gradle.org/browse/GRADLE-1682 100 | configurations.testRuntime.exclude group: 'org.apache.felix', module: 'org.osgi.foundation' 101 | --------------------------------------------------------------------------------