├── .gitignore ├── CONTRIBUTING.md ├── LICENSE.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── build.gradle ├── circle.yml ├── config └── checkstyle │ └── checkstyle.xml ├── core ├── build.gradle ├── gradle.properties ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── marklogic │ │ │ └── spring │ │ │ └── batch │ │ │ ├── bind │ │ │ ├── ExecutionContextAdapter.java │ │ │ ├── JobExecutionAdapter.java │ │ │ ├── JobInstanceAdapter.java │ │ │ ├── JobParametersAdapter.java │ │ │ └── StepExecutionAdapter.java │ │ │ ├── columnmap │ │ │ ├── ColumnMapSerializer.java │ │ │ ├── DefaultStaxColumnMapSerializer.java │ │ │ ├── JacksonColumnMapSerializer.java │ │ │ └── XmlStringColumnMapSerializer.java │ │ │ ├── config │ │ │ ├── MarkLogicBatchConfiguration.java │ │ │ ├── MarkLogicBatchConfigurer.java │ │ │ ├── MarkLogicConfiguration.java │ │ │ └── UseMarkLogicBatchCondition.java │ │ │ ├── core │ │ │ ├── AdaptedExecutionContext.java │ │ │ ├── AdaptedJobExecution.java │ │ │ ├── AdaptedJobInstance.java │ │ │ ├── AdaptedJobParameters.java │ │ │ ├── AdaptedStepExecution.java │ │ │ ├── MarkLogicSpringBatch.java │ │ │ ├── explore │ │ │ │ └── support │ │ │ │ │ └── MarkLogicJobExplorerFactoryBean.java │ │ │ ├── launch │ │ │ │ └── support │ │ │ │ │ └── CommandLineJobRunner.java │ │ │ ├── package-info.java │ │ │ └── repository │ │ │ │ ├── dao │ │ │ │ ├── MarkLogicExecutionContextDao.java │ │ │ │ ├── MarkLogicJobExecutionDao.java │ │ │ │ ├── MarkLogicJobInstanceDao.java │ │ │ │ └── MarkLogicStepExecutionDao.java │ │ │ │ └── support │ │ │ │ ├── MarkLogicJobRepositoryFactoryBean.java │ │ │ │ └── MarkLogicJobRepositoryProperties.java │ │ │ └── item │ │ │ ├── listener │ │ │ └── SimpleChunkListener.java │ │ │ ├── processor │ │ │ ├── AbstractMarkLogicItemProcessor.java │ │ │ ├── ColumnMapProcessor.java │ │ │ ├── MarkLogicItemProcessor.java │ │ │ ├── ResourceToDocumentWriteOperationItemProcessor.java │ │ │ ├── XmlElementItemProcessor.java │ │ │ └── support │ │ │ │ ├── DocumentUriGenerator.java │ │ │ │ └── UriGenerator.java │ │ │ ├── reader │ │ │ ├── DocumentItemReader.java │ │ │ ├── InvokeModuleItemReader.java │ │ │ └── ValuesItemReader.java │ │ │ ├── tasklet │ │ │ ├── DeleteDocumentsTasklet.java │ │ │ └── EvaluateServerSideStringTasklet.java │ │ │ └── writer │ │ │ ├── DataHubItemWriter.java │ │ │ ├── InvokeModuleItemWriter.java │ │ │ ├── MarkLogicItemWriter.java │ │ │ ├── MarkLogicPatchItemWriter.java │ │ │ ├── XccItemWriter.java │ │ │ └── support │ │ │ ├── DefaultUriTransformer.java │ │ │ └── UriTransformer.java │ └── resources │ │ └── logback.xml │ └── test │ ├── java │ └── com │ │ └── marklogic │ │ ├── client │ │ └── JavaClientApiIssue388Test.java │ │ └── spring │ │ └── batch │ │ ├── JobExecutionTestUtils.java │ │ ├── JobParametersTestUtils.java │ │ ├── MarkLogicSpringBatchConfigTests.java │ │ ├── SpringBatchNamespaceProvider.java │ │ ├── bind │ │ ├── JaxbConfiguration.java │ │ ├── MarshallSpringBatchPojoToXmlTest.java │ │ └── UnmarshallSpringBatchPojoTest.java │ │ ├── config │ │ ├── DoNotUseMarkLogicBatchConfigurerTest.java │ │ └── UseMarkLogicBatchConfigurerTest.java │ │ ├── core │ │ ├── explore │ │ │ ├── GetJobInstancesTest.java │ │ │ └── support │ │ │ │ └── SimpleJobExplorerTests.java │ │ ├── job │ │ │ └── JobSupport.java │ │ ├── launch │ │ │ └── support │ │ │ │ ├── CommandLineJobRunnerTests.java │ │ │ │ └── TestJobParametersIncrementer.java │ │ ├── repository │ │ │ ├── dao │ │ │ │ ├── MarkLogicExecutionContextDaoTests.java │ │ │ │ ├── MarkLogicJobDaoTests.java │ │ │ │ ├── MarkLogicJobExecutionDaoTests.java │ │ │ │ ├── MarkLogicJobInstanceDaoTests.java │ │ │ │ └── MarkLogicStepExecutionsDaoTest.java │ │ │ └── support │ │ │ │ └── SimpleJobRepositoryIntegrationTests.java │ │ └── step │ │ │ └── StepSupport.java │ │ ├── item │ │ ├── columnmap │ │ │ └── JacksonColumnMapSerializerTest.java │ │ ├── processor │ │ │ ├── ColumnMapProcessorTest.java │ │ │ └── support │ │ │ │ └── DocumentUriGeneratorTest.java │ │ ├── reader │ │ │ └── ValuesItemReaderTest.java │ │ └── writer │ │ │ ├── InvokeModuleItemWriterTest.java │ │ │ ├── MarkLogicItemWriterTest.java │ │ │ ├── MarkLogicPatchItemWriterTest.java │ │ │ ├── UriTransformerTest.java │ │ │ └── XccItemWriterTest.java │ │ └── test │ │ └── LoggingItemWriter.java │ └── resources │ ├── config │ ├── job.xml │ ├── job2.xml │ ├── launcher-with-environment.xml │ ├── launcher-with-locator.xml │ └── test-environment.xml │ ├── job.properties │ ├── options │ └── sample-options.properties │ ├── transforms │ └── simple.xqy │ └── xml │ ├── execution-context.xml │ ├── job-execution-2.xml │ ├── job-execution.xml │ ├── job-instance.xml │ ├── job-parameters.xml │ └── step-execution.xml ├── file ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ ├── main │ └── java │ │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── file │ │ ├── AvroItemReader.java │ │ ├── EnhancedResourcesItemReader.java │ │ ├── SplitXmlDocumentReader.java │ │ ├── TikaParserItemProcessor.java │ │ └── support │ │ ├── MapFileVisitor.java │ │ └── TikaParser.java │ └── test │ ├── java │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── file │ │ ├── AvroItemReaderTest.java │ │ ├── EnhancedResourcesItemReaderTest.java │ │ └── support │ │ ├── MapFileVisitorTest.java │ │ └── TikaParserTest.java │ └── resources │ ├── avro │ ├── sampleAvroSerialized.avro │ └── sampleOutput.txt │ └── word │ └── test-1.docx ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── http ├── build.gradle ├── gradle.properties ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── marklogic │ │ │ └── spring │ │ │ └── batch │ │ │ └── item │ │ │ └── http │ │ │ ├── HttpJsonItemReader.java │ │ │ └── HttpXmlItemReader.java │ └── resources │ │ └── job.properties │ └── test │ └── java │ └── com │ └── marklogic │ └── spring │ └── batch │ └── item │ └── http │ ├── HttpJsonItemReaderTest.java │ └── HttpXmlItemReaderTest.java ├── job.properties ├── mlJobRepo ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ └── main │ ├── java │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ ├── DeployMarkLogicJobRepository.java │ │ ├── MarkLogicSimpleJobRepositoryAppDeployer.java │ │ └── MarkLogicSimpleJobRepositoryConfig.java │ └── resources │ ├── content-database.json │ ├── logback.xml │ └── options.xml ├── rdbms ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ ├── main │ └── java │ │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── rdbms │ │ └── AllTablesItemReader.java │ └── test │ ├── java │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── rdbms │ │ ├── AllTablesItemReaderTest.java │ │ └── config │ │ └── H2DatabaseConfiguration.java │ └── resources │ └── db │ └── sampledata.sql ├── rdf ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── marklogic │ │ │ └── spring │ │ │ └── batch │ │ │ └── item │ │ │ └── rdf │ │ │ ├── AbstractDocumentWriter.java │ │ │ ├── GraphItemWriter.java │ │ │ ├── RdfTripleItemReader.java │ │ │ └── RdfTripleItemWriter.java │ └── resources │ │ └── logback.xml │ └── test │ ├── java │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── rdf │ │ ├── ImportRdfFromFileJob.java │ │ └── ImportRdfFromFileJobTest.java │ └── resources │ ├── job.properties │ ├── logback.xml │ └── triple │ ├── test1.ttl │ └── tigers.ttl ├── samples ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src │ ├── main │ └── java │ │ ├── YourJobConfig.java │ │ └── YourTwoStepJobConfig.java │ └── test │ ├── java │ ├── TwoStepJobTest.java │ ├── YourJobTest.java │ └── YourJobWithNoMarkLogicJobRepoTest.java │ └── resources │ └── job.properties ├── settings.gradle ├── shapefiles ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── shapefile │ │ ├── ShapefileProcessor.java │ │ └── support │ │ ├── HttpClientOgreProxy.java │ │ └── OgreProxy.java │ └── test │ ├── java │ └── com │ │ └── marklogic │ │ └── spring │ │ └── batch │ │ └── item │ │ └── shapefile │ │ └── ShapefileProcessorTest.java │ └── resources │ ├── logback.xml │ └── shapefiles │ └── Colleges_Syria.zip └── test ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src └── main └── java └── com └── marklogic └── spring └── batch └── test ├── AbstractJobRepositoryTest.java ├── AbstractJobRunnerTest.java ├── AbstractSpringBatchTest.java ├── EmptyItemWriter.java ├── JobRepositorySupport.java ├── JobRunnerContext.java └── SpringBatchNamespaceProvider.java /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | build/ 3 | classes/ 4 | out/ 5 | .classpath 6 | .project 7 | .gradle 8 | .settings 9 | .idea/ 10 | *.iml 11 | *.ipr 12 | *.iws 13 | marklogic-spring-batch.sublime* 14 | output-*.xml 15 | .nb-gradle/ 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![GitHub version](https://badge.fury.io/gh/marklogic-community%2Fmarklogic-spring-batch.svg) 2 | 3 | | Branch | Status | 4 | | ------------- | ------------- | 5 | | master | ![master](https://circleci.com/gh/sastafford/marklogic-spring-batch/tree/master.png?circle-token=e1b8b3198d3416fcb535509f2e7d600444ef153e) | 6 | | dev | ![dev](https://circleci.com/gh/sastafford/marklogic-spring-batch/tree/dev.png?circle-token=e1b8b3198d3416fcb535509f2e7d600444ef153e) | 7 | 8 | # MarkLogic Spring Batch 9 | 10 | The MarkLogic Spring Batch project is an extension of the CORE and INFRASTRUCTURE components of Spring Batch to make it easier to write batch processing programs using MarkLogic. 11 | 12 | Start with the [project home page](https://github.com/marklogic-community/marklogic-spring-batch/wiki) to get started. 13 | 14 | ## Prerequisites 15 | 16 | * MarkLogic 8+ 17 | * JDK 1.8+ 18 | 19 | ## Installing the Test Environment 20 | 21 | Open $PROJECT_ROOT/gradle.properties. Review the mlHost, mlRestPort, mlJobRepoPort properties to confirm there are no conflicts 22 | 23 | gradlew mlDeploy 24 | 25 | This command will set up your test database and application server. 26 | 27 | ## Running the tests 28 | 29 | Review the host/port/user credentials properties for the following property files. 30 | 31 | * ./core/src/test/resources/job.properties 32 | * ./infrastructure/src/test/resources/job.properties 33 | * ./rdf/src/test/resources/job.properties 34 | * ./samples/src/test/resources/job.properties 35 | 36 | Run the following command to execute all project tests. All tests should pass. 37 | 38 | gradlew test 39 | 40 | ## Coding Style Tests 41 | 42 | This project uses both checkstyle and PMD. 43 | 44 | gradlew check 45 | 46 | ## Deployment 47 | 48 | The product of this project are jar files that are published to [bintray](https://dl.bintray.com/sastafford/maven/). The following libraries are created. 49 | 50 | | Group | Artifact | 51 | | ------|----------| 52 | | com.marklogic | marklogic-spring-batch-core | 53 | | com.marklogic | marklogic-spring-batch-test | 54 | | com.marklogic | spring-batch-http | 55 | | com.marklogic | spring-batch-rdbms | 56 | | com.marklogic | spring-batch-file | 57 | 58 | The gradle bintray plugin is used to publish to bintray. 59 | 60 | gradlew :core:bintrayUpload 61 | gradlew :file:bintrayUpload 62 | gradlew :test:bintrayUpload 63 | gradlew :http:bintrayUpload 64 | gradlew :rdbms:bintrayUpload 65 | 66 | ## How do I use these libraries? 67 | 68 | ### Gradle 69 | 70 | ```groovy 71 | dependencies { 72 | compile 'com.marklogic:marklogic-spring-batch-core:1.+' 73 | testCompile 'com.marklogic:marklogic-spring-batch-test:1.+' 74 | } 75 | ``` 76 | 77 | 78 | ### Maven 79 | 80 | ```xml 81 | 82 | 83 | com.marklogic 84 | marklogic-spring-batch-core 85 | 1.5.0 86 | 87 | 88 | ``` 89 | 90 | 91 | # How can I contribute to the project? 92 | 93 | Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute code to this project and the process for submitting pull requests to us. 94 | 95 | # What license does MarkLogic Spring Batch use? 96 | 97 | See the [LICENSE.md](LICENSE.md) file for details 98 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | maven { 4 | url "https://plugins.gradle.org/m2/" 5 | jcenter() 6 | } 7 | } 8 | dependencies { 9 | classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7.3" 10 | classpath "com.marklogic:ml-gradle:3.2.0" 11 | } 12 | } 13 | 14 | plugins { 15 | id "com.marklogic.ml-gradle" version "3.3.0" 16 | } 17 | 18 | mlPostDeploy.dependsOn ":mlJobRepo:deployMarkLogicJobRepository" 19 | mlUndeploy.dependsOn "mlJobRepo:undeployMarkLogicJobRepository" 20 | 21 | allprojects { 22 | apply plugin: "checkstyle" 23 | apply plugin: "pmd" 24 | 25 | repositories { 26 | jcenter() 27 | mavenCentral() 28 | maven { url "http://repo.spring.io/release" } 29 | maven { url "http://developer.marklogic.com/maven2/" } 30 | } 31 | 32 | task wrapper(type: Wrapper) { 33 | gradleVersion = '4.5' 34 | } 35 | 36 | pmd { 37 | ruleSets = ["java-basic", "java-braces", "java-unusedcode", "java-imports"] 38 | } 39 | 40 | checkstyle { 41 | configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml") 42 | toolVersion = '7.1.2' 43 | } 44 | 45 | } 46 | 47 | subprojects { 48 | apply plugin: "java" 49 | apply plugin: "maven-publish" 50 | apply plugin: "idea" 51 | 52 | ext { 53 | springBatchVersion = '4.0.0.RELEASE' 54 | springVersion = '5.0.2.RELEASE' 55 | } 56 | 57 | dependencies { 58 | compile "ch.qos.logback:logback-classic:1.2.3" 59 | testCompile "junit:junit:4.12" 60 | } 61 | 62 | sourceCompatibility = 1.8 63 | targetCompatibility = 1.8 64 | } 65 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | services: 3 | - docker 4 | java: 5 | version: oraclejdk8 6 | 7 | dependencies: 8 | override: 9 | # Install git-lfs 10 | #- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash 11 | #- sudo apt-get install git-lfs=1.1.0 12 | #- ssh git@github.com git-lfs-authenticate sastafford/marklogic-spring-batch.git download 13 | # Docker commands 14 | - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASSWORD 15 | - docker info 16 | - docker pull $DOCKER_MARKLOGIC_IMAGE 17 | 18 | test: 19 | override: 20 | - docker run --hostname grover --name ml -d -p 8000:8000 -p 8001:8001 -p 8002:8002 -p 8200:8200 -p 8201:8201 $DOCKER_MARKLOGIC_IMAGE 21 | - sleep 10 22 | #- git lfs env 23 | #- git lfs pull 24 | - pwd 25 | - chmod +x gradlew 26 | - ./gradlew mlInit -PmlHost=localhost --info 27 | - ./gradlew mlInstallAdmin -PmlHost=localhost -PmlAdminPassword=admin -PmlAdminUsername=admin --info 28 | - ./gradlew mlDeploy -PmlHost=localhost --info 29 | - ./gradlew test --info 30 | 31 | post: 32 | - mkdir -p $CIRCLE_TEST_REPORTS/junit/ 33 | - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \; -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 30 | 31 | -------------------------------------------------------------------------------- /core/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "com.jfrog.bintray" 2 | 3 | dependencies { 4 | // For the CommandLineInterface main program 5 | compile "net.sf.jopt-simple:jopt-simple:5.0.1" 6 | 7 | compile "org.springframework.batch:spring-batch-core:$springBatchVersion" 8 | compile "com.marklogic:marklogic-client-api:4.0.3" 9 | compile "com.marklogic:ml-javaclient-util:3.2.0" 10 | compile "org.apache.commons:commons-lang3:3.5" 11 | 12 | testCompile "org.springframework.batch:spring-batch-test:$springBatchVersion" 13 | testCompile "com.marklogic:ml-junit:3.0.0" 14 | 15 | // Used by marshall/unmarshall tests, which depends on Jaxb2Marshaller 16 | testCompile "org.springframework:spring-oxm:$springVersion" 17 | testCompile project(":test") 18 | 19 | runtime "org.springframework.batch:spring-batch-core:$springBatchVersion" 20 | runtime "net.sf.jopt-simple:jopt-simple:5.0.1" 21 | } 22 | 23 | ext { 24 | artifact = "marklogic-spring-batch-core" 25 | } 26 | 27 | javadoc { 28 | excludes = ["com/marklogic/spring/batch/bind", "com/marklogic/spring/batch/core" ] 29 | } 30 | 31 | task sourcesJar(type: Jar, dependsOn: classes) { 32 | baseName = artifactId 33 | version = project.version 34 | classifier = 'sources' 35 | from sourceSets.main.allSource 36 | } 37 | 38 | jar { 39 | baseName = artifact 40 | version = project.version 41 | } 42 | 43 | publishing { 44 | publications { 45 | mainJavaWithSources(MavenPublication) { 46 | groupId group 47 | artifactId artifact 48 | version project.version 49 | from components.java 50 | artifact sourcesJar 51 | } 52 | } 53 | } 54 | 55 | bintray { 56 | user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 57 | key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_KEY') 58 | publications = [bintrayPublicationName] 59 | pkg { 60 | repo = bintrayRepo 61 | name = artifact 62 | userOrg = bintrayUserOrg 63 | licenses = [bintrayLicense] 64 | vcsUrl = bintrayVcsUrl 65 | version { 66 | name = project.version 67 | released = new Date() 68 | vcsTag = project.version 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /core/gradle.properties: -------------------------------------------------------------------------------- 1 | artifactId=marklogic-spring-batch-core 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /core/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/core/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /core/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /core/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/bind/ExecutionContextAdapter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.bind; 2 | 3 | import javax.xml.bind.annotation.adapters.XmlAdapter; 4 | 5 | import org.springframework.batch.item.ExecutionContext; 6 | 7 | import com.marklogic.spring.batch.core.AdaptedExecutionContext; 8 | 9 | public class ExecutionContextAdapter extends XmlAdapter { 10 | 11 | @Override 12 | public ExecutionContext unmarshal(AdaptedExecutionContext v) throws Exception { 13 | return new ExecutionContext(v.getMap()); 14 | } 15 | 16 | @Override 17 | public AdaptedExecutionContext marshal(ExecutionContext v) throws Exception { 18 | return new AdaptedExecutionContext(v); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/bind/JobExecutionAdapter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.bind; 2 | 3 | import javax.xml.bind.annotation.adapters.XmlAdapter; 4 | import org.springframework.batch.core.BatchStatus; 5 | import org.springframework.batch.core.ExitStatus; 6 | import org.springframework.batch.core.JobExecution; 7 | 8 | import com.marklogic.spring.batch.core.AdaptedJobExecution; 9 | 10 | public class JobExecutionAdapter extends XmlAdapter{ 11 | 12 | @Override 13 | public JobExecution unmarshal(AdaptedJobExecution v) throws Exception { 14 | JobExecution jobExec = new JobExecution(v.getId(), v.getJobParameters()); 15 | jobExec.setJobInstance(v.getJobInstance()); 16 | jobExec.setCreateTime(v.getCreateDateTime()); 17 | jobExec.setEndTime(v.getEndDateTime()); 18 | jobExec.setLastUpdated(v.getLastUpdatedDateTime()); 19 | jobExec.setStartTime(v.getStartDateTime()); 20 | jobExec.setStatus(BatchStatus.valueOf(v.getStatus())); 21 | jobExec.setExitStatus(new ExitStatus(v.getExitCode(), "")); 22 | jobExec.addStepExecutions(v.getStepExecutions()); 23 | jobExec.setVersion(v.getVersion()); 24 | jobExec.setExecutionContext(v.getExecutionContext()); 25 | return jobExec; 26 | } 27 | 28 | @Override 29 | public AdaptedJobExecution marshal(JobExecution v) throws Exception { 30 | return new AdaptedJobExecution(v); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/bind/JobInstanceAdapter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.bind; 2 | 3 | import com.marklogic.spring.batch.core.AdaptedJobInstance; 4 | import org.springframework.batch.core.JobInstance; 5 | import org.springframework.batch.core.JobParameters; 6 | 7 | import javax.xml.bind.annotation.adapters.XmlAdapter; 8 | 9 | public class JobInstanceAdapter extends XmlAdapter { 10 | 11 | private JobParameters jobParameters; 12 | 13 | public JobInstanceAdapter() { 14 | 15 | } 16 | 17 | public JobInstanceAdapter(JobParameters jobParameters) { 18 | this.jobParameters = jobParameters; 19 | } 20 | 21 | @Override 22 | public JobInstance unmarshal(AdaptedJobInstance v) throws Exception { 23 | JobInstance jobInstance = new JobInstance(v.getId(), v.getJobName()); 24 | jobInstance.setVersion(v.getVersion()); 25 | return jobInstance; 26 | } 27 | 28 | @Override 29 | public AdaptedJobInstance marshal(JobInstance v) throws Exception { 30 | AdaptedJobInstance aji = new AdaptedJobInstance(v); 31 | if (!(jobParameters == null)) { 32 | aji.setJobKey(jobParameters); 33 | } 34 | return aji; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/bind/JobParametersAdapter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.bind; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import javax.xml.bind.annotation.adapters.XmlAdapter; 10 | 11 | import org.springframework.batch.core.JobParameter; 12 | import org.springframework.batch.core.JobParameters; 13 | import org.springframework.batch.core.JobParametersBuilder; 14 | 15 | import com.marklogic.spring.batch.core.AdaptedJobParameters; 16 | 17 | public class JobParametersAdapter extends XmlAdapter { 18 | 19 | @Override 20 | public JobParameters unmarshal(AdaptedJobParameters params) throws Exception { 21 | JobParametersBuilder jobParametersBuilder = new JobParametersBuilder(); 22 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX"); 23 | for (AdaptedJobParameters.AdaptedJobParameter param : params.getParameters()) { 24 | switch (param.type) { 25 | case "STRING": 26 | jobParametersBuilder.addString(param.key, param.value, param.identifier); 27 | break; 28 | case "DATE": 29 | jobParametersBuilder.addDate(param.key, df.parse(param.value), param.identifier); 30 | break; 31 | case "DOUBLE": 32 | jobParametersBuilder.addDouble(param.key, Double.valueOf(param.value), param.identifier); 33 | break; 34 | case "LONG": 35 | jobParametersBuilder.addLong(param.key, Long.valueOf(param.value), param.identifier); 36 | break; 37 | } 38 | } 39 | return jobParametersBuilder.toJobParameters(); 40 | } 41 | 42 | @Override 43 | public AdaptedJobParameters marshal(JobParameters jobParams) throws Exception { 44 | AdaptedJobParameters adaptedJobParams = new AdaptedJobParameters(); 45 | List listOfParameters = new ArrayList<>(); 46 | if (jobParams == null) { 47 | return adaptedJobParams; 48 | } else { 49 | for (Map.Entry entry : jobParams.getParameters().entrySet()) { 50 | AdaptedJobParameters.AdaptedJobParameter param = new AdaptedJobParameters.AdaptedJobParameter(); 51 | param.key = entry.getKey(); 52 | JobParameter jobParam = entry.getValue(); 53 | param.type = jobParam.getType().toString(); 54 | param.identifier = jobParam.isIdentifying(); 55 | switch (jobParam.getType()) { 56 | case STRING: 57 | param.value = jobParams.getString(entry.getKey()); 58 | break; 59 | case DATE: 60 | param.value = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(jobParams.getDate(entry.getKey())); 61 | break; 62 | case DOUBLE: 63 | param.value = jobParams.getDouble(entry.getKey()).toString(); 64 | break; 65 | case LONG: 66 | param.value = jobParams.getLong(entry.getKey()).toString(); 67 | break; 68 | } 69 | listOfParameters.add(param); 70 | } 71 | adaptedJobParams.setParameters(listOfParameters); 72 | } 73 | return adaptedJobParams; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/bind/StepExecutionAdapter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.bind; 2 | 3 | import javax.xml.bind.annotation.adapters.XmlAdapter; 4 | 5 | import org.springframework.batch.core.ExitStatus; 6 | import org.springframework.batch.core.JobExecution; 7 | import org.springframework.batch.core.JobInstance; 8 | import org.springframework.batch.core.StepExecution; 9 | 10 | import com.marklogic.spring.batch.core.AdaptedStepExecution; 11 | 12 | public class StepExecutionAdapter extends XmlAdapter { 13 | 14 | @Override 15 | public StepExecution unmarshal(AdaptedStepExecution v) throws Exception { 16 | JobExecution je = new JobExecution(v.getJobExecutionId()); 17 | JobInstance ji = new JobInstance(v.getJobInstanceId(), v.getJobName()); 18 | je.setJobInstance(ji); 19 | StepExecution step = new StepExecution(v.getStepName(), je); 20 | step.setId(v.getId()); 21 | step.setStartTime(v.getStartTime()); 22 | step.setEndTime(v.getEndTime()); 23 | step.setReadSkipCount(v.getReadSkipCount()); 24 | step.setWriteSkipCount(v.getWriteSkipCount()); 25 | step.setProcessSkipCount(v.getProcessSkipCount()); 26 | step.setReadCount(v.getReadCount()); 27 | step.setWriteCount(v.getWriteCount()); 28 | step.setFilterCount(v.getFilterCount()); 29 | step.setRollbackCount(v.getRollbackCount()); 30 | step.setExitStatus(new ExitStatus(v.getExitCode())); 31 | step.setLastUpdated(v.getLastUpdated()); 32 | step.setVersion(v.getVersion()); 33 | step.setStatus(v.getStatus()); 34 | step.setExecutionContext(v.getExecutionContext()); 35 | return step; 36 | } 37 | 38 | @Override 39 | public AdaptedStepExecution marshal(StepExecution v) throws Exception { 40 | return new AdaptedStepExecution(v); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/columnmap/ColumnMapSerializer.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.columnmap; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Strategy interface for how a Spring "column map" can be written to a String of JSON or XML or, really, whatever you 7 | * want. 8 | */ 9 | public interface ColumnMapSerializer { 10 | 11 | String serializeColumnMap(Map columnMap, String rootLocalName); 12 | } 13 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/columnmap/JacksonColumnMapSerializer.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.columnmap; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * Simple implementation that uses Jackson's writeValueAsString method. The outputted JSON string can be customized by 10 | * passing in your own ObjectMapper implementation. 11 | */ 12 | public class JacksonColumnMapSerializer implements ColumnMapSerializer { 13 | 14 | private ObjectMapper objectMapper; 15 | 16 | public JacksonColumnMapSerializer() { 17 | this(new ObjectMapper()); 18 | } 19 | 20 | public JacksonColumnMapSerializer(ObjectMapper objectMapper) { 21 | this.objectMapper = objectMapper; 22 | } 23 | 24 | @Override 25 | public String serializeColumnMap(Map columnMap, String rootLocalName) { 26 | try { 27 | return objectMapper.writeValueAsString(columnMap); 28 | } catch (JsonProcessingException e) { 29 | throw new RuntimeException("Unable to write map as JSON: " + e.getMessage(), e); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/columnmap/XmlStringColumnMapSerializer.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.columnmap; 2 | 3 | import org.apache.commons.lang3.StringEscapeUtils; 4 | 5 | import java.util.Map; 6 | 7 | public class XmlStringColumnMapSerializer implements ColumnMapSerializer { 8 | 9 | @Override 10 | public String serializeColumnMap(Map columnMap, String rootLocalName) { 11 | String content = ""; 12 | String rootName = rootLocalName.length() == 0 ? "record" : rootLocalName.replaceAll("[^A-Za-z0-9\\_\\-]", ""); 13 | content = "<" + rootName + ">\n"; 14 | 15 | for (Map.Entry entry : transformColumnMap(columnMap).entrySet()) { 16 | String elName = entry.getKey().replaceAll("[^A-Za-z0-9\\_\\-]", ""); 17 | String value = entry.getValue() == null ? "" : StringEscapeUtils.escapeXml11(entry.getValue().toString()); 18 | content += "<" + elName + ">" + value + "\n"; 19 | } 20 | 21 | content += ""; 22 | return content; 23 | } 24 | 25 | //The strategy is to extend this class and overwrite this method. 26 | protected Map transformColumnMap(Map columnMap) { 27 | return columnMap; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/config/MarkLogicBatchConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.config; 2 | 3 | import com.marklogic.client.ext.DatabaseClientConfig; 4 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 5 | import com.marklogic.client.ext.spring.SimpleDatabaseClientProvider; 6 | import org.springframework.beans.factory.annotation.Qualifier; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.ComponentScan; 10 | import org.springframework.context.annotation.Conditional; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; 13 | 14 | import java.util.List; 15 | 16 | @Configuration 17 | @ComponentScan( 18 | basePackageClasses = { 19 | com.marklogic.spring.batch.config.MarkLogicBatchConfigurer.class, 20 | com.marklogic.spring.batch.core.repository.support.MarkLogicJobRepositoryProperties.class 21 | }) 22 | public class MarkLogicBatchConfiguration { 23 | 24 | @Bean 25 | public static PropertySourcesPlaceholderConfigurer propertyPlaceHolderConfigurer() { 26 | return new PropertySourcesPlaceholderConfigurer(); 27 | } 28 | 29 | @Conditional(UseMarkLogicBatchCondition.class) 30 | @Bean(name = "markLogicJobRepositoryDatabaseClientConfig") 31 | @Qualifier("markLogicJobRepositoryDatabaseClientConfig") 32 | public DatabaseClientConfig markLogicJobRepositoryDatabaseClientConfig( 33 | @Value("#{'${marklogic.jobrepo.host:localhost}'.split(',')}") List hosts, 34 | @Value("${marklogic.jobrepo.port:8201}") int port, 35 | @Value("${marklogic.jobrepo.username:admin}") String username, 36 | @Value("${marklogic.jobrepo.password:admin}") String password) { 37 | return new DatabaseClientConfig(hosts.get(0), port, username, password); 38 | } 39 | 40 | @Conditional(UseMarkLogicBatchCondition.class) 41 | @Bean(name = "markLogicJobRepositoryDatabaseClientProvider") 42 | @Qualifier("markLogicJobRepositoryDatabaseClientProvider") 43 | public DatabaseClientProvider markLogicJobRepositoryDatabaseClientProvider( 44 | @Qualifier("markLogicJobRepositoryDatabaseClientConfig") 45 | DatabaseClientConfig marklogicJobRepositoryClientConfig) { 46 | return new SimpleDatabaseClientProvider(marklogicJobRepositoryClientConfig); 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/config/MarkLogicConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.config; 2 | 3 | import com.marklogic.client.ext.DatabaseClientConfig; 4 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 5 | import com.marklogic.client.ext.spring.SimpleDatabaseClientProvider; 6 | import org.springframework.beans.factory.annotation.Qualifier; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | 11 | import java.util.List; 12 | 13 | @Configuration 14 | public class MarkLogicConfiguration { 15 | 16 | @Bean 17 | public DatabaseClientConfig databaseClientConfig( 18 | @Value("#{'${marklogic.host:localhost}'.split(',')}") List hosts, 19 | @Value("${marklogic.port:8000}") int port, 20 | @Value("${marklogic.username:admin}") String username, 21 | @Value("${marklogic.password:admin}") String password) { 22 | return new DatabaseClientConfig(hosts.get(0), port, username, password); 23 | } 24 | 25 | @Bean 26 | @Qualifier("targetDatabaseClientProvider") 27 | public DatabaseClientProvider databaseClientProvider(DatabaseClientConfig databaseClientConfig) { 28 | return new SimpleDatabaseClientProvider(databaseClientConfig); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/config/UseMarkLogicBatchCondition.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.config; 2 | 3 | import org.springframework.context.annotation.Condition; 4 | import org.springframework.context.annotation.ConditionContext; 5 | import org.springframework.core.env.Environment; 6 | import org.springframework.core.type.AnnotatedTypeMetadata; 7 | 8 | public class UseMarkLogicBatchCondition implements Condition { 9 | 10 | @Override 11 | public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { 12 | Environment env = context.getEnvironment(); 13 | Boolean enabled = env.getProperty("marklogic.batch.config.enabled", Boolean.class); 14 | if (enabled == null) { 15 | return false; 16 | } else { 17 | return enabled.booleanValue(); 18 | } 19 | 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/AdaptedExecutionContext.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core; 2 | 3 | import org.springframework.batch.item.ExecutionContext; 4 | 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | import javax.xml.bind.annotation.XmlType; 7 | import java.util.HashMap; 8 | import java.util.Iterator; 9 | import java.util.Map; 10 | import java.util.Map.Entry; 11 | import java.util.Set; 12 | 13 | @XmlRootElement(name = "executionContext", namespace = MarkLogicSpringBatch.JOB_NAMESPACE) 14 | @XmlType(namespace = MarkLogicSpringBatch.JOB_NAMESPACE) 15 | public class AdaptedExecutionContext { 16 | 17 | private Map map = new HashMap<>(); 18 | private int hashCode; 19 | private boolean dirtyFlag; 20 | 21 | public AdaptedExecutionContext() { 22 | 23 | } 24 | 25 | public AdaptedExecutionContext(ExecutionContext exeContext) throws InstantiationException, IllegalAccessException { 26 | this.hashCode = exeContext.hashCode(); 27 | this.dirtyFlag = exeContext.isDirty(); 28 | 29 | // Get a set of the entries 30 | Set> set = exeContext.entrySet(); 31 | // Get an iterator 32 | Iterator> i = set.iterator(); 33 | // Display elements 34 | while (i.hasNext()) { 35 | Entry me = i.next(); 36 | String name = me.getKey(); 37 | Object obj = getValue(me.getValue()); 38 | map.put(name, obj); 39 | } 40 | } 41 | 42 | public Map getMap() { 43 | return map; 44 | } 45 | 46 | public void setMap(Map map) { 47 | this.map = map; 48 | } 49 | 50 | public int getHashCode() { 51 | return hashCode; 52 | } 53 | 54 | public void setHashCode(int hashCode) { 55 | this.hashCode = hashCode; 56 | } 57 | 58 | public boolean isDirtyFlag() { 59 | return dirtyFlag; 60 | } 61 | 62 | public void setDirtyFlag(boolean dirtyFlag) { 63 | this.dirtyFlag = dirtyFlag; 64 | } 65 | 66 | private Object getValue(Object value) { 67 | 68 | if (String.class.isInstance(value)) { 69 | value = value; 70 | } else if (Long.class.isInstance(value)) { 71 | value = value; 72 | } else if (Double.class.isInstance(value)) { 73 | value = value; 74 | } else if (Integer.class.isInstance(value)) { 75 | value = value; 76 | } else if (Boolean.class.isInstance(value)) { 77 | value = value; 78 | } else { 79 | value = value; 80 | } 81 | return value; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/AdaptedJobInstance.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core; 2 | 3 | import javax.xml.bind.annotation.XmlRootElement; 4 | import javax.xml.bind.annotation.XmlType; 5 | 6 | import org.springframework.batch.core.DefaultJobKeyGenerator; 7 | import org.springframework.batch.core.JobInstance; 8 | import org.springframework.batch.core.JobKeyGenerator; 9 | import org.springframework.batch.core.JobParameters; 10 | 11 | import java.util.Date; 12 | 13 | @XmlRootElement(name = "jobInstance", namespace=MarkLogicSpringBatch.JOB_NAMESPACE) 14 | @XmlType(namespace=MarkLogicSpringBatch.JOB_NAMESPACE) 15 | public class AdaptedJobInstance { 16 | 17 | private Long id; 18 | private Integer version = 0; 19 | private String jobName; 20 | private String jobKey; 21 | private Date createDateTime; 22 | 23 | private JobKeyGenerator jobKeyGenerator = new DefaultJobKeyGenerator(); 24 | 25 | public AdaptedJobInstance() { } 26 | 27 | public AdaptedJobInstance(JobInstance jobInstance) { 28 | this.id = jobInstance.getId(); 29 | if (jobInstance.getVersion() == null) { 30 | jobInstance.setVersion(0); 31 | } else { 32 | this.setVersion(jobInstance.getVersion()); 33 | } 34 | this.jobName = jobInstance.getJobName(); 35 | } 36 | 37 | public AdaptedJobInstance(JobInstance jobInstance, JobParameters jobParameters) { 38 | this(jobInstance); 39 | setJobKey(jobKeyGenerator.generateKey(jobParameters)); 40 | } 41 | 42 | public String getJobName() { 43 | return jobName; 44 | } 45 | 46 | public void setJobName(String jobName) { 47 | this.jobName = jobName; 48 | } 49 | 50 | public Long getId() { 51 | return id; 52 | } 53 | 54 | public void setId(Long id) { 55 | this.id = id; 56 | } 57 | 58 | public Integer getVersion() { 59 | return version; 60 | } 61 | 62 | public void setVersion(Integer version) { 63 | this.version = version; 64 | } 65 | 66 | public String getJobKey() { return jobKey; } 67 | 68 | public void setJobKey(String jobKey) { 69 | this.jobKey = jobKey; 70 | } 71 | 72 | public void setJobKey(JobParameters jobParameters) { 73 | this.jobKey = jobKeyGenerator.generateKey(jobParameters); 74 | } 75 | 76 | public Date getCreateDateTime() { 77 | return createDateTime; 78 | } 79 | 80 | public void setCreateDateTime(Date createDateTime) { 81 | this.createDateTime = createDateTime; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/AdaptedJobParameters.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core; 2 | 3 | import java.util.List; 4 | 5 | import javax.xml.bind.annotation.XmlAttribute; 6 | import javax.xml.bind.annotation.XmlElement; 7 | import javax.xml.bind.annotation.XmlRootElement; 8 | import javax.xml.bind.annotation.XmlType; 9 | import javax.xml.bind.annotation.XmlValue; 10 | 11 | @XmlRootElement(name="jobParameters", namespace=MarkLogicSpringBatch.JOB_NAMESPACE) 12 | @XmlType(namespace=MarkLogicSpringBatch.JOB_NAMESPACE) 13 | public class AdaptedJobParameters { 14 | 15 | private List parameters; 16 | 17 | @XmlElement(name="jobParameter") 18 | public List getParameters() { 19 | return parameters; 20 | } 21 | 22 | public void setParameters(List parameters) { 23 | this.parameters = parameters; 24 | } 25 | 26 | public AdaptedJobParameters() { 27 | 28 | } 29 | 30 | public static class AdaptedJobParameter { 31 | @XmlAttribute 32 | public String key; 33 | 34 | @XmlAttribute 35 | public String type; 36 | 37 | @XmlAttribute 38 | public boolean identifier; 39 | 40 | @XmlValue 41 | public String value; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/MarkLogicSpringBatch.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core; 2 | 3 | public interface MarkLogicSpringBatch { 4 | 5 | String JOB_NAMESPACE = "http://marklogic.com/spring-batch"; 6 | String JOB_NAMESPACE_PREFIX = "msb"; 7 | String SPRING_BATCH_DIR = "/projects.spring.io/spring-batch/"; 8 | 9 | 10 | } 11 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/explore/support/MarkLogicJobExplorerFactoryBean.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core.explore.support; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicExecutionContextDao; 5 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicJobExecutionDao; 6 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicJobInstanceDao; 7 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicStepExecutionDao; 8 | import com.marklogic.spring.batch.core.repository.support.MarkLogicJobRepositoryProperties; 9 | import org.springframework.batch.core.explore.JobExplorer; 10 | import org.springframework.batch.core.explore.support.AbstractJobExplorerFactoryBean; 11 | import org.springframework.batch.core.explore.support.SimpleJobExplorer; 12 | import org.springframework.batch.core.repository.dao.ExecutionContextDao; 13 | import org.springframework.batch.core.repository.dao.JobExecutionDao; 14 | import org.springframework.batch.core.repository.dao.JobInstanceDao; 15 | import org.springframework.batch.core.repository.dao.StepExecutionDao; 16 | 17 | public class MarkLogicJobExplorerFactoryBean extends AbstractJobExplorerFactoryBean { 18 | 19 | private DatabaseClient databaseClient; 20 | private MarkLogicJobRepositoryProperties properties; 21 | 22 | public void setDatabaseClient(DatabaseClient databaseClient) { 23 | this.databaseClient = databaseClient; 24 | } 25 | 26 | public void setMarkLogicJobRepositoryProperties(MarkLogicJobRepositoryProperties properties) { 27 | this.properties = properties; 28 | } 29 | 30 | @Override 31 | protected JobInstanceDao createJobInstanceDao() throws Exception { 32 | return new MarkLogicJobInstanceDao(databaseClient, properties); 33 | } 34 | 35 | @Override 36 | protected JobExecutionDao createJobExecutionDao() throws Exception { 37 | return new MarkLogicJobExecutionDao(databaseClient, properties); 38 | } 39 | 40 | @Override 41 | protected StepExecutionDao createStepExecutionDao() throws Exception { 42 | return new MarkLogicStepExecutionDao(databaseClient, properties); 43 | } 44 | 45 | @Override 46 | protected ExecutionContextDao createExecutionContextDao() throws Exception { 47 | return new MarkLogicExecutionContextDao(databaseClient, properties); 48 | } 49 | 50 | @Override 51 | public JobExplorer getObject() throws Exception { 52 | return new SimpleJobExplorer(createJobInstanceDao(), 53 | createJobExecutionDao(), createStepExecutionDao(), 54 | createExecutionContextDao()); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/package-info.java: -------------------------------------------------------------------------------- 1 | @XmlSchema( 2 | namespace=MarkLogicSpringBatch.JOB_NAMESPACE, 3 | elementFormDefault=javax.xml.bind.annotation.XmlNsForm.QUALIFIED, 4 | xmlns = { 5 | @XmlNs(prefix = MarkLogicSpringBatch.JOB_NAMESPACE_PREFIX, namespaceURI=MarkLogicSpringBatch.JOB_NAMESPACE), 6 | @XmlNs(prefix = "xs", namespaceURI="http://www.w3.org/2001/XMLSchema"), 7 | @XmlNs(prefix = "xsi", namespaceURI="http://www.w3.org/2001/XMLSchema-instance") 8 | } 9 | ) 10 | 11 | /** 12 | * Provides the adapted JAXB classes necessary to persist JobExecution data in MarkLogic 13 | * 14 | * @since 1.0 15 | 16 | */ 17 | 18 | package com.marklogic.spring.batch.core; 19 | 20 | import javax.xml.bind.annotation.XmlSchema; 21 | import javax.xml.bind.annotation.XmlNs; 22 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/core/repository/support/MarkLogicJobRepositoryFactoryBean.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core.repository.support; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicExecutionContextDao; 5 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicJobExecutionDao; 6 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicJobInstanceDao; 7 | import com.marklogic.spring.batch.core.repository.dao.MarkLogicStepExecutionDao; 8 | import org.springframework.batch.core.repository.dao.ExecutionContextDao; 9 | import org.springframework.batch.core.repository.dao.JobExecutionDao; 10 | import org.springframework.batch.core.repository.dao.JobInstanceDao; 11 | import org.springframework.batch.core.repository.dao.StepExecutionDao; 12 | import org.springframework.batch.core.repository.support.AbstractJobRepositoryFactoryBean; 13 | import org.springframework.beans.factory.InitializingBean; 14 | import org.springframework.util.Assert; 15 | 16 | public class MarkLogicJobRepositoryFactoryBean extends AbstractJobRepositoryFactoryBean implements InitializingBean { 17 | 18 | private DatabaseClient databaseClient; 19 | private MarkLogicJobRepositoryProperties properties; 20 | 21 | public void setDatabaseClient(DatabaseClient databaseClient) { 22 | this.databaseClient = databaseClient; 23 | } 24 | 25 | public void setMarkLogicJobRepositoryProperties(MarkLogicJobRepositoryProperties properties) { 26 | this.properties = properties; 27 | } 28 | 29 | @Override 30 | protected JobInstanceDao createJobInstanceDao() throws Exception { 31 | return new MarkLogicJobInstanceDao(databaseClient, properties); 32 | } 33 | 34 | @Override 35 | protected JobExecutionDao createJobExecutionDao() throws Exception { 36 | return new MarkLogicJobExecutionDao(databaseClient, properties); 37 | } 38 | 39 | @Override 40 | protected StepExecutionDao createStepExecutionDao() throws Exception { 41 | return new MarkLogicStepExecutionDao(databaseClient, properties); 42 | } 43 | 44 | @Override 45 | protected ExecutionContextDao createExecutionContextDao() throws Exception { 46 | return new MarkLogicExecutionContextDao(databaseClient, properties); 47 | } 48 | 49 | @Override 50 | public void afterPropertiesSet() throws Exception { 51 | Assert.notNull(databaseClient, "DatabaseClient cannot be NULL"); 52 | super.afterPropertiesSet(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/listener/SimpleChunkListener.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.listener; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.batch.core.ChunkListener; 6 | import org.springframework.batch.core.scope.context.ChunkContext; 7 | 8 | public class SimpleChunkListener implements ChunkListener { 9 | 10 | Logger logger = LoggerFactory.getLogger(SimpleChunkListener.class); 11 | 12 | @Override 13 | public void beforeChunk(ChunkContext context) { 14 | 15 | } 16 | 17 | @Override 18 | public void afterChunk(ChunkContext context) { 19 | logger.info("Chunk Processed"); 20 | } 21 | 22 | @Override 23 | public void afterChunkError(ChunkContext context) { 24 | 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/processor/ColumnMapProcessor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor; 2 | 3 | import com.marklogic.client.io.StringHandle; 4 | import com.marklogic.client.io.marker.AbstractWriteHandle; 5 | import com.marklogic.spring.batch.columnmap.ColumnMapSerializer; 6 | import com.marklogic.spring.batch.item.processor.support.UriGenerator; 7 | 8 | import java.util.Map; 9 | import java.util.UUID; 10 | 11 | public class ColumnMapProcessor extends AbstractMarkLogicItemProcessor> { 12 | 13 | private ColumnMapSerializer columnMapSerializer; 14 | private String rootLocalName = "CHANGEME"; 15 | 16 | public ColumnMapProcessor(ColumnMapSerializer columnMapSerializer) { 17 | super(); 18 | this.columnMapSerializer = columnMapSerializer; 19 | setUriGenerator( 20 | new UriGenerator() { 21 | @Override 22 | public String generateUri(Object o) { 23 | String uuid = UUID.randomUUID().toString(); 24 | return "/" + rootLocalName.replaceAll("[^A-Za-z0-9\\_\\-]", "") + "/" + uuid + ".xml"; 25 | } 26 | }); 27 | } 28 | 29 | public ColumnMapProcessor(ColumnMapSerializer columnMapSerializer, UriGenerator uriGenerator) { 30 | super(uriGenerator); 31 | this.columnMapSerializer = columnMapSerializer; 32 | } 33 | 34 | @Override 35 | public AbstractWriteHandle getContentHandle(Map item) throws Exception { 36 | String rootElement = item.containsKey(rootLocalName) ? item.get(rootLocalName).toString() : rootLocalName; 37 | return new StringHandle(columnMapSerializer.serializeColumnMap(item, rootElement)); 38 | } 39 | 40 | public void setRootLocalName(String rootName) { 41 | this.rootLocalName = rootName; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/processor/MarkLogicItemProcessor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor; 2 | 3 | import com.marklogic.client.document.DocumentWriteOperation; 4 | import org.springframework.batch.item.ItemProcessor; 5 | 6 | public interface MarkLogicItemProcessor extends ItemProcessor { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/processor/ResourceToDocumentWriteOperationItemProcessor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor; 2 | 3 | import com.marklogic.client.io.FileHandle; 4 | import com.marklogic.client.io.Format; 5 | import com.marklogic.client.io.marker.AbstractWriteHandle; 6 | import com.marklogic.spring.batch.item.processor.support.UriGenerator; 7 | import org.springframework.core.io.Resource; 8 | 9 | import java.util.UUID; 10 | 11 | public class ResourceToDocumentWriteOperationItemProcessor extends AbstractMarkLogicItemProcessor { 12 | 13 | public ResourceToDocumentWriteOperationItemProcessor() { 14 | super(new UriGenerator(){ 15 | @Override 16 | public String generateUri(Resource resource) { 17 | try { 18 | return resource.getURL().getPath(); 19 | } catch (Exception ex) { 20 | return UUID.randomUUID().toString(); 21 | } 22 | } 23 | }); 24 | } 25 | 26 | @Override 27 | public AbstractWriteHandle getContentHandle(Resource item) throws Exception { 28 | String fileName = uriGenerator.generateUri(item); 29 | FileHandle handle = new FileHandle(item.getFile()); 30 | int i = fileName.lastIndexOf('.'); 31 | String extension = (i >= 0) ? fileName.substring(i+1) : ".xyz"; 32 | if (Format.XML.equals(getFormat()) || extension.equals("xml")) { 33 | handle.setFormat(Format.XML); 34 | } else if (Format.JSON.equals(getFormat()) || extension.equals("json")) { 35 | handle.setFormat(Format.JSON); 36 | } else if (Format.TEXT.equals(getFormat()) || extension.equals("txt")) { 37 | handle.setFormat(Format.TEXT); 38 | } else if (Format.BINARY.equals(getFormat())) { 39 | handle.setFormat(Format.BINARY); 40 | } else { 41 | handle.setFormat(Format.UNKNOWN); 42 | } 43 | return handle; 44 | } 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/processor/XmlElementItemProcessor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor; 2 | 3 | import com.marklogic.client.extra.jdom.JDOMHandle; 4 | import com.marklogic.client.io.marker.AbstractWriteHandle; 5 | import com.marklogic.spring.batch.item.processor.support.UriGenerator; 6 | import org.jdom2.Document; 7 | 8 | public class XmlElementItemProcessor extends AbstractMarkLogicItemProcessor { 9 | 10 | public XmlElementItemProcessor() { 11 | super(); 12 | } 13 | 14 | public XmlElementItemProcessor(UriGenerator uriGenerator) { 15 | super(uriGenerator); 16 | } 17 | 18 | @Override 19 | public AbstractWriteHandle getContentHandle(Document item) throws Exception { 20 | return new JDOMHandle(item); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/processor/support/DocumentUriGenerator.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor.support; 2 | 3 | import org.jdom2.Document; 4 | import org.jdom2.Element; 5 | import org.jdom2.Namespace; 6 | 7 | public class DocumentUriGenerator implements UriGenerator { 8 | 9 | private String elementId; 10 | private String namespace; 11 | 12 | public DocumentUriGenerator(String namespace, String elementId) { 13 | this.namespace = namespace; 14 | this.elementId = elementId; 15 | } 16 | 17 | public DocumentUriGenerator(String elementId) { 18 | this.elementId = elementId; 19 | } 20 | 21 | @Override 22 | public String generateUri(Document document) throws Exception { 23 | Element el = null; 24 | if (namespace == null) { 25 | el = document.getRootElement().getChild(elementId); 26 | } else { 27 | el = document.getRootElement().getChild(elementId, Namespace.getNamespace(namespace)); 28 | } 29 | if (el == null) { 30 | throw new RuntimeException(namespace + ":" + elementId + " does not exist"); 31 | } else if (el.getText().isEmpty()) { 32 | throw new RuntimeException(namespace + ":" + elementId + " is an empty string"); 33 | } else { 34 | return el.getText(); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/processor/support/UriGenerator.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor.support; 2 | 3 | public interface UriGenerator { 4 | 5 | /** 6 | * @param t Object to generate a URI for - can be anything - a File, a String, etc. 7 | * @return a string to be used for a URI 8 | * @throws Exception error generating the URI 9 | * 10 | */ 11 | public String generateUri(T t) throws Exception; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/reader/DocumentItemReader.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.reader; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.document.DocumentManager; 5 | import com.marklogic.client.document.DocumentPage; 6 | import com.marklogic.client.document.DocumentRecord; 7 | import com.marklogic.client.document.GenericDocumentManager; 8 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 9 | import com.marklogic.client.query.StructuredQueryDefinition; 10 | import org.springframework.batch.item.*; 11 | 12 | public class DocumentItemReader implements ItemStreamReader { 13 | 14 | private DatabaseClientProvider databaseClientProvider; 15 | private GenericDocumentManager docMgr; 16 | private StructuredQueryDefinition queryDef; 17 | private DocumentPage page; 18 | private long start = 1L; 19 | 20 | public DocumentItemReader(DatabaseClientProvider databaseClientProvider, StructuredQueryDefinition queryDef) { 21 | this.databaseClientProvider = databaseClientProvider; 22 | this.queryDef = queryDef; 23 | } 24 | 25 | @Override 26 | public DocumentRecord read() throws Exception { 27 | if (page.hasNext()) { 28 | return page.next(); 29 | } else if (page.hasNextPage()) { 30 | page = docMgr.search(queryDef, start); 31 | start += page.getPageSize(); 32 | return page.next(); 33 | } else { 34 | return null; 35 | } 36 | } 37 | 38 | @Override 39 | public void open(ExecutionContext executionContext) throws ItemStreamException { 40 | DatabaseClient databaseClient = databaseClientProvider.getDatabaseClient(); 41 | docMgr = databaseClient.newDocumentManager(); 42 | docMgr.setMetadataCategories(DocumentManager.Metadata.ALL); 43 | page = docMgr.search(queryDef, start); 44 | start += page.getPageSize(); 45 | executionContext.put("PAGE_INDEX", page.getPageNumber()); 46 | } 47 | 48 | @Override 49 | public void update(ExecutionContext executionContext) throws ItemStreamException { 50 | executionContext.put("PAGE_INDEX", page.getPageNumber()); 51 | } 52 | 53 | @Override 54 | public void close() throws ItemStreamException { 55 | page.close(); 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/tasklet/DeleteDocumentsTasklet.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.tasklet; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.datamovement.DataMovementManager; 5 | import com.marklogic.client.datamovement.DeleteListener; 6 | import com.marklogic.client.datamovement.JobTicket; 7 | import com.marklogic.client.datamovement.QueryBatcher; 8 | import com.marklogic.client.document.GenericDocumentManager; 9 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 10 | import com.marklogic.client.io.SearchHandle; 11 | import com.marklogic.client.query.MatchDocumentSummary; 12 | import com.marklogic.client.query.QueryManager; 13 | import com.marklogic.client.query.StructuredQueryDefinition; 14 | import org.springframework.batch.core.StepContribution; 15 | import org.springframework.batch.core.scope.context.ChunkContext; 16 | import org.springframework.batch.core.step.tasklet.Tasklet; 17 | import org.springframework.batch.repeat.RepeatStatus; 18 | 19 | public class DeleteDocumentsTasklet implements Tasklet { 20 | 21 | private DatabaseClient databaseClient; 22 | private StructuredQueryDefinition queryDefinition; 23 | private String marklogicVersion = "8"; 24 | 25 | public DeleteDocumentsTasklet(DatabaseClientProvider databaseClientProvider, StructuredQueryDefinition queryDef) { 26 | this.databaseClient = databaseClientProvider.getDatabaseClient(); 27 | this.queryDefinition = queryDef; 28 | } 29 | 30 | @Override 31 | public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { 32 | if (marklogicVersion.equals("8")) { 33 | QueryManager qryMgr = databaseClient.newQueryManager(); 34 | GenericDocumentManager docMgr = databaseClient.newDocumentManager(); 35 | SearchHandle handle = qryMgr.search(queryDefinition, new SearchHandle()); 36 | MatchDocumentSummary[] results = handle.getMatchResults(); 37 | for (int i = 0; i < results.length; i++) { 38 | docMgr.delete(results[i].getUri()); 39 | } 40 | } else if (marklogicVersion.equals("9")) { 41 | final DataMovementManager dataMovementManager = databaseClient.newDataMovementManager(); 42 | QueryBatcher qb = dataMovementManager.newQueryBatcher(queryDefinition) 43 | .withBatchSize(2500) 44 | .withConsistentSnapshot() 45 | .onUrisReady(new DeleteListener()) 46 | .onQueryFailure(throwable -> throwable.printStackTrace()); 47 | JobTicket ticket = dataMovementManager.startJob(qb); 48 | qb.awaitCompletion(); 49 | dataMovementManager.stopJob(ticket); 50 | 51 | } 52 | return RepeatStatus.FINISHED; 53 | } 54 | 55 | 56 | public void setMarklogicVersion(String marklogicVersion) { 57 | this.marklogicVersion = marklogicVersion; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/tasklet/EvaluateServerSideStringTasklet.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.tasklet; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.eval.ServerEvaluationCall; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.batch.core.StepContribution; 8 | import org.springframework.batch.core.scope.context.ChunkContext; 9 | import org.springframework.batch.core.step.tasklet.Tasklet; 10 | import org.springframework.batch.repeat.RepeatStatus; 11 | 12 | public class EvaluateServerSideStringTasklet implements Tasklet { 13 | 14 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 15 | private DatabaseClient databaseClient; 16 | private String evalString; 17 | private ServerSideLanguage language; 18 | 19 | public enum ServerSideLanguage { JAVASCRIPT, XQUERY } 20 | 21 | public EvaluateServerSideStringTasklet( 22 | DatabaseClient databaseClient, 23 | ServerSideLanguage language, 24 | String evalString) { 25 | this.databaseClient = databaseClient; 26 | this.evalString = evalString; 27 | this.language = language; 28 | } 29 | 30 | @Override 31 | public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { 32 | ServerEvaluationCall theCall = databaseClient.newServerEval(); 33 | if (language.equals(ServerSideLanguage.JAVASCRIPT)) { 34 | theCall.javascript(evalString); 35 | } else if (language.equals(ServerSideLanguage.XQUERY)) { 36 | theCall.xquery(evalString); 37 | } else { 38 | throw new IllegalArgumentException("Server side language not recognized"); 39 | } 40 | String response = theCall.evalAs(String.class); 41 | logger.info(evalString); 42 | logger.info(response); 43 | return RepeatStatus.FINISHED; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/writer/DataHubItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.document.ServerTransform; 5 | 6 | import java.util.Map; 7 | 8 | public class DataHubItemWriter extends MarkLogicItemWriter { 9 | 10 | protected final String TRANSFORM_NAME = "run-flow"; 11 | protected final String ENTITY_PARAM = "entity-name"; 12 | protected final String FLOW_PARAM = "flow-name"; 13 | protected final String JOB_ID_PARAM = "job-id"; 14 | 15 | protected String entity; 16 | protected String flow; 17 | protected String jobId; 18 | protected ServerTransform serverTransform; 19 | 20 | public DataHubItemWriter(DatabaseClient client, String entity, String flow, String jobId) { 21 | this(client, entity, flow, jobId, null); 22 | } 23 | 24 | public DataHubItemWriter(DatabaseClient client, String entity, String flow, String jobId, Map transformParams) { 25 | super(client); 26 | this.entity = entity; 27 | this.flow = flow; 28 | ServerTransform serverTransform = new ServerTransform(TRANSFORM_NAME); 29 | serverTransform.addParameter(ENTITY_PARAM, entity); 30 | serverTransform.addParameter(FLOW_PARAM, flow); 31 | serverTransform.addParameter(JOB_ID_PARAM, jobId); 32 | this.setServerTransform(serverTransform); 33 | if (transformParams != null) { 34 | for (String key : transformParams.keySet()) { 35 | serverTransform.addParameter(key, transformParams.get(key)); 36 | } 37 | } 38 | } 39 | 40 | public String getEntity() { 41 | return entity; 42 | } 43 | 44 | public void setEntity(String entity) { 45 | this.entity = entity; 46 | } 47 | 48 | public String getFlow() { 49 | return flow; 50 | } 51 | 52 | public void setFlow(String flow) { 53 | this.flow = flow; 54 | } 55 | 56 | public String getJobId() { 57 | return jobId; 58 | } 59 | 60 | public void setJobId(String jobId) { 61 | this.jobId = jobId; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/writer/InvokeModuleItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.eval.EvalResult; 5 | import com.marklogic.client.eval.EvalResultIterator; 6 | import com.marklogic.client.eval.ServerEvaluationCall; 7 | import com.marklogic.client.io.StringHandle; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.batch.item.ItemWriter; 11 | 12 | import org.jdom2.Namespace; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | public class InvokeModuleItemWriter implements ItemWriter> { 17 | 18 | protected String module; 19 | protected DatabaseClient databaseClient; 20 | protected ServerEvaluationCall invoker; 21 | protected List namespaces; 22 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 23 | 24 | public InvokeModuleItemWriter(DatabaseClient databaseClient, String module) { 25 | this.module = module; 26 | this.databaseClient = databaseClient; 27 | invoker = databaseClient.newServerEval(); 28 | invoker.modulePath(module); 29 | } 30 | 31 | public InvokeModuleItemWriter(DatabaseClient databaseClient, String module, List namespaces) { 32 | this(databaseClient, module); 33 | for (Namespace namespace : namespaces) { 34 | invoker.addNamespace(namespace.getPrefix(), namespace.getURI()); 35 | } 36 | } 37 | 38 | @Override 39 | public void write(List> items) throws Exception { 40 | for (Map item : items) { 41 | for (Map.Entry name : item.entrySet()) { 42 | invoker.addVariable(name.getKey(), name.getValue()); 43 | } 44 | processResults(invoker.eval()); 45 | 46 | } 47 | } 48 | 49 | public void processResults(EvalResultIterator itr) { 50 | String item = null; 51 | while (itr.hasNext()) { 52 | EvalResult result = itr.next(); 53 | switch (result.getType()) { 54 | case STRING: { 55 | item = result.getAs(String.class); 56 | logger.info(item); 57 | break; 58 | } 59 | default: { 60 | StringHandle handle = result.get(new StringHandle()); 61 | logger.info(handle.get()); 62 | break; 63 | } 64 | } 65 | } 66 | } 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/writer/MarkLogicPatchItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.document.DocumentPatchBuilder; 5 | import com.marklogic.client.document.XMLDocumentManager; 6 | import com.marklogic.client.io.marker.DocumentPatchHandle; 7 | import com.marklogic.client.util.EditableNamespaceContext; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.batch.item.ItemWriter; 11 | 12 | import java.util.List; 13 | 14 | public class MarkLogicPatchItemWriter implements ItemWriter { 15 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 16 | 17 | private DatabaseClient databaseClient; 18 | 19 | public MarkLogicPatchItemWriter(DatabaseClient client) { 20 | this.databaseClient = client; 21 | } 22 | 23 | @Override 24 | public void write(List items) throws Exception { 25 | for (String[] item : items) { 26 | String uri = item[0]; 27 | String xmlPatch = item[1]; 28 | logger.info(uri); 29 | XMLDocumentManager docMgr = databaseClient.newXMLDocumentManager(); 30 | EditableNamespaceContext namespaces = new EditableNamespaceContext(); 31 | namespaces.put("html", "http://www.w3.org/1999/xhtml"); 32 | 33 | DocumentPatchBuilder xmlPatchBldr = docMgr.newPatchBuilder(); 34 | xmlPatchBldr.setNamespaces(namespaces); 35 | 36 | //note the root element is referenced in the first parameter of this call, you may need to change based on your document 37 | DocumentPatchHandle patchHandle = xmlPatchBldr.insertFragment("/doc", DocumentPatchBuilder.Position.LAST_CHILD, xmlPatch).build(); 38 | docMgr.patch(uri, patchHandle); 39 | } 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/writer/XccItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.client.document.DocumentWriteOperation; 4 | import com.marklogic.client.ext.batch.XccBatchWriter; 5 | import com.marklogic.xcc.ContentSource; 6 | import org.springframework.batch.item.ExecutionContext; 7 | import org.springframework.batch.item.ItemStream; 8 | import org.springframework.batch.item.ItemStreamException; 9 | import org.springframework.batch.item.ItemWriter; 10 | 11 | import java.util.List; 12 | 13 | public class XccItemWriter implements ItemWriter, ItemStream { 14 | 15 | //Used for XCC 16 | private XccBatchWriter xccBatchWriter; 17 | private List contentSources; 18 | 19 | public XccItemWriter(List contentSources) { 20 | this.contentSources = contentSources; 21 | } 22 | 23 | @Override 24 | public void open(ExecutionContext executionContext) throws ItemStreamException { 25 | xccBatchWriter = new XccBatchWriter(contentSources); 26 | } 27 | 28 | @Override 29 | public void update(ExecutionContext executionContext) throws ItemStreamException { 30 | 31 | } 32 | 33 | @Override 34 | public void close() throws ItemStreamException { 35 | 36 | } 37 | 38 | @Override 39 | public void write(List items) throws Exception { 40 | xccBatchWriter.initialize(); 41 | xccBatchWriter.write(items); 42 | xccBatchWriter.waitForCompletion(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/writer/support/DefaultUriTransformer.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer.support; 2 | 3 | public class DefaultUriTransformer implements UriTransformer { 4 | 5 | private String outputUriPrefix; 6 | private String outputUriReplace; 7 | private String outputUriSuffix; 8 | 9 | public DefaultUriTransformer() { 10 | 11 | } 12 | 13 | public DefaultUriTransformer (String outputUriPrefix, String outputUriSuffix, String outputUriReplace) { 14 | this.outputUriPrefix = outputUriPrefix; 15 | this.outputUriSuffix = outputUriSuffix; 16 | this.outputUriReplace = outputUriReplace; 17 | } 18 | 19 | @Override 20 | public String transform(String uri) { 21 | uri = (outputUriReplace != null) && !outputUriReplace.isEmpty() ? applyOutputUriReplace(uri, outputUriReplace) : uri; 22 | uri = (outputUriPrefix != null) && !outputUriPrefix.isEmpty() ? outputUriPrefix + uri : uri; 23 | uri = (outputUriSuffix != null) && !outputUriSuffix.isEmpty() ? uri + outputUriSuffix : uri; 24 | return uri; 25 | } 26 | 27 | private String applyOutputUriReplace(String uri, String outputUriReplace) { 28 | String[] regexReplace = outputUriReplace.split(","); 29 | for (int i = 0; i < regexReplace.length; i=i+2) { 30 | String regex = regexReplace[i]; 31 | String replace = regexReplace[i+1]; 32 | uri = uri.replaceAll(regex, replace); 33 | } 34 | return uri; 35 | } 36 | 37 | /* 38 | Prepends a string to the URI after substitution 39 | */ 40 | public void setOutputUriPrefix(String outputUriPrefix) { 41 | this.outputUriPrefix = outputUriPrefix; 42 | } 43 | 44 | /* 45 | The outputUriReplace option accepts a comma delimited list of regular expression and replacement string pairs 46 | @param outputUriReplace Example regex,'replaceString',regex,'replaceString' 47 | */ 48 | public void setOutputUriReplace(String outputUriReplace) { 49 | this.outputUriReplace = outputUriReplace; 50 | } 51 | 52 | /* 53 | Appends a string to the URI after substitution 54 | */ 55 | public void setOutputUriSuffix(String outputUriSuffix) { 56 | this.outputUriSuffix = outputUriSuffix; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /core/src/main/java/com/marklogic/spring/batch/item/writer/support/UriTransformer.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer.support; 2 | 3 | public interface UriTransformer { 4 | 5 | public String transform(String uri); 6 | } 7 | -------------------------------------------------------------------------------- /core/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/client/JavaClientApiIssue388Test.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.client; 2 | 3 | import com.marklogic.junit.MarkLogicNamespaceProvider; 4 | import org.junit.Test; 5 | 6 | import com.marklogic.client.query.StructuredQueryBuilder; 7 | import com.marklogic.client.query.StructuredQueryDefinition; 8 | import com.marklogic.junit.Fragment; 9 | 10 | /* 11 | * Used to test Java Client Api Issue 388 12 | * https://github.com/marklogic/java-client-api/issues/388 13 | */ 14 | 15 | public class JavaClientApiIssue388Test { 16 | 17 | @Test 18 | public void replicateIssue388Test() { 19 | StructuredQueryBuilder qb = new StructuredQueryBuilder("default"); 20 | StructuredQueryDefinition querydef = 21 | qb.and(qb.term("neighborhood"), 22 | qb.valueConstraint("industry", "Real Estate")); 23 | Fragment frag = new Fragment(querydef.serialize(), new MarkLogicNamespaceProvider().getNamespaces()); 24 | frag.assertElementValue("//search:text", "neighborhood"); 25 | System.out.println(frag.getPrettyXml()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/JobExecutionTestUtils.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch; 2 | 3 | import java.util.Random; 4 | 5 | import org.springframework.batch.core.JobExecution; 6 | import org.springframework.batch.core.JobInstance; 7 | import org.springframework.batch.item.ExecutionContext; 8 | 9 | public class JobExecutionTestUtils { 10 | 11 | public static JobExecution getJobExecution() { 12 | JobInstance jobInstance = new JobInstance(Math.abs(new Random(100L).nextLong()), "sampleJob"); 13 | JobExecution jobExecution = new JobExecution(jobInstance, 123L, JobParametersTestUtils.getJobParameters(), "abc"); 14 | jobExecution.createStepExecution("sampleStep1"); 15 | jobExecution.createStepExecution("sampleStep2"); 16 | ExecutionContext ec = new ExecutionContext(); 17 | ec.put("testName", "testValue"); 18 | jobExecution.setExecutionContext(ec); 19 | return jobExecution; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/JobParametersTestUtils.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch; 2 | 3 | import java.util.Date; 4 | 5 | import org.springframework.batch.core.JobParameters; 6 | import org.springframework.batch.core.JobParametersBuilder; 7 | import org.springframework.batch.core.JobParametersIncrementer; 8 | 9 | public class JobParametersTestUtils implements JobParametersIncrementer { 10 | 11 | private static JobParameters jobParameters = new JobParametersBuilder().toJobParameters(); 12 | 13 | public static JobParameters getJobParameters() { 14 | JobParametersBuilder jobParametersBuilder = new JobParametersBuilder(); 15 | if (jobParameters.isEmpty()) { 16 | jobParametersBuilder.addLong("id", 1L, true); 17 | jobParametersBuilder.addString("stringTest", "Joe Cool", true); 18 | jobParametersBuilder.addDate("start", new Date(), false); 19 | jobParametersBuilder.addLong("longTest", 1239L, false); 20 | jobParametersBuilder.addDouble("doubleTest", 1.35D, false); 21 | jobParameters = jobParametersBuilder.toJobParameters(); 22 | } else { 23 | JobParametersTestUtils utils = new JobParametersTestUtils(); 24 | jobParameters = utils.getNext(jobParameters); 25 | } 26 | return jobParameters; 27 | 28 | } 29 | 30 | @Override 31 | public JobParameters getNext(JobParameters parameters) { 32 | JobParametersBuilder builder = new JobParametersBuilder(parameters); 33 | builder.addLong("id", parameters.getLong("id") + 1); 34 | return builder.toJobParameters(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/MarkLogicSpringBatchConfigTests.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch; 2 | 3 | import com.marklogic.spring.batch.test.AbstractJobRepositoryTest; 4 | import org.junit.Test; 5 | 6 | import com.marklogic.client.admin.ServerConfigurationManager; 7 | import com.marklogic.client.admin.ServerConfigurationManager.UpdatePolicy; 8 | 9 | public class MarkLogicSpringBatchConfigTests extends AbstractJobRepositoryTest { 10 | 11 | 12 | 13 | @Test 14 | public void isVersionOptimalUpdatePolicyTest() { 15 | 16 | // create server config manager 17 | ServerConfigurationManager configMgr = getClient().newServerConfigManager(); 18 | 19 | // read the server config from the database 20 | configMgr.readConfiguration(); 21 | 22 | assertEquals(UpdatePolicy.VERSION_REQUIRED, configMgr.getUpdatePolicy()); 23 | 24 | // release the client 25 | //client.release(); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/SpringBatchNamespaceProvider.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch; 2 | 3 | import java.util.List; 4 | 5 | import org.jdom2.Namespace; 6 | 7 | import com.marklogic.junit.MarkLogicNamespaceProvider; 8 | import com.marklogic.spring.batch.core.MarkLogicSpringBatch; 9 | 10 | public class SpringBatchNamespaceProvider extends MarkLogicNamespaceProvider { 11 | 12 | @Override 13 | protected List buildListOfNamespaces() { 14 | List list = super.buildListOfNamespaces(); 15 | list.add(Namespace.getNamespace("geo", "http://geonames.org")); 16 | list.add(Namespace.getNamespace(MarkLogicSpringBatch.JOB_NAMESPACE_PREFIX, MarkLogicSpringBatch.JOB_NAMESPACE)); 17 | list.add(Namespace.getNamespace("xs", "http://www.w3.org/2001/XMLSchema")); 18 | list.add(Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance")); 19 | list.add(Namespace.getNamespace("search", "http://marklogic.com/appservices/search")); 20 | list.add(Namespace.getNamespace("html", "http://www.w3.org/1999/xhtml")); 21 | return list; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/bind/JaxbConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.bind; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.oxm.jaxb.Jaxb2Marshaller; 6 | 7 | @Configuration 8 | public class JaxbConfiguration { 9 | 10 | @Bean 11 | public Jaxb2Marshaller jaxb2Marshaller() { 12 | Jaxb2Marshaller marshaller = new Jaxb2Marshaller(); 13 | marshaller.setClassesToBeBound( 14 | com.marklogic.spring.batch.core.AdaptedJobExecution.class, 15 | com.marklogic.spring.batch.core.AdaptedJobInstance.class, 16 | com.marklogic.spring.batch.core.AdaptedJobParameters.class, 17 | com.marklogic.spring.batch.core.AdaptedStepExecution.class, 18 | com.marklogic.spring.batch.core.AdaptedExecutionContext.class); 19 | marshaller.setAdapters( 20 | new ExecutionContextAdapter(), 21 | new JobExecutionAdapter(), 22 | new JobInstanceAdapter(), 23 | new JobParametersAdapter(), 24 | new StepExecutionAdapter()); 25 | //marshaller.setMarshallerProperties(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); 26 | return marshaller; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/config/DoNotUseMarkLogicBatchConfigurerTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.config; 2 | 3 | import com.marklogic.spring.batch.core.repository.support.MarkLogicJobRepositoryProperties; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.BeansException; 9 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 10 | import org.springframework.context.ApplicationContext; 11 | import org.springframework.context.ApplicationContextAware; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.TestPropertySource; 14 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 15 | 16 | import java.util.Arrays; 17 | 18 | import static org.hamcrest.MatcherAssert.assertThat; 19 | import static org.hamcrest.CoreMatchers.*; 20 | 21 | 22 | @RunWith(SpringJUnit4ClassRunner.class) 23 | @ContextConfiguration(classes = {MarkLogicBatchConfigurer.class, MarkLogicBatchConfiguration.class, MarkLogicJobRepositoryProperties.class}) 24 | @TestPropertySource(properties = { "marklogic.batch.config.enabled=false"}) 25 | public class DoNotUseMarkLogicBatchConfigurerTest implements ApplicationContextAware { 26 | 27 | private final static Logger logger = LoggerFactory.getLogger(UseMarkLogicBatchConfigurerTest.class); 28 | 29 | ApplicationContext ctx; 30 | 31 | @Test 32 | public void markLogicBatchConfigurerDoesNotExistTest() throws NoSuchBeanDefinitionException { 33 | for (String beanName : ctx.getBeanDefinitionNames()) { 34 | logger.info(beanName); 35 | } 36 | assertThat(Arrays.asList(ctx.getBeanDefinitionNames()), not(hasItem("markLogicBatchConfigurer"))); 37 | } 38 | 39 | @Override 40 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 41 | this.ctx = applicationContext; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/config/UseMarkLogicBatchConfigurerTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.config; 2 | 3 | import com.marklogic.spring.batch.core.repository.support.MarkLogicJobRepositoryProperties; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.BeansException; 9 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 10 | import org.springframework.context.ApplicationContext; 11 | import org.springframework.context.ApplicationContextAware; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.TestPropertySource; 14 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 15 | import org.springframework.util.Assert; 16 | 17 | @RunWith(SpringJUnit4ClassRunner.class) 18 | @ContextConfiguration(classes = {MarkLogicBatchConfigurer.class, MarkLogicBatchConfiguration.class, MarkLogicJobRepositoryProperties.class}) 19 | @TestPropertySource(properties = { "marklogic.batch.config.enabled=true"}) 20 | public class UseMarkLogicBatchConfigurerTest implements ApplicationContextAware { 21 | 22 | private final static Logger logger = LoggerFactory.getLogger(UseMarkLogicBatchConfigurerTest.class); 23 | 24 | ApplicationContext ctx; 25 | 26 | @Test 27 | public void doBeansExistTest() throws NoSuchBeanDefinitionException { 28 | for (String beanName : ctx.getBeanDefinitionNames()) { 29 | logger.info(beanName); 30 | } 31 | Assert.isTrue(ctx.containsBeanDefinition("markLogicBatchConfigurer"), "Cannot find batchConfigurer bean"); 32 | } 33 | 34 | @Override 35 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 36 | this.ctx = applicationContext; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/core/explore/GetJobInstancesTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core.explore; 2 | 3 | import com.marklogic.spring.batch.JobParametersTestUtils; 4 | import com.marklogic.spring.batch.test.AbstractJobRepositoryTest; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import org.springframework.batch.core.JobInstance; 8 | import org.springframework.batch.core.explore.JobExplorer; 9 | import org.springframework.batch.core.launch.NoSuchJobException; 10 | import org.springframework.batch.core.repository.JobRepository; 11 | 12 | import java.util.List; 13 | 14 | public class GetJobInstancesTest extends AbstractJobRepositoryTest { 15 | 16 | private final String JOB_NAME = "testJob"; 17 | private final String JOB_NAME_2 = JOB_NAME + "2"; 18 | private final String JOB_NAME_3 = JOB_NAME + "3"; 19 | 20 | private JobRepository jobRepository; 21 | private JobExplorer jobExplorer; 22 | 23 | @Before 24 | public void givenAJobExplorer() throws Exception { 25 | jobRepository = getJobRepository(); 26 | jobExplorer = getJobExplorer(); 27 | } 28 | 29 | @Test 30 | public void retrieveJobInstanceByIdTest() { 31 | JobInstance expectedJobInstance = jobRepository.createJobInstance(JOB_NAME, JobParametersTestUtils.getJobParameters()); 32 | JobInstance actualJobInstance = jobExplorer.getJobInstance(expectedJobInstance.getId()); 33 | assertTrue(expectedJobInstance.equals(actualJobInstance)); 34 | } 35 | 36 | @Test 37 | public void getJobInstanceCountTest() throws NoSuchJobException { 38 | createJobInstances(); 39 | assertEquals(3, jobExplorer.getJobInstanceCount(JOB_NAME)); 40 | } 41 | 42 | @Test(expected = NoSuchJobException.class) 43 | public void getJobInstanceCountNoJobException() throws NoSuchJobException { 44 | jobExplorer.getJobInstanceCount("NoJobs"); 45 | } 46 | 47 | @Test 48 | public void getJobInstancesTest() { 49 | createJobInstances(); 50 | List jobInstances = jobExplorer.getJobInstances(JOB_NAME, 1, 2); 51 | assertEquals(2, jobInstances.size()); 52 | } 53 | 54 | @Test 55 | public void findJobInstancesTest() { 56 | createJobInstances(); 57 | List jobInstances = jobExplorer.findJobInstancesByJobName(JOB_NAME, 1, 2); 58 | assertEquals(2, jobInstances.size()); 59 | } 60 | 61 | @Test 62 | public void getJobNamesTest() { 63 | createJobInstances(); 64 | List jobNames = jobExplorer.getJobNames(); 65 | assertEquals(3, jobNames.size()); 66 | assertTrue(jobNames.get(0).equals(JOB_NAME)); 67 | assertTrue(jobNames.get(1).equals(JOB_NAME_2)); 68 | assertTrue(jobNames.get(2).equals(JOB_NAME_3)); 69 | } 70 | 71 | private void createJobInstances() { 72 | jobRepository.createJobInstance(JOB_NAME, JobParametersTestUtils.getJobParameters()); 73 | jobRepository.createJobInstance(JOB_NAME, JobParametersTestUtils.getJobParameters()); 74 | jobRepository.createJobInstance(JOB_NAME, JobParametersTestUtils.getJobParameters()); 75 | jobRepository.createJobInstance(JOB_NAME_2, JobParametersTestUtils.getJobParameters()); 76 | jobRepository.createJobInstance(JOB_NAME_3, JobParametersTestUtils.getJobParameters()); 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/core/launch/support/TestJobParametersIncrementer.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core.launch.support; 2 | 3 | import org.springframework.batch.core.JobParameters; 4 | import org.springframework.batch.core.JobParametersBuilder; 5 | import org.springframework.batch.core.JobParametersIncrementer; 6 | 7 | public class TestJobParametersIncrementer implements JobParametersIncrementer { 8 | 9 | @Override 10 | public JobParameters getNext(JobParameters parameters) { 11 | return new JobParametersBuilder().addString("foo", "spam").toJobParameters(); 12 | } 13 | 14 | } -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/core/step/StepSupport.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.core.step; 2 | 3 | import org.springframework.batch.core.JobInterruptedException; 4 | import org.springframework.batch.core.Step; 5 | import org.springframework.batch.core.StepExecution; 6 | import org.springframework.batch.core.UnexpectedJobExecutionException; 7 | import org.springframework.beans.factory.BeanNameAware; 8 | 9 | public class StepSupport implements Step, BeanNameAware { 10 | 11 | private String name; 12 | 13 | private int startLimit = Integer.MAX_VALUE; 14 | 15 | private boolean allowStartIfComplete; 16 | 17 | /** 18 | * Default constructor for {@link StepSupport}. 19 | */ 20 | public StepSupport() { 21 | super(); 22 | } 23 | 24 | /** 25 | * @param string 26 | */ 27 | public StepSupport(String string) { 28 | super(); 29 | this.name = string; 30 | } 31 | 32 | @Override 33 | public String getName() { 34 | return this.name; 35 | } 36 | 37 | /** 38 | * Set the name property if it is not already set. Because of the order of the callbacks in a Spring container the 39 | * name property will be set first if it is present. Care is needed with bean definition inheritance - if a parent 40 | * bean has a name, then its children need an explicit name as well, otherwise they will not be unique. 41 | * 42 | * @see org.springframework.beans.factory.BeanNameAware#setBeanName(java.lang.String) 43 | */ 44 | @Override 45 | public void setBeanName(String name) { 46 | if (this.name == null) { 47 | this.name = name; 48 | } 49 | } 50 | 51 | /** 52 | * Set the name property. Always overrides the default value if this object is a Spring bean. 53 | * 54 | * @see #setBeanName(java.lang.String) 55 | */ 56 | public void setName(String name) { 57 | this.name = name; 58 | } 59 | 60 | @Override 61 | public int getStartLimit() { 62 | return this.startLimit; 63 | } 64 | 65 | /** 66 | * Public setter for the startLimit. 67 | * 68 | * @param startLimit the startLimit to set 69 | */ 70 | public void setStartLimit(int startLimit) { 71 | this.startLimit = startLimit; 72 | } 73 | 74 | @Override 75 | public boolean isAllowStartIfComplete() { 76 | return this.allowStartIfComplete; 77 | } 78 | 79 | /** 80 | * Public setter for the shouldAllowStartIfComplete. 81 | * 82 | * @param allowStartIfComplete the shouldAllowStartIfComplete to set 83 | */ 84 | public void setAllowStartIfComplete(boolean allowStartIfComplete) { 85 | this.allowStartIfComplete = allowStartIfComplete; 86 | } 87 | 88 | /** 89 | * Not supported but provided so that tests can easily create a step. 90 | * 91 | * @throws UnsupportedOperationException always 92 | * 93 | * @see org.springframework.batch.core.Step#execute(org.springframework.batch.core.StepExecution) 94 | */ 95 | @Override 96 | public void execute(StepExecution stepExecution) throws JobInterruptedException, UnexpectedJobExecutionException { 97 | throw new UnsupportedOperationException( 98 | "Cannot process a StepExecution. Use a smarter subclass of StepSupport."); 99 | } 100 | } -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/item/columnmap/JacksonColumnMapSerializerTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.columnmap; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.marklogic.spring.batch.columnmap.ColumnMapSerializer; 6 | import com.marklogic.spring.batch.columnmap.JacksonColumnMapSerializer; 7 | import org.junit.Assert; 8 | import org.junit.Test; 9 | 10 | import java.util.Map; 11 | 12 | public class JacksonColumnMapSerializerTest extends Assert { 13 | 14 | private ColumnMapSerializer sut = new JacksonColumnMapSerializer(); 15 | 16 | @Test 17 | public void test() throws Exception { 18 | Map map = new java.util.HashMap<>(); 19 | map.put("color", "red"); 20 | map.put("size", 10); 21 | map.put("boolean", true); 22 | 23 | String json = sut.serializeColumnMap(map, "ignored"); 24 | 25 | JsonNode node = new ObjectMapper().readTree(json); 26 | assertEquals(true, node.get("boolean").asBoolean()); 27 | assertEquals(10, node.get("size").asInt()); 28 | assertEquals("red", node.get("color").asText()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/item/processor/ColumnMapProcessorTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor; 2 | 3 | import com.marklogic.client.document.DocumentWriteOperation; 4 | import com.marklogic.client.io.StringHandle; 5 | import com.marklogic.spring.batch.columnmap.DefaultStaxColumnMapSerializer; 6 | import com.marklogic.spring.batch.columnmap.JacksonColumnMapSerializer; 7 | import org.junit.Assert; 8 | import org.junit.Test; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | public class ColumnMapProcessorTest extends Assert { 16 | 17 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 18 | 19 | @Test 20 | public void columnMapSimpleXmlTest() throws Exception { 21 | ColumnMapProcessor columnMapProcessor = new ColumnMapProcessor(new DefaultStaxColumnMapSerializer()); 22 | columnMapProcessor.setRootLocalName("test"); 23 | 24 | Map map = new HashMap(); 25 | map.put("sample", "value"); 26 | 27 | DocumentWriteOperation handle = columnMapProcessor.process(map); 28 | logger.info(handle.getUri()); 29 | StringHandle strHandle = (StringHandle) handle.getContent(); 30 | logger.info(strHandle.get()); 31 | assertTrue("value".equals(strHandle.get())); 32 | assertNotNull(handle); 33 | 34 | } 35 | 36 | @Test 37 | public void getRootNameFromMapTest() throws Exception { 38 | ColumnMapProcessor columnMapProcessor = new ColumnMapProcessor(new DefaultStaxColumnMapSerializer()); 39 | columnMapProcessor.setRootLocalName("type"); 40 | 41 | Map map = new HashMap(); 42 | map.put("type", "junk"); 43 | map.put("sample", "value"); 44 | 45 | DocumentWriteOperation handle = columnMapProcessor.process(map); 46 | logger.info(handle.getUri()); 47 | StringHandle strHandle = (StringHandle) handle.getContent(); 48 | logger.info(strHandle.get()); 49 | assertTrue("junkvalue".equals(strHandle.get())); 50 | assertNotNull(handle); 51 | 52 | } 53 | 54 | @Test 55 | public void columnMapSimpleJsonTest() throws Exception { 56 | ColumnMapProcessor columnMapProcessor = new ColumnMapProcessor(new JacksonColumnMapSerializer()); 57 | 58 | Map map = new HashMap(); 59 | map.put("sample", "value"); 60 | 61 | DocumentWriteOperation handle = columnMapProcessor.process(map); 62 | logger.info(handle.getUri()); 63 | StringHandle strHandle = (StringHandle) handle.getContent(); 64 | logger.info(strHandle.get()); 65 | assertTrue("{\"sample\":\"value\"}".equals(strHandle.get())); 66 | assertNotNull(handle); 67 | 68 | } 69 | 70 | 71 | } 72 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/item/processor/support/DocumentUriGeneratorTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.processor.support; 2 | 3 | import org.jdom2.Document; 4 | import org.jdom2.input.SAXBuilder; 5 | import org.junit.Test; 6 | 7 | import java.io.StringReader; 8 | 9 | import static org.hamcrest.MatcherAssert.assertThat; 10 | 11 | public class DocumentUriGeneratorTest { 12 | 13 | @Test 14 | public void generateIdFromDocumentWithoutNamespaceTest() throws Exception { 15 | SAXBuilder jdomBuilder = new SAXBuilder(); 16 | Document document = jdomBuilder.build(new StringReader("ABC123")); 17 | DocumentUriGenerator uriGenerator = new DocumentUriGenerator("id"); 18 | assertThat("Expecting ABC is generated URI", uriGenerator.generateUri(document).equals("ABC")); 19 | } 20 | 21 | @Test 22 | public void generateIdFromDocumentWithNamespaceTest() throws Exception { 23 | SAXBuilder jdomBuilder = new SAXBuilder(); 24 | Document document = jdomBuilder.build(new StringReader("ABC123")); 25 | DocumentUriGenerator uriGenerator = new DocumentUriGenerator("abc","id"); 26 | assertThat("Expecting ABC is generated uri", uriGenerator.generateUri(document).equals("ABC")); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/item/writer/InvokeModuleItemWriterTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.admin.ExtensionLibrariesManager; 5 | import com.marklogic.client.ext.DatabaseClientConfig; 6 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 7 | import com.marklogic.client.io.Format; 8 | import com.marklogic.client.io.StringHandle; 9 | import com.marklogic.client.ext.spring.SimpleDatabaseClientProvider; 10 | import com.marklogic.junit.ClientTestHelper; 11 | import com.marklogic.spring.batch.test.AbstractSpringBatchTest; 12 | import com.marklogic.spring.batch.test.SpringBatchNamespaceProvider; 13 | import org.junit.After; 14 | import org.junit.Before; 15 | import org.junit.Test; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.test.context.ContextConfiguration; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | @ContextConfiguration(classes = {com.marklogic.spring.batch.config.MarkLogicConfiguration.class}) 25 | public class InvokeModuleItemWriterTest extends AbstractSpringBatchTest { 26 | 27 | final String MODULE_PATH = "/ext/test.xqy"; 28 | InvokeModuleItemWriter itemWriter; 29 | ExtensionLibrariesManager libMgr; 30 | ClientTestHelper clientTestHelper; 31 | 32 | @Autowired 33 | public void setClientTestHelper(DatabaseClientConfig batchDatabaseClientConfig) { 34 | clientTestHelper = new ClientTestHelper(); 35 | DatabaseClientProvider databaseClientProvider = new SimpleDatabaseClientProvider(batchDatabaseClientConfig); 36 | clientTestHelper.setDatabaseClientProvider(databaseClientProvider); 37 | clientTestHelper.setNamespaceProvider(new SpringBatchNamespaceProvider()); 38 | return; 39 | } 40 | 41 | @Before 42 | public void setup() { 43 | DatabaseClient client = getClient(); 44 | libMgr = client.newServerConfigManager().newExtensionLibrariesManager(); 45 | StringHandle handle = new StringHandle("xdmp:document-insert('hello.xml', , (), 'test')").withFormat(Format.TEXT); 46 | libMgr.writeAs(MODULE_PATH, handle); 47 | itemWriter = new InvokeModuleItemWriter(client, MODULE_PATH); 48 | } 49 | 50 | @Test 51 | public void invokeModuleItemWriterTest() throws Exception { 52 | Map map = new HashMap(); 53 | List> maps = new ArrayList>(); 54 | maps.add(map); 55 | itemWriter.write(maps); 56 | clientTestHelper.assertInCollections("hello.xml", "test"); 57 | 58 | } 59 | 60 | @After 61 | public void teardown() { 62 | libMgr.delete(MODULE_PATH); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/item/writer/MarkLogicPatchItemWriterTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.client.DatabaseClient; 4 | import com.marklogic.client.DatabaseClientFactory; 5 | import com.marklogic.client.document.XMLDocumentManager; 6 | import com.marklogic.client.ext.DatabaseClientConfig; 7 | import com.marklogic.client.io.StringHandle; 8 | import com.marklogic.junit.Fragment; 9 | import com.marklogic.spring.batch.test.AbstractSpringBatchTest; 10 | import org.junit.Before; 11 | import org.junit.Test; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.test.context.ContextConfiguration; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | @ContextConfiguration(classes = {com.marklogic.spring.batch.config.MarkLogicConfiguration.class}) 19 | public class MarkLogicPatchItemWriterTest extends AbstractSpringBatchTest { 20 | 21 | DatabaseClient client; 22 | XMLDocumentManager docMgr; 23 | 24 | @Autowired 25 | DatabaseClientConfig batchDatabaseClientConfig; 26 | 27 | @Before 28 | public void setup() { 29 | DatabaseClientFactory.SecurityContext securityContext = 30 | new DatabaseClientFactory.DigestAuthContext( 31 | batchDatabaseClientConfig.getUsername(), batchDatabaseClientConfig.getPassword()); 32 | client = DatabaseClientFactory.newClient(batchDatabaseClientConfig.getHost(), 33 | batchDatabaseClientConfig.getPort(), securityContext); 34 | docMgr = client.newXMLDocumentManager(); 35 | StringHandle text1 = new StringHandle("Abbey D'Agostino finished the race Tuesday after helping Nikki Hamblin of New Zealand back up and urging her to finish. The two clipped heels during the late part of the race and tumbled to the ground. Hamblin has indicated she will run in the final. Emma Coburn, who took bronze in the women's 3,000 steeplechase, becoming the first American woman to medal in the event, reacted Wednesday"); 36 | docMgr.write("hello.xml", text1); 37 | } 38 | 39 | @Test 40 | public void patchItemWriterTest() throws Exception { 41 | MarkLogicPatchItemWriter itemWriter = new MarkLogicPatchItemWriter(client); 42 | String[] info = new String[2]; 43 | info[0] = "hello.xml"; 44 | info[1] = "content"; 45 | List strings = new ArrayList(); 46 | strings.add(info); 47 | itemWriter.write(strings); 48 | StringHandle handle = docMgr.read("hello.xml", new StringHandle()); 49 | logger.info(handle.toString()); 50 | Fragment frag = new Fragment(handle.toString()); 51 | frag.assertElementExists("Expecting new content", "//new[text() = 'content']"); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/item/writer/UriTransformerTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.writer; 2 | 3 | import com.marklogic.spring.batch.item.writer.support.DefaultUriTransformer; 4 | import com.marklogic.spring.batch.item.writer.support.UriTransformer; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | public class UriTransformerTest extends Assert { 9 | 10 | @Test 11 | public void uriPrefixWithNullValuesTest() { 12 | UriTransformer transformer = new DefaultUriTransformer("pre", null, null); 13 | String uri = "test"; 14 | assertEquals("pretest", transformer.transform(uri)); 15 | } 16 | 17 | @Test 18 | public void uriPrefixWithEmptyStringValuesTest() { 19 | UriTransformer transformer = new DefaultUriTransformer("pre", "", ""); 20 | String uri = "test"; 21 | assertEquals("pretest", transformer.transform(uri)); 22 | } 23 | 24 | @Test 25 | public void uriSuffixTest() { 26 | UriTransformer transformer = new DefaultUriTransformer("", "post", ""); 27 | String uri = "test"; 28 | assertEquals("testpost", transformer.transform(uri)); 29 | } 30 | 31 | @Test 32 | public void uriReplaceTest() { 33 | UriTransformer transformer = new DefaultUriTransformer("", "", "test,ABC"); 34 | String uri = "thisisatest"; 35 | assertEquals("thisisaABC", transformer.transform(uri)); 36 | } 37 | 38 | @Test 39 | public void uriPrefixAndReplaceTest() { 40 | UriTransformer transformer = new DefaultUriTransformer("pre", "", "test,ABC"); 41 | String uri = "thisisatest"; 42 | assertEquals("prethisisaABC", transformer.transform(uri)); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /core/src/test/java/com/marklogic/spring/batch/test/LoggingItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.test; 2 | 3 | import java.util.List; 4 | import java.util.logging.Logger; 5 | 6 | import org.springframework.batch.item.ItemWriter; 7 | 8 | public class LoggingItemWriter implements ItemWriter { 9 | 10 | Logger logger = Logger.getLogger("com.marklogic.spring.batch.test.LoggingItemWriter"); 11 | 12 | @Override 13 | public void write(List items) throws Exception { 14 | for (T item : items) { 15 | logger.info(item.toString()); 16 | } 17 | 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /core/src/test/resources/config/job.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 17 | 18 | 19 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /core/src/test/resources/config/job2.xml: -------------------------------------------------------------------------------- 1 | 2 | 11 | 12 | 14 | 15 | 16 | 17 | 18 | 20 | 21 | 23 | 24 | 25 | 26 | 27 | 29 | 30 | 31 | 33 | 34 | 35 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /core/src/test/resources/config/launcher-with-environment.xml: -------------------------------------------------------------------------------- 1 | 2 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /core/src/test/resources/config/launcher-with-locator.xml: -------------------------------------------------------------------------------- 1 | 2 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /core/src/test/resources/config/test-environment.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /core/src/test/resources/job.properties: -------------------------------------------------------------------------------- 1 | marklogic.host=localhost 2 | marklogic.port=8200 3 | marklogic.username=admin 4 | marklogic.password=admin 5 | marklogic.database=msb-content 6 | 7 | marklogic.jobrepo.host=localhost 8 | marklogic.jobrepo.port=8201 9 | marklogic.jobrepo.username=admin 10 | marklogic.jobrepo.password=admin 11 | marklogic.jobrepo.database=mlJobRepo-content 12 | 13 | marklogic.batch.config.enabled=true 14 | 15 | -------------------------------------------------------------------------------- /core/src/test/resources/options/sample-options.properties: -------------------------------------------------------------------------------- 1 | --host=some-host 2 | # A benefit of using a Java properties file is we can stretch values across multiple lines 3 | # And we can of course include comments too 4 | --multi_line_property=Value on \ 5 | two lines 6 | -------------------------------------------------------------------------------- /core/src/test/resources/transforms/simple.xqy: -------------------------------------------------------------------------------- 1 | xquery version "1.0-ml"; 2 | 3 | module namespace x = "http://marklogic.com/rest-api/transform/simple"; 4 | 5 | declare function x:transform( 6 | $context as map:map, 7 | $params as map:map, 8 | $content as document-node()) 9 | as document-node() { 10 | ( 11 | xdmp:log(xdmp:quote($content)), 12 | document { 13 | element { fn:local-name($content/element()) } { 14 | $content/element(), 15 | element transform {}, 16 | for $param in map:keys($params) 17 | return (element { $param } { map:get($params, $param) }, xdmp:log(fn:concat($param, ": ", map:get($params, $param))) ) 18 | } 19 | } 20 | ) 21 | 22 | }; -------------------------------------------------------------------------------- /core/src/test/resources/xml/execution-context.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | true 5 | 683181905 6 | 7 | 8 | testInteger 9 | 123 10 | 11 | 12 | testDouble 13 | 123.0 14 | 15 | 16 | testName 17 | testValue 18 | 19 | 20 | testLong 21 | 123 22 | 23 | 24 | -------------------------------------------------------------------------------- /core/src/test/resources/xml/job-execution-2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 2016-05-02T21:24:53.225-04:00 4 | UNKNOWN 5 | exitCode=UNKNOWN;exitDescription= 6 | 6275688569778434925 7 | 8 | 0 9 | 5036934157084150201 10 | job 11 | 12 | 13 | 14 | 2016-05-02T21:24:53.225-04:00 15 | STARTING 16 | 17 | 18 | 5540325527973473631 19 | 0 20 | 0 21 | 874520070930268575 22 | 1791948404679980561 23 | job 24 | 2016-05-02T21:24:53.386-04:00 25 | 0 26 | 0 27 | 7 28 | 3 29 | 2016-05-02T21:24:53.386-04:00 30 | STARTED 31 | foo 32 | false 33 | 0 34 | 5 35 | 36 | 37 | /projects.spring.io/spring-batch/6275688569778434925.xml 38 | 1 39 | -------------------------------------------------------------------------------- /core/src/test/resources/xml/job-instance.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 123 4 | test 5 | 0 6 | -------------------------------------------------------------------------------- /core/src/test/resources/xml/job-parameters.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 2 4 | Joe Cool 5 | 2016-05-05T20:49:24-0400 6 | 1239 7 | 1.35 8 | -------------------------------------------------------------------------------- /core/src/test/resources/xml/step-execution.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 0 4 | 5 | false 6 | 0 7 | 8 | 9 | 0 10 | 123 11 | 1234 12 | test 13 | 2016-05-05T20:49:24.117-04:00 14 | 0 15 | 0 16 | 0 17 | 0 18 | 2016-05-05T20:49:24.117-04:00 19 | STARTING 20 | testStep 21 | false 22 | 0 23 | 0 24 | -------------------------------------------------------------------------------- /file/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "com.jfrog.bintray" 2 | 3 | dependencies { 4 | compile project(":core") 5 | compile "org.springframework.batch:spring-batch-core:$springBatchVersion" 6 | compile "com.marklogic:marklogic-client-api:4.0.3" 7 | compile "org.apache.tika:tika-parsers:1.17" 8 | compile "org.apache.avro:avro:1.8.2" 9 | } 10 | 11 | ext { 12 | artifact = "spring-batch-file" 13 | } 14 | 15 | task sourcesJar(type: Jar, dependsOn: classes) { 16 | baseName = artifact 17 | version = project.version 18 | classifier = 'sources' 19 | from sourceSets.main.allSource 20 | } 21 | 22 | jar { 23 | baseName = artifact 24 | version = project.version 25 | } 26 | 27 | publishing { 28 | publications { 29 | mainJavaWithSources(MavenPublication) { 30 | groupId group 31 | artifactId artifact 32 | version project.version 33 | from components.java 34 | artifact sourcesJar 35 | } 36 | } 37 | } 38 | 39 | bintray { 40 | user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 41 | key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_KEY') 42 | publications = [bintrayPublicationName] 43 | pkg { 44 | repo = bintrayRepo 45 | name = artifact 46 | userOrg = bintrayUserOrg 47 | licenses = [bintrayLicense] 48 | vcsUrl = bintrayVcsUrl 49 | version { 50 | name = project.version 51 | released = new Date() 52 | vcsTag = project.version 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /file/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/file/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /file/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /file/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /file/src/main/java/com/marklogic/spring/batch/item/file/AvroItemReader.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file; 2 | 3 | import org.apache.avro.file.DataFileReader; 4 | import org.apache.avro.generic.GenericDatumReader; 5 | import org.apache.avro.generic.GenericRecord; 6 | import org.apache.avro.io.DatumReader; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.batch.item.ItemReader; 10 | 11 | import java.io.File; 12 | import java.io.IOException; 13 | 14 | public class AvroItemReader implements ItemReader { 15 | 16 | protected final static Logger logger = LoggerFactory.getLogger(AvroItemReader.class); 17 | 18 | private GenericRecord record; 19 | private DataFileReader dataFileReader; 20 | 21 | public AvroItemReader(File inputFile) throws IOException { 22 | DatumReader datumReader = new GenericDatumReader<>(); 23 | this.dataFileReader = new DataFileReader(inputFile, datumReader); 24 | 25 | this.record = null; 26 | } 27 | 28 | @Override 29 | public synchronized String read() throws Exception { 30 | 31 | if (this.dataFileReader.hasNext()) { 32 | this.record = this.dataFileReader.next(this.record); 33 | 34 | logger.info(record.toString()); 35 | 36 | return this.record.toString(); 37 | } else { 38 | return null; 39 | } 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /file/src/main/java/com/marklogic/spring/batch/item/file/TikaParserItemProcessor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file; 2 | 3 | import com.marklogic.client.io.InputSourceHandle; 4 | import com.marklogic.client.io.marker.AbstractWriteHandle; 5 | import com.marklogic.spring.batch.item.file.support.TikaParser; 6 | import com.marklogic.spring.batch.item.processor.AbstractMarkLogicItemProcessor; 7 | import com.marklogic.spring.batch.item.processor.support.UriGenerator; 8 | import org.springframework.core.io.Resource; 9 | import org.xml.sax.InputSource; 10 | 11 | import java.io.StringReader; 12 | 13 | public class TikaParserItemProcessor extends AbstractMarkLogicItemProcessor { 14 | 15 | public TikaParserItemProcessor() { 16 | super(); 17 | } 18 | 19 | public TikaParserItemProcessor(UriGenerator uriGenerator) { 20 | super(uriGenerator); 21 | } 22 | 23 | @Override 24 | public AbstractWriteHandle getContentHandle(Resource item) throws Exception { 25 | String parsedContent = TikaParser.parseToXML(item.getInputStream()); 26 | InputSource inputSource = new InputSource(new StringReader(parsedContent.toString())); 27 | InputSourceHandle handle = new InputSourceHandle(inputSource); 28 | return handle; 29 | } 30 | 31 | 32 | } 33 | -------------------------------------------------------------------------------- /file/src/main/java/com/marklogic/spring/batch/item/file/support/MapFileVisitor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file.support; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.nio.file.*; 9 | import java.nio.file.attribute.BasicFileAttributes; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class MapFileVisitor extends SimpleFileVisitor { 14 | 15 | private final static Logger logger = LoggerFactory.getLogger(MapFileVisitor.class); 16 | 17 | private PathMatcher matcher; 18 | private Map fileMap; 19 | private int numberOfFiles = 0; 20 | private int numberOfMatches = 0; 21 | 22 | public MapFileVisitor() { 23 | this("**.*"); 24 | } 25 | 26 | public MapFileVisitor(String globPattern) { 27 | fileMap = new HashMap(); 28 | matcher = FileSystems.getDefault().getPathMatcher("glob:" + globPattern); 29 | } 30 | 31 | public Map getFileMap() { 32 | return fileMap; 33 | } 34 | 35 | public int getNumberOfMatches() { 36 | return numberOfMatches; 37 | } 38 | 39 | public int getNumberOfFiles() { 40 | return numberOfFiles; 41 | } 42 | 43 | @Override 44 | public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { 45 | numberOfFiles++; 46 | if (match(file)) { 47 | numberOfMatches++; 48 | String fileName = file.getFileName().toString(); 49 | fileMap.put(file.getParent().toString() + File.separator + fileName, file); 50 | logger.debug(fileName); 51 | } 52 | return FileVisitResult.CONTINUE; 53 | } 54 | 55 | // Invoke the pattern matching method on each directory. 56 | @Override 57 | public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { 58 | match(dir); 59 | return FileVisitResult.CONTINUE; 60 | } 61 | 62 | // Compares the glob pattern against the file or directory name. 63 | // https://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob 64 | private boolean match(Path file) { 65 | if (matcher.matches(file)) { 66 | logger.debug("MATCH: " + file.getFileName()); 67 | return true; 68 | } else { 69 | return false; 70 | } 71 | 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /file/src/main/java/com/marklogic/spring/batch/item/file/support/TikaParser.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file.support; 2 | 3 | import org.apache.tika.exception.TikaException; 4 | import org.apache.tika.metadata.Metadata; 5 | import org.apache.tika.parser.AutoDetectParser; 6 | import org.apache.tika.sax.ToXMLContentHandler; 7 | import org.xml.sax.ContentHandler; 8 | import org.xml.sax.SAXException; 9 | 10 | import java.io.IOException; 11 | import java.io.InputStream; 12 | 13 | public class TikaParser { 14 | 15 | public static String parseToXML(InputStream inputStream) throws IOException, SAXException, TikaException { 16 | ContentHandler handler = new ToXMLContentHandler(); 17 | AutoDetectParser parser = new AutoDetectParser(); 18 | Metadata metadata = new Metadata(); 19 | parser.parse(inputStream, handler, metadata); 20 | return handler.toString(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /file/src/test/java/com/marklogic/spring/batch/item/file/AvroItemReaderTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | import org.springframework.core.io.ClassPathResource; 6 | import org.springframework.core.io.Resource; 7 | 8 | import java.io.File; 9 | import java.io.IOException; 10 | import java.nio.file.Files; 11 | import java.util.stream.Stream; 12 | 13 | import static org.junit.Assert.assertEquals; 14 | import static org.junit.Assert.assertNull; 15 | 16 | public class AvroItemReaderTest { 17 | private AvroItemReader itemReader; 18 | private Stream expectedOutputStream; 19 | 20 | @Before 21 | public void setUp() throws Exception { 22 | Resource sampleResource = new ClassPathResource("avro/sampleAvroSerialized.avro"); 23 | File sampleAvroFile = sampleResource.getFile(); 24 | 25 | this.itemReader = new AvroItemReader(sampleAvroFile); 26 | 27 | Resource expectedOutputResource = new ClassPathResource("avro/sampleOutput.txt"); 28 | this.expectedOutputStream = Files.lines(expectedOutputResource.getFile().toPath()); 29 | } 30 | 31 | @Test 32 | public void testRead() throws Exception { 33 | 34 | this.expectedOutputStream.forEach((String s) -> { 35 | try { 36 | assertEquals(itemReader.read(), s); 37 | } catch (Exception e) { 38 | e.printStackTrace(); 39 | } 40 | }); 41 | 42 | assertNull(itemReader.read()); 43 | 44 | } 45 | 46 | @Test(expected = IOException.class) 47 | public void testIOException() throws IOException { 48 | File sampleAvroFile = new File(AvroItemReaderTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(), "DOESNOTEXIST"); 49 | itemReader = new AvroItemReader(sampleAvroFile); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /file/src/test/java/com/marklogic/spring/batch/item/file/EnhancedResourcesItemReaderTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Before; 5 | import org.junit.Rule; 6 | import org.junit.Test; 7 | import org.junit.rules.TemporaryFolder; 8 | import org.springframework.batch.item.ExecutionContext; 9 | import org.springframework.core.io.Resource; 10 | 11 | import java.io.File; 12 | import java.io.IOException; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | /** 17 | * Unit test for verifying we construct the pattern correctly. 18 | */ 19 | public class EnhancedResourcesItemReaderTest extends Assert { 20 | 21 | private EnhancedResourcesItemReader sut = new EnhancedResourcesItemReader(); 22 | 23 | @Rule 24 | public TemporaryFolder file = new TemporaryFolder(); 25 | 26 | @Before 27 | public void before() throws IOException { 28 | file.newFolder("A"); 29 | file.newFolder("B"); 30 | file.newFolder("A", "A1"); 31 | 32 | file.newFile("/A/red.txt"); 33 | file.newFile("/B/blue.jpg"); 34 | file.newFile("/B/yellow.txt"); 35 | file.newFile("/A/A1/green.txt"); 36 | } 37 | 38 | @Test 39 | public void fileDirectory() { 40 | assertEquals("file:src/**", test("src")); 41 | assertEquals("file:src" + File.separator + "/**", test("src" + File.separator)); 42 | } 43 | 44 | @Test 45 | public void specificFile() { 46 | assertEquals("file:build.gradle", test("build.gradle")); 47 | assertEquals("file:./build.gradle", test("./build.gradle")); 48 | } 49 | 50 | @Test 51 | public void classpathReference() { 52 | assertEquals("classpath:org/example", test("classpath:org/example")); 53 | assertEquals("classpath:org/example/", test("classpath:org/example/")); 54 | } 55 | 56 | @Test 57 | public void withFilePrefix() { 58 | assertEquals("If the client includes file:, then a pattern won't be included automatically", 59 | "file:src", test("file:src")); 60 | } 61 | 62 | private String test(String inputFilePath) { 63 | sut.setInputFilePath(inputFilePath); 64 | return sut.buildPattern(); 65 | } 66 | 67 | @Test 68 | public void recursiveTest() { 69 | sut.setInputFilePath(file.getRoot().getAbsolutePath()); 70 | sut.setInputFilePattern(".*\\.txt"); 71 | 72 | ExecutionContext context = new ExecutionContext(); 73 | sut.open(context); 74 | 75 | try { 76 | List resourceList = new ArrayList<>(); 77 | Resource resource = null; 78 | while ( (resource = sut.read()) != null ) { 79 | resourceList.add(resource); 80 | } 81 | 82 | assertEquals("Expecting 3 .txt files", resourceList.size(), 3); 83 | } catch (Exception e) { 84 | fail("Cannot get resource list from EnhancedResourcesItemReader: " + e); 85 | e.printStackTrace(); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /file/src/test/java/com/marklogic/spring/batch/item/file/support/MapFileVisitorTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file.support; 2 | 3 | import org.junit.Before; 4 | import org.junit.Rule; 5 | import org.junit.Test; 6 | import org.junit.rules.TemporaryFolder; 7 | 8 | import java.io.IOException; 9 | import java.nio.file.*; 10 | import java.util.Map; 11 | 12 | import static org.hamcrest.MatcherAssert.assertThat; 13 | 14 | public class MapFileVisitorTest { 15 | 16 | @Rule 17 | public TemporaryFolder file = new TemporaryFolder(); 18 | 19 | @Before 20 | public void before() throws IOException { 21 | file.newFolder("A"); 22 | file.newFolder("B"); 23 | file.newFolder("A", "A1"); 24 | 25 | file.newFile("/A/red.txt"); 26 | file.newFile("/B/blue.jpg"); 27 | file.newFile("/A/A1/green.txt"); 28 | file.newFile("/B/yellow.txt"); 29 | } 30 | 31 | @Test 32 | public void recursiveTraverseFileTreeTest() throws IOException { 33 | Path startingDir = Paths.get(file.getRoot().getAbsolutePath()); 34 | MapFileVisitor fileVisitor = new MapFileVisitor(); 35 | Files.walkFileTree(startingDir, fileVisitor); 36 | Map fileMap = fileVisitor.getFileMap(); 37 | System.out.println("SIZE: " + fileMap.size()); 38 | for (Map.Entry item : fileMap.entrySet()) { 39 | System.out.println(item.getKey()); 40 | } 41 | assertThat("Expecting 4 files", fileVisitor.getNumberOfFiles() == 4); 42 | assertThat("Expecting 4 matched files", fileVisitor.getNumberOfMatches() == 4); 43 | } 44 | 45 | @Test 46 | public void recursiveTraverseFileTreeWithPatternTest() throws IOException { 47 | Path startingDir = Paths.get(file.getRoot().getAbsolutePath()); 48 | MapFileVisitor fileVisitor = new MapFileVisitor("**/**.txt"); 49 | Files.walkFileTree(startingDir, fileVisitor); 50 | Map fileMap = fileVisitor.getFileMap(); 51 | System.out.println("SIZE: " + fileMap.size()); 52 | for (Map.Entry item : fileMap.entrySet()) { 53 | System.out.println(item.getKey()); 54 | } 55 | assertThat("Expecting 3 matched files", fileVisitor.getNumberOfMatches() == 3); 56 | assertThat("Expecting 4 files", fileVisitor.getNumberOfFiles() == 4); 57 | 58 | } 59 | 60 | @Test 61 | public void recursiveTraverseFileTreeWithDirectoryPatternTest() throws IOException { 62 | Path startingDir = Paths.get(file.getRoot().getAbsolutePath()); 63 | MapFileVisitor fileVisitor = new MapFileVisitor("**/A/**.txt"); 64 | Files.walkFileTree(startingDir, fileVisitor); 65 | Map fileMap = fileVisitor.getFileMap(); 66 | System.out.println("SIZE: " + fileMap.size()); 67 | for (Map.Entry item : fileMap.entrySet()) { 68 | System.out.println(item.getKey()); 69 | } 70 | assertThat("Expecting 2 matched files", fileVisitor.getNumberOfMatches() == 2); 71 | assertThat("Expecting 4 files", fileVisitor.getNumberOfFiles() == 4); 72 | 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /file/src/test/java/com/marklogic/spring/batch/item/file/support/TikaParserTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.file.support; 2 | 3 | import org.junit.Test; 4 | import org.springframework.core.io.ClassPathResource; 5 | import org.springframework.core.io.Resource; 6 | 7 | import static org.hamcrest.MatcherAssert.assertThat; 8 | import static org.hamcrest.CoreMatchers.*; 9 | 10 | public class TikaParserTest { 11 | 12 | private Resource wordDocument; 13 | private String parsedXml; 14 | 15 | @Test 16 | public void parseWordDocumentTest() throws Exception { 17 | givenWordDocument("word/test-1.docx"); 18 | whenDocumentIsParsed(); 19 | thenContainsText(); 20 | } 21 | 22 | public void givenWordDocument(String path) { 23 | wordDocument = new ClassPathResource(path); 24 | } 25 | 26 | public void whenDocumentIsParsed() throws Exception { 27 | parsedXml = TikaParser.parseToXML(wordDocument.getInputStream()); 28 | } 29 | 30 | public void thenContainsText() { 31 | assertThat(parsedXml, containsString("The quick brown fox")); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /file/src/test/resources/avro/sampleAvroSerialized.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/file/src/test/resources/avro/sampleAvroSerialized.avro -------------------------------------------------------------------------------- /file/src/test/resources/avro/sampleOutput.txt: -------------------------------------------------------------------------------- 1 | {"id": "A91D021BA58444B29D4D42CA5E39F7BF", "student_id": 100, "university_id": 908, "course_details": {"course_id": 100, "enroll_date": "2012-02-13 00:00:00.000000000", "verb": "completed", "result_score": 0.9}} 2 | {"id": "502A77CC99B241CB94CA356F5218F1A9", "student_id": 101, "university_id": 112, "course_details": {"course_id": 233, "enroll_date": "2011-06-08 00:00:00.000000000", "verb": "started", "result_score": 0.65}} 3 | {"id": "5D04CD5ABF014D6EBA237766F9B470DE", "student_id": 102, "university_id": 340, "course_details": {"course_id": 339, "enroll_date": "2012-03-06 00:00:00.000000000", "verb": "started", "result_score": 0.57}} -------------------------------------------------------------------------------- /file/src/test/resources/word/test-1.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/file/src/test/resources/word/test-1.docx -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | group=com.marklogic 2 | version=1.7.0 3 | 4 | # Subprojects must override this so that publishing works properly 5 | # This is used instead of project.name, which cannot be overridden 6 | artifactId= 7 | 8 | # Setup target database to run test jobs 9 | mlHost=localhost 10 | mlAppName=msb 11 | mlRestPort=8200 12 | mlUsername=admin 13 | mlPassword=admin 14 | 15 | # Specify port for the MarkLogicJobRepository 16 | mlJobRepoPort=8201 17 | 18 | bintrayRepo=maven 19 | bintrayUserOrg=sastafford 20 | bintrayVcsUrl=https://github.com/marklogic-community/marklogic-spring-batch.git 21 | bintrayPublicationName=mainJavaWithSources 22 | bintrayLicense=Apache-2.0 23 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /http/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "com.jfrog.bintray" 2 | 3 | dependencies { 4 | compile "org.springframework:spring-web:$springVersion" 5 | compile "org.springframework.batch:spring-batch-core:$springBatchVersion" 6 | compile "org.jdom:jdom2:2.0.6" 7 | testCompile "org.springframework:spring-test:$springVersion" 8 | compile 'com.google.code.gson:gson:2.8.2' 9 | compile 'org.codehaus.jackson:jackson-core-asl:1.9.13' 10 | compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' 11 | compile 'com.fasterxml.jackson.core:jackson-databind:2.9.2' 12 | } 13 | 14 | ext { 15 | artifact = "spring-batch-http" 16 | } 17 | 18 | task sourcesJar(type: Jar, dependsOn: classes) { 19 | baseName = artifact 20 | version = project.version 21 | classifier = 'sources' 22 | from sourceSets.main.allSource 23 | } 24 | 25 | jar { 26 | baseName = artifact 27 | version = project.version 28 | } 29 | 30 | publishing { 31 | publications { 32 | mainJavaWithSources(MavenPublication) { 33 | groupId group 34 | artifactId artifact 35 | version project.version 36 | from components.java 37 | artifact sourcesJar 38 | } 39 | } 40 | } 41 | 42 | bintray { 43 | user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 44 | key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_KEY') 45 | publications = [bintrayPublicationName] 46 | pkg { 47 | repo = bintrayRepo 48 | name = artifact 49 | userOrg = bintrayUserOrg 50 | licenses = [bintrayLicense] 51 | vcsUrl = bintrayVcsUrl 52 | version { 53 | name = project.version 54 | released = new Date() 55 | vcsTag = project.version 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /http/gradle.properties: -------------------------------------------------------------------------------- 1 | artifactId=marklogic-spring-batch-http -------------------------------------------------------------------------------- /http/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/http/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /http/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /http/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /http/src/main/java/com/marklogic/spring/batch/item/http/HttpXmlItemReader.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.http; 2 | 3 | import org.jdom2.Document; 4 | import org.jdom2.Element; 5 | import org.jdom2.filter.ElementFilter; 6 | import org.jdom2.input.SAXBuilder; 7 | import org.springframework.batch.item.ExecutionContext; 8 | import org.springframework.batch.item.ItemStreamException; 9 | import org.springframework.batch.item.ItemStreamReader; 10 | import org.springframework.http.HttpStatus; 11 | import org.springframework.http.MediaType; 12 | import org.springframework.http.ResponseEntity; 13 | import org.springframework.web.client.RestTemplate; 14 | 15 | import java.io.StringReader; 16 | import java.net.URI; 17 | import java.util.Iterator; 18 | 19 | public class HttpXmlItemReader implements ItemStreamReader { 20 | 21 | private URI uri; 22 | private String aggregateRecordElement; 23 | private RestTemplate restTemplate; 24 | private Iterator aggregateRecordElementItr; 25 | private int count = 0; 26 | 27 | public HttpXmlItemReader(RestTemplate restTemplate, URI uri, String aggregateRecordElement) { 28 | this.restTemplate = restTemplate; 29 | this.uri = uri; 30 | this.aggregateRecordElement = aggregateRecordElement; 31 | } 32 | 33 | @Override 34 | public Document read() throws Exception { 35 | Document doc = null; 36 | if (aggregateRecordElementItr.hasNext()) { 37 | Element elem = aggregateRecordElementItr.next(); 38 | doc = new Document(elem.clone()); 39 | count++; 40 | } 41 | return doc; 42 | } 43 | 44 | @Override 45 | public void open(ExecutionContext executionContext) throws ItemStreamException { 46 | ResponseEntity response = restTemplate.getForEntity(uri, String.class); 47 | if (!response.getStatusCode().equals(HttpStatus.OK)) { 48 | throw new ItemStreamException(response.getStatusCode() + " HTTP response is returned"); 49 | } 50 | ; 51 | MediaType type = response.getHeaders().getContentType(); 52 | if (MediaType.APPLICATION_XML.equals(type) || 53 | MediaType.APPLICATION_ATOM_XML.equals(type) || 54 | MediaType.APPLICATION_XHTML_XML.equals(type) || 55 | MediaType.APPLICATION_RSS_XML.equals(type) || 56 | "text/xml".equals(type.toString())) { 57 | SAXBuilder jdomBuilder = new SAXBuilder(); 58 | try { 59 | Document document = jdomBuilder.build(new StringReader(response.getBody())); 60 | ElementFilter ef = new ElementFilter(aggregateRecordElement); 61 | aggregateRecordElementItr = document.getRootElement().getDescendants(ef); 62 | } catch (Exception ex) { 63 | throw new ItemStreamException(ex); 64 | } 65 | } 66 | } 67 | 68 | @Override 69 | public void update(ExecutionContext executionContext) throws ItemStreamException { 70 | executionContext.putInt("count", count); 71 | } 72 | 73 | @Override 74 | public void close() throws ItemStreamException { 75 | 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /http/src/main/resources/job.properties: -------------------------------------------------------------------------------- 1 | marklogic.host=localhost 2 | marklogic.port=8200 3 | marklogic.username=admin 4 | marklogic.password=admin 5 | marklogic.database=marklogic-spring-batch-test-content 6 | 7 | marklogic.jobrepo.host=localhost 8 | marklogic.jobrepo.port=8201 9 | marklogic.jobrepo.username=admin 10 | marklogic.jobrepo.password=admin 11 | marklogic.jobrepo.database=ml-job-repository 12 | 13 | marklogic.batch.config.enabled=true 14 | 15 | -------------------------------------------------------------------------------- /http/src/test/java/com/marklogic/spring/batch/item/http/HttpJsonItemReaderTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.http; 2 | import com.fasterxml.jackson.databind.JsonNode; 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import org.springframework.batch.item.ExecutionContext; 7 | import org.springframework.http.HttpMethod; 8 | import org.springframework.http.MediaType; 9 | import org.springframework.test.web.client.MockRestServiceServer; 10 | import org.springframework.web.client.RestTemplate; 11 | 12 | import java.io.IOException; 13 | import java.net.URI; 14 | 15 | import static org.hamcrest.CoreMatchers.equalTo; 16 | import static org.junit.Assert.assertThat; 17 | import static org.springframework.test.web.client.ExpectedCount.manyTimes; 18 | import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; 19 | import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; 20 | import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; 21 | 22 | public class HttpJsonItemReaderTest { 23 | RestTemplate mockRestTemplate; 24 | 25 | 26 | String json = "{\n" + 27 | " \"name\" : \"Ronaldo\",\n" + 28 | " \"sport\" : \"soccer\",\n" + 29 | " \"age\" : 25,\n" + 30 | " \"id\" : 121,\n" + 31 | " \"lastScores\" : [ 2, 1, 3, 5, 0, 0, 1, 1 ]" + 32 | "}"; 33 | ObjectMapper mapper = new ObjectMapper(); 34 | 35 | private JsonNode actualObj = mapper.readTree(json); 36 | 37 | public HttpJsonItemReaderTest() throws IOException { 38 | } 39 | 40 | 41 | @Before 42 | public void before() { 43 | 44 | mockRestTemplate = new RestTemplate(); 45 | MockRestServiceServer server = MockRestServiceServer.bindTo(mockRestTemplate).build(); 46 | server.expect(manyTimes(), requestTo("/dummy")).andExpect(method(HttpMethod.GET)). 47 | andRespond(withSuccess(json, MediaType.APPLICATION_JSON)); 48 | } 49 | 50 | @Test 51 | public void getJsonHttpResponseTest() throws Exception { 52 | HttpJsonItemReader itemReader = new HttpJsonItemReader(mockRestTemplate, new URI("/dummy"),actualObj); 53 | itemReader.open(new ExecutionContext()); 54 | 55 | JsonNode jsonData1 = itemReader.read(); 56 | assertThat(jsonData1.asText(), equalTo("Ronaldo")); 57 | 58 | JsonNode jsonData2 = itemReader.read(); 59 | assertThat(jsonData2.asText(), equalTo("soccer")); 60 | 61 | JsonNode jsonData3 = itemReader.read(); 62 | assertThat(jsonData3.asText(), equalTo("25")); 63 | 64 | JsonNode jsonData4 = itemReader.read(); 65 | assertThat(jsonData4.asText(), equalTo("121")); 66 | 67 | JsonNode jsonData5 = itemReader.read(); 68 | assertThat(jsonData5.isArray(), equalTo(true)); 69 | } 70 | 71 | 72 | 73 | 74 | } 75 | -------------------------------------------------------------------------------- /http/src/test/java/com/marklogic/spring/batch/item/http/HttpXmlItemReaderTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.http; 2 | 3 | import org.jdom2.Document; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import org.springframework.batch.item.ExecutionContext; 7 | import org.springframework.http.HttpMethod; 8 | import org.springframework.http.MediaType; 9 | import org.springframework.test.web.client.MockRestServiceServer; 10 | import org.springframework.web.client.RestTemplate; 11 | 12 | import java.net.URI; 13 | 14 | import static org.hamcrest.MatcherAssert.assertThat; 15 | import static org.springframework.test.web.client.ExpectedCount.manyTimes; 16 | import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; 17 | import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; 18 | import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; 19 | 20 | public class HttpXmlItemReaderTest { 21 | 22 | RestTemplate mockRestTemplate; 23 | 24 | private String xml = "abc"; 25 | 26 | @Before 27 | public void before() { 28 | mockRestTemplate = new RestTemplate(); 29 | MockRestServiceServer server = MockRestServiceServer.bindTo(mockRestTemplate).build(); 30 | server.expect(manyTimes(), requestTo("/dummy")).andExpect(method(HttpMethod.GET)) 31 | .andRespond(withSuccess(xml, MediaType.APPLICATION_XML)); 32 | } 33 | 34 | @Test 35 | public void getXmlHttpResponseTest() throws Exception { 36 | HttpXmlItemReader itemReader = new HttpXmlItemReader(mockRestTemplate, new URI("/dummy"), "entry"); 37 | itemReader.open(new ExecutionContext()); 38 | Document doc = itemReader.read(); 39 | assertThat("Expect value of 'a'", doc.getRootElement().getText().equals("a")); 40 | 41 | doc = itemReader.read(); 42 | assertThat("Expect value of 'b'", doc.getRootElement().getText().equals("b")); 43 | 44 | doc = itemReader.read(); 45 | assertThat("Expect value of 'c'", doc.getRootElement().getText().equals("c")); 46 | 47 | doc = itemReader.read(); 48 | assertThat("Expect null", doc == null); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /job.properties: -------------------------------------------------------------------------------- 1 | marklogic.host=localhost 2 | marklogic.port=8200 3 | marklogic.username=admin 4 | marklogic.password=admin 5 | marklogic.database=marklogic-spring-batch-test-content 6 | 7 | marklogic.jobrepo.host=localhost 8 | marklogic.jobrepo.port=8201 9 | marklogic.jobrepo.username=admin 10 | marklogic.jobrepo.password=admin 11 | marklogic.jobrepo.database=mlJobRepo-content 12 | 13 | marklogic.batch.config.enabled=true 14 | -------------------------------------------------------------------------------- /mlJobRepo/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "application" 2 | 3 | task deployMarkLogicJobRepository(type: JavaExec) { 4 | classpath = sourceSets.main.runtimeClasspath 5 | main = 'com.marklogic.spring.batch.DeployMarkLogicJobRepository' 6 | args = ["--name", "mlJobRepo", "--host", mlHost, "--port", mlJobRepoPort, "--username", mlUsername, "--password", mlPassword] 7 | } 8 | 9 | task undeployMarkLogicJobRepository(type: JavaExec) { 10 | classpath = sourceSets.main.runtimeClasspath 11 | main = 'com.marklogic.spring.batch.DeployMarkLogicJobRepository' 12 | args = ["--undeploy", "--name", "mlJobRepo", "--host", mlHost, "--port", mlJobRepoPort, "--username", mlUsername, "--password", mlPassword] 13 | } 14 | 15 | dependencies { 16 | // For deploying the ML Job Repo 17 | compile "com.marklogic:ml-app-deployer:3.4.1" 18 | 19 | // For the CLI 20 | compile "net.sf.jopt-simple:jopt-simple:5.0.1" 21 | } 22 | 23 | mainClassName = "com.marklogic.spring.batch.DeployMarkLogicJobRepository" -------------------------------------------------------------------------------- /mlJobRepo/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/mlJobRepo/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /mlJobRepo/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /mlJobRepo/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /mlJobRepo/src/main/resources/content-database.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection-lexicon": true, 3 | "database-name": "%s", 4 | "maintain-last-modified": true, 5 | "path-namespace": [ 6 | { 7 | "namespace-uri": "http://marklogic.com/spring-batch", 8 | "prefix": "msb" 9 | } 10 | ], 11 | "range-element-index": [ 12 | { 13 | "collation": "", 14 | "invalid-values": "reject", 15 | "localname": "createDateTime", 16 | "namespace-uri": "http://marklogic.com/spring-batch", 17 | "range-value-positions": false, 18 | "scalar-type": "dateTime" 19 | }, 20 | { 21 | "collation": "", 22 | "invalid-values": "reject", 23 | "localname": "endDateTime", 24 | "namespace-uri": "http://marklogic.com/spring-batch", 25 | "range-value-positions": false, 26 | "scalar-type": "dateTime" 27 | }, 28 | { 29 | "collation": "", 30 | "invalid-values": "reject", 31 | "localname": "id", 32 | "namespace-uri": "http://marklogic.com/spring-batch", 33 | "range-value-positions": false, 34 | "scalar-type": "unsignedLong" 35 | }, 36 | { 37 | "collation": "http://marklogic.com/collation/", 38 | "invalid-values": "reject", 39 | "localname": "jobName", 40 | "namespace-uri": "http://marklogic.com/spring-batch", 41 | "range-value-positions": false, 42 | "scalar-type": "string" 43 | } 44 | ], 45 | "triple-index": true, 46 | "uri-lexicon": true 47 | } -------------------------------------------------------------------------------- /mlJobRepo/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /mlJobRepo/src/main/resources/options.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /rdbms/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "com.jfrog.bintray" 2 | 3 | dependencies { 4 | compile "org.springframework:spring-jdbc:$springVersion" 5 | compile "org.springframework.batch:spring-batch-core:$springBatchVersion" 6 | 7 | testCompile "com.h2database:h2:1.4.194" 8 | testCompile "org.springframework:spring-test:$springVersion" 9 | testCompile 'org.hamcrest:hamcrest-library:1.3' 10 | 11 | } 12 | 13 | ext { 14 | artifact = "spring-batch-rdbms" 15 | } 16 | 17 | task sourcesJar(type: Jar, dependsOn: classes) { 18 | baseName = artifact 19 | version = project.version 20 | classifier = 'sources' 21 | from sourceSets.main.allSource 22 | } 23 | 24 | jar { 25 | baseName = artifact 26 | version = project.version 27 | } 28 | 29 | publishing { 30 | publications { 31 | mainJavaWithSources(MavenPublication) { 32 | groupId group 33 | artifactId artifact 34 | version project.version 35 | from components.java 36 | artifact sourcesJar 37 | } 38 | } 39 | } 40 | 41 | bintray { 42 | user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 43 | key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_KEY') 44 | publications = [bintrayPublicationName] 45 | pkg { 46 | repo = bintrayRepo 47 | name = artifact 48 | userOrg = bintrayUserOrg 49 | licenses = [bintrayLicense] 50 | vcsUrl = bintrayVcsUrl 51 | version { 52 | name = project.version 53 | released = new Date() 54 | vcsTag = project.version 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /rdbms/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/rdbms/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /rdbms/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /rdbms/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /rdbms/src/test/java/com/marklogic/spring/batch/item/rdbms/config/H2DatabaseConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.rdbms.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.jdbc.core.JdbcTemplate; 6 | import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase; 7 | import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; 8 | import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; 9 | 10 | import javax.sql.DataSource; 11 | 12 | @Configuration 13 | public class H2DatabaseConfiguration { 14 | 15 | @Bean 16 | public DataSource dataSource() { 17 | // no need shutdown, EmbeddedDatabaseFactoryBean will take care of this 18 | EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); 19 | EmbeddedDatabase dataSource = builder 20 | .setType(EmbeddedDatabaseType.H2) 21 | .addScript("db/sampledata.sql") 22 | .build(); 23 | return dataSource; 24 | } 25 | 26 | @Bean 27 | public JdbcTemplate jdbcTemplate(DataSource dataSource) { 28 | return new JdbcTemplate(dataSource); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /rdbms/src/test/resources/db/sampledata.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE Item IF EXISTS; 2 | DROP TABLE Invoice IF EXISTS; 3 | DROP TABLE Product IF EXISTS; 4 | DROP TABLE Customer IF EXISTS; 5 | 6 | CREATE TABLE Customer(ID INTEGER PRIMARY KEY,FirstName VARCHAR(20),LastName VARCHAR(30),Street VARCHAR(50),City VARCHAR(25)); 7 | CREATE TABLE Product(ID INTEGER PRIMARY KEY,Name VARCHAR(30),Price DECIMAL); 8 | CREATE TABLE Invoice(ID INTEGER PRIMARY KEY,CustomerID INTEGER,Total DECIMAL, FOREIGN KEY (CustomerId) REFERENCES Customer(ID) ON DELETE CASCADE); 9 | CREATE TABLE Item(InvoiceID INTEGER,Item INTEGER,ProductID INTEGER,Quantity INTEGER,Cost DECIMAL,PRIMARY KEY(InvoiceID,Item), FOREIGN KEY (InvoiceId) REFERENCES Invoice (ID) ON DELETE CASCADE, FOREIGN KEY (ProductId) REFERENCES Product(ID) ON DELETE CASCADE); 10 | COMMIT; 11 | 12 | 13 | INSERT INTO Customer VALUES(0,'Laura','Steel','429 Seventh Av.','Dallas'); 14 | INSERT INTO Product VALUES(0,'Iron Iron',54); 15 | INSERT INTO Customer VALUES(1,'Susanne','King','366 - 20th Ave.','Olten'); 16 | INSERT INTO Product VALUES(1,'Chair Shoe',248); 17 | INSERT INTO Customer VALUES(2,'Anne','Miller','20 Upland Pl.','Lyon'); 18 | INSERT INTO Product VALUES(2,'Telephone Clock',248); 19 | INSERT INTO Customer VALUES(3,'Michael','Clancy','542 Upland Pl.','San Francisco'); 20 | INSERT INTO Product VALUES(3,'Chair Chair',254); 21 | INSERT INTO Customer VALUES(4,'Sylvia','Ringer','365 College Av.','Dallas'); 22 | INSERT INTO Product VALUES(4,'Ice Tea Shoe',128); 23 | 24 | INSERT INTO Invoice VALUES(0,0,0.0); 25 | INSERT INTO Invoice VALUES(1,2,0.0); 26 | 27 | INSERT INTO Item VALUES(0,12,1,11,1.5); 28 | 29 | UPDATE Product SET Price=ROUND(Price*.1,2); 30 | UPDATE Item SET Cost=Cost*(SELECT Price FROM Product prod WHERE ProductID=prod.ID); 31 | UPDATE Invoice SET Total=SELECT SUM(Cost*Quantity) FROM Item WHERE InvoiceID=Invoice.ID; 32 | 33 | COMMIT; 34 | -------------------------------------------------------------------------------- /rdf/build.gradle: -------------------------------------------------------------------------------- 1 | //apply plugin: "com.jfrog.bintray" 2 | 3 | dependencies { 4 | compile "org.springframework.batch:spring-batch-core:${springBatchVersion}" 5 | compile('com.marklogic:marklogic-jena:3.0.5') 6 | testCompile project(':test') 7 | } 8 | 9 | ext { 10 | artifact = "spring-batch-rdf" 11 | } 12 | 13 | task sourcesJar(type: Jar, dependsOn: classes) { 14 | baseName = artifact 15 | version = project.version 16 | classifier = 'sources' 17 | from sourceSets.main.allSource 18 | } 19 | 20 | jar { 21 | baseName = artifact 22 | version = project.version 23 | } 24 | 25 | publishing { 26 | publications { 27 | mainJavaWithSources(MavenPublication) { 28 | groupId group 29 | artifactId artifact 30 | version project.version 31 | from components.java 32 | artifact sourcesJar 33 | } 34 | } 35 | } 36 | /* 37 | bintray { 38 | user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 39 | key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_KEY') 40 | publications = [bintrayPublicationName] 41 | pkg { 42 | repo = bintrayRepo 43 | name = artifact 44 | userOrg = bintrayUserOrg 45 | licenses = [bintrayLicense] 46 | vcsUrl = bintrayVcsUrl 47 | version { 48 | name = project.version 49 | released = new Date() 50 | vcsTag = project.version 51 | } 52 | } 53 | } 54 | */ 55 | -------------------------------------------------------------------------------- /rdf/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/rdf/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /rdf/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /rdf/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /rdf/src/main/java/com/marklogic/spring/batch/item/rdf/AbstractDocumentWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.rdf; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.batch.item.ItemStreamSupport; 6 | 7 | import com.marklogic.client.io.DocumentMetadataHandle; 8 | import com.marklogic.client.io.DocumentMetadataHandle.Capability; 9 | 10 | /** 11 | * Base class for writing documents. Should be able to support both the Client API and XCC. 12 | */ 13 | public abstract class AbstractDocumentWriter extends ItemStreamSupport { 14 | 15 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 16 | 17 | private String[] collections; 18 | 19 | // Comma-separated list of role,read,role,update, just like in Client API 20 | private String permissions; 21 | 22 | protected DocumentMetadataHandle buildMetadata() { 23 | DocumentMetadataHandle h = new DocumentMetadataHandle(); 24 | h = h.withCollections(collections); 25 | if (permissions != null) { 26 | String[] array = permissions.split(","); 27 | for (int i = 0; i < array.length; i += 2) { 28 | h.getPermissions().add(array[i], Capability.valueOf(array[i + 1].toUpperCase())); 29 | } 30 | } 31 | return h; 32 | } 33 | 34 | public void setCollections(String[] collections) { 35 | this.collections = collections; 36 | } 37 | 38 | public void setPermissions(String permissions) { 39 | this.permissions = permissions; 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /rdf/src/main/java/com/marklogic/spring/batch/item/rdf/GraphItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.rdf; 2 | 3 | import org.apache.jena.graph.Graph; 4 | 5 | import org.springframework.batch.item.ExecutionContext; 6 | import org.springframework.batch.item.ItemStreamException; 7 | import org.springframework.batch.item.ItemStreamWriter; 8 | 9 | import java.util.List; 10 | 11 | public class GraphItemWriter implements ItemStreamWriter { 12 | 13 | @Override 14 | public void open(ExecutionContext executionContext) throws ItemStreamException { 15 | 16 | } 17 | 18 | @Override 19 | public void update(ExecutionContext executionContext) throws ItemStreamException { 20 | 21 | } 22 | 23 | @Override 24 | public void close() throws ItemStreamException { 25 | 26 | } 27 | 28 | @Override 29 | public void write(List items) throws Exception { 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rdf/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /rdf/src/test/java/com/marklogic/spring/batch/item/rdf/ImportRdfFromFileJob.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.rdf; 2 | 3 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 4 | import org.springframework.batch.core.Job; 5 | import org.springframework.batch.core.Step; 6 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 7 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 8 | import org.springframework.batch.core.configuration.annotation.JobScope; 9 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 10 | import org.springframework.beans.factory.annotation.Qualifier; 11 | import org.springframework.beans.factory.annotation.Value; 12 | import org.springframework.context.annotation.Bean; 13 | import org.springframework.context.annotation.Import; 14 | import org.springframework.context.annotation.PropertySource; 15 | 16 | import java.util.Map; 17 | 18 | @EnableBatchProcessing 19 | @Import(value = { 20 | com.marklogic.spring.batch.config.MarkLogicBatchConfiguration.class, 21 | com.marklogic.spring.batch.config.MarkLogicConfiguration.class}) 22 | @PropertySource("classpath:job.properties") 23 | public class ImportRdfFromFileJob { 24 | 25 | @Bean 26 | public Job job(JobBuilderFactory jobBuilderFactory, @Qualifier("step1") Step step1) { 27 | return jobBuilderFactory.get("importRdfFromFileJob").start(step1).build(); 28 | } 29 | 30 | @Bean 31 | @JobScope 32 | public Step step1( 33 | StepBuilderFactory stepBuilderFactory, 34 | DatabaseClientProvider databaseClientProvider, 35 | @Value("#{jobParameters['input_file_path']}") String inputFilePath, 36 | @Value("#{jobParameters['graph_name']}") String graphName) { 37 | RdfTripleItemReader> reader = new RdfTripleItemReader>(); 38 | reader.setFileName(inputFilePath); 39 | 40 | RdfTripleItemWriter writer = new RdfTripleItemWriter(databaseClientProvider.getDatabaseClient(), graphName); 41 | 42 | return stepBuilderFactory.get("step1") 43 | ., Map>chunk(10) 44 | .reader(reader) 45 | .writer(writer) 46 | .build(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /rdf/src/test/java/com/marklogic/spring/batch/item/rdf/ImportRdfFromFileJobTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.rdf; 2 | 3 | import com.marklogic.junit.Fragment; 4 | import com.marklogic.spring.batch.test.AbstractJobRunnerTest; 5 | import org.junit.Test; 6 | import org.springframework.batch.core.JobParametersBuilder; 7 | import org.springframework.test.context.ContextConfiguration; 8 | 9 | @ContextConfiguration(classes = {ImportRdfFromFileJob.class} ) 10 | public class ImportRdfFromFileJobTest extends AbstractJobRunnerTest { 11 | 12 | @Test 13 | public void test() throws Exception { 14 | JobParametersBuilder jpb = new JobParametersBuilder(); 15 | jpb.addString("input_file_path", "src/test/resources/triple/test1.ttl"); 16 | jpb.addString("graph_name", "myTestGraph"); 17 | getJobLauncherTestUtils().launchJob(jpb.toJobParameters()); 18 | 19 | String xml = getClient().newServerEval().xquery("collection('myTestGraph')").evalAs(String.class); 20 | Fragment f = parse(xml); 21 | f.assertElementExists("/sem:triples/sem:triple[sem:subject = 'http://example.org/kennedy/person1']"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /rdf/src/test/resources/job.properties: -------------------------------------------------------------------------------- 1 | marklogic.host=localhost 2 | marklogic.port=8200 3 | marklogic.username=admin 4 | marklogic.password=admin 5 | marklogic.database=marklogic-spring-batch-test-content 6 | 7 | marklogic.jobrepo.host=localhost 8 | marklogic.jobrepo.port=8201 9 | marklogic.jobrepo.username=admin 10 | marklogic.jobrepo.password=admin 11 | marklogic.jobrepo.database=mlJobRepo-content 12 | 13 | marklogic.batch.config.enabled=true -------------------------------------------------------------------------------- /rdf/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /rdf/src/test/resources/triple/test1.ttl: -------------------------------------------------------------------------------- 1 | @prefix ken: . 2 | ken:person1 ken:spouse ken:person13 . 3 | 4 | -------------------------------------------------------------------------------- /samples/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "application" 2 | 3 | dependencies { 4 | compile project(':core') 5 | 6 | testCompile project(':test') 7 | testCompile "com.marklogic:ml-junit:3.0.0" 8 | 9 | } 10 | 11 | distributions { 12 | main { 13 | baseName = "samples" 14 | } 15 | } 16 | 17 | jar { 18 | baseName = "samples" 19 | } 20 | 21 | mainClassName = "com.marklogic.spring.batch.core.launch.support.CommandLineJobRunner" 22 | 23 | applicationName = "samples" 24 | 25 | task runYourJob(type: JavaExec) { 26 | main = 'com.marklogic.spring.batch.core.launch.support.CommandLineJobRunner' 27 | classpath = sourceSets.test.runtimeClasspath 28 | args = ["--job_path", "YourJobConfig", 29 | "--job_id", "yourJob", 30 | "--output_collections", "yourJob", 31 | "--chunk_size", "50", 32 | "--thread_count", "1", 33 | "--next"] 34 | } 35 | -------------------------------------------------------------------------------- /samples/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/samples/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /samples/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /samples/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /samples/src/test/java/TwoStepJobTest.java: -------------------------------------------------------------------------------- 1 | import com.marklogic.spring.batch.test.JobRunnerContext; 2 | import org.junit.Assert; 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.batch.core.BatchStatus; 6 | import org.springframework.batch.core.JobExecution; 7 | import org.springframework.batch.core.JobParametersBuilder; 8 | import org.springframework.batch.test.JobLauncherTestUtils; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.test.context.ContextConfiguration; 11 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 12 | 13 | @RunWith(SpringJUnit4ClassRunner.class) 14 | @ContextConfiguration(classes = { YourTwoStepJobConfig.class, JobRunnerContext.class } ) 15 | public class TwoStepJobTest { 16 | 17 | @Autowired 18 | JobLauncherTestUtils jobLauncherTestUtils; 19 | 20 | @Test 21 | public void runJobTest() throws Exception { 22 | JobParametersBuilder jpb = new JobParametersBuilder(); 23 | jpb.addString("output_collections", "monster"); 24 | JobExecution jobExecution = jobLauncherTestUtils.launchJob(jpb.toJobParameters()); 25 | Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /samples/src/test/java/YourJobTest.java: -------------------------------------------------------------------------------- 1 | 2 | import com.marklogic.spring.batch.test.AbstractJobRunnerTest; 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | import org.springframework.batch.core.BatchStatus; 6 | import org.springframework.batch.core.JobExecution; 7 | import org.springframework.batch.core.JobParametersBuilder; 8 | import org.springframework.test.context.ContextConfiguration; 9 | 10 | @ContextConfiguration(classes = {YourJobConfig.class} ) 11 | public class YourJobTest extends AbstractJobRunnerTest { 12 | 13 | @Test 14 | public void findOneMonsterInDatabaseTest() throws Exception { 15 | JobParametersBuilder jpb = new JobParametersBuilder(); 16 | jpb.addString("output_collections", "monster"); 17 | JobExecution jobExecution = getJobLauncherTestUtils().launchJob(jpb.toJobParameters()); 18 | Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); 19 | getClientTestHelper().assertCollectionSize("Expecting 100 items in monster collection", "monster", 100); 20 | getJobRepoClientTestHelper().assertCollectionSize("Expecting 5 items in batch collection", "batch", 5); 21 | } 22 | 23 | @Test 24 | public void runJobTwiceTest() throws Exception { 25 | JobParametersBuilder jpb = new JobParametersBuilder(); 26 | jpb.addString("output_collections", "monster", false); 27 | jpb.addLong("run_id", 1L, true); 28 | JobExecution jobExecution = getJobLauncherTestUtils().launchJob(jpb.toJobParameters()); 29 | jpb.addLong("run_id", 2L); 30 | JobExecution jobExecution2 = getJobLauncherTestUtils().launchJob(jpb.toJobParameters()); 31 | Assert.assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); 32 | Assert.assertEquals(BatchStatus.COMPLETED, jobExecution2.getStatus()); 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /samples/src/test/java/YourJobWithNoMarkLogicJobRepoTest.java: -------------------------------------------------------------------------------- 1 | import com.marklogic.spring.batch.test.AbstractJobRunnerTest; 2 | import org.junit.Before; 3 | import org.junit.Test; 4 | import org.springframework.batch.core.BatchStatus; 5 | import org.springframework.batch.core.JobExecution; 6 | import org.springframework.batch.core.JobParametersBuilder; 7 | import org.springframework.test.context.ContextConfiguration; 8 | import org.springframework.test.context.TestPropertySource; 9 | 10 | import static org.hamcrest.CoreMatchers.*; 11 | 12 | @ContextConfiguration(classes = {YourJobConfig.class} ) 13 | @TestPropertySource(properties = { "marklogic.batch.config.enabled=false"}) 14 | public class YourJobWithNoMarkLogicJobRepoTest extends AbstractJobRunnerTest { 15 | 16 | JobExecution jobExecution; 17 | JobParametersBuilder jpb = new JobParametersBuilder(); 18 | 19 | @Test 20 | public void jobRepoIsNullTest() throws Exception { 21 | givenYourJob(); 22 | whenYourJobExecutes(); 23 | thenMonstersExist(); 24 | } 25 | 26 | @Before 27 | public void givenYourJob() { 28 | jpb.addString("output_collections", "monster"); 29 | } 30 | 31 | public void whenYourJobExecutes() throws Exception { 32 | jobExecution = getJobLauncherTestUtils().launchJob(jpb.toJobParameters()); 33 | } 34 | 35 | public void thenMonstersExist() { 36 | assertThat(jobExecution.getStatus(), is(BatchStatus.COMPLETED)); 37 | getClientTestHelper().assertCollectionSize("Expecting 100 items in monster collection", "monster", 100); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /samples/src/test/resources/job.properties: -------------------------------------------------------------------------------- 1 | marklogic.host=localhost 2 | marklogic.port=8200 3 | marklogic.username=admin 4 | marklogic.password=admin 5 | marklogic.database=msb-content 6 | 7 | marklogic.jobrepo.host=localhost 8 | marklogic.jobrepo.port=8201 9 | marklogic.jobrepo.username=admin 10 | marklogic.jobrepo.password=admin 11 | marklogic.jobrepo.database=mlJobRepo-content 12 | 13 | marklogic.batch.config.enabled=true 14 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include "core" 2 | include "file" 3 | include "http" 4 | include "mlJobRepo" 5 | include "rdbms" 6 | include "rdf" 7 | include "samples" 8 | include "shapefiles" 9 | include "test" 10 | -------------------------------------------------------------------------------- /shapefiles/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | compile "org.apache.httpcomponents:httpmime:4.5.2" 3 | compile project(':core') 4 | testCompile project(':test') 5 | } -------------------------------------------------------------------------------- /shapefiles/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/shapefiles/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /shapefiles/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /shapefiles/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /shapefiles/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'infrastructure.shapefiles' 2 | -------------------------------------------------------------------------------- /shapefiles/src/main/java/com/marklogic/spring/batch/item/shapefile/ShapefileProcessor.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.shapefile; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.marklogic.client.io.JacksonHandle; 6 | import com.marklogic.client.io.marker.AbstractWriteHandle; 7 | import com.marklogic.spring.batch.item.processor.AbstractMarkLogicItemProcessor; 8 | import com.marklogic.spring.batch.item.shapefile.support.OgreProxy; 9 | 10 | import java.io.File; 11 | 12 | /** 13 | * Depends on an instance of OgreProxy for extracting GeoJSON from the given File, which is assumed to be a 14 | * valid shapefile. 15 | */ 16 | public class ShapefileProcessor extends AbstractMarkLogicItemProcessor { 17 | 18 | private OgreProxy ogreProxy; 19 | public ShapefileProcessor(OgreProxy ogreProxy) { 20 | this.ogreProxy = ogreProxy; 21 | } 22 | 23 | @Override 24 | public AbstractWriteHandle getContentHandle(File item) throws Exception { 25 | ObjectMapper mapper = new ObjectMapper(); 26 | JsonNode json = mapper.readTree(ogreProxy.extractGeoJson(item)); 27 | return new JacksonHandle(json); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /shapefiles/src/main/java/com/marklogic/spring/batch/item/shapefile/support/HttpClientOgreProxy.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.shapefile.support; 2 | 3 | import org.apache.http.HttpResponse; 4 | import org.apache.http.client.HttpClient; 5 | import org.apache.http.client.methods.HttpPost; 6 | import org.apache.http.entity.mime.MultipartEntityBuilder; 7 | import org.apache.http.impl.client.HttpClientBuilder; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.util.FileCopyUtils; 11 | 12 | import java.io.File; 13 | import java.io.IOException; 14 | 15 | /** 16 | * Using HttpClient, had trouble getting this to work with Spring's RestTemplate. 17 | */ 18 | public class HttpClientOgreProxy implements OgreProxy { 19 | 20 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 21 | 22 | private String url; 23 | private HttpClient httpClient; 24 | 25 | public HttpClientOgreProxy() { 26 | this("http://ogre.adc4gis.com/convert"); 27 | } 28 | 29 | public HttpClientOgreProxy(String url) { 30 | this.url = url; 31 | this.httpClient = HttpClientBuilder.create().build(); 32 | } 33 | 34 | @Override 35 | public String extractGeoJson(File file) throws IOException { 36 | HttpPost post = new HttpPost(url); 37 | MultipartEntityBuilder builder = MultipartEntityBuilder.create(); 38 | builder.addBinaryBody("upload", file); 39 | post.setEntity(builder.build()); 40 | HttpResponse response = httpClient.execute(post); 41 | return new String(FileCopyUtils.copyToByteArray(response.getEntity().getContent())); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /shapefiles/src/main/java/com/marklogic/spring/batch/item/shapefile/support/OgreProxy.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.shapefile.support; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | public interface OgreProxy { 7 | 8 | /** 9 | * @param file requiring a File, as the httpmime library didn't seem to work properly when submitting 10 | * a byte array or an InputStream 11 | * @return 12 | */ 13 | String extractGeoJson(File file) throws IOException; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /shapefiles/src/test/java/com/marklogic/spring/batch/item/shapefile/ShapefileProcessorTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.item.shapefile; 2 | 3 | import com.marklogic.client.document.DocumentWriteOperation; 4 | import com.marklogic.spring.batch.item.shapefile.support.HttpClientOgreProxy; 5 | import com.marklogic.spring.batch.item.shapefile.support.OgreProxy; 6 | import org.junit.Ignore; 7 | import org.junit.Test; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.core.io.ClassPathResource; 11 | 12 | import java.io.File; 13 | 14 | import static org.junit.Assert.assertTrue; 15 | 16 | public class ShapefileProcessorTest { 17 | 18 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 19 | /* 20 | private final static String MOCK_OGRE_RESPONSE = "{\n" + 21 | " \"type\": \"FeatureCollection\",\n" + 22 | " \"features\": [\n" + 23 | " {\n" + 24 | " \"type\": \"Feature\",\n" + 25 | " \"properties\": {\n" + 26 | " \"category\": \"Government and Public Services\",\n" + 27 | " \"name\": \"College:Arab Academy for Science and Technologe and Maritime Transport\",\n" + 28 | " \"ntype\": \"College\"\n" + 29 | " },\n" + 30 | " \"geometry\": {\n" + 31 | " \"type\": \"Point\",\n" + 32 | " \"coordinates\": [\n" + 33 | " 3982629.6882999986,\n" + 34 | " 4236277.941399999\n" + 35 | " ]\n" + 36 | " }\n" + 37 | " }\n" + 38 | " ]\n" + 39 | "}"; 40 | */ 41 | //Todo - mock the httpClientOgreProxy (Requires internet!!) 42 | @Ignore 43 | @Test 44 | public void syriaCollegeShapefileProcessorTest() throws Exception { 45 | File shapefile = new ClassPathResource("./shapefiles/Colleges_Syria.zip").getFile(); 46 | 47 | OgreProxy proxy = new HttpClientOgreProxy(); 48 | ShapefileProcessor processor = new ShapefileProcessor(proxy); 49 | DocumentWriteOperation op = processor.process(shapefile); 50 | String jsonString = op.getContent().toString(); 51 | logger.info(jsonString); 52 | 53 | assertTrue(jsonString.contains("College:Arab Academy for Science and Technologe and Maritime Transport")); 54 | 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /shapefiles/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /shapefiles/src/test/resources/shapefiles/Colleges_Syria.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/shapefiles/src/test/resources/shapefiles/Colleges_Syria.zip -------------------------------------------------------------------------------- /test/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "com.jfrog.bintray" 2 | 3 | ext { 4 | artifact = "marklogic-spring-batch-test" 5 | } 6 | 7 | dependencies { 8 | compile project(':core') 9 | compile "org.springframework.batch:spring-batch-test:$springBatchVersion" 10 | compile "com.marklogic:ml-junit:3.0.0" 11 | 12 | // For detecting version of MarkLogic 13 | compile "com.marklogic:ml-app-deployer:3.2.0" 14 | } 15 | 16 | task sourcesJar(type: Jar, dependsOn: classes) { 17 | baseName = artifact 18 | version = project.version 19 | classifier = 'sources' 20 | from sourceSets.main.allSource 21 | } 22 | 23 | jar { 24 | baseName = artifact 25 | version = project.version 26 | } 27 | 28 | publishing { 29 | publications { 30 | mainJavaWithSources(MavenPublication) { 31 | groupId group 32 | artifactId artifact 33 | version project.version 34 | from components.java 35 | artifact sourcesJar 36 | } 37 | } 38 | } 39 | 40 | bintray { 41 | user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 42 | key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_KEY') 43 | publications = [bintrayPublicationName] 44 | pkg { 45 | repo = bintrayRepo 46 | name = artifact 47 | licenses = [bintrayLicense] 48 | userOrg = bintrayUserOrg 49 | vcsUrl = bintrayVcsUrl 50 | version { 51 | name = project.version 52 | released = new Date() 53 | vcsTag = project.version 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /test/gradle.properties: -------------------------------------------------------------------------------- 1 | artifactId=marklogic-spring-batch-test -------------------------------------------------------------------------------- /test/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marklogic-community/marklogic-spring-batch/9ae06d957312714cf2df40cb8eb523449d3841db/test/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /test/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /test/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /test/src/main/java/com/marklogic/spring/batch/test/AbstractSpringBatchTest.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.test; 2 | 3 | import com.marklogic.client.ext.DatabaseClientConfig; 4 | import com.marklogic.client.ext.helper.DatabaseClientProvider; 5 | import com.marklogic.junit.NamespaceProvider; 6 | import com.marklogic.junit.spring.AbstractSpringTest; 7 | import com.marklogic.spring.batch.config.MarkLogicConfiguration; 8 | import com.marklogic.xcc.template.XccTemplate; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.beans.factory.annotation.Value; 11 | import org.springframework.context.ApplicationContext; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.TestPropertySource; 14 | 15 | /** 16 | * Base class for any "core" test. 17 | */ 18 | @ContextConfiguration(classes = { MarkLogicConfiguration.class }) 19 | @TestPropertySource("classpath:job.properties") 20 | public abstract class AbstractSpringBatchTest extends AbstractSpringTest { 21 | 22 | protected ApplicationContext applicationContext; 23 | protected XccTemplate xccTemplate; 24 | 25 | @Override 26 | protected NamespaceProvider getNamespaceProvider() { 27 | return new SpringBatchNamespaceProvider(); 28 | } 29 | 30 | @Override 31 | protected ApplicationContext getApplicationContext() { 32 | return applicationContext; 33 | } 34 | 35 | @Override 36 | public void setApplicationContext(ApplicationContext applicationContext) { 37 | this.applicationContext = applicationContext; 38 | setDatabaseClientProvider(applicationContext.getBean("databaseClientProvider", DatabaseClientProvider.class)); 39 | setXccTemplate(xccTemplate); 40 | } 41 | 42 | @Autowired 43 | public XccTemplate xccTemplate(DatabaseClientConfig batchDatabaseClientConfig, 44 | @Value("${marklogic.database:Documents}") String databaseName) { 45 | this.xccTemplate = new XccTemplate( 46 | String.format("xcc://%s:%s@%s:%s/%s", 47 | batchDatabaseClientConfig.getUsername(), 48 | batchDatabaseClientConfig.getPassword(), 49 | batchDatabaseClientConfig.getHost(), 50 | batchDatabaseClientConfig.getPort(), 51 | databaseName)); 52 | return xccTemplate; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /test/src/main/java/com/marklogic/spring/batch/test/EmptyItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.test; 2 | 3 | /* 4 | * Copyright 2006-2013 the original author or authors. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | import org.springframework.batch.item.ItemWriter; 22 | import org.springframework.batch.support.transaction.TransactionAwareProxyFactory; 23 | import org.springframework.beans.factory.InitializingBean; 24 | 25 | import java.util.List; 26 | 27 | /** 28 | * Mock {@link ItemWriter} that will throw an exception when a certain number of 29 | * items have been written. 30 | */ 31 | public class EmptyItemWriter implements ItemWriter, InitializingBean { 32 | 33 | protected Logger logger = LoggerFactory.getLogger(EmptyItemWriter.class); 34 | List list; 35 | private boolean failed = false; 36 | // point at which to fail... 37 | private int failurePoint = Integer.MAX_VALUE; 38 | 39 | @Override 40 | public void afterPropertiesSet() throws Exception { 41 | list = TransactionAwareProxyFactory.createTransactionalList(); 42 | } 43 | 44 | public void setFailurePoint(int failurePoint) { 45 | this.failurePoint = failurePoint; 46 | } 47 | 48 | @Override 49 | public void write(List items) { 50 | for (T data : items) { 51 | if (!failed && list.size() == failurePoint) { 52 | failed = true; 53 | throw new RuntimeException("Failed processing: [" + data + "]"); 54 | } 55 | logger.info("Processing: [" + data + "]"); 56 | list.add(data); 57 | } 58 | } 59 | 60 | public List getList() { 61 | return list; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /test/src/main/java/com/marklogic/spring/batch/test/JobRunnerContext.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.test; 2 | 3 | import org.springframework.batch.test.JobLauncherTestUtils; 4 | 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Configuration 9 | public class JobRunnerContext { 10 | 11 | @Bean 12 | public JobLauncherTestUtils jobLauncherTestUtils() { 13 | return new JobLauncherTestUtils(); 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/src/main/java/com/marklogic/spring/batch/test/SpringBatchNamespaceProvider.java: -------------------------------------------------------------------------------- 1 | package com.marklogic.spring.batch.test; 2 | 3 | import com.marklogic.junit.MarkLogicNamespaceProvider; 4 | import com.marklogic.spring.batch.core.MarkLogicSpringBatch; 5 | import org.jdom2.Namespace; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * Registers commonly used namespaces for marklogic-spring-batch tests. 11 | */ 12 | public class SpringBatchNamespaceProvider extends MarkLogicNamespaceProvider { 13 | 14 | @Override 15 | protected List buildListOfNamespaces() { 16 | List list = super.buildListOfNamespaces(); 17 | list.add(Namespace.getNamespace(MarkLogicSpringBatch.JOB_NAMESPACE_PREFIX, MarkLogicSpringBatch.JOB_NAMESPACE)); 18 | list.add(Namespace.getNamespace("geo", "http://geonames.org")); 19 | list.add(Namespace.getNamespace("html", "http://www.w3.org/1999/xhtml")); 20 | list.add(Namespace.getNamespace("xs", "http://www.w3.org/2001/XMLSchema")); 21 | list.add(Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance")); 22 | list.add(Namespace.getNamespace("search", "http://marklogic.com/appservices/search")); 23 | list.add(Namespace.getNamespace("sem", "http://marklogic.com/semantics")); 24 | return list; 25 | } 26 | } 27 | --------------------------------------------------------------------------------