├── .gitignore ├── .mvn ├── jvm.config ├── maven.config └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── LICENSE ├── README.adoc ├── generateApps.sh ├── mvnw ├── mvnw.cmd ├── pom.xml ├── settings.xml ├── spring-cloud-starter-task-jdbchdfs-local ├── README.adoc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── jdbchdfs │ │ │ └── local │ │ │ └── JdbchdfsLocalTaskConfiguration.java │ └── resources │ │ └── META-INF │ │ └── spring-configuration-metadata-whitelist.properties │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── task │ │ └── app │ │ └── jdbc │ │ └── hdfs │ │ └── local │ │ ├── JdbcHdfsDatabaseConfiguration.java │ │ ├── JdbcHdfsIntegrationTest.java │ │ └── JdbcHdfsTaskIntegrationTests.java │ └── resources │ ├── application.properties │ ├── jdbchdfs-data.sql │ └── jdbchdfs-schema.sql ├── spring-cloud-starter-task-spark-client ├── README.adoc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── spark │ │ │ └── client │ │ │ ├── SparkClientTaskConfiguration.java │ │ │ └── SparkClientTaskProperties.java │ └── resources │ │ └── META-INF │ │ └── spring-configuration-metadata-whitelist.properties │ └── test │ └── java │ └── org │ ├── apache │ └── spark │ │ └── examples │ │ └── JavaSparkPi.java │ └── springframework │ └── cloud │ └── task │ └── app │ └── spark │ └── client │ ├── SparkClientRunnerTests.java │ └── SparkClientTaskPropertiesTests.java ├── spring-cloud-starter-task-spark-cluster ├── README.adoc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── spark │ │ │ └── cluster │ │ │ ├── SparkClusterTaskConfiguration.java │ │ │ └── SparkClusterTaskProperties.java │ └── resources │ │ └── META-INF │ │ └── spring-configuration-metadata-whitelist.properties │ └── test │ └── java │ └── org │ └── springframework │ └── cloud │ └── task │ └── app │ └── spark │ └── cluster │ └── SparkClusterTaskPropertiesTests.java ├── spring-cloud-starter-task-spark-yarn ├── README.adoc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── spark │ │ │ └── yarn │ │ │ ├── SparkYarnTaskConfiguration.java │ │ │ └── SparkYarnTaskProperties.java │ └── resources │ │ └── META-INF │ │ └── spring-configuration-metadata-whitelist.properties │ └── test │ └── java │ └── org │ └── springframework │ └── cloud │ └── task │ └── app │ └── spark │ └── yarn │ └── SparkYarnTaskPropertiesTests.java ├── spring-cloud-starter-task-sqoop-job ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── sqoop │ │ │ └── job │ │ │ ├── SqoopJobTaskConfiguration.java │ │ │ └── SqoopJobTaskProperties.java │ └── resources │ │ └── log4j.properties │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── task │ │ └── app │ │ └── sqoop │ │ └── job │ │ ├── SqoopToolJobApplicationTests.java │ │ └── SqoopToolJobPropertiesTests.java │ └── resources │ ├── sqoop-root-data.sql │ ├── sqoop-schema-ddl.sql │ └── sqoop-sessions-data.sql ├── spring-cloud-starter-task-sqoop-tool ├── data │ ├── _SUCCESS │ └── part-m-00000 ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── sqoop │ │ │ └── tool │ │ │ ├── SqoopToolTaskConfiguration.java │ │ │ └── SqoopToolTaskProperties.java │ └── resources │ │ └── log4j.properties │ └── test │ └── java │ └── org │ └── springframework │ └── cloud │ └── task │ └── app │ └── sqoop │ └── tool │ ├── SqoopToolTaskApplicationTests.java │ └── SqoopToolTaskPropertiesTests.java ├── spring-cloud-starter-task-timestamp ├── README.adoc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── task │ │ │ └── app │ │ │ └── timestamp │ │ │ ├── TimestampTaskConfiguration.java │ │ │ └── TimestampTaskProperties.java │ └── resources │ │ └── META-INF │ │ └── spring-configuration-metadata-whitelist.properties │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── task │ │ └── app │ │ └── timestamp │ │ ├── TimestampTaskPropertiesTests.java │ │ └── TimestampTaskTests.java │ └── resources │ └── application.properties ├── spring-cloud-task-app-dependencies └── pom.xml ├── spring-cloud-task-app-descriptor ├── pom.xml └── src │ └── main │ └── resources │ └── META-INF │ ├── task-apps-docker.properties │ └── task-apps-maven.properties ├── spring-cloud-task-app-generator └── pom.xml ├── spring-cloud-task-app-starters-docs ├── pom.xml └── src │ └── main │ ├── asciidoc │ ├── .gitignore │ ├── Guardfile │ ├── appendix.adoc │ ├── building.adoc │ ├── contributing.adoc │ ├── index-docinfo.xml │ ├── index.adoc │ ├── overview.adoc │ └── tasks.adoc │ ├── docbook │ ├── css │ │ ├── highlight.css │ │ ├── manual-multipage.css │ │ ├── manual-singlepage.css │ │ └── manual.css │ ├── images │ │ ├── background.png │ │ ├── caution.png │ │ ├── important.png │ │ ├── logo.png │ │ ├── note.png │ │ ├── tip.png │ │ └── warning.png │ └── xsl │ │ ├── common.xsl │ │ ├── epub.xsl │ │ ├── html-multipage.xsl │ │ ├── html-singlepage.xsl │ │ ├── html.xsl │ │ ├── pdf.xsl │ │ ├── xslthl-config.xml │ │ └── xslthl │ │ ├── asciidoc-hl.xml │ │ ├── bourne-hl.xml │ │ ├── c-hl.xml │ │ ├── cpp-hl.xml │ │ ├── csharp-hl.xml │ │ ├── css-hl.xml │ │ ├── html-hl.xml │ │ ├── ini-hl.xml │ │ ├── java-hl.xml │ │ ├── javascript-hl.xml │ │ ├── json-hl.xml │ │ ├── perl-hl.xml │ │ ├── php-hl.xml │ │ ├── properties-hl.xml │ │ ├── python-hl.xml │ │ ├── ruby-hl.xml │ │ ├── sql2003-hl.xml │ │ └── yaml-hl.xml │ └── javadoc │ └── spring-javadoc.css ├── spring-cloud-task-jdbchdfs-common ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── springframework │ │ └── cloud │ │ └── task │ │ └── jdbchdfs │ │ └── common │ │ ├── HdfsTextItemWriter.java │ │ ├── HdfsTextItemWriterFactory.java │ │ ├── IncrementalColumnRangePartitioner.java │ │ ├── JdbcHdfsConfiguration.java │ │ ├── JdbcHdfsTaskProperties.java │ │ ├── NamedColumnJdbcItemReader.java │ │ ├── NamedColumnJdbcItemReaderFactory.java │ │ └── support │ │ ├── JdbcHdfsDataSourceConfiguration.java │ │ └── JdbcHdfsDataSourceProperties.java │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── task │ │ └── jdbchdfs │ │ └── common │ │ ├── HdfsTextWriterTests.java │ │ ├── IncrementalColumnRangePartitionerTests.java │ │ ├── JdbcHdfsDataSourceConfigurationTests.java │ │ ├── JdbcHdfsDataSourcePropertiesTests.java │ │ ├── JdbcHdfsTaskPropertyTests.java │ │ └── NamedColumnJdbcItemReaderTests.java │ └── resources │ ├── application.properties │ └── schema.sql └── spring-cloud-task-sqoop-common ├── pom.xml └── src ├── main └── java │ └── org │ └── springframework │ └── cloud │ └── task │ └── sqoop │ └── common │ ├── SqoopCommonRunnerUtils.java │ └── SqoopCommonTaskProperties.java └── test ├── java └── org │ └── springframework │ └── cloud │ └── task │ └── sqoop │ └── common │ ├── SqoopCommonTaskPropertiesTests.java │ └── SqoopToolDatabaseConfiguration.java └── resources ├── sqoop-test-data.sql └── sqoop-test-schema-ddl.sql /.gitignore: -------------------------------------------------------------------------------- 1 | apps/ 2 | /application.yml 3 | /application.properties 4 | asciidoctor.css 5 | *~ 6 | .#* 7 | *# 8 | target/ 9 | build/ 10 | bin/ 11 | _site/ 12 | .classpath 13 | .project 14 | .settings 15 | .springBeans 16 | .DS_Store 17 | *.sw* 18 | *.iml 19 | *.ipr 20 | *.iws 21 | .idea/ 22 | .factorypath 23 | spring-xd-samples/*/xd 24 | dump.rdb 25 | coverage-error.log 26 | .apt_generated 27 | -------------------------------------------------------------------------------- /.mvn/jvm.config: -------------------------------------------------------------------------------- 1 | -Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom -------------------------------------------------------------------------------- /.mvn/maven.config: -------------------------------------------------------------------------------- 1 | -DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring 2 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.3/apache-maven-3.3.3-bin.zip -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | # spring-cloud-task-app-starters is no longer actively maintained by VMware, Inc. 2 | 3 | == ANNOUNCEMENT: As of 1.1.0.GA / Addison.GA release, this project is now moved to link:https://github.com/spring-cloud-task-app-starters[spring-cloud-task-app-starters] organization and each of the Spring Cloud Task App Starters is now maintained in a separate repo by itself. 4 | 5 | === This change gives us the flexibility to bug-fix, update, and release individual applications independently as opposed to a time-consuming release of all the applications at once. The link:https://docs.spring.io/spring-cloud-task-app-starters/docs/[reference guide] for all the releases are still available from the same location, though. 6 | 7 | === This project will not be actively tracked going forward. 8 | - For any application specific issues, please find the respective repo under link:https://github.com/spring-cloud-task-app-starters[spring-cloud-task-app-starters] organization and report it there. 9 | - For new feature requests, please use the common link:https://github.com/spring-cloud-task-app-starters/app-starters-release/issues[spring-cloud-task-app-starter/app-starters-release] project backlog. 10 | -------------------------------------------------------------------------------- /generateApps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | cd $DIR 6 | 7 | #execute mvn plugin with app geneartion 8 | mvn clean install scs:generate-app -pl :spring-cloud-task-app-generator 9 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | org.springframework.cloud.task.app 5 | spring-cloud-task-app-starters 6 | 1.0.2.BUILD-SNAPSHOT 7 | pom 8 | 9 | 10 | org.springframework.cloud 11 | spring-cloud-build 12 | 1.1.1.RELEASE 13 | 14 | 15 | 16 | 17 | scm:git:git://github.com/spring-cloud/spring-cloud-task-app-starters.git 18 | scm:git:ssh://git@github.com/spring-cloud/spring-cloud-task-app-starters.git 19 | https://github.com/spring-cloud/spring-cloud-task-app-starters 20 | 21 | 22 | 23 | 1.7 24 | 1.0.2.RELEASE 25 | 1.0.3.RELEASE 26 | 27 | 28 | 29 | spring-cloud-starter-task-timestamp 30 | spring-cloud-starter-task-jdbchdfs-local 31 | spring-cloud-task-jdbchdfs-common 32 | spring-cloud-starter-task-spark-client 33 | spring-cloud-starter-task-spark-cluster 34 | spring-cloud-starter-task-spark-yarn 35 | spring-cloud-starter-task-sqoop-job 36 | spring-cloud-starter-task-sqoop-tool 37 | spring-cloud-task-sqoop-common 38 | spring-cloud-task-app-dependencies 39 | spring-cloud-task-app-descriptor 40 | spring-cloud-task-app-generator 41 | spring-cloud-task-app-starters-docs 42 | 43 | 44 | 45 | 46 | 47 | org.springframework.cloud.task.app 48 | spring-cloud-task-app-dependencies 49 | 1.0.2.BUILD-SNAPSHOT 50 | pom 51 | import 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | maven-javadoc-plugin 61 | 62 | true 63 | 64 | 65 | 66 | maven-surefire-plugin 67 | 68 | true 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | spring 78 | 79 | 80 | 81 | true 82 | 83 | spring-snapshots 84 | Spring Snapshots 85 | https://repo.spring.io/libs-snapshot-local 86 | 87 | 88 | 89 | false 90 | 91 | spring-milestones 92 | Spring Milestones 93 | https://repo.spring.io/libs-milestone-local 94 | 95 | 96 | 97 | false 98 | 99 | spring-releases 100 | Spring Releases 101 | https://repo.spring.io/release 102 | 103 | 104 | 105 | 106 | 107 | true 108 | 109 | spring-snapshots 110 | Spring Snapshots 111 | https://repo.spring.io/libs-snapshot-local 112 | 113 | 114 | 115 | false 116 | 117 | spring-milestones 118 | Spring Milestones 119 | https://repo.spring.io/libs-milestone-local 120 | 121 | 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | repo.spring.io 24 | ${env.CI_DEPLOY_USERNAME} 25 | ${env.CI_DEPLOY_PASSWORD} 26 | 27 | 28 | 29 | 30 | 36 | spring 37 | true 38 | 39 | 40 | spring-snapshots 41 | Spring Snapshots 42 | https://repo.spring.io/libs-snapshot-local 43 | 44 | true 45 | 46 | 47 | 48 | spring-milestones 49 | Spring Milestones 50 | https://repo.spring.io/libs-milestone-local 51 | 52 | false 53 | 54 | 55 | 56 | spring-releases 57 | Spring Releases 58 | https://repo.spring.io/release 59 | 60 | false 61 | 62 | 63 | 64 | 65 | 66 | spring-snapshots 67 | Spring Snapshots 68 | https://repo.spring.io/libs-snapshot-local 69 | 70 | true 71 | 72 | 73 | 74 | spring-milestones 75 | Spring Milestones 76 | https://repo.spring.io/libs-milestone-local 77 | 78 | false 79 | 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/README.adoc: -------------------------------------------------------------------------------- 1 | //tag::ref-doc[] 2 | = JdbcHdfs Task 3 | 4 | This task exports data from a Jdbc table(s) into CSV files on an HDFS using a single batch job. When launching the task, you must 5 | either supply the select statement by setting the sql parameter, or you can supply both tableName and columns options 6 | (which will be used to build the SQL statement). 7 | 8 | == Options 9 | 10 | // see syntax (soon to be automatically generated) in spring-cloud-task starters 11 | The **$$jdbchdfs$$** $$task$$ has the following options: 12 | 13 | //tag::configuration-properties[] 14 | $$jdbchdfs.datasource.password$$:: $$The password of the datasource that will be used by jdbhdfs app to retrieve table input.$$ *($$String$$, default: `$$value of spring.datasource.password$$`)* 15 | $$jdbchdfs.datasource.driverClassName$$:: $$The driver of the datasource that will be used by jdbhdfs app to retrieve table input.$$ *($$String$$, default: `$$value of spring.datasource.driverClassName$$`)* 16 | $$jdbchdfs.datasource.username$$:: $$The username of the datasource that will be used by jdbhdfs app to retrieve table input.$$ *($$String$$, default: `$$value of spring.datasource.username$$`)* 17 | $$jdbchdfs.datasource.url$$:: $$The url of the datasource that will be used by jdbhdfs app to retrieve table input.$$ *($$String$$, default: `$$value of spring.datasource.url$$`)* 18 | $$spring.datasource.password$$:: $$The password of the datasource that will be used by spring cloud task to record task and job information.$$ *($$String$$, default: `$$$$`)* 19 | $$spring.datasource.url$$:: $$The url of the datasource that will be used by spring cloud task to record task and job information.$$ *($$String$$, default: `$$$$`)* 20 | $$spring.datasource.username$$:: $$The username of the datasource that will be used by spring cloud task to record task and job information.$$ *($$String$$, default: `$$$$`)* 21 | $$spring.datasource.driverClassName$$:: $$The driver of the datasource that will be used by spring cloud task to record task and job information.$$ *($$String$$, default: `$$$$`)* 22 | $$jdbchdfs.user-key-tab$$:: $$The user key tab to be used.$$ *($$String$$, default: `$$$$`)* 23 | $$jdbchdfs.column-names$$:: $$The name of the columns to be queried.$$ *($$String$$, default: `$$$$`)* 24 | $$jdbchdfs.fs-uri$$:: $$The URI to the hadoop file system.$$ *($$String$$, default: `$$hdfs://localhost:8020$$`)* 25 | $$jdbchdfs.properties-location$$:: $$The properties location to be used.$$ *($$String$$, default: `$$$$`)* 26 | $$jdbchdfs.name-node-principal$$:: $$The name node principal to be used.$$ *($$String$$, default: `$$$$`)* 27 | $$jdbchdfs.security-method$$:: $$The security method to be used.$$ *($$String$$, default: `$$$$`)* 28 | $$jdbchdfs.file-name$$:: $$The name of the file to be written to the file system.$$ *($$String$$, default: `$$jdbchdfs$$`)* 29 | $$jdbchdfs.commit-interval$$:: $$The commit interval for the application.$$ *($$Integer$$, default: `$$1000$$`)* 30 | $$jdbchdfs.check-column$$:: $$The name of the column used to determine if the data should be read.$$ *($$String$$, default: `$$$$`)* 31 | $$jdbchdfs.register-url-handler $$:: $$The register url handler to be used.$$ *($$String$$, default: `$$$$`)* 32 | $$jdbchdfs.directory$$:: $$The directory the files are to be written.$$ *($$String$$, default: `$$/data$$`)* 33 | $$jdbchdfs.partitions$$:: $$The number of partitions to be created.$$ *($$SInteger$$, default: `$$4$$`)* 34 | $$jdbchdfs.rm-manager-principal $$:: $$The rm manager principal to be used.$$ *($$String$$, default: `$$$$`)* 35 | $$jdbchdfs.file-extension $$:: $$The extension that will be applied to the files.$$ *($$String$$, default: `$$csv$$`)* 36 | $$jdbchdfs.partition-column$$:: $$The name of the column used to partition the data.$$ *($$String$$, default: `$$$$`)* 37 | $$jdbchdfs.user-principal $$:: $$The user principal to be used.$$ *($$String$$, default: `$$$$`)* 38 | $$jdbchdfs.restartable $$:: $$Is the batch job restartable.$$ *($$Boolean$$, default: `$$false$$`)* 39 | $$jdbchdfs.rollover $$:: $$The number of bytes to be written before file rollsover.$$ *($$Long$$, default: `$$1000000000$$`)* 40 | $$jdbchdfs.table-name$$:: $$The name of the table to be queried.$$ *($$String$$, default: `$$$$`)* 41 | $$jdbchdfs.delimiter$$:: $$The delimiter used to split the columns of data in the output file.$$ *($$String$$, default: `$$,$$`)* 42 | $$jdbchdfs.max-workers $$:: $$Maximum number of concurrent workers.$$ *($$Integer$$, default: `$$2$$`)* 43 | $$jdbchdfs.sql$$:: $$Sql to be used to retrieve the data.$$ *($$String$$, default: `$$$$`)* 44 | 45 | //end::configuration-properties[] 46 | 47 | NOTE: If the jdbchdfs.datasource properties are not set the application will use the spring.datasource properties as 48 | the settings for the datasource that retrieves table data. 49 | 50 | //end::ref-doc[] 51 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | spring-cloud-starter-task-jdbchdfs-local 7 | jar 8 | Spring Cloud Starter Task Jdbchdfs Local 9 | Spring Cloud Task Jdbc Hdfs Starter 10 | 11 | 12 | org.springframework.cloud.task.app 13 | spring-cloud-task-app-starters 14 | 1.0.2.BUILD-SNAPSHOT 15 | 16 | 17 | 18 | 1.0.0.BUILD-SNAPSHOT 19 | 20 | 21 | 22 | 23 | 24 | org.springframework.boot 25 | spring-boot-starter-test 26 | test 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-jdbc 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-web 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-configuration-processor 39 | true 40 | 41 | 42 | org.springframework.cloud.task.app 43 | spring-cloud-task-jdbchdfs-common 44 | 1.0.2.BUILD-SNAPSHOT 45 | 46 | 47 | org.springframework.cloud 48 | spring-cloud-deployer-local 49 | ${local.deployer.version} 50 | 51 | 52 | org.springframework.cloud 53 | spring-cloud-task-core 54 | ${spring.cloud.task.core.version} 55 | 56 | 57 | 58 | com.h2database 59 | h2 60 | runtime 61 | 62 | 63 | org.mariadb.jdbc 64 | mariadb-java-client 65 | runtime 66 | 67 | 68 | org.postgresql 69 | postgresql 70 | runtime 71 | 72 | 73 | org.hsqldb 74 | hsqldb 75 | 76 | 77 | 78 | 79 | 80 | integration-tests 81 | 82 | 83 | 84 | maven-surefire-plugin 85 | 86 | true 87 | 88 | 89 | 90 | 91 | 92 | 93 | no-integration-tests 94 | 95 | true 96 | 97 | 98 | 99 | 100 | maven-surefire-plugin 101 | 102 | true 103 | org.springframework.cloud.task.app.jdbc.hdfs.local.JdbcHdfsIntegrationTest 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/main/java/org/springframework/cloud/task/app/jdbchdfs/local/JdbchdfsLocalTaskConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.jdbchdfs.local; 18 | 19 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 20 | import org.springframework.cloud.deployer.spi.local.LocalDeployerProperties; 21 | import org.springframework.cloud.deployer.spi.local.LocalTaskLauncher; 22 | import org.springframework.cloud.deployer.spi.task.TaskLauncher; 23 | import org.springframework.cloud.task.jdbchdfs.common.JdbcHdfsConfiguration; 24 | import org.springframework.cloud.task.jdbchdfs.common.JdbcHdfsTaskProperties; 25 | import org.springframework.context.annotation.Bean; 26 | import org.springframework.context.annotation.Configuration; 27 | import org.springframework.context.annotation.Import; 28 | 29 | /** 30 | * A batch job task that migrates data from a Jdbc repository to a hdfs file. 31 | */ 32 | @Configuration 33 | @EnableConfigurationProperties({JdbcHdfsTaskProperties.class}) 34 | @Import(JdbcHdfsConfiguration.class) 35 | public class JdbchdfsLocalTaskConfiguration { 36 | 37 | @Bean 38 | public TaskLauncher taskLauncher() { 39 | LocalDeployerProperties localDeployerProperties = new LocalDeployerProperties(); 40 | 41 | localDeployerProperties.setDeleteFilesOnExit(false); 42 | 43 | return new LocalTaskLauncher(localDeployerProperties); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | configuration-properties.classes=org.springframework.cloud.task.jdbchdfs.common.JdbcHdfsTaskProperties, \ 18 | org.springframework.cloud.task.jdbchdfs.common.support.JdbcHdfsDataSourceProperties 19 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/test/java/org/springframework/cloud/task/app/jdbc/hdfs/local/JdbcHdfsDatabaseConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.jdbc.hdfs.local; 18 | 19 | import java.io.IOException; 20 | import java.sql.DriverManager; 21 | import java.sql.SQLException; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | import org.hsqldb.Server; 26 | import org.hsqldb.persist.HsqlProperties; 27 | import org.hsqldb.server.ServerAcl; 28 | 29 | import org.springframework.context.annotation.Bean; 30 | import org.springframework.context.annotation.Configuration; 31 | import org.springframework.util.SocketUtils; 32 | 33 | /** 34 | * @author Glenn Renfro 35 | */ 36 | @Configuration 37 | public class JdbcHdfsDatabaseConfiguration { 38 | 39 | private static final Log logger = LogFactory.getLog(JdbcHdfsDatabaseConfiguration.class); 40 | 41 | @Bean(destroyMethod = "stop") 42 | public Server databaseServer() throws SQLException, IOException, ServerAcl.AclFormatException { 43 | DriverManager.registerDriver(new org.hsqldb.jdbcDriver()); 44 | int hsqldbPort = SocketUtils.findAvailableTcpPort(10000); 45 | System.setProperty("db.server.port", Integer.toString(hsqldbPort)); 46 | logger.info("Database is using port: " + Integer.toString(hsqldbPort)); 47 | HsqlProperties configProps = new HsqlProperties(); 48 | configProps.setProperty("server.port", hsqldbPort); 49 | configProps.setProperty("server.database.0", "file:target/db/test"); 50 | configProps.setProperty("server.dbname.0", "test"); 51 | Server server = new Server(); 52 | server.setLogWriter(null); 53 | server.setErrWriter(null); 54 | server.setRestartOnShutdown(false); 55 | server.setNoSystemExit(true); 56 | server.setProperties(configProps); 57 | server.start(); 58 | return server; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/test/java/org/springframework/cloud/task/app/jdbc/hdfs/local/JdbcHdfsIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.jdbc.hdfs.local; 18 | 19 | /** 20 | * The interface that Categorizes a test in the JdbcHdfs Task as an integration test. 21 | * @author Glenn Renfro 22 | */ 23 | public interface JdbcHdfsIntegrationTest { 24 | } 25 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/test/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | logging.level.root=DEBUG 18 | spring.application.name=Demo JdbcHdfs Task 19 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/test/resources/jdbchdfs-data.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO FOO VALUES(1, 'ONE', 'ONEVALUE'); 2 | INSERT INTO FOO VALUES(2, 'TWO', 'TWOVALUE'); 3 | INSERT INTO FOO VALUES(3, 'THREE', 'THREEVALUE'); 4 | INSERT INTO FOO VALUES(4, 'FOUR', 'FOURVALUE'); 5 | INSERT INTO FOO VALUES(5, 'FIVE', 'FIVEVALUE'); 6 | INSERT INTO FOO VALUES(6, 'SIX', 'SIXVALUE'); 7 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-jdbchdfs-local/src/test/resources/jdbchdfs-schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS FOO; 2 | CREATE TABLE FOO (ID smallint NOT NULL, PROPKEY varchar(255), PROPVALUE varchar(255), PRIMARY KEY (ID)); 3 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/README.adoc: -------------------------------------------------------------------------------- 1 | //tag::ref-doc[] 2 | = Spark Client Task 3 | 4 | This task is intended to launch a Spark application. The task submits the Spark application for local execution. This task is appropriate for a local deployment where any local file references can be resolved. It is not meant for any type of cluster deployments. 5 | 6 | == Options 7 | 8 | // see syntax (soon to be automatically generated) in spring-cloud-stream starters 9 | The **$$spark-client$$** $$task$$ has the following options: 10 | 11 | //tag::configuration-properties[] 12 | $$spark.app-args$$:: $$The arguments for the Spark application.$$ *($$String$$, default: `$$[]$$`)* 13 | $$spark.app-class$$:: $$The main class for the Spark application.$$ *($$String$$, default: `$$$$`)* 14 | $$spark.app-jar$$:: $$The path to a bundled jar that includes your application and its dependencies, excluding any Spark dependencies.$$ *($$String$$, default: `$$$$`)* 15 | $$spark.app-name$$:: $$The name to use for the Spark application submission.$$ *($$String$$, default: `$$$$`)* 16 | $$spark.executor-memory$$:: $$The memory setting to be used for each executor.$$ *($$String$$, default: `$$1024M$$`)* 17 | $$spark.master$$:: $$The master setting to be used (local, local[N] or local[*]).$$ *($$String$$, default: `$$local$$`)* 18 | $$spark.resource-archives$$:: $$A comma separated list of archive files to be included with the app submission.$$ *($$String$$, default: `$$$$`)* 19 | $$spark.resource-files $$:: $$A comma separated list of files to be included with the application submission.$$ *($$String$$, default: `$$$$`)* 20 | //end::configuration-properties[] 21 | 22 | //end::ref-doc[] 23 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-starter-task-spark-client 6 | jar 7 | spring-cloud-starter-task-spark-client 8 | Spring Cloud Task Spark Client Starter 9 | 10 | 11 | org.springframework.cloud.task.app 12 | spring-cloud-task-app-starters 13 | 1.0.2.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 1.6.2 18 | 19 | 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-test 28 | test 29 | 30 | 31 | 32 | org.springframework.cloud 33 | spring-cloud-task-core 34 | ${spring.cloud.task.core.version} 35 | 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-starter-jdbc 40 | 41 | 42 | com.h2database 43 | h2 44 | runtime 45 | 46 | 47 | org.mariadb.jdbc 48 | mariadb-java-client 49 | runtime 50 | 51 | 52 | org.postgresql 53 | postgresql 54 | runtime 55 | 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-configuration-processor 60 | true 61 | 62 | 63 | 64 | org.apache.spark 65 | spark-core_2.10 66 | ${spark.version} 67 | 68 | 69 | log4j 70 | log4j 71 | 72 | 73 | org.slf4j 74 | slf4j-log4j12 75 | 76 | 77 | 78 | 79 | com.fasterxml.jackson.core 80 | jackson-core 81 | 2.4.4 82 | 83 | 84 | com.fasterxml.jackson.core 85 | jackson-databind 86 | 2.4.4 87 | 88 | 89 | com.fasterxml.jackson.core 90 | jackson-annotations 91 | 2.4.4 92 | 93 | 94 | org.hibernate 95 | hibernate-validator 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/src/main/java/org/springframework/cloud/task/app/spark/client/SparkClientTaskConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.spark.client; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Arrays; 21 | 22 | import org.apache.commons.logging.Log; 23 | import org.apache.commons.logging.LogFactory; 24 | import org.apache.spark.deploy.SparkSubmit; 25 | 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.boot.CommandLineRunner; 28 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 29 | import org.springframework.cloud.task.configuration.EnableTask; 30 | import org.springframework.context.annotation.Bean; 31 | import org.springframework.context.annotation.Configuration; 32 | import org.springframework.util.StringUtils; 33 | 34 | /** 35 | * {@link CommandLineRunner} implementation that will run a Spark App in client mode using 36 | * configuration properties provided. 37 | * 38 | * @author Thomas Risberg 39 | */ 40 | @EnableTask 41 | @Configuration 42 | @EnableConfigurationProperties(SparkClientTaskProperties.class) 43 | public class SparkClientTaskConfiguration { 44 | 45 | @Bean 46 | public CommandLineRunner commandLineRunner() { 47 | return new SparkAppClientRunner(); 48 | } 49 | 50 | private class SparkAppClientRunner implements CommandLineRunner { 51 | 52 | private final Log logger = LogFactory.getLog(SparkAppClientRunner.class); 53 | 54 | @Autowired 55 | private SparkClientTaskProperties config; 56 | 57 | 58 | @Override 59 | public void run(String... args) throws Exception { 60 | ArrayList argList = new ArrayList<>(); 61 | if (StringUtils.hasText(config.getAppName())) { 62 | argList.add("--name"); 63 | argList.add(config.getAppName()); 64 | } 65 | argList.add("--class"); 66 | argList.add(config.getAppClass()); 67 | argList.add("--master"); 68 | argList.add(config.getMaster()); 69 | argList.add("--deploy-mode"); 70 | argList.add("client"); 71 | 72 | argList.add(config.getAppJar()); 73 | 74 | if (StringUtils.hasText(config.getResourceFiles())) { 75 | argList.add("--files"); 76 | argList.add(config.getResourceFiles()); 77 | } 78 | 79 | if (StringUtils.hasText(config.getResourceArchives())) { 80 | argList.add("--jars"); 81 | argList.add(config.getResourceArchives()); 82 | } 83 | 84 | argList.addAll(Arrays.asList(config.getAppArgs())); 85 | 86 | try { 87 | SparkSubmit.main(argList.toArray(new String[argList.size()])); 88 | } catch (Throwable t) { 89 | logger.error("Spark Application failed: " + t.getMessage(), t); 90 | throw new RuntimeException("Spark Application failed", t); 91 | } 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/src/main/java/org/springframework/cloud/task/app/spark/client/SparkClientTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.spark.client; 18 | 19 | import javax.validation.constraints.NotNull; 20 | 21 | import org.springframework.beans.factory.annotation.Value; 22 | import org.springframework.boot.context.properties.ConfigurationProperties; 23 | 24 | /** 25 | * Configuration properties to be used for cluster submission. These are in addition to the ones 26 | * defined in the common one. 27 | * 28 | * @author Thomas Risberg 29 | */ 30 | @ConfigurationProperties("spark") 31 | public class SparkClientTaskProperties { 32 | 33 | /** 34 | * The master setting to be used (local, local[N] or local[*]). 35 | */ 36 | private String master = "local"; 37 | 38 | /** 39 | * The name to use for the Spark application submission. 40 | */ 41 | @Value("${spring.application.name:sparkapp-task}") 42 | private String appName; 43 | 44 | /** 45 | * The main class for the Spark application. 46 | */ 47 | private String appClass; 48 | 49 | /** 50 | * The path to a bundled jar that includes your application and its dependencies, excluding any Spark dependencies. 51 | */ 52 | private String appJar; 53 | 54 | /** 55 | * The arguments for the Spark application. 56 | */ 57 | private String[] appArgs = new String[]{}; 58 | 59 | /** 60 | * A comma separated list of files to be included with the application submission. 61 | */ 62 | private String resourceFiles; 63 | 64 | /** 65 | * A comma separated list of archive files to be included with the app submission. 66 | */ 67 | private String resourceArchives; 68 | 69 | /** 70 | * The memory setting to be used for each executor. 71 | */ 72 | private String executorMemory = "1024M"; 73 | 74 | public String getMaster() { 75 | return master; 76 | } 77 | 78 | public void setMaster(String master) { 79 | this.master = master; 80 | } 81 | 82 | public String getAppName() { 83 | return appName; 84 | } 85 | 86 | public void setAppName(String appName) { 87 | this.appName = appName; 88 | } 89 | 90 | @NotNull 91 | public String getAppClass() { 92 | return appClass; 93 | } 94 | 95 | public void setAppClass(String appClass) { 96 | this.appClass = appClass; 97 | } 98 | 99 | @NotNull 100 | public String getAppJar() { 101 | return appJar; 102 | } 103 | 104 | public void setAppJar(String appJar) { 105 | this.appJar = appJar; 106 | } 107 | 108 | public String[] getAppArgs() { 109 | return appArgs; 110 | } 111 | 112 | public void setAppArgs(String[] appArgs) { 113 | this.appArgs = appArgs; 114 | } 115 | 116 | public String getResourceFiles() { 117 | return resourceFiles; 118 | } 119 | 120 | public void setResourceFiles(String resourceFiles) { 121 | this.resourceFiles = resourceFiles; 122 | } 123 | 124 | public String getResourceArchives() { 125 | return resourceArchives; 126 | } 127 | 128 | public void setResourceArchives(String resourceArchives) { 129 | this.resourceArchives = resourceArchives; 130 | } 131 | 132 | public String getExecutorMemory() { 133 | return executorMemory; 134 | } 135 | 136 | public void setExecutorMemory(String executorMemory) { 137 | this.executorMemory = executorMemory; 138 | } 139 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | configuration-properties.classes=org.springframework.cloud.task.app.spark.client.SparkClientTaskProperties, \ 18 | org.springframework.cloud.task.sparkapp.common.SparkAppCommonTaskProperties 19 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/src/test/java/org/apache/spark/examples/JavaSparkPi.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.spark.examples; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import org.apache.spark.SparkConf; 24 | import org.apache.spark.api.java.JavaRDD; 25 | import org.apache.spark.api.java.JavaSparkContext; 26 | import org.apache.spark.api.java.function.Function; 27 | import org.apache.spark.api.java.function.Function2; 28 | 29 | /* 30 | * NOTE: Copied from the Apache Spark (https://spark.apache.org/) source code since 31 | * this class is not available in the Maven Central repo. 32 | */ 33 | 34 | /** 35 | * Computes an approximation to pi 36 | * Usage: JavaSparkPi [slices] 37 | */ 38 | public final class JavaSparkPi { 39 | 40 | public static void main(String[] args) throws Exception { 41 | SparkConf sparkConf = new SparkConf().setAppName("JavaSparkPi"); 42 | JavaSparkContext jsc = new JavaSparkContext(sparkConf); 43 | 44 | int slices = (args.length == 1) ? Integer.parseInt(args[0]) : 2; 45 | int n = 100000 * slices; 46 | List l = new ArrayList(n); 47 | for (int i = 0; i < n; i++) { 48 | l.add(i); 49 | } 50 | 51 | JavaRDD dataSet = jsc.parallelize(l, slices); 52 | 53 | int count = dataSet.map(new Function() { 54 | @Override 55 | public Integer call(Integer integer) { 56 | double x = Math.random() * 2 - 1; 57 | double y = Math.random() * 2 - 1; 58 | return (x * x + y * y < 1) ? 1 : 0; 59 | } 60 | }).reduce(new Function2() { 61 | @Override 62 | public Integer call(Integer integer, Integer integer2) { 63 | return integer + integer2; 64 | } 65 | }); 66 | 67 | System.out.println("Pi is roughly " + 4.0 * count / n); 68 | 69 | jsc.stop(); 70 | } 71 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-client/src/test/java/org/springframework/cloud/task/app/spark/client/SparkClientRunnerTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.spark.client; 18 | 19 | import org.junit.Ignore; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.springframework.boot.SpringApplication; 23 | import org.springframework.boot.autoconfigure.SpringBootApplication; 24 | import org.springframework.boot.test.IntegrationTest; 25 | import org.springframework.boot.test.SpringApplicationConfiguration; 26 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 27 | 28 | /** 29 | * @author Thomas Risberg 30 | * @author Soby Chacko 31 | */ 32 | @RunWith(SpringJUnit4ClassRunner.class) 33 | @SpringApplicationConfiguration(classes=SparkClientRunnerTests.TestSparkClientTaskApplication.class) 34 | @IntegrationTest({"spark.app-name:pi", 35 | "spark.app-jar:dummy.jar", 36 | "spark.app-class:org.apache.spark.examples.JavaSparkPi", 37 | "spark.app-args:10"}) 38 | @Ignore 39 | public class SparkClientRunnerTests { 40 | 41 | @Test 42 | public void testExampleSparkApp() throws Exception { 43 | } 44 | 45 | @SpringBootApplication 46 | public static class TestSparkClientTaskApplication { 47 | public static void main(String[] args) { 48 | SpringApplication.run(TestSparkClientTaskApplication.class, args); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-cluster/README.adoc: -------------------------------------------------------------------------------- 1 | //tag::ref-doc[] 2 | = Spark Cluster Task 3 | 4 | This task is intended to launch a Spark application. The task submits the Spark application for execution in a Spark cluster. This task is appropriate for a deployments where any file references can be resolved to a shared location. 5 | 6 | == Options 7 | 8 | // see syntax (soon to be automatically generated) in spring-cloud-stream starters 9 | The **$$spark-cluster$$** $$task$$ has the following options: 10 | 11 | //tag::configuration-properties[] 12 | $$spark.app-args$$:: $$The arguments for the Spark application.$$ *($$String$$, default: `$$[]$$`)* 13 | $$spark.app-class$$:: $$The main class for the Spark application.$$ *($$String$$, default: `$$$$`)* 14 | $$spark.app-jar$$:: $$The path to a bundled jar that includes your application and its dependencies, excluding any Spark dependencies.$$ *($$String$$, default: `$$$$`)* 15 | $$spark.app-name$$:: $$The name to use for the Spark application submission.$$ *($$String$$, default: `$$$$`)* 16 | $$spark.executor-memory$$:: $$The memory setting to be used for each executor.$$ *($$String$$, default: `$$1024M$$`)* 17 | $$spark.master$$:: $$The master setting to be used (spark://host:port).$$ *($$String$$, default: `$$spark://localhost:7077$$`)* 18 | $$spark.resource-archives$$:: $$A comma separated list of archive files to be included with the app submission.$$ *($$String$$, default: `$$$$`)* 19 | $$spark.resource-files $$:: $$A comma separated list of files to be included with the application submission.$$ *($$String$$, default: `$$$$`)* 20 | $$spark.rest-url$$:: $$The URL for the Spark REST API to be used (spark://host:port).$$ *($$String$$, default: `$$spark://localhost:6066$$`)* 21 | $$spark.app-status-poll-interval$$:: $$The interval (ms) to use for polling for the App status.$$ *($$long$$, default: `$$1000L$$`)* 22 | //end::configuration-properties[] 23 | 24 | //end::ref-doc[] 25 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-cluster/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-starter-task-spark-cluster 6 | jar 7 | spring-cloud-starter-task-spark-cluster 8 | Spring Cloud Task Spark Cluster Starter 9 | 10 | 11 | org.springframework.cloud.task.app 12 | spring-cloud-task-app-starters 13 | 1.0.2.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 1.6.2 18 | 19 | 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-test 28 | test 29 | 30 | 31 | 32 | org.springframework.cloud 33 | spring-cloud-task-core 34 | ${spring.cloud.task.core.version} 35 | 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-starter-jdbc 40 | 41 | 42 | com.h2database 43 | h2 44 | runtime 45 | 46 | 47 | org.mariadb.jdbc 48 | mariadb-java-client 49 | runtime 50 | 51 | 52 | org.postgresql 53 | postgresql 54 | runtime 55 | 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-configuration-processor 60 | true 61 | 62 | 63 | 64 | org.apache.spark 65 | spark-core_2.10 66 | ${spark.version} 67 | 68 | 69 | log4j 70 | log4j 71 | 72 | 73 | org.slf4j 74 | slf4j-log4j12 75 | 76 | 77 | 78 | 79 | com.fasterxml.jackson.core 80 | jackson-core 81 | 2.4.4 82 | 83 | 84 | com.fasterxml.jackson.core 85 | jackson-databind 86 | 2.4.4 87 | 88 | 89 | com.fasterxml.jackson.core 90 | jackson-annotations 91 | 2.4.4 92 | 93 | 94 | org.hibernate 95 | hibernate-validator 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-cluster/src/main/java/org/springframework/cloud/task/app/spark/cluster/SparkClusterTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.spark.cluster; 18 | 19 | import javax.validation.constraints.NotNull; 20 | 21 | import org.springframework.beans.factory.annotation.Value; 22 | import org.springframework.boot.context.properties.ConfigurationProperties; 23 | 24 | /** 25 | * Configuration properties to be used for cluster submission. These are in addition to the ones 26 | * defined in the common one. 27 | * 28 | * @author Thomas Risberg 29 | */ 30 | @ConfigurationProperties("spark") 31 | public class SparkClusterTaskProperties { 32 | 33 | /** 34 | * The master setting to be used (spark://host:port). 35 | */ 36 | private String master = "spark://localhost:7077"; 37 | 38 | /** 39 | * The URL for the Spark REST API to be used (spark://host:port). 40 | */ 41 | private String restUrl = "spark://localhost:6066"; 42 | 43 | /** 44 | * The interval (ms) to use for polling for the App status. 45 | */ 46 | private long appStatusPollInterval = 10000L; 47 | 48 | /** 49 | * The name to use for the Spark application submission. 50 | */ 51 | @Value("${spring.application.name:sparkapp-task}") 52 | private String appName; 53 | 54 | /** 55 | * The main class for the Spark application. 56 | */ 57 | private String appClass; 58 | 59 | /** 60 | * The path to a bundled jar that includes your application and its dependencies, excluding any Spark dependencies. 61 | */ 62 | private String appJar; 63 | 64 | /** 65 | * The arguments for the Spark application. 66 | */ 67 | private String[] appArgs = new String[]{}; 68 | 69 | /** 70 | * A comma separated list of files to be included with the application submission. 71 | */ 72 | private String resourceFiles; 73 | 74 | /** 75 | * A comma separated list of archive files to be included with the app submission. 76 | */ 77 | private String resourceArchives; 78 | 79 | /** 80 | * The memory setting to be used for each executor. 81 | */ 82 | private String executorMemory = "1024M"; 83 | 84 | public String getMaster() { 85 | return master; 86 | } 87 | 88 | public void setMaster(String master) { 89 | this.master = master; 90 | } 91 | 92 | public String getRestUrl() { 93 | return restUrl; 94 | } 95 | 96 | public void setRestUrl(String restUrl) { 97 | this.restUrl = restUrl; 98 | } 99 | 100 | public long getAppStatusPollInterval() { 101 | return appStatusPollInterval; 102 | } 103 | 104 | public void setAppStatusPollInterval(long appStatusPollInterval) { 105 | this.appStatusPollInterval = appStatusPollInterval; 106 | } 107 | 108 | public String getAppName() { 109 | return appName; 110 | } 111 | 112 | public void setAppName(String appName) { 113 | this.appName = appName; 114 | } 115 | 116 | @NotNull 117 | public String getAppClass() { 118 | return appClass; 119 | } 120 | 121 | public void setAppClass(String appClass) { 122 | this.appClass = appClass; 123 | } 124 | 125 | @NotNull 126 | public String getAppJar() { 127 | return appJar; 128 | } 129 | 130 | public void setAppJar(String appJar) { 131 | this.appJar = appJar; 132 | } 133 | 134 | public String[] getAppArgs() { 135 | return appArgs; 136 | } 137 | 138 | public void setAppArgs(String[] appArgs) { 139 | this.appArgs = appArgs; 140 | } 141 | 142 | public String getResourceFiles() { 143 | return resourceFiles; 144 | } 145 | 146 | public void setResourceFiles(String resourceFiles) { 147 | this.resourceFiles = resourceFiles; 148 | } 149 | 150 | public String getResourceArchives() { 151 | return resourceArchives; 152 | } 153 | 154 | public void setResourceArchives(String resourceArchives) { 155 | this.resourceArchives = resourceArchives; 156 | } 157 | 158 | public String getExecutorMemory() { 159 | return executorMemory; 160 | } 161 | 162 | public void setExecutorMemory(String executorMemory) { 163 | this.executorMemory = executorMemory; 164 | } 165 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-cluster/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | configuration-properties.classes=org.springframework.cloud.task.app.spark.cluster.SparkClusterTaskProperties, \ 18 | org.springframework.cloud.task.sparkapp.common.SparkAppCommonTaskProperties 19 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-yarn/README.adoc: -------------------------------------------------------------------------------- 1 | //tag::ref-doc[] 2 | = Spark YARN Task 3 | 4 | This task is intended to launch a Spark application. The task submits the Spark application to a YARN cluster 5 | for execution. This task is appropriate for a deployment that has access to a Hadoop YARN cluster. The Spark application jar and the Spark Assembly jar should be referenced from an HDFS location. 6 | 7 | == Options 8 | 9 | // see syntax (soon to be automatically generated) in spring-cloud-stream starters 10 | The **$$spark-yarn$$** $$task$$ has the following options: 11 | 12 | //tag::configuration-properties[] 13 | $$spark.app-args$$:: $$The arguments for the Spark application.$$ *($$String$$, default: `$$[]$$`)* 14 | $$spark.app-class$$:: $$The main class for the Spark application.$$ *($$String$$, default: `$$$$`)* 15 | $$spark.app-jar$$:: $$The path to a bundled jar that includes your application and its dependencies, excluding any Spark dependencies.$$ *($$String$$, default: `$$$$`)* 16 | $$spark.app-name$$:: $$The name to use for the Spark application submission.$$ *($$String$$, default: `$$$$`)* 17 | $$spark.assembly-jar$$:: $$The path for the Spark Assembly jar to use.$$ *($$String$$, default: `$$$$`)* 18 | $$spark.executor-memory$$:: $$The memory setting to be used for each executor.$$ *($$String$$, default: `$$1024M$$`)* 19 | $$spark.num-executors$$:: $$The number of executors to use.$$ *($$Integer$$, default: `$$1$$`)* 20 | $$spark.resource-archives$$:: $$A comma separated list of archive files to be included with the app submission.$$ *($$String$$, default: `$$$$`)* 21 | $$spark.resource-files $$:: $$A comma separated list of files to be included with the application submission.$$ *($$String$$, default: `$$$$`)* 22 | //end::configuration-properties[] 23 | 24 | //end::ref-doc[] 25 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-yarn/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-starter-task-spark-yarn 6 | jar 7 | spring-cloud-starter-task-spark-yarn 8 | Spring Cloud Task Spark Yarn Starter 9 | 10 | 11 | org.springframework.cloud.task.app 12 | spring-cloud-task-app-starters 13 | 1.0.2.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 1.6.2 18 | 19 | 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-test 28 | test 29 | 30 | 31 | 32 | org.springframework.cloud 33 | spring-cloud-task-core 34 | ${spring.cloud.task.core.version} 35 | 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-starter-jdbc 40 | 41 | 42 | com.h2database 43 | h2 44 | runtime 45 | 46 | 47 | org.mariadb.jdbc 48 | mariadb-java-client 49 | runtime 50 | 51 | 52 | org.postgresql 53 | postgresql 54 | runtime 55 | 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-configuration-processor 60 | true 61 | 62 | 63 | 64 | org.apache.spark 65 | spark-yarn_2.10 66 | ${spark.version} 67 | 68 | 69 | org.apache.hadoop 70 | hadoop-client 71 | 72 | 73 | org.apache.hadoop 74 | hadoop-yarn-common 75 | 76 | 77 | org.apache.hadoop 78 | hadoop-yarn-client 79 | 80 | 81 | org.apache.hadoop 82 | hadoop-mapreduce-client-app 83 | 84 | 85 | org.apache.hadoop 86 | hadoop-yarn-server-web-proxy 87 | 88 | 89 | org.apache.hadoop 90 | hadoop-yarn-api 91 | 92 | 93 | log4j 94 | log4j 95 | 96 | 97 | org.slf4j 98 | slf4j-log4j12 99 | 100 | 101 | 102 | 103 | org.springframework.data 104 | spring-data-hadoop-boot 105 | 106 | 107 | commons-logging 108 | commons-logging 109 | 110 | 111 | 112 | 113 | org.hibernate 114 | hibernate-validator 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-yarn/src/main/java/org/springframework/cloud/task/app/spark/yarn/SparkYarnTaskConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.spark.yarn; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Arrays; 21 | import java.util.List; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | import org.apache.hadoop.conf.Configuration; 26 | import org.apache.spark.SparkConf; 27 | import org.apache.spark.deploy.yarn.Client; 28 | import org.apache.spark.deploy.yarn.ClientArguments; 29 | 30 | import org.springframework.beans.factory.annotation.Autowired; 31 | import org.springframework.boot.CommandLineRunner; 32 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 33 | import org.springframework.cloud.task.configuration.EnableTask; 34 | import org.springframework.context.annotation.Bean; 35 | import org.springframework.util.StringUtils; 36 | 37 | /** 38 | * {@link CommandLineRunner} implementation that will run a Spark App in YARN mode using 39 | * configuration properties provided. 40 | * 41 | * @author Thomas Risberg 42 | */ 43 | @EnableTask 44 | @org.springframework.context.annotation.Configuration 45 | @EnableConfigurationProperties(SparkYarnTaskProperties.class) 46 | public class SparkYarnTaskConfiguration { 47 | 48 | @Bean 49 | public CommandLineRunner commandLineRunner() { 50 | return new SparkAppYarnRunner(); 51 | } 52 | 53 | private class SparkAppYarnRunner implements CommandLineRunner { 54 | 55 | private final Log logger = LogFactory.getLog(SparkAppYarnRunner.class); 56 | 57 | @Autowired 58 | private Configuration hadoopConfiguration; 59 | 60 | @Autowired 61 | private SparkYarnTaskProperties config; 62 | 63 | @Override 64 | public void run(String... args) throws Exception { 65 | SparkConf sparkConf = new SparkConf(); 66 | sparkConf.set("spark.yarn.jar", config.getAssemblyJar()); 67 | 68 | List submitArgs = new ArrayList(); 69 | if (StringUtils.hasText(config.getAppName())) { 70 | submitArgs.add("--name"); 71 | submitArgs.add(config.getAppName()); 72 | } 73 | submitArgs.add("--jar"); 74 | submitArgs.add(config.getAppJar()); 75 | submitArgs.add("--class"); 76 | submitArgs.add(config.getAppClass()); 77 | if (StringUtils.hasText(config.getResourceFiles())) { 78 | submitArgs.add("--files"); 79 | submitArgs.add(config.getResourceFiles()); 80 | } 81 | if (StringUtils.hasText(config.getResourceArchives())) { 82 | submitArgs.add("--archives"); 83 | submitArgs.add(config.getResourceArchives()); 84 | } 85 | submitArgs.add("--executor-memory"); 86 | submitArgs.add(config.getExecutorMemory()); 87 | submitArgs.add("--num-executors"); 88 | submitArgs.add("" + config.getNumExecutors()); 89 | for (String arg : config.getAppArgs()) { 90 | submitArgs.add("--arg"); 91 | submitArgs.add(arg); 92 | } 93 | logger.info("Submit App with args: " + Arrays.asList(submitArgs)); 94 | ClientArguments clientArguments = 95 | new ClientArguments(submitArgs.toArray(new String[submitArgs.size()]), sparkConf); 96 | clientArguments.isClusterMode(); 97 | Client client = new Client(clientArguments, hadoopConfiguration, sparkConf); 98 | System.setProperty("SPARK_YARN_MODE", "true"); 99 | try { 100 | client.run(); 101 | } catch (Throwable t) { 102 | logger.error("Spark Application failed: " + t.getMessage(), t); 103 | throw new RuntimeException("Spark Application failed", t); 104 | } 105 | } 106 | } 107 | 108 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-yarn/src/main/java/org/springframework/cloud/task/app/spark/yarn/SparkYarnTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.spark.yarn; 18 | 19 | import javax.validation.constraints.NotNull; 20 | 21 | import org.springframework.beans.factory.annotation.Value; 22 | import org.springframework.boot.context.properties.ConfigurationProperties; 23 | 24 | /** 25 | * Configuration properties to be used for YARN submission. These are in addition to the ones 26 | * defined in the common one. 27 | * 28 | * @author Thomas Risberg 29 | */ 30 | @ConfigurationProperties("spark") 31 | public class SparkYarnTaskProperties { 32 | 33 | /** 34 | * The path for the Spark Assembly jar to use. 35 | */ 36 | private String assemblyJar; 37 | 38 | /** 39 | * The number of executors to use. 40 | */ 41 | private int numExecutors = 1; 42 | 43 | /** 44 | * The name to use for the Spark application submission. 45 | */ 46 | @Value("${spring.application.name:sparkapp-task}") 47 | private String appName; 48 | 49 | /** 50 | * The main class for the Spark application. 51 | */ 52 | private String appClass; 53 | 54 | /** 55 | * The path to a bundled jar that includes your application and its dependencies, excluding any Spark dependencies. 56 | */ 57 | private String appJar; 58 | 59 | /** 60 | * The arguments for the Spark application. 61 | */ 62 | private String[] appArgs = new String[]{}; 63 | 64 | /** 65 | * A comma separated list of files to be included with the application submission. 66 | */ 67 | private String resourceFiles; 68 | 69 | /** 70 | * A comma separated list of archive files to be included with the app submission. 71 | */ 72 | private String resourceArchives; 73 | 74 | /** 75 | * The memory setting to be used for each executor. 76 | */ 77 | private String executorMemory = "1024M"; 78 | 79 | @NotNull 80 | public String getAssemblyJar() { 81 | return assemblyJar; 82 | } 83 | 84 | public void setAssemblyJar(String assemblyJar) { 85 | this.assemblyJar = assemblyJar; 86 | } 87 | 88 | public int getNumExecutors() { 89 | return numExecutors; 90 | } 91 | 92 | public void setNumExecutors(int numExecutors) { 93 | this.numExecutors = numExecutors; 94 | } 95 | 96 | public String getAppName() { 97 | return appName; 98 | } 99 | 100 | public void setAppName(String appName) { 101 | this.appName = appName; 102 | } 103 | 104 | @NotNull 105 | public String getAppClass() { 106 | return appClass; 107 | } 108 | 109 | public void setAppClass(String appClass) { 110 | this.appClass = appClass; 111 | } 112 | 113 | @NotNull 114 | public String getAppJar() { 115 | return appJar; 116 | } 117 | 118 | public void setAppJar(String appJar) { 119 | this.appJar = appJar; 120 | } 121 | 122 | public String[] getAppArgs() { 123 | return appArgs; 124 | } 125 | 126 | public void setAppArgs(String[] appArgs) { 127 | this.appArgs = appArgs; 128 | } 129 | 130 | public String getResourceFiles() { 131 | return resourceFiles; 132 | } 133 | 134 | public void setResourceFiles(String resourceFiles) { 135 | this.resourceFiles = resourceFiles; 136 | } 137 | 138 | public String getResourceArchives() { 139 | return resourceArchives; 140 | } 141 | 142 | public void setResourceArchives(String resourceArchives) { 143 | this.resourceArchives = resourceArchives; 144 | } 145 | 146 | public String getExecutorMemory() { 147 | return executorMemory; 148 | } 149 | 150 | public void setExecutorMemory(String executorMemory) { 151 | this.executorMemory = executorMemory; 152 | } 153 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-spark-yarn/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | configuration-properties.classes=org.springframework.cloud.task.app.spark.yarn.SparkYarnTaskProperties, \ 18 | org.springframework.cloud.task.sparkapp.common.SparkAppCommonTaskProperties 19 | 20 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-job/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-starter-task-sqoop-job 6 | jar 7 | spring-cloud-starter-task-sqoop-job 8 | Spring Cloud Task Sqoop Job Starter 9 | 10 | 11 | org.springframework.cloud.task.app 12 | spring-cloud-task-app-starters 13 | 1.0.2.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 1.4.6 18 | 1.8.0.10 19 | 20 | 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-logging 29 | 30 | 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-log4j 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-test 39 | test 40 | 41 | 42 | 43 | org.springframework.cloud 44 | spring-cloud-task-core 45 | ${spring.cloud.task.core.version} 46 | 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-starter-jdbc 51 | 52 | 53 | com.h2database 54 | h2 55 | runtime 56 | 57 | 58 | org.mariadb.jdbc 59 | mariadb-java-client 60 | runtime 61 | 62 | 63 | org.postgresql 64 | postgresql 65 | runtime 66 | 67 | 68 | org.hsqldb 69 | hsqldb 70 | ${sqoop-hsqldb.version} 71 | runtime 72 | 73 | 74 | 75 | org.springframework.boot 76 | spring-boot-configuration-processor 77 | true 78 | 79 | 80 | 81 | org.springframework.cloud.task.app 82 | spring-cloud-task-sqoop-common 83 | 1.0.2.BUILD-SNAPSHOT 84 | 85 | 86 | org.springframework.cloud.task.app 87 | spring-cloud-task-sqoop-common 88 | 1.0.2.BUILD-SNAPSHOT 89 | tests 90 | test 91 | 92 | 93 | org.apache.sqoop 94 | sqoop 95 | ${sqoop.version} 96 | hadoop200 97 | 98 | 99 | org.springframework.data 100 | spring-data-hadoop-boot 101 | 102 | 103 | commons-logging 104 | commons-logging 105 | 106 | 107 | 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-job/src/main/java/org/springframework/cloud/task/app/sqoop/job/SqoopJobTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.sqoop.job; 18 | 19 | import javax.validation.constraints.NotNull; 20 | 21 | import org.springframework.boot.context.properties.ConfigurationProperties; 22 | import org.springframework.cloud.task.sqoop.common.SqoopCommonTaskProperties; 23 | 24 | /** 25 | * Configuration properties to be used by the SqoopJobTaskConfiguration. 26 | * 27 | * @author Thomas Risberg 28 | */ 29 | @ConfigurationProperties 30 | public class SqoopJobTaskProperties extends SqoopCommonTaskProperties { 31 | 32 | /** 33 | * The Sqoop job action to execute. 34 | */ 35 | private String action; 36 | 37 | /** 38 | * The Sqoop job name to be used for this job action. 39 | */ 40 | private String jobName; 41 | 42 | /** 43 | * The Sqoop Metastore URL to be used for this job action. 44 | */ 45 | private String metastoreUrl; 46 | 47 | /** 48 | * The Sqoop Metastore username to be used for this job action. 49 | */ 50 | private String metastoreUsername; 51 | 52 | /** 53 | * The Sqoop Metastore password to be used for this job action. 54 | */ 55 | private String metastorePassword; 56 | 57 | 58 | @NotNull 59 | public String getAction() { 60 | return action; 61 | } 62 | 63 | public void setAction(String action) { 64 | this.action = action; 65 | } 66 | 67 | @NotNull 68 | public String getJobName() { 69 | return jobName; 70 | } 71 | 72 | public void setJobName(String jobName) { 73 | this.jobName = jobName; 74 | } 75 | 76 | @NotNull 77 | public String getMetastoreUrl() { 78 | return metastoreUrl; 79 | } 80 | 81 | public void setMetastoreUrl(String metastoreUrl) { 82 | this.metastoreUrl = metastoreUrl; 83 | } 84 | 85 | public String getMetastoreUsername() { 86 | return metastoreUsername; 87 | } 88 | 89 | public void setMetastoreUsername(String metastoreUsername) { 90 | this.metastoreUsername = metastoreUsername; 91 | } 92 | 93 | public String getMetastorePassword() { 94 | return metastorePassword; 95 | } 96 | 97 | public void setMetastorePassword(String metastorePassword) { 98 | this.metastorePassword = metastorePassword; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-job/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%15.15t] %40.40c : %m%n 6 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-job/src/test/java/org/springframework/cloud/task/app/sqoop/job/SqoopToolJobPropertiesTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.sqoop.job; 18 | 19 | import org.junit.Test; 20 | 21 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 22 | import org.springframework.boot.test.EnvironmentTestUtils; 23 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 24 | import org.springframework.context.annotation.Configuration; 25 | 26 | import static org.hamcrest.CoreMatchers.equalTo; 27 | import static org.hamcrest.MatcherAssert.assertThat; 28 | 29 | /** 30 | * @author Thomas Risberg 31 | */ 32 | public class SqoopToolJobPropertiesTests { 33 | 34 | @Test 35 | public void testActionCanBeCustomized() { 36 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 37 | EnvironmentTestUtils.addEnvironment(context, "action: create"); 38 | EnvironmentTestUtils.addEnvironment(context, "job-name: myjob"); 39 | EnvironmentTestUtils.addEnvironment(context, "metastore-url: jdbc:hsqldb:hsql://localhost:16000/sqoop"); 40 | context.register(Conf.class); 41 | context.refresh(); 42 | SqoopJobTaskProperties properties = context.getBean(SqoopJobTaskProperties.class); 43 | assertThat(properties.getAction(), equalTo("create")); 44 | } 45 | 46 | @Test 47 | public void testJobNameCanBeCustomized() { 48 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 49 | EnvironmentTestUtils.addEnvironment(context, "action: exec"); 50 | EnvironmentTestUtils.addEnvironment(context, "job-name: anotherjob"); 51 | EnvironmentTestUtils.addEnvironment(context, "metastore-url: jdbc:hsqldb:hsql://localhost:16000/sqoop"); 52 | context.register(Conf.class); 53 | context.refresh(); 54 | SqoopJobTaskProperties properties = context.getBean(SqoopJobTaskProperties.class); 55 | assertThat(properties.getJobName(), equalTo("anotherjob")); 56 | } 57 | 58 | @Test 59 | public void testMetastoreUrlCanBeCustomized() { 60 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 61 | EnvironmentTestUtils.addEnvironment(context, "action: exec"); 62 | EnvironmentTestUtils.addEnvironment(context, "job-name: myjob"); 63 | EnvironmentTestUtils.addEnvironment(context, "metastore-url: jdbc:hsqldb:hsql://localhost:12345/sqoop"); 64 | context.register(Conf.class); 65 | context.refresh(); 66 | SqoopJobTaskProperties properties = context.getBean(SqoopJobTaskProperties.class); 67 | assertThat(properties.getMetastoreUrl(), equalTo("jdbc:hsqldb:hsql://localhost:12345/sqoop")); 68 | } 69 | 70 | @Test 71 | public void testMetastoreUsernameCanBeCustomized() { 72 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 73 | EnvironmentTestUtils.addEnvironment(context, "action: exec"); 74 | EnvironmentTestUtils.addEnvironment(context, "job-name: myjob"); 75 | EnvironmentTestUtils.addEnvironment(context, "metastore-url: jdbc:hsqldb:hsql://localhost:16000/sqoop"); 76 | EnvironmentTestUtils.addEnvironment(context, "metastore-username: spring"); 77 | context.register(Conf.class); 78 | context.refresh(); 79 | SqoopJobTaskProperties properties = context.getBean(SqoopJobTaskProperties.class); 80 | assertThat(properties.getMetastoreUsername(), equalTo("spring")); 81 | } 82 | 83 | @Test 84 | public void testMetastorePasswordCanBeCustomized() { 85 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 86 | EnvironmentTestUtils.addEnvironment(context, "action: exec"); 87 | EnvironmentTestUtils.addEnvironment(context, "job-name: myjob"); 88 | EnvironmentTestUtils.addEnvironment(context, "metastore-url: jdbc:hsqldb:hsql://localhost:16000/sqoop"); 89 | EnvironmentTestUtils.addEnvironment(context, "metastore-password: secret"); 90 | context.register(Conf.class); 91 | context.refresh(); 92 | SqoopJobTaskProperties properties = context.getBean(SqoopJobTaskProperties.class); 93 | assertThat(properties.getMetastorePassword(), equalTo("secret")); 94 | } 95 | 96 | @Configuration 97 | @EnableConfigurationProperties(SqoopJobTaskProperties.class) 98 | static class Conf { 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-job/src/test/resources/sqoop-root-data.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO SQOOP_ROOT (VERSION,PROPNAME,PROPVAL) VALUES(NULL,'sqoop.hsqldb.job.storage.version','0'); 2 | INSERT INTO SQOOP_ROOT (VERSION,PROPNAME,PROPVAL) VALUES(0,'sqoop.hsqldb.job.info.table','SQOOP_SESSIONS'); -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-job/src/test/resources/sqoop-schema-ddl.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS SQOOP_SESSIONS; 2 | DROP TABLE IF EXISTS SQOOP_ROOT; 3 | CREATE TABLE SQOOP_ROOT ( 4 | VERSION INTEGER, 5 | PROPNAME VARCHAR(128) NOT NULL, 6 | PROPVAL VARCHAR(256) 7 | ); 8 | CREATE INDEX IX_SQOOP_ROOT ON SQOOP_ROOT (VERSION, PROPNAME); 9 | CREATE TABLE SQOOP_SESSIONS ( 10 | JOB_NAME VARCHAR(64) NOT NULL, 11 | PROPNAME VARCHAR(128) NOT NULL, 12 | PROPVAL VARCHAR(1024), 13 | PROPCLASS VARCHAR(32) NOT NULL 14 | ); 15 | CREATE INDEX IX_SQOOP_SESSIONS ON SQOOP_SESSIONS (JOB_NAME, PROPNAME, PROPCLASS); 16 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/data/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-starter-task-sqoop-tool/data/_SUCCESS -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/data/part-m-00000: -------------------------------------------------------------------------------- 1 | 1,The Beatles,Let it be 2 | 2,The Doors,Light my fire 3 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-starter-task-sqoop-tool 6 | jar 7 | spring-cloud-starter-task-sqoop-tool 8 | Spring Cloud Task Sqoop Tool Starter 9 | 10 | 11 | org.springframework.cloud.task.app 12 | spring-cloud-task-app-starters 13 | 1.0.2.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 1.4.6 18 | 1.8.0.10 19 | 20 | 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-logging 29 | 30 | 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-log4j 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-test 39 | test 40 | 41 | 42 | 43 | org.springframework.cloud 44 | spring-cloud-task-core 45 | ${spring.cloud.task.core.version} 46 | 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-starter-jdbc 51 | 52 | 53 | com.h2database 54 | h2 55 | runtime 56 | 57 | 58 | org.mariadb.jdbc 59 | mariadb-java-client 60 | runtime 61 | 62 | 63 | org.postgresql 64 | postgresql 65 | runtime 66 | 67 | 68 | org.hsqldb 69 | hsqldb 70 | ${sqoop-hsqldb.version} 71 | runtime 72 | 73 | 74 | 75 | org.springframework.boot 76 | spring-boot-configuration-processor 77 | true 78 | 79 | 80 | 81 | org.springframework.cloud.task.app 82 | spring-cloud-task-sqoop-common 83 | 84 | 85 | org.springframework.cloud.task.app 86 | spring-cloud-task-sqoop-common 87 | tests 88 | test 89 | 1.0.2.BUILD-SNAPSHOT 90 | 91 | 92 | org.apache.sqoop 93 | sqoop 94 | ${sqoop.version} 95 | hadoop200 96 | 97 | 98 | org.springframework.data 99 | spring-data-hadoop-boot 100 | 101 | 102 | commons-logging 103 | commons-logging 104 | 105 | 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/src/main/java/org/springframework/cloud/task/app/sqoop/tool/SqoopToolTaskConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.sqoop.tool; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import org.apache.commons.logging.Log; 23 | import org.apache.commons.logging.LogFactory; 24 | import org.apache.hadoop.conf.Configuration; 25 | import org.apache.sqoop.Sqoop; 26 | import org.apache.sqoop.util.Jars; 27 | 28 | import org.springframework.beans.factory.annotation.Autowired; 29 | import org.springframework.beans.factory.annotation.Value; 30 | import org.springframework.boot.CommandLineRunner; 31 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 32 | import org.springframework.cloud.task.configuration.EnableTask; 33 | import org.springframework.cloud.task.sqoop.common.SqoopCommonRunnerUtils; 34 | import org.springframework.context.annotation.Bean; 35 | import org.springframework.util.StringUtils; 36 | 37 | /** 38 | * {@link CommandLineRunner} implementation that will run a Sqoop Tool. 39 | * This implementation currently supports import and export commands. 40 | * 41 | * @author Thomas Risberg 42 | */ 43 | @EnableTask 44 | @org.springframework.context.annotation.Configuration 45 | @EnableConfigurationProperties({ SqoopToolTaskProperties.class }) 46 | public class SqoopToolTaskConfiguration { 47 | 48 | @Bean 49 | public CommandLineRunner commandLineRunner() { 50 | return new SqoopToolRunner(); 51 | } 52 | 53 | private class SqoopToolRunner implements CommandLineRunner { 54 | 55 | private final Log logger = LogFactory.getLog(SqoopToolRunner.class); 56 | 57 | @Autowired 58 | private SqoopToolTaskProperties props; 59 | 60 | @Autowired 61 | private Configuration hadoopConfiguration; 62 | 63 | @Value("${spring.hadoop.config.mapreduce.framework.name:yarn}") 64 | private String mapreduceFramework; 65 | 66 | 67 | @Override 68 | public void run(String... args) throws Exception { 69 | 70 | List finalArguments = createFinalArguments(); 71 | 72 | logger.info("Running Sqoop tool with arguments: " + finalArguments); 73 | 74 | Configuration configuration = new Configuration(hadoopConfiguration); 75 | logger.info("Setting mapreduce.framework.name to " + mapreduceFramework); 76 | configuration.set("mapreduce.framework.name", mapreduceFramework); 77 | 78 | final int ret = Sqoop.runTool(finalArguments.toArray(new String[finalArguments.size()]), configuration); 79 | 80 | logger.info("Sqoop tool completed with return code: " + ret); 81 | 82 | if (ret != 0) { 83 | throw new RuntimeException("Sqoop job failed - return code " + ret); 84 | } 85 | } 86 | 87 | protected List createFinalArguments() { 88 | List finalArguments = new ArrayList(); 89 | String command = props.getCommand(); 90 | finalArguments.add(command); 91 | SqoopCommonRunnerUtils.setConnectProperties(props, finalArguments); 92 | if (command.toLowerCase().startsWith("import") || command.toLowerCase().startsWith("export")) { 93 | finalArguments.add("--hadoop-mapred-home=" + Jars.getJarPathForClass(Jars.class) 94 | .substring(0, Jars.getJarPathForClass(Jars.class).lastIndexOf("/"))); 95 | } 96 | List toolArguments = new ArrayList(); 97 | if (StringUtils.hasText(props.getToolArgs())) { 98 | String[] args = props.getToolArgs().split("\\s+"); 99 | for (String arg : args) { 100 | if (arg.startsWith("--hive") || arg.startsWith("--hcatalog") || 101 | arg.startsWith("--hbase") || arg.startsWith("--accumulo")) { 102 | throw new IllegalArgumentException( 103 | arg + " is incompatible with tool execution from a distributed task."); 104 | } 105 | toolArguments.add(arg); 106 | } 107 | } 108 | finalArguments.addAll(toolArguments); 109 | return finalArguments; 110 | } 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/src/main/java/org/springframework/cloud/task/app/sqoop/tool/SqoopToolTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.sqoop.tool; 18 | 19 | import javax.validation.constraints.NotNull; 20 | 21 | import org.springframework.boot.context.properties.ConfigurationProperties; 22 | import org.springframework.cloud.task.sqoop.common.SqoopCommonTaskProperties; 23 | 24 | /** 25 | * Configuration properties to be used by the SqoopToolTaskConfiguration. 26 | * 27 | * @author Thomas Risberg 28 | */ 29 | @ConfigurationProperties 30 | public class SqoopToolTaskProperties extends SqoopCommonTaskProperties { 31 | 32 | @Override 33 | @NotNull 34 | public String getCommand() { 35 | return super.getCommand(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%15.15t] %40.40c : %m%n 6 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-sqoop-tool/src/test/java/org/springframework/cloud/task/app/sqoop/tool/SqoopToolTaskPropertiesTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.sqoop.tool; 18 | 19 | import org.junit.Test; 20 | 21 | import org.springframework.beans.factory.BeanCreationException; 22 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 23 | import org.springframework.boot.test.EnvironmentTestUtils; 24 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 25 | import org.springframework.context.annotation.Configuration; 26 | 27 | import static org.hamcrest.CoreMatchers.equalTo; 28 | import static org.hamcrest.MatcherAssert.assertThat; 29 | 30 | /** 31 | * @author Thomas Risberg 32 | */ 33 | public class SqoopToolTaskPropertiesTests { 34 | 35 | @Test 36 | public void testCommandCanBeCustomized() { 37 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 38 | EnvironmentTestUtils.addEnvironment(context, "command: export"); 39 | context.register(Conf.class); 40 | context.refresh(); 41 | SqoopToolTaskProperties properties = context.getBean(SqoopToolTaskProperties.class); 42 | assertThat(properties.getCommand(), equalTo("export")); 43 | } 44 | 45 | @Test(expected=BeanCreationException.class) 46 | public void testCommandIsRequired() { 47 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 48 | context.register(Conf.class); 49 | context.refresh(); 50 | } 51 | 52 | @Configuration 53 | @EnableConfigurationProperties(SqoopToolTaskProperties.class) 54 | static class Conf { 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/README.adoc: -------------------------------------------------------------------------------- 1 | //tag::ref-doc[] 2 | = Timestamp Task 3 | 4 | A task that prints a timestamp to `stdout`. Intended to primarily be used for testing. 5 | 6 | == Options 7 | 8 | // see syntax (soon to be automatically generated) in spring-cloud-stream starters 9 | The **$$timestamp$$** $$task$$ has the following options: 10 | 11 | //tag::configuration-properties[] 12 | $$timestamp.format$$:: $$The timestamp format, "yyyy-MM-dd HH:mm:ss.SSS" by default.$$ *($$String$$, default: `$$yyyy-MM-dd HH:mm:ss.SSS$$`)* 13 | //end::configuration-properties[] 14 | 15 | //end::ref-doc[] 16 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-starter-task-timestamp 6 | jar 7 | spring-cloud-starter-task-timestamp 8 | Spring Cloud Task Timestamp Starter 9 | 10 | 11 | org.springframework.cloud.task.app 12 | spring-cloud-task-app-starters 13 | 1.0.2.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter-test 24 | test 25 | 26 | 27 | 28 | org.springframework.cloud 29 | spring-cloud-task-core 30 | ${spring.cloud.task.core.version} 31 | 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-jdbc 36 | 37 | 38 | com.h2database 39 | h2 40 | runtime 41 | 42 | 43 | org.mariadb.jdbc 44 | mariadb-java-client 45 | runtime 46 | 47 | 48 | org.postgresql 49 | postgresql 50 | runtime 51 | 52 | 53 | 54 | org.springframework.boot 55 | spring-boot-configuration-processor 56 | true 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/src/main/java/org/springframework/cloud/task/app/timestamp/TimestampTaskConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.timestamp; 18 | 19 | import java.text.DateFormat; 20 | import java.text.SimpleDateFormat; 21 | import java.util.Date; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.boot.CommandLineRunner; 28 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 29 | import org.springframework.cloud.task.configuration.EnableTask; 30 | import org.springframework.context.annotation.Bean; 31 | import org.springframework.context.annotation.Configuration; 32 | 33 | /** 34 | * A commandline runner that prints a timestamp. 35 | */ 36 | @EnableTask 37 | @Configuration 38 | @EnableConfigurationProperties({ TimestampTaskProperties.class }) 39 | public class TimestampTaskConfiguration { 40 | 41 | @Bean 42 | public TimestampTask timeStampTask() { 43 | return new TimestampTask(); 44 | } 45 | 46 | /** 47 | * A commandline runner that prints a timestamp. 48 | */ 49 | public class TimestampTask implements CommandLineRunner { 50 | private final Log logger = LogFactory.getLog(TimestampTask.class); 51 | 52 | @Autowired 53 | private TimestampTaskProperties config; 54 | 55 | @Override 56 | public void run(String... strings) throws Exception { 57 | DateFormat dateFormat = new SimpleDateFormat(config.getFormat()); 58 | logger.info(dateFormat.format(new Date())); 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/src/main/java/org/springframework/cloud/task/app/timestamp/TimestampTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.timestamp; 18 | 19 | import org.springframework.boot.context.properties.ConfigurationProperties; 20 | import org.springframework.util.Assert; 21 | 22 | /** 23 | * @author Glenn Renfro 24 | */ 25 | @ConfigurationProperties("timestamp") 26 | public class TimestampTaskProperties { 27 | 28 | /** 29 | * The timestamp format, "yyyy-MM-dd HH:mm:ss.SSS" by default. 30 | */ 31 | private String format = "yyyy-MM-dd HH:mm:ss.SSS"; 32 | 33 | public String getFormat() { 34 | Assert.hasText(format, "format must not be empty nor null"); 35 | return format; 36 | } 37 | 38 | public void setFormat(String format) { 39 | this.format = format; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | configuration-properties.classes=org.springframework.cloud.task.app.timestamp.TimestampTaskProperties 18 | -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/src/test/java/org/springframework/cloud/task/app/timestamp/TimestampTaskPropertiesTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.timestamp; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | 21 | import org.junit.Test; 22 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 23 | import org.springframework.boot.test.EnvironmentTestUtils; 24 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 25 | import org.springframework.context.annotation.Configuration; 26 | 27 | /** 28 | * @author Glenn Renfro 29 | */ 30 | public class TimestampTaskPropertiesTests { 31 | 32 | @Test(expected = IllegalArgumentException.class) 33 | public void testEmptyFormat() { 34 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 35 | EnvironmentTestUtils.addEnvironment(context, "timestamp.format:"); 36 | context.register(Conf.class); 37 | context.refresh(); 38 | TimestampTaskProperties properties = context.getBean(TimestampTaskProperties.class); 39 | properties.getFormat(); 40 | } 41 | 42 | @Test 43 | public void testFormatDefault() { 44 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 45 | context.register(Conf.class); 46 | context.refresh(); 47 | TimestampTaskProperties properties = context.getBean(TimestampTaskProperties.class); 48 | assertEquals("result does not match default format.", "yyyy-MM-dd HH:mm:ss.SSS", 49 | properties.getFormat()); 50 | } 51 | 52 | @Test 53 | public void testFormatSet() { 54 | final String FORMAT = "yyyy"; 55 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 56 | context.register(Conf.class); 57 | context.refresh(); 58 | TimestampTaskProperties properties = context.getBean(TimestampTaskProperties.class); 59 | properties.setFormat(FORMAT); 60 | assertEquals("result does not match established format.", FORMAT, 61 | properties.getFormat()); 62 | } 63 | 64 | @Configuration 65 | @EnableConfigurationProperties(TimestampTaskProperties.class) 66 | static class Conf { 67 | } 68 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/src/test/java/org/springframework/cloud/task/app/timestamp/TimestampTaskTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.app.timestamp; 18 | 19 | import org.junit.Rule; 20 | import org.junit.Test; 21 | import org.springframework.boot.SpringApplication; 22 | import org.springframework.boot.autoconfigure.SpringBootApplication; 23 | import org.springframework.boot.test.OutputCapture; 24 | 25 | import static junit.framework.Assert.assertTrue; 26 | 27 | /** 28 | * Verifies that the Task Application outputs the correct task log entries. 29 | * 30 | * @author Glenn Renfro 31 | * 32 | */ 33 | public class TimestampTaskTests { 34 | 35 | @Rule 36 | public OutputCapture outputCapture = new OutputCapture(); 37 | 38 | @Test 39 | public void testTimeStampApp() throws Exception { 40 | final String TEST_DATE_DOTS = "......."; 41 | final String CREATE_TASK_MESSAGE = "Creating: TaskExecution{executionId="; 42 | final String UPDATE_TASK_MESSAGE = "Updating: TaskExecution with executionId=1 with the following"; 43 | String[] args = { "--format=yyyy" + TEST_DATE_DOTS }; 44 | 45 | SpringApplication 46 | .run(TestTimestampTaskApplication.class, args); 47 | 48 | String output = this.outputCapture.toString(); 49 | 50 | assertTrue("Unable to find the timestamp: " + output, 51 | output.contains(TEST_DATE_DOTS)); 52 | assertTrue("Test results do not show create task message: " + output, 53 | output.contains(CREATE_TASK_MESSAGE)); 54 | assertTrue("Test results do not show success message: " + output, 55 | output.contains(UPDATE_TASK_MESSAGE)); 56 | } 57 | 58 | 59 | 60 | @SpringBootApplication 61 | public static class TestTimestampTaskApplication { 62 | public static void main(String[] args) { 63 | SpringApplication.run(TestTimestampTaskApplication.class, args); 64 | } 65 | } 66 | 67 | } -------------------------------------------------------------------------------- /spring-cloud-starter-task-timestamp/src/test/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | logging.level.root=DEBUG 18 | spring.application.name=Demo Timestamp Task -------------------------------------------------------------------------------- /spring-cloud-task-app-descriptor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | spring-cloud-task-app-starters 5 | org.springframework.cloud.task.app 6 | 1.0.2.BUILD-SNAPSHOT 7 | 8 | 4.0.0 9 | 10 | spring-cloud-task-app-descriptor 11 | spring-cloud-task-app-descriptor 12 | 13 | 14 | 15 | 16 | src/main/resources 17 | true 18 | 19 | META-INF/task-apps-maven.properties 20 | META-INF/task-apps-docker.properties 21 | 22 | 23 | 24 | 25 | 26 | 27 | org.codehaus.gmaven 28 | gmaven-plugin 29 | 1.5 30 | 31 | 32 | validate 33 | 34 | execute 35 | 36 | 37 | 41 | 42 | 43 | 44 | 45 | 46 | org.codehaus.mojo 47 | build-helper-maven-plugin 48 | 1.8 49 | 50 | 51 | attach-artifacts 52 | package 53 | 54 | attach-artifact 55 | 56 | 57 | 58 | 59 | target/classes/META-INF/task-apps-maven.properties 60 | task-apps-maven 61 | 62 | 63 | 64 | target/classes/META-INF/task-apps-docker.properties 65 | task-apps-docker 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /spring-cloud-task-app-descriptor/src/main/resources/META-INF/task-apps-docker.properties: -------------------------------------------------------------------------------- 1 | task.timestamp=docker:springcloudtask/timestamp-task:@docker.tag@ 2 | task.spark-client=docker:springcloudtask/spark-client-task:@docker.tag@ 3 | task.spark-cluster=docker:springcloudtask/spark-cluster-task:@docker.tag@ 4 | task.spark-yarn=docker:springcloudtask/spark-yarn-task:@docker.tag@ 5 | -------------------------------------------------------------------------------- /spring-cloud-task-app-descriptor/src/main/resources/META-INF/task-apps-maven.properties: -------------------------------------------------------------------------------- 1 | task.timestamp=maven://org.springframework.cloud.task.app:timestamp-task:@project.version@ 2 | task.spark-client=maven://org.springframework.cloud.task.app:spark-client-task:@project.version@ 3 | task.spark-cluster=maven://org.springframework.cloud.task.app:spark-cluster-task:@project.version@ 4 | task.spark-yarn=maven://org.springframework.cloud.task.app:spark-yarn-task:@project.version@ 5 | -------------------------------------------------------------------------------- /spring-cloud-task-app-generator/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | spring-cloud-task-app-starters 5 | org.springframework.cloud.task.app 6 | 1.0.2.BUILD-SNAPSHOT 7 | 8 | 4.0.0 9 | 10 | spring-cloud-task-app-generator 11 | pom 12 | spring-cloud-task-app-generator 13 | 14 | 15 | 16 | 17 | org.springframework.cloud.stream.app.plugin 18 | spring-cloud-stream-app-maven-plugin 19 | ${scs-app-maven-plugin.version} 20 | 21 | 1.7 22 | 1.3.5.RELEASE 23 | ${session.executionRootDirectory}/apps 24 | ${project.version} 25 | task 26 | 27 | scs-bom 28 | org.springframework.cloud.task.app 29 | spring-cloud-task-app-dependencies 30 | ${project.version} 31 | 32 | 33 | 34 | 35 | 36 | 37 | com.fasterxml.jackson.core 38 | jackson-core 39 | 2.4.4 40 | 41 | 42 | com.fasterxml.jackson.core 43 | jackson-databind 44 | 2.4.4 45 | 46 | 47 | com.fasterxml.jackson.core 48 | jackson-annotations 49 | 2.4.4 50 | 51 | 52 | 53 | 54 | 55 | 56 | com.fasterxml.jackson.core 57 | jackson-core 58 | 2.4.4 59 | 60 | 61 | com.fasterxml.jackson.core 62 | jackson-databind 63 | 2.4.4 64 | 65 | 66 | com.fasterxml.jackson.core 67 | jackson-annotations 68 | 2.4.4 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.css 3 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/Guardfile: -------------------------------------------------------------------------------- 1 | require 'asciidoctor' 2 | require 'erb' 3 | 4 | guard 'shell' do 5 | watch(/.*\.adoc$/) {|m| 6 | Asciidoctor.render_file('index.adoc', \ 7 | :in_place => true, \ 8 | :safe => Asciidoctor::SafeMode::UNSAFE, \ 9 | :attributes=> { \ 10 | 'source-highlighter' => 'prettify', \ 11 | 'icons' => 'font', \ 12 | 'linkcss'=> 'true', \ 13 | 'copycss' => 'true', \ 14 | 'doctype' => 'book'}) 15 | } 16 | end 17 | 18 | guard 'livereload' do 19 | watch(%r{^.+\.(css|js|html)$}) 20 | end 21 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/appendix.adoc: -------------------------------------------------------------------------------- 1 | [[appendix]] 2 | = Appendices 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/building.adoc: -------------------------------------------------------------------------------- 1 | [[building]] 2 | == Building 3 | 4 | :jdkversion: 1.7 5 | 6 | === Basic Compile and Test 7 | 8 | To build the source you will need to install JDK {jdkversion}. 9 | 10 | The build uses the Maven wrapper so you don't have to install a specific 11 | version of Maven. To enable the tests for Redis you should run the server 12 | before bulding. See below for more information on how run Redis. 13 | 14 | The main build command is 15 | 16 | ---- 17 | $ ./mvnw clean install 18 | ---- 19 | 20 | You can also add '-DskipTests' if you like, to avoid running the tests. 21 | 22 | NOTE: You can also install Maven (>=3.3.3) yourself and run the `mvn` command 23 | in place of `./mvnw` in the examples below. If you do that you also 24 | might need to add `-P spring` if your local Maven settings do not 25 | contain repository declarations for spring pre-release artifacts. 26 | 27 | NOTE: Be aware that you might need to increase the amount of memory 28 | available to Maven by setting a `MAVEN_OPTS` environment variable with 29 | a value like `-Xmx512m -XX:MaxPermSize=128m`. We try to cover this in 30 | the `.mvn` configuration, so if you find you have to do it to make a 31 | build succeed, please raise a ticket to get the settings added to 32 | source control. 33 | 34 | 35 | The projects that require middleware generally include a 36 | `docker-compose.yml`, so consider using 37 | https://compose.docker.io/[Docker Compose] to run the middeware servers 38 | in Docker containers. See the README in the 39 | https://github.com/spring-cloud-samples/scripts[scripts demo 40 | repository] for specific instructions about the common cases of mongo, 41 | rabbit and redis. 42 | 43 | === Documentation 44 | 45 | There is a "full" profile that will generate documentation. You can build just the documentation by executing 46 | 47 | [listing, subs=attributes] 48 | ---- 49 | $ ./mvnw package -DskipTests=true -P full -pl {project-artifactId} -am 50 | ---- 51 | 52 | === Working with the code 53 | If you don't have an IDE preference we would recommend that you use 54 | https://www.springsource.com/developer/sts[Spring Tools Suite] or 55 | https://eclipse.org[Eclipse] when working with the code. We use the 56 | https://eclipse.org/m2e/[m2eclipe] eclipse plugin for maven support. Other IDEs and tools 57 | should also work without issue. 58 | 59 | ==== Importing into eclipse with m2eclipse 60 | We recommend the https://eclipse.org/m2e/[m2eclipe] eclipse plugin when working with 61 | eclipse. If you don't already have m2eclipse installed it is available from the "eclipse 62 | marketplace". 63 | 64 | Unfortunately m2e does not yet support Maven 3.3, so once the projects 65 | are imported into Eclipse you will also need to tell m2eclipse to use 66 | the `.settings.xml` file for the projects. If you do not do this you 67 | may see many different errors related to the POMs in the 68 | projects. Open your Eclipse preferences, expand the Maven 69 | preferences, and select User Settings. In the User Settings field 70 | click Browse and navigate to the Spring Cloud project you imported 71 | selecting the `.settings.xml` file in that project. Click Apply and 72 | then OK to save the preference changes. 73 | 74 | NOTE: Alternatively you can copy the repository settings from https://github.com/spring-cloud/spring-cloud-build/blob/master/.settings.xml[`.settings.xml`] into your own `~/.m2/settings.xml`. 75 | 76 | ==== Importing into eclipse without m2eclipse 77 | If you prefer not to use m2eclipse you can generate eclipse project metadata using the 78 | following command: 79 | 80 | ---- 81 | $ ./mvnw eclipse:eclipse 82 | ---- 83 | 84 | The generated eclipse projects can be imported by selecting `import existing projects` 85 | from the `file` menu. 86 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/contributing.adoc: -------------------------------------------------------------------------------- 1 | [[contributing]] 2 | == Contributing 3 | 4 | Spring Cloud is released under the non-restrictive Apache 2.0 license, 5 | and follows a very standard Github development process, using Github 6 | tracker for issues and merging pull requests into master. If you want 7 | to contribute even something trivial please do not hesitate, but 8 | follow the guidelines below. 9 | 10 | === Sign the Contributor License Agreement 11 | Before we accept a non-trivial patch or pull request we will need you to sign the 12 | https://support.springsource.com/spring_committer_signup[contributor's agreement]. 13 | Signing the contributor's agreement does not grant anyone commit rights to the main 14 | repository, but it does mean that we can accept your contributions, and you will get an 15 | author credit if we do. Active contributors might be asked to join the core team, and 16 | given the ability to merge pull requests. 17 | 18 | === Code Conventions and Housekeeping 19 | None of these is essential for a pull request, but they will all help. They can also be 20 | added after the original pull request but before a merge. 21 | 22 | * Use the Spring Framework code format conventions. If you use Eclipse 23 | you can import formatter settings using the 24 | `eclipse-code-formatter.xml` file from the 25 | https://github.com/spring-cloud/build/tree/master/eclipse-coding-conventions.xml[Spring 26 | Cloud Build] project. If using IntelliJ, you can use the 27 | https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter 28 | Plugin] to import the same file. 29 | * Make sure all new `.java` files to have a simple Javadoc class comment with at least an 30 | `@author` tag identifying you, and preferably at least a paragraph on what the class is 31 | for. 32 | * Add the ASF license header comment to all new `.java` files (copy from existing files 33 | in the project) 34 | * Add yourself as an `@author` to the .java files that you modify substantially (more 35 | than cosmetic changes). 36 | * Add some Javadocs and, if you change the namespace, some XSD doc elements. 37 | * A few unit tests would help a lot as well -- someone has to do it. 38 | * If no-one else is using your branch, please rebase it against the current master (or 39 | other target branch in the main project). 40 | * When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions], 41 | if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit 42 | message (where XXXX is the issue number). 43 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/index-docinfo.xml: -------------------------------------------------------------------------------- 1 | Spring Cloud Data Flow 2 | {project-version} 3 | 4 | 2013-2016 5 | Pivotal Software, Inc. 6 | 7 | 8 | 9 | Copies of this document may be made for your own use and for distribution to 10 | others, provided that you do not charge any fee for such copies and further 11 | provided that each copy contains this Copyright Notice, whether distributed in 12 | print or electronically. 13 | 14 | 15 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/index.adoc: -------------------------------------------------------------------------------- 1 | = Spring Cloud Task Reference Guide 2 | Sabby Anandan, Artem Bilan, Marius Bogoevici, Eric Bottard, Mark Fisher, Ilayaperumal Gopinathan, Gunnar Hillert, Mark Pollack, Patrick Peralta, Glenn Renfro, Gary Russell, Thomas Risberg, David Turanski, Janne Valkealahti 3 | :doctype: book 4 | :toc: 5 | :toclevels: 4 6 | :source-highlighter: prettify 7 | :numbered: 8 | :icons: font 9 | :hide-uri-scheme: 10 | :repo-root: ../../../../ 11 | // ====================================================================================== 12 | 13 | = Reference Guide 14 | include::overview.adoc[] 15 | 16 | = Tasks 17 | include::tasks.adoc[] 18 | 19 | = Appendices 20 | [appendix] 21 | include::building.adoc[] 22 | 23 | include::contributing.adoc[] 24 | 25 | // ====================================================================================== 26 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/asciidoc/tasks.adoc: -------------------------------------------------------------------------------- 1 | [[spring-cloud-task-modules-tasks]] 2 | 3 | :leveloffset: +1 4 | :app-starters-root: https://raw.githubusercontent.com/spring-cloud/spring-cloud-task-app-starters/master 5 | 6 | [[spring-cloud-task-modules-spark-client]] 7 | include::{app-starters-root}/spring-cloud-starter-task-spark-client/README.adoc[tags=ref-doc] 8 | include::{app-starters-root}/spring-cloud-starter-task-spark-cluster/README.adoc[tags=ref-doc] 9 | include::{app-starters-root}/spring-cloud-starter-task-spark-yarn/README.adoc[tags=ref-doc] 10 | include::{app-starters-root}/spring-cloud-starter-task-timestamp/README.adoc[tags=ref-doc] 11 | 12 | :leveloffset: -1 -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/css/highlight.css: -------------------------------------------------------------------------------- 1 | /* 2 | code highlight CSS resemblign the Eclipse IDE default color schema 3 | @author Costin Leau 4 | */ 5 | 6 | .hl-keyword { 7 | color: #7F0055; 8 | font-weight: bold; 9 | } 10 | 11 | .hl-comment { 12 | color: #3F5F5F; 13 | font-style: italic; 14 | } 15 | 16 | .hl-multiline-comment { 17 | color: #3F5FBF; 18 | font-style: italic; 19 | } 20 | 21 | .hl-tag { 22 | color: #3F7F7F; 23 | } 24 | 25 | .hl-attribute { 26 | color: #7F007F; 27 | } 28 | 29 | .hl-value { 30 | color: #2A00FF; 31 | } 32 | 33 | .hl-string { 34 | color: #2A00FF; 35 | } -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/css/manual-multipage.css: -------------------------------------------------------------------------------- 1 | @IMPORT url("manual.css"); 2 | 3 | body.firstpage { 4 | background: url("../images/background.png") no-repeat center top; 5 | } 6 | 7 | div.part h1 { 8 | border-top: none; 9 | } 10 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/css/manual-singlepage.css: -------------------------------------------------------------------------------- 1 | @IMPORT url("manual.css"); 2 | 3 | body { 4 | background: url("../images/background.png") no-repeat center top; 5 | } 6 | 7 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/background.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/caution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/caution.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/important.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/important.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/logo.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/note.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/note.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/tip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/tip.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/images/warning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-task-app-starters/4e7bedb33b5ecad8633deafedf9bd1fc05916348/spring-cloud-task-app-starters-docs/src/main/docbook/images/warning.png -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/common.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 21 | 22 | 27 | 28 | 29 | 1 30 | 0 31 | 1 32 | 33 | 34 | 35 | images/ 36 | .png 37 | 38 | 39 | book toc,title 40 | 3 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/epub.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 21 | 22 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/html-multipage.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 21 | 22 | 24 | 25 | 26 | 27 | 28 | css/manual-multipage.css 29 | 30 | '5' 31 | '1' 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | firstpage 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/html-singlepage.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 21 | 22 | 24 | 25 | 26 | 27 | 28 | css/manual-singlepage.css 29 | 30 | 31 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/asciidoc-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | //// 10 | //// 11 | 12 | 13 | // 14 | 15 | 16 | 17 | ^(={1,6} .+)$ 18 | 19 | MULTILINE 20 | 21 | 22 | ^(\.[^\.\s].+)$ 23 | 24 | MULTILINE 25 | 26 | 27 | ^(:!?\w.*?:) 28 | 29 | MULTILINE 30 | 31 | 32 | ^(-|\*{1,5}|\d*\.{1,5})(?= .+$) 33 | 34 | MULTILINE 35 | 36 | 37 | ^(\[.+\])$ 38 | 39 | MULTILINE 40 | 41 | 42 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/bourne-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 27 | 28 | # 29 | 30 | << 31 | ' 32 | " 33 | - 34 | 35 | 36 | 37 | 38 | " 39 | \ 40 | 41 | 42 | ' 43 | \ 44 | 45 | 46 | 47 | 0x 48 | 49 | 50 | 51 | . 52 | 53 | 54 | 55 | 56 | 57 | if 58 | then 59 | else 60 | elif 61 | fi 62 | case 63 | esac 64 | for 65 | while 66 | until 67 | do 68 | done 69 | 70 | exec 71 | shift 72 | exit 73 | times 74 | break 75 | export 76 | trap 77 | continue 78 | readonly 79 | wait 80 | eval 81 | return 82 | 83 | cd 84 | echo 85 | hash 86 | pwd 87 | read 88 | set 89 | test 90 | type 91 | ulimit 92 | umask 93 | unset 94 | 95 | 96 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/c-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 29 | 30 | 31 | /** 32 | */ 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | /* 41 | */ 42 | 43 | // 44 | 45 | 46 | # 47 | \ 48 | 49 | 50 | 51 | 52 | " 53 | \ 54 | 55 | 56 | ' 57 | \ 58 | 59 | 60 | 0x 61 | ul 62 | lu 63 | u 64 | l 65 | 66 | 67 | 68 | . 69 | 70 | e 71 | ul 72 | lu 73 | u 74 | f 75 | l 76 | 77 | 78 | 79 | auto 80 | _Bool 81 | break 82 | case 83 | char 84 | _Complex 85 | const 86 | continue 87 | default 88 | do 89 | double 90 | else 91 | enum 92 | extern 93 | float 94 | for 95 | goto 96 | if 97 | _Imaginary 98 | inline 99 | int 100 | long 101 | register 102 | restrict 103 | return 104 | short 105 | signed 106 | sizeof 107 | static 108 | struct 109 | switch 110 | typedef 111 | union 112 | unsigned 113 | void 114 | volatile 115 | while 116 | 117 | 118 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/html-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 12 | 13 | 14 | 15 | 16 | a 17 | abbr 18 | address 19 | area 20 | article 21 | aside 22 | audio 23 | b 24 | base 25 | bdi 26 | blockquote 27 | body 28 | br 29 | button 30 | caption 31 | canvas 32 | cite 33 | code 34 | command 35 | col 36 | colgroup 37 | dd 38 | del 39 | dialog 40 | div 41 | dl 42 | dt 43 | em 44 | embed 45 | fieldset 46 | figcaption 47 | figure 48 | font 49 | form 50 | footer 51 | h1 52 | h2 53 | h3 54 | h4 55 | h5 56 | h6 57 | head 58 | header 59 | hr 60 | html 61 | i 62 | iframe 63 | img 64 | input 65 | ins 66 | kbd 67 | label 68 | legend 69 | li 70 | link 71 | map 72 | mark 73 | menu 74 | menu 75 | meta 76 | nav 77 | noscript 78 | object 79 | ol 80 | optgroup 81 | option 82 | p 83 | param 84 | pre 85 | q 86 | samp 87 | script 88 | section 89 | select 90 | small 91 | source 92 | span 93 | strong 94 | style 95 | sub 96 | summary 97 | sup 98 | table 99 | tbody 100 | td 101 | textarea 102 | tfoot 103 | th 104 | thead 105 | time 106 | title 107 | tr 108 | track 109 | u 110 | ul 111 | var 112 | video 113 | wbr 114 | xmp 115 | 116 | 117 | 118 | 119 | xsl: 120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/ini-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 31 | 32 | ; 33 | 34 | 35 | ^(\[.+\]\s*)$ 36 | 37 | MULTILINE 38 | 39 | 40 | 41 | ^(.+)(?==) 42 | 43 | MULTILINE 44 | 45 | 46 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/java-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 31 | 32 | 33 | /** 34 | */ 35 | 36 | 37 | 38 | /* 39 | */ 40 | 41 | // 42 | 43 | " 44 | \ 45 | 46 | 47 | ' 48 | \ 49 | 50 | 51 | @ 52 | ( 53 | ) 54 | 55 | 56 | 0x 57 | 58 | 59 | 60 | . 61 | e 62 | f 63 | d 64 | l 65 | 66 | 67 | 68 | abstract 69 | boolean 70 | break 71 | byte 72 | case 73 | catch 74 | char 75 | class 76 | const 77 | continue 78 | default 79 | do 80 | double 81 | else 82 | extends 83 | final 84 | finally 85 | float 86 | for 87 | goto 88 | if 89 | implements 90 | import 91 | instanceof 92 | int 93 | interface 94 | long 95 | native 96 | new 97 | package 98 | private 99 | protected 100 | public 101 | return 102 | short 103 | static 104 | strictfp 105 | super 106 | switch 107 | synchronized 108 | this 109 | throw 110 | throws 111 | transient 112 | try 113 | void 114 | volatile 115 | while 116 | 117 | 118 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/json-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | # 4 | 5 | " 6 | \ 7 | 8 | 9 | ' 10 | \ 11 | 12 | 13 | @ 14 | ( 15 | ) 16 | 17 | 18 | . 19 | e 20 | f 21 | d 22 | l 23 | 24 | 25 | 26 | true 27 | false 28 | 29 | 30 | { 31 | } 32 | , 33 | [ 34 | ] 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/perl-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 31 | 32 | # 33 | 34 | << 35 | ' 36 | " 37 | 38 | 39 | 40 | " 41 | \ 42 | 43 | 44 | ' 45 | \ 46 | 47 | 48 | 49 | 0x 50 | 51 | 52 | 53 | . 54 | 55 | 56 | 57 | 58 | if 59 | unless 60 | while 61 | until 62 | foreach 63 | else 64 | elsif 65 | for 66 | when 67 | default 68 | given 69 | 70 | caller 71 | continue 72 | die 73 | do 74 | dump 75 | eval 76 | exit 77 | goto 78 | last 79 | next 80 | redo 81 | return 82 | sub 83 | wantarray 84 | 85 | caller 86 | import 87 | local 88 | my 89 | package 90 | use 91 | 92 | do 93 | import 94 | no 95 | package 96 | require 97 | use 98 | 99 | bless 100 | dbmclose 101 | dbmopen 102 | package 103 | ref 104 | tie 105 | tied 106 | untie 107 | use 108 | 109 | and 110 | or 111 | not 112 | eq 113 | ne 114 | lt 115 | gt 116 | le 117 | ge 118 | cmp 119 | 120 | 121 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/properties-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 31 | 32 | # 33 | 34 | ^(.+?)(?==|:) 35 | 36 | MULTILINE 37 | 38 | 39 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/python-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 31 | 32 | 33 | 34 | @ 35 | ( 36 | ) 37 | 38 | # 39 | 40 | """ 41 | 42 | 43 | 44 | ''' 45 | 46 | 47 | 48 | " 49 | \ 50 | 51 | 52 | ' 53 | \ 54 | 55 | 56 | 0x 57 | l 58 | 59 | 60 | 61 | . 62 | 63 | e 64 | l 65 | 66 | 67 | 68 | and 69 | del 70 | from 71 | not 72 | while 73 | as 74 | elif 75 | global 76 | or 77 | with 78 | assert 79 | else 80 | if 81 | pass 82 | yield 83 | break 84 | except 85 | import 86 | print 87 | class 88 | exec 89 | in 90 | raise 91 | continue 92 | finally 93 | is 94 | return 95 | def 96 | for 97 | lambda 98 | try 99 | 100 | 101 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/ruby-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 31 | 32 | # 33 | 34 | << 35 | 36 | 37 | 38 | " 39 | \ 40 | 41 | 42 | %Q{ 43 | } 44 | \ 45 | 46 | 47 | %/ 48 | / 49 | \ 50 | 51 | 52 | ' 53 | \ 54 | 55 | 56 | %q{ 57 | } 58 | \ 59 | 60 | 61 | 0x 62 | 63 | 64 | 65 | . 66 | e 67 | 68 | 69 | 70 | alias 71 | and 72 | BEGIN 73 | begin 74 | break 75 | case 76 | class 77 | def 78 | defined 79 | do 80 | else 81 | elsif 82 | END 83 | end 84 | ensure 85 | false 86 | for 87 | if 88 | in 89 | module 90 | next 91 | nil 92 | not 93 | or 94 | redo 95 | rescue 96 | retry 97 | return 98 | self 99 | super 100 | then 101 | true 102 | undef 103 | unless 104 | until 105 | when 106 | while 107 | yield 108 | 109 | 110 | -------------------------------------------------------------------------------- /spring-cloud-task-app-starters-docs/src/main/docbook/xsl/xslthl/yaml-hl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | # 4 | 5 | " 6 | \ 7 | 8 | 9 | ' 10 | \ 11 | 12 | 13 | @ 14 | ( 15 | ) 16 | 17 | 18 | . 19 | e 20 | f 21 | d 22 | l 23 | 24 | 25 | 26 | true 27 | false 28 | 29 | 30 | { 31 | } 32 | , 33 | [ 34 | ] 35 | 36 | 37 | 38 | ^(---)$ 39 | 40 | MULTILINE 41 | 42 | 43 | ^(.+?)(?==|:) 44 | 45 | MULTILINE 46 | 47 | 48 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.springframework.cloud.task.app 5 | spring-cloud-task-app-starters 6 | 1.0.2.BUILD-SNAPSHOT 7 | ../ 8 | 9 | 4.0.0 10 | 11 | spring-cloud-task-jdbchdfs-common 12 | jar 13 | spring-cloud-task-jdbchdfs-common 14 | 15 | 16 | 2.4.0.RELEASE 17 | 1.0.1.RELEASE 18 | 1.0.0.RELEASE 19 | 20 | 21 | 22 | 23 | org.hibernate 24 | hibernate-validator 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-batch 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-test 33 | test 34 | 35 | 36 | org.springframework.batch 37 | spring-batch-infrastructure 38 | 39 | 40 | org.springframework.cloud 41 | spring-cloud-task-core 42 | ${spring.cloud.task.core.version} 43 | 44 | 45 | org.springframework.cloud 46 | spring-cloud-task-batch 47 | ${spring.cloud.task.core.version} 48 | 49 | 50 | org.springframework.batch 51 | spring-batch-integration 52 | 53 | 54 | 55 | org.springframework.data 56 | spring-data-hadoop-store 57 | ${spring.data.hadoop.store} 58 | 59 | 60 | org.springframework.data 61 | spring-data-hadoop-boot 62 | 63 | 64 | commons-logging 65 | commons-logging 66 | 67 | 68 | 69 | 70 | 71 | org.springframework.cloud 72 | spring-cloud-deployer-spi 73 | ${spring.cloud.deployer.spi} 74 | 75 | 76 | org.springframework.cloud 77 | spring-cloud-deployer-resource-maven 78 | ${spring.cloud.deployer.resource.maven} 79 | 80 | 81 | org.springframework.boot 82 | spring-boot-configuration-processor 83 | true 84 | 85 | 86 | org.hsqldb 87 | hsqldb 88 | test 89 | 90 | 91 | 92 | org.apache.tomcat 93 | tomcat-jdbc 94 | test 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/main/java/org/springframework/cloud/task/jdbchdfs/common/HdfsTextItemWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.jdbchdfs.common; 18 | 19 | import java.io.IOException; 20 | import java.io.UnsupportedEncodingException; 21 | import java.util.List; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import org.springframework.batch.item.ExecutionContext; 27 | import org.springframework.batch.item.WriteFailedException; 28 | import org.springframework.batch.item.file.transform.LineAggregator; 29 | import org.springframework.batch.item.support.AbstractItemStreamItemWriter; 30 | import org.springframework.data.hadoop.store.DataStoreWriter; 31 | import org.springframework.data.hadoop.store.output.OutputStreamWriter; 32 | 33 | /** 34 | * Writes items as a byte array to a destination as specified by the fsURI. 35 | * 36 | * @author Glenn Renfro 37 | */ 38 | public class HdfsTextItemWriter extends AbstractItemStreamItemWriter { 39 | 40 | private static final String DEFAULT_LINE_SEPARATOR = System.getProperty("line.separator"); 41 | 42 | private volatile String charset = "UTF-8"; 43 | 44 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 45 | 46 | private volatile DataStoreWriter storeWriter; 47 | 48 | private LineAggregator lineAggregator; 49 | 50 | private String lineSeparator = DEFAULT_LINE_SEPARATOR; 51 | 52 | /** 53 | * Aggregates the items to a single string and writes the data out using {@link OutputStreamWriter} as a byte array. 54 | * 55 | * @param list of objects that will be written to the specified fsUri. 56 | */ 57 | @Override 58 | public void write(List list) throws Exception { 59 | storeWriter.write(getItemsAsByte(list)); 60 | } 61 | 62 | @Override 63 | public void update(ExecutionContext executionContext) { 64 | logger.debug("Flushing store writer"); 65 | if (storeWriter != null) { 66 | try { 67 | storeWriter.flush(); 68 | } 69 | catch (IOException e) { 70 | throw new IllegalStateException("Error while flushing store writer", e); 71 | } 72 | } 73 | } 74 | 75 | @Override 76 | public void close() { 77 | logger.debug("Closing item writer"); 78 | if (storeWriter != null) { 79 | try { 80 | storeWriter.flush(); 81 | storeWriter.close(); 82 | } 83 | catch (IOException e) { 84 | throw new IllegalStateException("Error while closing writer", e); 85 | } 86 | finally { 87 | storeWriter = null; 88 | } 89 | } 90 | } 91 | 92 | /** 93 | * Converts the list of items to a byte array. 94 | * 95 | * @param items 96 | * @return string of items 97 | */ 98 | private byte[] getItemsAsByte(List items) { 99 | 100 | StringBuilder lines = new StringBuilder(); 101 | for (T item : items) { 102 | lines.append(lineAggregator.aggregate(item) + lineSeparator); 103 | } 104 | try { 105 | return lines.toString().getBytes(this.charset); 106 | } 107 | catch (UnsupportedEncodingException e) { 108 | throw new WriteFailedException("Could not write data.", e); 109 | } 110 | } 111 | 112 | /** 113 | * Public setter for the {@link LineAggregator}. This will be used to translate the item into a line for output. 114 | * 115 | * @param lineAggregator the {@link LineAggregator} to set 116 | */ 117 | public void setLineAggregator(LineAggregator lineAggregator) { 118 | this.lineAggregator = lineAggregator; 119 | } 120 | 121 | public DataStoreWriter getStoreWriter() { 122 | return storeWriter; 123 | } 124 | 125 | public void setStoreWriter(DataStoreWriter storeWriter) { 126 | this.storeWriter = storeWriter; 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/main/java/org/springframework/cloud/task/jdbchdfs/common/NamedColumnJdbcItemReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.springframework.cloud.task.jdbchdfs.common; 17 | 18 | import java.sql.ResultSet; 19 | import java.sql.SQLException; 20 | 21 | import org.springframework.batch.item.database.JdbcCursorItemReader; 22 | import org.springframework.jdbc.core.RowMapper; 23 | import org.springframework.jdbc.support.JdbcUtils; 24 | 25 | /** 26 | * Reader which reads a row from a database as a delimited string from a 27 | * predefined list of column names. 28 | * 29 | * @author Luke Taylor 30 | * @author Thomas Risberg 31 | * @author Michael Minella 32 | */ 33 | public class NamedColumnJdbcItemReader extends JdbcCursorItemReader { 34 | 35 | private String delimiter; 36 | 37 | @Override 38 | public void afterPropertiesSet() throws Exception { 39 | setRowMapper(new RowMapper() { 40 | @Override 41 | public String mapRow(ResultSet rs, int rowNum) throws SQLException { 42 | StringBuilder builder = new StringBuilder(); 43 | 44 | for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { 45 | builder.append(JdbcUtils.getResultSetValue(rs, i, String.class)) 46 | .append(delimiter); 47 | } 48 | 49 | return builder.substring(0, builder.length() - delimiter.length()); 50 | } 51 | }); 52 | 53 | super.afterPropertiesSet(); 54 | } 55 | 56 | public void setDelimiter(String delimiter) { 57 | this.delimiter = delimiter; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/main/java/org/springframework/cloud/task/jdbchdfs/common/support/JdbcHdfsDataSourceConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.jdbchdfs.common.support; 18 | 19 | import javax.sql.DataSource; 20 | 21 | import org.springframework.beans.factory.annotation.Autowired; 22 | import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder; 23 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 24 | import org.springframework.context.annotation.Bean; 25 | import org.springframework.context.annotation.Primary; 26 | import org.springframework.core.env.Environment; 27 | 28 | /** 29 | * Supports the datasource configurations required for the JdbcHdfs application. 30 | * 31 | * @author Glenn Renfro 32 | */ 33 | @EnableConfigurationProperties({JdbcHdfsDataSourceProperties.class}) 34 | public class JdbcHdfsDataSourceConfiguration { 35 | 36 | @Autowired 37 | private JdbcHdfsDataSourceProperties props; 38 | 39 | @Autowired 40 | private Environment environment; 41 | 42 | @Bean(name="taskDataSource") 43 | @Primary 44 | public DataSource taskDataSource() { 45 | return getDefaultDataSource(); 46 | } 47 | 48 | 49 | @Bean(name="jdbchdfsDataSource") 50 | public DataSource jdbcHdfsDataSource() { 51 | DataSource dataSource; 52 | if(props.getUrl() != null && props.getUsername() != null) { 53 | dataSource = DataSourceBuilder.create().driverClassName(props.getDriverClassName()) 54 | .url(props.getUrl()) 55 | .username(props.getUsername()) 56 | .password(props.getPassword()).build(); 57 | } else { 58 | dataSource = getDefaultDataSource(); 59 | } 60 | return dataSource; 61 | } 62 | 63 | private DataSource getDefaultDataSource() { 64 | return DataSourceBuilder.create().driverClassName(environment.getProperty("spring.datasource.driverClassName")) 65 | .url(environment.getProperty("spring.datasource.url")) 66 | .username(environment.getProperty("spring.datasource.username")) 67 | .password(environment.getProperty("spring.datasource.password")).build(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/main/java/org/springframework/cloud/task/jdbchdfs/common/support/JdbcHdfsDataSourceProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.jdbchdfs.common.support; 18 | 19 | import org.springframework.boot.context.properties.ConfigurationProperties; 20 | 21 | /** 22 | * Establishes the Configuration Properties for the JdbcHdfs task datasources. 23 | * @author Glenn Renfro 24 | */ 25 | @ConfigurationProperties("jdbchdfs.datasource") 26 | public class JdbcHdfsDataSourceProperties { 27 | 28 | /** 29 | * The url of the datasource that will be used by jdbhdfs app to retrieve table input. 30 | */ 31 | private String url; 32 | 33 | /** 34 | * The driver of the datasource that will be used by jdbhdfs app to retrieve table input. 35 | */ 36 | private String driverClassName; 37 | 38 | /** 39 | * The username of the datasource that will be used by jdbhdfs app to retrieve table input. 40 | */ 41 | private String username; 42 | 43 | /** 44 | * The password of the datasource that will be used by jdbhdfs app to retrieve table input. 45 | */ 46 | private String password; 47 | 48 | public String getUrl() { 49 | return url; 50 | } 51 | 52 | public void setUrl(String url) { 53 | this.url = url; 54 | } 55 | 56 | public String getDriverClassName() { 57 | return driverClassName; 58 | } 59 | 60 | public void setDriverClassName(String driverClassName) { 61 | this.driverClassName = driverClassName; 62 | } 63 | 64 | public String getUsername() { 65 | return username; 66 | } 67 | 68 | public void setUsername(String username) { 69 | this.username = username; 70 | } 71 | 72 | public String getPassword() { 73 | return password; 74 | } 75 | 76 | public void setPassword(String password) { 77 | this.password = password; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/test/java/org/springframework/cloud/task/jdbchdfs/common/HdfsTextWriterTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.jdbchdfs.common; 18 | 19 | import java.io.File; 20 | import java.io.FilenameFilter; 21 | import java.io.IOException; 22 | import java.nio.charset.Charset; 23 | import java.nio.file.Files; 24 | import java.nio.file.Paths; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | import org.junit.After; 29 | import org.junit.Assert; 30 | import org.junit.Before; 31 | import org.junit.Test; 32 | 33 | import org.springframework.util.FileSystemUtils; 34 | 35 | import static org.hamcrest.CoreMatchers.equalTo; 36 | import static org.junit.Assert.assertNotNull; 37 | 38 | /** 39 | * @author Glenn Renfro 40 | */ 41 | public class HdfsTextWriterTests { 42 | 43 | public static final String ROW_1 = "Hello world you rock"; 44 | public static final String ROW_2 = "And Rock on"; 45 | public static final String ROW_TERMINATOR = "\n"; 46 | 47 | private String tmpDir; 48 | 49 | private HdfsTextItemWriter writer; 50 | 51 | JdbcHdfsTaskProperties props; 52 | 53 | 54 | @Before 55 | public void setup() throws Exception { 56 | tmpDir = System.getProperty("java.io.tmpdir") + "/jdbchdfs-task"; 57 | File file = new File(tmpDir); 58 | if (file.exists()) { 59 | FileSystemUtils.deleteRecursively(file); 60 | } 61 | props = new JdbcHdfsTaskProperties(); 62 | props.setFsUri("file:///"); 63 | props.setDirectory(tmpDir); 64 | props.setFileName("dataWriterBasicTest"); 65 | } 66 | 67 | @After 68 | public void tearDown() { 69 | if (writer != null) { 70 | writer.close(); 71 | } 72 | File file = new File(tmpDir); 73 | if (file.exists()) { 74 | FileSystemUtils.deleteRecursively(file); 75 | } 76 | } 77 | 78 | @Test 79 | public void testDataWriterBasic() throws Exception { 80 | props.setRollover(100); 81 | HdfsTextItemWriterFactory factory = new HdfsTextItemWriterFactory(new org.apache.hadoop.conf.Configuration(), props, "part1"); 82 | writer = factory.getObject(); 83 | List list = new ArrayList(); 84 | list.add(ROW_1); 85 | writer.write(list); 86 | list = new ArrayList(); 87 | list.add(ROW_2); 88 | writer.write(list); 89 | writer.close(); 90 | checkPartitionInstance(tmpDir, "-0.csv", ROW_1 + ROW_TERMINATOR + ROW_2 + ROW_TERMINATOR); 91 | } 92 | 93 | @Test 94 | public void testDataWriterRollover() throws Exception { 95 | props.setRollover(1); 96 | HdfsTextItemWriterFactory factory = new HdfsTextItemWriterFactory(new org.apache.hadoop.conf.Configuration(), props, "part1"); 97 | writer = factory.getObject(); 98 | List list = new ArrayList(); 99 | list.add(ROW_1); 100 | writer.write(list); 101 | list = new ArrayList(); 102 | list.add(ROW_2); 103 | writer.write(list); 104 | writer.close(); 105 | checkPartitionInstance(tmpDir, "-0.csv", ROW_1 + ROW_TERMINATOR); 106 | checkPartitionInstance(tmpDir, "-1.csv", ROW_2 + ROW_TERMINATOR); 107 | } 108 | 109 | private void checkPartitionInstance(String testDir, final String fileSuffix, String expectedData) throws Exception { 110 | File testOutput = new File(testDir); 111 | Assert.assertTrue(testOutput.exists()); 112 | File[] partitionFile = testOutput.listFiles(new FilenameFilter() { 113 | 114 | @Override 115 | public boolean accept(File dir, String name) { 116 | return name.endsWith(fileSuffix); 117 | } 118 | 119 | }); 120 | Assert.assertTrue(partitionFile.length > 0); 121 | File dataFile = partitionFile[0]; 122 | assertNotNull(dataFile); 123 | Assert.assertThat(readFile(dataFile.getPath(), Charset.forName("UTF-8")), equalTo(expectedData)); 124 | } 125 | 126 | private String readFile(String path, Charset encoding) throws IOException { 127 | byte[] encoded = Files.readAllBytes(Paths.get(path)); 128 | return new String(encoded, encoding); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/test/java/org/springframework/cloud/task/jdbchdfs/common/JdbcHdfsDataSourcePropertiesTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.jdbchdfs.common; 18 | 19 | import org.junit.Test; 20 | 21 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 22 | import org.springframework.cloud.task.jdbchdfs.common.support.JdbcHdfsDataSourceProperties; 23 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 24 | import org.springframework.context.annotation.Configuration; 25 | 26 | import static org.junit.Assert.assertEquals; 27 | import static org.junit.Assert.assertFalse; 28 | import static org.junit.Assert.assertNull; 29 | 30 | /** 31 | * Verifies that configurations for the data source properties are 32 | * correct. 33 | * 34 | * @author Glenn Renfro 35 | */ 36 | public class JdbcHdfsDataSourcePropertiesTests { 37 | private static final String DRIVER_CLASS_ONE = "driverClassOne"; 38 | private static final String USER_NAME_ONE= "userNameOne"; 39 | private static final String PASSWORD_ONE= "passwordOne"; 40 | private static final String URL_ONE= "urlOne"; 41 | 42 | 43 | /** 44 | * Verify that the defaults for the properties are properly set. 45 | */ 46 | @Test 47 | public void defaultPropertyTests() { 48 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 49 | context.register(Conf.class); 50 | context.refresh(); 51 | JdbcHdfsDataSourceProperties properties = context.getBean(JdbcHdfsDataSourceProperties.class); 52 | assertNull(properties.getDriverClassName()); 53 | assertNull(properties.getPassword()); 54 | assertNull(properties.getUrl()); 55 | assertNull(properties.getUsername()); 56 | } 57 | 58 | /** 59 | * Verify setters and getters are working properly. 60 | */ 61 | @Test 62 | public void testSetters() { 63 | JdbcHdfsDataSourceProperties properties = new JdbcHdfsDataSourceProperties(); 64 | properties.setDriverClassName(DRIVER_CLASS_ONE); 65 | properties.setUsername(USER_NAME_ONE); 66 | properties.setPassword(PASSWORD_ONE); 67 | properties.setUrl(URL_ONE); 68 | assertEquals(DRIVER_CLASS_ONE, properties.getDriverClassName()); 69 | assertEquals(USER_NAME_ONE, properties.getUsername()); 70 | assertEquals(PASSWORD_ONE, properties.getPassword()); 71 | assertEquals(URL_ONE, properties.getUrl()); 72 | 73 | } 74 | 75 | @Configuration 76 | @EnableConfigurationProperties(JdbcHdfsDataSourceProperties.class) 77 | static class Conf { 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/test/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 the original author or authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | spring.cloud.task.closecontext.enable:false 18 | -------------------------------------------------------------------------------- /spring-cloud-task-jdbchdfs-common/src/test/resources/schema.sql: -------------------------------------------------------------------------------- 1 | -- Run by default by Boot infrastructure 2 | 3 | create table test( 4 | id bigint, 5 | name varchar (20) 6 | ); 7 | insert into test values (1, 'Bob'); 8 | insert into test values (2, 'Jane'); 9 | insert into test values (3, 'John'); 10 | -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.springframework.cloud.task.app 5 | spring-cloud-task-app-starters 6 | 1.0.2.BUILD-SNAPSHOT 7 | ../ 8 | 9 | 4.0.0 10 | 11 | spring-cloud-task-sqoop-common 12 | jar 13 | spring-cloud-task-sqoop-common 14 | 15 | 16 | 17 | org.hibernate 18 | hibernate-validator 19 | 20 | 21 | org.hsqldb 22 | hsqldb 23 | 1.8.0.10 24 | test 25 | 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-test 34 | test 35 | 36 | 37 | 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-jar-plugin 43 | 44 | 45 | 46 | test-jar 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/src/main/java/org/springframework/cloud/task/sqoop/common/SqoopCommonRunnerUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.sqoop.common; 18 | 19 | import java.util.List; 20 | 21 | import org.springframework.util.StringUtils; 22 | 23 | /** 24 | * Common utility methods for configuraing a Sqoop CommandLineRunner. 25 | * 26 | * @author Thomas Risberg 27 | */ 28 | public class SqoopCommonRunnerUtils { 29 | 30 | public static void setConnectProperties(SqoopCommonTaskProperties props, List sqoopArguments) { 31 | if (StringUtils.hasText(props.getConnect())) { 32 | sqoopArguments.add("--connect"); 33 | sqoopArguments.add(props.getConnect()); 34 | } 35 | if (StringUtils.hasText(props.getUsername())) { 36 | sqoopArguments.add("--username"); 37 | sqoopArguments.add(props.getUsername()); 38 | } 39 | if (StringUtils.hasText(props.getPassword())) { 40 | sqoopArguments.add("--password"); 41 | sqoopArguments.add(props.getPassword()); 42 | } 43 | if (StringUtils.hasText(props.getPasswordFile())) { 44 | sqoopArguments.add("--password-file"); 45 | sqoopArguments.add(props.getPasswordFile()); 46 | } 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/src/main/java/org/springframework/cloud/task/sqoop/common/SqoopCommonTaskProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.sqoop.common; 18 | 19 | import javax.validation.constraints.AssertTrue; 20 | 21 | import org.springframework.boot.context.properties.ConfigurationProperties; 22 | import org.springframework.util.StringUtils; 23 | 24 | /** 25 | * Configuration properties to be used by the SqoopJobRunner. 26 | * 27 | * @author Thomas Risberg 28 | */ 29 | @ConfigurationProperties 30 | public class SqoopCommonTaskProperties { 31 | 32 | /** 33 | * The Sqoop command to execute. 34 | */ 35 | private String command; 36 | 37 | /** 38 | * The Sqoop connect string (JDBC URL) to use. 39 | */ 40 | private String connect; 41 | 42 | /** 43 | * The database username to use. 44 | */ 45 | String username; 46 | 47 | /** 48 | * The database password to use. 49 | */ 50 | String password; 51 | 52 | /** 53 | * The database password-file to use. 54 | */ 55 | String passwordFile; 56 | 57 | /** 58 | * The Sqoop specific job arguments as a complete String, passed as is to the command job. 59 | */ 60 | private String toolArgs; 61 | 62 | public String getCommand() { 63 | return command; 64 | } 65 | 66 | public void setCommand(String command) { 67 | this.command = command; 68 | } 69 | 70 | public String getConnect() { 71 | return connect; 72 | } 73 | 74 | public void setConnect(String connect) { 75 | this.connect = connect; 76 | } 77 | 78 | public String getUsername() { 79 | return username; 80 | } 81 | 82 | public void setUsername(String username) { 83 | this.username = username; 84 | } 85 | 86 | public String getPassword() { 87 | return password; 88 | } 89 | 90 | public void setPassword(String password) { 91 | this.password = password; 92 | } 93 | 94 | public String getPasswordFile() { 95 | return passwordFile; 96 | } 97 | 98 | public void setPasswordFile(String passwordFile) { 99 | this.passwordFile = passwordFile; 100 | } 101 | 102 | public String getToolArgs() { 103 | return toolArgs; 104 | } 105 | 106 | public void setToolArgs(String toolArgs) { 107 | this.toolArgs = toolArgs; 108 | } 109 | 110 | @AssertTrue(message = "Unsupported command, the Sqoop Task currently only supports 'import' and 'export'.") 111 | private boolean isSupportedCommand() { 112 | return (StringUtils.isEmpty(command) || command.equals("import") || command.equals("export")); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/src/test/java/org/springframework/cloud/task/sqoop/common/SqoopCommonTaskPropertiesTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.sqoop.common; 18 | 19 | import org.junit.Test; 20 | 21 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 22 | import org.springframework.boot.test.EnvironmentTestUtils; 23 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 24 | import org.springframework.context.annotation.Configuration; 25 | 26 | import static org.hamcrest.CoreMatchers.equalTo; 27 | import static org.hamcrest.MatcherAssert.assertThat; 28 | 29 | /** 30 | * @author Thomas Risberg 31 | */ 32 | public class SqoopCommonTaskPropertiesTests { 33 | 34 | @Test 35 | public void testCommandCanBeCustomized() { 36 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 37 | EnvironmentTestUtils.addEnvironment(context, "command: export"); 38 | context.register(Conf.class); 39 | context.refresh(); 40 | SqoopCommonTaskProperties properties = context.getBean(SqoopCommonTaskProperties.class); 41 | assertThat(properties.getCommand(), equalTo("export")); 42 | } 43 | 44 | @Test 45 | public void testConnectCanBeCustomized() { 46 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 47 | EnvironmentTestUtils.addEnvironment(context, "connect: jdbc:foo://bar:1234"); 48 | context.register(Conf.class); 49 | context.refresh(); 50 | SqoopCommonTaskProperties properties = context.getBean(SqoopCommonTaskProperties.class); 51 | assertThat(properties.getConnect(), equalTo("jdbc:foo://bar:1234")); 52 | } 53 | 54 | @Test 55 | public void testToolArgsCanBeCustomized() { 56 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 57 | EnvironmentTestUtils.addEnvironment(context, "tool-args: --table foo"); 58 | context.register(Conf.class); 59 | context.refresh(); 60 | SqoopCommonTaskProperties properties = context.getBean(SqoopCommonTaskProperties.class); 61 | assertThat(properties.getToolArgs(), equalTo("--table foo")); 62 | } 63 | 64 | @Configuration 65 | @EnableConfigurationProperties(SqoopCommonTaskProperties.class) 66 | static class Conf { 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/src/test/java/org/springframework/cloud/task/sqoop/common/SqoopToolDatabaseConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.task.sqoop.common; 18 | 19 | import java.io.IOException; 20 | import java.sql.DriverManager; 21 | import java.sql.SQLException; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | import org.hsqldb.Server; 26 | import org.hsqldb.persist.HsqlProperties; 27 | 28 | import org.springframework.context.annotation.Bean; 29 | import org.springframework.context.annotation.Configuration; 30 | import org.springframework.util.SocketUtils; 31 | 32 | /** 33 | * @author Thomas Risberg 34 | */ 35 | @Configuration 36 | public class SqoopToolDatabaseConfiguration { 37 | 38 | private static final Log logger = LogFactory.getLog(SqoopToolDatabaseConfiguration.class); 39 | 40 | @Bean(destroyMethod = "stop") 41 | public Server databaseServer() throws SQLException, IOException { 42 | DriverManager.registerDriver(new org.hsqldb.jdbcDriver()); 43 | int hsqldbPort = SocketUtils.findAvailableTcpPort(10000); 44 | System.setProperty("db.server.port", Integer.toString(hsqldbPort)); 45 | logger.info("Database is using port: " + Integer.toString(hsqldbPort)); 46 | HsqlProperties configProps = new HsqlProperties(); 47 | configProps.setProperty("server.port", hsqldbPort); 48 | configProps.setProperty("server.database.0", "file:target/db/test"); 49 | configProps.setProperty("server.dbname.0", "test"); 50 | Server server = new org.hsqldb.Server(); 51 | server.setLogWriter(null); 52 | server.setErrWriter(null); 53 | server.setRestartOnShutdown(false); 54 | server.setNoSystemExit(true); 55 | server.setProperties(configProps); 56 | server.start(); 57 | return server; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/src/test/resources/sqoop-test-data.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO FOO VALUES(1, 'The Beatles', 'Let it be'); 2 | INSERT INTO FOO VALUES(2, 'The Doors', 'Light my fire'); -------------------------------------------------------------------------------- /spring-cloud-task-sqoop-common/src/test/resources/sqoop-test-schema-ddl.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS FOO; 2 | DROP TABLE IF EXISTS BAR; 3 | CREATE TABLE FOO (ID smallint NOT NULL, ARTIST varchar(255), TITLE varchar(255), PRIMARY KEY (ID)); 4 | CREATE TABLE BAR (ID smallint NOT NULL, ARTIST varchar(255), TITLE varchar(255), PRIMARY KEY (ID)); --------------------------------------------------------------------------------