├── .gitignore ├── .mvn ├── jvm.config ├── maven.config └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .settings.xml ├── .travis.yml ├── LICENSE ├── NOTICE ├── README.adoc ├── mvnw ├── mvnw.cmd ├── pom.xml ├── spring-cloud-deployer-yarn-appdeployerappmaster ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── deployer │ │ └── spi │ │ └── yarn │ │ └── appdeployer │ │ ├── StreamAppmaster.java │ │ ├── StreamAppmasterApplication.java │ │ └── StreamAppmasterProperties.java │ └── resources │ └── application.yml ├── spring-cloud-deployer-yarn-autoconfig ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── deployer │ │ └── spi │ │ └── yarn │ │ └── autoconfigure │ │ └── YarnDeployerAutoConfiguration.java │ └── resources │ └── META-INF │ └── spring.factories ├── spring-cloud-deployer-yarn-build-tests ├── pom.xml └── src │ ├── main │ └── assembly │ │ └── assembly.xml │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── dataflow │ │ └── yarn │ │ └── buildtests │ │ ├── AbstractCliBootYarnClusterTests.java │ │ ├── AppDeployerIT.java │ │ └── TaskLauncherIT.java │ └── resources │ ├── log4j.properties │ └── yarn-site.xml ├── spring-cloud-deployer-yarn-tasklauncherappmaster ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── deployer │ │ └── spi │ │ └── yarn │ │ └── tasklauncher │ │ ├── TaskAppmaster.java │ │ ├── TaskAppmasterApplication.java │ │ └── TaskAppmasterProperties.java │ └── resources │ └── application.yml ├── spring-cloud-deployer-yarn ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── deployer │ │ │ └── spi │ │ │ └── yarn │ │ │ ├── AbstractDeployerStateMachine.java │ │ │ ├── AppDeployerStateMachine.java │ │ │ ├── DefaultYarnCloudAppService.java │ │ │ ├── DeployerApplicationYarnClient.java │ │ │ ├── StreamClustersInfoReportData.java │ │ │ ├── TaskLauncherStateMachine.java │ │ │ ├── YarnAppDeployer.java │ │ │ ├── YarnCloudAppService.java │ │ │ ├── YarnCloudAppServiceApplication.java │ │ │ ├── YarnDeployerProperties.java │ │ │ └── YarnTaskLauncher.java │ └── resources │ │ ├── appdeployer-model.di │ │ ├── appdeployer-model.notation │ │ ├── appdeployer-model.uml │ │ ├── stream.yml │ │ ├── task.yml │ │ ├── tasklauncher-model.di │ │ ├── tasklauncher-model.notation │ │ └── tasklauncher-model.uml │ └── test │ └── java │ └── org │ └── springframework │ └── cloud │ └── deployer │ └── spi │ └── yarn │ ├── AbstractStateMachineTests.java │ ├── AppDeployerStateMachineTests.java │ └── TaskLauncherStateMachineTests.java └── spring-cloud-starter-deployer-yarn └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .#* 3 | *# 4 | *.sw* 5 | _site/ 6 | .factorypath 7 | .gradletasknamecache 8 | .DS_Store 9 | /application.yml 10 | /application.properties 11 | asciidoctor.css 12 | atlassian-ide-plugin.xml 13 | build/ 14 | dump.rdb 15 | out 16 | spring-shell.log 17 | target/ 18 | test-output 19 | spring-*/bin 20 | 21 | # Eclipse artifacts, including WTP generated manifests 22 | .classpath 23 | .project 24 | .settings/ 25 | .springBeans 26 | spring-*/src/main/java/META-INF/MANIFEST.MF 27 | 28 | # IDEA artifacts and output dirs 29 | *.iml 30 | *.ipr 31 | *.iws 32 | .idea/* 33 | /.idea/ 34 | -------------------------------------------------------------------------------- /.mvn/jvm.config: -------------------------------------------------------------------------------- 1 | -Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom -------------------------------------------------------------------------------- /.mvn/maven.config: -------------------------------------------------------------------------------- 1 | -DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -s .settings.xml -P spring 2 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-deployer-yarn/175449d482bd9a45b79669cdb0b80dca569f8c8c/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.3/apache-maven-3.3.3-bin.zip -------------------------------------------------------------------------------- /.settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | repo.spring.io 6 | ${env.CI_DEPLOY_USERNAME} 7 | ${env.CI_DEPLOY_PASSWORD} 8 | 9 | 10 | 11 | 12 | spring 13 | true 14 | 15 | 16 | spring-snapshots 17 | Spring Snapshots 18 | http://repo.spring.io/libs-snapshot-local 19 | 20 | true 21 | 22 | 23 | 24 | spring-milestones 25 | Spring Milestones 26 | http://repo.spring.io/libs-milestone-local 27 | 28 | false 29 | 30 | 31 | 32 | spring-releases 33 | Spring Releases 34 | http://repo.spring.io/release 35 | 36 | false 37 | 38 | 39 | 40 | 41 | 42 | spring-snapshots 43 | Spring Snapshots 44 | http://repo.spring.io/libs-snapshot-local 45 | 46 | true 47 | 48 | 49 | 50 | spring-milestones 51 | Spring Milestones 52 | http://repo.spring.io/libs-milestone-local 53 | 54 | false 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | cache: 3 | directories: 4 | - $HOME/.m2 5 | language: java 6 | jdk: 7 | - oraclejdk8 8 | services: 9 | - redis-server 10 | install: true 11 | script: 12 | - '[ "${TRAVIS_PULL_REQUEST}" != "false" ] || ./mvnw package -DskipITs -Pfull -U -Dmaven.test.redirectTestOutputToFile=false' 13 | - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] || ./mvnw package -DskipTests -U -Dmaven.test.redirectTestOutputToFile=false' 14 | 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | spring-cloud-deployer-yarn 2 | 3 | Copyright (c) 2016-Present Pivotal Software, Inc. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | # spring-cloud-deployer-yarn is no longer actively maintained by VMware, Inc. 2 | 3 | == Spring Cloud Deployer for Apache Yarn 4 | 5 | 6 | The Spring Cloud Data Flow team have been happy stewards of this project, but we are focusing our efforts towards Local, Kubernetes and Cloud Foundry implementations going forward, so the Spring Cloud Data Flow team will no longer maintain it. 7 | 8 | However, we are looking to donate the project to the community. If you are interested in forking the project to continue to develop and maintaining it, we would be happy to cross-link and promote your forked project. Please ping us at https://gitter.im/spring-cloud/spring-cloud-dataflow[Gitter] to discuss it. 9 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # 58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look 59 | # for the new JDKs provided by Oracle. 60 | # 61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then 62 | # 63 | # Apple JDKs 64 | # 65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home 66 | fi 67 | 68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then 69 | # 70 | # Apple JDKs 71 | # 72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 73 | fi 74 | 75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then 76 | # 77 | # Oracle JDKs 78 | # 79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 80 | fi 81 | 82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then 83 | # 84 | # Apple JDKs 85 | # 86 | export JAVA_HOME=`/usr/libexec/java_home` 87 | fi 88 | ;; 89 | esac 90 | 91 | if [ -z "$JAVA_HOME" ] ; then 92 | if [ -r /etc/gentoo-release ] ; then 93 | JAVA_HOME=`java-config --jre-home` 94 | fi 95 | fi 96 | 97 | if [ -z "$M2_HOME" ] ; then 98 | ## resolve links - $0 may be a link to maven's home 99 | PRG="$0" 100 | 101 | # need this for relative symlinks 102 | while [ -h "$PRG" ] ; do 103 | ls=`ls -ld "$PRG"` 104 | link=`expr "$ls" : '.*-> \(.*\)$'` 105 | if expr "$link" : '/.*' > /dev/null; then 106 | PRG="$link" 107 | else 108 | PRG="`dirname "$PRG"`/$link" 109 | fi 110 | done 111 | 112 | saveddir=`pwd` 113 | 114 | M2_HOME=`dirname "$PRG"`/.. 115 | 116 | # make it fully qualified 117 | M2_HOME=`cd "$M2_HOME" && pwd` 118 | 119 | cd "$saveddir" 120 | # echo Using m2 at $M2_HOME 121 | fi 122 | 123 | # For Cygwin, ensure paths are in UNIX format before anything is touched 124 | if $cygwin ; then 125 | [ -n "$M2_HOME" ] && 126 | M2_HOME=`cygpath --unix "$M2_HOME"` 127 | [ -n "$JAVA_HOME" ] && 128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 129 | [ -n "$CLASSPATH" ] && 130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 131 | fi 132 | 133 | # For Migwn, ensure paths are in UNIX format before anything is touched 134 | if $mingw ; then 135 | [ -n "$M2_HOME" ] && 136 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 137 | [ -n "$JAVA_HOME" ] && 138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 139 | # TODO classpath? 140 | fi 141 | 142 | if [ -z "$JAVA_HOME" ]; then 143 | javaExecutable="`which javac`" 144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 145 | # readlink(1) is not available as standard on Solaris 10. 146 | readLink=`which readlink` 147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 148 | if $darwin ; then 149 | javaHome="`dirname \"$javaExecutable\"`" 150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 151 | else 152 | javaExecutable="`readlink -f \"$javaExecutable\"`" 153 | fi 154 | javaHome="`dirname \"$javaExecutable\"`" 155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 156 | JAVA_HOME="$javaHome" 157 | export JAVA_HOME 158 | fi 159 | fi 160 | fi 161 | 162 | if [ -z "$JAVACMD" ] ; then 163 | if [ -n "$JAVA_HOME" ] ; then 164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 165 | # IBM's JDK on AIX uses strange locations for the executables 166 | JAVACMD="$JAVA_HOME/jre/sh/java" 167 | else 168 | JAVACMD="$JAVA_HOME/bin/java" 169 | fi 170 | else 171 | JAVACMD="`which java`" 172 | fi 173 | fi 174 | 175 | if [ ! -x "$JAVACMD" ] ; then 176 | echo "Error: JAVA_HOME is not defined correctly." >&2 177 | echo " We cannot execute $JAVACMD" >&2 178 | exit 1 179 | fi 180 | 181 | if [ -z "$JAVA_HOME" ] ; then 182 | echo "Warning: JAVA_HOME environment variable is not set." 183 | fi 184 | 185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 186 | 187 | # For Cygwin, switch paths to Windows format before running java 188 | if $cygwin; then 189 | [ -n "$M2_HOME" ] && 190 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 191 | [ -n "$JAVA_HOME" ] && 192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 193 | [ -n "$CLASSPATH" ] && 194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 195 | fi 196 | 197 | # traverses directory structure from process work directory to filesystem root 198 | # first directory with .mvn subdirectory is considered project base directory 199 | find_maven_basedir() { 200 | local basedir=$(pwd) 201 | local wdir=$(pwd) 202 | while [ "$wdir" != '/' ] ; do 203 | if [ -d "$wdir"/.mvn ] ; then 204 | basedir=$wdir 205 | break 206 | fi 207 | wdir=$(cd "$wdir/.."; pwd) 208 | done 209 | echo "${basedir}" 210 | } 211 | 212 | # concatenates all lines of a file 213 | concat_lines() { 214 | if [ -f "$1" ]; then 215 | echo "$(tr -s '\n' ' ' < "$1")" 216 | fi 217 | } 218 | 219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} 220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 221 | 222 | # Provide a "standardized" way to retrieve the CLI args that will 223 | # work with both Windows and non-Windows executions. 224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 225 | export MAVEN_CMD_LINE_ARGS 226 | 227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 228 | 229 | exec "$JAVACMD" \ 230 | $MAVEN_OPTS \ 231 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 233 | ${WRAPPER_LAUNCHER} "$@" 234 | 235 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | set MAVEN_CMD_LINE_ARGS=%* 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | 121 | set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" 122 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 123 | 124 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% 125 | if ERRORLEVEL 1 goto error 126 | goto end 127 | 128 | :error 129 | set ERROR_CODE=1 130 | 131 | :end 132 | @endlocal & set ERROR_CODE=%ERROR_CODE% 133 | 134 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 135 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 136 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 137 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 138 | :skipRcPost 139 | 140 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 141 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 142 | 143 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 144 | 145 | exit /B %ERROR_CODE% 146 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | org.springframework.cloud 6 | spring-cloud-deployer-yarn-parent 7 | 1.2.3.BUILD-SNAPSHOT 8 | pom 9 | 10 | 11 | org.springframework.cloud 12 | spring-cloud-deployer-parent 13 | 1.2.1.RELEASE 14 | 15 | 16 | 17 | 18 | 1.2.1.RELEASE 19 | 2.4.0.RELEASE 20 | 1.1.0.RELEASE 21 | 22 | 23 | 24 | spring-cloud-deployer-yarn-appdeployerappmaster 25 | spring-cloud-deployer-yarn-tasklauncherappmaster 26 | spring-cloud-deployer-yarn 27 | spring-cloud-deployer-yarn-autoconfig 28 | spring-cloud-deployer-yarn-build-tests 29 | spring-cloud-starter-deployer-yarn 30 | 31 | 32 | 33 | 34 | 35 | org.springframework.boot 36 | spring-boot-starter-parent 37 | ${spring-boot.version} 38 | pom 39 | import 40 | 41 | 42 | 43 | org.springframework.data 44 | spring-yarn-boot 45 | ${spring-data-hadoop.version} 46 | 47 | 48 | org.springframework.data 49 | spring-yarn-boot-cli 50 | ${spring-data-hadoop.version} 51 | 52 | 53 | org.springframework.cloud 54 | spring-cloud-deployer-spi 55 | ${spring-cloud-deployer.version} 56 | 57 | 58 | org.springframework.cloud 59 | spring-cloud-deployer-resource-support 60 | ${spring-cloud-deployer.version} 61 | 62 | 63 | org.springframework.cloud 64 | spring-cloud-deployer-resource-maven 65 | ${spring-cloud-deployer.version} 66 | 67 | 68 | org.springframework.cloud 69 | spring-cloud-dataflow-server-core 70 | ${spring-cloud-dataflow.version} 71 | 72 | 73 | org.springframework.statemachine 74 | spring-statemachine-core 75 | ${spring-statemachine.version} 76 | 77 | 78 | org.springframework.statemachine 79 | spring-statemachine-uml 80 | ${spring-statemachine.version} 81 | 82 | 83 | org.springframework.statemachine 84 | spring-statemachine-test 85 | ${spring-statemachine.version} 86 | test 87 | 88 | 89 | org.springframework.data 90 | spring-yarn-boot-test 91 | ${spring-data-hadoop.version} 92 | test 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-appdeployerappmaster/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | org.springframework.cloud 6 | spring-cloud-deployer-yarn-appdeployerappmaster 7 | 1.2.3.BUILD-SNAPSHOT 8 | 9 | org.springframework.cloud 10 | spring-cloud-deployer-yarn-parent 11 | 1.2.3.BUILD-SNAPSHOT 12 | 13 | 14 | 15 | 16 | org.springframework.data 17 | spring-yarn-boot 18 | 19 | 20 | javax.servlet 21 | servlet-api 22 | 23 | 24 | org.slf4j 25 | slf4j-log4j12 26 | 27 | 28 | 29 | 30 | org.springframework.boot 31 | spring-boot-starter-actuator 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-web 36 | 37 | 38 | 39 | 40 | 41 | 42 | src/main/resources 43 | true 44 | 45 | application.yml 46 | 47 | 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-resources-plugin 53 | 54 | 55 | org.springframework.boot 56 | spring-boot-maven-plugin 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-appdeployerappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/appdeployer/StreamAppmaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2017 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.appdeployer; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.Iterator; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.Map.Entry; 25 | 26 | import org.apache.commons.logging.Log; 27 | import org.apache.commons.logging.LogFactory; 28 | import org.apache.hadoop.yarn.api.records.Container; 29 | import org.apache.hadoop.yarn.api.records.ContainerId; 30 | import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; 31 | import org.apache.hadoop.yarn.api.records.ContainerStatus; 32 | import org.apache.hadoop.yarn.api.records.LocalResource; 33 | import org.apache.hadoop.yarn.api.records.LocalResourceType; 34 | import org.springframework.beans.factory.annotation.Autowired; 35 | import org.springframework.util.StringUtils; 36 | import org.springframework.yarn.am.ContainerLauncherInterceptor; 37 | import org.springframework.yarn.am.cluster.ContainerCluster; 38 | import org.springframework.yarn.am.cluster.ManagedContainerClusterAppmaster; 39 | import org.springframework.yarn.am.container.AbstractLauncher; 40 | import org.springframework.yarn.am.grid.GridMember; 41 | import org.springframework.yarn.am.grid.support.ProjectionData; 42 | import org.springframework.yarn.fs.LocalResourcesFactoryBean; 43 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.CopyEntry; 44 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.TransferEntry; 45 | import org.springframework.yarn.fs.ResourceLocalizer; 46 | import org.springframework.yarn.listener.ContainerMonitorListener; 47 | 48 | /** 49 | * Custom yarn appmaster tweaking container launch settings. 50 | * 51 | * @author Janne Valkealahti 52 | * 53 | */ 54 | public class StreamAppmaster extends ManagedContainerClusterAppmaster { 55 | 56 | private final static Log log = LogFactory.getLog(StreamAppmaster.class); 57 | private final Map artifactLocalizers = new HashMap<>(); 58 | private final ContainerIndexTracker indexTracker = new ContainerIndexTracker(); 59 | 60 | /** ContainerId to ContainerCluster id map */ 61 | private final Map containerIdMap = new HashMap<>(); 62 | 63 | @Autowired 64 | private StreamAppmasterProperties streamAppmasterProperties; 65 | 66 | @Override 67 | protected void onInit() throws Exception { 68 | super.onInit(); 69 | 70 | // TODO: we want to have a proper support in base classes to gracefully 71 | // shutdown appmaster when it has nothing to do. this trick 72 | // here is solely a workaround not being able to access internal 73 | // structures of base classes. this is pretty much all we can do 74 | // from a subclass. 75 | // potentially we want to make it configurable with a grace period, etc. 76 | getMonitor().addContainerMonitorStateListener(new ContainerMonitorListener() { 77 | 78 | @Override 79 | public void state(ContainerMonitorState state) { 80 | if (log.isDebugEnabled()) { 81 | log.info("Received monitor state " + state + " and container clusters size is " + getContainerClusters().size()); 82 | } 83 | if (state.getRunning() == 0 && getContainerClusters().size() == 0) { 84 | // this state is valid at start but we know it's not gonna 85 | // get called until we have had at least one container running 86 | log.info("No running containers and no container clusters, initiate app shutdown"); 87 | notifyCompleted(); 88 | } 89 | } 90 | }); 91 | 92 | if (getLauncher() instanceof AbstractLauncher) { 93 | ((AbstractLauncher)getLauncher()).addInterceptor(new IndexAddingContainerLauncherInterceptor()); 94 | } 95 | } 96 | 97 | @Override 98 | public ContainerCluster createContainerCluster(String clusterId, String clusterDef, ProjectionData projectionData, 99 | Map extraProperties) { 100 | log.info("intercept createContainerCluster " + clusterId); 101 | String artifactPath = streamAppmasterProperties.getArtifact(); 102 | try { 103 | LocalResourcesFactoryBean lrfb = new LocalResourcesFactoryBean(); 104 | lrfb.setConfiguration(getConfiguration()); 105 | 106 | String containerArtifact = (String) extraProperties.get("containerArtifact"); 107 | TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifactPath + "/" + containerArtifact, false); 108 | ArrayList hdfsEntries = new ArrayList(); 109 | hdfsEntries.add(te); 110 | lrfb.setHdfsEntries(hdfsEntries); 111 | lrfb.setCopyEntries(new ArrayList()); 112 | lrfb.afterPropertiesSet(); 113 | ResourceLocalizer rl = lrfb.getObject(); 114 | log.info("Adding localizer for " + clusterId + " / " + rl); 115 | artifactLocalizers.put(clusterId, rl); 116 | } catch (Exception e) { 117 | log.error("Error creating localizer", e); 118 | } 119 | 120 | return super.createContainerCluster(clusterId, clusterDef, projectionData, extraProperties); 121 | } 122 | 123 | @Override 124 | protected List onContainerLaunchCommands(Container container, ContainerCluster cluster, 125 | List commands) { 126 | log.info("onContainerLaunchCommands commands=" + StringUtils.collectionToCommaDelimitedString(commands)); 127 | ArrayList list = new ArrayList(); 128 | Map extraProperties = cluster.getExtraProperties(); 129 | String artifact = (String) extraProperties.get("containerArtifact"); 130 | 131 | for (String command : commands) { 132 | if (command.contains("placeholder.jar")) { 133 | list.add(command.replace("placeholder.jar", artifact)); 134 | } else { 135 | list.add(command); 136 | } 137 | } 138 | 139 | if (extraProperties != null) { 140 | for (Entry entry : extraProperties.entrySet()) { 141 | if (entry.getKey().startsWith("containerArg")) { 142 | log.info("onContainerLaunchCommands adding command=--" + entry.getValue().toString()); 143 | list.add(Math.max(list.size() - 2, 0), "--" + entry.getValue().toString()); 144 | } 145 | } 146 | } 147 | return list; 148 | } 149 | 150 | @Override 151 | protected Map buildLocalizedResources(ContainerCluster cluster) { 152 | Map resources = super.buildLocalizedResources(cluster); 153 | ResourceLocalizer rl = artifactLocalizers.get(cluster.getId()); 154 | log.info("Localizer for " + cluster.getId() + " is " + rl); 155 | resources.putAll(rl.getResources()); 156 | return resources; 157 | } 158 | 159 | @Override 160 | protected void onContainerCompleted(ContainerStatus status) { 161 | super.onContainerCompleted(status); 162 | String containerClusterId = containerIdMap.get(status.getContainerId()); 163 | if (containerClusterId != null) { 164 | synchronized (indexTracker) { 165 | indexTracker.freeIndex(status.getContainerId(), containerClusterId); 166 | } 167 | } 168 | } 169 | 170 | private ContainerCluster findContainerClusterByContainerId(ContainerId containerId) { 171 | for (Entry entry : getContainerClusters().entrySet()) { 172 | for (GridMember member : entry.getValue().getGridProjection().getMembers()) { 173 | if (member.getId().equals(containerId)) { 174 | return entry.getValue(); 175 | } 176 | } 177 | } 178 | return null; 179 | } 180 | 181 | /** 182 | * Interceptor adding INSTANCE_INDEX as env variable based on ContainerIndexTracker. 183 | */ 184 | private class IndexAddingContainerLauncherInterceptor implements ContainerLauncherInterceptor { 185 | 186 | @Override 187 | public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) { 188 | ContainerCluster containerCluster = findContainerClusterByContainerId(container.getId()); 189 | if (containerCluster == null) { 190 | return context; 191 | } 192 | containerIdMap.put(container.getId(), containerCluster.getId()); 193 | 194 | Map environment = context.getEnvironment(); 195 | Map indexEnv = new HashMap<>(); 196 | indexEnv.putAll(environment); 197 | Integer reservedIndex; 198 | synchronized (indexTracker) { 199 | reservedIndex = indexTracker.reserveIndex(container.getId(), containerCluster); 200 | } 201 | indexEnv.put("SPRING_CLOUD_APPLICATION_GUID", container.getId().toString()); 202 | indexEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(reservedIndex)); 203 | indexEnv.put("INSTANCE_INDEX", Integer.toString(reservedIndex)); 204 | context.setEnvironment(indexEnv); 205 | return context; 206 | } 207 | } 208 | 209 | /** 210 | * Support class tracking containers per group and reserving an index sequence. 211 | */ 212 | private static class ContainerIndexTracker { 213 | // TODO: move this feature to spring-yarn where scaling can be 214 | // implemented accurately. scaling up/down is anyway not 215 | // supported in sc stream at this moment. 216 | Map> reservationsMap = new HashMap<>(); 217 | 218 | Integer reserveIndex(ContainerId containerId, ContainerCluster containerCluster) { 219 | ArrayList reservationList = reservationsMap.get(containerCluster.getId()); 220 | if (reservationList == null) { 221 | reservationList = new ArrayList<>(); 222 | reservationsMap.put(containerCluster.getId(), reservationList); 223 | } 224 | 225 | // we always increment index at least once 226 | Iterator iterator = reservationList.iterator(); 227 | int index = -1; 228 | ContainerId n = null; 229 | while(iterator.hasNext()) { 230 | ContainerId nn = iterator.next(); 231 | index++; 232 | if (nn == null) { 233 | // we found existing nullified reservation, use that 234 | n = containerId; 235 | break; 236 | } 237 | } 238 | 239 | // all resevations in use, add new 240 | if (n == null) { 241 | reservationList.add(containerId); 242 | index++; 243 | } 244 | return index; 245 | } 246 | 247 | void freeIndex(ContainerId containerId, String containerClusterId) { 248 | ArrayList reservationList = reservationsMap.get(containerClusterId); 249 | if (reservationList != null) { 250 | for (int index = 0; index < reservationList.size(); index++) { 251 | if (containerId.equals(reservationList.get(index))) { 252 | // nullify existing reservation 253 | reservationList.set(index, null); 254 | return; 255 | } 256 | } 257 | } 258 | } 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-appdeployerappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/appdeployer/StreamAppmasterApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.appdeployer; 18 | 19 | import org.springframework.boot.SpringApplication; 20 | import org.springframework.boot.autoconfigure.SpringBootApplication; 21 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 22 | 23 | /** 24 | * Yarn application bootstrapping appmaster. 25 | * 26 | * @author Janne Valkealahti 27 | * 28 | */ 29 | @SpringBootApplication 30 | @EnableConfigurationProperties({ StreamAppmasterProperties.class }) 31 | public class StreamAppmasterApplication { 32 | 33 | public static void main(String[] args) { 34 | SpringApplication.run(StreamAppmasterApplication.class, args); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-appdeployerappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/appdeployer/StreamAppmasterProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.appdeployer; 18 | 19 | import org.springframework.boot.context.properties.ConfigurationProperties; 20 | 21 | /** 22 | * Configuration properties passed from a deployer into an appmaster. 23 | * 24 | * @author Janne Valkealahti 25 | */ 26 | @ConfigurationProperties(value = "spring.cloud.deployer.yarn.appmaster") 27 | public class StreamAppmasterProperties { 28 | 29 | private String artifact; 30 | 31 | public String getArtifact() { 32 | return artifact; 33 | } 34 | 35 | public void setArtifact(String artifact) { 36 | this.artifact = artifact; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-appdeployerappmaster/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 0 3 | management: 4 | security: 5 | enabled: false 6 | endpoints: 7 | shutdown: 8 | enabled: true 9 | spring: 10 | main: 11 | show_banner: false 12 | cloud: 13 | dataflow: 14 | yarn: 15 | version: "@project.version@" 16 | yarn: 17 | appName: spring-cloud-dataflow-yarn-app 18 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/stream/ 19 | appmaster: 20 | appmasterClass: org.springframework.cloud.deployer.spi.yarn.appdeployer.StreamAppmaster 21 | keepContextAlive: true 22 | localizer: 23 | patterns: 24 | - "servers.yml" 25 | containercluster: 26 | enabled: true 27 | clusters: 28 | module-template: 29 | resource: 30 | priority: ${spring.cloud.deployer.yarn.app.streamcontainer.priority:5} 31 | memory: ${spring.cloud.deployer.yarn.app.streamcontainer.memory:256m} 32 | virtualCores: ${spring.cloud.deployer.yarn.app.streamcontainer.virtualCores:1} 33 | labelExpression: ${spring.cloud.deployer.yarn.app.streamcontainer.labelExpression:} 34 | launchcontext: 35 | options: 36 | - ${spring.cloud.deployer.yarn.app.streamcontainer.javaOpts:} 37 | - ${spring.cloud.deployer.yarn.app.streamcontainer.saslConfig:} 38 | arguments: 39 | -Dserver.port: 0 40 | -Dendpoints.jmx.unique-names: true 41 | -Dspring.config.location: servers.yml 42 | locality: false 43 | archiveFile: placeholder.jar 44 | endpoints: 45 | containercluster: 46 | enabled: true 47 | containerregister: 48 | enabled: false 49 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-autoconfig/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | spring-cloud-deployer-yarn-autoconfig 6 | 1.2.3.BUILD-SNAPSHOT 7 | jar 8 | spring-cloud-deployer-yarn-autoconfig 9 | Data Flow Deployer Autoconfig for Apache Yarn 10 | 11 | 12 | org.springframework.cloud 13 | spring-cloud-deployer-yarn-parent 14 | 1.2.3.BUILD-SNAPSHOT 15 | 16 | 17 | 18 | 19 | org.springframework.cloud 20 | spring-cloud-deployer-yarn 21 | 1.2.3.BUILD-SNAPSHOT 22 | 23 | 24 | org.springframework.cloud 25 | spring-cloud-deployer-resource-support 26 | 27 | 28 | org.springframework.cloud 29 | spring-cloud-deployer-resource-maven 30 | 31 | 32 | 33 | 34 | 35 | org.apache.maven.plugins 36 | maven-resources-plugin 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-autoconfig/src/main/java/org/springframework/cloud/deployer/spi/yarn/autoconfigure/YarnDeployerAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.autoconfigure; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.springframework.beans.factory.BeanFactory; 23 | import org.springframework.beans.factory.annotation.Autowired; 24 | import org.springframework.beans.factory.annotation.Value; 25 | import org.springframework.boot.autoconfigure.AutoConfigureOrder; 26 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 27 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 28 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; 29 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 30 | import org.springframework.cloud.deployer.resource.maven.MavenResourceLoader; 31 | import org.springframework.cloud.deployer.resource.support.DelegatingResourceLoader; 32 | import org.springframework.cloud.deployer.spi.app.AppDeployer; 33 | import org.springframework.cloud.deployer.spi.task.TaskLauncher; 34 | import org.springframework.cloud.deployer.spi.yarn.AppDeployerStateMachine; 35 | import org.springframework.cloud.deployer.spi.yarn.DefaultYarnCloudAppService; 36 | import org.springframework.cloud.deployer.spi.yarn.TaskLauncherStateMachine; 37 | import org.springframework.cloud.deployer.spi.yarn.YarnAppDeployer; 38 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService; 39 | import org.springframework.cloud.deployer.spi.yarn.YarnDeployerProperties; 40 | import org.springframework.cloud.deployer.spi.yarn.YarnTaskLauncher; 41 | import org.springframework.context.ApplicationContext; 42 | import org.springframework.context.annotation.Bean; 43 | import org.springframework.context.annotation.Configuration; 44 | import org.springframework.core.Ordered; 45 | import org.springframework.core.io.DefaultResourceLoader; 46 | import org.springframework.core.io.ResourceLoader; 47 | import org.springframework.core.task.TaskExecutor; 48 | import org.springframework.data.hadoop.fs.HdfsResourceLoader; 49 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 50 | 51 | /** 52 | * {@link EnableAutoConfiguration Auto-configuration} for dataflow 53 | * YARN deployer. 54 | * 55 | * @author Janne Valkealahti 56 | * 57 | */ 58 | @Configuration 59 | @AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE) 60 | @ConditionalOnClass({ AppDeployer.class, TaskLauncher.class }) 61 | @EnableConfigurationProperties({ YarnDeployerProperties.class }) 62 | public class YarnDeployerAutoConfiguration { 63 | 64 | @Value("${spring.cloud.deployer.yarn.version:}") 65 | private String deployerVersion; 66 | 67 | @Autowired(required = false) 68 | private MavenResourceLoader mavenResourceLoader; 69 | 70 | @Bean 71 | public DelegatingResourceLoader delegatingResourceLoader(org.apache.hadoop.conf.Configuration configuration) { 72 | DefaultResourceLoader defaultLoader = new DefaultResourceLoader(); 73 | Map loaders = new HashMap<>(); 74 | loaders.put("hdfs", new HdfsResourceLoader(configuration)); 75 | if (mavenResourceLoader != null) { 76 | loaders.put("maven", mavenResourceLoader); 77 | } 78 | loaders.put("file", defaultLoader); 79 | loaders.put("http", defaultLoader); 80 | return new DelegatingResourceLoader(loaders); 81 | } 82 | 83 | @Bean 84 | public YarnCloudAppService yarnCloudAppService() { 85 | return new DefaultYarnCloudAppService(deployerVersion); 86 | } 87 | 88 | @Bean 89 | public TaskExecutor yarnModuleDeployerTaskExecutor() { 90 | ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); 91 | executor.setCorePoolSize(1); 92 | return executor; 93 | } 94 | 95 | @Configuration 96 | @ConditionalOnMissingBean(name = "appDeployer") 97 | public static class ProcessModuleDeployerConfig { 98 | 99 | @Bean 100 | public AppDeployerStateMachine appDeployerStateMachine(YarnCloudAppService yarnCloudAppService, 101 | TaskExecutor yarnModuleDeployerTaskExecutor, BeanFactory beanFactory, ApplicationContext applicationContext) throws Exception { 102 | return new AppDeployerStateMachine(yarnCloudAppService, yarnModuleDeployerTaskExecutor, beanFactory, applicationContext); 103 | } 104 | 105 | @Bean 106 | public AppDeployer appDeployer(YarnCloudAppService yarnCloudAppService, 107 | AppDeployerStateMachine appDeployerStateMachine) throws Exception { 108 | return new YarnAppDeployer(yarnCloudAppService, appDeployerStateMachine.buildStateMachine()); 109 | } 110 | } 111 | 112 | @Configuration 113 | @ConditionalOnMissingBean(name = "taskLauncher") 114 | public static class TaskModuleDeployerConfig { 115 | 116 | @Bean 117 | public TaskLauncherStateMachine taskLauncherStateMachine(YarnCloudAppService yarnCloudAppService, 118 | TaskExecutor yarnModuleDeployerTaskExecutor, BeanFactory beanFactory, ApplicationContext applicationContext) throws Exception { 119 | return new TaskLauncherStateMachine(yarnCloudAppService, yarnModuleDeployerTaskExecutor, beanFactory, applicationContext); 120 | } 121 | 122 | @Bean 123 | public TaskLauncher taskLauncher(YarnCloudAppService yarnCloudAppService, 124 | TaskLauncherStateMachine taskLauncherStateMachine) throws Exception { 125 | return new YarnTaskLauncher(yarnCloudAppService, taskLauncherStateMachine.buildStateMachine()); 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-autoconfig/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.springframework.cloud.deployer.spi.yarn.autoconfigure.YarnDeployerAutoConfiguration 2 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-build-tests/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | spring-cloud-deployer-yarn-build-tests 6 | jar 7 | spring-cloud-deployer-yarn-build-tests 8 | Yarn SPI tests 9 | 10 | 11 | org.springframework.cloud 12 | spring-cloud-deployer-yarn-parent 13 | 1.2.3.BUILD-SNAPSHOT 14 | 15 | 16 | 17 | 18 | org.springframework.data 19 | spring-yarn-boot-test 20 | test 21 | 22 | 23 | org.slf4j 24 | slf4j-log4j12 25 | 26 | 27 | javax.servlet 28 | servlet-api 29 | 30 | 31 | 32 | 33 | com.sun.jersey 34 | jersey-core 35 | 1.9 36 | test 37 | 38 | 39 | com.sun.jersey 40 | jersey-client 41 | 1.9 42 | test 43 | 44 | 45 | com.sun.jersey 46 | jersey-server 47 | 1.9 48 | test 49 | 50 | 51 | org.springframework.cloud 52 | spring-cloud-deployer-yarn 53 | 1.2.3.BUILD-SNAPSHOT 54 | test 55 | 56 | 57 | org.springframework.cloud 58 | spring-cloud-deployer-resource-maven 59 | test 60 | 61 | 62 | org.springframework.cloud 63 | spring-cloud-deployer-yarn-appdeployerappmaster 64 | 1.2.3.BUILD-SNAPSHOT 65 | test 66 | 67 | 68 | org.springframework.cloud 69 | spring-cloud-deployer-yarn-tasklauncherappmaster 70 | 1.2.3.BUILD-SNAPSHOT 71 | test 72 | 73 | 74 | org.springframework.boot 75 | spring-boot-starter-test 76 | test 77 | 78 | 79 | 80 | 81 | 82 | 83 | maven-assembly-plugin 84 | 2.5.1 85 | 86 | 87 | src/main/assembly/assembly.xml 88 | 89 | 90 | 91 | 92 | test-assembly 93 | package 94 | 95 | single 96 | 97 | 98 | ${project.name} 99 | false 100 | 101 | 102 | 103 | 104 | 105 | maven-failsafe-plugin 106 | 107 | -Xmx2048m -XX:MaxPermSize=768m 108 | 109 | 110 | 111 | package 112 | 113 | integration-test 114 | verify 115 | 116 | 117 | 118 | 119 | 120 | com.internetitem 121 | write-properties-file-maven-plugin 122 | 1.0.1 123 | 124 | 125 | one 126 | compile 127 | 128 | write-properties-file 129 | 130 | 131 | ${basedir}/target/spring-cloud-deployer-yarn-build-tests 132 | test.properties 133 | 134 | 135 | projectVersion 136 | ${project.version} 137 | 138 | 139 | artifactVersion 140 | 1.0.0.BUILD-SNAPSHOT 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-build-tests/src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | dist 7 | 8 | dir 9 | 10 | false 11 | 12 | 13 | true 14 | 15 | org.springframework.cloud:spring-cloud-deployer-yarn-appdeployerappmaster 16 | org.springframework.cloud:spring-cloud-deployer-yarn-tasklauncherappmaster 17 | 18 | 19 | false 20 | false 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-build-tests/src/test/java/org/springframework/cloud/dataflow/yarn/buildtests/AbstractCliBootYarnClusterTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.dataflow.yarn.buildtests; 18 | 19 | import static org.hamcrest.CoreMatchers.notNullValue; 20 | import static org.hamcrest.Matchers.is; 21 | import static org.junit.Assert.assertThat; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | import java.util.List; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.fs.FileSystem; 30 | import org.apache.hadoop.fs.LocatedFileStatus; 31 | import org.apache.hadoop.fs.Path; 32 | import org.apache.hadoop.fs.RemoteIterator; 33 | import org.apache.hadoop.yarn.api.records.ApplicationId; 34 | import org.apache.hadoop.yarn.api.records.ApplicationReport; 35 | import org.apache.hadoop.yarn.api.records.YarnApplicationState; 36 | import org.junit.Before; 37 | import org.junit.runner.RunWith; 38 | import org.springframework.beans.factory.annotation.Autowired; 39 | import org.springframework.boot.builder.SpringApplicationBuilder; 40 | import org.springframework.context.ApplicationContext; 41 | import org.springframework.context.ApplicationContextAware; 42 | import org.springframework.context.ApplicationContextInitializer; 43 | import org.springframework.context.ConfigurableApplicationContext; 44 | import org.springframework.context.EnvironmentAware; 45 | import org.springframework.core.env.Environment; 46 | import org.springframework.core.io.Resource; 47 | import org.springframework.test.annotation.DirtiesContext; 48 | import org.springframework.test.annotation.DirtiesContext.ClassMode; 49 | import org.springframework.test.context.TestPropertySource; 50 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 51 | import org.springframework.util.Assert; 52 | import org.springframework.yarn.boot.SpringApplicationCallback; 53 | import org.springframework.yarn.boot.SpringApplicationTemplate; 54 | import org.springframework.yarn.client.YarnClient; 55 | import org.springframework.yarn.test.context.MiniYarnClusterTest; 56 | import org.springframework.yarn.test.context.YarnCluster; 57 | import org.springframework.yarn.test.junit.ApplicationInfo; 58 | import org.springframework.yarn.test.support.ContainerLogUtils; 59 | 60 | @RunWith(SpringJUnit4ClassRunner.class) 61 | @MiniYarnClusterTest 62 | @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) 63 | @TestPropertySource("file:target/spring-cloud-deployer-yarn-build-tests/test.properties") 64 | public class AbstractCliBootYarnClusterTests implements ApplicationContextAware, EnvironmentAware { 65 | 66 | private ApplicationContext applicationContext; 67 | private Environment environment; 68 | private Configuration configuration; 69 | private YarnCluster yarnCluster; 70 | private YarnClient yarnClient; 71 | private String projectVersion; 72 | 73 | @Before 74 | public void setup() { 75 | projectVersion = getEnvironment().getProperty("projectVersion"); 76 | } 77 | 78 | @Override 79 | public final void setApplicationContext(ApplicationContext applicationContext) { 80 | this.applicationContext = applicationContext; 81 | } 82 | 83 | @Autowired 84 | public void setConfiguration(Configuration configuration) { 85 | this.configuration = configuration; 86 | } 87 | 88 | @Autowired 89 | public void setYarnCluster(YarnCluster yarnCluster) { 90 | this.yarnCluster = yarnCluster; 91 | } 92 | 93 | @Override 94 | public void setEnvironment(Environment environment) { 95 | this.environment = environment; 96 | } 97 | 98 | public ApplicationContext getApplicationContext() { 99 | return applicationContext; 100 | } 101 | 102 | public Environment getEnvironment() { 103 | return environment; 104 | } 105 | 106 | public Configuration getConfiguration() { 107 | return configuration; 108 | } 109 | 110 | public void setYarnClient(YarnClient yarnClient) { 111 | this.yarnClient = yarnClient; 112 | } 113 | 114 | public YarnClient getYarnClient() { 115 | return yarnClient; 116 | } 117 | 118 | public YarnCluster getYarnCluster() { 119 | return yarnCluster; 120 | } 121 | 122 | public String getProjectVersion() { 123 | return projectVersion; 124 | } 125 | 126 | protected ApplicationInfo submitApplicationAndWait(Object source, String[] args) throws Exception { 127 | return submitApplicationAndWait(source, args, 60, TimeUnit.SECONDS); 128 | } 129 | 130 | protected ApplicationInfo submitApplicationAndWait(Object source, String[] args, long timeout, final TimeUnit unit) throws Exception { 131 | return submitApplicationAndWaitState(source, args, timeout, unit, YarnApplicationState.FINISHED, YarnApplicationState.FAILED); 132 | } 133 | 134 | protected ApplicationInfo submitApplicationAndWaitState(long timeout, TimeUnit unit, YarnApplicationState... applicationStates) throws Exception { 135 | Assert.notEmpty(applicationStates, "Need to have atleast one state"); 136 | Assert.notNull(getYarnClient(), "Yarn client must be set"); 137 | 138 | YarnApplicationState state = null; 139 | ApplicationReport report = null; 140 | 141 | ApplicationId applicationId = submitApplication(); 142 | Assert.notNull(applicationId, "Failed to get application id from submit"); 143 | 144 | long end = System.currentTimeMillis() + unit.toMillis(timeout); 145 | 146 | // break label for inner loop 147 | done: 148 | do { 149 | report = findApplicationReport(getYarnClient(), applicationId); 150 | if (report == null) { 151 | break; 152 | } 153 | state = report.getYarnApplicationState(); 154 | for (YarnApplicationState stateCheck : applicationStates) { 155 | if (state.equals(stateCheck)) { 156 | break done; 157 | } 158 | } 159 | Thread.sleep(1000); 160 | } while (System.currentTimeMillis() < end); 161 | return new ApplicationInfo(applicationId, report); 162 | } 163 | 164 | protected ApplicationInfo waitState(ApplicationId applicationId, long timeout, TimeUnit unit, YarnApplicationState... applicationStates) throws Exception { 165 | YarnApplicationState state = null; 166 | ApplicationReport report = null; 167 | long end = System.currentTimeMillis() + unit.toMillis(timeout); 168 | 169 | // break label for inner loop 170 | done: 171 | do { 172 | report = findApplicationReport(getYarnClient(), applicationId); 173 | if (report == null) { 174 | break; 175 | } 176 | state = report.getYarnApplicationState(); 177 | for (YarnApplicationState stateCheck : applicationStates) { 178 | if (state.equals(stateCheck)) { 179 | break done; 180 | } 181 | } 182 | Thread.sleep(1000); 183 | } while (System.currentTimeMillis() < end); 184 | return new ApplicationInfo(applicationId, report); 185 | } 186 | 187 | 188 | protected ApplicationInfo submitApplicationAndWaitState(Object source, String[] args, final long timeout, 189 | final TimeUnit unit, final YarnApplicationState... applicationStates) throws Exception { 190 | 191 | SpringApplicationBuilder builder = new SpringApplicationBuilder(source); 192 | builder.initializers(new HadoopConfigurationInjectingInitializer(getConfiguration())); 193 | 194 | SpringApplicationTemplate template = new SpringApplicationTemplate(builder); 195 | return template.execute(new SpringApplicationCallback() { 196 | 197 | @Override 198 | public ApplicationInfo runWithSpringApplication(ApplicationContext context) throws Exception { 199 | setYarnClient(context.getBean(YarnClient.class)); 200 | return submitApplicationAndWaitState(timeout, unit, applicationStates); 201 | } 202 | 203 | }, args); 204 | } 205 | 206 | protected File assertWaitFileContent(long timeout, TimeUnit unit, ApplicationId applicationId, String search) throws Exception { 207 | File file = null; 208 | 209 | long end = System.currentTimeMillis() + unit.toMillis(timeout); 210 | done: 211 | do { 212 | 213 | List resources = ContainerLogUtils.queryContainerLogs( 214 | getYarnCluster(), applicationId); 215 | for (Resource res : resources) { 216 | File f = res.getFile(); 217 | String content = ContainerLogUtils.getFileContent(f); 218 | if (content.contains(search)) { 219 | file = f; 220 | break done; 221 | } 222 | } 223 | 224 | Thread.sleep(1000); 225 | } while (System.currentTimeMillis() < end); 226 | 227 | 228 | assertThat(file, notNullValue()); 229 | return file; 230 | } 231 | 232 | protected String dumpFs() throws IOException { 233 | StringBuilder buf = new StringBuilder(); 234 | FileSystem fs = FileSystem.get(getConfiguration()); 235 | RemoteIterator files = fs.listFiles(new Path("/"), true); 236 | while (files.hasNext()) { 237 | buf.append(files.next().toString()); 238 | buf.append("\n"); 239 | } 240 | return buf.toString(); 241 | } 242 | 243 | protected void waitHdfsFile(String path, long timeout, TimeUnit unit) throws Exception { 244 | Path p = new Path(path); 245 | FileSystem fs = FileSystem.get(getConfiguration()); 246 | long end = System.currentTimeMillis() + unit.toMillis(timeout); 247 | boolean found = false; 248 | do { 249 | if (fs.exists(p)) { 250 | found = true; 251 | break; 252 | } 253 | Thread.sleep(1000); 254 | } while (System.currentTimeMillis() < end); 255 | assertThat(found, is(true)); 256 | } 257 | 258 | private ApplicationReport findApplicationReport(YarnClient client, ApplicationId applicationId) { 259 | Assert.notNull(getYarnClient(), "Yarn client must be set"); 260 | for (ApplicationReport report : client.listApplications()) { 261 | if (report.getApplicationId().equals(applicationId)) { 262 | return report; 263 | } 264 | } 265 | return null; 266 | } 267 | 268 | protected ApplicationId submitApplication() { 269 | Assert.notNull(getYarnClient(), "Yarn client must be set"); 270 | ApplicationId applicationId = getYarnClient().submitApplication(); 271 | Assert.notNull(applicationId, "Failed to get application id from submit"); 272 | return applicationId; 273 | } 274 | 275 | public static class HadoopConfigurationInjectingInitializer 276 | implements ApplicationContextInitializer { 277 | 278 | private final Configuration configuration; 279 | 280 | public HadoopConfigurationInjectingInitializer(Configuration configuration) { 281 | this.configuration = configuration; 282 | } 283 | 284 | @Override 285 | public void initialize(ConfigurableApplicationContext applicationContext) { 286 | applicationContext.getBeanFactory().registerSingleton("miniYarnConfiguration", configuration); 287 | } 288 | 289 | } 290 | 291 | } 292 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-build-tests/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootCategory=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=%d %p [%C{1}] [%t] - %m%n 6 | 7 | log4j.category.org.apache.hadoop=INFO 8 | log4j.category.org.springframework.statemachine=DEBUG 9 | log4j.category.org.springframework.yarn=DEBUG 10 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-build-tests/src/test/resources/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | yarn.nodemanager.sleep-delay-before-sigkill.ms 9 | 10000 10 | 11 | 12 | yarn.nodemanager.delete.debug-delay-sec 13 | 600 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-tasklauncherappmaster/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | org.springframework.cloud 6 | spring-cloud-deployer-yarn-tasklauncherappmaster 7 | 1.2.3.BUILD-SNAPSHOT 8 | 9 | org.springframework.cloud 10 | spring-cloud-deployer-yarn-parent 11 | 1.2.3.BUILD-SNAPSHOT 12 | 13 | 14 | 15 | 16 | org.springframework.data 17 | spring-yarn-boot 18 | 19 | 20 | javax.servlet 21 | servlet-api 22 | 23 | 24 | org.slf4j 25 | slf4j-log4j12 26 | 27 | 28 | 29 | 30 | org.springframework.boot 31 | spring-boot-starter-actuator 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-web 36 | 37 | 38 | 39 | 40 | 41 | 42 | src/main/resources 43 | true 44 | 45 | application.yml 46 | 47 | 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-resources-plugin 53 | 54 | 55 | org.springframework.boot 56 | spring-boot-maven-plugin 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/tasklauncher/TaskAppmaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.tasklauncher; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Map.Entry; 24 | 25 | import org.apache.commons.logging.Log; 26 | import org.apache.commons.logging.LogFactory; 27 | import org.apache.hadoop.yarn.api.records.Container; 28 | import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; 29 | import org.apache.hadoop.yarn.api.records.LocalResource; 30 | import org.apache.hadoop.yarn.api.records.LocalResourceType; 31 | import org.springframework.beans.factory.annotation.Autowired; 32 | import org.springframework.context.Lifecycle; 33 | import org.springframework.util.Assert; 34 | import org.springframework.util.StringUtils; 35 | import org.springframework.yarn.am.ContainerLauncherInterceptor; 36 | import org.springframework.yarn.am.StaticEventingAppmaster; 37 | import org.springframework.yarn.am.container.AbstractLauncher; 38 | import org.springframework.yarn.fs.LocalResourcesFactoryBean; 39 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.CopyEntry; 40 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.TransferEntry; 41 | import org.springframework.yarn.fs.ResourceLocalizer; 42 | 43 | /** 44 | * Spring YARN application master class which does a specific 45 | * handling of a task containers. 46 | * 47 | * @author Janne Valkealahti 48 | */ 49 | public class TaskAppmaster extends StaticEventingAppmaster { 50 | 51 | private final static Log log = LogFactory.getLog(TaskAppmaster.class); 52 | 53 | @Autowired 54 | private TaskAppmasterProperties taskAppmasterProperties; 55 | 56 | private ResourceLocalizer artifactResourceLocalizer; 57 | 58 | private boolean finishRequested = false; 59 | 60 | @Override 61 | protected void onInit() throws Exception { 62 | super.onInit(); 63 | Assert.hasText(taskAppmasterProperties.getArtifact(), "Artifact must be set"); 64 | Assert.hasText(taskAppmasterProperties.getArtifactName(), "Artifact name must be set"); 65 | artifactResourceLocalizer = buildArtifactResourceLocalizer(); 66 | if (getLauncher() instanceof AbstractLauncher) { 67 | ((AbstractLauncher) getLauncher()).addInterceptor(new ArtifactResourceContainerLaunchInterceptor()); 68 | } 69 | } 70 | 71 | @Override 72 | protected void doStop() { 73 | if (getAllocator() instanceof Lifecycle) { 74 | ((Lifecycle)getAllocator()).stop(); 75 | } 76 | if (!finishRequested) { 77 | super.doStop(); 78 | } 79 | } 80 | 81 | @Override 82 | public List getCommands() { 83 | List list = new ArrayList<>(); 84 | for (String command : super.getCommands()) { 85 | if (command.contains("placeholder.jar")) { 86 | list.add(command.replace("placeholder.jar", taskAppmasterProperties.getArtifactName())); 87 | } else { 88 | list.add(command); 89 | } 90 | } 91 | if (taskAppmasterProperties.getParameters() != null) { 92 | for (Entry entry : taskAppmasterProperties.getParameters().entrySet()) { 93 | if (StringUtils.hasText(entry.getValue())) { 94 | // don't pass empty values as those will most likely fail 95 | // with spring/boot command-line parser 96 | list.add(Math.max(list.size() - 2, 0), "--" + entry.getKey() + "='" + entry.getValue() + "'"); 97 | } 98 | } 99 | } 100 | if (taskAppmasterProperties.getCommandlineArguments() != null) { 101 | for (String commandlineArgument : taskAppmasterProperties.getCommandlineArguments()) { 102 | list.add(Math.max(list.size() - 2, 0), "'" + commandlineArgument + "'"); 103 | } 104 | } 105 | log.info("Using command list for task container: " + StringUtils.collectionToCommaDelimitedString(list)); 106 | return list; 107 | } 108 | 109 | @Override 110 | protected boolean shutdownContainers() { 111 | // SHDP-561 112 | return true; 113 | } 114 | 115 | @Override 116 | protected void notifyCompleted() { 117 | // SHDP-561 118 | finishAppmaster(); 119 | finishRequested = true; 120 | super.notifyCompleted(); 121 | } 122 | 123 | private ResourceLocalizer buildArtifactResourceLocalizer() throws Exception { 124 | String artifact = taskAppmasterProperties.getArtifact(); 125 | log.info("Building localizer for artifact " + artifact); 126 | LocalResourcesFactoryBean fb = new LocalResourcesFactoryBean(); 127 | fb.setConfiguration(getConfiguration()); 128 | TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifact, false); 129 | ArrayList hdfsEntries = new ArrayList(); 130 | hdfsEntries.add(te); 131 | fb.setHdfsEntries(hdfsEntries); 132 | fb.setCopyEntries(new ArrayList()); 133 | fb.afterPropertiesSet(); 134 | return fb.getObject(); 135 | } 136 | 137 | private class ArtifactResourceContainerLaunchInterceptor implements ContainerLauncherInterceptor { 138 | 139 | @Override 140 | public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) { 141 | Map resources = new HashMap(); 142 | resources.putAll(context.getLocalResources()); 143 | resources.putAll(artifactResourceLocalizer.getResources()); 144 | context.setLocalResources(resources); 145 | return context; 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/tasklauncher/TaskAppmasterApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.tasklauncher; 18 | 19 | import org.springframework.boot.SpringApplication; 20 | import org.springframework.boot.autoconfigure.SpringBootApplication; 21 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 22 | 23 | /** 24 | * Yarn application bootstrapping appmaster. 25 | * 26 | * @author Janne Valkealahti 27 | */ 28 | @SpringBootApplication 29 | @EnableConfigurationProperties({ TaskAppmasterProperties.class }) 30 | public class TaskAppmasterApplication { 31 | 32 | public static void main(String[] args) { 33 | SpringApplication.run(TaskAppmasterApplication.class, args); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/tasklauncher/TaskAppmasterProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn.tasklauncher; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | import org.springframework.boot.context.properties.ConfigurationProperties; 23 | 24 | /** 25 | * Configuration properties passed from a deployer into an appmaster. 26 | * 27 | * @author Janne Valkealahti 28 | */ 29 | @ConfigurationProperties(value = "spring.cloud.deployer.yarn.appmaster") 30 | public class TaskAppmasterProperties { 31 | 32 | private String artifact; 33 | private String artifactName; 34 | private Map parameters; 35 | private List commandlineArguments; 36 | 37 | public String getArtifact() { 38 | return artifact; 39 | } 40 | 41 | public void setArtifact(String artifact) { 42 | this.artifact = artifact; 43 | } 44 | 45 | public String getArtifactName() { 46 | return artifactName; 47 | } 48 | 49 | public void setArtifactName(String artifactName) { 50 | this.artifactName = artifactName; 51 | } 52 | 53 | public Map getParameters() { 54 | return parameters; 55 | } 56 | 57 | public void setParameters(Map parameters) { 58 | this.parameters = parameters; 59 | } 60 | 61 | public List getCommandlineArguments() { 62 | return commandlineArguments; 63 | } 64 | 65 | public void setCommandlineArguments(List commandlineArguments) { 66 | this.commandlineArguments = commandlineArguments; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 0 3 | management: 4 | security: 5 | enabled: false 6 | endpoints: 7 | shutdown: 8 | enabled: true 9 | spring: 10 | main: 11 | show_banner: false 12 | cloud: 13 | dataflow: 14 | yarn: 15 | version: "@project.version@" 16 | yarn: 17 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/task/ 18 | appmaster: 19 | containerCount: 1 20 | appmasterClass: org.springframework.cloud.deployer.spi.yarn.tasklauncher.TaskAppmaster 21 | localizer: 22 | patterns: 23 | - "servers.yml" 24 | resource: 25 | priority: ${spring.cloud.deployer.yarn.app.taskcontainer.priority:10} 26 | labelExpression: ${spring.cloud.deployer.yarn.app.taskcontainer.labelExpression:} 27 | memory: ${spring.cloud.deployer.yarn.app.taskcontainer.memory:256m} 28 | virtualCores: ${spring.cloud.deployer.yarn.app.taskcontainer.virtualCores:1} 29 | launchcontext: 30 | options: 31 | - ${spring.cloud.deployer.yarn.app.taskcontainer.javaOpts:} 32 | arguments: 33 | -Dserver.port: 0 34 | -Dspring.jmx.enabled: false 35 | -Dspring.config.location: servers.yml 36 | locality: false 37 | archiveFile: placeholder.jar 38 | endpoints: 39 | containerregister: 40 | enabled: false 41 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | spring-cloud-deployer-yarn 6 | 1.2.3.BUILD-SNAPSHOT 7 | jar 8 | spring-cloud-deployer-yarn 9 | Data Flow Deployer for Apache Yarn 10 | 11 | 12 | org.springframework.cloud 13 | spring-cloud-deployer-yarn-parent 14 | 1.2.3.BUILD-SNAPSHOT 15 | 16 | 17 | 18 | 19 | org.springframework.cloud 20 | spring-cloud-deployer-spi 21 | 22 | 23 | org.springframework.data 24 | spring-yarn-boot 25 | 26 | 27 | org.slf4j 28 | slf4j-log4j12 29 | 30 | 31 | javax.servlet 32 | servlet-api 33 | 34 | 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-web 39 | 40 | 41 | org.springframework.statemachine 42 | spring-statemachine-uml 43 | 44 | 45 | org.springframework.statemachine 46 | spring-statemachine-test 47 | test 48 | 49 | 50 | 51 | 52 | 53 | src/main/resources 54 | true 55 | 56 | task.yml 57 | stream.yml 58 | appdeployer-model.uml 59 | tasklauncher-model.uml 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/AbstractDeployerStateMachine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.io.IOException; 20 | import java.util.Collection; 21 | import java.util.Map; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | import org.springframework.beans.factory.BeanFactory; 26 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppInfo; 27 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType; 28 | import org.springframework.core.io.Resource; 29 | import org.springframework.core.io.ResourceLoader; 30 | import org.springframework.core.task.TaskExecutor; 31 | import org.springframework.statemachine.StateContext; 32 | import org.springframework.statemachine.StateMachine; 33 | import org.springframework.statemachine.action.Action; 34 | import org.springframework.statemachine.config.StateMachineBuilder; 35 | import org.springframework.statemachine.config.StateMachineBuilder.Builder; 36 | import org.springframework.statemachine.guard.Guard; 37 | import org.springframework.statemachine.uml.UmlStateMachineModelFactory; 38 | import org.springframework.util.Assert; 39 | import org.springframework.util.StringUtils; 40 | 41 | /** 42 | * Shared base functionality for state machines handling app deployment 43 | * and task launching. 44 | * 45 | * @author Janne Valkealahti 46 | * 47 | */ 48 | public abstract class AbstractDeployerStateMachine { 49 | 50 | private static final Logger logger = LoggerFactory.getLogger(AbstractDeployerStateMachine.class); 51 | 52 | // common states 53 | public final static String STATE_READY = "READY"; 54 | public final static String STATE_ERROR = "ERROR"; 55 | 56 | // common extended state variables 57 | public static final String VAR_ERROR = "error"; 58 | public static final String VAR_APP_VERSION = "appVersion"; 59 | public static final String VAR_APPLICATION_ID = "applicationId"; 60 | public static final String VAR_ARTIFACT = "artifact"; 61 | public static final String VAR_MESSAGE_ID = "messageId"; 62 | 63 | // common event headers 64 | public static final String HEADER_APPLICATION_ID = "applicationId"; 65 | public static final String HEADER_APP_VERSION = "appVersion"; 66 | public static final String HEADER_GROUP_ID = "groupId"; 67 | public static final String HEADER_ERROR = "error"; 68 | public static final String HEADER_ARTIFACT = "artifact"; 69 | public static final String HEADER_ARTIFACT_DIR = "artifactDir"; 70 | public static final String HEADER_CONTEXT_RUN_ARGS = "contextRunArgs"; 71 | public static final String HEADER_DEFINITION_PARAMETERS = "definitionParameters"; 72 | 73 | private final YarnCloudAppService yarnCloudAppService; 74 | private final TaskExecutor taskExecutor; 75 | private final BeanFactory beanFactory; 76 | private final ResourceLoader resourceLoader; 77 | private final String modelLocation; 78 | 79 | private boolean autoStart = true; 80 | 81 | /** 82 | * Instantiates a new abstract deployer state machine. 83 | * 84 | * @param yarnCloudAppService the yarn cloud app service 85 | * @param taskExecutor the task executor 86 | * @param beanFactory the bean factory 87 | * @param resourceLoader the resource loader 88 | * @param modelLocation the model location 89 | */ 90 | public AbstractDeployerStateMachine(YarnCloudAppService yarnCloudAppService, TaskExecutor taskExecutor, BeanFactory beanFactory, 91 | ResourceLoader resourceLoader, String modelLocation) { 92 | Assert.notNull(yarnCloudAppService, "YarnCloudAppService must be set"); 93 | Assert.notNull(taskExecutor, "TaskExecutor must be set"); 94 | Assert.notNull(beanFactory, "BeanFactory must be set"); 95 | Assert.notNull(resourceLoader, "ResourceLoader must be set"); 96 | Assert.notNull(modelLocation, "Model must be set"); 97 | this.yarnCloudAppService = yarnCloudAppService; 98 | this.taskExecutor = taskExecutor; 99 | this.beanFactory = beanFactory; 100 | this.resourceLoader = resourceLoader; 101 | this.modelLocation = modelLocation; 102 | } 103 | 104 | /** 105 | * Builds the state machine. 106 | * 107 | * @return the state machine 108 | * @throws Exception the exception 109 | */ 110 | public final StateMachine buildStateMachine() throws Exception { 111 | Builder builder = StateMachineBuilder.builder(); 112 | 113 | builder.configureConfiguration() 114 | .withConfiguration() 115 | .autoStartup(autoStart) 116 | .taskExecutor(taskExecutor) 117 | .beanFactory(beanFactory); 118 | 119 | UmlStateMachineModelFactory modelFactory = new UmlStateMachineModelFactory(modelLocation); 120 | 121 | Map> registeredActions = getRegisteredActions(); 122 | if (registeredActions != null) { 123 | for (Map.Entry> entry : registeredActions.entrySet()) { 124 | // wraps actions to catch errors 125 | modelFactory.registerAction(entry.getKey(), new ExceptionCatchingAction(entry.getValue())); 126 | } 127 | } 128 | 129 | Map> registeredGuards = getRegisteredGuards(); 130 | if (registeredGuards != null) { 131 | for (Map.Entry> entry : registeredGuards.entrySet()) { 132 | modelFactory.registerGuard(entry.getKey(), entry.getValue()); 133 | } 134 | } 135 | 136 | modelFactory.setResourceLoader(resourceLoader); 137 | builder.configureModel() 138 | .withModel() 139 | .factory(modelFactory); 140 | 141 | return builder.build(); 142 | } 143 | 144 | /** 145 | * Sets the auto start flag for builder. 146 | * 147 | * @param autoStart the new auto start 148 | */ 149 | public void setAutoStart(boolean autoStart) { 150 | this.autoStart = autoStart; 151 | } 152 | 153 | /** 154 | * Gets the registered actions. 155 | * 156 | * @return the registered actions 157 | */ 158 | protected abstract Map> getRegisteredActions(); 159 | 160 | /** 161 | * Gets the registered guards. 162 | * 163 | * @return the registered guards 164 | */ 165 | protected abstract Map> getRegisteredGuards(); 166 | 167 | /** 168 | * Gets the yarn cloud app service. 169 | * 170 | * @return the yarn cloud app service 171 | */ 172 | protected YarnCloudAppService getYarnCloudAppService() { 173 | return yarnCloudAppService; 174 | } 175 | 176 | 177 | /** 178 | * {@link Action} which clears existing extended state variables. 179 | */ 180 | protected class ResetVariablesAction implements Action { 181 | 182 | @Override 183 | public void execute(StateContext context) { 184 | context.getExtendedState().getVariables().clear(); 185 | } 186 | } 187 | 188 | /** 189 | * {@link Action} which queries {@link YarnCloudAppService} and checks if 190 | * passed {@code appVersion} from event headers exists and sends {@code ERROR} 191 | * event into state machine if it doesn't exist. Add to be used {@code appVersion} 192 | * into extended state variables which later used by other guards and actions. 193 | */ 194 | protected class CheckAppAction implements Action { 195 | 196 | private final CloudAppType cloudAppType; 197 | 198 | public CheckAppAction(CloudAppType cloudAppType) { 199 | this.cloudAppType = cloudAppType; 200 | } 201 | 202 | @Override 203 | public void execute(StateContext context) { 204 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION); 205 | 206 | if (!StringUtils.hasText(appVersion)) { 207 | context.getExtendedState().getVariables().put(VAR_ERROR, new RuntimeException("appVersion not defined")); 208 | } else { 209 | Collection appInfos = yarnCloudAppService.getApplications(cloudAppType); 210 | for (CloudAppInfo appInfo : appInfos) { 211 | if (appInfo.getName().equals(appVersion)) { 212 | context.getExtendedState().getVariables().put(VAR_APP_VERSION, appVersion); 213 | } 214 | } 215 | } 216 | } 217 | } 218 | 219 | /** 220 | * {@link Action} which pushes application version into hdfs found 221 | * from variable {@code appVersion}. 222 | */ 223 | protected class PushAppAction implements Action { 224 | 225 | private final CloudAppType cloudAppType; 226 | 227 | public PushAppAction(CloudAppType cloudAppType) { 228 | this.cloudAppType = cloudAppType; 229 | } 230 | 231 | @Override 232 | public void execute(StateContext context) { 233 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION); 234 | yarnCloudAppService.pushApplication(appVersion, cloudAppType); 235 | } 236 | } 237 | 238 | /** 239 | * {@link Action} which pushes artifact into hdfs. 240 | */ 241 | protected class PushArtifactAction implements Action { 242 | 243 | @Override 244 | public void execute(StateContext context) { 245 | Resource artifact = (Resource) context.getMessageHeader(HEADER_ARTIFACT); 246 | String artifactDir = (String) context.getMessageHeader(HEADER_ARTIFACT_DIR); 247 | if (!isHdfsResource(artifact)) { 248 | yarnCloudAppService.pushArtifact(artifact, artifactDir); 249 | } else { 250 | if (!artifact.exists()) { 251 | context.getExtendedState().getVariables().put(VAR_ERROR, new RuntimeException("hdfs artifact missing")); 252 | } 253 | } 254 | } 255 | } 256 | 257 | /** 258 | * {@link Guard} which is used to protect state where application push 259 | * into hdfs would happen. Assumes that if {@code appVersion} variable 260 | * exists, application is installed. 261 | */ 262 | protected class PushAppGuard implements Guard { 263 | 264 | @Override 265 | public boolean evaluate(StateContext context) { 266 | return !context.getExtendedState().getVariables().containsKey(VAR_APP_VERSION); 267 | } 268 | } 269 | 270 | /** 271 | * {@link Guard} which is used to protect transitions happening 272 | * in case on error set in variables. 273 | */ 274 | protected class ErrorGuard implements Guard { 275 | 276 | @Override 277 | public boolean evaluate(StateContext context) { 278 | return context.getExtendedState().getVariables().containsKey(VAR_ERROR); 279 | } 280 | } 281 | 282 | /** 283 | * {@link Action} which simply logs context when machine passes 284 | * through error state. 285 | */ 286 | protected class ErrorAction implements Action { 287 | 288 | @Override 289 | public void execute(StateContext context) { 290 | logger.error("Passing through error state {}", context); 291 | } 292 | } 293 | 294 | /** 295 | * {@link Action} which wraps a delegating action and sets an error if its 296 | * execution results an exception. 297 | */ 298 | private class ExceptionCatchingAction implements Action { 299 | 300 | private final Action delegate; 301 | 302 | public ExceptionCatchingAction(Action delegate) { 303 | Assert.notNull(delegate, "Delegate action must be set"); 304 | this.delegate = delegate; 305 | } 306 | 307 | @Override 308 | public void execute(StateContext context) { 309 | try { 310 | delegate.execute(context); 311 | } catch (Exception e) { 312 | context.getExtendedState().getVariables().put(VAR_ERROR, e); 313 | } 314 | } 315 | } 316 | 317 | private boolean isHdfsResource(Resource resource) { 318 | try { 319 | return resource != null && resource.getURI().getScheme().equals("hdfs"); 320 | } catch (IOException e) { 321 | return false; 322 | } 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/AppDeployerStateMachine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | import org.springframework.beans.factory.BeanFactory; 26 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppInstanceInfo; 27 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType; 28 | import org.springframework.core.io.Resource; 29 | import org.springframework.core.io.ResourceLoader; 30 | import org.springframework.core.task.TaskExecutor; 31 | import org.springframework.statemachine.StateContext; 32 | import org.springframework.statemachine.action.Action; 33 | import org.springframework.statemachine.guard.Guard; 34 | import org.springframework.util.StringUtils; 35 | 36 | /** 37 | * Statemachine for app deployments. 38 | * 39 | * @author Janne Valkealahti 40 | * 41 | */ 42 | public class AppDeployerStateMachine extends AbstractDeployerStateMachine { 43 | 44 | private static final Logger logger = LoggerFactory.getLogger(AppDeployerStateMachine.class); 45 | 46 | private final static String MODEL_LOCATION = "classpath:appdeployer-model.uml"; 47 | 48 | public final static String EVENT_DEPLOY = "DEPLOY"; 49 | public final static String EVENT_UNDEPLOY = "UNDEPLOY"; 50 | 51 | public static final String VAR_INSTANCE_ADDRESS = "instanceAddress"; 52 | public static final String VAR_COUNT = "count"; 53 | public static final String VAR_APPNAME = "appname"; 54 | public static final String VAR_CLUSTER_ID = "clusterId"; 55 | public static final String VAR_DEFINITION_PARAMETERS = "definitionParameters"; 56 | 57 | public static final String HEADER_CLUSTER_ID = "clusterId"; 58 | public static final String HEADER_COUNT = "count"; 59 | public static final String HEADER_DEFINITION_PARAMETERS = "definitionParameters"; 60 | 61 | /** 62 | * Instantiates a new app deployer state machine. 63 | * 64 | * @param yarnCloudAppService the yarn cloud app service 65 | * @param taskExecutor the task executor 66 | * @param beanFactory the bean factory 67 | * @param resourceLoader the resource loader 68 | */ 69 | public AppDeployerStateMachine(YarnCloudAppService yarnCloudAppService, TaskExecutor taskExecutor, BeanFactory beanFactory, 70 | ResourceLoader resourceLoader) { 71 | super(yarnCloudAppService, taskExecutor, beanFactory, resourceLoader, MODEL_LOCATION); 72 | } 73 | 74 | @Override 75 | protected Map> getRegisteredActions() { 76 | HashMap> actions = new HashMap<>(); 77 | actions.put("resetVariablesAction", new ResetVariablesAction()); 78 | actions.put("deployAction", new DeployAction()); 79 | actions.put("checkAppAction", new CheckAppAction(CloudAppType.STREAM)); 80 | actions.put("pushAppAction", new PushAppAction(CloudAppType.STREAM)); 81 | actions.put("checkInstanceAction", new CheckInstanceAction()); 82 | actions.put("pushArtifactAction", new PushArtifactAction()); 83 | actions.put("startInstanceAction", new StartInstanceAction()); 84 | actions.put("waitInstanceAction", new WaitInstanceAction()); 85 | actions.put("resolveInstanceAction", new ResolveInstanceAction()); 86 | actions.put("createClusterAction", new CreateClusterAction()); 87 | actions.put("startClusterAction", new StartClusterAction()); 88 | actions.put("stopClusterAction", new StopClusterAction()); 89 | actions.put("destroyClusterAction", new DestroyClusterAction()); 90 | actions.put("errorHandlingAction", new ErrorAction()); 91 | return actions; 92 | } 93 | 94 | @Override 95 | protected Map> getRegisteredGuards() { 96 | HashMap> guards = new HashMap<>(); 97 | guards.put("pushAppGuard", new PushAppGuard()); 98 | guards.put("startInstanceGuard", new StartInstanceGuard()); 99 | guards.put("errorGuard", new ErrorGuard()); 100 | guards.put("instanceGuard", new InstanceGuard()); 101 | return guards; 102 | } 103 | 104 | /** 105 | * {@link Action} which queries {@link YarnCloudAppService} for existing 106 | * running instances. 107 | */ 108 | private class CheckInstanceAction implements Action { 109 | 110 | @Override 111 | public void execute(StateContext context) { 112 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION); 113 | String groupId = (String) context.getMessageHeader(HEADER_GROUP_ID); 114 | String appName = "scdstream:" + appVersion + ":" + groupId; 115 | CloudAppInstanceInfo appInstanceInfo = findRunningInstance(appName); 116 | if (appInstanceInfo != null) { 117 | context.getExtendedState().getVariables().put(VAR_APPLICATION_ID, appInstanceInfo.getApplicationId()); 118 | } 119 | } 120 | } 121 | 122 | private CloudAppInstanceInfo findRunningInstance(String appName) { 123 | for (CloudAppInstanceInfo appInstanceInfo : getYarnCloudAppService().getInstances(CloudAppType.STREAM)) { 124 | logger.info("Checking instance {} for appName {}", appInstanceInfo, appName); 125 | if (appInstanceInfo.getName().equals(appName) && appInstanceInfo.getState().equals("RUNNING") 126 | && appInstanceInfo.getAddress().contains("http")) { 127 | logger.info("Using instance {}", appInstanceInfo); 128 | return appInstanceInfo; 129 | } 130 | } 131 | return null; 132 | } 133 | 134 | private class StartInstanceAction implements Action { 135 | 136 | @Override 137 | public void execute(StateContext context) { 138 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION); 139 | 140 | // we control type so casting is safe 141 | @SuppressWarnings("unchecked") 142 | List contextRunArgs = (List) context.getMessageHeader(HEADER_CONTEXT_RUN_ARGS); 143 | String applicationId = getYarnCloudAppService().submitApplication(appVersion, CloudAppType.STREAM, contextRunArgs); 144 | context.getExtendedState().getVariables().put(VAR_APPLICATION_ID, applicationId); 145 | } 146 | } 147 | 148 | private class WaitInstanceAction implements Action { 149 | 150 | @Override 151 | public void execute(StateContext context) { 152 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION); 153 | String groupId = (String) context.getMessageHeader(HEADER_GROUP_ID); 154 | if (StringUtils.hasText(appVersion) && StringUtils.hasText(groupId)) { 155 | String appName = "scdstream:" + appVersion + ":" + groupId; 156 | context.getExtendedState().getVariables().put(VAR_APPNAME, appName); 157 | } 158 | } 159 | } 160 | 161 | private class ResolveInstanceAction implements Action { 162 | 163 | @Override 164 | public void execute(StateContext context) { 165 | String appName = context.getExtendedState().get(VAR_APPNAME, String.class); 166 | String applicationId = context.getExtendedState().get(VAR_APPLICATION_ID, String.class); 167 | CloudAppInstanceInfo appInstanceInfo = findRunningInstance(appName); 168 | if (appInstanceInfo != null && appInstanceInfo.getApplicationId().equals(applicationId)) { 169 | context.getExtendedState().getVariables().put(VAR_INSTANCE_ADDRESS, appInstanceInfo.getAddress()); 170 | } 171 | } 172 | } 173 | 174 | private class CreateClusterAction implements Action { 175 | 176 | @SuppressWarnings("unchecked") 177 | @Override 178 | public void execute(StateContext context) { 179 | Resource artifact = context.getExtendedState().get(VAR_ARTIFACT, Resource.class); 180 | getYarnCloudAppService().createCluster( 181 | context.getExtendedState().get(VAR_APPLICATION_ID, String.class), 182 | context.getExtendedState().get(HEADER_CLUSTER_ID, String.class), 183 | context.getExtendedState().get(VAR_COUNT, Integer.class), 184 | artifact != null ? artifact.getFilename() : null, 185 | context.getExtendedState().get(HEADER_DEFINITION_PARAMETERS, Map.class)); 186 | } 187 | } 188 | 189 | private class StartClusterAction implements Action { 190 | 191 | @Override 192 | public void execute(StateContext context) { 193 | getYarnCloudAppService().startCluster(context.getExtendedState().get(VAR_APPLICATION_ID, String.class), context 194 | .getExtendedState().get(HEADER_CLUSTER_ID, String.class)); 195 | } 196 | } 197 | 198 | private class DeployAction implements Action { 199 | 200 | @Override 201 | public void execute(StateContext context) { 202 | Integer count = context.getMessageHeaders().get(HEADER_COUNT, Integer.class); 203 | String clusterId = context.getMessageHeaders().get(HEADER_CLUSTER_ID, String.class); 204 | Map definitionParameters = context.getMessageHeaders().get(HEADER_DEFINITION_PARAMETERS, Map.class); 205 | Resource artifact = context.getMessageHeaders().get(HEADER_ARTIFACT, Resource.class); 206 | context.getExtendedState().getVariables().put(VAR_COUNT, count != null ? count : 1); 207 | context.getExtendedState().getVariables().put(VAR_CLUSTER_ID, clusterId); 208 | context.getExtendedState().getVariables().put(VAR_DEFINITION_PARAMETERS, definitionParameters); 209 | if (artifact != null) { 210 | context.getExtendedState().getVariables().put(VAR_ARTIFACT, artifact); 211 | } 212 | context.getExtendedState().getVariables().put(VAR_MESSAGE_ID, context.getMessageHeaders().getId().toString()); 213 | } 214 | } 215 | 216 | private class StartInstanceGuard implements Guard { 217 | 218 | @Override 219 | public boolean evaluate(StateContext context) { 220 | return !context.getExtendedState().getVariables().containsKey(VAR_APPLICATION_ID); 221 | } 222 | } 223 | 224 | private class StopClusterAction implements Action { 225 | 226 | @Override 227 | public void execute(StateContext context) { 228 | String clusterId = context.getMessageHeaders().get(HEADER_CLUSTER_ID, String.class); 229 | String applicationId = (String) context.getMessageHeader(HEADER_APPLICATION_ID); 230 | getYarnCloudAppService().stopCluster(applicationId, clusterId); 231 | } 232 | } 233 | 234 | private class DestroyClusterAction implements Action { 235 | 236 | @Override 237 | public void execute(StateContext context) { 238 | String clusterId = context.getMessageHeaders().get(HEADER_CLUSTER_ID, String.class); 239 | String applicationId = (String) context.getMessageHeader(HEADER_APPLICATION_ID); 240 | getYarnCloudAppService().destroyCluster(applicationId, clusterId); 241 | } 242 | } 243 | 244 | private class InstanceGuard implements Guard { 245 | 246 | @Override 247 | public boolean evaluate(StateContext context) { 248 | return context.getExtendedState().getVariables().containsKey(VAR_INSTANCE_ADDRESS); 249 | } 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/DefaultYarnCloudAppService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2017 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.springframework.cloud.deployer.spi.yarn; 17 | 18 | import java.io.File; 19 | import java.io.FileOutputStream; 20 | import java.util.ArrayList; 21 | import java.util.Collection; 22 | import java.util.HashMap; 23 | import java.util.List; 24 | import java.util.Map; 25 | import java.util.Properties; 26 | import java.util.UUID; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.yarn.util.ConverterUtils; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | import org.springframework.beans.factory.InitializingBean; 33 | import org.springframework.beans.factory.annotation.Autowired; 34 | import org.springframework.context.ApplicationContextInitializer; 35 | import org.springframework.core.io.Resource; 36 | import org.springframework.data.hadoop.fs.FsShell; 37 | import org.springframework.util.FileCopyUtils; 38 | import org.springframework.util.ObjectUtils; 39 | import org.springframework.util.StringUtils; 40 | 41 | /** 42 | * Default implementation of {@link YarnCloudAppService} which talks to rest 43 | * api's exposed by specific yarn controlling container clusters. 44 | * 45 | * @author Janne Valkealahti 46 | * @author Mark Fisher 47 | */ 48 | public class DefaultYarnCloudAppService implements YarnCloudAppService, InitializingBean { 49 | 50 | private static final Logger logger = LoggerFactory.getLogger(DefaultYarnCloudAppService.class); 51 | private final ApplicationContextInitializer[] initializers; 52 | private final String deployerVersion; 53 | private final Map appCache = new HashMap(); 54 | 55 | private Configuration configuration; 56 | 57 | /** 58 | * Instantiates a new default yarn cloud app service. 59 | * 60 | * @param deployerVersion the deployer version 61 | */ 62 | public DefaultYarnCloudAppService(String deployerVersion) { 63 | this(deployerVersion, null); 64 | } 65 | 66 | /** 67 | * Instantiates a new default yarn cloud app service. 68 | * 69 | * @param deployerVersion the deployer version 70 | * @param initializers the initializers 71 | */ 72 | public DefaultYarnCloudAppService(String deployerVersion, ApplicationContextInitializer[] initializers) { 73 | this.deployerVersion = deployerVersion; 74 | this.initializers = initializers; 75 | } 76 | 77 | @Autowired 78 | public void setConfiguration(Configuration configuration) { 79 | this.configuration = configuration; 80 | } 81 | 82 | @Override 83 | public void pushArtifact(Resource artifact, String dir) { 84 | File tmp = null; 85 | try { 86 | tmp = File.createTempFile(UUID.randomUUID().toString(), null); 87 | tmp.deleteOnExit(); 88 | FileCopyUtils.copy(artifact.getInputStream(), new FileOutputStream(tmp)); 89 | @SuppressWarnings("resource") 90 | FsShell shell = new FsShell(configuration); 91 | String artifactPath = dir + "/" + artifact.getFile().getName(); 92 | if (!shell.test(artifactPath)) { 93 | logger.info("Pushing artifact {} into dir {}", artifact, dir); 94 | shell.copyFromLocal(tmp.getAbsolutePath(), artifactPath); 95 | } 96 | } catch (Exception e) { 97 | logger.error("Error pushing artifact", e); 98 | } finally { 99 | if (tmp != null) { 100 | try { 101 | tmp.delete(); 102 | } catch (Exception e) { 103 | } 104 | } 105 | } 106 | } 107 | 108 | @Override 109 | public void afterPropertiesSet() throws Exception { 110 | } 111 | 112 | @Override 113 | public Collection getApplications(CloudAppType cloudAppType) { 114 | return getApp(null, null, cloudAppType).getPushedApplications(); 115 | } 116 | 117 | @Override 118 | public Collection getInstances(CloudAppType cloudAppType) { 119 | return getApp(null, null, cloudAppType).getSubmittedApplications(); 120 | } 121 | 122 | @Override 123 | public void pushApplication(String appVersion, CloudAppType cloudAppType) { 124 | getApp(appVersion, deployerVersion, cloudAppType, null).pushApplication(appVersion); 125 | } 126 | 127 | @Override 128 | public String submitApplication(String appVersion, CloudAppType cloudAppType) { 129 | return submitApplication(appVersion, cloudAppType, null); 130 | } 131 | 132 | @Override 133 | public String submitApplication(String appVersion, CloudAppType cloudAppType, List contextRunArgs) { 134 | return getApp(appVersion, deployerVersion, cloudAppType, contextRunArgs).submitApplication(appVersion); 135 | } 136 | 137 | @Override 138 | public void killApplications(String appName, CloudAppType cloudAppType) { 139 | YarnCloudAppServiceApplication app = getApp(null, null, cloudAppType); 140 | Collection submittedApplications = app.getSubmittedApplications(); 141 | for (CloudAppInstanceInfo info : submittedApplications) { 142 | if (info.getName() == appName) { 143 | app.killApplication(info.getApplicationId()); 144 | } 145 | } 146 | } 147 | 148 | @Override 149 | public void killApplication(String yarnApplicationId, CloudAppType cloudAppType) { 150 | YarnCloudAppServiceApplication app = getApp(null, null, cloudAppType); 151 | app.killApplication(yarnApplicationId); 152 | } 153 | 154 | @Override 155 | public void createCluster(String yarnApplicationId, String clusterId, int count, String artifact, 156 | Map definitionParameters) { 157 | 158 | Map extraProperties = new HashMap(); 159 | extraProperties.put("containerArtifact", artifact); 160 | 161 | int i = 0; 162 | for (Map.Entry entry : definitionParameters.entrySet()) { 163 | String value = entry.getValue(); 164 | if (value.startsWith("\"") && value.endsWith("\"")) { 165 | // prevent ${xxx.ddd} bad substitution with bash 166 | value = value.replace("$", "\\\\\\$"); 167 | // escape existing double quotes 168 | extraProperties.put("containerArg" + i++, entry.getKey() + "=\\" + value.substring(0, value.length()-1) + "\\\""); 169 | } else { 170 | // prevent ${xxx.ddd} bad substitution with bash 171 | value = value.replace("$", "\\\\\\$"); 172 | // escape with extra double quotes 173 | extraProperties.put("containerArg" + i++, entry.getKey() + "=\\\"" + value + "\\\""); 174 | } 175 | } 176 | getApp(null, null, CloudAppType.STREAM).createCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId, "module-template", 177 | "default", count, null, null, null, extraProperties); 178 | } 179 | 180 | @Override 181 | public void startCluster(String yarnApplicationId, String clusterId) { 182 | getApp(null, null, CloudAppType.STREAM).startCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId); 183 | } 184 | 185 | @Override 186 | public void stopCluster(String yarnApplicationId, String clusterId) { 187 | getApp(null, null, CloudAppType.STREAM).stopCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId); 188 | } 189 | 190 | @Override 191 | public Map getClustersStates(String yarnApplicationId) { 192 | HashMap states = new HashMap(); 193 | Collection submittedApplications = getApp(null, null, CloudAppType.STREAM).getSubmittedApplications(yarnApplicationId); 194 | for (CloudAppInstanceInfo instanceInfo : submittedApplications) { 195 | if (instanceInfo.getName().startsWith("scdstream:") && instanceInfo.getState().equals("RUNNING")) { 196 | for (String cluster : getClusters(instanceInfo.getApplicationId())) { 197 | states.putAll(getInstanceClustersStates(instanceInfo.getApplicationId(), cluster)); 198 | } 199 | } 200 | } 201 | return states; 202 | } 203 | 204 | @Override 205 | public Collection getClusters(String yarnApplicationId) { 206 | return getApp(null, null, CloudAppType.STREAM).getClustersInfo(ConverterUtils.toApplicationId(yarnApplicationId)); 207 | } 208 | 209 | @Override 210 | public void destroyCluster(String yarnApplicationId, String clusterId) { 211 | getApp(null, null, CloudAppType.STREAM).destroyCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId); 212 | } 213 | 214 | private Map getInstanceClustersStates(String yarnApplicationId, String clusterId) { 215 | HashMap states = new HashMap(); 216 | List clusterInfo = getApp(null, null, CloudAppType.STREAM) 217 | .getClusterInfo(ConverterUtils.toApplicationId(yarnApplicationId), clusterId); 218 | if (clusterInfo.size() == 1) { 219 | states.put(clusterId, clusterInfo.get(0)); 220 | } 221 | return states; 222 | } 223 | 224 | protected List processContextRunArgs(List contextRunArgs) { 225 | return contextRunArgs; 226 | } 227 | 228 | private synchronized YarnCloudAppServiceApplication getApp(String appVersion, String dataflowVersion, CloudAppType cloudAppType) { 229 | return getApp(appVersion, dataflowVersion, cloudAppType, null); 230 | } 231 | 232 | private synchronized YarnCloudAppServiceApplication getApp(String appVersion, String dataflowVersion, CloudAppType cloudAppType, 233 | List contextRunArgs) { 234 | contextRunArgs = processContextRunArgs(contextRunArgs); 235 | String cacheKey = cloudAppType + appVersion + StringUtils.collectionToCommaDelimitedString(contextRunArgs); 236 | YarnCloudAppServiceApplication app = appCache.get(cacheKey); 237 | logger.info("Cachekey {} found YarnCloudAppServiceApplication {}", cacheKey, app); 238 | if (app == null) { 239 | Properties configFileProperties = new Properties(); 240 | if (StringUtils.hasText(appVersion)) { 241 | configFileProperties.setProperty("spring.yarn.applicationVersion", appVersion); 242 | } 243 | if (StringUtils.hasText(dataflowVersion)) { 244 | configFileProperties.setProperty("spring.cloud.deployer.yarn.version", dataflowVersion); 245 | } 246 | 247 | logger.info("Bootsrapping YarnCloudAppServiceApplication with {}", cloudAppType.toString().toLowerCase()); 248 | ArrayList runArgs = new ArrayList(); 249 | runArgs.add("--spring.config.name=" + cloudAppType.toString().toLowerCase()); 250 | runArgs.add("--spring.jmx.enabled=false"); 251 | 252 | if (!ObjectUtils.isEmpty(contextRunArgs)) { 253 | runArgs.addAll(contextRunArgs); 254 | } 255 | 256 | app = new YarnCloudAppServiceApplication(appVersion, dataflowVersion, "application.properties", configFileProperties, 257 | runArgs.toArray(new String[0]), initializers); 258 | try { 259 | app.afterPropertiesSet(); 260 | } catch (Exception e) { 261 | throw new RuntimeException("Error initializing YarnCloudAppServiceApplication", e); 262 | } 263 | logger.info("Set cache with key {} and YarnCloudAppServiceApplication {}", cacheKey, app); 264 | appCache.put(cacheKey, app); 265 | } 266 | return app; 267 | } 268 | 269 | } 270 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/DeployerApplicationYarnClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import org.apache.hadoop.yarn.api.records.ApplicationId; 20 | import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; 21 | import org.springframework.yarn.client.ClientRmOperations; 22 | import org.springframework.yarn.client.DefaultApplicationYarnClient; 23 | import org.springframework.yarn.client.YarnClient; 24 | 25 | /** 26 | * Custom {@link YarnClient} handling deployer specific client operations. 27 | * 28 | * @author Janne Valkealahti 29 | * 30 | */ 31 | public class DeployerApplicationYarnClient extends DefaultApplicationYarnClient { 32 | 33 | /** 34 | * Instantiates a new deployer application yarn client. 35 | * 36 | * @param clientRmOperations the client rm operations 37 | */ 38 | public DeployerApplicationYarnClient(ClientRmOperations clientRmOperations) { 39 | super(clientRmOperations); 40 | } 41 | 42 | @Override 43 | protected ApplicationSubmissionContext getSubmissionContext(ApplicationId applicationId) { 44 | ApplicationSubmissionContext submissionContext = super.getSubmissionContext(applicationId); 45 | // force app fail after first appmaster failure regardles 46 | // what's set on yarn level. 47 | // TODO: should make configurable in spring yarn if possible 48 | submissionContext.setMaxAppAttempts(1); 49 | return submissionContext; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/StreamClustersInfoReportData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.springframework.cloud.deployer.spi.yarn; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData; 22 | 23 | public class StreamClustersInfoReportData extends ClustersInfoReportData { 24 | 25 | private final List members; 26 | 27 | public StreamClustersInfoReportData(String state, Integer count, Integer pany, Map phosts, Map pracks, 28 | Integer sany, Map shosts, Map sracks, List members) { 29 | super(state, count, pany, phosts, pracks, sany, shosts, sracks); 30 | this.members = members; 31 | } 32 | 33 | public List getMembers() { 34 | return members; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/TaskLauncherStateMachine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | import org.springframework.beans.factory.BeanFactory; 26 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType; 27 | import org.springframework.core.io.ResourceLoader; 28 | import org.springframework.core.task.TaskExecutor; 29 | import org.springframework.statemachine.StateContext; 30 | import org.springframework.statemachine.action.Action; 31 | import org.springframework.statemachine.guard.Guard; 32 | 33 | /** 34 | * Statemachine for task launching. 35 | * 36 | * @author Janne Valkealahti 37 | * 38 | */ 39 | public class TaskLauncherStateMachine extends AbstractDeployerStateMachine { 40 | 41 | private static final Logger logger = LoggerFactory.getLogger(TaskLauncherStateMachine.class); 42 | private final static String MODEL_LOCATION = "classpath:tasklauncher-model.uml"; 43 | 44 | public final static String EVENT_LAUNCH = "LAUNCH"; 45 | public final static String EVENT_CANCEL = "CANCEL"; 46 | 47 | /** 48 | * Instantiates a new task launcher state machine. 49 | * 50 | * @param yarnCloudAppService the yarn cloud app service 51 | * @param taskExecutor the task executor 52 | * @param beanFactory the bean factory 53 | * @param resourceLoader the resource loader 54 | */ 55 | public TaskLauncherStateMachine(YarnCloudAppService yarnCloudAppService, TaskExecutor taskExecutor, BeanFactory beanFactory, 56 | ResourceLoader resourceLoader) { 57 | super(yarnCloudAppService, taskExecutor, beanFactory, resourceLoader, MODEL_LOCATION); 58 | } 59 | 60 | @Override 61 | protected Map> getRegisteredActions() { 62 | HashMap> actions = new HashMap<>(); 63 | actions.put("resetVariablesAction", new ResetVariablesAction()); 64 | actions.put("launchAction", new LaunchAction()); 65 | actions.put("cancelAction", new CancelAction()); 66 | actions.put("checkAppAction", new CheckAppAction(CloudAppType.TASK)); 67 | actions.put("pushAppAction", new PushAppAction(CloudAppType.TASK)); 68 | actions.put("pushArtifactAction", new PushArtifactAction()); 69 | actions.put("startAppAction", new StartAppAction()); 70 | actions.put("stopAppAction", new StopAppAction()); 71 | actions.put("errorHandlingAction", new ErrorAction()); 72 | return actions; 73 | } 74 | 75 | @Override 76 | protected Map> getRegisteredGuards() { 77 | HashMap> guards = new HashMap<>(); 78 | guards.put("pushAppGuard", new PushAppGuard()); 79 | guards.put("errorGuard", new ErrorGuard()); 80 | return guards; 81 | } 82 | 83 | private class LaunchAction implements Action { 84 | 85 | @Override 86 | public void execute(StateContext context) { 87 | context.getExtendedState().getVariables().put(VAR_MESSAGE_ID, context.getMessageHeaders().getId().toString()); 88 | } 89 | } 90 | 91 | private class CancelAction implements Action { 92 | 93 | @Override 94 | public void execute(StateContext context) { 95 | context.getExtendedState().getVariables().put(VAR_MESSAGE_ID, context.getMessageHeaders().getId().toString()); 96 | } 97 | } 98 | 99 | private class StartAppAction implements Action { 100 | 101 | @Override 102 | public void execute(StateContext context) { 103 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION); 104 | 105 | // we control type so casting is safe 106 | @SuppressWarnings("unchecked") 107 | List contextRunArgs = (List) context.getMessageHeader(HEADER_CONTEXT_RUN_ARGS); 108 | 109 | String applicationId = getYarnCloudAppService().submitApplication(appVersion, CloudAppType.TASK, contextRunArgs); 110 | logger.info("New application id is {}", applicationId); 111 | context.getExtendedState().getVariables().put(VAR_APPLICATION_ID, applicationId); 112 | } 113 | } 114 | 115 | private class StopAppAction implements Action { 116 | 117 | @Override 118 | public void execute(StateContext context) { 119 | String applicationId = (String) context.getMessageHeader(HEADER_APPLICATION_ID); 120 | logger.info("Killing application {}", applicationId); 121 | getYarnCloudAppService().killApplication(applicationId, CloudAppType.TASK); 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnAppDeployer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2017 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.io.IOException; 20 | import java.util.ArrayList; 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.Map.Entry; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | import org.apache.commons.io.FilenameUtils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | import org.springframework.beans.factory.annotation.Autowired; 31 | import org.springframework.cloud.deployer.spi.app.AppDeployer; 32 | import org.springframework.cloud.deployer.spi.app.AppInstanceStatus; 33 | import org.springframework.cloud.deployer.spi.app.AppStatus; 34 | import org.springframework.cloud.deployer.spi.app.AppStatus.Builder; 35 | import org.springframework.cloud.deployer.spi.app.DeploymentState; 36 | import org.springframework.cloud.deployer.spi.core.AppDefinition; 37 | import org.springframework.cloud.deployer.spi.core.AppDeploymentRequest; 38 | import org.springframework.cloud.deployer.spi.core.RuntimeEnvironmentInfo; 39 | import org.springframework.cloud.deployer.spi.util.RuntimeVersionUtils; 40 | import org.springframework.core.io.Resource; 41 | import org.springframework.messaging.Message; 42 | import org.springframework.messaging.support.MessageBuilder; 43 | import org.springframework.statemachine.StateContext; 44 | import org.springframework.statemachine.StateMachine; 45 | import org.springframework.statemachine.StateContext.Stage; 46 | import org.springframework.statemachine.listener.StateMachineListener; 47 | import org.springframework.statemachine.listener.StateMachineListenerAdapter; 48 | import org.springframework.util.Assert; 49 | import org.springframework.util.ObjectUtils; 50 | import org.springframework.util.StringUtils; 51 | import org.springframework.util.concurrent.ListenableFutureCallback; 52 | import org.springframework.util.concurrent.SettableListenableFuture; 53 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData; 54 | 55 | /** 56 | * {@link ModuleDeployer} which communicates with a Yarn app running 57 | * on a Hadoop cluster waiting for deployment requests. This app 58 | * uses Spring Yarn's container grouping functionality to create a 59 | * new group per module type. This allows all modules to share the 60 | * same settings and the group itself can controlled, i.e. ramp up/down 61 | * or shutdown/destroy a whole group. 62 | * 63 | * @author Janne Valkealahti 64 | */ 65 | public class YarnAppDeployer implements AppDeployer { 66 | 67 | private static final Logger logger = LoggerFactory.getLogger(YarnAppDeployer.class); 68 | private final YarnCloudAppService yarnCloudAppService; 69 | private final StateMachine stateMachine; 70 | 71 | @Autowired 72 | private YarnDeployerProperties yarnDeployerProperties; 73 | 74 | /** 75 | * Instantiates a new yarn stream module deployer. 76 | * 77 | * @param yarnCloudAppService the yarn cloud app service 78 | * @param stateMachine the state machine 79 | */ 80 | public YarnAppDeployer(YarnCloudAppService yarnCloudAppService, StateMachine stateMachine) { 81 | this.yarnCloudAppService = yarnCloudAppService; 82 | this.stateMachine = stateMachine; 83 | } 84 | 85 | @Override 86 | public String deploy(AppDeploymentRequest request) { 87 | logger.info("Deploy request for {}", request); 88 | final AppDefinition definition = request.getDefinition(); 89 | Map definitionParameters = definition.getProperties(); 90 | Map deploymentProperties = request.getDeploymentProperties(); 91 | logger.info("Deploying request for definition {}", definition); 92 | logger.info("Parameters for definition {}", definitionParameters); 93 | logger.info("Deployment properties for request {}", deploymentProperties); 94 | 95 | int count = 1; 96 | String countString = request.getDeploymentProperties().get(AppDeployer.COUNT_PROPERTY_KEY); 97 | if (StringUtils.hasText(countString)) { 98 | count = Integer.parseInt(countString); 99 | } 100 | final String group = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY); 101 | Resource resource = request.getResource(); 102 | final String clusterId = group + ":" + definition.getName(); 103 | 104 | String appVersion = StringUtils.hasText(yarnDeployerProperties.getAppVersion()) ? yarnDeployerProperties.getAppVersion() : "app"; 105 | if (deploymentProperties.containsKey("spring.cloud.deployer.yarn.app.appVersion")) { 106 | appVersion = deploymentProperties.get("spring.cloud.deployer.yarn.app.appVersion"); 107 | } 108 | // contextRunArgs are passed to boot app ran to control yarn apps 109 | ArrayList contextRunArgs = new ArrayList(); 110 | contextRunArgs.add("--spring.yarn.appName=scdstream:" + appVersion + ":" + group); 111 | 112 | // deployment properties override servers.yml which overrides application.yml 113 | for (Entry entry : deploymentProperties.entrySet()) { 114 | if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.streamappmaster")) { 115 | contextRunArgs.add("--" + entry.getKey() + "=" + entry.getValue()); 116 | } else if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.streamcontainer")) { 117 | // weird format with '--' is just straight pass to appmaster 118 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--" + entry.getKey() + "='" + entry.getValue() + "'"); 119 | } 120 | } 121 | 122 | String baseDir = yarnDeployerProperties.getBaseDir(); 123 | if (!baseDir.endsWith("/")) { 124 | baseDir = baseDir + "/"; 125 | } 126 | 127 | String artifactPath = isHdfsResource(resource) ? getHdfsArtifactPath(resource) : baseDir + "/artifacts/cache/"; 128 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.artifact=" + artifactPath); 129 | 130 | // add group as 'spring.cloud.application.group' 131 | definitionParameters = new HashMap<>(definitionParameters); 132 | definitionParameters.put("spring.cloud.application.group", group); 133 | 134 | final Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_DEPLOY) 135 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, appVersion) 136 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, clusterId) 137 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, group) 138 | .setHeader(AppDeployerStateMachine.HEADER_COUNT, count) 139 | .setHeader(AppDeployerStateMachine.HEADER_ARTIFACT, resource) 140 | .setHeader(AppDeployerStateMachine.HEADER_ARTIFACT_DIR, artifactPath) 141 | .setHeader(AppDeployerStateMachine.HEADER_DEFINITION_PARAMETERS, definitionParameters) 142 | .setHeader(AppDeployerStateMachine.HEADER_CONTEXT_RUN_ARGS, contextRunArgs) 143 | .build(); 144 | 145 | // Use of future here is to set id when it becomes available from machine 146 | final SettableListenableFuture id = new SettableListenableFuture<>(); 147 | final StateMachineListener listener = new StateMachineListenerAdapter() { 148 | 149 | @Override 150 | public void stateContext(StateContext stateContext) { 151 | if (stateContext.getStage() == Stage.STATE_ENTRY && stateContext.getTarget().getId().equals(AppDeployerStateMachine.STATE_READY)) { 152 | if (ObjectUtils.nullSafeEquals(message.getHeaders().getId().toString(), 153 | stateContext.getExtendedState().get(AppDeployerStateMachine.VAR_MESSAGE_ID, String.class))) { 154 | Exception exception = stateContext.getExtendedState().get(AppDeployerStateMachine.VAR_ERROR, Exception.class); 155 | if (exception != null) { 156 | id.setException(exception); 157 | } else { 158 | String applicationId = stateContext.getStateMachine().getExtendedState() 159 | .get(AppDeployerStateMachine.VAR_APPLICATION_ID, String.class); 160 | DeploymentKey key = new DeploymentKey(group, definition.getName(), applicationId); 161 | id.set(key.toString()); 162 | } 163 | } 164 | } 165 | } 166 | }; 167 | stateMachine.addStateListener(listener); 168 | id.addCallback(new ListenableFutureCallback() { 169 | 170 | @Override 171 | public void onSuccess(String result) { 172 | stateMachine.removeStateListener(listener); 173 | } 174 | 175 | @Override 176 | public void onFailure(Throwable ex) { 177 | stateMachine.removeStateListener(listener); 178 | } 179 | }); 180 | 181 | stateMachine.sendEvent(message); 182 | // we need to block here until SPI supports 183 | // returning id asynchronously 184 | try { 185 | return id.get(2, TimeUnit.MINUTES); 186 | } catch (Exception e) { 187 | throw new RuntimeException(e); 188 | } 189 | } 190 | 191 | @Override 192 | public void undeploy(String id) { 193 | logger.info("Undeploy request for id {}", id); 194 | DeploymentKey key = new DeploymentKey(id); 195 | Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_UNDEPLOY) 196 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, key.getClusterId()) 197 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, "app") 198 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, key.group) 199 | .setHeader(AppDeployerStateMachine.HEADER_APPLICATION_ID, key.applicationId) 200 | .build(); 201 | stateMachine.sendEvent(message); 202 | } 203 | 204 | @Override 205 | public AppStatus status(String id) { 206 | logger.info("Checking status of {}", id); 207 | DeploymentKey key = new DeploymentKey(id); 208 | Builder builder = AppStatus.of(id); 209 | for (Entry entry : yarnCloudAppService.getClustersStates(key.applicationId).entrySet()) { 210 | if (ObjectUtils.nullSafeEquals(entry.getKey(), key.getClusterId())) { 211 | ClustersInfoReportData data = entry.getValue(); 212 | List members = entry.getValue().getMembers(); 213 | for (int i = 0 ; i < data.getProjectionAny(); i++) { 214 | Map attributes = new HashMap<>(); 215 | if (i < members.size()) { 216 | attributes.put("container", members.get(i)); 217 | attributes.put("guid", members.get(i)); 218 | } 219 | InstanceStatus instanceStatus = new InstanceStatus(key.getClusterId() + "-" + i, i < data.getCount(), attributes); 220 | builder.with(instanceStatus); 221 | } 222 | break; 223 | } 224 | } 225 | return builder.build(); 226 | } 227 | 228 | @Override 229 | public RuntimeEnvironmentInfo environmentInfo() { 230 | return new RuntimeEnvironmentInfo.Builder() 231 | .spiClass(AppDeployer.class) 232 | .implementationName(getClass().getSimpleName()) 233 | .implementationVersion(RuntimeVersionUtils.getVersion(this.getClass())) 234 | .platformType("Yarn") 235 | .platformApiVersion(System.getProperty("os.name") + " " + System.getProperty("os.version")) 236 | .platformClientVersion(System.getProperty("os.version")) 237 | .platformHostVersion(System.getProperty("os.version")) 238 | .build(); 239 | } 240 | 241 | private boolean isHdfsResource(Resource resource) { 242 | try { 243 | return resource != null && resource.getURI().getScheme().equals("hdfs"); 244 | } catch (IOException e) { 245 | return false; 246 | } 247 | } 248 | 249 | private String getHdfsArtifactPath(Resource resource) { 250 | String path = null; 251 | try { 252 | path = "/" + FilenameUtils.getPath(resource.getURI().getPath()); 253 | } catch (IOException e) { 254 | } 255 | return path; 256 | } 257 | 258 | private static class DeploymentKey { 259 | final static String SEPARATOR = ":"; 260 | final String group; 261 | final String name; 262 | final String applicationId; 263 | final String clusterId; 264 | 265 | public DeploymentKey(String id) { 266 | String[] split = id.split(SEPARATOR); 267 | Assert.isTrue(split.length == 3, "Unable to parse deployment key " + id); 268 | group = split[0]; 269 | name = split[1]; 270 | applicationId = split[2]; 271 | clusterId = group + SEPARATOR + name; 272 | } 273 | 274 | public DeploymentKey(String group, String name, String applicationId) { 275 | Assert.notNull(group, "Group must be set"); 276 | Assert.notNull(name, "Name must be set"); 277 | Assert.notNull(applicationId, "Application id must be set"); 278 | this.group = group; 279 | this.name = name; 280 | this.applicationId = applicationId; 281 | clusterId = group + SEPARATOR + name; 282 | } 283 | 284 | public String getClusterId() { 285 | return clusterId; 286 | } 287 | 288 | @Override 289 | public String toString() { 290 | return group + SEPARATOR + name + SEPARATOR + applicationId; 291 | } 292 | } 293 | 294 | private static class InstanceStatus implements AppInstanceStatus { 295 | private final String id; 296 | private final DeploymentState state; 297 | private final Map attributes = new HashMap(); 298 | 299 | public InstanceStatus(String id, boolean deployed, Map attributes) { 300 | this.id = id; 301 | this.state = deployed ? DeploymentState.deployed : DeploymentState.unknown; 302 | if (attributes != null) { 303 | this.attributes.putAll(attributes); 304 | } 305 | } 306 | 307 | @Override 308 | public String getId() { 309 | return id; 310 | } 311 | 312 | @Override 313 | public DeploymentState getState() { 314 | return state; 315 | } 316 | 317 | @Override 318 | public Map getAttributes() { 319 | return attributes; 320 | } 321 | 322 | @Override 323 | public String toString() { 324 | return "InstanceStatus [id=" + id + ", state=" + state + ", attributes=" + attributes + "]"; 325 | } 326 | } 327 | } 328 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnCloudAppService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.util.Collection; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import org.springframework.core.io.Resource; 24 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData; 25 | 26 | /** 27 | * Interface used to glue a state machine and yarn application logic together. 28 | * 29 | * @author Janne Valkealahti 30 | */ 31 | public interface YarnCloudAppService { 32 | 33 | /** 34 | * Get applications pushed to hdfs. 35 | * 36 | * @return the applications 37 | */ 38 | Collection getApplications(CloudAppType cloudAppType); 39 | 40 | /** 41 | * Get running application instances. 42 | * 43 | * @return the instances 44 | */ 45 | Collection getInstances(CloudAppType cloudAppType); 46 | 47 | /** 48 | * Push new application into hdfs. Push operation is copying needed 49 | * files into hdfs without trying to start a new application instance. 50 | * 51 | * @param appVersion the app version 52 | * @param cloudAppType the cloud app type 53 | */ 54 | void pushApplication(String appVersion, CloudAppType cloudAppType); 55 | 56 | /** 57 | * Submit new application instance into yarn. Prior to calling a new 58 | * submit operation, application has to exist in hdfs which can be done 59 | * i.e. using {@link #pushApplication(String)} method. 60 | * 61 | * @param appVersion the app version 62 | * @param cloudAppType the cloud app type 63 | * @return the application id 64 | */ 65 | String submitApplication(String appVersion, CloudAppType cloudAppType); 66 | 67 | /** 68 | * Submit new application instance into yarn. Prior to calling a new 69 | * submit operation, application has to exist in hdfs which can be done 70 | * i.e. using {@link #pushApplication(String)} method. 71 | * 72 | * @param appVersion the app version 73 | * @param cloudAppType the cloud app type 74 | * @param contextRunArgs the context run arguments 75 | * @return the application id 76 | */ 77 | String submitApplication(String appVersion, CloudAppType cloudAppType, List contextRunArgs); 78 | 79 | /** 80 | * Kill all applications matching given application name. 81 | * 82 | * @param appName the application name 83 | */ 84 | void killApplications(String appName, CloudAppType cloudAppType); 85 | 86 | /** 87 | * Kill application with given yarn application id. 88 | * 89 | * @param yarnApplicationId the yarn application id 90 | * @param cloudAppType the cloud app type 91 | */ 92 | void killApplication(String yarnApplicationId, CloudAppType cloudAppType); 93 | 94 | /** 95 | * Creates the container cluster. 96 | * 97 | * @param yarnApplicationId the yarn application id 98 | * @param clusterId the cluster id 99 | * @param count the count 100 | * @param module the module 101 | * @param definitionParameters the definition parameters 102 | */ 103 | void createCluster(String yarnApplicationId, String clusterId, int count, String artifact, 104 | Map definitionParameters); 105 | 106 | /** 107 | * Start a container cluster. 108 | * 109 | * @param yarnApplicationId the yarn application id 110 | * @param clusterId the cluster id 111 | */ 112 | void startCluster(String yarnApplicationId, String clusterId); 113 | 114 | /** 115 | * Stop a container cluster. 116 | * 117 | * @param yarnApplicationId the yarn application id 118 | * @param clusterId the cluster id 119 | */ 120 | void stopCluster(String yarnApplicationId, String clusterId); 121 | 122 | /** 123 | * Gets the clusters states. Returned map has a mapping between 124 | * yarn container cluster id and its state known by application master 125 | * using container clusters. 126 | * 127 | * @param yarnApplicationId the yarn application id 128 | * @return the clusters states 129 | */ 130 | Map getClustersStates(String yarnApplicationId); 131 | 132 | /** 133 | * Gets the clusters. 134 | * 135 | * @param yarnApplicationId the yarn application id 136 | * @return the clusters 137 | */ 138 | Collection getClusters(String yarnApplicationId); 139 | 140 | /** 141 | * Destroy cluster. 142 | * 143 | * @param yarnApplicationId the yarn application id 144 | * @param clusterId the cluster id 145 | */ 146 | void destroyCluster(String yarnApplicationId, String clusterId); 147 | 148 | /** 149 | * Push artifact into a hdfs directory. 150 | * 151 | * @param artifact the artifact 152 | * @param dir the dir 153 | */ 154 | void pushArtifact(Resource artifact, String dir); 155 | 156 | /** 157 | * Wrapping info about application pushed into hdfs. 158 | */ 159 | public class CloudAppInfo { 160 | 161 | private final String name; 162 | 163 | public CloudAppInfo(String name) { 164 | this.name = name; 165 | } 166 | 167 | public String getName() { 168 | return name; 169 | } 170 | 171 | } 172 | 173 | /** 174 | * Wrapping info about running application. 175 | */ 176 | public class CloudAppInstanceInfo { 177 | 178 | private final String applicationId; 179 | private final String name; 180 | private final String state; 181 | private final String address; 182 | 183 | public CloudAppInstanceInfo(String applicationId, String name, String state, String address) { 184 | this.applicationId = applicationId; 185 | this.name = name; 186 | this.state = state; 187 | this.address = address; 188 | } 189 | 190 | public String getApplicationId() { 191 | return applicationId; 192 | } 193 | 194 | public String getName() { 195 | return name; 196 | } 197 | 198 | public String getState() { 199 | return state; 200 | } 201 | 202 | public String getAddress() { 203 | return address; 204 | } 205 | 206 | @Override 207 | public String toString() { 208 | return "CloudAppInstanceInfo [applicationId=" + applicationId + ", name=" + name + ", state=" + state + ", address=" + address 209 | + "]"; 210 | } 211 | 212 | } 213 | 214 | public enum CloudAppType { 215 | STREAM, 216 | TASK 217 | } 218 | 219 | } 220 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnDeployerProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import org.springframework.boot.context.properties.ConfigurationProperties; 20 | 21 | @ConfigurationProperties(value = "spring.cloud.deployer.yarn.app") 22 | public class YarnDeployerProperties { 23 | 24 | private String baseDir = "/dataflow"; 25 | private String appVersion; 26 | 27 | public String getBaseDir() { 28 | return baseDir; 29 | } 30 | 31 | public void setBaseDir(String baseDir) { 32 | this.baseDir = baseDir; 33 | } 34 | 35 | public String getAppVersion() { 36 | return appVersion; 37 | } 38 | 39 | public void setAppVersion(String appVersion) { 40 | this.appVersion = appVersion; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnTaskLauncher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.io.IOException; 20 | import java.util.ArrayList; 21 | import java.util.Collection; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.Map.Entry; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | import org.apache.commons.io.FilenameUtils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | import org.springframework.beans.factory.annotation.Autowired; 31 | import org.springframework.cloud.deployer.spi.core.AppDefinition; 32 | import org.springframework.cloud.deployer.spi.core.AppDeploymentRequest; 33 | import org.springframework.cloud.deployer.spi.core.RuntimeEnvironmentInfo; 34 | import org.springframework.cloud.deployer.spi.task.LaunchState; 35 | import org.springframework.cloud.deployer.spi.task.TaskLauncher; 36 | import org.springframework.cloud.deployer.spi.task.TaskStatus; 37 | import org.springframework.cloud.deployer.spi.util.RuntimeVersionUtils; 38 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppInstanceInfo; 39 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType; 40 | import org.springframework.core.io.Resource; 41 | import org.springframework.messaging.Message; 42 | import org.springframework.messaging.support.MessageBuilder; 43 | import org.springframework.statemachine.StateContext; 44 | import org.springframework.statemachine.StateContext.Stage; 45 | import org.springframework.statemachine.StateMachine; 46 | import org.springframework.statemachine.listener.StateMachineListener; 47 | import org.springframework.statemachine.listener.StateMachineListenerAdapter; 48 | import org.springframework.util.Assert; 49 | import org.springframework.util.ObjectUtils; 50 | import org.springframework.util.StringUtils; 51 | import org.springframework.util.concurrent.ListenableFutureCallback; 52 | import org.springframework.util.concurrent.SettableListenableFuture; 53 | 54 | /** 55 | * {@link TaskLauncher} which is launching tasks as Yarn applications. 56 | * 57 | * @author Janne Valkealahti 58 | */ 59 | public class YarnTaskLauncher implements TaskLauncher { 60 | 61 | private static final Logger logger = LoggerFactory.getLogger(YarnTaskLauncher.class); 62 | private final YarnCloudAppService yarnCloudAppService; 63 | private final StateMachine stateMachine; 64 | 65 | @Autowired 66 | private YarnDeployerProperties yarnDeployerProperties; 67 | 68 | /** 69 | * Instantiates a new yarn task module deployer. 70 | * 71 | * @param yarnCloudAppService the yarn cloud app service 72 | * @param stateMachine the state machine 73 | */ 74 | public YarnTaskLauncher(YarnCloudAppService yarnCloudAppService, StateMachine stateMachine) { 75 | this.yarnCloudAppService = yarnCloudAppService; 76 | this.stateMachine = stateMachine; 77 | } 78 | 79 | @Override 80 | public String launch(AppDeploymentRequest request) { 81 | logger.info("Deploy request for {}", request); 82 | logger.info("Deploy request deployment properties {}", request.getDeploymentProperties()); 83 | logger.info("Deploy definition {}", request.getDefinition()); 84 | Resource resource = request.getResource(); 85 | AppDefinition definition = request.getDefinition(); 86 | 87 | String artifact = resource.getFilename(); 88 | 89 | final String name = definition.getName(); 90 | Map definitionParameters = definition.getProperties(); 91 | Map deploymentProperties = request.getDeploymentProperties(); 92 | List commandlineArguments = request.getCommandlineArguments(); 93 | String appName = "scdtask:" + name; 94 | 95 | // contextRunArgs are passed to boot app ran to control yarn apps 96 | // we pass needed args to control module coordinates and params, 97 | // weird format with '--' is just straight pass to container 98 | ArrayList contextRunArgs = new ArrayList(); 99 | 100 | contextRunArgs.add("--spring.yarn.appName=" + appName); 101 | for (Entry entry : definitionParameters.entrySet()) { 102 | if (StringUtils.hasText(entry.getValue())) { 103 | contextRunArgs 104 | .add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.parameters." 105 | + entry.getKey() + ".='" + entry.getValue() + "'"); 106 | } 107 | } 108 | 109 | int index = 0; 110 | for (String commandlineArgument : commandlineArguments) { 111 | contextRunArgs.add("--spring.yarn.client.launchcontext.argumentsList[" + index 112 | + "]='--spring.cloud.deployer.yarn.appmaster.commandlineArguments[" + index + "]=" + commandlineArgument + "'"); 113 | index++; 114 | } 115 | 116 | String baseDir = yarnDeployerProperties.getBaseDir(); 117 | if (!baseDir.endsWith("/")) { 118 | baseDir = baseDir + "/"; 119 | } 120 | String artifactPath = isHdfsResource(resource) ? getHdfsArtifactPath(resource) : baseDir + "/artifacts/cache/"; 121 | 122 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.artifact=" + artifactPath + artifact); 123 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.artifactName=" + artifact); 124 | 125 | // deployment properties override servers.yml which overrides application.yml 126 | for (Entry entry : deploymentProperties.entrySet()) { 127 | if (StringUtils.hasText(entry.getValue())) { 128 | if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.taskcontainer")) { 129 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--" + entry.getKey() + "='" + entry.getValue() + "'"); 130 | } else if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.taskappmaster")) { 131 | contextRunArgs.add("--" + entry.getKey() + "=" + entry.getValue()); 132 | } 133 | } 134 | } 135 | 136 | String appVersion = StringUtils.hasText(yarnDeployerProperties.getAppVersion()) ? yarnDeployerProperties.getAppVersion() : "app"; 137 | if (deploymentProperties.containsKey("spring.cloud.deployer.yarn.app.appVersion")) { 138 | appVersion = deploymentProperties.get("spring.cloud.deployer.yarn.app.appVersion"); 139 | } 140 | 141 | final Message message = MessageBuilder.withPayload(TaskLauncherStateMachine.EVENT_LAUNCH) 142 | .setHeader(TaskLauncherStateMachine.HEADER_APP_VERSION, appVersion) 143 | .setHeader(TaskLauncherStateMachine.HEADER_ARTIFACT, resource) 144 | .setHeader(TaskLauncherStateMachine.HEADER_ARTIFACT_DIR, artifactPath) 145 | .setHeader(TaskLauncherStateMachine.HEADER_DEFINITION_PARAMETERS, definitionParameters) 146 | .setHeader(TaskLauncherStateMachine.HEADER_CONTEXT_RUN_ARGS, contextRunArgs) 147 | .build(); 148 | 149 | // setup future, listen event from machine and finally unregister listener, 150 | // and set future value 151 | final SettableListenableFuture id = new SettableListenableFuture<>(); 152 | final StateMachineListener listener = new StateMachineListenerAdapter() { 153 | 154 | @Override 155 | public void stateContext(StateContext stateContext) { 156 | if (stateContext.getStage() == Stage.STATE_ENTRY && stateContext.getTarget().getId().equals(TaskLauncherStateMachine.STATE_READY)) { 157 | if (ObjectUtils.nullSafeEquals(message.getHeaders().getId().toString(), 158 | stateContext.getExtendedState().get(TaskLauncherStateMachine.VAR_MESSAGE_ID, String.class))) { 159 | Exception exception = stateContext.getExtendedState().get(TaskLauncherStateMachine.VAR_ERROR, Exception.class); 160 | if (exception != null) { 161 | id.setException(exception); 162 | } else { 163 | String applicationId = stateContext.getStateMachine().getExtendedState() 164 | .get(TaskLauncherStateMachine.VAR_APPLICATION_ID, String.class); 165 | DeploymentKey key = new DeploymentKey(name, applicationId); 166 | id.set(key.toString()); 167 | } 168 | } 169 | } 170 | } 171 | }; 172 | 173 | stateMachine.addStateListener(listener); 174 | id.addCallback(new ListenableFutureCallback() { 175 | 176 | @Override 177 | public void onSuccess(String result) { 178 | stateMachine.removeStateListener(listener); 179 | } 180 | 181 | @Override 182 | public void onFailure(Throwable ex) { 183 | stateMachine.removeStateListener(listener); 184 | } 185 | }); 186 | 187 | stateMachine.sendEvent(message); 188 | // we need to block here until SPI supports 189 | // returning id asynchronously 190 | try { 191 | return id.get(2, TimeUnit.MINUTES); 192 | } catch (Exception e) { 193 | throw new RuntimeException(e); 194 | } 195 | } 196 | 197 | private boolean isHdfsResource(Resource resource) { 198 | try { 199 | return resource != null && resource.getURI().getScheme().equals("hdfs"); 200 | } catch (IOException e) { 201 | return false; 202 | } 203 | } 204 | 205 | private String getHdfsArtifactPath(Resource resource) { 206 | String path = null; 207 | try { 208 | path = "/" + FilenameUtils.getPath(resource.getURI().getPath()); 209 | } catch (IOException e) { 210 | } 211 | return path; 212 | } 213 | 214 | @Override 215 | public void cancel(String id) { 216 | logger.info("Undeploy request for task {}", id); 217 | DeploymentKey key = new DeploymentKey(id); 218 | Message message = MessageBuilder.withPayload(TaskLauncherStateMachine.EVENT_CANCEL) 219 | .setHeader(TaskLauncherStateMachine.HEADER_APPLICATION_ID, key.applicationId) 220 | .build(); 221 | stateMachine.sendEvent(message); 222 | } 223 | 224 | @Override 225 | public TaskStatus status(String id) { 226 | logger.info("Status request for module {}", id); 227 | DeploymentKey key = new DeploymentKey(id); 228 | 229 | Collection instances = yarnCloudAppService.getInstances(CloudAppType.TASK); 230 | for (CloudAppInstanceInfo instance : instances) { 231 | if (instance.getApplicationId().equals(key.applicationId)) { 232 | if (instance.getState() == "RUNNING") { 233 | return new TaskStatus(id, LaunchState.running, null); 234 | } 235 | } 236 | } 237 | return new TaskStatus(id, LaunchState.unknown, null); 238 | } 239 | 240 | @Override 241 | public void cleanup(String id) { 242 | } 243 | 244 | @Override 245 | public void destroy(String appName) { 246 | } 247 | 248 | @Override 249 | public RuntimeEnvironmentInfo environmentInfo() { 250 | return new RuntimeEnvironmentInfo.Builder() 251 | .spiClass(TaskLauncher.class) 252 | .implementationName(getClass().getSimpleName()) 253 | .implementationVersion(RuntimeVersionUtils.getVersion(this.getClass())) 254 | .platformType("Yarn") 255 | .platformApiVersion(System.getProperty("os.name") + " " + System.getProperty("os.version")) 256 | .platformClientVersion(System.getProperty("os.version")) 257 | .platformHostVersion(System.getProperty("os.version")) 258 | .build(); 259 | } 260 | 261 | private static class DeploymentKey { 262 | final static String SEPARATOR = ":"; 263 | final String name; 264 | final String applicationId; 265 | 266 | public DeploymentKey(String id) { 267 | String[] split = id.split(SEPARATOR); 268 | Assert.isTrue(split.length == 2, "Unable to parse deployment key " + id); 269 | name = split[0]; 270 | applicationId = split[1]; 271 | } 272 | 273 | public DeploymentKey(String name, String applicationId) { 274 | Assert.notNull(name, "Name must be set"); 275 | Assert.notNull(applicationId, "Application id must be set"); 276 | this.name = name; 277 | this.applicationId = applicationId; 278 | } 279 | 280 | @Override 281 | public String toString() { 282 | return name + SEPARATOR + applicationId; 283 | } 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/resources/appdeployer-model.di: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/resources/appdeployer-model.uml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | bean 18 | errorGuard 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | bean 28 | errorGuard 29 | 30 | 31 | 32 | 33 | 34 | 35 | bean 36 | errorGuard 37 | 38 | 39 | 40 | 41 | 42 | bean 43 | resetVariablesAction 44 | 45 | 46 | 47 | 48 | bean 49 | errorHandlingAction 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | bean 58 | deployAction 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | bean 68 | pushAppGuard 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | bean 83 | instanceGuard 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | bean 93 | startInstanceGuard 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | bean 103 | checkAppAction 104 | 105 | 106 | 107 | 108 | bean 109 | pushAppAction 110 | 111 | 112 | 113 | 114 | bean 115 | checkInstanceAction 116 | 117 | 118 | 119 | 120 | bean 121 | startInstanceAction 122 | 123 | 124 | 125 | 126 | bean 127 | pushArtifactAction 128 | 129 | 130 | 131 | 132 | bean 133 | createClusterAction 134 | 135 | 136 | 137 | 138 | bean 139 | startClusterAction 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | bean 148 | waitInstanceAction 149 | 150 | 151 | 152 | 153 | 154 | bean 155 | resolveInstanceAction 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | bean 171 | stopClusterAction 172 | 173 | 174 | 175 | 176 | bean 177 | destroyClusterAction 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/resources/stream.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | main: 3 | show_banner: false 4 | cloud: 5 | deployer: 6 | yarn: 7 | version: "@project.version@" 8 | yarn: 9 | appType: DATAFLOW 10 | appName: scdstream:app 11 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/stream/ 12 | client: 13 | clientClass: org.springframework.cloud.deployer.spi.yarn.DeployerApplicationYarnClient 14 | files: 15 | - "file:${deployer.yarn.app.appmaster.path:lib}/spring-cloud-deployer-yarn-appdeployerappmaster-*.jar" 16 | - "file:${deployer.yarn.app.config.path:config}/servers.yml*" 17 | queue: ${spring.cloud.deployer.yarn.app.streamappmaster.queue:default} 18 | labelExpression: ${spring.cloud.deployer.yarn.app.streamappmaster.labelExpression:} 19 | launchcontext: 20 | options: 21 | - ${spring.cloud.deployer.yarn.app.streamappmaster.javaOpts:} 22 | arguments: 23 | -Dspring.config.location: servers.yml 24 | archiveFile: spring-cloud-deployer-yarn-appdeployerappmaster-${spring.cloud.deployer.yarn.version}.jar 25 | localizer: 26 | patterns: 27 | - "spring-cloud-deployer-yarn-appdeployerappmaster-*.jar" 28 | - "servers.yml" 29 | resource: 30 | memory: ${spring.cloud.deployer.yarn.app.streamappmaster.memory:512m} 31 | virtualCores: ${spring.cloud.deployer.yarn.app.streamappmaster.virtualCores:1} 32 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/resources/task.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | main: 3 | show_banner: false 4 | cloud: 5 | deployer: 6 | yarn: 7 | version: "@project.version@" 8 | yarn: 9 | appType: DATAFLOW 10 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/task/ 11 | client: 12 | enabled: true 13 | clientClass: org.springframework.cloud.deployer.spi.yarn.DeployerApplicationYarnClient 14 | files: 15 | - "file:${deployer.yarn.app.appmaster.path:lib}/spring-cloud-deployer-yarn-tasklauncherappmaster-*.jar" 16 | - "file:${deployer.yarn.app.config.path:config}/servers.yml*" 17 | queue: ${spring.cloud.deployer.yarn.app.taskappmaster.queue:default} 18 | labelExpression: ${spring.cloud.deployer.yarn.app.taskappmaster.labelExpression:} 19 | launchcontext: 20 | options: 21 | - ${spring.cloud.deployer.yarn.app.taskappmaster.javaOpts:} 22 | arguments: 23 | -Dspring.config.location: servers.yml 24 | archiveFile: spring-cloud-deployer-yarn-tasklauncherappmaster-${spring.cloud.deployer.yarn.version}.jar 25 | localizer: 26 | patterns: 27 | - "spring-cloud-deployer-yarn-tasklauncherappmaster-*.jar" 28 | - "servers.yml" 29 | resource: 30 | memory: ${spring.cloud.deployer.yarn.app.taskappmaster.memory:512m} 31 | virtualCores: ${spring.cloud.deployer.yarn.app.taskappmaster.virtualCores:1} 32 | container: 33 | enabled: false 34 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/resources/tasklauncher-model.di: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/main/resources/tasklauncher-model.uml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | bean 11 | errorGuard 12 | 13 | 14 | 15 | 16 | 17 | 18 | bean 19 | errorGuard 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | bean 29 | errorGuard 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | bean 43 | resetVariablesAction 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | bean 52 | launchAction 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | bean 61 | pushAppGuard 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | bean 72 | checkAppAction 73 | 74 | 75 | 76 | 77 | bean 78 | pushAppAction 79 | 80 | 81 | 82 | 83 | bean 84 | pushArtifactAction 85 | 86 | 87 | 88 | 89 | bean 90 | startAppAction 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | bean 103 | cancelAction 104 | 105 | 106 | 107 | 108 | 109 | 110 | bean 111 | stopAppAction 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | bean 120 | errorHandlingAction 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/test/java/org/springframework/cloud/deployer/spi/yarn/AbstractStateMachineTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Collection; 21 | import java.util.Collections; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.concurrent.CountDownLatch; 25 | 26 | import org.junit.After; 27 | import org.junit.Before; 28 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 29 | import org.springframework.core.io.Resource; 30 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData; 31 | 32 | public class AbstractStateMachineTests { 33 | 34 | protected AnnotationConfigApplicationContext context; 35 | 36 | @Before 37 | public void setup() { 38 | context = buildContext(); 39 | } 40 | 41 | @After 42 | public void clean() { 43 | if (context != null) { 44 | context.close(); 45 | } 46 | context = null; 47 | } 48 | 49 | protected AnnotationConfigApplicationContext buildContext() { 50 | return null; 51 | } 52 | 53 | protected static class TestYarnCloudAppService implements YarnCloudAppService { 54 | 55 | volatile String app = null; 56 | volatile String instance = null; 57 | 58 | final CountDownLatch getApplicationsLatch = new CountDownLatch(1); 59 | final CountDownLatch getInstancesLatch = new CountDownLatch(2); 60 | final CountDownLatch pushApplicationLatch = new CountDownLatch(1); 61 | final CountDownLatch submitApplicationLatch = new CountDownLatch(1); 62 | final CountDownLatch createClusterLatch = new CountDownLatch(1); 63 | final CountDownLatch startClusterLatch = new CountDownLatch(1); 64 | final CountDownLatch stopClusterLatch = new CountDownLatch(1); 65 | 66 | volatile int getApplicationsCount = 0; 67 | volatile int getInstancesCount = 0; 68 | volatile int getInstancesCountBeforeReturn = 0; 69 | 70 | final List pushApplicationCount = Collections.synchronizedList(new ArrayList()); 71 | final List submitApplicationCount = Collections.synchronizedList(new ArrayList()); 72 | final List createClusterCount = Collections.synchronizedList(new ArrayList()); 73 | final List startClusterCount = Collections.synchronizedList(new ArrayList()); 74 | final List stopClusterCount = Collections.synchronizedList(new ArrayList()); 75 | 76 | @Override 77 | public Collection getApplications(CloudAppType cloudAppType) { 78 | ArrayList infos = new ArrayList(); 79 | if (app != null) { 80 | infos.add(new CloudAppInfo(app)); 81 | } 82 | getApplicationsCount++; 83 | getApplicationsLatch.countDown(); 84 | return infos; 85 | } 86 | 87 | @Override 88 | public Collection getInstances(CloudAppType cloudAppType) { 89 | ArrayList infos = new ArrayList(); 90 | if (instance != null) { 91 | if (getInstancesCount >= getInstancesCountBeforeReturn) { 92 | infos.add(new CloudAppInstanceInfo("fakeApplicationId", instance, "RUNNING", "http://fakeAddress")); 93 | } 94 | } 95 | getInstancesCount++; 96 | getInstancesLatch.countDown(); 97 | return infos; 98 | } 99 | 100 | @Override 101 | public void pushApplication(String appVersion, CloudAppType cloudAppType) { 102 | app = appVersion; 103 | pushApplicationCount.add(new Wrapper(appVersion)); 104 | pushApplicationLatch.countDown(); 105 | } 106 | 107 | @Override 108 | public String submitApplication(String appVersion, CloudAppType cloudAppType) { 109 | instance = "scdstream:" + appVersion; 110 | submitApplicationCount.add(new Wrapper(appVersion)); 111 | submitApplicationLatch.countDown(); 112 | return "fakeApplicationId"; 113 | } 114 | 115 | @Override 116 | public String submitApplication(String appVersion, CloudAppType cloudAppType, List contextRunArgs) { 117 | instance = "scdstream:" + appVersion + ":fakeGroup"; 118 | submitApplicationCount.add(new Wrapper(appVersion)); 119 | submitApplicationLatch.countDown(); 120 | return "fakeApplicationId"; 121 | } 122 | 123 | @Override 124 | public void killApplications(String appName, CloudAppType cloudAppType) { 125 | } 126 | 127 | @Override 128 | public void createCluster(String yarnApplicationId, String clusterId, int count, String artifact, 129 | Map definitionParameters) { 130 | createClusterCount.add(new Wrapper(yarnApplicationId, clusterId, count, definitionParameters)); 131 | createClusterLatch.countDown(); 132 | } 133 | 134 | @Override 135 | public void startCluster(String yarnApplicationId, String clusterId) { 136 | startClusterCount.add(new Wrapper(yarnApplicationId, clusterId)); 137 | startClusterLatch.countDown(); 138 | } 139 | 140 | @Override 141 | public void stopCluster(String yarnApplicationId, String clusterId) { 142 | stopClusterCount.add(new Wrapper(yarnApplicationId, clusterId)); 143 | stopClusterLatch.countDown(); 144 | } 145 | 146 | @Override 147 | public Collection getClusters(String yarnApplicationId) { 148 | return null; 149 | } 150 | 151 | @Override 152 | public void destroyCluster(String yarnApplicationId, String clusterId) { 153 | } 154 | 155 | @Override 156 | public void pushArtifact(Resource artifact, String dir) { 157 | } 158 | 159 | static class Wrapper { 160 | String appVersion; 161 | String yarnApplicationId; 162 | String clusterId; 163 | int count; 164 | Map definitionParameters; 165 | 166 | public Wrapper(String appVersion) { 167 | this.appVersion = appVersion; 168 | } 169 | 170 | public Wrapper(String yarnApplicationId, String clusterId) { 171 | this.yarnApplicationId = yarnApplicationId; 172 | this.clusterId = clusterId; 173 | } 174 | 175 | public Wrapper(String yarnApplicationId, String clusterId, int count, 176 | Map definitionParameters) { 177 | this.yarnApplicationId = yarnApplicationId; 178 | this.clusterId = clusterId; 179 | this.count = count; 180 | this.definitionParameters = definitionParameters; 181 | } 182 | 183 | } 184 | 185 | @Override 186 | public Map getClustersStates(String yarnApplicationId) { 187 | return null; 188 | } 189 | 190 | @Override 191 | public void killApplication(String yarnApplicationId, CloudAppType cloudAppType) { 192 | } 193 | 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/test/java/org/springframework/cloud/deployer/spi/yarn/AppDeployerStateMachineTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.deployer.spi.yarn; 18 | 19 | import java.util.HashMap; 20 | 21 | import org.junit.Test; 22 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 23 | import org.springframework.context.annotation.Bean; 24 | import org.springframework.context.annotation.Configuration; 25 | import org.springframework.core.task.TaskExecutor; 26 | import org.springframework.messaging.Message; 27 | import org.springframework.messaging.support.MessageBuilder; 28 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 29 | import org.springframework.statemachine.StateMachine; 30 | import org.springframework.statemachine.test.StateMachineTestPlan; 31 | import org.springframework.statemachine.test.StateMachineTestPlanBuilder; 32 | 33 | public class AppDeployerStateMachineTests extends AbstractStateMachineTests { 34 | 35 | @Test 36 | public void testInitial() throws Exception { 37 | context.register(Config.class); 38 | context.refresh(); 39 | 40 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService(); 41 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class); 42 | AppDeployerStateMachine ycasm = new AppDeployerStateMachine(yarnCloudAppService, taskExecutor, context, context); 43 | ycasm.setAutoStart(false); 44 | StateMachine stateMachine = ycasm.buildStateMachine(); 45 | 46 | StateMachineTestPlan plan = 47 | StateMachineTestPlanBuilder.builder() 48 | .defaultAwaitTime(10) 49 | .stateMachine(stateMachine) 50 | .step() 51 | .expectStateMachineStarted(1) 52 | .expectStates("READY") 53 | .and() 54 | .build(); 55 | plan.test(); 56 | } 57 | 58 | @Test 59 | public void testDeployShouldPushAndStart() throws Exception { 60 | context.register(Config.class); 61 | context.refresh(); 62 | 63 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService(); 64 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class); 65 | AppDeployerStateMachine ycasm = new AppDeployerStateMachine(yarnCloudAppService, taskExecutor, context, context); 66 | ycasm.setAutoStart(false); 67 | StateMachine stateMachine = ycasm.buildStateMachine(); 68 | 69 | Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_DEPLOY) 70 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, "app") 71 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, "fakeClusterId") 72 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, "fakeGroup") 73 | .setHeader(AppDeployerStateMachine.HEADER_COUNT, 1) 74 | .setHeader(AppDeployerStateMachine.HEADER_DEFINITION_PARAMETERS, new HashMap()) 75 | .build(); 76 | 77 | StateMachineTestPlan plan = 78 | StateMachineTestPlanBuilder.builder() 79 | .defaultAwaitTime(10) 80 | .stateMachine(stateMachine) 81 | .step() 82 | .expectStateMachineStarted(1) 83 | .expectStates(AppDeployerStateMachine.STATE_READY) 84 | .and() 85 | .step() 86 | .sendEvent(message) 87 | .expectStateChanged(12) 88 | .expectStates(AppDeployerStateMachine.STATE_READY) 89 | .and() 90 | .build(); 91 | plan.test(); 92 | } 93 | 94 | @Test 95 | public void testDeployShouldPushAndStartLoopWaitInstance() throws Exception { 96 | context.register(Config.class); 97 | context.refresh(); 98 | 99 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService(); 100 | yarnCloudAppService.getInstancesCountBeforeReturn = 2; 101 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class); 102 | AppDeployerStateMachine ycasm = new AppDeployerStateMachine(yarnCloudAppService, taskExecutor, context, context); 103 | ycasm.setAutoStart(false); 104 | StateMachine stateMachine = ycasm.buildStateMachine(); 105 | 106 | Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_DEPLOY) 107 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, "app") 108 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, "fakeClusterId") 109 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, "fakeGroup") 110 | .setHeader(AppDeployerStateMachine.HEADER_COUNT, 1) 111 | .setHeader(AppDeployerStateMachine.HEADER_DEFINITION_PARAMETERS, new HashMap()) 112 | .build(); 113 | 114 | StateMachineTestPlan plan = 115 | StateMachineTestPlanBuilder.builder() 116 | .defaultAwaitTime(10) 117 | .stateMachine(stateMachine) 118 | .step() 119 | .expectStateMachineStarted(1) 120 | .expectStates(AppDeployerStateMachine.STATE_READY) 121 | .and() 122 | .step() 123 | .sendEvent(message) 124 | .expectStateChanged(14) 125 | .expectStates(AppDeployerStateMachine.STATE_READY) 126 | .and() 127 | .build(); 128 | plan.test(); 129 | } 130 | 131 | @Configuration 132 | static class Config { 133 | 134 | @Bean 135 | TaskExecutor taskExecutor() { 136 | ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); 137 | taskExecutor.setCorePoolSize(1); 138 | return taskExecutor; 139 | } 140 | 141 | } 142 | 143 | @Override 144 | protected AnnotationConfigApplicationContext buildContext() { 145 | return new AnnotationConfigApplicationContext(); 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /spring-cloud-deployer-yarn/src/test/java/org/springframework/cloud/deployer/spi/yarn/TaskLauncherStateMachineTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.springframework.cloud.deployer.spi.yarn; 17 | 18 | import java.util.ArrayList; 19 | import java.util.HashMap; 20 | 21 | import org.junit.Test; 22 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 23 | import org.springframework.context.annotation.Bean; 24 | import org.springframework.context.annotation.Configuration; 25 | import org.springframework.core.task.TaskExecutor; 26 | import org.springframework.messaging.Message; 27 | import org.springframework.messaging.support.MessageBuilder; 28 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 29 | import org.springframework.statemachine.StateMachine; 30 | import org.springframework.statemachine.test.StateMachineTestPlan; 31 | import org.springframework.statemachine.test.StateMachineTestPlanBuilder; 32 | 33 | public class TaskLauncherStateMachineTests extends AbstractStateMachineTests { 34 | 35 | @Test 36 | public void testInitial() throws Exception { 37 | context.register(Config.class); 38 | context.refresh(); 39 | 40 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService(); 41 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class); 42 | TaskLauncherStateMachine ycasm = new TaskLauncherStateMachine(yarnCloudAppService, taskExecutor, context, context); 43 | ycasm.setAutoStart(false); 44 | StateMachine stateMachine = ycasm.buildStateMachine(); 45 | 46 | StateMachineTestPlan plan = 47 | StateMachineTestPlanBuilder.builder() 48 | .defaultAwaitTime(10) 49 | .stateMachine(stateMachine) 50 | .step() 51 | .expectStateMachineStarted(1) 52 | .expectStates(TaskLauncherStateMachine.STATE_READY) 53 | .and() 54 | .build(); 55 | plan.test(); 56 | } 57 | 58 | @Test 59 | public void testLaunchShouldPushAndStart() throws Exception { 60 | context.register(Config.class); 61 | context.refresh(); 62 | 63 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService(); 64 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class); 65 | TaskLauncherStateMachine ycasm = new TaskLauncherStateMachine(yarnCloudAppService, taskExecutor, context, context); 66 | ycasm.setAutoStart(false); 67 | StateMachine stateMachine = ycasm.buildStateMachine(); 68 | 69 | ArrayList contextRunArgs = new ArrayList(); 70 | Message launchMessage = MessageBuilder.withPayload(TaskLauncherStateMachine.EVENT_LAUNCH) 71 | .setHeader(TaskLauncherStateMachine.HEADER_APP_VERSION, "fakeApp") 72 | .setHeader(TaskLauncherStateMachine.HEADER_DEFINITION_PARAMETERS, new HashMap()) 73 | .setHeader(TaskLauncherStateMachine.HEADER_CONTEXT_RUN_ARGS, contextRunArgs) 74 | .build(); 75 | 76 | StateMachineTestPlan plan = 77 | StateMachineTestPlanBuilder.builder() 78 | .defaultAwaitTime(10) 79 | .stateMachine(stateMachine) 80 | .step() 81 | .expectStateMachineStarted(1) 82 | .expectStates(TaskLauncherStateMachine.STATE_READY) 83 | .and() 84 | .step() 85 | .sendEvent(launchMessage) 86 | .expectStateChanged(6) 87 | .expectStates(TaskLauncherStateMachine.STATE_READY) 88 | .and() 89 | .build(); 90 | plan.test(); 91 | } 92 | 93 | @Configuration 94 | static class Config { 95 | 96 | @Bean 97 | TaskExecutor taskExecutor() { 98 | ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); 99 | taskExecutor.setCorePoolSize(1); 100 | return taskExecutor; 101 | } 102 | 103 | } 104 | 105 | @Override 106 | protected AnnotationConfigApplicationContext buildContext() { 107 | return new AnnotationConfigApplicationContext(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /spring-cloud-starter-deployer-yarn/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | spring-cloud-starter-deployer-yarn 6 | 1.2.3.BUILD-SNAPSHOT 7 | spring-cloud-starter-deployer-yarn 8 | Data Flow Deployer Starter for Apache Yarn 9 | http://projects.spring.io/spring-cloud/ 10 | 11 | Pivotal Software, Inc. 12 | http://www.spring.io 13 | 14 | 15 | 16 | org.springframework.cloud 17 | spring-cloud-deployer-yarn-parent 18 | 1.2.3.BUILD-SNAPSHOT 19 | 20 | 21 | 22 | 23 | org.springframework.cloud 24 | spring-cloud-deployer-yarn-autoconfig 25 | 1.2.3.BUILD-SNAPSHOT 26 | 27 | 28 | 29 | --------------------------------------------------------------------------------