├── .gitignore
├── .mvn
├── jvm.config
├── maven.config
└── wrapper
│ ├── maven-wrapper.jar
│ └── maven-wrapper.properties
├── .settings.xml
├── .travis.yml
├── LICENSE
├── NOTICE
├── README.adoc
├── mvnw
├── mvnw.cmd
├── pom.xml
├── spring-cloud-deployer-yarn-appdeployerappmaster
├── pom.xml
└── src
│ └── main
│ ├── java
│ └── org
│ │ └── springframework
│ │ └── cloud
│ │ └── deployer
│ │ └── spi
│ │ └── yarn
│ │ └── appdeployer
│ │ ├── StreamAppmaster.java
│ │ ├── StreamAppmasterApplication.java
│ │ └── StreamAppmasterProperties.java
│ └── resources
│ └── application.yml
├── spring-cloud-deployer-yarn-autoconfig
├── pom.xml
└── src
│ └── main
│ ├── java
│ └── org
│ │ └── springframework
│ │ └── cloud
│ │ └── deployer
│ │ └── spi
│ │ └── yarn
│ │ └── autoconfigure
│ │ └── YarnDeployerAutoConfiguration.java
│ └── resources
│ └── META-INF
│ └── spring.factories
├── spring-cloud-deployer-yarn-build-tests
├── pom.xml
└── src
│ ├── main
│ └── assembly
│ │ └── assembly.xml
│ └── test
│ ├── java
│ └── org
│ │ └── springframework
│ │ └── cloud
│ │ └── dataflow
│ │ └── yarn
│ │ └── buildtests
│ │ ├── AbstractCliBootYarnClusterTests.java
│ │ ├── AppDeployerIT.java
│ │ └── TaskLauncherIT.java
│ └── resources
│ ├── log4j.properties
│ └── yarn-site.xml
├── spring-cloud-deployer-yarn-tasklauncherappmaster
├── pom.xml
└── src
│ └── main
│ ├── java
│ └── org
│ │ └── springframework
│ │ └── cloud
│ │ └── deployer
│ │ └── spi
│ │ └── yarn
│ │ └── tasklauncher
│ │ ├── TaskAppmaster.java
│ │ ├── TaskAppmasterApplication.java
│ │ └── TaskAppmasterProperties.java
│ └── resources
│ └── application.yml
├── spring-cloud-deployer-yarn
├── pom.xml
└── src
│ ├── main
│ ├── java
│ │ └── org
│ │ │ └── springframework
│ │ │ └── cloud
│ │ │ └── deployer
│ │ │ └── spi
│ │ │ └── yarn
│ │ │ ├── AbstractDeployerStateMachine.java
│ │ │ ├── AppDeployerStateMachine.java
│ │ │ ├── DefaultYarnCloudAppService.java
│ │ │ ├── DeployerApplicationYarnClient.java
│ │ │ ├── StreamClustersInfoReportData.java
│ │ │ ├── TaskLauncherStateMachine.java
│ │ │ ├── YarnAppDeployer.java
│ │ │ ├── YarnCloudAppService.java
│ │ │ ├── YarnCloudAppServiceApplication.java
│ │ │ ├── YarnDeployerProperties.java
│ │ │ └── YarnTaskLauncher.java
│ └── resources
│ │ ├── appdeployer-model.di
│ │ ├── appdeployer-model.notation
│ │ ├── appdeployer-model.uml
│ │ ├── stream.yml
│ │ ├── task.yml
│ │ ├── tasklauncher-model.di
│ │ ├── tasklauncher-model.notation
│ │ └── tasklauncher-model.uml
│ └── test
│ └── java
│ └── org
│ └── springframework
│ └── cloud
│ └── deployer
│ └── spi
│ └── yarn
│ ├── AbstractStateMachineTests.java
│ ├── AppDeployerStateMachineTests.java
│ └── TaskLauncherStateMachineTests.java
└── spring-cloud-starter-deployer-yarn
└── pom.xml
/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | .#*
3 | *#
4 | *.sw*
5 | _site/
6 | .factorypath
7 | .gradletasknamecache
8 | .DS_Store
9 | /application.yml
10 | /application.properties
11 | asciidoctor.css
12 | atlassian-ide-plugin.xml
13 | build/
14 | dump.rdb
15 | out
16 | spring-shell.log
17 | target/
18 | test-output
19 | spring-*/bin
20 |
21 | # Eclipse artifacts, including WTP generated manifests
22 | .classpath
23 | .project
24 | .settings/
25 | .springBeans
26 | spring-*/src/main/java/META-INF/MANIFEST.MF
27 |
28 | # IDEA artifacts and output dirs
29 | *.iml
30 | *.ipr
31 | *.iws
32 | .idea/*
33 | /.idea/
34 |
--------------------------------------------------------------------------------
/.mvn/jvm.config:
--------------------------------------------------------------------------------
1 | -Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom
--------------------------------------------------------------------------------
/.mvn/maven.config:
--------------------------------------------------------------------------------
1 | -DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -s .settings.xml -P spring
2 |
--------------------------------------------------------------------------------
/.mvn/wrapper/maven-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/spring-attic/spring-cloud-deployer-yarn/175449d482bd9a45b79669cdb0b80dca569f8c8c/.mvn/wrapper/maven-wrapper.jar
--------------------------------------------------------------------------------
/.mvn/wrapper/maven-wrapper.properties:
--------------------------------------------------------------------------------
1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.3/apache-maven-3.3.3-bin.zip
--------------------------------------------------------------------------------
/.settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | repo.spring.io
6 | ${env.CI_DEPLOY_USERNAME}
7 | ${env.CI_DEPLOY_PASSWORD}
8 |
9 |
10 |
11 |
12 | spring
13 | true
14 |
15 |
16 | spring-snapshots
17 | Spring Snapshots
18 | http://repo.spring.io/libs-snapshot-local
19 |
20 | true
21 |
22 |
23 |
24 | spring-milestones
25 | Spring Milestones
26 | http://repo.spring.io/libs-milestone-local
27 |
28 | false
29 |
30 |
31 |
32 | spring-releases
33 | Spring Releases
34 | http://repo.spring.io/release
35 |
36 | false
37 |
38 |
39 |
40 |
41 |
42 | spring-snapshots
43 | Spring Snapshots
44 | http://repo.spring.io/libs-snapshot-local
45 |
46 | true
47 |
48 |
49 |
50 | spring-milestones
51 | Spring Milestones
52 | http://repo.spring.io/libs-milestone-local
53 |
54 | false
55 |
56 |
57 |
58 |
59 |
60 |
61 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | sudo: false
2 | cache:
3 | directories:
4 | - $HOME/.m2
5 | language: java
6 | jdk:
7 | - oraclejdk8
8 | services:
9 | - redis-server
10 | install: true
11 | script:
12 | - '[ "${TRAVIS_PULL_REQUEST}" != "false" ] || ./mvnw package -DskipITs -Pfull -U -Dmaven.test.redirectTestOutputToFile=false'
13 | - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] || ./mvnw package -DskipTests -U -Dmaven.test.redirectTestOutputToFile=false'
14 |
15 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
203 |
--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
1 | spring-cloud-deployer-yarn
2 |
3 | Copyright (c) 2016-Present Pivotal Software, Inc. All Rights Reserved.
4 |
5 | Licensed under the Apache License, Version 2.0 (the "License");
6 | you may not use this file except in compliance with the License.
7 | You may obtain a copy of the License at
8 |
9 | http://www.apache.org/licenses/LICENSE-2.0
10 |
11 | Unless required by applicable law or agreed to in writing, software
12 | distributed under the License is distributed on an "AS IS" BASIS,
13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | See the License for the specific language governing permissions and
15 | limitations under the License.
--------------------------------------------------------------------------------
/README.adoc:
--------------------------------------------------------------------------------
1 | # spring-cloud-deployer-yarn is no longer actively maintained by VMware, Inc.
2 |
3 | == Spring Cloud Deployer for Apache Yarn
4 |
5 |
6 | The Spring Cloud Data Flow team have been happy stewards of this project, but we are focusing our efforts towards Local, Kubernetes and Cloud Foundry implementations going forward, so the Spring Cloud Data Flow team will no longer maintain it.
7 |
8 | However, we are looking to donate the project to the community. If you are interested in forking the project to continue to develop and maintaining it, we would be happy to cross-link and promote your forked project. Please ping us at https://gitter.im/spring-cloud/spring-cloud-dataflow[Gitter] to discuss it.
9 |
--------------------------------------------------------------------------------
/mvnw:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # ----------------------------------------------------------------------------
3 | # Licensed to the Apache Software Foundation (ASF) under one
4 | # or more contributor license agreements. See the NOTICE file
5 | # distributed with this work for additional information
6 | # regarding copyright ownership. The ASF licenses this file
7 | # to you under the Apache License, Version 2.0 (the
8 | # "License"); you may not use this file except in compliance
9 | # with the License. You may obtain a copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 | # ----------------------------------------------------------------------------
20 |
21 | # ----------------------------------------------------------------------------
22 | # Maven2 Start Up Batch script
23 | #
24 | # Required ENV vars:
25 | # ------------------
26 | # JAVA_HOME - location of a JDK home dir
27 | #
28 | # Optional ENV vars
29 | # -----------------
30 | # M2_HOME - location of maven2's installed home dir
31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven
32 | # e.g. to debug Maven itself, use
33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files
35 | # ----------------------------------------------------------------------------
36 |
37 | if [ -z "$MAVEN_SKIP_RC" ] ; then
38 |
39 | if [ -f /etc/mavenrc ] ; then
40 | . /etc/mavenrc
41 | fi
42 |
43 | if [ -f "$HOME/.mavenrc" ] ; then
44 | . "$HOME/.mavenrc"
45 | fi
46 |
47 | fi
48 |
49 | # OS specific support. $var _must_ be set to either true or false.
50 | cygwin=false;
51 | darwin=false;
52 | mingw=false
53 | case "`uname`" in
54 | CYGWIN*) cygwin=true ;;
55 | MINGW*) mingw=true;;
56 | Darwin*) darwin=true
57 | #
58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look
59 | # for the new JDKs provided by Oracle.
60 | #
61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then
62 | #
63 | # Apple JDKs
64 | #
65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home
66 | fi
67 |
68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then
69 | #
70 | # Apple JDKs
71 | #
72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home
73 | fi
74 |
75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then
76 | #
77 | # Oracle JDKs
78 | #
79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home
80 | fi
81 |
82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then
83 | #
84 | # Apple JDKs
85 | #
86 | export JAVA_HOME=`/usr/libexec/java_home`
87 | fi
88 | ;;
89 | esac
90 |
91 | if [ -z "$JAVA_HOME" ] ; then
92 | if [ -r /etc/gentoo-release ] ; then
93 | JAVA_HOME=`java-config --jre-home`
94 | fi
95 | fi
96 |
97 | if [ -z "$M2_HOME" ] ; then
98 | ## resolve links - $0 may be a link to maven's home
99 | PRG="$0"
100 |
101 | # need this for relative symlinks
102 | while [ -h "$PRG" ] ; do
103 | ls=`ls -ld "$PRG"`
104 | link=`expr "$ls" : '.*-> \(.*\)$'`
105 | if expr "$link" : '/.*' > /dev/null; then
106 | PRG="$link"
107 | else
108 | PRG="`dirname "$PRG"`/$link"
109 | fi
110 | done
111 |
112 | saveddir=`pwd`
113 |
114 | M2_HOME=`dirname "$PRG"`/..
115 |
116 | # make it fully qualified
117 | M2_HOME=`cd "$M2_HOME" && pwd`
118 |
119 | cd "$saveddir"
120 | # echo Using m2 at $M2_HOME
121 | fi
122 |
123 | # For Cygwin, ensure paths are in UNIX format before anything is touched
124 | if $cygwin ; then
125 | [ -n "$M2_HOME" ] &&
126 | M2_HOME=`cygpath --unix "$M2_HOME"`
127 | [ -n "$JAVA_HOME" ] &&
128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
129 | [ -n "$CLASSPATH" ] &&
130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
131 | fi
132 |
133 | # For Migwn, ensure paths are in UNIX format before anything is touched
134 | if $mingw ; then
135 | [ -n "$M2_HOME" ] &&
136 | M2_HOME="`(cd "$M2_HOME"; pwd)`"
137 | [ -n "$JAVA_HOME" ] &&
138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
139 | # TODO classpath?
140 | fi
141 |
142 | if [ -z "$JAVA_HOME" ]; then
143 | javaExecutable="`which javac`"
144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
145 | # readlink(1) is not available as standard on Solaris 10.
146 | readLink=`which readlink`
147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
148 | if $darwin ; then
149 | javaHome="`dirname \"$javaExecutable\"`"
150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
151 | else
152 | javaExecutable="`readlink -f \"$javaExecutable\"`"
153 | fi
154 | javaHome="`dirname \"$javaExecutable\"`"
155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'`
156 | JAVA_HOME="$javaHome"
157 | export JAVA_HOME
158 | fi
159 | fi
160 | fi
161 |
162 | if [ -z "$JAVACMD" ] ; then
163 | if [ -n "$JAVA_HOME" ] ; then
164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
165 | # IBM's JDK on AIX uses strange locations for the executables
166 | JAVACMD="$JAVA_HOME/jre/sh/java"
167 | else
168 | JAVACMD="$JAVA_HOME/bin/java"
169 | fi
170 | else
171 | JAVACMD="`which java`"
172 | fi
173 | fi
174 |
175 | if [ ! -x "$JAVACMD" ] ; then
176 | echo "Error: JAVA_HOME is not defined correctly." >&2
177 | echo " We cannot execute $JAVACMD" >&2
178 | exit 1
179 | fi
180 |
181 | if [ -z "$JAVA_HOME" ] ; then
182 | echo "Warning: JAVA_HOME environment variable is not set."
183 | fi
184 |
185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
186 |
187 | # For Cygwin, switch paths to Windows format before running java
188 | if $cygwin; then
189 | [ -n "$M2_HOME" ] &&
190 | M2_HOME=`cygpath --path --windows "$M2_HOME"`
191 | [ -n "$JAVA_HOME" ] &&
192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
193 | [ -n "$CLASSPATH" ] &&
194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
195 | fi
196 |
197 | # traverses directory structure from process work directory to filesystem root
198 | # first directory with .mvn subdirectory is considered project base directory
199 | find_maven_basedir() {
200 | local basedir=$(pwd)
201 | local wdir=$(pwd)
202 | while [ "$wdir" != '/' ] ; do
203 | if [ -d "$wdir"/.mvn ] ; then
204 | basedir=$wdir
205 | break
206 | fi
207 | wdir=$(cd "$wdir/.."; pwd)
208 | done
209 | echo "${basedir}"
210 | }
211 |
212 | # concatenates all lines of a file
213 | concat_lines() {
214 | if [ -f "$1" ]; then
215 | echo "$(tr -s '\n' ' ' < "$1")"
216 | fi
217 | }
218 |
219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)}
220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
221 |
222 | # Provide a "standardized" way to retrieve the CLI args that will
223 | # work with both Windows and non-Windows executions.
224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
225 | export MAVEN_CMD_LINE_ARGS
226 |
227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
228 |
229 | exec "$JAVACMD" \
230 | $MAVEN_OPTS \
231 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
233 | ${WRAPPER_LAUNCHER} "$@"
234 |
235 |
--------------------------------------------------------------------------------
/mvnw.cmd:
--------------------------------------------------------------------------------
1 | @REM ----------------------------------------------------------------------------
2 | @REM Licensed to the Apache Software Foundation (ASF) under one
3 | @REM or more contributor license agreements. See the NOTICE file
4 | @REM distributed with this work for additional information
5 | @REM regarding copyright ownership. The ASF licenses this file
6 | @REM to you under the Apache License, Version 2.0 (the
7 | @REM "License"); you may not use this file except in compliance
8 | @REM with the License. You may obtain a copy of the License at
9 | @REM
10 | @REM http://www.apache.org/licenses/LICENSE-2.0
11 | @REM
12 | @REM Unless required by applicable law or agreed to in writing,
13 | @REM software distributed under the License is distributed on an
14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | @REM KIND, either express or implied. See the License for the
16 | @REM specific language governing permissions and limitations
17 | @REM under the License.
18 | @REM ----------------------------------------------------------------------------
19 |
20 | @REM ----------------------------------------------------------------------------
21 | @REM Maven2 Start Up Batch script
22 | @REM
23 | @REM Required ENV vars:
24 | @REM JAVA_HOME - location of a JDK home dir
25 | @REM
26 | @REM Optional ENV vars
27 | @REM M2_HOME - location of maven2's installed home dir
28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
31 | @REM e.g. to debug Maven itself, use
32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
34 | @REM ----------------------------------------------------------------------------
35 |
36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
37 | @echo off
38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
40 |
41 | @REM set %HOME% to equivalent of $HOME
42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
43 |
44 | @REM Execute a user defined script before this one
45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending
47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
49 | :skipRcPre
50 |
51 | @setlocal
52 |
53 | set ERROR_CODE=0
54 |
55 | @REM To isolate internal variables from possible post scripts, we use another setlocal
56 | @setlocal
57 |
58 | @REM ==== START VALIDATION ====
59 | if not "%JAVA_HOME%" == "" goto OkJHome
60 |
61 | echo.
62 | echo Error: JAVA_HOME not found in your environment. >&2
63 | echo Please set the JAVA_HOME variable in your environment to match the >&2
64 | echo location of your Java installation. >&2
65 | echo.
66 | goto error
67 |
68 | :OkJHome
69 | if exist "%JAVA_HOME%\bin\java.exe" goto init
70 |
71 | echo.
72 | echo Error: JAVA_HOME is set to an invalid directory. >&2
73 | echo JAVA_HOME = "%JAVA_HOME%" >&2
74 | echo Please set the JAVA_HOME variable in your environment to match the >&2
75 | echo location of your Java installation. >&2
76 | echo.
77 | goto error
78 |
79 | @REM ==== END VALIDATION ====
80 |
81 | :init
82 |
83 | set MAVEN_CMD_LINE_ARGS=%*
84 |
85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
86 | @REM Fallback to current working directory if not found.
87 |
88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
90 |
91 | set EXEC_DIR=%CD%
92 | set WDIR=%EXEC_DIR%
93 | :findBaseDir
94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound
95 | cd ..
96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound
97 | set WDIR=%CD%
98 | goto findBaseDir
99 |
100 | :baseDirFound
101 | set MAVEN_PROJECTBASEDIR=%WDIR%
102 | cd "%EXEC_DIR%"
103 | goto endDetectBaseDir
104 |
105 | :baseDirNotFound
106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
107 | cd "%EXEC_DIR%"
108 |
109 | :endDetectBaseDir
110 |
111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
112 |
113 | @setlocal EnableExtensions EnableDelayedExpansion
114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
116 |
117 | :endReadAdditionalConfig
118 |
119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
120 |
121 | set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar""
122 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
123 |
124 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS%
125 | if ERRORLEVEL 1 goto error
126 | goto end
127 |
128 | :error
129 | set ERROR_CODE=1
130 |
131 | :end
132 | @endlocal & set ERROR_CODE=%ERROR_CODE%
133 |
134 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
135 | @REM check for post script, once with legacy .bat ending and once with .cmd ending
136 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
137 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
138 | :skipRcPost
139 |
140 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
141 | if "%MAVEN_BATCH_PAUSE%" == "on" pause
142 |
143 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
144 |
145 | exit /B %ERROR_CODE%
146 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 4.0.0
5 | org.springframework.cloud
6 | spring-cloud-deployer-yarn-parent
7 | 1.2.3.BUILD-SNAPSHOT
8 | pom
9 |
10 |
11 | org.springframework.cloud
12 | spring-cloud-deployer-parent
13 | 1.2.1.RELEASE
14 |
15 |
16 |
17 |
18 | 1.2.1.RELEASE
19 | 2.4.0.RELEASE
20 | 1.1.0.RELEASE
21 |
22 |
23 |
24 | spring-cloud-deployer-yarn-appdeployerappmaster
25 | spring-cloud-deployer-yarn-tasklauncherappmaster
26 | spring-cloud-deployer-yarn
27 | spring-cloud-deployer-yarn-autoconfig
28 | spring-cloud-deployer-yarn-build-tests
29 | spring-cloud-starter-deployer-yarn
30 |
31 |
32 |
33 |
34 |
35 | org.springframework.boot
36 | spring-boot-starter-parent
37 | ${spring-boot.version}
38 | pom
39 | import
40 |
41 |
42 |
43 | org.springframework.data
44 | spring-yarn-boot
45 | ${spring-data-hadoop.version}
46 |
47 |
48 | org.springframework.data
49 | spring-yarn-boot-cli
50 | ${spring-data-hadoop.version}
51 |
52 |
53 | org.springframework.cloud
54 | spring-cloud-deployer-spi
55 | ${spring-cloud-deployer.version}
56 |
57 |
58 | org.springframework.cloud
59 | spring-cloud-deployer-resource-support
60 | ${spring-cloud-deployer.version}
61 |
62 |
63 | org.springframework.cloud
64 | spring-cloud-deployer-resource-maven
65 | ${spring-cloud-deployer.version}
66 |
67 |
68 | org.springframework.cloud
69 | spring-cloud-dataflow-server-core
70 | ${spring-cloud-dataflow.version}
71 |
72 |
73 | org.springframework.statemachine
74 | spring-statemachine-core
75 | ${spring-statemachine.version}
76 |
77 |
78 | org.springframework.statemachine
79 | spring-statemachine-uml
80 | ${spring-statemachine.version}
81 |
82 |
83 | org.springframework.statemachine
84 | spring-statemachine-test
85 | ${spring-statemachine.version}
86 | test
87 |
88 |
89 | org.springframework.data
90 | spring-yarn-boot-test
91 | ${spring-data-hadoop.version}
92 | test
93 |
94 |
95 |
96 |
97 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-appdeployerappmaster/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 4.0.0
5 | org.springframework.cloud
6 | spring-cloud-deployer-yarn-appdeployerappmaster
7 | 1.2.3.BUILD-SNAPSHOT
8 |
9 | org.springframework.cloud
10 | spring-cloud-deployer-yarn-parent
11 | 1.2.3.BUILD-SNAPSHOT
12 |
13 |
14 |
15 |
16 | org.springframework.data
17 | spring-yarn-boot
18 |
19 |
20 | javax.servlet
21 | servlet-api
22 |
23 |
24 | org.slf4j
25 | slf4j-log4j12
26 |
27 |
28 |
29 |
30 | org.springframework.boot
31 | spring-boot-starter-actuator
32 |
33 |
34 | org.springframework.boot
35 | spring-boot-starter-web
36 |
37 |
38 |
39 |
40 |
41 |
42 | src/main/resources
43 | true
44 |
45 | application.yml
46 |
47 |
48 |
49 |
50 |
51 | org.apache.maven.plugins
52 | maven-resources-plugin
53 |
54 |
55 | org.springframework.boot
56 | spring-boot-maven-plugin
57 |
58 |
59 |
60 |
61 |
62 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-appdeployerappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/appdeployer/StreamAppmaster.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2017 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.appdeployer;
18 |
19 | import java.util.ArrayList;
20 | import java.util.HashMap;
21 | import java.util.Iterator;
22 | import java.util.List;
23 | import java.util.Map;
24 | import java.util.Map.Entry;
25 |
26 | import org.apache.commons.logging.Log;
27 | import org.apache.commons.logging.LogFactory;
28 | import org.apache.hadoop.yarn.api.records.Container;
29 | import org.apache.hadoop.yarn.api.records.ContainerId;
30 | import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
31 | import org.apache.hadoop.yarn.api.records.ContainerStatus;
32 | import org.apache.hadoop.yarn.api.records.LocalResource;
33 | import org.apache.hadoop.yarn.api.records.LocalResourceType;
34 | import org.springframework.beans.factory.annotation.Autowired;
35 | import org.springframework.util.StringUtils;
36 | import org.springframework.yarn.am.ContainerLauncherInterceptor;
37 | import org.springframework.yarn.am.cluster.ContainerCluster;
38 | import org.springframework.yarn.am.cluster.ManagedContainerClusterAppmaster;
39 | import org.springframework.yarn.am.container.AbstractLauncher;
40 | import org.springframework.yarn.am.grid.GridMember;
41 | import org.springframework.yarn.am.grid.support.ProjectionData;
42 | import org.springframework.yarn.fs.LocalResourcesFactoryBean;
43 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.CopyEntry;
44 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.TransferEntry;
45 | import org.springframework.yarn.fs.ResourceLocalizer;
46 | import org.springframework.yarn.listener.ContainerMonitorListener;
47 |
48 | /**
49 | * Custom yarn appmaster tweaking container launch settings.
50 | *
51 | * @author Janne Valkealahti
52 | *
53 | */
54 | public class StreamAppmaster extends ManagedContainerClusterAppmaster {
55 |
56 | private final static Log log = LogFactory.getLog(StreamAppmaster.class);
57 | private final Map artifactLocalizers = new HashMap<>();
58 | private final ContainerIndexTracker indexTracker = new ContainerIndexTracker();
59 |
60 | /** ContainerId to ContainerCluster id map */
61 | private final Map containerIdMap = new HashMap<>();
62 |
63 | @Autowired
64 | private StreamAppmasterProperties streamAppmasterProperties;
65 |
66 | @Override
67 | protected void onInit() throws Exception {
68 | super.onInit();
69 |
70 | // TODO: we want to have a proper support in base classes to gracefully
71 | // shutdown appmaster when it has nothing to do. this trick
72 | // here is solely a workaround not being able to access internal
73 | // structures of base classes. this is pretty much all we can do
74 | // from a subclass.
75 | // potentially we want to make it configurable with a grace period, etc.
76 | getMonitor().addContainerMonitorStateListener(new ContainerMonitorListener() {
77 |
78 | @Override
79 | public void state(ContainerMonitorState state) {
80 | if (log.isDebugEnabled()) {
81 | log.info("Received monitor state " + state + " and container clusters size is " + getContainerClusters().size());
82 | }
83 | if (state.getRunning() == 0 && getContainerClusters().size() == 0) {
84 | // this state is valid at start but we know it's not gonna
85 | // get called until we have had at least one container running
86 | log.info("No running containers and no container clusters, initiate app shutdown");
87 | notifyCompleted();
88 | }
89 | }
90 | });
91 |
92 | if (getLauncher() instanceof AbstractLauncher) {
93 | ((AbstractLauncher)getLauncher()).addInterceptor(new IndexAddingContainerLauncherInterceptor());
94 | }
95 | }
96 |
97 | @Override
98 | public ContainerCluster createContainerCluster(String clusterId, String clusterDef, ProjectionData projectionData,
99 | Map extraProperties) {
100 | log.info("intercept createContainerCluster " + clusterId);
101 | String artifactPath = streamAppmasterProperties.getArtifact();
102 | try {
103 | LocalResourcesFactoryBean lrfb = new LocalResourcesFactoryBean();
104 | lrfb.setConfiguration(getConfiguration());
105 |
106 | String containerArtifact = (String) extraProperties.get("containerArtifact");
107 | TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifactPath + "/" + containerArtifact, false);
108 | ArrayList hdfsEntries = new ArrayList();
109 | hdfsEntries.add(te);
110 | lrfb.setHdfsEntries(hdfsEntries);
111 | lrfb.setCopyEntries(new ArrayList());
112 | lrfb.afterPropertiesSet();
113 | ResourceLocalizer rl = lrfb.getObject();
114 | log.info("Adding localizer for " + clusterId + " / " + rl);
115 | artifactLocalizers.put(clusterId, rl);
116 | } catch (Exception e) {
117 | log.error("Error creating localizer", e);
118 | }
119 |
120 | return super.createContainerCluster(clusterId, clusterDef, projectionData, extraProperties);
121 | }
122 |
123 | @Override
124 | protected List onContainerLaunchCommands(Container container, ContainerCluster cluster,
125 | List commands) {
126 | log.info("onContainerLaunchCommands commands=" + StringUtils.collectionToCommaDelimitedString(commands));
127 | ArrayList list = new ArrayList();
128 | Map extraProperties = cluster.getExtraProperties();
129 | String artifact = (String) extraProperties.get("containerArtifact");
130 |
131 | for (String command : commands) {
132 | if (command.contains("placeholder.jar")) {
133 | list.add(command.replace("placeholder.jar", artifact));
134 | } else {
135 | list.add(command);
136 | }
137 | }
138 |
139 | if (extraProperties != null) {
140 | for (Entry entry : extraProperties.entrySet()) {
141 | if (entry.getKey().startsWith("containerArg")) {
142 | log.info("onContainerLaunchCommands adding command=--" + entry.getValue().toString());
143 | list.add(Math.max(list.size() - 2, 0), "--" + entry.getValue().toString());
144 | }
145 | }
146 | }
147 | return list;
148 | }
149 |
150 | @Override
151 | protected Map buildLocalizedResources(ContainerCluster cluster) {
152 | Map resources = super.buildLocalizedResources(cluster);
153 | ResourceLocalizer rl = artifactLocalizers.get(cluster.getId());
154 | log.info("Localizer for " + cluster.getId() + " is " + rl);
155 | resources.putAll(rl.getResources());
156 | return resources;
157 | }
158 |
159 | @Override
160 | protected void onContainerCompleted(ContainerStatus status) {
161 | super.onContainerCompleted(status);
162 | String containerClusterId = containerIdMap.get(status.getContainerId());
163 | if (containerClusterId != null) {
164 | synchronized (indexTracker) {
165 | indexTracker.freeIndex(status.getContainerId(), containerClusterId);
166 | }
167 | }
168 | }
169 |
170 | private ContainerCluster findContainerClusterByContainerId(ContainerId containerId) {
171 | for (Entry entry : getContainerClusters().entrySet()) {
172 | for (GridMember member : entry.getValue().getGridProjection().getMembers()) {
173 | if (member.getId().equals(containerId)) {
174 | return entry.getValue();
175 | }
176 | }
177 | }
178 | return null;
179 | }
180 |
181 | /**
182 | * Interceptor adding INSTANCE_INDEX as env variable based on ContainerIndexTracker.
183 | */
184 | private class IndexAddingContainerLauncherInterceptor implements ContainerLauncherInterceptor {
185 |
186 | @Override
187 | public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
188 | ContainerCluster containerCluster = findContainerClusterByContainerId(container.getId());
189 | if (containerCluster == null) {
190 | return context;
191 | }
192 | containerIdMap.put(container.getId(), containerCluster.getId());
193 |
194 | Map environment = context.getEnvironment();
195 | Map indexEnv = new HashMap<>();
196 | indexEnv.putAll(environment);
197 | Integer reservedIndex;
198 | synchronized (indexTracker) {
199 | reservedIndex = indexTracker.reserveIndex(container.getId(), containerCluster);
200 | }
201 | indexEnv.put("SPRING_CLOUD_APPLICATION_GUID", container.getId().toString());
202 | indexEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(reservedIndex));
203 | indexEnv.put("INSTANCE_INDEX", Integer.toString(reservedIndex));
204 | context.setEnvironment(indexEnv);
205 | return context;
206 | }
207 | }
208 |
209 | /**
210 | * Support class tracking containers per group and reserving an index sequence.
211 | */
212 | private static class ContainerIndexTracker {
213 | // TODO: move this feature to spring-yarn where scaling can be
214 | // implemented accurately. scaling up/down is anyway not
215 | // supported in sc stream at this moment.
216 | Map> reservationsMap = new HashMap<>();
217 |
218 | Integer reserveIndex(ContainerId containerId, ContainerCluster containerCluster) {
219 | ArrayList reservationList = reservationsMap.get(containerCluster.getId());
220 | if (reservationList == null) {
221 | reservationList = new ArrayList<>();
222 | reservationsMap.put(containerCluster.getId(), reservationList);
223 | }
224 |
225 | // we always increment index at least once
226 | Iterator iterator = reservationList.iterator();
227 | int index = -1;
228 | ContainerId n = null;
229 | while(iterator.hasNext()) {
230 | ContainerId nn = iterator.next();
231 | index++;
232 | if (nn == null) {
233 | // we found existing nullified reservation, use that
234 | n = containerId;
235 | break;
236 | }
237 | }
238 |
239 | // all resevations in use, add new
240 | if (n == null) {
241 | reservationList.add(containerId);
242 | index++;
243 | }
244 | return index;
245 | }
246 |
247 | void freeIndex(ContainerId containerId, String containerClusterId) {
248 | ArrayList reservationList = reservationsMap.get(containerClusterId);
249 | if (reservationList != null) {
250 | for (int index = 0; index < reservationList.size(); index++) {
251 | if (containerId.equals(reservationList.get(index))) {
252 | // nullify existing reservation
253 | reservationList.set(index, null);
254 | return;
255 | }
256 | }
257 | }
258 | }
259 | }
260 | }
261 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-appdeployerappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/appdeployer/StreamAppmasterApplication.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.appdeployer;
18 |
19 | import org.springframework.boot.SpringApplication;
20 | import org.springframework.boot.autoconfigure.SpringBootApplication;
21 | import org.springframework.boot.context.properties.EnableConfigurationProperties;
22 |
23 | /**
24 | * Yarn application bootstrapping appmaster.
25 | *
26 | * @author Janne Valkealahti
27 | *
28 | */
29 | @SpringBootApplication
30 | @EnableConfigurationProperties({ StreamAppmasterProperties.class })
31 | public class StreamAppmasterApplication {
32 |
33 | public static void main(String[] args) {
34 | SpringApplication.run(StreamAppmasterApplication.class, args);
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-appdeployerappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/appdeployer/StreamAppmasterProperties.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.appdeployer;
18 |
19 | import org.springframework.boot.context.properties.ConfigurationProperties;
20 |
21 | /**
22 | * Configuration properties passed from a deployer into an appmaster.
23 | *
24 | * @author Janne Valkealahti
25 | */
26 | @ConfigurationProperties(value = "spring.cloud.deployer.yarn.appmaster")
27 | public class StreamAppmasterProperties {
28 |
29 | private String artifact;
30 |
31 | public String getArtifact() {
32 | return artifact;
33 | }
34 |
35 | public void setArtifact(String artifact) {
36 | this.artifact = artifact;
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-appdeployerappmaster/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | server:
2 | port: 0
3 | management:
4 | security:
5 | enabled: false
6 | endpoints:
7 | shutdown:
8 | enabled: true
9 | spring:
10 | main:
11 | show_banner: false
12 | cloud:
13 | dataflow:
14 | yarn:
15 | version: "@project.version@"
16 | yarn:
17 | appName: spring-cloud-dataflow-yarn-app
18 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/stream/
19 | appmaster:
20 | appmasterClass: org.springframework.cloud.deployer.spi.yarn.appdeployer.StreamAppmaster
21 | keepContextAlive: true
22 | localizer:
23 | patterns:
24 | - "servers.yml"
25 | containercluster:
26 | enabled: true
27 | clusters:
28 | module-template:
29 | resource:
30 | priority: ${spring.cloud.deployer.yarn.app.streamcontainer.priority:5}
31 | memory: ${spring.cloud.deployer.yarn.app.streamcontainer.memory:256m}
32 | virtualCores: ${spring.cloud.deployer.yarn.app.streamcontainer.virtualCores:1}
33 | labelExpression: ${spring.cloud.deployer.yarn.app.streamcontainer.labelExpression:}
34 | launchcontext:
35 | options:
36 | - ${spring.cloud.deployer.yarn.app.streamcontainer.javaOpts:}
37 | - ${spring.cloud.deployer.yarn.app.streamcontainer.saslConfig:}
38 | arguments:
39 | -Dserver.port: 0
40 | -Dendpoints.jmx.unique-names: true
41 | -Dspring.config.location: servers.yml
42 | locality: false
43 | archiveFile: placeholder.jar
44 | endpoints:
45 | containercluster:
46 | enabled: true
47 | containerregister:
48 | enabled: false
49 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-autoconfig/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 4.0.0
5 | spring-cloud-deployer-yarn-autoconfig
6 | 1.2.3.BUILD-SNAPSHOT
7 | jar
8 | spring-cloud-deployer-yarn-autoconfig
9 | Data Flow Deployer Autoconfig for Apache Yarn
10 |
11 |
12 | org.springframework.cloud
13 | spring-cloud-deployer-yarn-parent
14 | 1.2.3.BUILD-SNAPSHOT
15 |
16 |
17 |
18 |
19 | org.springframework.cloud
20 | spring-cloud-deployer-yarn
21 | 1.2.3.BUILD-SNAPSHOT
22 |
23 |
24 | org.springframework.cloud
25 | spring-cloud-deployer-resource-support
26 |
27 |
28 | org.springframework.cloud
29 | spring-cloud-deployer-resource-maven
30 |
31 |
32 |
33 |
34 |
35 | org.apache.maven.plugins
36 | maven-resources-plugin
37 |
38 |
39 |
40 |
41 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-autoconfig/src/main/java/org/springframework/cloud/deployer/spi/yarn/autoconfigure/YarnDeployerAutoConfiguration.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.autoconfigure;
18 |
19 | import java.util.HashMap;
20 | import java.util.Map;
21 |
22 | import org.springframework.beans.factory.BeanFactory;
23 | import org.springframework.beans.factory.annotation.Autowired;
24 | import org.springframework.beans.factory.annotation.Value;
25 | import org.springframework.boot.autoconfigure.AutoConfigureOrder;
26 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
27 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
28 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
29 | import org.springframework.boot.context.properties.EnableConfigurationProperties;
30 | import org.springframework.cloud.deployer.resource.maven.MavenResourceLoader;
31 | import org.springframework.cloud.deployer.resource.support.DelegatingResourceLoader;
32 | import org.springframework.cloud.deployer.spi.app.AppDeployer;
33 | import org.springframework.cloud.deployer.spi.task.TaskLauncher;
34 | import org.springframework.cloud.deployer.spi.yarn.AppDeployerStateMachine;
35 | import org.springframework.cloud.deployer.spi.yarn.DefaultYarnCloudAppService;
36 | import org.springframework.cloud.deployer.spi.yarn.TaskLauncherStateMachine;
37 | import org.springframework.cloud.deployer.spi.yarn.YarnAppDeployer;
38 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService;
39 | import org.springframework.cloud.deployer.spi.yarn.YarnDeployerProperties;
40 | import org.springframework.cloud.deployer.spi.yarn.YarnTaskLauncher;
41 | import org.springframework.context.ApplicationContext;
42 | import org.springframework.context.annotation.Bean;
43 | import org.springframework.context.annotation.Configuration;
44 | import org.springframework.core.Ordered;
45 | import org.springframework.core.io.DefaultResourceLoader;
46 | import org.springframework.core.io.ResourceLoader;
47 | import org.springframework.core.task.TaskExecutor;
48 | import org.springframework.data.hadoop.fs.HdfsResourceLoader;
49 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
50 |
51 | /**
52 | * {@link EnableAutoConfiguration Auto-configuration} for dataflow
53 | * YARN deployer.
54 | *
55 | * @author Janne Valkealahti
56 | *
57 | */
58 | @Configuration
59 | @AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE)
60 | @ConditionalOnClass({ AppDeployer.class, TaskLauncher.class })
61 | @EnableConfigurationProperties({ YarnDeployerProperties.class })
62 | public class YarnDeployerAutoConfiguration {
63 |
64 | @Value("${spring.cloud.deployer.yarn.version:}")
65 | private String deployerVersion;
66 |
67 | @Autowired(required = false)
68 | private MavenResourceLoader mavenResourceLoader;
69 |
70 | @Bean
71 | public DelegatingResourceLoader delegatingResourceLoader(org.apache.hadoop.conf.Configuration configuration) {
72 | DefaultResourceLoader defaultLoader = new DefaultResourceLoader();
73 | Map loaders = new HashMap<>();
74 | loaders.put("hdfs", new HdfsResourceLoader(configuration));
75 | if (mavenResourceLoader != null) {
76 | loaders.put("maven", mavenResourceLoader);
77 | }
78 | loaders.put("file", defaultLoader);
79 | loaders.put("http", defaultLoader);
80 | return new DelegatingResourceLoader(loaders);
81 | }
82 |
83 | @Bean
84 | public YarnCloudAppService yarnCloudAppService() {
85 | return new DefaultYarnCloudAppService(deployerVersion);
86 | }
87 |
88 | @Bean
89 | public TaskExecutor yarnModuleDeployerTaskExecutor() {
90 | ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
91 | executor.setCorePoolSize(1);
92 | return executor;
93 | }
94 |
95 | @Configuration
96 | @ConditionalOnMissingBean(name = "appDeployer")
97 | public static class ProcessModuleDeployerConfig {
98 |
99 | @Bean
100 | public AppDeployerStateMachine appDeployerStateMachine(YarnCloudAppService yarnCloudAppService,
101 | TaskExecutor yarnModuleDeployerTaskExecutor, BeanFactory beanFactory, ApplicationContext applicationContext) throws Exception {
102 | return new AppDeployerStateMachine(yarnCloudAppService, yarnModuleDeployerTaskExecutor, beanFactory, applicationContext);
103 | }
104 |
105 | @Bean
106 | public AppDeployer appDeployer(YarnCloudAppService yarnCloudAppService,
107 | AppDeployerStateMachine appDeployerStateMachine) throws Exception {
108 | return new YarnAppDeployer(yarnCloudAppService, appDeployerStateMachine.buildStateMachine());
109 | }
110 | }
111 |
112 | @Configuration
113 | @ConditionalOnMissingBean(name = "taskLauncher")
114 | public static class TaskModuleDeployerConfig {
115 |
116 | @Bean
117 | public TaskLauncherStateMachine taskLauncherStateMachine(YarnCloudAppService yarnCloudAppService,
118 | TaskExecutor yarnModuleDeployerTaskExecutor, BeanFactory beanFactory, ApplicationContext applicationContext) throws Exception {
119 | return new TaskLauncherStateMachine(yarnCloudAppService, yarnModuleDeployerTaskExecutor, beanFactory, applicationContext);
120 | }
121 |
122 | @Bean
123 | public TaskLauncher taskLauncher(YarnCloudAppService yarnCloudAppService,
124 | TaskLauncherStateMachine taskLauncherStateMachine) throws Exception {
125 | return new YarnTaskLauncher(yarnCloudAppService, taskLauncherStateMachine.buildStateMachine());
126 | }
127 | }
128 | }
129 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-autoconfig/src/main/resources/META-INF/spring.factories:
--------------------------------------------------------------------------------
1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.springframework.cloud.deployer.spi.yarn.autoconfigure.YarnDeployerAutoConfiguration
2 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-build-tests/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 4.0.0
5 | spring-cloud-deployer-yarn-build-tests
6 | jar
7 | spring-cloud-deployer-yarn-build-tests
8 | Yarn SPI tests
9 |
10 |
11 | org.springframework.cloud
12 | spring-cloud-deployer-yarn-parent
13 | 1.2.3.BUILD-SNAPSHOT
14 |
15 |
16 |
17 |
18 | org.springframework.data
19 | spring-yarn-boot-test
20 | test
21 |
22 |
23 | org.slf4j
24 | slf4j-log4j12
25 |
26 |
27 | javax.servlet
28 | servlet-api
29 |
30 |
31 |
32 |
33 | com.sun.jersey
34 | jersey-core
35 | 1.9
36 | test
37 |
38 |
39 | com.sun.jersey
40 | jersey-client
41 | 1.9
42 | test
43 |
44 |
45 | com.sun.jersey
46 | jersey-server
47 | 1.9
48 | test
49 |
50 |
51 | org.springframework.cloud
52 | spring-cloud-deployer-yarn
53 | 1.2.3.BUILD-SNAPSHOT
54 | test
55 |
56 |
57 | org.springframework.cloud
58 | spring-cloud-deployer-resource-maven
59 | test
60 |
61 |
62 | org.springframework.cloud
63 | spring-cloud-deployer-yarn-appdeployerappmaster
64 | 1.2.3.BUILD-SNAPSHOT
65 | test
66 |
67 |
68 | org.springframework.cloud
69 | spring-cloud-deployer-yarn-tasklauncherappmaster
70 | 1.2.3.BUILD-SNAPSHOT
71 | test
72 |
73 |
74 | org.springframework.boot
75 | spring-boot-starter-test
76 | test
77 |
78 |
79 |
80 |
81 |
82 |
83 | maven-assembly-plugin
84 | 2.5.1
85 |
86 |
87 | src/main/assembly/assembly.xml
88 |
89 |
90 |
91 |
92 | test-assembly
93 | package
94 |
95 | single
96 |
97 |
98 | ${project.name}
99 | false
100 |
101 |
102 |
103 |
104 |
105 | maven-failsafe-plugin
106 |
107 | -Xmx2048m -XX:MaxPermSize=768m
108 |
109 |
110 |
111 | package
112 |
113 | integration-test
114 | verify
115 |
116 |
117 |
118 |
119 |
120 | com.internetitem
121 | write-properties-file-maven-plugin
122 | 1.0.1
123 |
124 |
125 | one
126 | compile
127 |
128 | write-properties-file
129 |
130 |
131 | ${basedir}/target/spring-cloud-deployer-yarn-build-tests
132 | test.properties
133 |
134 |
135 | projectVersion
136 | ${project.version}
137 |
138 |
139 | artifactVersion
140 | 1.0.0.BUILD-SNAPSHOT
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-build-tests/src/main/assembly/assembly.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 | dist
7 |
8 | dir
9 |
10 | false
11 |
12 |
13 | true
14 |
15 | org.springframework.cloud:spring-cloud-deployer-yarn-appdeployerappmaster
16 | org.springframework.cloud:spring-cloud-deployer-yarn-tasklauncherappmaster
17 |
18 |
19 | false
20 | false
21 |
22 |
23 |
24 |
25 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-build-tests/src/test/java/org/springframework/cloud/dataflow/yarn/buildtests/AbstractCliBootYarnClusterTests.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.dataflow.yarn.buildtests;
18 |
19 | import static org.hamcrest.CoreMatchers.notNullValue;
20 | import static org.hamcrest.Matchers.is;
21 | import static org.junit.Assert.assertThat;
22 |
23 | import java.io.File;
24 | import java.io.IOException;
25 | import java.util.List;
26 | import java.util.concurrent.TimeUnit;
27 |
28 | import org.apache.hadoop.conf.Configuration;
29 | import org.apache.hadoop.fs.FileSystem;
30 | import org.apache.hadoop.fs.LocatedFileStatus;
31 | import org.apache.hadoop.fs.Path;
32 | import org.apache.hadoop.fs.RemoteIterator;
33 | import org.apache.hadoop.yarn.api.records.ApplicationId;
34 | import org.apache.hadoop.yarn.api.records.ApplicationReport;
35 | import org.apache.hadoop.yarn.api.records.YarnApplicationState;
36 | import org.junit.Before;
37 | import org.junit.runner.RunWith;
38 | import org.springframework.beans.factory.annotation.Autowired;
39 | import org.springframework.boot.builder.SpringApplicationBuilder;
40 | import org.springframework.context.ApplicationContext;
41 | import org.springframework.context.ApplicationContextAware;
42 | import org.springframework.context.ApplicationContextInitializer;
43 | import org.springframework.context.ConfigurableApplicationContext;
44 | import org.springframework.context.EnvironmentAware;
45 | import org.springframework.core.env.Environment;
46 | import org.springframework.core.io.Resource;
47 | import org.springframework.test.annotation.DirtiesContext;
48 | import org.springframework.test.annotation.DirtiesContext.ClassMode;
49 | import org.springframework.test.context.TestPropertySource;
50 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
51 | import org.springframework.util.Assert;
52 | import org.springframework.yarn.boot.SpringApplicationCallback;
53 | import org.springframework.yarn.boot.SpringApplicationTemplate;
54 | import org.springframework.yarn.client.YarnClient;
55 | import org.springframework.yarn.test.context.MiniYarnClusterTest;
56 | import org.springframework.yarn.test.context.YarnCluster;
57 | import org.springframework.yarn.test.junit.ApplicationInfo;
58 | import org.springframework.yarn.test.support.ContainerLogUtils;
59 |
60 | @RunWith(SpringJUnit4ClassRunner.class)
61 | @MiniYarnClusterTest
62 | @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD)
63 | @TestPropertySource("file:target/spring-cloud-deployer-yarn-build-tests/test.properties")
64 | public class AbstractCliBootYarnClusterTests implements ApplicationContextAware, EnvironmentAware {
65 |
66 | private ApplicationContext applicationContext;
67 | private Environment environment;
68 | private Configuration configuration;
69 | private YarnCluster yarnCluster;
70 | private YarnClient yarnClient;
71 | private String projectVersion;
72 |
73 | @Before
74 | public void setup() {
75 | projectVersion = getEnvironment().getProperty("projectVersion");
76 | }
77 |
78 | @Override
79 | public final void setApplicationContext(ApplicationContext applicationContext) {
80 | this.applicationContext = applicationContext;
81 | }
82 |
83 | @Autowired
84 | public void setConfiguration(Configuration configuration) {
85 | this.configuration = configuration;
86 | }
87 |
88 | @Autowired
89 | public void setYarnCluster(YarnCluster yarnCluster) {
90 | this.yarnCluster = yarnCluster;
91 | }
92 |
93 | @Override
94 | public void setEnvironment(Environment environment) {
95 | this.environment = environment;
96 | }
97 |
98 | public ApplicationContext getApplicationContext() {
99 | return applicationContext;
100 | }
101 |
102 | public Environment getEnvironment() {
103 | return environment;
104 | }
105 |
106 | public Configuration getConfiguration() {
107 | return configuration;
108 | }
109 |
110 | public void setYarnClient(YarnClient yarnClient) {
111 | this.yarnClient = yarnClient;
112 | }
113 |
114 | public YarnClient getYarnClient() {
115 | return yarnClient;
116 | }
117 |
118 | public YarnCluster getYarnCluster() {
119 | return yarnCluster;
120 | }
121 |
122 | public String getProjectVersion() {
123 | return projectVersion;
124 | }
125 |
126 | protected ApplicationInfo submitApplicationAndWait(Object source, String[] args) throws Exception {
127 | return submitApplicationAndWait(source, args, 60, TimeUnit.SECONDS);
128 | }
129 |
130 | protected ApplicationInfo submitApplicationAndWait(Object source, String[] args, long timeout, final TimeUnit unit) throws Exception {
131 | return submitApplicationAndWaitState(source, args, timeout, unit, YarnApplicationState.FINISHED, YarnApplicationState.FAILED);
132 | }
133 |
134 | protected ApplicationInfo submitApplicationAndWaitState(long timeout, TimeUnit unit, YarnApplicationState... applicationStates) throws Exception {
135 | Assert.notEmpty(applicationStates, "Need to have atleast one state");
136 | Assert.notNull(getYarnClient(), "Yarn client must be set");
137 |
138 | YarnApplicationState state = null;
139 | ApplicationReport report = null;
140 |
141 | ApplicationId applicationId = submitApplication();
142 | Assert.notNull(applicationId, "Failed to get application id from submit");
143 |
144 | long end = System.currentTimeMillis() + unit.toMillis(timeout);
145 |
146 | // break label for inner loop
147 | done:
148 | do {
149 | report = findApplicationReport(getYarnClient(), applicationId);
150 | if (report == null) {
151 | break;
152 | }
153 | state = report.getYarnApplicationState();
154 | for (YarnApplicationState stateCheck : applicationStates) {
155 | if (state.equals(stateCheck)) {
156 | break done;
157 | }
158 | }
159 | Thread.sleep(1000);
160 | } while (System.currentTimeMillis() < end);
161 | return new ApplicationInfo(applicationId, report);
162 | }
163 |
164 | protected ApplicationInfo waitState(ApplicationId applicationId, long timeout, TimeUnit unit, YarnApplicationState... applicationStates) throws Exception {
165 | YarnApplicationState state = null;
166 | ApplicationReport report = null;
167 | long end = System.currentTimeMillis() + unit.toMillis(timeout);
168 |
169 | // break label for inner loop
170 | done:
171 | do {
172 | report = findApplicationReport(getYarnClient(), applicationId);
173 | if (report == null) {
174 | break;
175 | }
176 | state = report.getYarnApplicationState();
177 | for (YarnApplicationState stateCheck : applicationStates) {
178 | if (state.equals(stateCheck)) {
179 | break done;
180 | }
181 | }
182 | Thread.sleep(1000);
183 | } while (System.currentTimeMillis() < end);
184 | return new ApplicationInfo(applicationId, report);
185 | }
186 |
187 |
188 | protected ApplicationInfo submitApplicationAndWaitState(Object source, String[] args, final long timeout,
189 | final TimeUnit unit, final YarnApplicationState... applicationStates) throws Exception {
190 |
191 | SpringApplicationBuilder builder = new SpringApplicationBuilder(source);
192 | builder.initializers(new HadoopConfigurationInjectingInitializer(getConfiguration()));
193 |
194 | SpringApplicationTemplate template = new SpringApplicationTemplate(builder);
195 | return template.execute(new SpringApplicationCallback() {
196 |
197 | @Override
198 | public ApplicationInfo runWithSpringApplication(ApplicationContext context) throws Exception {
199 | setYarnClient(context.getBean(YarnClient.class));
200 | return submitApplicationAndWaitState(timeout, unit, applicationStates);
201 | }
202 |
203 | }, args);
204 | }
205 |
206 | protected File assertWaitFileContent(long timeout, TimeUnit unit, ApplicationId applicationId, String search) throws Exception {
207 | File file = null;
208 |
209 | long end = System.currentTimeMillis() + unit.toMillis(timeout);
210 | done:
211 | do {
212 |
213 | List resources = ContainerLogUtils.queryContainerLogs(
214 | getYarnCluster(), applicationId);
215 | for (Resource res : resources) {
216 | File f = res.getFile();
217 | String content = ContainerLogUtils.getFileContent(f);
218 | if (content.contains(search)) {
219 | file = f;
220 | break done;
221 | }
222 | }
223 |
224 | Thread.sleep(1000);
225 | } while (System.currentTimeMillis() < end);
226 |
227 |
228 | assertThat(file, notNullValue());
229 | return file;
230 | }
231 |
232 | protected String dumpFs() throws IOException {
233 | StringBuilder buf = new StringBuilder();
234 | FileSystem fs = FileSystem.get(getConfiguration());
235 | RemoteIterator files = fs.listFiles(new Path("/"), true);
236 | while (files.hasNext()) {
237 | buf.append(files.next().toString());
238 | buf.append("\n");
239 | }
240 | return buf.toString();
241 | }
242 |
243 | protected void waitHdfsFile(String path, long timeout, TimeUnit unit) throws Exception {
244 | Path p = new Path(path);
245 | FileSystem fs = FileSystem.get(getConfiguration());
246 | long end = System.currentTimeMillis() + unit.toMillis(timeout);
247 | boolean found = false;
248 | do {
249 | if (fs.exists(p)) {
250 | found = true;
251 | break;
252 | }
253 | Thread.sleep(1000);
254 | } while (System.currentTimeMillis() < end);
255 | assertThat(found, is(true));
256 | }
257 |
258 | private ApplicationReport findApplicationReport(YarnClient client, ApplicationId applicationId) {
259 | Assert.notNull(getYarnClient(), "Yarn client must be set");
260 | for (ApplicationReport report : client.listApplications()) {
261 | if (report.getApplicationId().equals(applicationId)) {
262 | return report;
263 | }
264 | }
265 | return null;
266 | }
267 |
268 | protected ApplicationId submitApplication() {
269 | Assert.notNull(getYarnClient(), "Yarn client must be set");
270 | ApplicationId applicationId = getYarnClient().submitApplication();
271 | Assert.notNull(applicationId, "Failed to get application id from submit");
272 | return applicationId;
273 | }
274 |
275 | public static class HadoopConfigurationInjectingInitializer
276 | implements ApplicationContextInitializer {
277 |
278 | private final Configuration configuration;
279 |
280 | public HadoopConfigurationInjectingInitializer(Configuration configuration) {
281 | this.configuration = configuration;
282 | }
283 |
284 | @Override
285 | public void initialize(ConfigurableApplicationContext applicationContext) {
286 | applicationContext.getBeanFactory().registerSingleton("miniYarnConfiguration", configuration);
287 | }
288 |
289 | }
290 |
291 | }
292 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-build-tests/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | log4j.rootCategory=INFO, stdout
2 |
3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
5 | log4j.appender.stdout.layout.ConversionPattern=%d %p [%C{1}] [%t] - %m%n
6 |
7 | log4j.category.org.apache.hadoop=INFO
8 | log4j.category.org.springframework.statemachine=DEBUG
9 | log4j.category.org.springframework.yarn=DEBUG
10 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-build-tests/src/test/resources/yarn-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | yarn.nodemanager.sleep-delay-before-sigkill.ms
9 | 10000
10 |
11 |
12 | yarn.nodemanager.delete.debug-delay-sec
13 | 600
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-tasklauncherappmaster/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 4.0.0
5 | org.springframework.cloud
6 | spring-cloud-deployer-yarn-tasklauncherappmaster
7 | 1.2.3.BUILD-SNAPSHOT
8 |
9 | org.springframework.cloud
10 | spring-cloud-deployer-yarn-parent
11 | 1.2.3.BUILD-SNAPSHOT
12 |
13 |
14 |
15 |
16 | org.springframework.data
17 | spring-yarn-boot
18 |
19 |
20 | javax.servlet
21 | servlet-api
22 |
23 |
24 | org.slf4j
25 | slf4j-log4j12
26 |
27 |
28 |
29 |
30 | org.springframework.boot
31 | spring-boot-starter-actuator
32 |
33 |
34 | org.springframework.boot
35 | spring-boot-starter-web
36 |
37 |
38 |
39 |
40 |
41 |
42 | src/main/resources
43 | true
44 |
45 | application.yml
46 |
47 |
48 |
49 |
50 |
51 | org.apache.maven.plugins
52 | maven-resources-plugin
53 |
54 |
55 | org.springframework.boot
56 | spring-boot-maven-plugin
57 |
58 |
59 |
60 |
61 |
62 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/tasklauncher/TaskAppmaster.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.tasklauncher;
18 |
19 | import java.util.ArrayList;
20 | import java.util.HashMap;
21 | import java.util.List;
22 | import java.util.Map;
23 | import java.util.Map.Entry;
24 |
25 | import org.apache.commons.logging.Log;
26 | import org.apache.commons.logging.LogFactory;
27 | import org.apache.hadoop.yarn.api.records.Container;
28 | import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
29 | import org.apache.hadoop.yarn.api.records.LocalResource;
30 | import org.apache.hadoop.yarn.api.records.LocalResourceType;
31 | import org.springframework.beans.factory.annotation.Autowired;
32 | import org.springframework.context.Lifecycle;
33 | import org.springframework.util.Assert;
34 | import org.springframework.util.StringUtils;
35 | import org.springframework.yarn.am.ContainerLauncherInterceptor;
36 | import org.springframework.yarn.am.StaticEventingAppmaster;
37 | import org.springframework.yarn.am.container.AbstractLauncher;
38 | import org.springframework.yarn.fs.LocalResourcesFactoryBean;
39 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.CopyEntry;
40 | import org.springframework.yarn.fs.LocalResourcesFactoryBean.TransferEntry;
41 | import org.springframework.yarn.fs.ResourceLocalizer;
42 |
43 | /**
44 | * Spring YARN application master class which does a specific
45 | * handling of a task containers.
46 | *
47 | * @author Janne Valkealahti
48 | */
49 | public class TaskAppmaster extends StaticEventingAppmaster {
50 |
51 | private final static Log log = LogFactory.getLog(TaskAppmaster.class);
52 |
53 | @Autowired
54 | private TaskAppmasterProperties taskAppmasterProperties;
55 |
56 | private ResourceLocalizer artifactResourceLocalizer;
57 |
58 | private boolean finishRequested = false;
59 |
60 | @Override
61 | protected void onInit() throws Exception {
62 | super.onInit();
63 | Assert.hasText(taskAppmasterProperties.getArtifact(), "Artifact must be set");
64 | Assert.hasText(taskAppmasterProperties.getArtifactName(), "Artifact name must be set");
65 | artifactResourceLocalizer = buildArtifactResourceLocalizer();
66 | if (getLauncher() instanceof AbstractLauncher) {
67 | ((AbstractLauncher) getLauncher()).addInterceptor(new ArtifactResourceContainerLaunchInterceptor());
68 | }
69 | }
70 |
71 | @Override
72 | protected void doStop() {
73 | if (getAllocator() instanceof Lifecycle) {
74 | ((Lifecycle)getAllocator()).stop();
75 | }
76 | if (!finishRequested) {
77 | super.doStop();
78 | }
79 | }
80 |
81 | @Override
82 | public List getCommands() {
83 | List list = new ArrayList<>();
84 | for (String command : super.getCommands()) {
85 | if (command.contains("placeholder.jar")) {
86 | list.add(command.replace("placeholder.jar", taskAppmasterProperties.getArtifactName()));
87 | } else {
88 | list.add(command);
89 | }
90 | }
91 | if (taskAppmasterProperties.getParameters() != null) {
92 | for (Entry entry : taskAppmasterProperties.getParameters().entrySet()) {
93 | if (StringUtils.hasText(entry.getValue())) {
94 | // don't pass empty values as those will most likely fail
95 | // with spring/boot command-line parser
96 | list.add(Math.max(list.size() - 2, 0), "--" + entry.getKey() + "='" + entry.getValue() + "'");
97 | }
98 | }
99 | }
100 | if (taskAppmasterProperties.getCommandlineArguments() != null) {
101 | for (String commandlineArgument : taskAppmasterProperties.getCommandlineArguments()) {
102 | list.add(Math.max(list.size() - 2, 0), "'" + commandlineArgument + "'");
103 | }
104 | }
105 | log.info("Using command list for task container: " + StringUtils.collectionToCommaDelimitedString(list));
106 | return list;
107 | }
108 |
109 | @Override
110 | protected boolean shutdownContainers() {
111 | // SHDP-561
112 | return true;
113 | }
114 |
115 | @Override
116 | protected void notifyCompleted() {
117 | // SHDP-561
118 | finishAppmaster();
119 | finishRequested = true;
120 | super.notifyCompleted();
121 | }
122 |
123 | private ResourceLocalizer buildArtifactResourceLocalizer() throws Exception {
124 | String artifact = taskAppmasterProperties.getArtifact();
125 | log.info("Building localizer for artifact " + artifact);
126 | LocalResourcesFactoryBean fb = new LocalResourcesFactoryBean();
127 | fb.setConfiguration(getConfiguration());
128 | TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifact, false);
129 | ArrayList hdfsEntries = new ArrayList();
130 | hdfsEntries.add(te);
131 | fb.setHdfsEntries(hdfsEntries);
132 | fb.setCopyEntries(new ArrayList());
133 | fb.afterPropertiesSet();
134 | return fb.getObject();
135 | }
136 |
137 | private class ArtifactResourceContainerLaunchInterceptor implements ContainerLauncherInterceptor {
138 |
139 | @Override
140 | public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
141 | Map resources = new HashMap();
142 | resources.putAll(context.getLocalResources());
143 | resources.putAll(artifactResourceLocalizer.getResources());
144 | context.setLocalResources(resources);
145 | return context;
146 | }
147 | }
148 | }
149 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/tasklauncher/TaskAppmasterApplication.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.tasklauncher;
18 |
19 | import org.springframework.boot.SpringApplication;
20 | import org.springframework.boot.autoconfigure.SpringBootApplication;
21 | import org.springframework.boot.context.properties.EnableConfigurationProperties;
22 |
23 | /**
24 | * Yarn application bootstrapping appmaster.
25 | *
26 | * @author Janne Valkealahti
27 | */
28 | @SpringBootApplication
29 | @EnableConfigurationProperties({ TaskAppmasterProperties.class })
30 | public class TaskAppmasterApplication {
31 |
32 | public static void main(String[] args) {
33 | SpringApplication.run(TaskAppmasterApplication.class, args);
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/java/org/springframework/cloud/deployer/spi/yarn/tasklauncher/TaskAppmasterProperties.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn.tasklauncher;
18 |
19 | import java.util.List;
20 | import java.util.Map;
21 |
22 | import org.springframework.boot.context.properties.ConfigurationProperties;
23 |
24 | /**
25 | * Configuration properties passed from a deployer into an appmaster.
26 | *
27 | * @author Janne Valkealahti
28 | */
29 | @ConfigurationProperties(value = "spring.cloud.deployer.yarn.appmaster")
30 | public class TaskAppmasterProperties {
31 |
32 | private String artifact;
33 | private String artifactName;
34 | private Map parameters;
35 | private List commandlineArguments;
36 |
37 | public String getArtifact() {
38 | return artifact;
39 | }
40 |
41 | public void setArtifact(String artifact) {
42 | this.artifact = artifact;
43 | }
44 |
45 | public String getArtifactName() {
46 | return artifactName;
47 | }
48 |
49 | public void setArtifactName(String artifactName) {
50 | this.artifactName = artifactName;
51 | }
52 |
53 | public Map getParameters() {
54 | return parameters;
55 | }
56 |
57 | public void setParameters(Map parameters) {
58 | this.parameters = parameters;
59 | }
60 |
61 | public List getCommandlineArguments() {
62 | return commandlineArguments;
63 | }
64 |
65 | public void setCommandlineArguments(List commandlineArguments) {
66 | this.commandlineArguments = commandlineArguments;
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn-tasklauncherappmaster/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | server:
2 | port: 0
3 | management:
4 | security:
5 | enabled: false
6 | endpoints:
7 | shutdown:
8 | enabled: true
9 | spring:
10 | main:
11 | show_banner: false
12 | cloud:
13 | dataflow:
14 | yarn:
15 | version: "@project.version@"
16 | yarn:
17 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/task/
18 | appmaster:
19 | containerCount: 1
20 | appmasterClass: org.springframework.cloud.deployer.spi.yarn.tasklauncher.TaskAppmaster
21 | localizer:
22 | patterns:
23 | - "servers.yml"
24 | resource:
25 | priority: ${spring.cloud.deployer.yarn.app.taskcontainer.priority:10}
26 | labelExpression: ${spring.cloud.deployer.yarn.app.taskcontainer.labelExpression:}
27 | memory: ${spring.cloud.deployer.yarn.app.taskcontainer.memory:256m}
28 | virtualCores: ${spring.cloud.deployer.yarn.app.taskcontainer.virtualCores:1}
29 | launchcontext:
30 | options:
31 | - ${spring.cloud.deployer.yarn.app.taskcontainer.javaOpts:}
32 | arguments:
33 | -Dserver.port: 0
34 | -Dspring.jmx.enabled: false
35 | -Dspring.config.location: servers.yml
36 | locality: false
37 | archiveFile: placeholder.jar
38 | endpoints:
39 | containerregister:
40 | enabled: false
41 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 4.0.0
5 | spring-cloud-deployer-yarn
6 | 1.2.3.BUILD-SNAPSHOT
7 | jar
8 | spring-cloud-deployer-yarn
9 | Data Flow Deployer for Apache Yarn
10 |
11 |
12 | org.springframework.cloud
13 | spring-cloud-deployer-yarn-parent
14 | 1.2.3.BUILD-SNAPSHOT
15 |
16 |
17 |
18 |
19 | org.springframework.cloud
20 | spring-cloud-deployer-spi
21 |
22 |
23 | org.springframework.data
24 | spring-yarn-boot
25 |
26 |
27 | org.slf4j
28 | slf4j-log4j12
29 |
30 |
31 | javax.servlet
32 | servlet-api
33 |
34 |
35 |
36 |
37 | org.springframework.boot
38 | spring-boot-starter-web
39 |
40 |
41 | org.springframework.statemachine
42 | spring-statemachine-uml
43 |
44 |
45 | org.springframework.statemachine
46 | spring-statemachine-test
47 | test
48 |
49 |
50 |
51 |
52 |
53 | src/main/resources
54 | true
55 |
56 | task.yml
57 | stream.yml
58 | appdeployer-model.uml
59 | tasklauncher-model.uml
60 |
61 |
62 |
63 |
64 |
65 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/AbstractDeployerStateMachine.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.io.IOException;
20 | import java.util.Collection;
21 | import java.util.Map;
22 |
23 | import org.slf4j.Logger;
24 | import org.slf4j.LoggerFactory;
25 | import org.springframework.beans.factory.BeanFactory;
26 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppInfo;
27 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType;
28 | import org.springframework.core.io.Resource;
29 | import org.springframework.core.io.ResourceLoader;
30 | import org.springframework.core.task.TaskExecutor;
31 | import org.springframework.statemachine.StateContext;
32 | import org.springframework.statemachine.StateMachine;
33 | import org.springframework.statemachine.action.Action;
34 | import org.springframework.statemachine.config.StateMachineBuilder;
35 | import org.springframework.statemachine.config.StateMachineBuilder.Builder;
36 | import org.springframework.statemachine.guard.Guard;
37 | import org.springframework.statemachine.uml.UmlStateMachineModelFactory;
38 | import org.springframework.util.Assert;
39 | import org.springframework.util.StringUtils;
40 |
41 | /**
42 | * Shared base functionality for state machines handling app deployment
43 | * and task launching.
44 | *
45 | * @author Janne Valkealahti
46 | *
47 | */
48 | public abstract class AbstractDeployerStateMachine {
49 |
50 | private static final Logger logger = LoggerFactory.getLogger(AbstractDeployerStateMachine.class);
51 |
52 | // common states
53 | public final static String STATE_READY = "READY";
54 | public final static String STATE_ERROR = "ERROR";
55 |
56 | // common extended state variables
57 | public static final String VAR_ERROR = "error";
58 | public static final String VAR_APP_VERSION = "appVersion";
59 | public static final String VAR_APPLICATION_ID = "applicationId";
60 | public static final String VAR_ARTIFACT = "artifact";
61 | public static final String VAR_MESSAGE_ID = "messageId";
62 |
63 | // common event headers
64 | public static final String HEADER_APPLICATION_ID = "applicationId";
65 | public static final String HEADER_APP_VERSION = "appVersion";
66 | public static final String HEADER_GROUP_ID = "groupId";
67 | public static final String HEADER_ERROR = "error";
68 | public static final String HEADER_ARTIFACT = "artifact";
69 | public static final String HEADER_ARTIFACT_DIR = "artifactDir";
70 | public static final String HEADER_CONTEXT_RUN_ARGS = "contextRunArgs";
71 | public static final String HEADER_DEFINITION_PARAMETERS = "definitionParameters";
72 |
73 | private final YarnCloudAppService yarnCloudAppService;
74 | private final TaskExecutor taskExecutor;
75 | private final BeanFactory beanFactory;
76 | private final ResourceLoader resourceLoader;
77 | private final String modelLocation;
78 |
79 | private boolean autoStart = true;
80 |
81 | /**
82 | * Instantiates a new abstract deployer state machine.
83 | *
84 | * @param yarnCloudAppService the yarn cloud app service
85 | * @param taskExecutor the task executor
86 | * @param beanFactory the bean factory
87 | * @param resourceLoader the resource loader
88 | * @param modelLocation the model location
89 | */
90 | public AbstractDeployerStateMachine(YarnCloudAppService yarnCloudAppService, TaskExecutor taskExecutor, BeanFactory beanFactory,
91 | ResourceLoader resourceLoader, String modelLocation) {
92 | Assert.notNull(yarnCloudAppService, "YarnCloudAppService must be set");
93 | Assert.notNull(taskExecutor, "TaskExecutor must be set");
94 | Assert.notNull(beanFactory, "BeanFactory must be set");
95 | Assert.notNull(resourceLoader, "ResourceLoader must be set");
96 | Assert.notNull(modelLocation, "Model must be set");
97 | this.yarnCloudAppService = yarnCloudAppService;
98 | this.taskExecutor = taskExecutor;
99 | this.beanFactory = beanFactory;
100 | this.resourceLoader = resourceLoader;
101 | this.modelLocation = modelLocation;
102 | }
103 |
104 | /**
105 | * Builds the state machine.
106 | *
107 | * @return the state machine
108 | * @throws Exception the exception
109 | */
110 | public final StateMachine buildStateMachine() throws Exception {
111 | Builder builder = StateMachineBuilder.builder();
112 |
113 | builder.configureConfiguration()
114 | .withConfiguration()
115 | .autoStartup(autoStart)
116 | .taskExecutor(taskExecutor)
117 | .beanFactory(beanFactory);
118 |
119 | UmlStateMachineModelFactory modelFactory = new UmlStateMachineModelFactory(modelLocation);
120 |
121 | Map> registeredActions = getRegisteredActions();
122 | if (registeredActions != null) {
123 | for (Map.Entry> entry : registeredActions.entrySet()) {
124 | // wraps actions to catch errors
125 | modelFactory.registerAction(entry.getKey(), new ExceptionCatchingAction(entry.getValue()));
126 | }
127 | }
128 |
129 | Map> registeredGuards = getRegisteredGuards();
130 | if (registeredGuards != null) {
131 | for (Map.Entry> entry : registeredGuards.entrySet()) {
132 | modelFactory.registerGuard(entry.getKey(), entry.getValue());
133 | }
134 | }
135 |
136 | modelFactory.setResourceLoader(resourceLoader);
137 | builder.configureModel()
138 | .withModel()
139 | .factory(modelFactory);
140 |
141 | return builder.build();
142 | }
143 |
144 | /**
145 | * Sets the auto start flag for builder.
146 | *
147 | * @param autoStart the new auto start
148 | */
149 | public void setAutoStart(boolean autoStart) {
150 | this.autoStart = autoStart;
151 | }
152 |
153 | /**
154 | * Gets the registered actions.
155 | *
156 | * @return the registered actions
157 | */
158 | protected abstract Map> getRegisteredActions();
159 |
160 | /**
161 | * Gets the registered guards.
162 | *
163 | * @return the registered guards
164 | */
165 | protected abstract Map> getRegisteredGuards();
166 |
167 | /**
168 | * Gets the yarn cloud app service.
169 | *
170 | * @return the yarn cloud app service
171 | */
172 | protected YarnCloudAppService getYarnCloudAppService() {
173 | return yarnCloudAppService;
174 | }
175 |
176 |
177 | /**
178 | * {@link Action} which clears existing extended state variables.
179 | */
180 | protected class ResetVariablesAction implements Action {
181 |
182 | @Override
183 | public void execute(StateContext context) {
184 | context.getExtendedState().getVariables().clear();
185 | }
186 | }
187 |
188 | /**
189 | * {@link Action} which queries {@link YarnCloudAppService} and checks if
190 | * passed {@code appVersion} from event headers exists and sends {@code ERROR}
191 | * event into state machine if it doesn't exist. Add to be used {@code appVersion}
192 | * into extended state variables which later used by other guards and actions.
193 | */
194 | protected class CheckAppAction implements Action {
195 |
196 | private final CloudAppType cloudAppType;
197 |
198 | public CheckAppAction(CloudAppType cloudAppType) {
199 | this.cloudAppType = cloudAppType;
200 | }
201 |
202 | @Override
203 | public void execute(StateContext context) {
204 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION);
205 |
206 | if (!StringUtils.hasText(appVersion)) {
207 | context.getExtendedState().getVariables().put(VAR_ERROR, new RuntimeException("appVersion not defined"));
208 | } else {
209 | Collection appInfos = yarnCloudAppService.getApplications(cloudAppType);
210 | for (CloudAppInfo appInfo : appInfos) {
211 | if (appInfo.getName().equals(appVersion)) {
212 | context.getExtendedState().getVariables().put(VAR_APP_VERSION, appVersion);
213 | }
214 | }
215 | }
216 | }
217 | }
218 |
219 | /**
220 | * {@link Action} which pushes application version into hdfs found
221 | * from variable {@code appVersion}.
222 | */
223 | protected class PushAppAction implements Action {
224 |
225 | private final CloudAppType cloudAppType;
226 |
227 | public PushAppAction(CloudAppType cloudAppType) {
228 | this.cloudAppType = cloudAppType;
229 | }
230 |
231 | @Override
232 | public void execute(StateContext context) {
233 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION);
234 | yarnCloudAppService.pushApplication(appVersion, cloudAppType);
235 | }
236 | }
237 |
238 | /**
239 | * {@link Action} which pushes artifact into hdfs.
240 | */
241 | protected class PushArtifactAction implements Action {
242 |
243 | @Override
244 | public void execute(StateContext context) {
245 | Resource artifact = (Resource) context.getMessageHeader(HEADER_ARTIFACT);
246 | String artifactDir = (String) context.getMessageHeader(HEADER_ARTIFACT_DIR);
247 | if (!isHdfsResource(artifact)) {
248 | yarnCloudAppService.pushArtifact(artifact, artifactDir);
249 | } else {
250 | if (!artifact.exists()) {
251 | context.getExtendedState().getVariables().put(VAR_ERROR, new RuntimeException("hdfs artifact missing"));
252 | }
253 | }
254 | }
255 | }
256 |
257 | /**
258 | * {@link Guard} which is used to protect state where application push
259 | * into hdfs would happen. Assumes that if {@code appVersion} variable
260 | * exists, application is installed.
261 | */
262 | protected class PushAppGuard implements Guard {
263 |
264 | @Override
265 | public boolean evaluate(StateContext context) {
266 | return !context.getExtendedState().getVariables().containsKey(VAR_APP_VERSION);
267 | }
268 | }
269 |
270 | /**
271 | * {@link Guard} which is used to protect transitions happening
272 | * in case on error set in variables.
273 | */
274 | protected class ErrorGuard implements Guard {
275 |
276 | @Override
277 | public boolean evaluate(StateContext context) {
278 | return context.getExtendedState().getVariables().containsKey(VAR_ERROR);
279 | }
280 | }
281 |
282 | /**
283 | * {@link Action} which simply logs context when machine passes
284 | * through error state.
285 | */
286 | protected class ErrorAction implements Action {
287 |
288 | @Override
289 | public void execute(StateContext context) {
290 | logger.error("Passing through error state {}", context);
291 | }
292 | }
293 |
294 | /**
295 | * {@link Action} which wraps a delegating action and sets an error if its
296 | * execution results an exception.
297 | */
298 | private class ExceptionCatchingAction implements Action {
299 |
300 | private final Action delegate;
301 |
302 | public ExceptionCatchingAction(Action delegate) {
303 | Assert.notNull(delegate, "Delegate action must be set");
304 | this.delegate = delegate;
305 | }
306 |
307 | @Override
308 | public void execute(StateContext context) {
309 | try {
310 | delegate.execute(context);
311 | } catch (Exception e) {
312 | context.getExtendedState().getVariables().put(VAR_ERROR, e);
313 | }
314 | }
315 | }
316 |
317 | private boolean isHdfsResource(Resource resource) {
318 | try {
319 | return resource != null && resource.getURI().getScheme().equals("hdfs");
320 | } catch (IOException e) {
321 | return false;
322 | }
323 | }
324 | }
325 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/AppDeployerStateMachine.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.util.HashMap;
20 | import java.util.List;
21 | import java.util.Map;
22 |
23 | import org.slf4j.Logger;
24 | import org.slf4j.LoggerFactory;
25 | import org.springframework.beans.factory.BeanFactory;
26 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppInstanceInfo;
27 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType;
28 | import org.springframework.core.io.Resource;
29 | import org.springframework.core.io.ResourceLoader;
30 | import org.springframework.core.task.TaskExecutor;
31 | import org.springframework.statemachine.StateContext;
32 | import org.springframework.statemachine.action.Action;
33 | import org.springframework.statemachine.guard.Guard;
34 | import org.springframework.util.StringUtils;
35 |
36 | /**
37 | * Statemachine for app deployments.
38 | *
39 | * @author Janne Valkealahti
40 | *
41 | */
42 | public class AppDeployerStateMachine extends AbstractDeployerStateMachine {
43 |
44 | private static final Logger logger = LoggerFactory.getLogger(AppDeployerStateMachine.class);
45 |
46 | private final static String MODEL_LOCATION = "classpath:appdeployer-model.uml";
47 |
48 | public final static String EVENT_DEPLOY = "DEPLOY";
49 | public final static String EVENT_UNDEPLOY = "UNDEPLOY";
50 |
51 | public static final String VAR_INSTANCE_ADDRESS = "instanceAddress";
52 | public static final String VAR_COUNT = "count";
53 | public static final String VAR_APPNAME = "appname";
54 | public static final String VAR_CLUSTER_ID = "clusterId";
55 | public static final String VAR_DEFINITION_PARAMETERS = "definitionParameters";
56 |
57 | public static final String HEADER_CLUSTER_ID = "clusterId";
58 | public static final String HEADER_COUNT = "count";
59 | public static final String HEADER_DEFINITION_PARAMETERS = "definitionParameters";
60 |
61 | /**
62 | * Instantiates a new app deployer state machine.
63 | *
64 | * @param yarnCloudAppService the yarn cloud app service
65 | * @param taskExecutor the task executor
66 | * @param beanFactory the bean factory
67 | * @param resourceLoader the resource loader
68 | */
69 | public AppDeployerStateMachine(YarnCloudAppService yarnCloudAppService, TaskExecutor taskExecutor, BeanFactory beanFactory,
70 | ResourceLoader resourceLoader) {
71 | super(yarnCloudAppService, taskExecutor, beanFactory, resourceLoader, MODEL_LOCATION);
72 | }
73 |
74 | @Override
75 | protected Map> getRegisteredActions() {
76 | HashMap> actions = new HashMap<>();
77 | actions.put("resetVariablesAction", new ResetVariablesAction());
78 | actions.put("deployAction", new DeployAction());
79 | actions.put("checkAppAction", new CheckAppAction(CloudAppType.STREAM));
80 | actions.put("pushAppAction", new PushAppAction(CloudAppType.STREAM));
81 | actions.put("checkInstanceAction", new CheckInstanceAction());
82 | actions.put("pushArtifactAction", new PushArtifactAction());
83 | actions.put("startInstanceAction", new StartInstanceAction());
84 | actions.put("waitInstanceAction", new WaitInstanceAction());
85 | actions.put("resolveInstanceAction", new ResolveInstanceAction());
86 | actions.put("createClusterAction", new CreateClusterAction());
87 | actions.put("startClusterAction", new StartClusterAction());
88 | actions.put("stopClusterAction", new StopClusterAction());
89 | actions.put("destroyClusterAction", new DestroyClusterAction());
90 | actions.put("errorHandlingAction", new ErrorAction());
91 | return actions;
92 | }
93 |
94 | @Override
95 | protected Map> getRegisteredGuards() {
96 | HashMap> guards = new HashMap<>();
97 | guards.put("pushAppGuard", new PushAppGuard());
98 | guards.put("startInstanceGuard", new StartInstanceGuard());
99 | guards.put("errorGuard", new ErrorGuard());
100 | guards.put("instanceGuard", new InstanceGuard());
101 | return guards;
102 | }
103 |
104 | /**
105 | * {@link Action} which queries {@link YarnCloudAppService} for existing
106 | * running instances.
107 | */
108 | private class CheckInstanceAction implements Action {
109 |
110 | @Override
111 | public void execute(StateContext context) {
112 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION);
113 | String groupId = (String) context.getMessageHeader(HEADER_GROUP_ID);
114 | String appName = "scdstream:" + appVersion + ":" + groupId;
115 | CloudAppInstanceInfo appInstanceInfo = findRunningInstance(appName);
116 | if (appInstanceInfo != null) {
117 | context.getExtendedState().getVariables().put(VAR_APPLICATION_ID, appInstanceInfo.getApplicationId());
118 | }
119 | }
120 | }
121 |
122 | private CloudAppInstanceInfo findRunningInstance(String appName) {
123 | for (CloudAppInstanceInfo appInstanceInfo : getYarnCloudAppService().getInstances(CloudAppType.STREAM)) {
124 | logger.info("Checking instance {} for appName {}", appInstanceInfo, appName);
125 | if (appInstanceInfo.getName().equals(appName) && appInstanceInfo.getState().equals("RUNNING")
126 | && appInstanceInfo.getAddress().contains("http")) {
127 | logger.info("Using instance {}", appInstanceInfo);
128 | return appInstanceInfo;
129 | }
130 | }
131 | return null;
132 | }
133 |
134 | private class StartInstanceAction implements Action {
135 |
136 | @Override
137 | public void execute(StateContext context) {
138 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION);
139 |
140 | // we control type so casting is safe
141 | @SuppressWarnings("unchecked")
142 | List contextRunArgs = (List) context.getMessageHeader(HEADER_CONTEXT_RUN_ARGS);
143 | String applicationId = getYarnCloudAppService().submitApplication(appVersion, CloudAppType.STREAM, contextRunArgs);
144 | context.getExtendedState().getVariables().put(VAR_APPLICATION_ID, applicationId);
145 | }
146 | }
147 |
148 | private class WaitInstanceAction implements Action {
149 |
150 | @Override
151 | public void execute(StateContext context) {
152 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION);
153 | String groupId = (String) context.getMessageHeader(HEADER_GROUP_ID);
154 | if (StringUtils.hasText(appVersion) && StringUtils.hasText(groupId)) {
155 | String appName = "scdstream:" + appVersion + ":" + groupId;
156 | context.getExtendedState().getVariables().put(VAR_APPNAME, appName);
157 | }
158 | }
159 | }
160 |
161 | private class ResolveInstanceAction implements Action {
162 |
163 | @Override
164 | public void execute(StateContext context) {
165 | String appName = context.getExtendedState().get(VAR_APPNAME, String.class);
166 | String applicationId = context.getExtendedState().get(VAR_APPLICATION_ID, String.class);
167 | CloudAppInstanceInfo appInstanceInfo = findRunningInstance(appName);
168 | if (appInstanceInfo != null && appInstanceInfo.getApplicationId().equals(applicationId)) {
169 | context.getExtendedState().getVariables().put(VAR_INSTANCE_ADDRESS, appInstanceInfo.getAddress());
170 | }
171 | }
172 | }
173 |
174 | private class CreateClusterAction implements Action {
175 |
176 | @SuppressWarnings("unchecked")
177 | @Override
178 | public void execute(StateContext context) {
179 | Resource artifact = context.getExtendedState().get(VAR_ARTIFACT, Resource.class);
180 | getYarnCloudAppService().createCluster(
181 | context.getExtendedState().get(VAR_APPLICATION_ID, String.class),
182 | context.getExtendedState().get(HEADER_CLUSTER_ID, String.class),
183 | context.getExtendedState().get(VAR_COUNT, Integer.class),
184 | artifact != null ? artifact.getFilename() : null,
185 | context.getExtendedState().get(HEADER_DEFINITION_PARAMETERS, Map.class));
186 | }
187 | }
188 |
189 | private class StartClusterAction implements Action {
190 |
191 | @Override
192 | public void execute(StateContext context) {
193 | getYarnCloudAppService().startCluster(context.getExtendedState().get(VAR_APPLICATION_ID, String.class), context
194 | .getExtendedState().get(HEADER_CLUSTER_ID, String.class));
195 | }
196 | }
197 |
198 | private class DeployAction implements Action {
199 |
200 | @Override
201 | public void execute(StateContext context) {
202 | Integer count = context.getMessageHeaders().get(HEADER_COUNT, Integer.class);
203 | String clusterId = context.getMessageHeaders().get(HEADER_CLUSTER_ID, String.class);
204 | Map, ?> definitionParameters = context.getMessageHeaders().get(HEADER_DEFINITION_PARAMETERS, Map.class);
205 | Resource artifact = context.getMessageHeaders().get(HEADER_ARTIFACT, Resource.class);
206 | context.getExtendedState().getVariables().put(VAR_COUNT, count != null ? count : 1);
207 | context.getExtendedState().getVariables().put(VAR_CLUSTER_ID, clusterId);
208 | context.getExtendedState().getVariables().put(VAR_DEFINITION_PARAMETERS, definitionParameters);
209 | if (artifact != null) {
210 | context.getExtendedState().getVariables().put(VAR_ARTIFACT, artifact);
211 | }
212 | context.getExtendedState().getVariables().put(VAR_MESSAGE_ID, context.getMessageHeaders().getId().toString());
213 | }
214 | }
215 |
216 | private class StartInstanceGuard implements Guard {
217 |
218 | @Override
219 | public boolean evaluate(StateContext context) {
220 | return !context.getExtendedState().getVariables().containsKey(VAR_APPLICATION_ID);
221 | }
222 | }
223 |
224 | private class StopClusterAction implements Action {
225 |
226 | @Override
227 | public void execute(StateContext context) {
228 | String clusterId = context.getMessageHeaders().get(HEADER_CLUSTER_ID, String.class);
229 | String applicationId = (String) context.getMessageHeader(HEADER_APPLICATION_ID);
230 | getYarnCloudAppService().stopCluster(applicationId, clusterId);
231 | }
232 | }
233 |
234 | private class DestroyClusterAction implements Action {
235 |
236 | @Override
237 | public void execute(StateContext context) {
238 | String clusterId = context.getMessageHeaders().get(HEADER_CLUSTER_ID, String.class);
239 | String applicationId = (String) context.getMessageHeader(HEADER_APPLICATION_ID);
240 | getYarnCloudAppService().destroyCluster(applicationId, clusterId);
241 | }
242 | }
243 |
244 | private class InstanceGuard implements Guard {
245 |
246 | @Override
247 | public boolean evaluate(StateContext context) {
248 | return context.getExtendedState().getVariables().containsKey(VAR_INSTANCE_ADDRESS);
249 | }
250 | }
251 | }
252 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/DefaultYarnCloudAppService.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2017 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package org.springframework.cloud.deployer.spi.yarn;
17 |
18 | import java.io.File;
19 | import java.io.FileOutputStream;
20 | import java.util.ArrayList;
21 | import java.util.Collection;
22 | import java.util.HashMap;
23 | import java.util.List;
24 | import java.util.Map;
25 | import java.util.Properties;
26 | import java.util.UUID;
27 |
28 | import org.apache.hadoop.conf.Configuration;
29 | import org.apache.hadoop.yarn.util.ConverterUtils;
30 | import org.slf4j.Logger;
31 | import org.slf4j.LoggerFactory;
32 | import org.springframework.beans.factory.InitializingBean;
33 | import org.springframework.beans.factory.annotation.Autowired;
34 | import org.springframework.context.ApplicationContextInitializer;
35 | import org.springframework.core.io.Resource;
36 | import org.springframework.data.hadoop.fs.FsShell;
37 | import org.springframework.util.FileCopyUtils;
38 | import org.springframework.util.ObjectUtils;
39 | import org.springframework.util.StringUtils;
40 |
41 | /**
42 | * Default implementation of {@link YarnCloudAppService} which talks to rest
43 | * api's exposed by specific yarn controlling container clusters.
44 | *
45 | * @author Janne Valkealahti
46 | * @author Mark Fisher
47 | */
48 | public class DefaultYarnCloudAppService implements YarnCloudAppService, InitializingBean {
49 |
50 | private static final Logger logger = LoggerFactory.getLogger(DefaultYarnCloudAppService.class);
51 | private final ApplicationContextInitializer>[] initializers;
52 | private final String deployerVersion;
53 | private final Map appCache = new HashMap();
54 |
55 | private Configuration configuration;
56 |
57 | /**
58 | * Instantiates a new default yarn cloud app service.
59 | *
60 | * @param deployerVersion the deployer version
61 | */
62 | public DefaultYarnCloudAppService(String deployerVersion) {
63 | this(deployerVersion, null);
64 | }
65 |
66 | /**
67 | * Instantiates a new default yarn cloud app service.
68 | *
69 | * @param deployerVersion the deployer version
70 | * @param initializers the initializers
71 | */
72 | public DefaultYarnCloudAppService(String deployerVersion, ApplicationContextInitializer>[] initializers) {
73 | this.deployerVersion = deployerVersion;
74 | this.initializers = initializers;
75 | }
76 |
77 | @Autowired
78 | public void setConfiguration(Configuration configuration) {
79 | this.configuration = configuration;
80 | }
81 |
82 | @Override
83 | public void pushArtifact(Resource artifact, String dir) {
84 | File tmp = null;
85 | try {
86 | tmp = File.createTempFile(UUID.randomUUID().toString(), null);
87 | tmp.deleteOnExit();
88 | FileCopyUtils.copy(artifact.getInputStream(), new FileOutputStream(tmp));
89 | @SuppressWarnings("resource")
90 | FsShell shell = new FsShell(configuration);
91 | String artifactPath = dir + "/" + artifact.getFile().getName();
92 | if (!shell.test(artifactPath)) {
93 | logger.info("Pushing artifact {} into dir {}", artifact, dir);
94 | shell.copyFromLocal(tmp.getAbsolutePath(), artifactPath);
95 | }
96 | } catch (Exception e) {
97 | logger.error("Error pushing artifact", e);
98 | } finally {
99 | if (tmp != null) {
100 | try {
101 | tmp.delete();
102 | } catch (Exception e) {
103 | }
104 | }
105 | }
106 | }
107 |
108 | @Override
109 | public void afterPropertiesSet() throws Exception {
110 | }
111 |
112 | @Override
113 | public Collection getApplications(CloudAppType cloudAppType) {
114 | return getApp(null, null, cloudAppType).getPushedApplications();
115 | }
116 |
117 | @Override
118 | public Collection getInstances(CloudAppType cloudAppType) {
119 | return getApp(null, null, cloudAppType).getSubmittedApplications();
120 | }
121 |
122 | @Override
123 | public void pushApplication(String appVersion, CloudAppType cloudAppType) {
124 | getApp(appVersion, deployerVersion, cloudAppType, null).pushApplication(appVersion);
125 | }
126 |
127 | @Override
128 | public String submitApplication(String appVersion, CloudAppType cloudAppType) {
129 | return submitApplication(appVersion, cloudAppType, null);
130 | }
131 |
132 | @Override
133 | public String submitApplication(String appVersion, CloudAppType cloudAppType, List contextRunArgs) {
134 | return getApp(appVersion, deployerVersion, cloudAppType, contextRunArgs).submitApplication(appVersion);
135 | }
136 |
137 | @Override
138 | public void killApplications(String appName, CloudAppType cloudAppType) {
139 | YarnCloudAppServiceApplication app = getApp(null, null, cloudAppType);
140 | Collection submittedApplications = app.getSubmittedApplications();
141 | for (CloudAppInstanceInfo info : submittedApplications) {
142 | if (info.getName() == appName) {
143 | app.killApplication(info.getApplicationId());
144 | }
145 | }
146 | }
147 |
148 | @Override
149 | public void killApplication(String yarnApplicationId, CloudAppType cloudAppType) {
150 | YarnCloudAppServiceApplication app = getApp(null, null, cloudAppType);
151 | app.killApplication(yarnApplicationId);
152 | }
153 |
154 | @Override
155 | public void createCluster(String yarnApplicationId, String clusterId, int count, String artifact,
156 | Map definitionParameters) {
157 |
158 | Map extraProperties = new HashMap();
159 | extraProperties.put("containerArtifact", artifact);
160 |
161 | int i = 0;
162 | for (Map.Entry entry : definitionParameters.entrySet()) {
163 | String value = entry.getValue();
164 | if (value.startsWith("\"") && value.endsWith("\"")) {
165 | // prevent ${xxx.ddd} bad substitution with bash
166 | value = value.replace("$", "\\\\\\$");
167 | // escape existing double quotes
168 | extraProperties.put("containerArg" + i++, entry.getKey() + "=\\" + value.substring(0, value.length()-1) + "\\\"");
169 | } else {
170 | // prevent ${xxx.ddd} bad substitution with bash
171 | value = value.replace("$", "\\\\\\$");
172 | // escape with extra double quotes
173 | extraProperties.put("containerArg" + i++, entry.getKey() + "=\\\"" + value + "\\\"");
174 | }
175 | }
176 | getApp(null, null, CloudAppType.STREAM).createCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId, "module-template",
177 | "default", count, null, null, null, extraProperties);
178 | }
179 |
180 | @Override
181 | public void startCluster(String yarnApplicationId, String clusterId) {
182 | getApp(null, null, CloudAppType.STREAM).startCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId);
183 | }
184 |
185 | @Override
186 | public void stopCluster(String yarnApplicationId, String clusterId) {
187 | getApp(null, null, CloudAppType.STREAM).stopCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId);
188 | }
189 |
190 | @Override
191 | public Map getClustersStates(String yarnApplicationId) {
192 | HashMap states = new HashMap();
193 | Collection submittedApplications = getApp(null, null, CloudAppType.STREAM).getSubmittedApplications(yarnApplicationId);
194 | for (CloudAppInstanceInfo instanceInfo : submittedApplications) {
195 | if (instanceInfo.getName().startsWith("scdstream:") && instanceInfo.getState().equals("RUNNING")) {
196 | for (String cluster : getClusters(instanceInfo.getApplicationId())) {
197 | states.putAll(getInstanceClustersStates(instanceInfo.getApplicationId(), cluster));
198 | }
199 | }
200 | }
201 | return states;
202 | }
203 |
204 | @Override
205 | public Collection getClusters(String yarnApplicationId) {
206 | return getApp(null, null, CloudAppType.STREAM).getClustersInfo(ConverterUtils.toApplicationId(yarnApplicationId));
207 | }
208 |
209 | @Override
210 | public void destroyCluster(String yarnApplicationId, String clusterId) {
211 | getApp(null, null, CloudAppType.STREAM).destroyCluster(ConverterUtils.toApplicationId(yarnApplicationId), clusterId);
212 | }
213 |
214 | private Map getInstanceClustersStates(String yarnApplicationId, String clusterId) {
215 | HashMap states = new HashMap();
216 | List clusterInfo = getApp(null, null, CloudAppType.STREAM)
217 | .getClusterInfo(ConverterUtils.toApplicationId(yarnApplicationId), clusterId);
218 | if (clusterInfo.size() == 1) {
219 | states.put(clusterId, clusterInfo.get(0));
220 | }
221 | return states;
222 | }
223 |
224 | protected List processContextRunArgs(List contextRunArgs) {
225 | return contextRunArgs;
226 | }
227 |
228 | private synchronized YarnCloudAppServiceApplication getApp(String appVersion, String dataflowVersion, CloudAppType cloudAppType) {
229 | return getApp(appVersion, dataflowVersion, cloudAppType, null);
230 | }
231 |
232 | private synchronized YarnCloudAppServiceApplication getApp(String appVersion, String dataflowVersion, CloudAppType cloudAppType,
233 | List contextRunArgs) {
234 | contextRunArgs = processContextRunArgs(contextRunArgs);
235 | String cacheKey = cloudAppType + appVersion + StringUtils.collectionToCommaDelimitedString(contextRunArgs);
236 | YarnCloudAppServiceApplication app = appCache.get(cacheKey);
237 | logger.info("Cachekey {} found YarnCloudAppServiceApplication {}", cacheKey, app);
238 | if (app == null) {
239 | Properties configFileProperties = new Properties();
240 | if (StringUtils.hasText(appVersion)) {
241 | configFileProperties.setProperty("spring.yarn.applicationVersion", appVersion);
242 | }
243 | if (StringUtils.hasText(dataflowVersion)) {
244 | configFileProperties.setProperty("spring.cloud.deployer.yarn.version", dataflowVersion);
245 | }
246 |
247 | logger.info("Bootsrapping YarnCloudAppServiceApplication with {}", cloudAppType.toString().toLowerCase());
248 | ArrayList runArgs = new ArrayList();
249 | runArgs.add("--spring.config.name=" + cloudAppType.toString().toLowerCase());
250 | runArgs.add("--spring.jmx.enabled=false");
251 |
252 | if (!ObjectUtils.isEmpty(contextRunArgs)) {
253 | runArgs.addAll(contextRunArgs);
254 | }
255 |
256 | app = new YarnCloudAppServiceApplication(appVersion, dataflowVersion, "application.properties", configFileProperties,
257 | runArgs.toArray(new String[0]), initializers);
258 | try {
259 | app.afterPropertiesSet();
260 | } catch (Exception e) {
261 | throw new RuntimeException("Error initializing YarnCloudAppServiceApplication", e);
262 | }
263 | logger.info("Set cache with key {} and YarnCloudAppServiceApplication {}", cacheKey, app);
264 | appCache.put(cacheKey, app);
265 | }
266 | return app;
267 | }
268 |
269 | }
270 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/DeployerApplicationYarnClient.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import org.apache.hadoop.yarn.api.records.ApplicationId;
20 | import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
21 | import org.springframework.yarn.client.ClientRmOperations;
22 | import org.springframework.yarn.client.DefaultApplicationYarnClient;
23 | import org.springframework.yarn.client.YarnClient;
24 |
25 | /**
26 | * Custom {@link YarnClient} handling deployer specific client operations.
27 | *
28 | * @author Janne Valkealahti
29 | *
30 | */
31 | public class DeployerApplicationYarnClient extends DefaultApplicationYarnClient {
32 |
33 | /**
34 | * Instantiates a new deployer application yarn client.
35 | *
36 | * @param clientRmOperations the client rm operations
37 | */
38 | public DeployerApplicationYarnClient(ClientRmOperations clientRmOperations) {
39 | super(clientRmOperations);
40 | }
41 |
42 | @Override
43 | protected ApplicationSubmissionContext getSubmissionContext(ApplicationId applicationId) {
44 | ApplicationSubmissionContext submissionContext = super.getSubmissionContext(applicationId);
45 | // force app fail after first appmaster failure regardles
46 | // what's set on yarn level.
47 | // TODO: should make configurable in spring yarn if possible
48 | submissionContext.setMaxAppAttempts(1);
49 | return submissionContext;
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/StreamClustersInfoReportData.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2017 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package org.springframework.cloud.deployer.spi.yarn;
17 |
18 | import java.util.List;
19 | import java.util.Map;
20 |
21 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData;
22 |
23 | public class StreamClustersInfoReportData extends ClustersInfoReportData {
24 |
25 | private final List members;
26 |
27 | public StreamClustersInfoReportData(String state, Integer count, Integer pany, Map phosts, Map pracks,
28 | Integer sany, Map shosts, Map sracks, List members) {
29 | super(state, count, pany, phosts, pracks, sany, shosts, sracks);
30 | this.members = members;
31 | }
32 |
33 | public List getMembers() {
34 | return members;
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/TaskLauncherStateMachine.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.util.HashMap;
20 | import java.util.List;
21 | import java.util.Map;
22 |
23 | import org.slf4j.Logger;
24 | import org.slf4j.LoggerFactory;
25 | import org.springframework.beans.factory.BeanFactory;
26 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType;
27 | import org.springframework.core.io.ResourceLoader;
28 | import org.springframework.core.task.TaskExecutor;
29 | import org.springframework.statemachine.StateContext;
30 | import org.springframework.statemachine.action.Action;
31 | import org.springframework.statemachine.guard.Guard;
32 |
33 | /**
34 | * Statemachine for task launching.
35 | *
36 | * @author Janne Valkealahti
37 | *
38 | */
39 | public class TaskLauncherStateMachine extends AbstractDeployerStateMachine {
40 |
41 | private static final Logger logger = LoggerFactory.getLogger(TaskLauncherStateMachine.class);
42 | private final static String MODEL_LOCATION = "classpath:tasklauncher-model.uml";
43 |
44 | public final static String EVENT_LAUNCH = "LAUNCH";
45 | public final static String EVENT_CANCEL = "CANCEL";
46 |
47 | /**
48 | * Instantiates a new task launcher state machine.
49 | *
50 | * @param yarnCloudAppService the yarn cloud app service
51 | * @param taskExecutor the task executor
52 | * @param beanFactory the bean factory
53 | * @param resourceLoader the resource loader
54 | */
55 | public TaskLauncherStateMachine(YarnCloudAppService yarnCloudAppService, TaskExecutor taskExecutor, BeanFactory beanFactory,
56 | ResourceLoader resourceLoader) {
57 | super(yarnCloudAppService, taskExecutor, beanFactory, resourceLoader, MODEL_LOCATION);
58 | }
59 |
60 | @Override
61 | protected Map> getRegisteredActions() {
62 | HashMap> actions = new HashMap<>();
63 | actions.put("resetVariablesAction", new ResetVariablesAction());
64 | actions.put("launchAction", new LaunchAction());
65 | actions.put("cancelAction", new CancelAction());
66 | actions.put("checkAppAction", new CheckAppAction(CloudAppType.TASK));
67 | actions.put("pushAppAction", new PushAppAction(CloudAppType.TASK));
68 | actions.put("pushArtifactAction", new PushArtifactAction());
69 | actions.put("startAppAction", new StartAppAction());
70 | actions.put("stopAppAction", new StopAppAction());
71 | actions.put("errorHandlingAction", new ErrorAction());
72 | return actions;
73 | }
74 |
75 | @Override
76 | protected Map> getRegisteredGuards() {
77 | HashMap> guards = new HashMap<>();
78 | guards.put("pushAppGuard", new PushAppGuard());
79 | guards.put("errorGuard", new ErrorGuard());
80 | return guards;
81 | }
82 |
83 | private class LaunchAction implements Action {
84 |
85 | @Override
86 | public void execute(StateContext context) {
87 | context.getExtendedState().getVariables().put(VAR_MESSAGE_ID, context.getMessageHeaders().getId().toString());
88 | }
89 | }
90 |
91 | private class CancelAction implements Action {
92 |
93 | @Override
94 | public void execute(StateContext context) {
95 | context.getExtendedState().getVariables().put(VAR_MESSAGE_ID, context.getMessageHeaders().getId().toString());
96 | }
97 | }
98 |
99 | private class StartAppAction implements Action {
100 |
101 | @Override
102 | public void execute(StateContext context) {
103 | String appVersion = (String) context.getMessageHeader(HEADER_APP_VERSION);
104 |
105 | // we control type so casting is safe
106 | @SuppressWarnings("unchecked")
107 | List contextRunArgs = (List) context.getMessageHeader(HEADER_CONTEXT_RUN_ARGS);
108 |
109 | String applicationId = getYarnCloudAppService().submitApplication(appVersion, CloudAppType.TASK, contextRunArgs);
110 | logger.info("New application id is {}", applicationId);
111 | context.getExtendedState().getVariables().put(VAR_APPLICATION_ID, applicationId);
112 | }
113 | }
114 |
115 | private class StopAppAction implements Action {
116 |
117 | @Override
118 | public void execute(StateContext context) {
119 | String applicationId = (String) context.getMessageHeader(HEADER_APPLICATION_ID);
120 | logger.info("Killing application {}", applicationId);
121 | getYarnCloudAppService().killApplication(applicationId, CloudAppType.TASK);
122 | }
123 | }
124 | }
125 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnAppDeployer.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2017 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.io.IOException;
20 | import java.util.ArrayList;
21 | import java.util.HashMap;
22 | import java.util.List;
23 | import java.util.Map;
24 | import java.util.Map.Entry;
25 | import java.util.concurrent.TimeUnit;
26 |
27 | import org.apache.commons.io.FilenameUtils;
28 | import org.slf4j.Logger;
29 | import org.slf4j.LoggerFactory;
30 | import org.springframework.beans.factory.annotation.Autowired;
31 | import org.springframework.cloud.deployer.spi.app.AppDeployer;
32 | import org.springframework.cloud.deployer.spi.app.AppInstanceStatus;
33 | import org.springframework.cloud.deployer.spi.app.AppStatus;
34 | import org.springframework.cloud.deployer.spi.app.AppStatus.Builder;
35 | import org.springframework.cloud.deployer.spi.app.DeploymentState;
36 | import org.springframework.cloud.deployer.spi.core.AppDefinition;
37 | import org.springframework.cloud.deployer.spi.core.AppDeploymentRequest;
38 | import org.springframework.cloud.deployer.spi.core.RuntimeEnvironmentInfo;
39 | import org.springframework.cloud.deployer.spi.util.RuntimeVersionUtils;
40 | import org.springframework.core.io.Resource;
41 | import org.springframework.messaging.Message;
42 | import org.springframework.messaging.support.MessageBuilder;
43 | import org.springframework.statemachine.StateContext;
44 | import org.springframework.statemachine.StateMachine;
45 | import org.springframework.statemachine.StateContext.Stage;
46 | import org.springframework.statemachine.listener.StateMachineListener;
47 | import org.springframework.statemachine.listener.StateMachineListenerAdapter;
48 | import org.springframework.util.Assert;
49 | import org.springframework.util.ObjectUtils;
50 | import org.springframework.util.StringUtils;
51 | import org.springframework.util.concurrent.ListenableFutureCallback;
52 | import org.springframework.util.concurrent.SettableListenableFuture;
53 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData;
54 |
55 | /**
56 | * {@link ModuleDeployer} which communicates with a Yarn app running
57 | * on a Hadoop cluster waiting for deployment requests. This app
58 | * uses Spring Yarn's container grouping functionality to create a
59 | * new group per module type. This allows all modules to share the
60 | * same settings and the group itself can controlled, i.e. ramp up/down
61 | * or shutdown/destroy a whole group.
62 | *
63 | * @author Janne Valkealahti
64 | */
65 | public class YarnAppDeployer implements AppDeployer {
66 |
67 | private static final Logger logger = LoggerFactory.getLogger(YarnAppDeployer.class);
68 | private final YarnCloudAppService yarnCloudAppService;
69 | private final StateMachine stateMachine;
70 |
71 | @Autowired
72 | private YarnDeployerProperties yarnDeployerProperties;
73 |
74 | /**
75 | * Instantiates a new yarn stream module deployer.
76 | *
77 | * @param yarnCloudAppService the yarn cloud app service
78 | * @param stateMachine the state machine
79 | */
80 | public YarnAppDeployer(YarnCloudAppService yarnCloudAppService, StateMachine stateMachine) {
81 | this.yarnCloudAppService = yarnCloudAppService;
82 | this.stateMachine = stateMachine;
83 | }
84 |
85 | @Override
86 | public String deploy(AppDeploymentRequest request) {
87 | logger.info("Deploy request for {}", request);
88 | final AppDefinition definition = request.getDefinition();
89 | Map definitionParameters = definition.getProperties();
90 | Map deploymentProperties = request.getDeploymentProperties();
91 | logger.info("Deploying request for definition {}", definition);
92 | logger.info("Parameters for definition {}", definitionParameters);
93 | logger.info("Deployment properties for request {}", deploymentProperties);
94 |
95 | int count = 1;
96 | String countString = request.getDeploymentProperties().get(AppDeployer.COUNT_PROPERTY_KEY);
97 | if (StringUtils.hasText(countString)) {
98 | count = Integer.parseInt(countString);
99 | }
100 | final String group = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY);
101 | Resource resource = request.getResource();
102 | final String clusterId = group + ":" + definition.getName();
103 |
104 | String appVersion = StringUtils.hasText(yarnDeployerProperties.getAppVersion()) ? yarnDeployerProperties.getAppVersion() : "app";
105 | if (deploymentProperties.containsKey("spring.cloud.deployer.yarn.app.appVersion")) {
106 | appVersion = deploymentProperties.get("spring.cloud.deployer.yarn.app.appVersion");
107 | }
108 | // contextRunArgs are passed to boot app ran to control yarn apps
109 | ArrayList contextRunArgs = new ArrayList();
110 | contextRunArgs.add("--spring.yarn.appName=scdstream:" + appVersion + ":" + group);
111 |
112 | // deployment properties override servers.yml which overrides application.yml
113 | for (Entry entry : deploymentProperties.entrySet()) {
114 | if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.streamappmaster")) {
115 | contextRunArgs.add("--" + entry.getKey() + "=" + entry.getValue());
116 | } else if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.streamcontainer")) {
117 | // weird format with '--' is just straight pass to appmaster
118 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--" + entry.getKey() + "='" + entry.getValue() + "'");
119 | }
120 | }
121 |
122 | String baseDir = yarnDeployerProperties.getBaseDir();
123 | if (!baseDir.endsWith("/")) {
124 | baseDir = baseDir + "/";
125 | }
126 |
127 | String artifactPath = isHdfsResource(resource) ? getHdfsArtifactPath(resource) : baseDir + "/artifacts/cache/";
128 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.artifact=" + artifactPath);
129 |
130 | // add group as 'spring.cloud.application.group'
131 | definitionParameters = new HashMap<>(definitionParameters);
132 | definitionParameters.put("spring.cloud.application.group", group);
133 |
134 | final Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_DEPLOY)
135 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, appVersion)
136 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, clusterId)
137 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, group)
138 | .setHeader(AppDeployerStateMachine.HEADER_COUNT, count)
139 | .setHeader(AppDeployerStateMachine.HEADER_ARTIFACT, resource)
140 | .setHeader(AppDeployerStateMachine.HEADER_ARTIFACT_DIR, artifactPath)
141 | .setHeader(AppDeployerStateMachine.HEADER_DEFINITION_PARAMETERS, definitionParameters)
142 | .setHeader(AppDeployerStateMachine.HEADER_CONTEXT_RUN_ARGS, contextRunArgs)
143 | .build();
144 |
145 | // Use of future here is to set id when it becomes available from machine
146 | final SettableListenableFuture id = new SettableListenableFuture<>();
147 | final StateMachineListener listener = new StateMachineListenerAdapter() {
148 |
149 | @Override
150 | public void stateContext(StateContext stateContext) {
151 | if (stateContext.getStage() == Stage.STATE_ENTRY && stateContext.getTarget().getId().equals(AppDeployerStateMachine.STATE_READY)) {
152 | if (ObjectUtils.nullSafeEquals(message.getHeaders().getId().toString(),
153 | stateContext.getExtendedState().get(AppDeployerStateMachine.VAR_MESSAGE_ID, String.class))) {
154 | Exception exception = stateContext.getExtendedState().get(AppDeployerStateMachine.VAR_ERROR, Exception.class);
155 | if (exception != null) {
156 | id.setException(exception);
157 | } else {
158 | String applicationId = stateContext.getStateMachine().getExtendedState()
159 | .get(AppDeployerStateMachine.VAR_APPLICATION_ID, String.class);
160 | DeploymentKey key = new DeploymentKey(group, definition.getName(), applicationId);
161 | id.set(key.toString());
162 | }
163 | }
164 | }
165 | }
166 | };
167 | stateMachine.addStateListener(listener);
168 | id.addCallback(new ListenableFutureCallback() {
169 |
170 | @Override
171 | public void onSuccess(String result) {
172 | stateMachine.removeStateListener(listener);
173 | }
174 |
175 | @Override
176 | public void onFailure(Throwable ex) {
177 | stateMachine.removeStateListener(listener);
178 | }
179 | });
180 |
181 | stateMachine.sendEvent(message);
182 | // we need to block here until SPI supports
183 | // returning id asynchronously
184 | try {
185 | return id.get(2, TimeUnit.MINUTES);
186 | } catch (Exception e) {
187 | throw new RuntimeException(e);
188 | }
189 | }
190 |
191 | @Override
192 | public void undeploy(String id) {
193 | logger.info("Undeploy request for id {}", id);
194 | DeploymentKey key = new DeploymentKey(id);
195 | Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_UNDEPLOY)
196 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, key.getClusterId())
197 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, "app")
198 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, key.group)
199 | .setHeader(AppDeployerStateMachine.HEADER_APPLICATION_ID, key.applicationId)
200 | .build();
201 | stateMachine.sendEvent(message);
202 | }
203 |
204 | @Override
205 | public AppStatus status(String id) {
206 | logger.info("Checking status of {}", id);
207 | DeploymentKey key = new DeploymentKey(id);
208 | Builder builder = AppStatus.of(id);
209 | for (Entry entry : yarnCloudAppService.getClustersStates(key.applicationId).entrySet()) {
210 | if (ObjectUtils.nullSafeEquals(entry.getKey(), key.getClusterId())) {
211 | ClustersInfoReportData data = entry.getValue();
212 | List members = entry.getValue().getMembers();
213 | for (int i = 0 ; i < data.getProjectionAny(); i++) {
214 | Map attributes = new HashMap<>();
215 | if (i < members.size()) {
216 | attributes.put("container", members.get(i));
217 | attributes.put("guid", members.get(i));
218 | }
219 | InstanceStatus instanceStatus = new InstanceStatus(key.getClusterId() + "-" + i, i < data.getCount(), attributes);
220 | builder.with(instanceStatus);
221 | }
222 | break;
223 | }
224 | }
225 | return builder.build();
226 | }
227 |
228 | @Override
229 | public RuntimeEnvironmentInfo environmentInfo() {
230 | return new RuntimeEnvironmentInfo.Builder()
231 | .spiClass(AppDeployer.class)
232 | .implementationName(getClass().getSimpleName())
233 | .implementationVersion(RuntimeVersionUtils.getVersion(this.getClass()))
234 | .platformType("Yarn")
235 | .platformApiVersion(System.getProperty("os.name") + " " + System.getProperty("os.version"))
236 | .platformClientVersion(System.getProperty("os.version"))
237 | .platformHostVersion(System.getProperty("os.version"))
238 | .build();
239 | }
240 |
241 | private boolean isHdfsResource(Resource resource) {
242 | try {
243 | return resource != null && resource.getURI().getScheme().equals("hdfs");
244 | } catch (IOException e) {
245 | return false;
246 | }
247 | }
248 |
249 | private String getHdfsArtifactPath(Resource resource) {
250 | String path = null;
251 | try {
252 | path = "/" + FilenameUtils.getPath(resource.getURI().getPath());
253 | } catch (IOException e) {
254 | }
255 | return path;
256 | }
257 |
258 | private static class DeploymentKey {
259 | final static String SEPARATOR = ":";
260 | final String group;
261 | final String name;
262 | final String applicationId;
263 | final String clusterId;
264 |
265 | public DeploymentKey(String id) {
266 | String[] split = id.split(SEPARATOR);
267 | Assert.isTrue(split.length == 3, "Unable to parse deployment key " + id);
268 | group = split[0];
269 | name = split[1];
270 | applicationId = split[2];
271 | clusterId = group + SEPARATOR + name;
272 | }
273 |
274 | public DeploymentKey(String group, String name, String applicationId) {
275 | Assert.notNull(group, "Group must be set");
276 | Assert.notNull(name, "Name must be set");
277 | Assert.notNull(applicationId, "Application id must be set");
278 | this.group = group;
279 | this.name = name;
280 | this.applicationId = applicationId;
281 | clusterId = group + SEPARATOR + name;
282 | }
283 |
284 | public String getClusterId() {
285 | return clusterId;
286 | }
287 |
288 | @Override
289 | public String toString() {
290 | return group + SEPARATOR + name + SEPARATOR + applicationId;
291 | }
292 | }
293 |
294 | private static class InstanceStatus implements AppInstanceStatus {
295 | private final String id;
296 | private final DeploymentState state;
297 | private final Map attributes = new HashMap();
298 |
299 | public InstanceStatus(String id, boolean deployed, Map attributes) {
300 | this.id = id;
301 | this.state = deployed ? DeploymentState.deployed : DeploymentState.unknown;
302 | if (attributes != null) {
303 | this.attributes.putAll(attributes);
304 | }
305 | }
306 |
307 | @Override
308 | public String getId() {
309 | return id;
310 | }
311 |
312 | @Override
313 | public DeploymentState getState() {
314 | return state;
315 | }
316 |
317 | @Override
318 | public Map getAttributes() {
319 | return attributes;
320 | }
321 |
322 | @Override
323 | public String toString() {
324 | return "InstanceStatus [id=" + id + ", state=" + state + ", attributes=" + attributes + "]";
325 | }
326 | }
327 | }
328 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnCloudAppService.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.util.Collection;
20 | import java.util.List;
21 | import java.util.Map;
22 |
23 | import org.springframework.core.io.Resource;
24 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData;
25 |
26 | /**
27 | * Interface used to glue a state machine and yarn application logic together.
28 | *
29 | * @author Janne Valkealahti
30 | */
31 | public interface YarnCloudAppService {
32 |
33 | /**
34 | * Get applications pushed to hdfs.
35 | *
36 | * @return the applications
37 | */
38 | Collection getApplications(CloudAppType cloudAppType);
39 |
40 | /**
41 | * Get running application instances.
42 | *
43 | * @return the instances
44 | */
45 | Collection getInstances(CloudAppType cloudAppType);
46 |
47 | /**
48 | * Push new application into hdfs. Push operation is copying needed
49 | * files into hdfs without trying to start a new application instance.
50 | *
51 | * @param appVersion the app version
52 | * @param cloudAppType the cloud app type
53 | */
54 | void pushApplication(String appVersion, CloudAppType cloudAppType);
55 |
56 | /**
57 | * Submit new application instance into yarn. Prior to calling a new
58 | * submit operation, application has to exist in hdfs which can be done
59 | * i.e. using {@link #pushApplication(String)} method.
60 | *
61 | * @param appVersion the app version
62 | * @param cloudAppType the cloud app type
63 | * @return the application id
64 | */
65 | String submitApplication(String appVersion, CloudAppType cloudAppType);
66 |
67 | /**
68 | * Submit new application instance into yarn. Prior to calling a new
69 | * submit operation, application has to exist in hdfs which can be done
70 | * i.e. using {@link #pushApplication(String)} method.
71 | *
72 | * @param appVersion the app version
73 | * @param cloudAppType the cloud app type
74 | * @param contextRunArgs the context run arguments
75 | * @return the application id
76 | */
77 | String submitApplication(String appVersion, CloudAppType cloudAppType, List contextRunArgs);
78 |
79 | /**
80 | * Kill all applications matching given application name.
81 | *
82 | * @param appName the application name
83 | */
84 | void killApplications(String appName, CloudAppType cloudAppType);
85 |
86 | /**
87 | * Kill application with given yarn application id.
88 | *
89 | * @param yarnApplicationId the yarn application id
90 | * @param cloudAppType the cloud app type
91 | */
92 | void killApplication(String yarnApplicationId, CloudAppType cloudAppType);
93 |
94 | /**
95 | * Creates the container cluster.
96 | *
97 | * @param yarnApplicationId the yarn application id
98 | * @param clusterId the cluster id
99 | * @param count the count
100 | * @param module the module
101 | * @param definitionParameters the definition parameters
102 | */
103 | void createCluster(String yarnApplicationId, String clusterId, int count, String artifact,
104 | Map definitionParameters);
105 |
106 | /**
107 | * Start a container cluster.
108 | *
109 | * @param yarnApplicationId the yarn application id
110 | * @param clusterId the cluster id
111 | */
112 | void startCluster(String yarnApplicationId, String clusterId);
113 |
114 | /**
115 | * Stop a container cluster.
116 | *
117 | * @param yarnApplicationId the yarn application id
118 | * @param clusterId the cluster id
119 | */
120 | void stopCluster(String yarnApplicationId, String clusterId);
121 |
122 | /**
123 | * Gets the clusters states. Returned map has a mapping between
124 | * yarn container cluster id and its state known by application master
125 | * using container clusters.
126 | *
127 | * @param yarnApplicationId the yarn application id
128 | * @return the clusters states
129 | */
130 | Map getClustersStates(String yarnApplicationId);
131 |
132 | /**
133 | * Gets the clusters.
134 | *
135 | * @param yarnApplicationId the yarn application id
136 | * @return the clusters
137 | */
138 | Collection getClusters(String yarnApplicationId);
139 |
140 | /**
141 | * Destroy cluster.
142 | *
143 | * @param yarnApplicationId the yarn application id
144 | * @param clusterId the cluster id
145 | */
146 | void destroyCluster(String yarnApplicationId, String clusterId);
147 |
148 | /**
149 | * Push artifact into a hdfs directory.
150 | *
151 | * @param artifact the artifact
152 | * @param dir the dir
153 | */
154 | void pushArtifact(Resource artifact, String dir);
155 |
156 | /**
157 | * Wrapping info about application pushed into hdfs.
158 | */
159 | public class CloudAppInfo {
160 |
161 | private final String name;
162 |
163 | public CloudAppInfo(String name) {
164 | this.name = name;
165 | }
166 |
167 | public String getName() {
168 | return name;
169 | }
170 |
171 | }
172 |
173 | /**
174 | * Wrapping info about running application.
175 | */
176 | public class CloudAppInstanceInfo {
177 |
178 | private final String applicationId;
179 | private final String name;
180 | private final String state;
181 | private final String address;
182 |
183 | public CloudAppInstanceInfo(String applicationId, String name, String state, String address) {
184 | this.applicationId = applicationId;
185 | this.name = name;
186 | this.state = state;
187 | this.address = address;
188 | }
189 |
190 | public String getApplicationId() {
191 | return applicationId;
192 | }
193 |
194 | public String getName() {
195 | return name;
196 | }
197 |
198 | public String getState() {
199 | return state;
200 | }
201 |
202 | public String getAddress() {
203 | return address;
204 | }
205 |
206 | @Override
207 | public String toString() {
208 | return "CloudAppInstanceInfo [applicationId=" + applicationId + ", name=" + name + ", state=" + state + ", address=" + address
209 | + "]";
210 | }
211 |
212 | }
213 |
214 | public enum CloudAppType {
215 | STREAM,
216 | TASK
217 | }
218 |
219 | }
220 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnDeployerProperties.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import org.springframework.boot.context.properties.ConfigurationProperties;
20 |
21 | @ConfigurationProperties(value = "spring.cloud.deployer.yarn.app")
22 | public class YarnDeployerProperties {
23 |
24 | private String baseDir = "/dataflow";
25 | private String appVersion;
26 |
27 | public String getBaseDir() {
28 | return baseDir;
29 | }
30 |
31 | public void setBaseDir(String baseDir) {
32 | this.baseDir = baseDir;
33 | }
34 |
35 | public String getAppVersion() {
36 | return appVersion;
37 | }
38 |
39 | public void setAppVersion(String appVersion) {
40 | this.appVersion = appVersion;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/java/org/springframework/cloud/deployer/spi/yarn/YarnTaskLauncher.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.io.IOException;
20 | import java.util.ArrayList;
21 | import java.util.Collection;
22 | import java.util.List;
23 | import java.util.Map;
24 | import java.util.Map.Entry;
25 | import java.util.concurrent.TimeUnit;
26 |
27 | import org.apache.commons.io.FilenameUtils;
28 | import org.slf4j.Logger;
29 | import org.slf4j.LoggerFactory;
30 | import org.springframework.beans.factory.annotation.Autowired;
31 | import org.springframework.cloud.deployer.spi.core.AppDefinition;
32 | import org.springframework.cloud.deployer.spi.core.AppDeploymentRequest;
33 | import org.springframework.cloud.deployer.spi.core.RuntimeEnvironmentInfo;
34 | import org.springframework.cloud.deployer.spi.task.LaunchState;
35 | import org.springframework.cloud.deployer.spi.task.TaskLauncher;
36 | import org.springframework.cloud.deployer.spi.task.TaskStatus;
37 | import org.springframework.cloud.deployer.spi.util.RuntimeVersionUtils;
38 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppInstanceInfo;
39 | import org.springframework.cloud.deployer.spi.yarn.YarnCloudAppService.CloudAppType;
40 | import org.springframework.core.io.Resource;
41 | import org.springframework.messaging.Message;
42 | import org.springframework.messaging.support.MessageBuilder;
43 | import org.springframework.statemachine.StateContext;
44 | import org.springframework.statemachine.StateContext.Stage;
45 | import org.springframework.statemachine.StateMachine;
46 | import org.springframework.statemachine.listener.StateMachineListener;
47 | import org.springframework.statemachine.listener.StateMachineListenerAdapter;
48 | import org.springframework.util.Assert;
49 | import org.springframework.util.ObjectUtils;
50 | import org.springframework.util.StringUtils;
51 | import org.springframework.util.concurrent.ListenableFutureCallback;
52 | import org.springframework.util.concurrent.SettableListenableFuture;
53 |
54 | /**
55 | * {@link TaskLauncher} which is launching tasks as Yarn applications.
56 | *
57 | * @author Janne Valkealahti
58 | */
59 | public class YarnTaskLauncher implements TaskLauncher {
60 |
61 | private static final Logger logger = LoggerFactory.getLogger(YarnTaskLauncher.class);
62 | private final YarnCloudAppService yarnCloudAppService;
63 | private final StateMachine stateMachine;
64 |
65 | @Autowired
66 | private YarnDeployerProperties yarnDeployerProperties;
67 |
68 | /**
69 | * Instantiates a new yarn task module deployer.
70 | *
71 | * @param yarnCloudAppService the yarn cloud app service
72 | * @param stateMachine the state machine
73 | */
74 | public YarnTaskLauncher(YarnCloudAppService yarnCloudAppService, StateMachine stateMachine) {
75 | this.yarnCloudAppService = yarnCloudAppService;
76 | this.stateMachine = stateMachine;
77 | }
78 |
79 | @Override
80 | public String launch(AppDeploymentRequest request) {
81 | logger.info("Deploy request for {}", request);
82 | logger.info("Deploy request deployment properties {}", request.getDeploymentProperties());
83 | logger.info("Deploy definition {}", request.getDefinition());
84 | Resource resource = request.getResource();
85 | AppDefinition definition = request.getDefinition();
86 |
87 | String artifact = resource.getFilename();
88 |
89 | final String name = definition.getName();
90 | Map definitionParameters = definition.getProperties();
91 | Map deploymentProperties = request.getDeploymentProperties();
92 | List commandlineArguments = request.getCommandlineArguments();
93 | String appName = "scdtask:" + name;
94 |
95 | // contextRunArgs are passed to boot app ran to control yarn apps
96 | // we pass needed args to control module coordinates and params,
97 | // weird format with '--' is just straight pass to container
98 | ArrayList contextRunArgs = new ArrayList();
99 |
100 | contextRunArgs.add("--spring.yarn.appName=" + appName);
101 | for (Entry entry : definitionParameters.entrySet()) {
102 | if (StringUtils.hasText(entry.getValue())) {
103 | contextRunArgs
104 | .add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.parameters."
105 | + entry.getKey() + ".='" + entry.getValue() + "'");
106 | }
107 | }
108 |
109 | int index = 0;
110 | for (String commandlineArgument : commandlineArguments) {
111 | contextRunArgs.add("--spring.yarn.client.launchcontext.argumentsList[" + index
112 | + "]='--spring.cloud.deployer.yarn.appmaster.commandlineArguments[" + index + "]=" + commandlineArgument + "'");
113 | index++;
114 | }
115 |
116 | String baseDir = yarnDeployerProperties.getBaseDir();
117 | if (!baseDir.endsWith("/")) {
118 | baseDir = baseDir + "/";
119 | }
120 | String artifactPath = isHdfsResource(resource) ? getHdfsArtifactPath(resource) : baseDir + "/artifacts/cache/";
121 |
122 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.artifact=" + artifactPath + artifact);
123 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--spring.cloud.deployer.yarn.appmaster.artifactName=" + artifact);
124 |
125 | // deployment properties override servers.yml which overrides application.yml
126 | for (Entry entry : deploymentProperties.entrySet()) {
127 | if (StringUtils.hasText(entry.getValue())) {
128 | if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.taskcontainer")) {
129 | contextRunArgs.add("--spring.yarn.client.launchcontext.arguments.--" + entry.getKey() + "='" + entry.getValue() + "'");
130 | } else if (entry.getKey().startsWith("spring.cloud.deployer.yarn.app.taskappmaster")) {
131 | contextRunArgs.add("--" + entry.getKey() + "=" + entry.getValue());
132 | }
133 | }
134 | }
135 |
136 | String appVersion = StringUtils.hasText(yarnDeployerProperties.getAppVersion()) ? yarnDeployerProperties.getAppVersion() : "app";
137 | if (deploymentProperties.containsKey("spring.cloud.deployer.yarn.app.appVersion")) {
138 | appVersion = deploymentProperties.get("spring.cloud.deployer.yarn.app.appVersion");
139 | }
140 |
141 | final Message message = MessageBuilder.withPayload(TaskLauncherStateMachine.EVENT_LAUNCH)
142 | .setHeader(TaskLauncherStateMachine.HEADER_APP_VERSION, appVersion)
143 | .setHeader(TaskLauncherStateMachine.HEADER_ARTIFACT, resource)
144 | .setHeader(TaskLauncherStateMachine.HEADER_ARTIFACT_DIR, artifactPath)
145 | .setHeader(TaskLauncherStateMachine.HEADER_DEFINITION_PARAMETERS, definitionParameters)
146 | .setHeader(TaskLauncherStateMachine.HEADER_CONTEXT_RUN_ARGS, contextRunArgs)
147 | .build();
148 |
149 | // setup future, listen event from machine and finally unregister listener,
150 | // and set future value
151 | final SettableListenableFuture id = new SettableListenableFuture<>();
152 | final StateMachineListener listener = new StateMachineListenerAdapter() {
153 |
154 | @Override
155 | public void stateContext(StateContext stateContext) {
156 | if (stateContext.getStage() == Stage.STATE_ENTRY && stateContext.getTarget().getId().equals(TaskLauncherStateMachine.STATE_READY)) {
157 | if (ObjectUtils.nullSafeEquals(message.getHeaders().getId().toString(),
158 | stateContext.getExtendedState().get(TaskLauncherStateMachine.VAR_MESSAGE_ID, String.class))) {
159 | Exception exception = stateContext.getExtendedState().get(TaskLauncherStateMachine.VAR_ERROR, Exception.class);
160 | if (exception != null) {
161 | id.setException(exception);
162 | } else {
163 | String applicationId = stateContext.getStateMachine().getExtendedState()
164 | .get(TaskLauncherStateMachine.VAR_APPLICATION_ID, String.class);
165 | DeploymentKey key = new DeploymentKey(name, applicationId);
166 | id.set(key.toString());
167 | }
168 | }
169 | }
170 | }
171 | };
172 |
173 | stateMachine.addStateListener(listener);
174 | id.addCallback(new ListenableFutureCallback() {
175 |
176 | @Override
177 | public void onSuccess(String result) {
178 | stateMachine.removeStateListener(listener);
179 | }
180 |
181 | @Override
182 | public void onFailure(Throwable ex) {
183 | stateMachine.removeStateListener(listener);
184 | }
185 | });
186 |
187 | stateMachine.sendEvent(message);
188 | // we need to block here until SPI supports
189 | // returning id asynchronously
190 | try {
191 | return id.get(2, TimeUnit.MINUTES);
192 | } catch (Exception e) {
193 | throw new RuntimeException(e);
194 | }
195 | }
196 |
197 | private boolean isHdfsResource(Resource resource) {
198 | try {
199 | return resource != null && resource.getURI().getScheme().equals("hdfs");
200 | } catch (IOException e) {
201 | return false;
202 | }
203 | }
204 |
205 | private String getHdfsArtifactPath(Resource resource) {
206 | String path = null;
207 | try {
208 | path = "/" + FilenameUtils.getPath(resource.getURI().getPath());
209 | } catch (IOException e) {
210 | }
211 | return path;
212 | }
213 |
214 | @Override
215 | public void cancel(String id) {
216 | logger.info("Undeploy request for task {}", id);
217 | DeploymentKey key = new DeploymentKey(id);
218 | Message message = MessageBuilder.withPayload(TaskLauncherStateMachine.EVENT_CANCEL)
219 | .setHeader(TaskLauncherStateMachine.HEADER_APPLICATION_ID, key.applicationId)
220 | .build();
221 | stateMachine.sendEvent(message);
222 | }
223 |
224 | @Override
225 | public TaskStatus status(String id) {
226 | logger.info("Status request for module {}", id);
227 | DeploymentKey key = new DeploymentKey(id);
228 |
229 | Collection instances = yarnCloudAppService.getInstances(CloudAppType.TASK);
230 | for (CloudAppInstanceInfo instance : instances) {
231 | if (instance.getApplicationId().equals(key.applicationId)) {
232 | if (instance.getState() == "RUNNING") {
233 | return new TaskStatus(id, LaunchState.running, null);
234 | }
235 | }
236 | }
237 | return new TaskStatus(id, LaunchState.unknown, null);
238 | }
239 |
240 | @Override
241 | public void cleanup(String id) {
242 | }
243 |
244 | @Override
245 | public void destroy(String appName) {
246 | }
247 |
248 | @Override
249 | public RuntimeEnvironmentInfo environmentInfo() {
250 | return new RuntimeEnvironmentInfo.Builder()
251 | .spiClass(TaskLauncher.class)
252 | .implementationName(getClass().getSimpleName())
253 | .implementationVersion(RuntimeVersionUtils.getVersion(this.getClass()))
254 | .platformType("Yarn")
255 | .platformApiVersion(System.getProperty("os.name") + " " + System.getProperty("os.version"))
256 | .platformClientVersion(System.getProperty("os.version"))
257 | .platformHostVersion(System.getProperty("os.version"))
258 | .build();
259 | }
260 |
261 | private static class DeploymentKey {
262 | final static String SEPARATOR = ":";
263 | final String name;
264 | final String applicationId;
265 |
266 | public DeploymentKey(String id) {
267 | String[] split = id.split(SEPARATOR);
268 | Assert.isTrue(split.length == 2, "Unable to parse deployment key " + id);
269 | name = split[0];
270 | applicationId = split[1];
271 | }
272 |
273 | public DeploymentKey(String name, String applicationId) {
274 | Assert.notNull(name, "Name must be set");
275 | Assert.notNull(applicationId, "Application id must be set");
276 | this.name = name;
277 | this.applicationId = applicationId;
278 | }
279 |
280 | @Override
281 | public String toString() {
282 | return name + SEPARATOR + applicationId;
283 | }
284 | }
285 | }
286 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/resources/appdeployer-model.di:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/resources/appdeployer-model.uml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | bean
18 | errorGuard
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 | bean
28 | errorGuard
29 |
30 |
31 |
32 |
33 |
34 |
35 | bean
36 | errorGuard
37 |
38 |
39 |
40 |
41 |
42 | bean
43 | resetVariablesAction
44 |
45 |
46 |
47 |
48 | bean
49 | errorHandlingAction
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 | bean
58 | deployAction
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 | bean
68 | pushAppGuard
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 | bean
83 | instanceGuard
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 | bean
93 | startInstanceGuard
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 | bean
103 | checkAppAction
104 |
105 |
106 |
107 |
108 | bean
109 | pushAppAction
110 |
111 |
112 |
113 |
114 | bean
115 | checkInstanceAction
116 |
117 |
118 |
119 |
120 | bean
121 | startInstanceAction
122 |
123 |
124 |
125 |
126 | bean
127 | pushArtifactAction
128 |
129 |
130 |
131 |
132 | bean
133 | createClusterAction
134 |
135 |
136 |
137 |
138 | bean
139 | startClusterAction
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 | bean
148 | waitInstanceAction
149 |
150 |
151 |
152 |
153 |
154 | bean
155 | resolveInstanceAction
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 | bean
171 | stopClusterAction
172 |
173 |
174 |
175 |
176 | bean
177 | destroyClusterAction
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/resources/stream.yml:
--------------------------------------------------------------------------------
1 | spring:
2 | main:
3 | show_banner: false
4 | cloud:
5 | deployer:
6 | yarn:
7 | version: "@project.version@"
8 | yarn:
9 | appType: DATAFLOW
10 | appName: scdstream:app
11 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/stream/
12 | client:
13 | clientClass: org.springframework.cloud.deployer.spi.yarn.DeployerApplicationYarnClient
14 | files:
15 | - "file:${deployer.yarn.app.appmaster.path:lib}/spring-cloud-deployer-yarn-appdeployerappmaster-*.jar"
16 | - "file:${deployer.yarn.app.config.path:config}/servers.yml*"
17 | queue: ${spring.cloud.deployer.yarn.app.streamappmaster.queue:default}
18 | labelExpression: ${spring.cloud.deployer.yarn.app.streamappmaster.labelExpression:}
19 | launchcontext:
20 | options:
21 | - ${spring.cloud.deployer.yarn.app.streamappmaster.javaOpts:}
22 | arguments:
23 | -Dspring.config.location: servers.yml
24 | archiveFile: spring-cloud-deployer-yarn-appdeployerappmaster-${spring.cloud.deployer.yarn.version}.jar
25 | localizer:
26 | patterns:
27 | - "spring-cloud-deployer-yarn-appdeployerappmaster-*.jar"
28 | - "servers.yml"
29 | resource:
30 | memory: ${spring.cloud.deployer.yarn.app.streamappmaster.memory:512m}
31 | virtualCores: ${spring.cloud.deployer.yarn.app.streamappmaster.virtualCores:1}
32 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/resources/task.yml:
--------------------------------------------------------------------------------
1 | spring:
2 | main:
3 | show_banner: false
4 | cloud:
5 | deployer:
6 | yarn:
7 | version: "@project.version@"
8 | yarn:
9 | appType: DATAFLOW
10 | applicationBaseDir: ${spring.cloud.deployer.yarn.app.baseDir:/dataflow}/apps/task/
11 | client:
12 | enabled: true
13 | clientClass: org.springframework.cloud.deployer.spi.yarn.DeployerApplicationYarnClient
14 | files:
15 | - "file:${deployer.yarn.app.appmaster.path:lib}/spring-cloud-deployer-yarn-tasklauncherappmaster-*.jar"
16 | - "file:${deployer.yarn.app.config.path:config}/servers.yml*"
17 | queue: ${spring.cloud.deployer.yarn.app.taskappmaster.queue:default}
18 | labelExpression: ${spring.cloud.deployer.yarn.app.taskappmaster.labelExpression:}
19 | launchcontext:
20 | options:
21 | - ${spring.cloud.deployer.yarn.app.taskappmaster.javaOpts:}
22 | arguments:
23 | -Dspring.config.location: servers.yml
24 | archiveFile: spring-cloud-deployer-yarn-tasklauncherappmaster-${spring.cloud.deployer.yarn.version}.jar
25 | localizer:
26 | patterns:
27 | - "spring-cloud-deployer-yarn-tasklauncherappmaster-*.jar"
28 | - "servers.yml"
29 | resource:
30 | memory: ${spring.cloud.deployer.yarn.app.taskappmaster.memory:512m}
31 | virtualCores: ${spring.cloud.deployer.yarn.app.taskappmaster.virtualCores:1}
32 | container:
33 | enabled: false
34 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/resources/tasklauncher-model.di:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/main/resources/tasklauncher-model.uml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 | bean
11 | errorGuard
12 |
13 |
14 |
15 |
16 |
17 |
18 | bean
19 | errorGuard
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | bean
29 | errorGuard
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 | bean
43 | resetVariablesAction
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 | bean
52 | launchAction
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 | bean
61 | pushAppGuard
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 | bean
72 | checkAppAction
73 |
74 |
75 |
76 |
77 | bean
78 | pushAppAction
79 |
80 |
81 |
82 |
83 | bean
84 | pushArtifactAction
85 |
86 |
87 |
88 |
89 | bean
90 | startAppAction
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 | bean
103 | cancelAction
104 |
105 |
106 |
107 |
108 |
109 |
110 | bean
111 | stopAppAction
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 | bean
120 | errorHandlingAction
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/test/java/org/springframework/cloud/deployer/spi/yarn/AbstractStateMachineTests.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.util.ArrayList;
20 | import java.util.Collection;
21 | import java.util.Collections;
22 | import java.util.List;
23 | import java.util.Map;
24 | import java.util.concurrent.CountDownLatch;
25 |
26 | import org.junit.After;
27 | import org.junit.Before;
28 | import org.springframework.context.annotation.AnnotationConfigApplicationContext;
29 | import org.springframework.core.io.Resource;
30 | import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData;
31 |
32 | public class AbstractStateMachineTests {
33 |
34 | protected AnnotationConfigApplicationContext context;
35 |
36 | @Before
37 | public void setup() {
38 | context = buildContext();
39 | }
40 |
41 | @After
42 | public void clean() {
43 | if (context != null) {
44 | context.close();
45 | }
46 | context = null;
47 | }
48 |
49 | protected AnnotationConfigApplicationContext buildContext() {
50 | return null;
51 | }
52 |
53 | protected static class TestYarnCloudAppService implements YarnCloudAppService {
54 |
55 | volatile String app = null;
56 | volatile String instance = null;
57 |
58 | final CountDownLatch getApplicationsLatch = new CountDownLatch(1);
59 | final CountDownLatch getInstancesLatch = new CountDownLatch(2);
60 | final CountDownLatch pushApplicationLatch = new CountDownLatch(1);
61 | final CountDownLatch submitApplicationLatch = new CountDownLatch(1);
62 | final CountDownLatch createClusterLatch = new CountDownLatch(1);
63 | final CountDownLatch startClusterLatch = new CountDownLatch(1);
64 | final CountDownLatch stopClusterLatch = new CountDownLatch(1);
65 |
66 | volatile int getApplicationsCount = 0;
67 | volatile int getInstancesCount = 0;
68 | volatile int getInstancesCountBeforeReturn = 0;
69 |
70 | final List pushApplicationCount = Collections.synchronizedList(new ArrayList());
71 | final List submitApplicationCount = Collections.synchronizedList(new ArrayList());
72 | final List createClusterCount = Collections.synchronizedList(new ArrayList());
73 | final List startClusterCount = Collections.synchronizedList(new ArrayList());
74 | final List stopClusterCount = Collections.synchronizedList(new ArrayList());
75 |
76 | @Override
77 | public Collection getApplications(CloudAppType cloudAppType) {
78 | ArrayList infos = new ArrayList();
79 | if (app != null) {
80 | infos.add(new CloudAppInfo(app));
81 | }
82 | getApplicationsCount++;
83 | getApplicationsLatch.countDown();
84 | return infos;
85 | }
86 |
87 | @Override
88 | public Collection getInstances(CloudAppType cloudAppType) {
89 | ArrayList infos = new ArrayList();
90 | if (instance != null) {
91 | if (getInstancesCount >= getInstancesCountBeforeReturn) {
92 | infos.add(new CloudAppInstanceInfo("fakeApplicationId", instance, "RUNNING", "http://fakeAddress"));
93 | }
94 | }
95 | getInstancesCount++;
96 | getInstancesLatch.countDown();
97 | return infos;
98 | }
99 |
100 | @Override
101 | public void pushApplication(String appVersion, CloudAppType cloudAppType) {
102 | app = appVersion;
103 | pushApplicationCount.add(new Wrapper(appVersion));
104 | pushApplicationLatch.countDown();
105 | }
106 |
107 | @Override
108 | public String submitApplication(String appVersion, CloudAppType cloudAppType) {
109 | instance = "scdstream:" + appVersion;
110 | submitApplicationCount.add(new Wrapper(appVersion));
111 | submitApplicationLatch.countDown();
112 | return "fakeApplicationId";
113 | }
114 |
115 | @Override
116 | public String submitApplication(String appVersion, CloudAppType cloudAppType, List contextRunArgs) {
117 | instance = "scdstream:" + appVersion + ":fakeGroup";
118 | submitApplicationCount.add(new Wrapper(appVersion));
119 | submitApplicationLatch.countDown();
120 | return "fakeApplicationId";
121 | }
122 |
123 | @Override
124 | public void killApplications(String appName, CloudAppType cloudAppType) {
125 | }
126 |
127 | @Override
128 | public void createCluster(String yarnApplicationId, String clusterId, int count, String artifact,
129 | Map definitionParameters) {
130 | createClusterCount.add(new Wrapper(yarnApplicationId, clusterId, count, definitionParameters));
131 | createClusterLatch.countDown();
132 | }
133 |
134 | @Override
135 | public void startCluster(String yarnApplicationId, String clusterId) {
136 | startClusterCount.add(new Wrapper(yarnApplicationId, clusterId));
137 | startClusterLatch.countDown();
138 | }
139 |
140 | @Override
141 | public void stopCluster(String yarnApplicationId, String clusterId) {
142 | stopClusterCount.add(new Wrapper(yarnApplicationId, clusterId));
143 | stopClusterLatch.countDown();
144 | }
145 |
146 | @Override
147 | public Collection getClusters(String yarnApplicationId) {
148 | return null;
149 | }
150 |
151 | @Override
152 | public void destroyCluster(String yarnApplicationId, String clusterId) {
153 | }
154 |
155 | @Override
156 | public void pushArtifact(Resource artifact, String dir) {
157 | }
158 |
159 | static class Wrapper {
160 | String appVersion;
161 | String yarnApplicationId;
162 | String clusterId;
163 | int count;
164 | Map, ?> definitionParameters;
165 |
166 | public Wrapper(String appVersion) {
167 | this.appVersion = appVersion;
168 | }
169 |
170 | public Wrapper(String yarnApplicationId, String clusterId) {
171 | this.yarnApplicationId = yarnApplicationId;
172 | this.clusterId = clusterId;
173 | }
174 |
175 | public Wrapper(String yarnApplicationId, String clusterId, int count,
176 | Map, ?> definitionParameters) {
177 | this.yarnApplicationId = yarnApplicationId;
178 | this.clusterId = clusterId;
179 | this.count = count;
180 | this.definitionParameters = definitionParameters;
181 | }
182 |
183 | }
184 |
185 | @Override
186 | public Map getClustersStates(String yarnApplicationId) {
187 | return null;
188 | }
189 |
190 | @Override
191 | public void killApplication(String yarnApplicationId, CloudAppType cloudAppType) {
192 | }
193 |
194 | }
195 | }
196 |
--------------------------------------------------------------------------------
/spring-cloud-deployer-yarn/src/test/java/org/springframework/cloud/deployer/spi/yarn/AppDeployerStateMachineTests.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015-2016 the original author or authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package org.springframework.cloud.deployer.spi.yarn;
18 |
19 | import java.util.HashMap;
20 |
21 | import org.junit.Test;
22 | import org.springframework.context.annotation.AnnotationConfigApplicationContext;
23 | import org.springframework.context.annotation.Bean;
24 | import org.springframework.context.annotation.Configuration;
25 | import org.springframework.core.task.TaskExecutor;
26 | import org.springframework.messaging.Message;
27 | import org.springframework.messaging.support.MessageBuilder;
28 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
29 | import org.springframework.statemachine.StateMachine;
30 | import org.springframework.statemachine.test.StateMachineTestPlan;
31 | import org.springframework.statemachine.test.StateMachineTestPlanBuilder;
32 |
33 | public class AppDeployerStateMachineTests extends AbstractStateMachineTests {
34 |
35 | @Test
36 | public void testInitial() throws Exception {
37 | context.register(Config.class);
38 | context.refresh();
39 |
40 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService();
41 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class);
42 | AppDeployerStateMachine ycasm = new AppDeployerStateMachine(yarnCloudAppService, taskExecutor, context, context);
43 | ycasm.setAutoStart(false);
44 | StateMachine stateMachine = ycasm.buildStateMachine();
45 |
46 | StateMachineTestPlan plan =
47 | StateMachineTestPlanBuilder.builder()
48 | .defaultAwaitTime(10)
49 | .stateMachine(stateMachine)
50 | .step()
51 | .expectStateMachineStarted(1)
52 | .expectStates("READY")
53 | .and()
54 | .build();
55 | plan.test();
56 | }
57 |
58 | @Test
59 | public void testDeployShouldPushAndStart() throws Exception {
60 | context.register(Config.class);
61 | context.refresh();
62 |
63 | TestYarnCloudAppService yarnCloudAppService = new TestYarnCloudAppService();
64 | TaskExecutor taskExecutor = context.getBean(TaskExecutor.class);
65 | AppDeployerStateMachine ycasm = new AppDeployerStateMachine(yarnCloudAppService, taskExecutor, context, context);
66 | ycasm.setAutoStart(false);
67 | StateMachine stateMachine = ycasm.buildStateMachine();
68 |
69 | Message message = MessageBuilder.withPayload(AppDeployerStateMachine.EVENT_DEPLOY)
70 | .setHeader(AppDeployerStateMachine.HEADER_APP_VERSION, "app")
71 | .setHeader(AppDeployerStateMachine.HEADER_CLUSTER_ID, "fakeClusterId")
72 | .setHeader(AppDeployerStateMachine.HEADER_GROUP_ID, "fakeGroup")
73 | .setHeader(AppDeployerStateMachine.HEADER_COUNT, 1)
74 | .setHeader(AppDeployerStateMachine.HEADER_DEFINITION_PARAMETERS, new HashMap