├── CLA.pdf ├── LICENSE.txt ├── README.md ├── bin ├── read-link └── storm-yarn ├── create-tarball.sh ├── image ├── applicationid.png ├── config.png ├── editpom.png ├── environment.png ├── stormhome.png ├── stormui.png └── yarnui.png ├── lib ├── storm-starter-topologies-1.0.1.jar └── storm.zip ├── pom.xml └── src ├── main ├── genthrift.sh ├── java │ └── com │ │ └── yahoo │ │ └── storm │ │ └── yarn │ │ ├── Client.java │ │ ├── Config.java │ │ ├── LaunchCommand.java │ │ ├── MasterClient.java │ │ ├── MasterServer.java │ │ ├── StormAMRMClient.java │ │ ├── StormClusterChecker.java │ │ ├── StormMasterCommand.java │ │ ├── StormMasterServerHandler.java │ │ ├── StormOnYarn.java │ │ ├── Util.java │ │ ├── Version.java │ │ ├── VersionCommand.java │ │ └── generated │ │ └── StormMaster.java ├── resources │ ├── log4j2.xml │ └── master_defaults.yaml └── storm_master.thrift └── test ├── java └── com │ └── yahoo │ └── storm │ └── yarn │ ├── EmbeddedZKServer.java │ ├── TestConfig.java │ ├── TestIntegration.java │ ├── TestStormCluster.java │ └── TestStormMaster.java └── resources ├── log4j.properties └── log4j2-test.xml /CLA.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/CLA.pdf -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 10 | 11 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 14 | 15 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 16 | 17 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 18 | 19 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 20 | 21 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 22 | 23 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 24 | 25 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 26 | 27 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 28 | 29 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 30 | 31 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 32 | 33 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 34 | 35 | (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and 36 | 37 | (b) You must cause any modified files to carry prominent notices stating that You changed the files; and 38 | 39 | (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | 41 | (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 42 | 43 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 44 | 45 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 46 | 47 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 48 | 49 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, eitherf any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 50 | 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS 54 | 55 | APPENDIX: How to apply the Apache License to your work. 56 | 57 | To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. 58 | 59 | Copyright [yyyy] [name of copyright owner] 60 | 61 | Licensed under the Apache License, Version 2.0 (the "License"); 62 | you may not use this file except in compliance with the License. 63 | You may obtain a copy of the License at 64 | 65 | http://www.apache.org/licenses/LICENSE-2.0 66 | 67 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 68 | 69 | See the License for the specific language governing permissions and limitations under the License. 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | Storm-yarn 18 | ================= 19 | Storm-yarn enables Storm clusters to be deployed into machines managed by Hadoop YARN. It is still a work in progress. 20 | 21 | ## Contributors 22 | 23 | * Andy Feng ([@anfeng](https://github.com/anfeng)) 24 | * Robert Evans ([@revans2](https://github.com/revans2)) 25 | * Derek Dagit ([@d2r](https://github.com/d2r)) 26 | * Nathan Roberts ([@ynroberts](https://github.com/ynroberts)) 27 | * Xin Wang ([@vesense](https://github.com/vesense)) 28 | 29 | ## Mailing list 30 | 31 | Feel free to ask questions on storm-yarn's mailing list: http://groups.google.com/group/storm-yarn 32 | 33 | ## New features: 34 | 35 | Based on the project developed by yahoo, we have added following new features. 36 | 37 | 1. We have updated the version of Apache Storm from 0.9.0 to 1.0.1. 38 | 39 | 2. We have added StormClusterChecker class, in order to monitor the storm cluster. It can adjust the number of supervisors based on the usage of system resources. 40 | 41 | 3. We have added the function, namely removeSupervisors() in order to monitor resources. Its functionality is just opposite to that of addSupervisors(). 42 | 43 | 4. We have updated the logging framework from logback to log4j2. 44 | 45 | ## How to install and use 46 | ### Prerequisite 47 | * Install Java 8 and Maven 3 first. These two software are necessary to compiling and packaging the source code of storm-on-yarn. 48 | 49 | * Make sure [Hadoop YARN](https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/ClusterSetup.html) have been properly launched. 50 | 51 | * The storm-on-yarn implementation does not include running Zookeeper on YARN. Make sure the Zookeeper service is independently launched beforehands. 52 | 53 | ### Download and build 54 | 55 | 1. Download the source code of storm-on-yarn, e.g., execute the command ``git clone `` to get the source code. 56 | 57 | 2. Edit pom.xml in storm-on-yarn root directory to set the Hadoop version. 58 | 59 | ![pom.xml](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/editpom.png) 60 | 61 | 3. To package items, please execute the following command under storm-on-yarn root directory. 62 | 63 | mvn package 64 | 65 | You will see the following execution messages. 66 | 
17:57:27.810 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn launch ./conf/storm.yaml --stormZip lib/storm.zip --appname storm-on-yarn-test --output target/appId.txt
 67 |   17:57:59.681 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn getStormConfig ./conf/storm.yaml --appId application_1372121842369_0001 --output ./lib/storm/storm.yaml
 68 |   17:58:04.382 [main] INFO  com.yahoo.storm.yarn.TestIntegration - ./lib/storm/bin/storm jar lib/storm-starter-0.0.1-SNAPSHOT.jar storm.starter.ExclamationTopology exclamation-topology
 69 |   17:58:04.382 [main] INFO  com.yahoo.storm.yarn.TestIntegration - ./lib/storm/bin/storm kill exclamation-topology
 70 |   17:58:07.798 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn stopNimbus ./conf/storm.yaml --appId application_1372121842369_0001
 71 |   17:58:10.131 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn startNimbus ./conf/storm.yaml --appId application_1372121842369_0001
 72 |   17:58:12.460 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn stopUI ./conf/storm.yaml --appId application_1372121842369_0001
 73 |   17:58:15.045 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn startUI ./conf/storm.yaml --appId application_1372121842369_0001
 74 |   17:58:17.390 [main] INFO  com.yahoo.storm.yarn.TestIntegration - bin/storm-yarn shutdown ./conf/storm.yaml --appId application_1372121842369_0001
 75 |
76 | 77 | If you want to skip the tests, please add ``-DskipTests ``. 78 | 79 | mvn package -DskipTests 80 | 81 | ### Deploy: 82 | 83 | After compiling and building the whole project of storm-on-yarn, next you need to install storm-on-yarn and Storm on the Storm Client machine, which is used for submitting the YARN applications to YARN ResourceManager (RM) later. 84 | 85 | Please refer to the following  guide, step by step to deploy on the Storm Client machine. 86 | 87 | 1. Copy the packaged storm-on-yarn project to Storm Client machine, downloading the project of [storm-1.0.1](http://www.apache.org/dyn/closer.lua/storm/apache-storm-1.0.1/apache-storm-1.0.1.tar.gz). and put the decompressed project of storm-1.0.1 into same directory as the storm-on-yarn project. 88 | As shown below, 89 | 90 | ![stormHome](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/stormhome.png) 91 | 92 | So far, you have put storm-on-yarn and storm in the right place on Storm Client machine. You do not need to start running the Storm cluster, as this will be done by running storm-on-yarn later on. 93 | 94 | 2. When executing storm-on-yarn commands, commands like "storm-yarn", "storm" and etc., will be frequently called. Therefore, all paths to the bin files containing these executable commands must be included to the PATH environment variable. 95 | 96 | Hence you are suggested to add storm-1.0.1/bin and $(storm-on-yarn root directory)/bin to your PATH environment variable, like this: 97 | 98 | ![environment](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/environment.png) 99 | 100 | 3. Storm-on-yarn will replicate a copy of Storm code throughout all the nodes of the YARN cluster using HDFS. However, the location of where to fetch such copy is hard-coded into the Storm-on-YARN client. Therefore, you will have to manually prepare the copy inside HDFS. 101 |  The storm.zip file (the copy of Storm code) can be stored in HDFS under path "/lib/storm/[storm version]/storm.zip". 102 | 103 | Following commands illustrate how to upload the storm.zip from the local directory to "/lib/storm/1.0.1" in HDFS. 104 | 105 | hadoop fs -mkdir /lib 106 | hadoop fs -mkdir /lib/storm 107 | hadoop fs -mkdir /lib/storm/1.0.1 108 | zip -r storm.zip storm-1.0.1 109 | hadoop fs -put storm.zip /lib/storm/1.0.1/ 110 | 111 | At this point, the deployment step is complete. 112 | 113 | ### Run: 114 | 115 | Everything should be ready. Now you can start your storm-on-yarn project. 116 | 117 | The storm-on-yarn project have a set of specify commands and it use **storm-yarn [command] -[arg] xxx** as the comman format. 118 | 119 | To launch the cluster you can run 120 | 121 | storm-yarn launch [storm-yarn-configuration] 122 | 123 | [storm-yarn-configuration], which is usually a .yaml file, including all the required configurations during the launch of the Storm cluster. 124 | 125 | In this project, we provide two quick ways to create the storm-yarn-configuration file: 126 | 127 | a) Edit the storm.yaml file under storm-1.0.1/conf folder 128 | 129 | b) Copy the $(storm-on-yarn root directory)/src/main/master_defaults.yaml to storm-1.0.1/conf and rename it to master.yaml, and then edit it where necessary. 130 | 131 | In the simplest case, the only configuration you need to add is the Zookeeper cluster information and set the port of supervisor: 132 | 133 | ![Configuration](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/config.png) 134 | 135 | When you write a right configure file and execute the command above, the Hadoop YARN will return a application ID, if you deploy completely right. like this: 136 | 137 | ![applicationid](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/applicationid.png) 138 | 139 | And, you can see the graphic below on YARN UI and Storm UI respectively. 140 | 141 | ![yarnui](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/yarnui.png) 142 | 143 | ![stormui](https://github.com/wendyshusband/storm-yarn/blob/storm-1.0.1/image/stormui.png) 144 | 145 | So far, the storm-on-yarn has run and you can submit topology. 146 | 147 | You can run the following command to submit a topology. 148 | 149 | storm jar ***.jar topologyName -c nimbus= 150 | 151 | For example : 152 | 153 | storm jar ~/WordCount2/testWord-1.0-SNAPSHOT.jar storm.topology.WordCountTopology wordcount -c nimbus=192.168.1.25 154 | 155 | 156 | ### Other details: 157 | 158 | 1. storm-yarn has a number of new options for configuring the storm ApplicationManager (AM), e.g., 159 | * master.initial-num-supervisors, which stands for the initial number of supervisors to launch with storm. 160 | * master.container.size-mb, which stands for the size of the containers to request (from the YARN RM). 161 | 162 | 2. The procedure of "storm-yarn launch" returns an Application ID, which uniquely identifies the newly launched Storm master. This Application ID will be used for accessing the Storm master. 163 | 164 | To obtain a storm.yaml from the newly launch Storm master, you can run 165 | 166 | storm-yarn getStormConfig --appId --output 167 | 168 | storm.yaml will be retrieved from Storm master. 169 | 170 | 3. For a full list of storm-yarn commands and options you can run 171 | 172 | storm-yarn help 173 | 174 | 4. Storm-on-yarn is now configured to use Netty for communication between spouts and bolts. 175 | 176 | It's pure JVM based, and thus OS independent. 177 | 178 | If you are running storm using zeromq (instead of Netty), you need to augment the standard storm.zip file the needed .so files. This can be done with the not ideally named create-tarball.sh script 179 | 180 | create-tarball.sh storm.zip 181 | 182 | 5. Ideally the storm.zip file is a world readable file installed by ops so there is only one copy in the distributed cache ever. 183 | 184 | ## Commands: 185 | 186 | | Command | Usage | 187 | | -------- | ------ | 188 | |storm-yarn launch | launch storm on yarn| 189 | |storm-yarn help | get help| 190 | |storm-yarn version | view storm version| 191 | |storm-yarn addSupervisors/removeSupervisor | add/remove supervisor| 192 | |storm-yarn startNimbus/stopNimbus | start/stop Nimbus| 193 | |storm-yarn startUI/stopUI | start/stop Web UI| 194 | |storm-yarn startSupervisors/stopSupervisor | start or stop all supervisors| 195 | |storm-yarn shutdown | shutdown storm cluster| 196 | 197 | ## Arguments： 198 | 199 | | Argument | Usage| 200 | | -------- | ------ | 201 | | -appname | (Only for storm-yarn launch) Application Name. Default value – "Storm-on-Yarn" | 202 | | -appId | (Required) The storm clusters app ID| 203 | | -output | Output file| 204 | | -container | (Required for removeSupervisors) the supervisor to be removed| 205 | | -supervisors | (Required for addSupervisors) The # of supervisors to be added| 206 | 207 | ## Known Issues: 208 | 209 | There is no failover when nimbus goes down. Still working on it. 210 | 211 | There is no simple way to get to the logs for the different processes. 212 | 213 | ## License 214 | 215 | The use and distribution terms for this software are covered by the 216 | Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0.html). 217 | 218 | ## Acknowledgement 219 | This work was partially supported by SingAREN/AWS Cloud Credit for Research Program 2016. 220 | -------------------------------------------------------------------------------- /bin/read-link: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. See accompanying LICENSE file. 15 | # 16 | 17 | cd `dirname $1` 18 | TARGET_FILE=`basename $1` 19 | 20 | while [ -L "$TARGET_FILE" ] 21 | do 22 | TARGET_FILE=`readlink $TARGET_FILE` 23 | cd `dirname $TARGET_FILE` 24 | TARGET_FILE=`basename $TARGET_FILE` 25 | done 26 | 27 | echo "$(pwd -P)/$TARGET_FILE" 28 | -------------------------------------------------------------------------------- /bin/storm-yarn: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. See accompanying LICENSE file. 15 | # 16 | readonly STORM_ON_YARN_BIN="$(dirname "$(read-link "$0")")" 17 | readonly MASTER_JAR="$(ls "$STORM_ON_YARN_BIN"/../storm-yarn-*.jar "$STORM_ON_YARN_BIN"/../target/storm-yarn-*.jar 2> /dev/null | head -1)" 18 | 19 | if [ `command -v storm` ]; then 20 | readonly STORM_BIN="$(dirname "$(read-link "$(which storm)")")" 21 | STORM_CLASSPATH="$(storm classpath)" 22 | else 23 | echo "storm is not installed" >&2 24 | exit 1 25 | fi 26 | 27 | if [ -n "${JAVA_HOME}" ]; then 28 | RUNNER="${JAVA_HOME}/bin/java" 29 | else 30 | if [ `command -v java` ]; then 31 | RUNNER="java" 32 | else 33 | echo "JAVA_HOME is not set" >&2 34 | exit 1 35 | fi 36 | fi 37 | 38 | if [ `command -v yarn` ]; then 39 | YARN_CLASSPATH="$(yarn classpath)" 40 | else 41 | echo "yarn is not installed" >&2 42 | exit 1 43 | fi 44 | 45 | CLASSPATH="$STORM_YARN_CONF_DIR:$MASTER_JAR:$YARN_CLASSPATH:$STORM_CLASSPATH:$HOME/.storm" 46 | 47 | #echo "$RUNNER" -cp "$CLASSPATH" -Dstorm.home="$STORM_BIN"/.. com.yahoo.storm.yarn.Client "$@" 48 | exec "$RUNNER" -cp "$CLASSPATH" -Dstorm.home="$STORM_BIN"/.. com.yahoo.storm.yarn.Client "$@" 49 | 50 | -------------------------------------------------------------------------------- /create-tarball.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. See accompanying LICENSE file. 15 | # 16 | 17 | function set_LibraryPaths() { 18 | # This is hard-coded from storm. 19 | LibraryPaths="/usr/local/lib:/opt/local/lib:/usr/lib" 20 | 21 | local readonly LDLibraryPaths="$(echo "$LD_LIBRARY_PATH" | column -s: -t)" 22 | LibraryPaths="$LibraryPaths:$LDLibraryPaths" 23 | } 24 | 25 | function resolveNativeDependency() { 26 | local readonly Dep="$1" 27 | 28 | if [[ -z "$LibraryPaths" ]]; then set_LibraryPaths; fi 29 | 30 | local readonly OLD_IFS="$IFS" 31 | IFS=':' 32 | for path in $LibraryPaths 33 | do 34 | local depPath='' 35 | depPath="$(readlink -f --no-newline "$path/$Dep")" 36 | if [[ "$?" -eq 0 && -r "$depPath" ]] 37 | then 38 | echo -n " $depPath" 39 | IFS="$OLD_IFS" 40 | return 41 | fi 42 | done 43 | IFS="$OLD_IFS" 44 | 45 | echo "Could not find '$Dep' in [$LibraryPaths]" 1>&2 46 | echo 'Consider adding the appropriate path to LD_LIBRARY_PATH.' 1>&2 47 | exit 42 48 | } 49 | 50 | function exitIfError() { 51 | local readonly ExitCode="$?" 52 | if [[ "$ExitCode" -ne 0 ]]; then exit "$ExitCode"; fi 53 | } 54 | 55 | function printHDFSInstructions() { 56 | local readonly HDFSPath='/lib/storm/1.0.1' 57 | cat < 69 | Updates the given zip file (in-place) to add storms native dependency libraries. 70 | USAGE_STRING 71 | 72 | echo 73 | printHDFSInstructions 74 | exit 1 75 | } 76 | 77 | function processArguments() { 78 | if [[ "$#" -ne 1 ]] 79 | then 80 | printUsage 81 | fi 82 | 83 | if [[ ! -r "$1" || ! -w "$1" ]] 84 | then 85 | echo "Cannot access '$1'" 86 | exit 1 87 | fi 88 | 89 | if [[ ! -w "$(dirname "$0")" ]] 90 | then 91 | echo "Cannot write to the working directory" 92 | exit 1 93 | fi 94 | 95 | ZipFileName="$(readlink -f "$1")" 96 | if [[ ! -r "$ZipFileName" ]] 97 | then 98 | echo "File '$ZipFileName' is not readable." 99 | exit 1 100 | fi 101 | } 102 | 103 | function set_NativeDepsPaths() { 104 | for dep in $Dependencies; 105 | do 106 | local DepPath 107 | DepPath="$(resolveNativeDependency "$dep" "$LibraryPaths")" 108 | exitIfError 109 | NativeDepsPaths="$NativeDepsPaths $DepPath" 110 | done 111 | } 112 | 113 | # This is hard-coded from storm. 114 | readonly Dependencies='libjzmq.so libzmq.so' 115 | 116 | 117 | function createTarball() { 118 | processArguments "$@" 119 | set_NativeDepsPaths 120 | printHDFSInstructions 121 | echo "Adding native dependencies to '$ZipFileName':" 122 | zip -v -j -u "$ZipFileName" $NativeDepsPaths 123 | exitIfError 124 | } 125 | 126 | readonly _ThisScript="$(basename "$(readlink -f "$0")")" 127 | 128 | if [[ "$_ThisScript" == 'create-tarball.sh' ]] 129 | then 130 | createTarball "$@" 131 | fi 132 | -------------------------------------------------------------------------------- /image/applicationid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/applicationid.png -------------------------------------------------------------------------------- /image/config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/config.png -------------------------------------------------------------------------------- /image/editpom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/editpom.png -------------------------------------------------------------------------------- /image/environment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/environment.png -------------------------------------------------------------------------------- /image/stormhome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/stormhome.png -------------------------------------------------------------------------------- /image/stormui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/stormui.png -------------------------------------------------------------------------------- /image/yarnui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/image/yarnui.png -------------------------------------------------------------------------------- /lib/storm-starter-topologies-1.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/lib/storm-starter-topologies-1.0.1.jar -------------------------------------------------------------------------------- /lib/storm.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/storm-yarn/3c72d1675f37b6229a7d3900c95fd948ce416e78/lib/storm.zip -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.yahoo.storm.yarn 8 | storm-yarn 9 | 1.0-SNAPSHOT 10 | stormOnYARNwithDRS 11 | jar 12 | 13 | 14 | central 15 | http://repo1.maven.org/maven2/ 16 | 17 | false 18 | 19 | 20 | true 21 | 22 | 23 | 24 | clojars 25 | https://clojars.org/repo/ 26 | 27 | true 28 | 29 | 30 | true 31 | 32 | 33 | 34 | apache.snapshots 35 | http://repository.apache.org/content/repositories/snapshots/ 36 | 37 | 39 | 40 | version99 41 | 42 | http://version99.qos.ch/ 43 | 44 | 45 | 46 | 1.2.2 47 | 2.8.2 48 | 49 | 50 | 51 | org.apache.storm 52 | storm-core 53 | ${storm.version} 54 | 55 | 56 | org.slf4j 57 | log4j-over-slf4j 58 | 59 | 60 | 61 | 62 | org.slf4j 63 | jcl-over-slf4j 64 | 1.7.2 65 | 66 | 72 | 73 | org.apache.hadoop 74 | hadoop-yarn-server-tests 75 | ${hadoop.version} 76 | test-jar 77 | test 78 | 79 | 80 | org.apache.hadoop 81 | hadoop-common 82 | ${hadoop.version} 83 | 84 | 85 | org.apache.hadoop 86 | hadoop-mapreduce-client-core 87 | ${hadoop.version} 88 | 89 | 90 | org.apache.hadoop 91 | hadoop-yarn-client 92 | ${hadoop.version} 93 | provided 94 | 95 | 96 | junit 97 | junit 98 | 4.13.1 99 | test 100 | 101 | 102 | org.apache.zookeeper 103 | zookeeper 104 | 3.4.14 105 | test 106 | 107 | 108 | org.mockito 109 | mockito-all 110 | 1.9.5 111 | test 112 | 113 | 114 | com.googlecode.json-simple 115 | json-simple 116 | 1.1.1 117 | 118 | 119 | org.yaml 120 | snakeyaml 121 | 1.16 122 | 123 | 124 | 125 | 126 | 127 | 128 | maven-jar-plugin 129 | 2.4 130 | 131 | 132 | 133 | jar 134 | 135 | 137 | test-compile 138 | 139 | 140 | 141 | 142 | org.apache.maven.plugins 143 | maven-compiler-plugin 144 | 3.0 145 | 146 | true 147 | 1.8 148 | 1.8 149 | 150 | 151 | 152 | org.apache.maven.plugins 153 | maven-surefire-plugin 154 | 2.10 155 | 156 | 157 | conf 158 | 159 | 160 | 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /src/main/genthrift.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. See accompanying LICENSE file. 15 | # 16 | rm -rf gen-javabean 17 | rm -rf java/com/yahoo/storm/yarn/generated 18 | thrift --gen java:beans,hashcode,nocamel storm_master.thrift 19 | mv gen-javabean/com/yahoo/storm/yarn/generated java/com/yahoo/storm/yarn/generated 20 | rm -rf gen-javabean 21 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/Client.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.storm.yarn; 2 | import org.apache.commons.cli.CommandLine; 3 | import org.apache.commons.cli.GnuParser; 4 | import org.apache.commons.cli.HelpFormatter; 5 | import org.apache.commons.cli.Options; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import java.util.Arrays; 9 | import java.util.Collection; 10 | import java.util.HashMap; 11 | 12 | public class Client { 13 | private static final Logger LOG = LoggerFactory.getLogger(Client.class); 14 | 15 | public static interface ClientCommand { 16 | 17 | /** 18 | * @return the options this client will process. 19 | */ 20 | public Options getOpts(); 21 | 22 | /** 23 | * @return header description for this command 24 | */ 25 | public String getHeaderDescription(); 26 | 27 | /** 28 | * Do the processing 29 | * @param cl the arguments to process 30 | * @throws Exception on any error 31 | */ 32 | public void process(CommandLine cl) throws Exception; 33 | } 34 | 35 | public static class HelpCommand implements ClientCommand { 36 | HashMap _commands; 37 | public HelpCommand(HashMap commands) { 38 | _commands = commands; 39 | } 40 | 41 | @Override 42 | public Options getOpts() { 43 | return new Options(); 44 | } 45 | 46 | @Override 47 | public String getHeaderDescription() { 48 | return "storm-yarn help"; 49 | } 50 | 51 | @SuppressWarnings("unchecked") 52 | @Override 53 | public void process(CommandLine cl) throws Exception { 54 | printHelpFor(cl.getArgList()); 55 | } 56 | 57 | public void printHelpFor(Collection args) { 58 | if(args == null || args.size() < 1) { 59 | args = _commands.keySet(); 60 | } 61 | HelpFormatter f = new HelpFormatter(); 62 | for(String command: args) { 63 | ClientCommand c = _commands.get(command); 64 | if (c != null) { 65 | //TODO Show any arguments to the commands. 66 | f.printHelp(command, c.getHeaderDescription(), c.getOpts(), null); 67 | } else { 68 | System.err.println("ERROR: " + c + " is not a supported command."); 69 | //TODO make this exit with an error at some point 70 | } 71 | } 72 | } 73 | } 74 | 75 | /** 76 | * @param args the command line arguments 77 | * @throws Exception 78 | */ 79 | @SuppressWarnings("rawtypes") 80 | public void execute(String[] args) throws Exception { 81 | HashMap commands = new HashMap(); 82 | HelpCommand help = new HelpCommand(commands); 83 | commands.put("help", help); 84 | commands.put("launch", new LaunchCommand()); 85 | commands.put("setStormConfig", new StormMasterCommand(StormMasterCommand.COMMAND.SET_STORM_CONFIG)); 86 | commands.put("getStormConfig", new StormMasterCommand(StormMasterCommand.COMMAND.GET_STORM_CONFIG)); 87 | commands.put("addSupervisors", new StormMasterCommand(StormMasterCommand.COMMAND.ADD_SUPERVISORS)); 88 | commands.put("startNimbus", new StormMasterCommand(StormMasterCommand.COMMAND.START_NIMBUS)); 89 | commands.put("stopNimbus", new StormMasterCommand(StormMasterCommand.COMMAND.STOP_NIMBUS)); 90 | commands.put("startUI", new StormMasterCommand(StormMasterCommand.COMMAND.START_UI)); 91 | commands.put("stopUI", new StormMasterCommand(StormMasterCommand.COMMAND.STOP_UI)); 92 | commands.put("startSupervisors", new StormMasterCommand(StormMasterCommand.COMMAND.START_SUPERVISORS)); 93 | commands.put("stopSupervisors", new StormMasterCommand(StormMasterCommand.COMMAND.STOP_SUPERVISORS)); 94 | commands.put("shutdown", new StormMasterCommand(StormMasterCommand.COMMAND.SHUTDOWN)); 95 | commands.put("version", new VersionCommand()); 96 | commands.put("removeSupervisors", new StormMasterCommand(StormMasterCommand.COMMAND.REMOVE_SUPERVISORS)); 97 | String commandName = null; 98 | String[] commandArgs = null; 99 | if (args.length < 1) { 100 | commandName = "help"; 101 | commandArgs = new String[0]; 102 | } else { 103 | commandName = args[0]; 104 | commandArgs = Arrays.copyOfRange(args, 1, args.length); 105 | } 106 | ClientCommand command = commands.get(commandName); 107 | if(command == null) { 108 | LOG.error("ERROR: " + commandName + " is not a supported command."); 109 | help.printHelpFor(null); 110 | System.exit(1); 111 | } 112 | Options opts = command.getOpts(); 113 | if(!opts.hasOption("h")) { 114 | opts.addOption("h", "help", false, "print out a help message"); 115 | } 116 | CommandLine cl = new GnuParser().parse(command.getOpts(), commandArgs); 117 | if(cl.hasOption("help")) { 118 | help.printHelpFor(Arrays.asList(commandName)); 119 | } else { 120 | 121 | command.process(cl); 122 | } 123 | } 124 | 125 | public static void main(String[] args) throws Exception { 126 | Client client = new Client(); 127 | client.execute(args); 128 | } 129 | } 130 | 131 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import org.apache.storm.security.auth.ThriftConnectionType; 20 | import org.apache.storm.utils.Utils; 21 | import org.yaml.snakeyaml.Yaml; 22 | 23 | import java.io.FileInputStream; 24 | import java.io.IOException; 25 | import java.util.Map; 26 | 27 | 28 | 29 | public class Config { 30 | final public static ThriftConnectionType MASTER_THRIFT_TYPE = ThriftConnectionType.DRPC; 31 | final public static String MASTER_DEFAULTS_CONFIG = "master_defaults.yaml"; 32 | final public static String MASTER_CONFIG = "master.yaml"; 33 | final public static String MASTER_HOST = "master.host"; 34 | final public static String MASTER_THRIFT_PORT = "master.thrift.port"; 35 | final public static String MASTER_TIMEOUT_SECS = "master.timeout.secs"; 36 | final public static String MASTER_SIZE_MB = "master.container.size-mb"; 37 | final public static String MASTER_NUM_SUPERVISORS = "master.initial-num-supervisors"; 38 | final public static String MASTER_CONTAINER_PRIORITY = "master.container.priority"; 39 | //# of milliseconds to wait for YARN report on Storm Master host/port 40 | final public static String YARN_REPORT_WAIT_MILLIS = "yarn.report.wait.millis"; 41 | final public static String MASTER_HEARTBEAT_INTERVAL_MILLIS = "master.heartbeat.interval.millis"; 42 | 43 | @SuppressWarnings("rawtypes") 44 | static public Map readStormConfig() { 45 | return readStormConfig(null); 46 | } 47 | 48 | @SuppressWarnings({ "rawtypes", "unchecked" }) 49 | static Map readStormConfig(String stormYarnConfigPath) { 50 | //default configurations 51 | Map ret = Utils.readDefaultConfig(); 52 | Map conf = Utils.findAndReadConfigFile(Config.MASTER_DEFAULTS_CONFIG); 53 | ret.putAll(conf); 54 | 55 | //standard storm configuration 56 | String confFile = System.getProperty("storm.conf.file"); 57 | Map storm_conf; 58 | if (confFile==null || confFile.equals("")) { 59 | storm_conf = Utils.findAndReadConfigFile("storm.yaml", false); 60 | } else { 61 | storm_conf = Utils.findAndReadConfigFile(confFile, true); 62 | } 63 | ret.putAll(storm_conf); 64 | 65 | //configuration file per command parameter 66 | if (stormYarnConfigPath == null) { 67 | Map master_conf = Utils.findAndReadConfigFile(Config.MASTER_CONFIG, false); 68 | ret.putAll(master_conf); 69 | } 70 | else { 71 | try { 72 | Yaml yaml = new Yaml(); 73 | FileInputStream is = new FileInputStream(stormYarnConfigPath); 74 | Map storm_yarn_config = (Map) yaml.load(is); 75 | if(storm_yarn_config!=null) 76 | ret.putAll(storm_yarn_config); 77 | is.close(); 78 | } catch (IOException e) { 79 | throw new RuntimeException(e); 80 | } 81 | } 82 | 83 | //other configuration settings via CLS opts per system property: storm.options 84 | ret.putAll(Utils.readCommandLineOpts()); 85 | return ret; 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/LaunchCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import org.apache.commons.cli.CommandLine; 21 | import org.apache.commons.cli.Options; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.io.PrintStream; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | class LaunchCommand implements Client.ClientCommand { 30 | private static final Logger LOG = LoggerFactory.getLogger(LaunchCommand.class); 31 | 32 | @Override 33 | public String getHeaderDescription() { 34 | return "storm-yarn launch "; 35 | } 36 | 37 | @Override 38 | public Options getOpts() { 39 | Options opts = new Options(); 40 | opts.addOption("appname", true, "Application Name. Default value - Storm-on-Yarn"); 41 | opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); 42 | opts.addOption("stormHome", true, "Storm Home Directory"); 43 | opts.addOption("output", true, "Output file"); 44 | opts.addOption("stormConfOutput", true, "storm.yaml file"); 45 | opts.addOption("stormZip", true, "file path of storm.zip"); 46 | return opts; 47 | } 48 | 49 | @Override 50 | public void process(CommandLine cl) throws Exception { 51 | 52 | String config_file = null; 53 | List remaining_args = cl.getArgList(); 54 | if (remaining_args!=null && !remaining_args.isEmpty()) { 55 | config_file = (String)remaining_args.get(0); 56 | } 57 | Map stormConf = Config.readStormConfig(config_file); 58 | 59 | String appName = cl.getOptionValue("appname", "Storm-on-Yarn"); 60 | String queue = cl.getOptionValue("queue", "default"); 61 | 62 | String storm_zip_location = cl.getOptionValue("stormZip"); 63 | Integer amSize = (Integer) stormConf.get(Config.MASTER_SIZE_MB); 64 | 65 | StormOnYarn storm = null; 66 | try { 67 | storm = StormOnYarn.launchApplication(appName, 68 | queue, amSize, 69 | stormConf, 70 | storm_zip_location); 71 | LOG.debug("Submitted application's ID:" + storm.getAppId()); 72 | 73 | //download storm.yaml file 74 | String storm_yaml_output = cl.getOptionValue("stormConfOutput"); 75 | if (storm_yaml_output != null && storm_yaml_output.length() > 0) { 76 | //try to download storm.yaml 77 | StormMaster.Client client = storm.getClient(); 78 | if (client != null) 79 | StormMasterCommand.downloadStormYaml(client, storm_yaml_output); 80 | else 81 | LOG.warn("No storm.yaml is downloaded"); 82 | } 83 | 84 | //store appID to output 85 | String output = cl.getOptionValue("output"); 86 | if (output == null) 87 | System.out.println(storm.getAppId()); 88 | else { 89 | PrintStream os = new PrintStream(output); 90 | os.println(storm.getAppId()); 91 | os.flush(); 92 | os.close(); 93 | } 94 | } finally { 95 | if (storm != null) { 96 | storm.stop(); 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/MasterClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import org.apache.storm.security.auth.ThriftClient; 21 | import org.apache.storm.thrift.transport.TTransportException; 22 | import org.apache.storm.utils.Utils; 23 | 24 | import java.util.Map; 25 | 26 | public class MasterClient extends ThriftClient { 27 | private StormMaster.Client _client; 28 | 29 | @SuppressWarnings("rawtypes") 30 | public static MasterClient getConfiguredClient(Map conf) { 31 | try { 32 | String masterHost = (String) conf.get(Config.MASTER_HOST); 33 | int masterPort = Utils.getInt(conf.get(Config.MASTER_THRIFT_PORT)); 34 | try { 35 | Integer timeout = Utils.getInt(conf.get(Config.MASTER_TIMEOUT_SECS)); 36 | return new MasterClient(conf, masterHost, masterPort, timeout); 37 | } catch (IllegalArgumentException e) { 38 | return new MasterClient(conf, masterHost, masterPort, null); 39 | } 40 | 41 | } catch (TTransportException ex) { 42 | throw new RuntimeException(ex); 43 | } 44 | } 45 | 46 | @SuppressWarnings("rawtypes") 47 | public MasterClient(Map conf, String host, int port, Integer timeout) throws TTransportException { 48 | super(conf, Config.MASTER_THRIFT_TYPE, host, port, timeout); 49 | _client = new StormMaster.Client(_protocol); 50 | } 51 | 52 | public StormMaster.Client getClient() { 53 | return _client; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/MasterServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import org.apache.commons.cli.CommandLine; 21 | import org.apache.commons.cli.GnuParser; 22 | import org.apache.commons.cli.Options; 23 | import org.apache.hadoop.net.NetUtils; 24 | import org.apache.hadoop.service.Service; 25 | import org.apache.hadoop.yarn.api.ApplicationConstants; 26 | import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; 27 | import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; 28 | import org.apache.hadoop.yarn.api.records.*; 29 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 30 | import org.apache.hadoop.yarn.util.ConverterUtils; 31 | import org.apache.storm.security.auth.ThriftServer; 32 | import org.apache.storm.utils.Utils; 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | import java.io.IOException; 37 | import java.net.InetAddress; 38 | import java.net.InetSocketAddress; 39 | import java.util.Arrays; 40 | import java.util.List; 41 | import java.util.Map; 42 | import java.util.concurrent.BlockingQueue; 43 | import java.util.concurrent.LinkedBlockingQueue; 44 | 45 | public class MasterServer extends ThriftServer { 46 | private static final Logger LOG = LoggerFactory.getLogger(MasterServer.class); 47 | 48 | private StormMasterServerHandler _handler; 49 | 50 | private Thread initAndStartHeartbeat(final StormAMRMClient client, 51 | final BlockingQueue launcherQueue, 52 | final int heartBeatIntervalMs) { 53 | Thread thread = new Thread() { 54 | @Override 55 | public void run() { 56 | try { 57 | while (client.getServiceState() == Service.STATE.STARTED && 58 | !Thread.currentThread().isInterrupted()) { 59 | 60 | Thread.sleep(heartBeatIntervalMs); 61 | 62 | // We always send 50% progress. 63 | AllocateResponse allocResponse = client.allocate(0.5f); 64 | 65 | AMCommand am_command = allocResponse.getAMCommand(); 66 | if (am_command != null && 67 | (am_command == AMCommand.AM_SHUTDOWN || am_command == AMCommand.AM_RESYNC)) { 68 | LOG.info("Got AM_SHUTDOWN or AM_RESYNC from the RM"); 69 | _handler.stop(); 70 | System.exit(0); 71 | } 72 | 73 | List allocatedContainers = allocResponse.getAllocatedContainers(); 74 | if (allocatedContainers.size() > 0) { 75 | // Add newly allocated containers to the client. 76 | LOG.info("HB: Received allocated containers (" + allocatedContainers.size() + ")"); 77 | client.addAllocatedContainers(allocatedContainers); 78 | if (client.supervisorsAreToRun()) { 79 | LOG.info("HB: Supervisors are to run, so queueing (" + allocatedContainers.size() + ") containers..."); 80 | launcherQueue.addAll(allocatedContainers); 81 | } else { 82 | LOG.info("HB: Supervisors are to stop, so releasing all containers..."); 83 | client.stopAllSupervisors(); 84 | } 85 | } 86 | 87 | List completedContainers = allocResponse.getCompletedContainersStatuses(); 88 | if (completedContainers.size() > 0 && client.supervisorsAreToRun()) { 89 | LOG.debug("HB: Containers completed (" + completedContainers.size() + "), so releasing them."); 90 | client.removeCompletedContainers(completedContainers); 91 | client.startAllSupervisors(); 92 | } 93 | 94 | } 95 | } catch (Throwable t) { 96 | // Something happened we could not handle. Make sure the AM goes 97 | // down so that we are not surprised later on that our heart 98 | // stopped.. 99 | LOG.error("Unhandled error in AM: ", t); 100 | _handler.stop(); 101 | System.exit(1); 102 | } 103 | } 104 | }; 105 | thread.start(); 106 | return thread; 107 | } 108 | 109 | @SuppressWarnings("unchecked") 110 | public static void main(String[] args) throws Exception { 111 | LOG.info("Starting the AM!!!!"); 112 | 113 | Options opts = new Options(); 114 | opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used " + 115 | "unless for testing purposes"); 116 | 117 | CommandLine cl = new GnuParser().parse(opts, args); 118 | 119 | ApplicationAttemptId appAttemptID; 120 | Map envs = System.getenv(); 121 | if (cl.hasOption("app_attempt_id")) { 122 | String appIdStr = cl.getOptionValue("app_attempt_id", ""); 123 | appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); 124 | } else if (envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { 125 | ContainerId containerId = ConverterUtils.toContainerId(envs 126 | .get(ApplicationConstants.Environment.CONTAINER_ID.name())); 127 | appAttemptID = containerId.getApplicationAttemptId(); 128 | LOG.info("appAttemptID from env:" + appAttemptID.toString()); 129 | } else { 130 | LOG.error("appAttemptID is not specified for storm master"); 131 | throw new Exception("appAttemptID is not specified for storm master"); 132 | } 133 | 134 | @SuppressWarnings("rawtypes") 135 | Map storm_conf = Config.readStormConfig(null);///here is get config 136 | 137 | Util.rmNulls(storm_conf); 138 | 139 | YarnConfiguration hadoopConf = new YarnConfiguration(); 140 | 141 | final String host = InetAddress.getLocalHost().getHostName(); 142 | 143 | storm_conf.put("nimbus.seeds", Arrays.asList(new String[]{host})); 144 | 145 | StormAMRMClient rmClient = 146 | new StormAMRMClient(appAttemptID, storm_conf, hadoopConf); 147 | rmClient.init(hadoopConf); 148 | rmClient.start(); 149 | 150 | BlockingQueue launcherQueue = new LinkedBlockingQueue(); 151 | 152 | MasterServer server = new MasterServer(storm_conf, rmClient); 153 | try { 154 | final int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT)); 155 | final String target = host + ":" + port; 156 | 157 | InetSocketAddress addr = NetUtils.createSocketAddr(target); 158 | RegisterApplicationMasterResponse resp = 159 | rmClient.registerApplicationMaster(addr.getHostName(), port, null); 160 | LOG.info("Got a registration response " + resp); 161 | LOG.info("Max Capability " + resp.getMaximumResourceCapability()); 162 | rmClient.setMaxResource(resp.getMaximumResourceCapability()); 163 | LOG.info("Starting HB thread"); 164 | server.initAndStartHeartbeat(rmClient, launcherQueue, 165 | (Integer) storm_conf 166 | .get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS)); 167 | LOG.info("Starting launcher"); 168 | initAndStartLauncher(rmClient, launcherQueue); 169 | rmClient.startAllSupervisors(); 170 | LOG.info("Starting Master Thrift Server"); 171 | server.serve(); 172 | LOG.info("StormAMRMClient::unregisterApplicationMaster"); 173 | rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, 174 | "AllDone", null); 175 | } finally { 176 | if (server.isServing()) { 177 | LOG.info("Stop Master Thrift Server"); 178 | server.stop(); 179 | } 180 | LOG.info("Stop RM client"); 181 | rmClient.stop(); 182 | } 183 | System.exit(0); 184 | } 185 | 186 | private static void initAndStartLauncher(final StormAMRMClient client, 187 | final BlockingQueue launcherQueue) { 188 | Thread thread = new Thread() { 189 | Container container; 190 | 191 | @Override 192 | public void run() { 193 | while (client.getServiceState() == Service.STATE.STARTED && 194 | !Thread.currentThread().isInterrupted()) { 195 | try { 196 | container = launcherQueue.take(); 197 | LOG.info("LAUNCHER: Taking container with id (" + container.getId() + ") from the queue."); 198 | if (client.supervisorsAreToRun()) { 199 | LOG.info("LAUNCHER: Supervisors are to run, so launching container id (" + container.getId() + ")"); 200 | client.launchSupervisorOnContainer(container); 201 | } else { 202 | // Do nothing 203 | LOG.info("LAUNCHER: Supervisors are not to run, so not launching container id (" + container.getId() + ")"); 204 | } 205 | } catch (InterruptedException e) { 206 | if (client.getServiceState() == Service.STATE.STARTED) { 207 | LOG.error("Launcher thread interrupted : ", e); 208 | System.exit(1); 209 | } 210 | return; 211 | } catch (IOException e) { 212 | LOG.error("Launcher thread I/O exception : ", e); 213 | System.exit(1); 214 | } 215 | } 216 | } 217 | }; 218 | thread.start(); 219 | } 220 | 221 | public MasterServer(@SuppressWarnings("rawtypes") Map storm_conf, 222 | StormAMRMClient client) { 223 | 224 | this(storm_conf, new StormMasterServerHandler(storm_conf, client)); 225 | } 226 | 227 | private MasterServer(@SuppressWarnings("rawtypes") Map storm_conf, 228 | StormMasterServerHandler handler) { 229 | super(storm_conf, 230 | new StormMaster.Processor(handler), 231 | Config.MASTER_THRIFT_TYPE); 232 | try { 233 | 234 | _handler = handler; 235 | _handler.init(this); 236 | 237 | LOG.info("launch nimbus"); 238 | _handler.startNimbus(); 239 | 240 | LOG.info("launch ui"); 241 | _handler.startUI(); 242 | 243 | int numSupervisors = 244 | Utils.getInt(storm_conf.get(Config.MASTER_NUM_SUPERVISORS)); 245 | LOG.info("launch " + numSupervisors + " supervisors"); 246 | _handler.addSupervisors(numSupervisors); 247 | new StormClusterChecker(storm_conf, _handler).start(); 248 | } catch (Exception e) { 249 | throw new RuntimeException(e); 250 | } 251 | } 252 | 253 | public void stop() { 254 | super.stop(); 255 | if (_handler != null) { 256 | _handler.stop(); 257 | _handler = null; 258 | } 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/StormAMRMClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import org.apache.hadoop.fs.FileSystem; 20 | import org.apache.hadoop.fs.Path; 21 | import org.apache.hadoop.io.DataOutputBuffer; 22 | import org.apache.hadoop.security.Credentials; 23 | import org.apache.hadoop.security.UserGroupInformation; 24 | import org.apache.hadoop.yarn.api.ApplicationConstants; 25 | import org.apache.hadoop.yarn.api.records.*; 26 | import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; 27 | import org.apache.hadoop.yarn.client.api.NMClient; 28 | import org.apache.hadoop.yarn.client.api.YarnClient; 29 | import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; 30 | import org.apache.hadoop.yarn.client.api.impl.NMClientImpl; 31 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 32 | import org.apache.hadoop.yarn.util.Records; 33 | import org.apache.storm.utils.Utils; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | import java.io.IOException; 38 | import java.nio.ByteBuffer; 39 | import java.util.*; 40 | import java.util.concurrent.atomic.AtomicInteger; 41 | import java.util.stream.Collectors; 42 | 43 | 44 | class StormAMRMClient extends AMRMClientImpl { 45 | private static final Logger LOG = LoggerFactory.getLogger(StormAMRMClient.class); 46 | 47 | @SuppressWarnings("rawtypes") 48 | private final Map storm_conf; 49 | private final YarnConfiguration hadoopConf; 50 | private final Priority DEFAULT_PRIORITY = Records.newRecord(Priority.class); 51 | private final Set containers; 52 | private volatile boolean supervisorsAreToRun = false; 53 | private AtomicInteger numSupervisors; 54 | private Resource maxResourceCapability; 55 | private ApplicationAttemptId appAttemptId; 56 | private NMClientImpl nmClient; 57 | private YarnClient yarnClient; 58 | private Set requestingNodes = new HashSet<>(); 59 | 60 | public StormAMRMClient(ApplicationAttemptId appAttemptId, 61 | @SuppressWarnings("rawtypes") Map storm_conf, 62 | YarnConfiguration hadoopConf) { 63 | this.appAttemptId = appAttemptId; 64 | this.storm_conf = storm_conf; 65 | this.hadoopConf = hadoopConf; 66 | Integer pri = Utils.getInt(storm_conf.get(Config.MASTER_CONTAINER_PRIORITY)); 67 | this.DEFAULT_PRIORITY.setPriority(pri); 68 | this.containers = new TreeSet(); 69 | numSupervisors = new AtomicInteger(0); 70 | 71 | // start am nm client 72 | nmClient = (NMClientImpl) NMClient.createNMClient(); 73 | nmClient.init(hadoopConf); 74 | nmClient.start(); 75 | yarnClient = YarnClient.createYarnClient(); 76 | yarnClient.init(hadoopConf); 77 | yarnClient.start(); 78 | } 79 | 80 | public synchronized void removeCompletedContainers(List completedContainers) { 81 | LOG.debug("remove completed containers..."); 82 | for (ContainerStatus status : completedContainers) { 83 | Iterator it = this.containers.iterator(); 84 | while (it.hasNext()) { 85 | if (it.next().getId().equals(status.getContainerId())) { 86 | it.remove(); 87 | LOG.info("Remove completed container {}", status.getContainerId()); 88 | break; 89 | } 90 | } 91 | } 92 | } 93 | 94 | public synchronized void startAllSupervisors() { 95 | LOG.debug("Starting all supervisors, requesting containers..."); 96 | this.supervisorsAreToRun = true; 97 | this.addSupervisorsRequest(); 98 | } 99 | 100 | public synchronized void stopAllSupervisors() { 101 | LOG.debug("Stopping all supervisors, releasing all containers..."); 102 | this.supervisorsAreToRun = false; 103 | releaseAllSupervisorsRequest(); 104 | } 105 | 106 | private void addSupervisorsRequest() { 107 | NodeReport[] availableNodes; 108 | try { 109 | Set hosts = containers.stream().map(c -> c.getNodeId().getHost()).collect(Collectors.toSet()); 110 | hosts.addAll(requestingNodes); 111 | availableNodes = yarnClient.getNodeReports(NodeState.RUNNING).stream() 112 | .filter(n -> !hosts.contains(n.getNodeId().getHost())).toArray(NodeReport[]::new); 113 | } catch (Exception e) { 114 | LOG.warn("can not get node list", e); 115 | return; 116 | } 117 | int requiredNumSupervisors = numSupervisors.getAndSet(0); 118 | int num = Math.min(requiredNumSupervisors, availableNodes.length); 119 | LOG.info("Require {} supervisors, and {} nodes available", requiredNumSupervisors, availableNodes.length); 120 | for (int i = 0; i < num; i++) { 121 | String node = availableNodes[i].getNodeId().getHost(); 122 | requestingNodes.add(node); 123 | ContainerRequest req = new ContainerRequest(this.maxResourceCapability, 124 | new String[]{node}, // String[] nodes, 125 | null, // String[] racks, 126 | DEFAULT_PRIORITY, false); 127 | super.addContainerRequest(req); 128 | } 129 | } 130 | 131 | public synchronized boolean addAllocatedContainers(List containers) { 132 | for (int i = 0; i < containers.size(); i++) { 133 | Container c = containers.get(i); 134 | ContainerRequest req = new ContainerRequest(this.maxResourceCapability, 135 | new String[]{c.getNodeId().getHost()}, // String[] nodes, 136 | null, // String[] racks, 137 | DEFAULT_PRIORITY, false); 138 | super.removeContainerRequest(req); 139 | requestingNodes.remove(c.getNodeId().getHost()); 140 | } 141 | return this.containers.addAll(containers); 142 | } 143 | 144 | private synchronized void releaseAllSupervisorsRequest() { 145 | Iterator it = this.containers.iterator(); 146 | ContainerId id; 147 | while (it.hasNext()) { 148 | id = it.next().getId(); 149 | LOG.debug("Releasing container (id:" + id + ")"); 150 | releaseAssignedContainer(id); 151 | it.remove(); 152 | } 153 | } 154 | 155 | public synchronized boolean supervisorsAreToRun() { 156 | return this.supervisorsAreToRun; 157 | } 158 | 159 | public synchronized void addSupervisors(int number) { 160 | int num = numSupervisors.addAndGet(number); 161 | if (this.supervisorsAreToRun) { 162 | LOG.info("Added " + num + " supervisors, and requesting containers..."); 163 | addSupervisorsRequest(); 164 | } else { 165 | LOG.info("Added " + num + " supervisors, but not requesting containers now."); 166 | } 167 | } 168 | 169 | public void launchSupervisorOnContainer(Container container) throws IOException { 170 | // containers. 171 | // create a container launch context 172 | ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); 173 | UserGroupInformation user = UserGroupInformation.getCurrentUser(); 174 | try { 175 | Credentials credentials = user.getCredentials(); 176 | DataOutputBuffer dob = new DataOutputBuffer(); 177 | credentials.writeTokenStorageToStream(dob); 178 | ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); 179 | launchContext.setTokens(securityTokens); 180 | } catch (IOException e) { 181 | LOG.warn("Getting current user info failed when trying to launch the container" 182 | + e.getMessage()); 183 | } 184 | 185 | // CLC: env 186 | Map env = new HashMap(); 187 | env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); 188 | 189 | String java_home = (String) storm_conf.get("storm.yarn.java_home"); 190 | if (java_home == null) 191 | java_home = System.getenv("JAVA_HOME"); 192 | if (java_home != null && !java_home.isEmpty()) 193 | env.put("JAVA_HOME", java_home); 194 | 195 | launchContext.setEnvironment(env); 196 | 197 | // CLC: local resources includes storm, conf 198 | Map localResources = new HashMap(); 199 | String storm_zip_path = (String) storm_conf.get("storm.zip.path"); 200 | Path zip = new Path(storm_zip_path); 201 | FileSystem fs = FileSystem.get(hadoopConf); 202 | String vis = (String) storm_conf.get("storm.zip.visibility"); 203 | if (vis.equals("PUBLIC")) 204 | localResources.put("storm", Util.newYarnAppResource(fs, zip, 205 | LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC)); 206 | else if (vis.equals("PRIVATE")) 207 | localResources.put("storm", Util.newYarnAppResource(fs, zip, 208 | LocalResourceType.ARCHIVE, LocalResourceVisibility.PRIVATE)); 209 | else if (vis.equals("APPLICATION")) 210 | localResources.put("storm", Util.newYarnAppResource(fs, zip, 211 | LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); 212 | 213 | String userShortName = user.getShortUserName(); 214 | String workingDirectory = hadoopConf.get("yarn.nodemanager.local-dirs") + "/usercache/"+ userShortName 215 | + "/appcache/" + appAttemptId.getApplicationId() + "/" + container.getId(); 216 | Version stormVersion = Util.getStormVersion(); 217 | String stormHomeInZip = Util.getStormHomeInZip(fs, zip, stormVersion.version()); 218 | String appHome = Util.getApplicationHomeForId(appAttemptId.toString()); 219 | String containerHome = appHome + Path.SEPARATOR + container.getId().getContainerId(); 220 | Map supervisorConf = new HashMap(this.storm_conf); 221 | Path confDst = Util.createConfigurationFileInFs(fs, containerHome, supervisorConf, this.hadoopConf); 222 | localResources.put("conf", Util.newYarnAppResource(fs, confDst)); 223 | 224 | launchContext.setLocalResources(localResources); 225 | 226 | // CLC: command 227 | List supervisorArgs = Util.buildSupervisorCommands(this.storm_conf,workingDirectory,stormHomeInZip); 228 | launchContext.setCommands(supervisorArgs); 229 | try { 230 | LOG.info("Use NMClient to launch supervisors in container. "); 231 | nmClient.startContainer(container, launchContext); 232 | if (userShortName != null) 233 | LOG.info("Supervisor log: http://" + container.getNodeHttpAddress() + "/node/containerlogs/" 234 | + container.getId().toString() + "/" + userShortName + "/supervisor.log"); 235 | 236 | } catch (Exception e) { 237 | LOG.error("Caught an exception while trying to start a container", e); 238 | System.exit(-1); 239 | } 240 | } 241 | 242 | public void setMaxResource(Resource maximumResourceCapability) { 243 | this.maxResourceCapability = maximumResourceCapability; 244 | LOG.info("Max Capability is now " + this.maxResourceCapability); 245 | } 246 | 247 | public synchronized void removeSupervisors(String hostname) { 248 | LOG.info("removing container id: " + hostname + this.supervisorsAreToRun); 249 | if (this.supervisorsAreToRun) { 250 | LOG.debug("remove the needless supervisors, stop the container..."); 251 | releaseSupervisorsRequest(hostname); 252 | }else{ 253 | LOG.error("No supervisor is running!!!"); 254 | } 255 | } 256 | 257 | private synchronized void releaseSupervisorsRequest(String hostname) { 258 | Iterator it = this.containers.iterator(); 259 | while (it.hasNext()) { 260 | ContainerId id; 261 | id = it.next().getId(); 262 | if(id.toString().equals(hostname)) { 263 | LOG.debug("Releasing container (id:" + id + ")"); 264 | releaseAssignedContainer(id); 265 | it.remove(); 266 | break; 267 | } 268 | } 269 | } 270 | 271 | public Set getAllContainerInfo() { 272 | return this.containers; 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/StormClusterChecker.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.storm.yarn; 2 | 3 | import org.apache.storm.generated.ClusterSummary; 4 | import org.apache.storm.generated.Nimbus; 5 | import org.apache.storm.generated.SupervisorSummary; 6 | import org.apache.storm.generated.TopologySummary; 7 | import org.apache.storm.utils.NimbusClient; 8 | import org.apache.hadoop.yarn.api.records.Container; 9 | import org.apache.storm.thrift.TException; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.Iterator; 14 | import java.util.Map; 15 | 16 | /** 17 | * Created by ding on 14/11/17. 18 | */ 19 | class StormClusterChecker extends Thread { 20 | 21 | private static Logger LOG = LoggerFactory.getLogger(StormClusterChecker.class); 22 | private Map stormConf; 23 | private final StormMasterServerHandler master; 24 | private Nimbus.Iface nimbus; 25 | 26 | public StormClusterChecker(Map stormConf, StormMasterServerHandler master) { 27 | this.stormConf = stormConf; 28 | this.master = master; 29 | setDaemon(true); 30 | setName("storm cluster checker thread"); 31 | } 32 | 33 | @Override 34 | public void run() { 35 | LOG.info("Try to connect storm nimbus"); 36 | while (true) { 37 | while (nimbus == null) { 38 | try { 39 | Thread.sleep(10000); 40 | nimbus = NimbusClient.getConfiguredClient(stormConf).getClient(); 41 | LOG.info("Connected to storm nimbus, start checker..."); 42 | } catch (Exception e) { 43 | } 44 | } 45 | try { 46 | Thread.sleep(10000); 47 | } catch (InterruptedException e) { 48 | break; 49 | } 50 | try { 51 | ClusterSummary stormCluster = nimbus.getClusterInfo(); 52 | int totalNumWorkers = stormCluster.get_supervisors().stream() 53 | .mapToInt(SupervisorSummary::get_num_workers).sum(); 54 | int totalUsed = stormCluster.get_topologies().stream().mapToInt(TopologySummary::get_num_workers).sum(); 55 | if (totalUsed >= totalNumWorkers) { 56 | LOG.info("Need more workers, add 1 supervisor"); 57 | master.addSupervisors(1); 58 | }else{ 59 | int oneSupervisorWorkersNum = stormCluster.get_supervisors().get(0).get_num_workers(); 60 | int numOfVacant = (totalNumWorkers - totalUsed) ; 61 | if(numOfVacant > oneSupervisorWorkersNum){ 62 | Iterator it = master.getContainerInfo().iterator(); 63 | String containerID; 64 | if (it.hasNext()) { 65 | containerID = it.next().getId().toString(); 66 | LOG.info("remove a supervisor " + containerID); 67 | master.removeSupervisors(containerID); 68 | } 69 | } 70 | } 71 | } catch (TException e) { 72 | nimbus = null; 73 | } 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/StormMasterCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import org.apache.commons.cli.CommandLine; 21 | import org.apache.commons.cli.Options; 22 | import org.apache.storm.thrift.TException; 23 | import org.apache.storm.thrift.transport.TTransportException; 24 | import org.json.simple.JSONValue; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | import org.yaml.snakeyaml.Yaml; 28 | 29 | import java.io.FileWriter; 30 | import java.util.List; 31 | import java.util.Map; 32 | 33 | class StormMasterCommand implements Client.ClientCommand { 34 | private static final Logger LOG = LoggerFactory.getLogger(StormMasterCommand.class); 35 | 36 | enum COMMAND { 37 | GET_STORM_CONFIG, 38 | SET_STORM_CONFIG, 39 | START_NIMBUS, 40 | STOP_NIMBUS, 41 | START_UI, 42 | STOP_UI, 43 | ADD_SUPERVISORS, 44 | START_SUPERVISORS, 45 | STOP_SUPERVISORS, 46 | SHUTDOWN, 47 | REMOVE_SUPERVISORS 48 | }; 49 | COMMAND cmd; 50 | 51 | StormMasterCommand(COMMAND cmd) { 52 | this.cmd = cmd; 53 | } 54 | 55 | @Override 56 | public Options getOpts() { 57 | Options opts = new Options(); 58 | //TODO can we make this required 59 | opts.addOption("appId", true, "(Required) The storm clusters app ID"); 60 | opts.addOption("output", true, "Output file"); 61 | opts.addOption("supervisors", true, "(Required for addSupervisors) The # of supervisors to be added"); 62 | opts.addOption("container", true, "(Required for removeSupervisors) the supervisor to be remove"); 63 | 64 | return opts; 65 | } 66 | 67 | @Override 68 | public String getHeaderDescription() { 69 | return null; 70 | } 71 | 72 | @Override 73 | public void process(CommandLine cl) throws Exception { 74 | 75 | String config_file = null; 76 | List remaining_args = cl.getArgList(); 77 | if (remaining_args!=null && !remaining_args.isEmpty()) { 78 | config_file = (String)remaining_args.get(0); 79 | } 80 | Map stormConf = Config.readStormConfig(null); 81 | 82 | String appId = cl.getOptionValue("appId"); 83 | if(appId == null) { 84 | throw new IllegalArgumentException("-appId is required"); 85 | } 86 | 87 | StormOnYarn storm = null; 88 | try { 89 | storm = StormOnYarn.attachToApp(appId, stormConf); 90 | StormMaster.Client client = storm.getClient(); 91 | switch (cmd) { 92 | case REMOVE_SUPERVISORS: 93 | String supversior = cl.getOptionValue("container"); 94 | try { 95 | client.removeSupervisors(supversior); 96 | } catch (TTransportException ex) { 97 | LOG.info(ex.toString()); 98 | } 99 | break; 100 | case GET_STORM_CONFIG: 101 | downloadStormYaml(client, cl.getOptionValue("output")); 102 | break; 103 | 104 | case SET_STORM_CONFIG: 105 | String storm_conf_str = JSONValue.toJSONString(stormConf); 106 | try { 107 | client.setStormConf(storm_conf_str); 108 | } catch (TTransportException ex) { 109 | LOG.info(ex.toString()); 110 | } 111 | break; 112 | 113 | case ADD_SUPERVISORS: 114 | String supversiors = cl.getOptionValue("supervisors", "1"); 115 | try { 116 | client.addSupervisors(new Integer(supversiors).intValue()); 117 | } catch (TTransportException ex) { 118 | LOG.info(ex.toString()); 119 | } 120 | break; 121 | 122 | case START_NIMBUS: 123 | try { 124 | client.startNimbus(); 125 | } catch (TTransportException ex) { 126 | LOG.info(ex.toString()); 127 | } 128 | break; 129 | 130 | case STOP_NIMBUS: 131 | try { 132 | client.stopNimbus(); 133 | } catch (TTransportException ex) { 134 | LOG.info(ex.toString()); 135 | } 136 | break; 137 | 138 | case START_UI: 139 | try { 140 | client.startUI(); 141 | } catch (TTransportException ex) { 142 | LOG.info(ex.toString()); 143 | } 144 | break; 145 | 146 | case STOP_UI: 147 | try { 148 | client.stopUI(); 149 | } catch (TTransportException ex) { 150 | LOG.info(ex.toString()); 151 | } 152 | break; 153 | 154 | case START_SUPERVISORS: 155 | try { 156 | client.startSupervisors(); 157 | } catch (TTransportException ex) { 158 | LOG.info(ex.toString()); 159 | } 160 | break; 161 | 162 | case STOP_SUPERVISORS: 163 | try { 164 | client.stopSupervisors(); 165 | } catch (TTransportException ex) { 166 | LOG.info(ex.toString()); 167 | } 168 | break; 169 | 170 | case SHUTDOWN: 171 | try { 172 | client.shutdown(); 173 | } catch (TTransportException ex) { 174 | LOG.info(ex.toString()); 175 | } 176 | break; 177 | } 178 | } finally { 179 | if (storm != null) { 180 | storm.stop(); 181 | } 182 | } 183 | } 184 | 185 | public static void downloadStormYaml(StormMaster.Client client, String storm_yaml_output) { 186 | String conf_str = "Not Avaialble"; 187 | 188 | //fetch storm.yaml from Master 189 | try { 190 | conf_str = client.getStormConf(); 191 | } catch (TTransportException ex) { 192 | LOG.error("Exception in downloading storm.yaml", ex); 193 | } catch (TException ex) { 194 | LOG.error("Exception in downloading storm.yaml", ex); 195 | } 196 | 197 | //storm the fetched storm.yaml into storm_yaml_output or stdout 198 | try { 199 | Object json = JSONValue.parse(conf_str); 200 | Map conf = (Map)json; 201 | Yaml yaml = new Yaml(); 202 | 203 | if (storm_yaml_output == null) { 204 | LOG.info("storm.yaml downloaded:"); 205 | System.out.println(yaml.dump(conf)); 206 | } else { 207 | FileWriter out = new FileWriter(storm_yaml_output); 208 | yaml.dump(conf, out); 209 | out.flush(); 210 | out.close(); 211 | LOG.info("storm.yaml downloaded into "+storm_yaml_output); 212 | } 213 | } catch (Exception ex) { 214 | LOG.error("Exception in storing storm.yaml. ", ex); 215 | } 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/StormMasterServerHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import org.apache.storm.Config; 21 | import com.google.common.base.Joiner; 22 | import org.apache.hadoop.yarn.api.records.Container; 23 | import org.apache.storm.thrift.TException; 24 | import org.json.simple.JSONValue; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.io.IOException; 29 | import java.net.InetAddress; 30 | import java.net.UnknownHostException; 31 | import java.util.Arrays; 32 | import java.util.List; 33 | import java.util.Map; 34 | import java.util.Set; 35 | 36 | public class StormMasterServerHandler implements StormMaster.Iface { 37 | private static final Logger LOG = LoggerFactory.getLogger(StormMasterServerHandler.class); 38 | 39 | @SuppressWarnings("rawtypes") 40 | Map _storm_conf; 41 | StormAMRMClient _client; 42 | MasterServer _masterServer; 43 | 44 | StormMasterServerHandler(@SuppressWarnings("rawtypes") Map storm_conf, StormAMRMClient client) { 45 | _storm_conf = storm_conf; 46 | setStormHostConf(); 47 | Util.rmNulls(_storm_conf); 48 | _client = client; 49 | } 50 | 51 | void init(MasterServer masterServer) { 52 | _masterServer = masterServer; 53 | } 54 | 55 | void stop() { 56 | try { 57 | stopSupervisors(); 58 | stopUI(); 59 | stopNimbus(); 60 | } catch (TException e) { 61 | // TODO Auto-generated catch block 62 | e.printStackTrace(); 63 | } 64 | } 65 | 66 | private void setStormHostConf() { 67 | try { 68 | String host_addr = InetAddress.getLocalHost().getHostAddress(); 69 | LOG.info("Storm master host:" + host_addr); 70 | _storm_conf.put(Config.NIMBUS_SEEDS, Arrays.asList(new String[]{host_addr})); 71 | 72 | } catch (UnknownHostException ex) { 73 | LOG.warn("Failed to get IP address of local host"); 74 | } 75 | } 76 | 77 | @Override 78 | public String getStormConf() throws TException { 79 | LOG.info("getting configuration..."); 80 | return JSONValue.toJSONString(_storm_conf); 81 | } 82 | 83 | @SuppressWarnings("unchecked") 84 | @Override 85 | public void setStormConf(String storm_conf) throws TException { 86 | LOG.info("setting configuration..."); 87 | 88 | // stop processes 89 | stopSupervisors(); 90 | stopUI(); 91 | stopNimbus(); 92 | 93 | Object json = JSONValue.parse(storm_conf); 94 | Map new_conf = (Map) json; 95 | _storm_conf.putAll(new_conf); 96 | Util.rmNulls(_storm_conf); 97 | setStormHostConf(); 98 | 99 | // start processes 100 | startNimbus(); 101 | startUI(); 102 | startSupervisors(); 103 | } 104 | 105 | @Override 106 | public void addSupervisors(int number) throws TException { 107 | LOG.info("adding " + number + " supervisors..."); 108 | _client.addSupervisors(number); 109 | } 110 | 111 | @Override 112 | public void removeSupervisors(String hostname) throws TException { 113 | LOG.info("remove supervisors..."); 114 | _client.removeSupervisors(hostname); 115 | } 116 | 117 | class StormProcess extends Thread { 118 | Process _process; 119 | String _name; 120 | 121 | public StormProcess(String name) { 122 | _name = name; 123 | } 124 | 125 | public void run() { 126 | startStormProcess(); 127 | try { 128 | _process.waitFor(); 129 | LOG.info("Storm process " + _name + " stopped"); 130 | } catch (InterruptedException e) { 131 | LOG.info("Interrupted => will stop the storm process too"); 132 | _process.destroy(); 133 | } 134 | } 135 | 136 | private void startStormProcess() { 137 | try { 138 | LOG.info("Running: " + Joiner.on(" ").join(buildCommands())); 139 | ProcessBuilder builder = 140 | new ProcessBuilder(buildCommands()); 141 | 142 | _process = builder.start(); 143 | Util.redirectStreamAsync(_process.getInputStream(), System.out); 144 | Util.redirectStreamAsync(_process.getErrorStream(), System.err); 145 | 146 | } catch (IOException e) { 147 | LOG.warn("Error starting nimbus process ", e); 148 | } 149 | } 150 | 151 | private List buildCommands() throws IOException { 152 | if (_name == "nimbus") { 153 | return Util.buildNimbusCommands(_storm_conf); 154 | } else if (_name == "ui") { 155 | return Util.buildUICommands(_storm_conf); 156 | } 157 | 158 | throw new IllegalArgumentException( 159 | "Cannot build command list for \"" + _name + "\""); 160 | } 161 | 162 | public void stopStormProcess() { 163 | _process.destroy(); 164 | } 165 | } 166 | 167 | StormProcess nimbusProcess; 168 | StormProcess uiProcess; 169 | 170 | @Override 171 | public void startNimbus() { 172 | LOG.info("starting nimbus..."); 173 | synchronized (this) { 174 | if (nimbusProcess != null && nimbusProcess.isAlive()) { 175 | LOG.info("Received a request to start nimbus, but it is running now"); 176 | return; 177 | } 178 | nimbusProcess = new StormProcess("nimbus"); 179 | nimbusProcess.start(); 180 | } 181 | } 182 | 183 | @Override 184 | public void stopNimbus() { 185 | synchronized (this) { 186 | if (nimbusProcess == null) return; 187 | LOG.info("stopping nimbus..."); 188 | if (!nimbusProcess.isAlive()) { 189 | LOG.info("Received a request to stop nimbus, but it is not running now"); 190 | return; 191 | } 192 | nimbusProcess.stopStormProcess(); 193 | nimbusProcess = null; 194 | } 195 | } 196 | 197 | @Override 198 | public void startUI() throws TException { 199 | LOG.info("starting UI..."); 200 | synchronized (this) { 201 | if (uiProcess != null && uiProcess.isAlive()) { 202 | LOG.info("Received a request to start UI, but it is running now"); 203 | return; 204 | } 205 | uiProcess = new StormProcess("ui"); 206 | uiProcess.start(); 207 | } 208 | } 209 | 210 | @Override 211 | public void stopUI() throws TException { 212 | synchronized (this) { 213 | if (uiProcess == null) return; 214 | LOG.info("stopping UI..."); 215 | if (!uiProcess.isAlive()) { 216 | LOG.info("Received a request to stop UI, but it is not running now"); 217 | return; 218 | } 219 | uiProcess.stopStormProcess(); 220 | uiProcess = null; 221 | } 222 | } 223 | 224 | @Override 225 | public void startSupervisors() throws TException { 226 | LOG.info("starting supervisors..."); 227 | _client.startAllSupervisors(); 228 | } 229 | 230 | @Override 231 | public void stopSupervisors() throws TException { 232 | LOG.info("stopping supervisors..."); 233 | _client.stopAllSupervisors(); 234 | } 235 | 236 | @Override 237 | public void shutdown() throws TException { 238 | LOG.info("shutdown storm master..."); 239 | _masterServer.stop(); 240 | } 241 | 242 | public Set getContainerInfo(){ 243 | LOG.info("get all container info..."); 244 | return _client.getAllContainerInfo(); 245 | } 246 | 247 | } 248 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/StormOnYarn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import org.apache.hadoop.fs.FileSystem; 21 | import org.apache.hadoop.fs.Path; 22 | import org.apache.hadoop.io.DataOutputBuffer; 23 | import org.apache.hadoop.mapreduce.security.TokenCache; 24 | import org.apache.hadoop.security.Credentials; 25 | import org.apache.hadoop.yarn.api.ApplicationConstants; 26 | import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; 27 | import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; 28 | import org.apache.hadoop.yarn.api.records.*; 29 | import org.apache.hadoop.yarn.client.api.YarnClient; 30 | import org.apache.hadoop.yarn.client.api.YarnClientApplication; 31 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 32 | import org.apache.hadoop.yarn.exceptions.YarnException; 33 | import org.apache.hadoop.yarn.util.Apps; 34 | import org.apache.hadoop.yarn.util.ConverterUtils; 35 | import org.apache.hadoop.yarn.util.Records; 36 | import org.apache.storm.utils.Utils; 37 | import org.slf4j.Logger; 38 | import org.slf4j.LoggerFactory; 39 | 40 | import java.io.BufferedReader; 41 | import java.io.IOException; 42 | import java.io.InputStreamReader; 43 | import java.net.URL; 44 | import java.net.URLDecoder; 45 | import java.nio.ByteBuffer; 46 | import java.util.*; 47 | 48 | public class StormOnYarn { 49 | private static final Logger LOG = LoggerFactory.getLogger(StormOnYarn.class); 50 | 51 | private YarnClient _yarn; 52 | private YarnConfiguration _hadoopConf; 53 | private ApplicationId _appId; 54 | @SuppressWarnings("rawtypes") 55 | private Map _stormConf; 56 | private MasterClient _client = null; 57 | 58 | private StormOnYarn(@SuppressWarnings("rawtypes") Map stormConf) { 59 | this(null, stormConf); 60 | } 61 | 62 | private StormOnYarn(ApplicationId appId, @SuppressWarnings("rawtypes") Map stormConf) { 63 | _hadoopConf = new YarnConfiguration(); 64 | _yarn = YarnClient.createYarnClient(); 65 | _stormConf = stormConf; 66 | _appId = appId; 67 | _yarn.init(_hadoopConf); 68 | _yarn.start(); 69 | } 70 | 71 | public void stop() { 72 | if (_client != null) { 73 | _client.close(); 74 | } 75 | _yarn.stop(); 76 | } 77 | 78 | public ApplicationId getAppId() { 79 | //TODO make this immutable 80 | return _appId; 81 | } 82 | 83 | @SuppressWarnings("unchecked") 84 | public synchronized StormMaster.Client getClient() throws YarnException, IOException { 85 | if (_client == null) { 86 | String host = null; 87 | int port = 0; 88 | //wait for application to be ready 89 | int max_wait_for_report = Utils.getInt(_stormConf.get(Config.YARN_REPORT_WAIT_MILLIS)); 90 | int waited = 0; 91 | while (waited < max_wait_for_report) { 92 | ApplicationReport report = _yarn.getApplicationReport(_appId); 93 | host = report.getHost(); 94 | port = report.getRpcPort(); 95 | if (host == null || port == 0) { 96 | try { 97 | Thread.sleep(1000); 98 | } catch (InterruptedException e) { 99 | } 100 | waited += 1000; 101 | } else { 102 | break; 103 | } 104 | } 105 | if (host == null || port == 0) { 106 | LOG.info("No host/port returned for Application Master " + _appId); 107 | return null; 108 | } 109 | 110 | LOG.info("application report for " + _appId + " :" + host + ":" + port); 111 | if (_stormConf == null) { 112 | _stormConf = new HashMap(); 113 | } 114 | _stormConf.put(Config.MASTER_HOST, host); 115 | _stormConf.put(Config.MASTER_THRIFT_PORT, port); 116 | LOG.info("Attaching to " + host + ":" + port + " to talk to app master " + _appId); 117 | _client = MasterClient.getConfiguredClient(_stormConf); 118 | } 119 | return _client.getClient(); 120 | } 121 | 122 | private void launchApp(String appName, String queue, int amMB, String storm_zip_location) throws Exception { 123 | LOG.debug("StormOnYarn:launchApp() ..."); 124 | YarnClientApplication client_app = _yarn.createApplication(); 125 | GetNewApplicationResponse app = client_app.getNewApplicationResponse(); 126 | _appId = app.getApplicationId(); 127 | LOG.debug("_appId:" + _appId); 128 | 129 | if (amMB > app.getMaximumResourceCapability().getMemory()) { 130 | //TODO need some sanity checks 131 | amMB = app.getMaximumResourceCapability().getMemory(); 132 | } 133 | ApplicationSubmissionContext appContext = 134 | Records.newRecord(ApplicationSubmissionContext.class); 135 | appContext.setApplicationId(app.getApplicationId()); 136 | appContext.setApplicationName(appName); 137 | appContext.setQueue(queue); 138 | 139 | // Set up the container launch context for the application master 140 | ContainerLaunchContext amContainer = Records 141 | .newRecord(ContainerLaunchContext.class); 142 | Map localResources = new HashMap(); 143 | 144 | // set local resources for the application master 145 | // local files or archives as needed 146 | // In this scenario, the jar file for the application master is part of the 147 | // local resources 148 | LOG.info("Copy App Master jar from local filesystem and add to local environment"); 149 | // Copy the application master jar to the filesystem 150 | // Create a local resource to point to the destination jar path 151 | String appMasterJar = findContainingJar(MasterServer.class); 152 | FileSystem fs = FileSystem.get(_hadoopConf); 153 | Path src = new Path(appMasterJar); 154 | String appHome = Util.getApplicationHomeForId(_appId.toString()); 155 | Path dst = new Path(fs.getHomeDirectory(), 156 | appHome + Path.SEPARATOR + "AppMaster.jar"); 157 | fs.copyFromLocalFile(false, true, src, dst); 158 | localResources.put("AppMaster.jar", Util.newYarnAppResource(fs, dst)); 159 | 160 | Version stormVersion = Util.getStormVersion(); 161 | LOG.info("Storm Home is " + Util.getStormHome()); 162 | LOG.info("Storm version is " + stormVersion.version()); 163 | Path zip; 164 | if (storm_zip_location != null) { 165 | zip = new Path(storm_zip_location); 166 | } else { 167 | zip = new Path("/lib/storm/" + stormVersion + "/storm.zip"); 168 | } 169 | _stormConf.put("storm.zip.path", zip.makeQualified(fs).toUri().getPath()); 170 | LocalResourceVisibility visibility = LocalResourceVisibility.PUBLIC; 171 | _stormConf.put("storm.zip.visibility", "PUBLIC"); 172 | if (!Util.isPublic(fs, zip)) { 173 | visibility = LocalResourceVisibility.APPLICATION; 174 | _stormConf.put("storm.zip.visibility", "APPLICATION"); 175 | } 176 | localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, visibility)); 177 | 178 | Path confDst = Util.createConfigurationFileInFs(fs, appHome, _stormConf, _hadoopConf); 179 | // establish a symbolic link to conf directory 180 | localResources.put("conf", Util.newYarnAppResource(fs, confDst)); 181 | 182 | // Setup security tokens 183 | Path[] paths = new Path[3]; 184 | paths[0] = dst; 185 | paths[1] = zip; 186 | paths[2] = confDst; 187 | Credentials credentials = new Credentials(); 188 | TokenCache.obtainTokensForNamenodes(credentials, paths, _hadoopConf); 189 | DataOutputBuffer dob = new DataOutputBuffer(); 190 | credentials.writeTokenStorageToStream(dob); 191 | ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); 192 | 193 | //security tokens for HDFS distributed cache 194 | amContainer.setTokens(securityTokens); 195 | 196 | // Set local resource info into app master container launch context 197 | amContainer.setLocalResources(localResources); 198 | 199 | // Set the env variables to be setup in the env where the application master 200 | // will be run 201 | LOG.info("Set the environment for the application master"); 202 | Map env = new HashMap(); 203 | // add the runtime classpath needed for tests to work 204 | Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./conf",":"); 205 | Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar",":"); 206 | //Make sure that AppMaster has access to all YARN JARs 207 | List yarn_classpath_cmd = Arrays.asList("yarn", "classpath"); 208 | ProcessBuilder pb = new ProcessBuilder(yarn_classpath_cmd); 209 | LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]"); 210 | pb.environment().putAll(System.getenv()); 211 | Process proc = pb.start(); 212 | Util.redirectStreamAsync(proc.getErrorStream(), System.err); 213 | BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), "UTF-8")); 214 | String line = ""; 215 | String yarn_class_path = (String) _stormConf.get("storm.yarn.yarn_classpath"); 216 | if (yarn_class_path == null) { 217 | StringBuilder yarn_class_path_builder = new StringBuilder(); 218 | while ((line = reader.readLine()) != null) { 219 | yarn_class_path_builder.append(line); 220 | } 221 | yarn_class_path = yarn_class_path_builder.toString(); 222 | } 223 | LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]"); 224 | proc.waitFor(); 225 | reader.close(); 226 | Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path); 227 | Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path,":"); 228 | String stormHomeInZip = Util.getStormHomeInZip(fs, zip, stormVersion.version()); 229 | Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/*",":"); 230 | Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/lib/*",":"); 231 | 232 | String java_home = (String) _stormConf.get("storm.yarn.java_home"); 233 | if (java_home == null) 234 | java_home = System.getenv("JAVA_HOME"); 235 | 236 | if (java_home != null && !java_home.isEmpty()) 237 | env.put("JAVA_HOME", java_home); 238 | LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]"); 239 | 240 | env.put("appJar", appMasterJar); 241 | env.put("appName", appName); 242 | env.put("appId", new Integer(_appId.getId()).toString()); 243 | env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); 244 | 245 | amContainer.setEnvironment(env); 246 | 247 | // Set the necessary command to execute the application master 248 | Vector vargs = new Vector(); 249 | if (java_home != null && !java_home.isEmpty()) 250 | vargs.add(env.get("JAVA_HOME") + "/bin/java"); 251 | else 252 | vargs.add("java"); 253 | vargs.add("-Dstorm.home=./storm/" + stormHomeInZip); 254 | vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log"); 255 | 256 | //vargs.add("-verbose:class"); 257 | vargs.add("com.yahoo.storm.yarn.MasterServer"); 258 | vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); 259 | vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); 260 | // Set java executable command 261 | LOG.info("Setting up app master command:" + vargs); 262 | 263 | amContainer.setCommands(vargs); 264 | 265 | // Set up resource type requirements 266 | // For now, only memory is supported so we set memory requirements 267 | Resource capability = Records.newRecord(Resource.class); 268 | capability.setMemory(amMB); 269 | appContext.setResource(capability); 270 | appContext.setAMContainerSpec(amContainer); 271 | 272 | _yarn.submitApplication(appContext); 273 | } 274 | 275 | 276 | /** 277 | * Wait until the application is successfully launched 278 | * 279 | * @throws YarnException 280 | */ 281 | public boolean waitUntilLaunched() throws YarnException, IOException { 282 | while (true) { 283 | 284 | // Check app status every 1 second. 285 | try { 286 | Thread.sleep(1000); 287 | } catch (InterruptedException e) { 288 | LOG.debug("Thread sleep in monitoring loop interrupted"); 289 | } 290 | 291 | // Get application report for the appId we are interested in 292 | ApplicationReport report = _yarn.getApplicationReport(_appId); 293 | YarnApplicationState state = report.getYarnApplicationState(); 294 | FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); 295 | if (YarnApplicationState.FINISHED == state) { 296 | if (FinalApplicationStatus.SUCCEEDED == dsStatus) { 297 | LOG.info("Application has completed successfully. Breaking monitoring loop"); 298 | return true; 299 | } else { 300 | LOG.info("Application did finished unsuccessfully." 301 | + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() 302 | + ". Breaking monitoring loop"); 303 | return false; 304 | } 305 | } else if (YarnApplicationState.KILLED == state 306 | || YarnApplicationState.FAILED == state) { 307 | LOG.info("Application did not finish." 308 | + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() 309 | + ". Breaking monitoring loop"); 310 | return false; 311 | } 312 | 313 | //announce application master's host and port 314 | if (state == YarnApplicationState.RUNNING) { 315 | return true; 316 | } 317 | } 318 | } 319 | 320 | 321 | /** 322 | * Find a jar that contains a class of the same name, if any. 323 | * It will return a jar file, even if that is not the first thing 324 | * on the class path that has a class with the same name. 325 | * 326 | * @param my_class the class to find. 327 | * @return a jar file that contains the class, or null. 328 | * @throws IOException on any error 329 | */ 330 | public static String findContainingJar(Class my_class) throws IOException { 331 | ClassLoader loader = my_class.getClassLoader(); 332 | String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; 333 | for (Enumeration itr = loader.getResources(class_file); 334 | itr.hasMoreElements(); ) { 335 | URL url = itr.nextElement(); 336 | if ("jar".equals(url.getProtocol())) { 337 | String toReturn = url.getPath(); 338 | if (toReturn.startsWith("file:")) { 339 | toReturn = toReturn.substring("file:".length()); 340 | } 341 | // URLDecoder is a misnamed class, since it actually decodes 342 | // x-www-form-urlencoded MIME type rather than actual 343 | // URL encoding (which the file path has). Therefore it would 344 | // decode +s to ' 's which is incorrect (spaces are actually 345 | // either unencoded or encoded as "%20"). Replace +s first, so 346 | // that they are kept sacred during the decoding process. 347 | toReturn = toReturn.replaceAll("\\+", "%2B"); 348 | toReturn = URLDecoder.decode(toReturn, "UTF-8"); 349 | return toReturn.replaceAll("!.*$", ""); 350 | } 351 | } 352 | 353 | throw new IOException("Fail to locate a JAR for class: " + my_class.getName()); 354 | } 355 | 356 | public static StormOnYarn launchApplication(String appName, String queue, 357 | int amMB, @SuppressWarnings("rawtypes") Map stormConf, String storm_zip_location) throws Exception { 358 | StormOnYarn storm = new StormOnYarn(stormConf); 359 | storm.launchApp(appName, queue, amMB, storm_zip_location); 360 | return storm; 361 | } 362 | 363 | public static StormOnYarn attachToApp(String appId, 364 | @SuppressWarnings("rawtypes") Map stormConf) { 365 | return new StormOnYarn(ConverterUtils.toApplicationId(appId), stormConf); 366 | } 367 | } 368 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/Util.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.google.common.base.Joiner; 20 | import org.apache.hadoop.fs.*; 21 | import org.apache.hadoop.fs.FileSystem; 22 | import org.apache.hadoop.fs.permission.FsAction; 23 | import org.apache.hadoop.fs.permission.FsPermission; 24 | import org.apache.hadoop.yarn.api.ApplicationConstants; 25 | import org.apache.hadoop.yarn.api.records.LocalResource; 26 | import org.apache.hadoop.yarn.api.records.LocalResourceType; 27 | import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; 28 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 29 | import org.apache.hadoop.yarn.util.ConverterUtils; 30 | import org.apache.hadoop.yarn.util.Records; 31 | import org.yaml.snakeyaml.Yaml; 32 | 33 | import java.io.*; 34 | import java.net.InetAddress; 35 | import java.net.URL; 36 | import java.util.*; 37 | import java.util.jar.JarEntry; 38 | import java.util.zip.ZipEntry; 39 | import java.util.zip.ZipInputStream; 40 | 41 | class Util { 42 | private static final String STORM_CONF_PATH_STRING = "conf" + Path.SEPARATOR + "storm.yaml"; 43 | 44 | static String getStormHome() { 45 | String ret = System.getProperty("storm.home"); 46 | if (ret == null) { 47 | throw new RuntimeException("storm.home is not set"); 48 | } 49 | return ret; 50 | } 51 | 52 | @SuppressWarnings("rawtypes") 53 | static Version getStormVersion() throws IOException { 54 | 55 | String versionNumber = "Unknown"; 56 | File releaseFile = new File(getStormHome(), "RELEASE"); 57 | if (releaseFile.exists()) { 58 | BufferedReader reader = new BufferedReader(new FileReader(releaseFile)); 59 | try { 60 | versionNumber = reader.readLine().trim(); 61 | } finally { 62 | reader.close(); 63 | } 64 | } 65 | 66 | File buildFile = new File(getStormHome(), "BUILD"); 67 | String buildNumber = null; 68 | if (buildFile.exists()) { 69 | BufferedReader reader = new BufferedReader(new FileReader(buildFile)); 70 | try { 71 | buildNumber = reader.readLine().trim(); 72 | } finally { 73 | reader.close(); 74 | } 75 | } 76 | 77 | Version version = new Version(versionNumber, buildNumber); 78 | return version; 79 | } 80 | 81 | static String getStormHomeInZip(FileSystem fs, Path zip, String stormVersion) throws IOException, RuntimeException { 82 | FSDataInputStream fsInputStream = fs.open(zip); 83 | ZipInputStream zipInputStream = new ZipInputStream(fsInputStream); 84 | ZipEntry entry = zipInputStream.getNextEntry(); 85 | while (entry != null) { 86 | String entryName = entry.getName(); 87 | if (entryName.matches("^storm(-" + stormVersion + ")?/")) { 88 | fsInputStream.close(); 89 | return entryName.replace("/", ""); 90 | } 91 | entry = zipInputStream.getNextEntry(); 92 | } 93 | fsInputStream.close(); 94 | throw new RuntimeException("Can not find storm home entry in storm zip file."); 95 | } 96 | 97 | static LocalResource newYarnAppResource(FileSystem fs, Path path, 98 | LocalResourceType type, LocalResourceVisibility vis) throws IOException { 99 | Path qualified = fs.makeQualified(path); 100 | FileStatus status = fs.getFileStatus(qualified); 101 | LocalResource resource = Records.newRecord(LocalResource.class); 102 | resource.setType(type); 103 | resource.setVisibility(vis); 104 | resource.setResource(ConverterUtils.getYarnUrlFromPath(qualified)); 105 | resource.setTimestamp(status.getModificationTime()); 106 | resource.setSize(status.getLen()); 107 | return resource; 108 | } 109 | 110 | @SuppressWarnings("rawtypes") 111 | static void rmNulls(Map map) { 112 | Set s = map.entrySet(); 113 | Iterator it = s.iterator(); 114 | while (it.hasNext()) { 115 | Map.Entry m =(Map.Entry)it.next(); 116 | if (m.getValue() == null) 117 | it.remove(); 118 | } 119 | } 120 | 121 | @SuppressWarnings("rawtypes") 122 | static Path createConfigurationFileInFs(FileSystem fs, 123 | String appHome, Map stormConf, YarnConfiguration yarnConf) 124 | throws IOException { 125 | // dump stringwriter's content into FS conf/storm.yaml 126 | Path confDst = new Path(fs.getHomeDirectory(), 127 | appHome + Path.SEPARATOR + STORM_CONF_PATH_STRING); 128 | Path dirDst = confDst.getParent(); 129 | fs.mkdirs(dirDst); 130 | 131 | //storm.yaml 132 | FSDataOutputStream out = fs.create(confDst); 133 | Yaml yaml = new Yaml(); 134 | OutputStreamWriter writer = new OutputStreamWriter(out); 135 | rmNulls(stormConf); 136 | yaml.dump(stormConf, writer); 137 | writer.close(); 138 | out.close(); 139 | 140 | //yarn-site.xml 141 | Path yarn_site_xml = new Path(dirDst, "yarn-site.xml"); 142 | out = fs.create(yarn_site_xml); 143 | writer = new OutputStreamWriter(out); 144 | yarnConf.writeXml(writer); 145 | writer.close(); 146 | out.close(); 147 | 148 | //log4j2.xml 149 | Path log4j2_xml = new Path(dirDst, "log4j2.xml"); 150 | out = fs.create(log4j2_xml); 151 | CreateLog4j2XML(out); 152 | out.close(); 153 | 154 | return dirDst; 155 | } 156 | 157 | static LocalResource newYarnAppResource(FileSystem fs, Path path) 158 | throws IOException { 159 | return Util.newYarnAppResource(fs, path, LocalResourceType.FILE, 160 | LocalResourceVisibility.APPLICATION); 161 | } 162 | 163 | private static void CreateLog4j2XML(OutputStream out) throws IOException { 164 | Enumeration log4j2_xml_urls; 165 | log4j2_xml_urls = Thread.currentThread().getContextClassLoader().getResources("log4j2.xml"); 166 | while (log4j2_xml_urls.hasMoreElements()) { 167 | URL log4j2_xml_url = log4j2_xml_urls.nextElement(); 168 | //Case 1: log4j2 as simple file 169 | if (log4j2_xml_url.getProtocol().equals("file")) { 170 | FileInputStream is = new FileInputStream(log4j2_xml_url.getPath()); 171 | while (is.available() > 0) { 172 | out.write(is.read()); 173 | } 174 | is.close(); 175 | return; 176 | } 177 | if (log4j2_xml_url.getProtocol().equals("jar")) { 178 | //Case 2: log4j2.xml included in a JAR 179 | String path = log4j2_xml_url.getPath(); 180 | String jarFile = path.substring("file:".length(), path.indexOf("!")); 181 | java.util.jar.JarFile jar = new java.util.jar.JarFile(jarFile); 182 | Enumeration enums = jar.entries(); 183 | while (enums.hasMoreElements()) { 184 | JarEntry file = enums.nextElement(); 185 | if (!file.isDirectory() && file.getName().equals("log4j2.xml")) { 186 | InputStream is = jar.getInputStream(file); // get the input stream 187 | while (is.available() > 0) { 188 | out.write(is.read()); 189 | } 190 | is.close(); 191 | jar.close(); 192 | return; 193 | } 194 | } 195 | jar.close(); 196 | } 197 | } 198 | throw new IOException("Failed to locate a log4j2.xml"); 199 | } 200 | 201 | @SuppressWarnings("rawtypes") 202 | private static List buildCommandPrefix(Map conf, String childOptsKey) 203 | throws IOException { 204 | String stormHomePath = getStormHome(); 205 | List toRet = new ArrayList(); 206 | if (System.getenv("JAVA_HOME") != null) 207 | toRet.add(System.getenv("JAVA_HOME") + "/bin/java"); 208 | else 209 | toRet.add("java"); 210 | toRet.add("-server"); 211 | toRet.add("-Dstorm.home=" + stormHomePath); 212 | toRet.add("-Djava.library.path=" + conf.get(org.apache.storm.Config.JAVA_LIBRARY_PATH)); 213 | toRet.add("-Dstorm.conf.file=" + new File(STORM_CONF_PATH_STRING).getName()); 214 | toRet.add("-cp"); 215 | toRet.add(buildClassPathArgument()); 216 | 217 | if (conf.containsKey(childOptsKey) 218 | && conf.get(childOptsKey) != null) { 219 | toRet.add((String) conf.get(childOptsKey)); 220 | } 221 | 222 | return toRet; 223 | } 224 | @SuppressWarnings("rawtypes") 225 | static List buildUICommands(Map conf) throws IOException { 226 | List toRet = 227 | buildCommandPrefix(conf, org.apache.storm.Config.UI_CHILDOPTS); 228 | final String host = InetAddress.getLocalHost().getHostName(); 229 | toRet.add("-Dstorm.options=" + org.apache.storm.Config.NIMBUS_SEEDS + "=[\""+host+"\"]"); 230 | toRet.add("-Dlogfile.name=" + System.getenv("STORM_LOG_DIR") + "/ui.log"); 231 | toRet.add("org.apache.storm.ui.core"); 232 | return toRet; 233 | } 234 | 235 | @SuppressWarnings("rawtypes") 236 | static List buildNimbusCommands(Map conf) throws IOException { 237 | List toRet = 238 | buildCommandPrefix(conf, org.apache.storm.Config.NIMBUS_CHILDOPTS); 239 | 240 | toRet.add("-Dlogfile.name=" + System.getenv("STORM_LOG_DIR") + "/nimbus.log"); 241 | toRet.add("org.apache.storm.daemon.nimbus"); 242 | 243 | return toRet; 244 | } 245 | 246 | @SuppressWarnings("rawtypes") 247 | static List buildSupervisorCommands(Map conf,String workingDirectory,String stormHomeInZip) throws IOException { 248 | List toRet = 249 | buildCommandPrefix(conf, org.apache.storm.Config.NIMBUS_CHILDOPTS); 250 | 251 | for(int i=0;i paths = new ArrayList(); 266 | paths.add(new File(STORM_CONF_PATH_STRING).getParent()); 267 | paths.add(getStormHome()); 268 | for (String jarPath : findAllJarsInPaths(getStormHome(), getStormHome() + File.separator + "lib")) { 269 | paths.add(jarPath); 270 | } 271 | return Joiner.on(File.pathSeparatorChar).join(paths); 272 | } 273 | 274 | private static interface FileVisitor { 275 | public void visit(File file); 276 | } 277 | 278 | private static List findAllJarsInPaths(String... pathStrs) { 279 | final LinkedHashSet pathSet = new LinkedHashSet(); 280 | 281 | FileVisitor visitor = new FileVisitor() { 282 | 283 | @Override 284 | public void visit(File file) { 285 | String name = file.getName(); 286 | if (name.endsWith(".jar")) { 287 | pathSet.add(file.getPath()); 288 | } 289 | } 290 | }; 291 | 292 | for (String path : pathStrs) { 293 | File file = new File(path); 294 | traverse(file, visitor); 295 | } 296 | 297 | final List toRet = new ArrayList(); 298 | for (String p : pathSet) { 299 | toRet.add(p); 300 | } 301 | return toRet; 302 | } 303 | 304 | private static void traverse(File file, FileVisitor visitor) { 305 | if (file.isDirectory()) { 306 | File childs[] = file.listFiles(); 307 | if (childs.length > 0) { 308 | for (int i = 0; i < childs.length; i++) { 309 | File child = childs[i]; 310 | traverse(child, visitor); 311 | } 312 | } 313 | } else { 314 | visitor.visit(file); 315 | } 316 | } 317 | 318 | static String getApplicationHomeForId(String id) { 319 | if (id.isEmpty()) { 320 | throw new IllegalArgumentException( 321 | "The ID of the application cannot be empty."); 322 | } 323 | return ".storm" + Path.SEPARATOR + id; 324 | } 325 | 326 | /** 327 | * Returns a boolean to denote whether a cache file is visible to all(public) 328 | * or not 329 | * @param fs Hadoop file system 330 | * @param path file path 331 | * @return true if the path is visible to all, false otherwise 332 | * @throws IOException 333 | */ 334 | static boolean isPublic(FileSystem fs, Path path) throws IOException { 335 | //the leaf level file should be readable by others 336 | if (!checkPermissionOfOther(fs, path, FsAction.READ)) { 337 | return false; 338 | } 339 | return ancestorsHaveExecutePermissions(fs, path.getParent()); 340 | } 341 | 342 | /** 343 | * Checks for a given path whether the Other permissions on it 344 | * imply the permission in the passed FsAction 345 | * @param fs 346 | * @param path 347 | * @param action 348 | * @return true if the path in the uri is visible to all, false otherwise 349 | * @throws IOException 350 | */ 351 | private static boolean checkPermissionOfOther(FileSystem fs, Path path, 352 | FsAction action) throws IOException { 353 | FileStatus status = fs.getFileStatus(path); 354 | FsPermission perms = status.getPermission(); 355 | FsAction otherAction = perms.getOtherAction(); 356 | if (otherAction.implies(action)) { 357 | return true; 358 | } 359 | return false; 360 | } 361 | 362 | /** 363 | * Returns true if all ancestors of the specified path have the 'execute' 364 | * permission set for all users (i.e. that other users can traverse 365 | * the directory hierarchy to the given path) 366 | */ 367 | static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path) throws IOException { 368 | Path current = path; 369 | while (current != null) { 370 | //the subdirs in the path should have execute permissions for others 371 | if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE)) { 372 | return false; 373 | } 374 | current = current.getParent(); 375 | } 376 | return true; 377 | } 378 | 379 | static void redirectStreamAsync(final InputStream input, final PrintStream output) { 380 | new Thread(new Runnable() { 381 | @Override 382 | public void run() { 383 | Scanner scanner = new Scanner(input); 384 | while (scanner.hasNextLine()) { 385 | output.println(scanner.nextLine()); 386 | } 387 | } 388 | }).start(); 389 | } 390 | } 391 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. See accompanying LICENSE file. 13 | */ 14 | package com.yahoo.storm.yarn; 15 | 16 | public class Version { 17 | 18 | private final String version; 19 | private final String build; 20 | 21 | public Version(String version, String build) { 22 | this.version = version; 23 | this.build = build; 24 | } 25 | 26 | public String version() { 27 | return this.version; 28 | } 29 | 30 | public String build() { 31 | return this.build; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | if (null == build || build.isEmpty()) { 37 | return version; 38 | } else { 39 | return version + "-" + build; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/storm/yarn/VersionCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. See accompanying LICENSE file. 13 | */ 14 | 15 | package com.yahoo.storm.yarn; 16 | 17 | import org.apache.commons.cli.CommandLine; 18 | import org.apache.commons.cli.Options; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | class VersionCommand implements Client.ClientCommand { 24 | private static final Logger LOG = LoggerFactory.getLogger(StormMasterCommand.class); 25 | 26 | VersionCommand() { 27 | } 28 | 29 | @Override 30 | public Options getOpts() { 31 | Options opts = new Options(); 32 | return opts; 33 | } 34 | 35 | @Override 36 | public String getHeaderDescription() { 37 | return "storm-yarn version"; 38 | } 39 | 40 | @Override 41 | public void process(CommandLine cl) throws Exception { 42 | Version version = Util.getStormVersion(); 43 | System.out.println(version.toString()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ${sys:logfile.name} 5 | 6 | 7 | 8 | 9 | %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/resources/master_defaults.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. See accompanying LICENSE file. 14 | # 15 | master.host: "localhost" 16 | master.thrift.port: 9000 17 | 18 | master.initial-num-supervisors: 1 19 | master.container.priority: 0 20 | master.container.size-mb: 4096 21 | master.heartbeat.interval.millis: 1000 22 | master.timeout.secs: 1000 23 | yarn.report.wait.millis: 10000 24 | nimbusui.startup.ms: 10000 25 | 26 | storm.thrift.transport: "org.apache.storm.security.auth.SimpleTransportPlugin" 27 | nimbus.thrift.port: 6627 28 | nimbus.queue.size: 100000 29 | nimbus.thrift.threads: 64 30 | nimbus.thrift.max_buffer_size: 1048576 31 | 32 | drpc.thrift.transport: "org.apache.storm.security.auth.SimpleTransportPlugin" 33 | drpc.port: 9000 34 | drpc.queue.size: 128 35 | drpc.worker.threads: 64 36 | drpc.max_buffer_size: 1048576 37 | 38 | ui.port: 7070 39 | 40 | storm.messaging.transport: "org.apache.storm.messaging.netty.Context" 41 | storm.messaging.netty.buffer_size: 1048576 42 | storm.messaging.netty.max_retries: 100 43 | storm.messaging.netty.min_wait_ms: 1000 44 | storm.messaging.netty.max_wait_ms: 5000 45 | 46 | # Configuration parameter that allows the launching machine to specify the JAVA_JOME 47 | # used when the application is executed on the YARN cluster. 48 | # 49 | #storm.yarn.java_home: "/usr/java/default" 50 | 51 | # Configuration parameter that allows the launching machine to specify the yarn classpath 52 | # used when the application is executed on the YARN cluster. To find this value, run 53 | # "yarn classpath" on the target machines. 54 | # 55 | #storm.yarn.yarn_classpath: "" -------------------------------------------------------------------------------- /src/main/storm_master.thrift: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/thrift --gen java:beans,nocamel,hashcode 2 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. See accompanying LICENSE file. 15 | # 16 | 17 | namespace java com.yahoo.storm.yarn.generated 18 | 19 | service StormMaster { 20 | // Storm configuration 21 | string getStormConf(); 22 | void setStormConf(1: string storm_conf); 23 | 24 | // supervisors 25 | void addSupervisors(1: i32 number); 26 | void removeSupervisors(1: string supervisor_host); 27 | 28 | // start/stop nimbus 29 | void startNimbus(); 30 | void stopNimbus(); 31 | 32 | // enable/disable ui 33 | void startUI(); 34 | void stopUI(); 35 | 36 | // enable/disable supervisors 37 | void startSupervisors(); 38 | void stopSupervisors(); 39 | 40 | // shutdown storm cluster 41 | void shutdown(); 42 | } 43 | 44 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/storm/yarn/EmbeddedZKServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | package com.yahoo.storm.yarn; 17 | 18 | import org.apache.commons.logging.Log; 19 | import org.apache.commons.logging.LogFactory; 20 | import org.apache.zookeeper.server.NIOServerCnxnFactory; 21 | import org.apache.zookeeper.server.ZooKeeperServer; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | import java.net.BindException; 26 | import java.net.InetSocketAddress; 27 | 28 | public class EmbeddedZKServer { 29 | 30 | private static final Log LOG = LogFactory.getLog(EmbeddedZKServer.class); 31 | private NIOServerCnxnFactory zkFactory; 32 | private int zkport; 33 | 34 | void start() throws IOException, InterruptedException { 35 | LOG.info("Starting up embedded Zookeeper server"); 36 | File localfile = new File("./target/zookeeper.data"); 37 | ZooKeeperServer zkServer; 38 | zkServer = new ZooKeeperServer(localfile, localfile, 2000); 39 | for (zkport = 60000; true; zkport++) 40 | try { 41 | zkFactory = new NIOServerCnxnFactory(); 42 | zkFactory.configure(new InetSocketAddress(zkport),60); 43 | break; 44 | } catch (BindException e) { 45 | if (zkport == 65535) throw new IOException("Fail to find a port for Zookeeper server to bind"); 46 | } 47 | LOG.info("Zookeeper port allocated:"+zkport); 48 | zkFactory.startup(zkServer); 49 | } 50 | 51 | int port() { 52 | return zkport; 53 | } 54 | 55 | void stop() { 56 | LOG.info("shutdown embedded zookeeper server with port "+zkport); 57 | zkFactory.shutdown(); 58 | zkFactory = null; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/storm/yarn/TestConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | package com.yahoo.storm.yarn; 17 | 18 | import java.io.BufferedInputStream; 19 | import java.io.File; 20 | import java.io.FileInputStream; 21 | import java.io.FileNotFoundException; 22 | import java.io.FileOutputStream; 23 | import java.io.FileWriter; 24 | import java.io.IOException; 25 | import java.util.Map; 26 | import java.util.zip.ZipEntry; 27 | import java.util.zip.ZipInputStream; 28 | 29 | import org.apache.commons.logging.Log; 30 | import org.apache.commons.logging.LogFactory; 31 | import org.apache.hadoop.conf.Configuration; 32 | import org.apache.hadoop.fs.Path; 33 | import org.yaml.snakeyaml.Yaml; 34 | 35 | public class TestConfig { 36 | private static final Log LOG = LogFactory.getLog(TestConfig.class); 37 | 38 | private File yarn_site_xml; 39 | private File storm_conf_file; 40 | private String storm_home = null; 41 | 42 | synchronized File createYarnSiteConfig(Configuration yarn_conf) throws IOException { 43 | yarn_site_xml = new File("./target/conf/yarn-site.xml"); 44 | yarn_site_xml.getParentFile().mkdirs(); 45 | FileWriter writer = new FileWriter(yarn_site_xml); 46 | yarn_conf.writeXml(writer); 47 | writer.flush(); 48 | writer.close(); 49 | return yarn_site_xml; 50 | } 51 | 52 | @SuppressWarnings("rawtypes") 53 | synchronized File createConfigFile(Map storm_conf) throws IOException { 54 | storm_conf_file = new File("./conf/storm.yaml"); 55 | storm_conf_file.getParentFile().mkdirs(); 56 | Yaml yaml = new Yaml(); 57 | FileWriter writer = new FileWriter(storm_conf_file); 58 | Util.rmNulls(storm_conf); 59 | yaml.dump(storm_conf, writer); 60 | writer.flush(); 61 | writer.close(); 62 | return storm_conf_file; 63 | } 64 | 65 | void cleanup() { 66 | if (storm_conf_file != null) 67 | deleteFolder(storm_conf_file.getParentFile()); 68 | if (storm_home != null) 69 | deleteFolder(new File(storm_home)); 70 | } 71 | 72 | synchronized String stormHomePath() throws IOException { 73 | unzipFile("./lib/storm.zip"); 74 | Runtime.getRuntime().exec( "chmod +x "+storm_home+"/bin/storm"); 75 | System.setProperty("storm.home", storm_home); 76 | return storm_home; 77 | } 78 | 79 | private void unzipFile(String filePath){ 80 | FileInputStream fis = null; 81 | ZipInputStream zipIs = null; 82 | ZipEntry zEntry = null; 83 | try { 84 | fis = new FileInputStream(filePath); 85 | zipIs = new ZipInputStream(new BufferedInputStream(fis)); 86 | while((zEntry = zipIs.getNextEntry()) != null){ 87 | try{ 88 | byte[] tmp = new byte[4*1024]; 89 | FileOutputStream fos = null; 90 | String opFilePath = "lib/"+zEntry.getName(); 91 | if (zEntry.isDirectory()) { 92 | LOG.debug("Create a folder "+opFilePath); 93 | if (zEntry.getName().indexOf(Path.SEPARATOR)==(zEntry.getName().length()-1)) 94 | storm_home = opFilePath.substring(0, opFilePath.length()-1); 95 | new File(opFilePath).mkdir(); 96 | } else { 97 | LOG.debug("Extracting file to "+opFilePath); 98 | fos = new FileOutputStream(opFilePath); 99 | int size = 0; 100 | while((size = zipIs.read(tmp)) != -1){ 101 | fos.write(tmp, 0 , size); 102 | } 103 | fos.flush(); 104 | fos.close(); 105 | } 106 | } catch(Exception ex){ 107 | ex.printStackTrace(); 108 | } 109 | } 110 | zipIs.close(); 111 | } catch (FileNotFoundException e) { 112 | LOG.warn(e.toString()); 113 | } catch (IOException e) { 114 | LOG.warn(e.toString()); 115 | } 116 | LOG.info("storm_home: "+storm_home); 117 | } 118 | 119 | private void deleteFolder(File file) { 120 | if (file.isDirectory()) { 121 | //directory is empty, then delete it 122 | if(file.list().length==0){ 123 | file.delete(); 124 | } else { 125 | //list all the directory contents 126 | String files[] = file.list(); 127 | for (String temp : files) { 128 | //construct the file structure 129 | File fileDelete = new File(file, temp); 130 | //recursive delete 131 | deleteFolder(fileDelete); 132 | } 133 | 134 | //check the directory again, if empty then delete it 135 | if(file.list().length==0){ 136 | file.delete(); 137 | } 138 | } 139 | } else { 140 | //if file, then delete it 141 | file.delete(); 142 | } 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/storm/yarn/TestIntegration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | package com.yahoo.storm.yarn; 17 | 18 | import org.apache.storm.generated.ClusterSummary; 19 | import org.apache.storm.generated.Nimbus; 20 | import org.apache.storm.generated.TopologySummary; 21 | import org.apache.storm.utils.NimbusClient; 22 | import org.apache.storm.utils.Utils; 23 | import com.google.common.base.Joiner; 24 | import org.apache.commons.io.FileUtils; 25 | import org.apache.hadoop.conf.Configuration; 26 | import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; 27 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 28 | import org.apache.hadoop.yarn.server.MiniYARNCluster; 29 | import org.junit.AfterClass; 30 | import org.junit.Assert; 31 | import org.junit.BeforeClass; 32 | import org.junit.Test; 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | import java.io.BufferedReader; 37 | import java.io.File; 38 | import java.io.FileReader; 39 | import java.io.IOException; 40 | import java.net.URL; 41 | import java.util.List; 42 | import java.util.Map; 43 | 44 | public class TestIntegration { 45 | private static final Logger LOG = LoggerFactory.getLogger(TestIntegration.class); 46 | private static final String STORM_YARN_CMD = "bin"+File.separator+"storm-yarn"; 47 | private static MiniYARNCluster yarnCluster = null; 48 | private static String appId; 49 | private static EmbeddedZKServer zkServer; 50 | private static File storm_conf_file; 51 | private static File yarn_site_xml; 52 | private static String storm_home; 53 | private static TestConfig testConf = new TestConfig(); 54 | 55 | @SuppressWarnings({ "rawtypes", "unchecked" }) 56 | @BeforeClass 57 | public static void setup() { 58 | try { 59 | zkServer = new EmbeddedZKServer(); 60 | zkServer.start(); 61 | LOG.info("Starting up MiniYARN cluster"); 62 | if (yarnCluster == null) { 63 | yarnCluster = new MiniYARNCluster(TestIntegration.class.getName(), 2, 1, 1); 64 | Configuration conf = new YarnConfiguration(); 65 | conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); 66 | conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 2*1024); 67 | yarnCluster.init(conf); 68 | yarnCluster.start(); 69 | } 70 | sleep(2000); 71 | 72 | Configuration miniyarn_conf = yarnCluster.getConfig(); 73 | yarn_site_xml = testConf.createYarnSiteConfig(miniyarn_conf); 74 | 75 | storm_home = testConf.stormHomePath(); 76 | LOG.info("Will be using storm found on PATH at "+storm_home); 77 | 78 | //create a storm configuration file with zkport 79 | final Map storm_conf = Config.readStormConfig(); 80 | storm_conf.put(org.apache.storm.Config.STORM_ZOOKEEPER_PORT, zkServer.port()); 81 | storm_conf_file = testConf.createConfigFile(storm_conf); 82 | 83 | List cmd = java.util.Arrays.asList("bin/storm-yarn", 84 | "launch", 85 | storm_conf_file.toString(), 86 | "--stormZip", 87 | "lib/storm.zip", 88 | "--appname", 89 | "storm-on-yarn-test", 90 | "--output", 91 | "target/appId.txt"); 92 | execute(cmd); 93 | 94 | //wait for Storm cluster to be fully luanched 95 | sleep(15000); 96 | 97 | BufferedReader reader = new BufferedReader(new FileReader ("target/appId.txt")); 98 | appId = reader.readLine(); 99 | reader.close(); 100 | if (appId!=null) appId = appId.trim(); 101 | LOG.info("application ID:"+appId); 102 | } catch (Exception ex) { 103 | LOG.error("setup failure", ex); 104 | Assert.assertEquals(null, ex); 105 | } 106 | } 107 | 108 | private static void sleep(int i) { 109 | try { 110 | Thread.sleep(i); 111 | } catch (InterruptedException e) { 112 | LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); 113 | } 114 | } 115 | 116 | @Test 117 | public void performActions() throws Exception { 118 | try { 119 | List cmd = java.util.Arrays.asList(STORM_YARN_CMD, 120 | "getStormConfig", 121 | storm_conf_file.toString(), 122 | "--appId", 123 | appId, 124 | "--output", 125 | storm_home+"/storm.yaml"); 126 | execute(cmd); 127 | sleep(1000); 128 | 129 | cmd = java.util.Arrays.asList(storm_home+"/bin/storm", 130 | "jar", 131 | "lib/storm-starter-topologies-1.0.1.jar", 132 | "org.apache.storm.starter.ExclamationTopology", 133 | "exclamation-topology"); 134 | execute(cmd); 135 | sleep(30000); 136 | 137 | if (new File(storm_home+"/storm.yaml").exists()) { 138 | Map storm_conf = Config.readStormConfig(storm_home+"/storm.yaml"); 139 | Nimbus.Client nimbus_client = NimbusClient.getConfiguredClient(storm_conf).getClient(); 140 | ClusterSummary cluster_summary = nimbus_client.getClusterInfo(); 141 | TopologySummary topology_summary = cluster_summary.get_topologies().get(0); 142 | Assert.assertEquals("ACTIVE", topology_summary.get_status()); 143 | } 144 | 145 | cmd = java.util.Arrays.asList(storm_home+"/bin/storm", 146 | "kill", 147 | "exclamation-topology"); 148 | execute(cmd); 149 | sleep(1000); 150 | 151 | cmd = java.util.Arrays.asList(STORM_YARN_CMD, 152 | "stopNimbus", 153 | storm_conf_file.toString(), 154 | "--appId", 155 | appId); 156 | execute(cmd); 157 | sleep(1000); 158 | 159 | cmd = java.util.Arrays.asList(STORM_YARN_CMD, 160 | "startNimbus", 161 | storm_conf_file.toString(), 162 | "--appId", 163 | appId); 164 | execute(cmd); 165 | sleep(1000); 166 | 167 | cmd = java.util.Arrays.asList(STORM_YARN_CMD, 168 | "stopUI", 169 | storm_conf_file.toString(), 170 | "--appId", 171 | appId); 172 | execute(cmd); 173 | sleep(1000); 174 | 175 | cmd = java.util.Arrays.asList(STORM_YARN_CMD, 176 | "startUI", 177 | storm_conf_file.toString(), 178 | "--appId", 179 | appId); 180 | execute(cmd); 181 | sleep(1000); 182 | } catch (Exception ex) { 183 | LOG.warn("Exception in Integration Test", ex); 184 | Assert.assertEquals(null, ex); 185 | } 186 | } 187 | 188 | @AfterClass 189 | public static void tearDown() { 190 | try { 191 | //shutdown Storm Cluster 192 | List cmd = java.util.Arrays.asList(STORM_YARN_CMD, 193 | "shutdown", 194 | storm_conf_file.toString(), 195 | "--appId", 196 | appId); 197 | execute(cmd); 198 | sleep(1000); 199 | } catch (Exception ex) { 200 | LOG.info(ex.toString()); 201 | } 202 | 203 | //shutdown Zookeeper server 204 | if (zkServer != null) { 205 | zkServer.stop(); 206 | zkServer = null; 207 | } 208 | 209 | //shutdown YARN cluster 210 | if (yarnCluster != null) { 211 | LOG.info("shutdown MiniYarn cluster"); 212 | yarnCluster.stop(); 213 | yarnCluster = null; 214 | } 215 | sleep(1000); 216 | 217 | //remove configuration file 218 | testConf.cleanup(); 219 | } 220 | 221 | @SuppressWarnings({ "rawtypes", "unchecked" }) 222 | private static int execute(List cmd) throws InterruptedException, IOException { 223 | LOG.info(Joiner.on(" ").join(cmd)); 224 | ProcessBuilder pb = new ProcessBuilder(cmd); 225 | Map env = pb.environment(); 226 | env.putAll(System.getenv()); 227 | env.put(Environment.PATH.name(), "bin:"+storm_home+File.separator+"bin:"+env.get(Environment.PATH.name())); 228 | String yarn_conf_dir = yarn_site_xml.getParent().toString(); 229 | env.put("STORM_YARN_CONF_DIR", yarn_conf_dir); 230 | List log4j2_xmls = Utils.findResources("log4j2.xml"); 231 | if (log4j2_xmls != null && log4j2_xmls.size()>=1) { 232 | String log4j2_xml = log4j2_xmls.get(0).getFile(); 233 | LOG.debug("log4j2_xml:"+yarn_conf_dir+File.separator+"log4j2.xml"); 234 | FileUtils.copyFile(new File(log4j2_xml), new File(yarn_conf_dir+File.separator+"log4j2.xml")); 235 | } 236 | List log4j_properties = Utils.findResources("log4j.properties"); 237 | if (log4j_properties != null && log4j_properties.size()>=1) { 238 | String log4j_properties_file = log4j_properties.get(0).getFile(); 239 | LOG.debug("log4j_properties_file:"+yarn_conf_dir+File.separator+"log4j.properties"); 240 | FileUtils.copyFile(new File(log4j_properties_file), new File(yarn_conf_dir+File.separator+"log4j.properties")); 241 | } 242 | 243 | Process proc = pb.start(); 244 | Util.redirectStreamAsync(proc.getInputStream(), System.out); 245 | Util.redirectStreamAsync(proc.getErrorStream(), System.err); 246 | int status = proc.waitFor(); 247 | return status; 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/storm/yarn/TestStormCluster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | 17 | package com.yahoo.storm.yarn; 18 | 19 | import com.yahoo.storm.yarn.generated.StormMaster; 20 | import junit.framework.Assert; 21 | import org.junit.AfterClass; 22 | import org.junit.BeforeClass; 23 | import org.junit.Test; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import java.io.File; 28 | import java.io.IOException; 29 | import java.net.Socket; 30 | import java.net.URL; 31 | import java.net.URLConnection; 32 | import java.net.UnknownHostException; 33 | import java.util.Map; 34 | 35 | import static org.mockito.Mockito.mock; 36 | 37 | public class TestStormCluster { 38 | static final Logger LOG = LoggerFactory.getLogger(TestStormCluster.class); 39 | 40 | private static EmbeddedZKServer zkServer; 41 | private static MasterServer server = null; 42 | private static MasterClient client = null; 43 | private static File storm_conf_file = null; 44 | private static TestConfig testConf = new TestConfig(); 45 | 46 | @SuppressWarnings({ "unchecked", "rawtypes" }) 47 | @BeforeClass 48 | public static void setup() throws InterruptedException, IOException { 49 | //start embedded ZK server 50 | zkServer = new EmbeddedZKServer(); 51 | zkServer.start(); 52 | 53 | String storm_home = testConf.stormHomePath(); 54 | if (storm_home == null) { 55 | throw new RuntimeException("Storm home was not found." 56 | + " Make sure to include storm in the PATH."); 57 | } 58 | LOG.info("Will be using storm found on PATH at "+storm_home); 59 | 60 | //simple configuration 61 | final Map storm_conf = Config.readStormConfig("src/main/resources/master_defaults.yaml"); 62 | storm_conf.put(org.apache.storm.Config.STORM_ZOOKEEPER_PORT, zkServer.port()); 63 | storm_conf_file = testConf.createConfigFile(storm_conf); 64 | 65 | confirmNothingIsRunning(storm_conf); 66 | 67 | StormAMRMClient mockClient = mock(StormAMRMClient.class); 68 | server = new MasterServer(storm_conf, mockClient); 69 | 70 | //launch server 71 | new Thread(new Runnable() { 72 | @Override 73 | public void run() { 74 | try { 75 | server.serve(); 76 | } catch (Exception ex) { 77 | tearDown(); 78 | } 79 | } 80 | }).start(); 81 | 82 | LOG.info("Sleep to wait for the server to startup"); 83 | final int timeoutSecs = 40; 84 | for (int elapsedSecs=0; elapsedSecs < timeoutSecs; elapsedSecs++) { 85 | Thread.sleep(1000); 86 | LOG.info("Slept " + elapsedSecs + " of " + timeoutSecs + "s."); 87 | try { 88 | checkZkConnection(storm_conf); 89 | } catch (IOException e) { 90 | LOG.warn("Could not connect to zookeeper server"); 91 | continue; 92 | } 93 | try { 94 | checkNimbusConnection(storm_conf); 95 | } catch (IOException e) { 96 | LOG.warn("Still cannot connect to nimbus server."); 97 | continue; 98 | } 99 | try { 100 | checkUiConnection(storm_conf); 101 | } catch (IOException e) { 102 | LOG.warn("Still cannot connect to UI server."); 103 | continue; 104 | } 105 | 106 | // The server appears to be up. Launch the client. 107 | client = MasterClient.getConfiguredClient(storm_conf); 108 | LOG.info("Connected to master to get client"); 109 | 110 | return; 111 | } 112 | 113 | throw new RuntimeException("Failed to connect to nimbus server in " 114 | + timeoutSecs + "seconds."); 115 | } 116 | 117 | private static void checkNimbusConnection( 118 | @SuppressWarnings("rawtypes") final Map storm_conf) 119 | throws IOException, UnknownHostException { 120 | // Try to open a TCP connection to the nimbus port. 121 | new Socket((String) storm_conf.get(Config.MASTER_HOST), 122 | (Integer) storm_conf 123 | .get(org.apache.storm.Config.NIMBUS_THRIFT_PORT)) 124 | .close(); 125 | } 126 | 127 | private static void checkZkConnection( 128 | @SuppressWarnings("rawtypes") final Map storm_conf) 129 | throws IOException, UnknownHostException { 130 | // Try to open a TCP connection to the zookeeper ports 131 | new Socket("localhost", 132 | (Integer) storm_conf 133 | .get(org.apache.storm.Config.STORM_ZOOKEEPER_PORT)) 134 | .close(); 135 | } 136 | 137 | private static void checkUiConnection(Map storm_conf) 138 | throws IOException, UnknownHostException { 139 | // Try to open a TCP connection to the UI port. 140 | new Socket((String) storm_conf.get(Config.MASTER_HOST), 141 | (Integer) storm_conf 142 | .get(org.apache.storm.Config.UI_PORT)) 143 | .close(); 144 | 145 | } 146 | 147 | private static void confirmNothingIsRunning(Map storm_conf) { 148 | try { 149 | checkNimbusConnection(storm_conf); 150 | throw new RuntimeException("Nimbus server already running."); 151 | } catch (IOException e) { 152 | LOG.info("OK: Nimbus does not seem to be running."); 153 | } 154 | 155 | try { 156 | checkUiConnection(storm_conf); 157 | throw new RuntimeException("UI server already running."); 158 | } catch (IOException e) { 159 | LOG.info("OK: UI does not seem to be running."); 160 | } 161 | 162 | } 163 | 164 | @AfterClass 165 | public static void tearDown() { 166 | //stop client 167 | if (client != null) { 168 | StormMaster.Client master_client = client.getClient(); 169 | try { 170 | master_client.stopSupervisors(); 171 | master_client.stopNimbus(); 172 | master_client.stopUI(); 173 | } catch (Exception e) { 174 | LOG.info("failure in tearn down:"+e.toString()); 175 | } 176 | client.close(); 177 | client = null; 178 | } 179 | 180 | //stop server 181 | if (server != null) { 182 | server.stop(); 183 | server = null; 184 | } 185 | 186 | //remove configuration file 187 | testConf.cleanup(); 188 | 189 | //shutdown Zookeeper server 190 | if (zkServer != null) { 191 | zkServer.stop(); 192 | zkServer = null; 193 | } 194 | } 195 | 196 | @Test 197 | public void testUI() throws Exception { 198 | LOG.info("Testing UI"); 199 | @SuppressWarnings("rawtypes") 200 | final Map storm_conf = Config.readStormConfig(storm_conf_file.toString()); 201 | LOG.info("Testing connection to UI ..."); 202 | String host = (String) storm_conf.get("ui.host"); 203 | if (host==null) host = "localhost"; 204 | URL url = new URL("http://"+host+":"+storm_conf.get("ui.port")+"/"); 205 | LOG.info("UI URL:"+url); 206 | URLConnection con = url.openConnection(); 207 | Assert.assertNotNull(con); 208 | Assert.assertNotNull(con.getContent()); 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/storm/yarn/TestStormMaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. See accompanying LICENSE file. 15 | */ 16 | package com.yahoo.storm.yarn; 17 | 18 | import com.yahoo.storm.yarn.generated.StormMaster; 19 | import junit.framework.Assert; 20 | import org.apache.commons.logging.Log; 21 | import org.apache.commons.logging.LogFactory; 22 | import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; 23 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 24 | import org.apache.hadoop.yarn.util.Records; 25 | import org.junit.AfterClass; 26 | import org.junit.BeforeClass; 27 | import org.junit.Test; 28 | 29 | import java.io.IOException; 30 | import java.util.Map; 31 | 32 | public class TestStormMaster { 33 | 34 | private static final Log LOG = LogFactory.getLog(TestStormMaster.class); 35 | private static EmbeddedZKServer zkServer; 36 | private static MasterServer server = null; 37 | private static MasterClient client = null; 38 | private static TestConfig testConf = new TestConfig(); 39 | 40 | @SuppressWarnings({ "unchecked", "rawtypes" }) 41 | @BeforeClass 42 | public static void setup() throws InterruptedException, IOException { 43 | //start embedded ZK server 44 | zkServer = new EmbeddedZKServer(); 45 | zkServer.start(); 46 | 47 | //simple configuration 48 | final Map storm_conf = Config.readStormConfig("src/main/resources/master_defaults.yaml"); 49 | storm_conf.put(org.apache.storm.Config.STORM_ZOOKEEPER_PORT, zkServer.port()); 50 | 51 | String storm_home = testConf.stormHomePath(); 52 | if (storm_home == null) { 53 | throw new RuntimeException("Storm home was not found." 54 | + " Make sure to include storm in the PATH."); 55 | } 56 | LOG.info("Will be using storm found on PATH at "+storm_home); 57 | 58 | final YarnConfiguration hadoopConf = new YarnConfiguration(); 59 | ApplicationAttemptId appAttemptId = Records.newRecord(ApplicationAttemptId.class); 60 | 61 | StormAMRMClient client = new StormAMRMClient(appAttemptId, storm_conf, hadoopConf); 62 | 63 | LOG.info("Storm server attaching to port: "+ storm_conf.get(Config.MASTER_THRIFT_PORT)); 64 | //launch server 65 | server = new MasterServer(storm_conf, client); 66 | new Thread(new Runnable() { 67 | @Override 68 | public void run() { 69 | try { 70 | server.serve(); 71 | } catch (Exception e) { 72 | LOG.error(e); 73 | } 74 | } 75 | }).start(); 76 | while (!server.isServing()) 77 | Thread.sleep(10); 78 | LOG.info("Storm server started at port: "+ storm_conf.get(Config.MASTER_THRIFT_PORT)); 79 | 80 | //launch client 81 | TestStormMaster.client = MasterClient.getConfiguredClient(storm_conf); 82 | } 83 | 84 | @AfterClass 85 | public static void tearDown() throws IOException { 86 | //stop client 87 | if (client != null) { 88 | client.close(); 89 | client = null; 90 | } 91 | 92 | //stop server 93 | if (server != null) { 94 | server.stop(); 95 | server = null; 96 | } 97 | 98 | //remove configuration file 99 | testConf.cleanup(); 100 | 101 | //shutdown Zookeeper server 102 | if (zkServer != null) { 103 | zkServer.stop(); 104 | zkServer = null; 105 | } 106 | } 107 | 108 | @Test 109 | public void test1() throws Exception { 110 | StormMaster.Client master_client = client.getClient(); 111 | 112 | LOG.info("getStormConf"); 113 | Assert.assertNotNull(master_client.getStormConf()); 114 | 115 | LOG.info("addSupervisors(1)"); 116 | master_client.addSupervisors(1); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. See accompanying LICENSE file. 14 | 15 | log4j.rootLogger=warn,stdout 16 | log4j.threshhold=ALL 17 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 18 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 19 | -------------------------------------------------------------------------------- /src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | --------------------------------------------------------------------------------