├── .gitignore ├── .dockerignore ├── vagrant ├── submit-topology.sh ├── start-nimbus.sh ├── start-mesos-master-and-slave.sh └── start-mesos-slave.sh ├── storm-shim ├── src │ └── main │ │ └── java │ │ └── storm │ │ └── mesos │ │ └── shims │ │ ├── ILocalStateShim.java │ │ └── ICommandLineShim.java └── pom.xml ├── bin ├── storm-mesos ├── run-with-marathon.sh └── build-release.sh ├── storm ├── src │ ├── main │ │ └── storm │ │ │ └── mesos │ │ │ ├── resources │ │ │ ├── ReservationType.java │ │ │ ├── ResourceNotAvailableException.java │ │ │ ├── ResourceEntry.java │ │ │ ├── DefaultReservationTypeComparator.java │ │ │ ├── ResourceType.java │ │ │ ├── Resource.java │ │ │ ├── ResourceEntries.java │ │ │ ├── ScalarResource.java │ │ │ ├── AggregatedOffers.java │ │ │ └── RangeResource.java │ │ │ ├── shims │ │ │ └── CommandLineShimFactory.java │ │ │ ├── ResourceRoleComparator.java │ │ │ ├── schedulers │ │ │ ├── MesosWorkerSlot.java │ │ │ ├── IMesosStormScheduler.java │ │ │ └── SchedulerUtils.java │ │ │ ├── LaunchTask.java │ │ │ ├── LocalFileServer.java │ │ │ ├── util │ │ │ ├── ZKClient.java │ │ │ ├── PrettyProtobuf.java │ │ │ └── MesosCommon.java │ │ │ ├── TaskAssignments.java │ │ │ └── NimbusMesosScheduler.java │ └── test │ │ └── storm │ │ └── mesos │ │ ├── OfferRoleComparatorTest.java │ │ ├── shims │ │ └── CommandLineShimTest.java │ │ ├── schedulers │ │ └── SchedulerUtilsTest.java │ │ ├── LaunchTaskTest.java │ │ ├── resources │ │ ├── AggregatedOffersTest.java │ │ └── ResourceEntryTest.java │ │ └── ZKClientTest.java └── pom.xml ├── storm-shim-1x ├── src │ └── main │ │ └── java │ │ └── storm │ │ └── mesos │ │ └── shims │ │ ├── CommandLineShim.java │ │ ├── DockerCommandLineShim.java │ │ └── LocalStateShim.java └── pom.xml ├── .travis.yml ├── storm.yaml ├── Makefile ├── Dockerfile ├── onbuild └── Dockerfile ├── Vagrantfile ├── pom.xml ├── docs └── vagrant.md ├── config └── checkstyle.xml └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vimrc 3 | target 4 | *.swp 5 | _release 6 | *.iml 7 | *.zip 8 | *.tgz 9 | dependency-reduced-pom.xml 10 | /build/ 11 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | !storm-mesos-*.tgz 2 | .idea 3 | .vimrc 4 | *.swp 5 | _release 6 | build 7 | target 8 | *.iml 9 | *.zip 10 | *.tgz 11 | *.tar.gz 12 | dependency-reduced-pom.xml 13 | -------------------------------------------------------------------------------- /vagrant/submit-topology.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RELEASE=`grep -1 -A 0 -B 0 '' /vagrant/pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` 4 | 5 | cd /vagrant/_release/storm-mesos-${RELEASE}-* 6 | bin/storm jar /vagrant/vagrant/$* 7 | -------------------------------------------------------------------------------- /storm-shim/src/main/java/storm/mesos/shims/ILocalStateShim.java: -------------------------------------------------------------------------------- 1 | package storm.mesos.shims; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Wraps LocalState 7 | */ 8 | public interface ILocalStateShim { 9 | 10 | public String get(String key); 11 | 12 | public void put(String key, String value); 13 | 14 | public void remove(String key); 15 | 16 | public Map snapshot(); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /bin/storm-mesos: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys 4 | import os 5 | import subprocess 6 | 7 | STORM_PATH = "/".join(os.path.abspath(__file__).split("/")[0:-1]) 8 | STORM_CMD = STORM_PATH + "/storm" 9 | 10 | 11 | def nimbus(*args): 12 | os.chdir(STORM_PATH + "/..") 13 | args = [STORM_CMD, "nimbus", "storm.mesos.MesosNimbus"] + list(args) 14 | os.execv(STORM_CMD, args) 15 | 16 | COMMANDS = {"nimbus": nimbus} 17 | 18 | 19 | def main(): 20 | COMMAND = sys.argv[1] 21 | ARGS = sys.argv[2:] 22 | (COMMANDS.get(COMMAND))(*ARGS) 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /bin/run-with-marathon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o errexit -o nounset -o pipefail 3 | 4 | MESOS_MASTER_ZK="${MESOS_MASTER_ZK:-master.mesos}" 5 | 6 | echo "Running storm with MESOS_MASTER_ZK=${MESOS_MASTER_ZK}" 7 | 8 | function run_ui () { 9 | while [ true ] ; do 10 | ./bin/storm ui -c ui.port=$PORT0 -c nimbus.thrift.port=$PORT1 -c storm.log.dir=$MESOS_SANDBOX/logs ${STORM_UI_OPTS:-} 11 | done 12 | } 13 | 14 | function run_nimbus () { 15 | ./bin/storm-mesos nimbus -c mesos.master.url=zk://${MESOS_MASTER_ZK}:2181/mesos -c storm.zookeeper.servers="[\"${MESOS_MASTER_ZK}\"]" -c nimbus.thrift.port=$PORT1 -c storm.log.dir=$MESOS_SANDBOX/logs ${STORM_NIMBUS_OPTS:-} 16 | } 17 | 18 | function run () { 19 | run_ui & 20 | run_nimbus 21 | } 22 | 23 | run 24 | -------------------------------------------------------------------------------- /storm-shim/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | org.apache.mesos 5 | storm-shim 6 | 0.2.10-SNAPSHOT 7 | storm-shim 8 | Shim to ease Storm Version differences 9 | 10 | 11 | storm-parent 12 | org.apache.mesos 13 | 0.2.10-SNAPSHOT 14 | 15 | 16 | 17 | org.apache.storm 18 | storm-core 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/ReservationType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | public enum ReservationType { 21 | UNRESERVED, 22 | STATIC, 23 | DYNAMIC; 24 | 25 | public String toString() { 26 | return name().toLowerCase(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/ResourceNotAvailableException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | public class ResourceNotAvailableException extends Exception { 21 | public ResourceNotAvailableException(String message) { 22 | super(message); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /storm-shim/src/main/java/storm/mesos/shims/ICommandLineShim.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.shims; 19 | 20 | /** 21 | * Returns the command line, shimmed and 0.9.X and 0.10.X have slightly a different command 22 | */ 23 | public interface ICommandLineShim { 24 | 25 | public String getCommandLine(String topologyId); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /vagrant/start-nimbus.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $EUID -ne 0 ]]; then 4 | echo "$0 must be run as root" 5 | exit 1 6 | fi 7 | 8 | RELEASE=`grep -1 -A 0 -B 0 '' /vagrant/pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` 9 | 10 | echo "`date` - Starting nimbus!!" > start-nimbus.log 11 | 12 | cd /vagrant/_release/storm-mesos-${RELEASE}-* 13 | 14 | # kill existing MesosNimbus and storm UI processes 15 | kill `ps aux | grep MesosNimbu[s] | awk '{print $2}'` &> /dev/null || /bin/true 16 | # Grr... the below *was* working, but now the jar paths are too long with the long package version name. 17 | kill `ps aux | grep org.apache.storm.ui.cor[e] | awk '{print $2}'` &> /dev/null || /bin/true 18 | # So using this more aggressive form now. 19 | kill `ps aux | grep stor[m] | grep -v grep | awk '{print $2}'` &> /dev/null || /bin/true 20 | 21 | # Start storm nimbus, which also acts as the mesos scheduler in this case. 22 | # Point the STORM_CONF_DIR to where the repo's storm.yaml lives, so we can modify it 23 | # without having to rebuild the framework tarball and fully reprovision. 24 | MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so STORM_CONF_DIR=/vagrant bin/storm-mesos nimbus & 25 | 26 | # Start storm UI 27 | bin/storm ui & 28 | -------------------------------------------------------------------------------- /vagrant/start-mesos-master-and-slave.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $EUID -ne 0 ]]; then 4 | echo "$0 must be run as root" 5 | exit 1 6 | fi 7 | 8 | RELEASE=`grep -1 -A 0 -B 0 '' /vagrant/pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` 9 | 10 | echo "`date` - Starting mesos-master & mesos-slave!! $@" > start-mesos-master-and-slave.log 11 | 12 | # enable running of docker images 13 | echo 'docker,mesos' > /etc/mesos-slave/containerizers 14 | echo "zk://$1:2181/mesos" > /etc/mesos/zk 15 | grep -q "$1 master" /etc/hosts || echo "$1 master" >> /etc/hosts 16 | grep -q "$2 slave" /etc/hosts || echo "$2 slave" >> /etc/hosts 17 | echo $1 > /etc/mesos-master/ip 18 | echo $1 > /etc/mesos-slave/ip 19 | 20 | # Allow executor to be loaded from the tarball built by build-release.sh 21 | mkdir -p /usr/local/storm 22 | rm -rf /usr/local/storm/* || /bin/true 23 | cp /vagrant/storm-mesos-${RELEASE}-*.tgz /usr/local/storm 24 | 25 | # clean up previous runs' mesos-slave work_dir 26 | rm -rf /tmp/mesos/* 27 | 28 | # Restart (or start) mesos-master & mesos-slave after the configurations above 29 | status mesos-master | grep -q running && restart mesos-master || start mesos-master 30 | status mesos-slave | grep -q running && restart mesos-slave || start mesos-slave 31 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/ResourceEntry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | /** 21 | * Interface to represent various resource types defined in 22 | * {@link storm.mesos.resources.ResourceType} 23 | */ 24 | public interface ResourceEntry { 25 | 26 | public ResourceEntry add(ResourceEntry entry); 27 | 28 | public ResourceEntry remove(ResourceEntry entry); 29 | 30 | public ReservationType getReservationType(); 31 | } 32 | -------------------------------------------------------------------------------- /vagrant/start-mesos-slave.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $EUID -ne 0 ]]; then 4 | echo "$0 must be run as root" 5 | exit 1 6 | fi 7 | 8 | RELEASE=`grep -1 -A 0 -B 0 '' /vagrant/pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` 9 | 10 | echo "`date` - Starting mesos-slave!! $@" > start-mesos-slave.log 11 | 12 | # enable running of docker images 13 | echo 'docker,mesos' > /etc/mesos-slave/containerizers 14 | echo "zk://$1:2181/mesos" > /etc/mesos/zk 15 | grep -q "$1 master" /etc/hosts || echo "$1 master" >> /etc/hosts 16 | grep -q "$2 slave" /etc/hosts || echo "$2 slave" >> /etc/hosts 17 | echo "$2" > /etc/mesos-slave/ip 18 | 19 | # Allow executor to be loaded from the tarball built by build-release.sh 20 | mkdir -p /usr/local/storm 21 | rm -rf /usr/local/storm/* || /bin/true 22 | cp /vagrant/storm-mesos-${RELEASE}-*.tgz /usr/local/storm 23 | 24 | # clean up previous runs' mesos-slave work_dir 25 | rm -rf /tmp/mesos/* 26 | 27 | # Ensure mesos-master isn't running, and that it won't start automatically later 28 | echo manual > /etc/init/mesos-master.override 29 | status mesos-master | grep -q running && stop mesos-master 30 | 31 | # Restart (or start) mesos-slave after the configurations above 32 | status mesos-slave | grep -q running && restart mesos-slave || start mesos-slave 33 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/shims/CommandLineShimFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.shims; 19 | 20 | public final class CommandLineShimFactory { 21 | 22 | private CommandLineShimFactory() { 23 | 24 | } 25 | 26 | public static ICommandLineShim makeCommandLineShim(boolean docker, String extraConfig) { 27 | if (docker) { 28 | return new DockerCommandLineShim(extraConfig); 29 | } else { 30 | return new CommandLineShim(extraConfig); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/DefaultReservationTypeComparator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | import java.util.Comparator; 21 | 22 | public class DefaultReservationTypeComparator implements Comparator { 23 | 24 | public int compare(ReservationType left, ReservationType right) { 25 | if (left.equals(right)) { 26 | return 0; 27 | } else if (left.equals(ReservationType.UNRESERVED)) { 28 | return 1; 29 | } 30 | return -1; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/ResourceRoleComparator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos; 19 | 20 | import org.apache.mesos.Protos.Resource; 21 | 22 | import java.util.Comparator; 23 | 24 | public class ResourceRoleComparator implements Comparator { 25 | @Override 26 | public int compare(Resource left, Resource right) { 27 | if (left.getRole().equals(right.getRole())) { 28 | return 0; 29 | } else if (left.getRole().equals("*")) { 30 | return 1; 31 | } 32 | return -1; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /storm-shim-1x/src/main/java/storm/mesos/shims/CommandLineShim.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.shims; 19 | 20 | public class CommandLineShim implements ICommandLineShim { 21 | String extraConfig; 22 | 23 | public CommandLineShim(String extraConfig) { 24 | this.extraConfig = extraConfig; 25 | } 26 | 27 | public String getCommandLine(String topologyId) { 28 | return String.format( 29 | "export STORM_SUPERVISOR_LOG_FILE=%s-supervisor.log" + 30 | " && cp storm.yaml storm-mesos*/conf" + 31 | " && cd storm-mesos*" + 32 | " && bin/storm supervisor storm.mesos.MesosSupervisor%s", 33 | topologyId, extraConfig); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /storm-shim-1x/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | org.apache.mesos 5 | storm-shim-1x 6 | 0.2.10-SNAPSHOT 7 | storm-shim-1x 8 | 0.1.x impl shim to ease Storm Version differences 9 | 10 | 11 | scm:git:git://github.com/mesos/storm.git 12 | scm:git:ssh://git@github.com/mesos/storm.git 13 | https://github.com/mesos/storm 14 | HEAD 15 | 16 | 17 | 18 | storm-parent 19 | org.apache.mesos 20 | 0.2.10-SNAPSHOT 21 | 22 | 23 | 24 | UTF-8 25 | 26 | 1.7 27 | 28 | 29 | 30 | 31 | org.apache.storm 32 | storm-core 33 | ${storm.version} 34 | provided 35 | 36 | 37 | org.apache.mesos 38 | storm-shim 39 | ${project.parent.version} 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/schedulers/MesosWorkerSlot.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.schedulers; 19 | 20 | import org.apache.storm.scheduler.WorkerSlot; 21 | 22 | public class MesosWorkerSlot extends WorkerSlot { 23 | private String topologyId; 24 | 25 | public MesosWorkerSlot(String nodeId, Number port, String topologyId) { 26 | super(nodeId, port); 27 | this.topologyId = topologyId; 28 | } 29 | 30 | public String getTopologyId() { 31 | return this.topologyId; 32 | } 33 | 34 | public String getAssignmentAsString() { 35 | return String.format("%s:%s", super.getNodeId(), super.getPort()); 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return String.format("%s:%s topologyId: %s", super.getNodeId(), super.getPort(), topologyId); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/LaunchTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos; 19 | 20 | import org.apache.mesos.Protos; 21 | 22 | import static storm.mesos.util.PrettyProtobuf.offerToString; 23 | import static storm.mesos.util.PrettyProtobuf.taskInfoToString; 24 | 25 | class LaunchTask { 26 | private final Protos.TaskInfo task; 27 | private final Protos.Offer offer; 28 | 29 | public LaunchTask(final Protos.TaskInfo task, final Protos.Offer offer) { 30 | this.task = task; 31 | this.offer = offer; 32 | } 33 | 34 | public Protos.TaskInfo getTask() { 35 | return task; 36 | } 37 | 38 | public Protos.Offer getOffer() { 39 | return offer; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return String.format("Offer: %s TaskInfo: %s", offerToString(offer), taskInfoToString(task)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | sudo: required 3 | dist: trusty 4 | cache: 5 | directories: 6 | - $HOME/.m2/repository 7 | services: 8 | - docker 9 | env: 10 | global: 11 | - DOCKER_REPO=$TRAVIS_REPO_SLUG 12 | before_install: 13 | - apt-cache madison docker-ce 14 | - travis_retry sudo apt-get -o Dpkg::Options::="--force-confnew" install -y docker-ce 15 | install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true 16 | script: 17 | - jdk_switcher use oraclejdk8 18 | - mvn test 19 | - jdk_switcher use openjdk7 20 | - mvn test 21 | - travis_retry bin/build-release.sh 22 | matrix: 23 | fast_finish: true 24 | include: 25 | - env: STORM_RELEASE="1.0.6" MESOS_RELEASE="1.1.0" STORM_URL="https://github.com/erikdw/storm/releases/download/v1.0.6-storm-mesos4/apache-storm-1.0.6-storm-mesos4.tar.gz" 26 | - env: STORM_RELEASE="1.0.6" MESOS_RELEASE="0.28.2" STORM_URL="https://github.com/erikdw/storm/releases/download/v1.0.6-storm-mesos4/apache-storm-1.0.6-storm-mesos4.tar.gz" 27 | before_deploy: 28 | - travis_retry make images 29 | - travis_retry make images JAVA_PRODUCT_VERSION=8 30 | - docker images 31 | - travis_retry docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" 32 | - travis_retry make push 33 | - travis_retry make push JAVA_PRODUCT_VERSION=8 34 | deploy: 35 | provider: releases 36 | api_key: 37 | secure: SEKAUlMBRXrm9EY8JUYKrXQYC7v1+nJ2/0H23KvcZGbvBL6tMaY1jPJZ8vlCJjCwd69WUCc9a1p/IzYMkt6jz4xx2vB13MYL7GnQgltzEwCAVl7I6uw1r08SDyNdeCeBqZFTcPXmrWM5ByQ0q/4SqEqLMw1uCTfOQV1/XQeivkA= 38 | file: "storm-mesos-$(grep -1 -A 0 -B 0 '' pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's|.*||')-storm$STORM_RELEASE-mesos$MESOS_RELEASE.tgz" 39 | skip_cleanup: true 40 | on: 41 | tags: true 42 | repo: $TRAVIS_REPO_SLUG 43 | -------------------------------------------------------------------------------- /storm/src/test/storm/mesos/OfferRoleComparatorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos; 19 | 20 | import org.apache.mesos.Protos.Offer; 21 | import org.apache.mesos.Protos.Resource; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.Collections; 26 | import java.util.List; 27 | 28 | import static org.junit.Assert.assertEquals; 29 | 30 | public class OfferRoleComparatorTest { 31 | 32 | 33 | 34 | @Test 35 | public void testCompare() throws Exception { 36 | List aggregatedOffers = new ArrayList<>(); 37 | Offer offer = TestUtils.buildOffer(); 38 | aggregatedOffers.addAll(offer.getResourcesList()); 39 | Collections.sort(aggregatedOffers, new ResourceRoleComparator()); 40 | 41 | assertEquals( 42 | "*", 43 | aggregatedOffers.get(5).getRole() 44 | ); 45 | assertEquals( 46 | "*", 47 | aggregatedOffers.get(4).getRole() 48 | ); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /storm-shim-1x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.shims; 19 | import java.util.Map; 20 | 21 | public class DockerCommandLineShim implements ICommandLineShim { 22 | String extraConfig; 23 | 24 | public DockerCommandLineShim(String extraConfig) { 25 | this.extraConfig = extraConfig; 26 | } 27 | 28 | public String getCommandLine(String topologyId) { 29 | // An ugly workaround for a bug in DCOS 30 | Map env = System.getenv(); 31 | String javaLibPath = env.get("MESOS_NATIVE_JAVA_LIBRARY"); 32 | return String.format( 33 | "export MESOS_NATIVE_JAVA_LIBRARY=%s" + 34 | " && export STORM_SUPERVISOR_LOG_FILE=%s-supervisor.log" + 35 | " && /bin/cp $MESOS_SANDBOX/storm.yaml conf " + 36 | " && bin/storm supervisor storm.mesos.MesosSupervisor " + 37 | "-c storm.log.dir=$MESOS_SANDBOX/logs%s", 38 | javaLibPath, topologyId, extraConfig); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /storm/src/test/storm/mesos/shims/CommandLineShimTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.shims; 19 | 20 | import org.junit.Test; 21 | 22 | import static org.hamcrest.CoreMatchers.is; 23 | import static org.junit.Assert.assertThat; 24 | 25 | public final class CommandLineShimTest { 26 | 27 | @Test 28 | public void getCommandLine() { 29 | String extraConfig = " -c storm.local.dir=/tmp"; 30 | String topologyId = "sample-id"; 31 | CommandLineShim target = new CommandLineShim(extraConfig); 32 | String actualCmd = target.getCommandLine(topologyId); 33 | String expectedCmd = 34 | "export STORM_SUPERVISOR_LOG_FILE=" + 35 | topologyId + 36 | "-supervisor.log " + 37 | "&& cp storm.yaml storm-mesos*/conf " + 38 | "&& cd storm-mesos* " + 39 | "&& bin/storm supervisor storm.mesos.MesosSupervisor" + 40 | extraConfig; 41 | assertThat(actualCmd, is(expectedCmd)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/schedulers/IMesosStormScheduler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.schedulers; 19 | 20 | 21 | import org.apache.storm.scheduler.SupervisorDetails; 22 | import org.apache.storm.scheduler.Topologies; 23 | import org.apache.storm.scheduler.WorkerSlot; 24 | import org.apache.mesos.Protos; 25 | 26 | import java.util.Collection; 27 | import java.util.List; 28 | import java.util.Map; 29 | import java.util.Set; 30 | 31 | /** 32 | * A scheduler needs to implement the following interface for it to be MesosNimbus compatible. 33 | */ 34 | public interface IMesosStormScheduler { 35 | 36 | /** 37 | * This method is invoked by Nimbus when it wants to get a list of worker slots that are available for assigning the 38 | * topology workers. In Nimbus's view, a "WorkerSlot" is a host and port that it can use to assign a worker. 39 | */ 40 | public List allSlotsAvailableForScheduling(Map offers, 41 | Collection existingSupervisors, 42 | Topologies topologies, Set topologiesMissingAssignments); 43 | } 44 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/ResourceType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | public enum ResourceType { 27 | CPU("cpus"), 28 | MEM("mem"), 29 | PORTS("ports"), 30 | DISK("disk"); 31 | 32 | private final String resourceType; 33 | private static final Logger LOG = LoggerFactory.getLogger(ResourceType.class); 34 | 35 | private static final Map availableResourceTypes = new HashMap<>(ResourceType.values().length); 36 | 37 | static { 38 | for (ResourceType r : ResourceType.values()) { 39 | availableResourceTypes.put(r.resourceType, r); 40 | } 41 | } 42 | 43 | 44 | public static ResourceType of(String name) { 45 | ResourceType result = availableResourceTypes.get(name); 46 | if (result == null) { 47 | LOG.warn("Unexpected resource type {}", name); 48 | } 49 | return result; 50 | } 51 | 52 | 53 | ResourceType(String resourceType) { 54 | this.resourceType = resourceType; 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return resourceType; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /storm.yaml: -------------------------------------------------------------------------------- 1 | # Please change these for your cluster 2 | # to reflect your cluster settings 3 | # ----------------------------------------------------------- 4 | mesos.master.url: "zk://localhost:2181/mesos" 5 | storm.zookeeper.servers: 6 | - "localhost" 7 | # ----------------------------------------------------------- 8 | 9 | 10 | # Worker resources 11 | topology.mesos.worker.cpu: 1.0 12 | # Worker heap with 25% overhead 13 | topology.mesos.worker.mem.mb: 512 14 | worker.childopts: "-Xmx384m" 15 | 16 | # Supervisor resources 17 | topology.mesos.executor.cpu: 0.1 18 | topology.mesos.executor.mem.mb: 500 # Supervisor memory, with 20% overhead 19 | supervisor.childopts: "-Xmx400m" 20 | 21 | # Configs for Resource Aware Scheduler 22 | topology.worker.max.heap.size.mb: 768.0 # default max heap size, can be overriden in topologies' configs 23 | 24 | # By default this framework will launch a logviewer mesos executor on each mesos host where Storm workers are launched. 25 | # Set to false to disable logviewer from being automatically launched. 26 | mesos.logviewer.sidecar.enabled: true 27 | # This port must be available on every mesos host where Storm workers are launched. 28 | logviewer.port: 8000 29 | logviewer.childopts: "-Xmx128m" 30 | logviewer.cleanup.age.mins: 10080 31 | logviewer.appender.name: "A1" 32 | 33 | # Use the public Mesosphere Storm build 34 | # Please note that it won't work with other distributions. 35 | # You may want to make this empty if you use `mesos.container.docker.image` instead. 36 | # mesos.executor.uri: "file:///usr/local/storm/storm-mesos-0.9.6.tgz" 37 | 38 | # Alternatively, use a Docker image instead of URI. If an image is specified, 39 | # Docker will be used instead of Mesos containers. 40 | mesos.container.docker.image: "mesosphere/storm" 41 | 42 | # Use Netty to avoid ZMQ dependencies 43 | storm.messaging.transport: "org.apache.storm.messaging.netty.Context" 44 | 45 | storm.local.dir: "storm-local" 46 | 47 | # role must be one of the mesos-master's roles defined in the --roles flag 48 | # 49 | mesos.framework.role: "*" 50 | mesos.framework.checkpoint: true 51 | mesos.framework.name: "Storm" 52 | 53 | # For setting up the necessary mesos authentication see mesos authentication page 54 | # and set the mesos-master flags --credentials, --authenticate, --acls, and --roles. 55 | # 56 | #mesos.framework.principal: "storm" 57 | 58 | # The "secret" phrase cannot be followed by a NL 59 | # 60 | #mesos.framework.secret.file: "storm-local/secret" 61 | 62 | #mesos.allowed.hosts: 63 | # - host1 64 | #mesos.disallowed.hosts: 65 | # - host1 66 | -------------------------------------------------------------------------------- /storm-shim-1x/src/main/java/storm/mesos/shims/LocalStateShim.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.shims; 19 | 20 | import org.apache.storm.generated.ComponentObject; 21 | import org.apache.storm.generated.JavaObject; 22 | import org.apache.storm.generated.JavaObjectArg; 23 | import org.apache.storm.utils.LocalState; 24 | import org.apache.storm.thrift.TBase; 25 | 26 | import java.io.IOException; 27 | import java.util.ArrayList; 28 | import java.util.HashMap; 29 | import java.util.List; 30 | import java.util.Map; 31 | 32 | public class LocalStateShim implements ILocalStateShim { 33 | LocalState _state; 34 | 35 | public LocalStateShim(String localDir) throws IOException { 36 | _state = new LocalState(localDir); 37 | } 38 | 39 | public String get(String key) { 40 | TBase tBase = _state.get(key); 41 | if (tBase != null) { 42 | JavaObject jo = ((JavaObject) tBase); 43 | ComponentObject co = ComponentObject.java_object(jo); 44 | List l = jo.get_args_list(); 45 | if (l != null && l.size() >= 0 && l.get(0) != null) { 46 | return l.get(0).get_string_arg(); 47 | } 48 | } 49 | return null; 50 | } 51 | 52 | public void put(String key, String value) { 53 | ArrayList tmpList = new ArrayList(); 54 | tmpList.add(JavaObjectArg.string_arg(value)); 55 | _state.put(key, new JavaObject("java.lang.String", tmpList)); 56 | } 57 | 58 | public void remove(String key) { 59 | _state.remove(key); 60 | } 61 | 62 | public Map snapshot() { 63 | Map snapshot = _state.snapshot(); 64 | HashMap retVal = new HashMap<>(); 65 | for (String key: snapshot.keySet()) { 66 | retVal.put(key, get(key)); 67 | } 68 | return retVal; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/Resource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | import java.util.Comparator; 21 | import java.util.List; 22 | 23 | /** 24 | * An interface for managing various kinds of resources defined 25 | * in {@link storm.mesos.resources.ResourceType} 26 | */ 27 | public interface Resource> { 28 | 29 | public boolean isAvailable(T resourceEntry); 30 | 31 | public boolean isAvailable(T resourceEntry, ReservationType reservationType); 32 | 33 | public List getAllAvailableResources(); 34 | 35 | public List getAllAvailableResources(ReservationType reservationType); 36 | 37 | public void add(T resourceEntry, ReservationType reservationType); 38 | 39 | public List removeAndGet(T resourceEntry) throws ResourceNotAvailableException; 40 | 41 | /** 42 | * Unused Method - Exists for the purpose of facilitating support of reservations. 43 | * TODO: Support reservations (https://github.com/mesos/storm/issues/148) 44 | * For more information about why this unused code exists, see discussion: https://github.com/mesos/storm/pull/146#issuecomment-225496075 45 | */ 46 | public List removeAndGet(T value, ReservationType reservationType) throws ResourceNotAvailableException; 47 | 48 | /** 49 | * Unused Method - Exists for the purpose of facilitating support of reservations. 50 | * TODO: Support reservations (https://github.com/mesos/storm/issues/148) 51 | * For more information about why this unused code exists, see discussion: https://github.com/mesos/storm/pull/146#issuecomment-225496075 52 | */ 53 | public List removeAndGet(T value, Comparator reservationTypeComparator) throws ResourceNotAvailableException; 54 | 55 | } 56 | -------------------------------------------------------------------------------- /storm/src/test/storm/mesos/schedulers/SchedulerUtilsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.schedulers; 19 | 20 | import org.apache.storm.scheduler.SupervisorDetails; 21 | import org.apache.mesos.Protos.Offer; 22 | import org.apache.mesos.Protos.OfferID; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.mockito.runners.MockitoJUnitRunner; 27 | import storm.mesos.util.MesosCommon; 28 | 29 | import java.util.ArrayList; 30 | import java.util.Collection; 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | 34 | import static org.junit.Assert.assertEquals; 35 | import static storm.mesos.TestUtils.buildOffer; 36 | 37 | @RunWith(MockitoJUnitRunner.class) 38 | public class SchedulerUtilsTest { 39 | 40 | Map map; 41 | private final String sampleHost = "host1.east"; 42 | 43 | @Before 44 | public void initialize() { 45 | map = new HashMap(); 46 | } 47 | 48 | private void buildOfferAndUpdateMap(String offerId, String hostName, double cpus, double memory) { 49 | Offer offer = buildOffer(offerId, hostName, cpus, memory); 50 | map.put(offer.getId(), offer); 51 | } 52 | 53 | @Test 54 | public void testSupervisorExists() throws Exception { 55 | Collection existingSupervisors = new ArrayList<>(); 56 | String hostName = "host1.east"; 57 | String frameworkName = "Storm!!!"; 58 | 59 | existingSupervisors.add(new SupervisorDetails(MesosCommon.supervisorId(frameworkName, hostName, "test-topology1-65-1442255385"), hostName)); 60 | existingSupervisors.add(new SupervisorDetails(MesosCommon.supervisorId(frameworkName, hostName, "test-topology10-65-1442255385"), hostName)); 61 | 62 | assertEquals(true, SchedulerUtils.supervisorExists(frameworkName, hostName, existingSupervisors, "test-topology1-65-1442255385")); 63 | assertEquals(false, SchedulerUtils.supervisorExists(frameworkName, hostName, existingSupervisors, "test-topology2-65-1442255385")); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile for building storm docker images 3 | # 4 | # make help 5 | # 6 | # 7 | RELEASE ?= `grep -1 -A 0 -B 0 '' pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` 8 | DOCKER_REPO ?= mesos/storm 9 | MIRROR ?= 10 | STORM_URL ?= 11 | JAVA_PRODUCT_VERSION ?= 7 12 | 13 | all: help 14 | 15 | help: 16 | @echo 'Options available:' 17 | @echo ' make images STORM_RELEASE=0.10.1 MESOS_RELEASE=0.28.2 DOCKER_REPO=mesos/storm' 18 | @echo ' make push STORM_RELEASE=0.10.1 MESOS_RELEASE=0.28.2 DOCKER_REPO=mesos/storm' 19 | @echo '' 20 | @echo 'ENV' 21 | @echo ' STORM_RELEASE The targeted release version of Storm' 22 | @echo ' Default: 0.10.1' 23 | @echo ' MESOS_RELEASE The targeted release version of MESOS' 24 | @echo ' Default: 0.28.2' 25 | @echo ' DOCKER_REPO The docker repo for which to build the docker image' 26 | @echo ' Default: mesos/storm' 27 | @echo ' RELEASE The targeted release version of Storm' 28 | @echo ' Default: current version in pom.xml' 29 | @echo ' JAVA_PRODUCT_VERSION The java product version to use in the docker image' 30 | @echo ' Default: 7' 31 | @echo ' MIRROR Where to download the apache storm packages' 32 | @echo ' Default: http://www.gtlib.gatech.edu/pub' 33 | @echo ' STORM_URL The url to use to download the apache storm packages' 34 | @echo ' Default: $$MIRROR/apache/storm/apache-storm-$$STORM_RELEASE/apache-storm-$$STORM_RELEASE.tar.gz' 35 | 36 | check-version: 37 | ifndef STORM_RELEASE 38 | @echo "Error: STORM_RELEASE is undefined." 39 | @make --no-print-directory help 40 | @exit 1 41 | endif 42 | ifndef MESOS_RELEASE 43 | @echo "Error: MESOS_RELEASE is undefined." 44 | @make --no-print-directory help 45 | @exit 1 46 | endif 47 | 48 | images: check-version 49 | docker build \ 50 | --rm \ 51 | --build-arg STORM_RELEASE=$(STORM_RELEASE) \ 52 | --build-arg MESOS_RELEASE=$(MESOS_RELEASE) \ 53 | --build-arg RELEASE=$(RELEASE) \ 54 | --build-arg JAVA_PRODUCT_VERSION=$(JAVA_PRODUCT_VERSION) \ 55 | --build-arg MIRROR=$(MIRROR) \ 56 | --build-arg STORM_URL=$(STORM_URL) \ 57 | -t $(DOCKER_REPO):$(RELEASE)-$(STORM_RELEASE)-$(MESOS_RELEASE)-jdk$(JAVA_PRODUCT_VERSION) . 58 | docker build \ 59 | --rm \ 60 | -f onbuild/Dockerfile \ 61 | -t $(DOCKER_REPO):$(RELEASE)-$(STORM_RELEASE)-$(MESOS_RELEASE)-jdk$(JAVA_PRODUCT_VERSION)-onbuild . 62 | 63 | push: check-version 64 | docker push $(DOCKER_REPO):$(RELEASE)-$(STORM_RELEASE)-$(MESOS_RELEASE)-jdk$(JAVA_PRODUCT_VERSION) 65 | docker push $(DOCKER_REPO):$(RELEASE)-$(STORM_RELEASE)-$(MESOS_RELEASE)-jdk$(JAVA_PRODUCT_VERSION)-onbuild 66 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Dockerfile for Storm Mesos framework 3 | # 4 | FROM ubuntu:14.04 5 | MAINTAINER Salimane Adjao Moustapha 6 | 7 | # set JAVA_HOME depending if JAVA_PRODUCT_VERSION is set 8 | ARG JAVA_PRODUCT_VERSION='' 9 | ENV TARGET_JAVA_HOME ${JAVA_PRODUCT_VERSION:+/usr/lib/jvm/java-$JAVA_PRODUCT_VERSION-openjdk-amd64} 10 | ENV JAVA_HOME ${TARGET_JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64} 11 | ENV JAVA_PRODUCT_VERSION ${JAVA_PRODUCT_VERSION:-7} 12 | 13 | # Add mesosphere 14 | # install mesos version, build arg, MESOS_RELEASE with a default value 15 | ARG MESOS_RELEASE=0.28.2 16 | RUN DISTRO=$(lsb_release -is | tr "[:upper:]" "[:lower:]") && \ 17 | CODENAME=$(lsb_release -cs) && \ 18 | echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" > /etc/apt/sources.list.d/mesosphere.list && \ 19 | apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF && \ 20 | apt-get -y update && \ 21 | apt-get -y install mesos=`apt-cache madison mesos | grep " ${MESOS_RELEASE}" | head -1 | awk '{print $3}'` && \ 22 | apt-get clean 23 | 24 | # set build arg STORM_RELEASE, MIRROR with defaults values 25 | ARG STORM_RELEASE=0.10.1 26 | ARG MIRROR='' 27 | ARG STORM_URL='' 28 | ARG RELEASE='' 29 | 30 | # set runtime environment variables 31 | ENV MESOS_RELEASE ${MESOS_RELEASE:-0.28.2} 32 | ENV STORM_RELEASE ${STORM_RELEASE:-0.10.1} 33 | ENV DEBIAN_FRONTEND ${DEBIAN_FRONTEND:-noninteractive} 34 | ENV MESOS_NATIVE_JAVA_LIBRARY ${MESOS_NATIVE_JAVA_LIBRARY:-/usr/lib/libmesos.so} 35 | 36 | # Add files for storm-mesos package building 37 | COPY . /tmp 38 | 39 | # storm-mesos package building if file storm-mesos*.tgz not found 40 | RUN cd /tmp && \ 41 | RELEASE=`grep -1 -A 0 -B 0 '' pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` && \ 42 | ([ -f storm-mesos-${RELEASE}-storm${STORM_RELEASE}-mesos${MESOS_RELEASE}.tgz ] || \ 43 | (apt-get -y install software-properties-common && \ 44 | add-apt-repository ppa:openjdk-r/ppa && \ 45 | apt-get -y update && \ 46 | apt-get -y install openjdk-${JAVA_PRODUCT_VERSION}-jdk maven wget curl && \ 47 | update-alternatives --install /usr/bin/java java ${JAVA_HOME%/}/bin/java 20000 && \ 48 | update-alternatives --install /usr/bin/javac javac ${JAVA_HOME%/}/bin/javac 20000 && \ 49 | STORM_RELEASE=$STORM_RELEASE MESOS_RELEASE=$MESOS_RELEASE ./bin/build-release.sh main && \ 50 | apt-get -yf autoremove openjdk-${JAVA_PRODUCT_VERSION}-jdk maven \ 51 | ) \ 52 | ) && \ 53 | mkdir -p /opt/storm && \ 54 | tar xf /tmp/storm-mesos-${RELEASE}-storm${STORM_RELEASE}-mesos${MESOS_RELEASE}.tgz -C /opt/storm --strip=1 && \ 55 | rm -rf /tmp/* ~/.m2 && \ 56 | apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 57 | 58 | WORKDIR /opt/storm 59 | -------------------------------------------------------------------------------- /onbuild/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Dockerfile for Storm Mesos framework 3 | # 4 | FROM ubuntu:14.04 5 | MAINTAINER Salimane Adjao Moustapha 6 | 7 | # set JAVA_HOME depending if JAVA_PRODUCT_VERSION is set 8 | ONBUILD ARG JAVA_PRODUCT_VERSION='' 9 | ONBUILD ENV TARGET_JAVA_HOME ${JAVA_PRODUCT_VERSION:+/usr/lib/jvm/java-$JAVA_PRODUCT_VERSION-openjdk-amd64} 10 | ONBUILD ENV JAVA_HOME ${TARGET_JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64} 11 | ONBUILD ENV JAVA_PRODUCT_VERSION ${JAVA_PRODUCT_VERSION:-7} 12 | 13 | # Add mesosphere 14 | # install mesos version, build arg, MESOS_RELEASE with a default value 15 | ONBUILD ARG MESOS_RELEASE=0.28.2 16 | ONBUILD RUN DISTRO=$(lsb_release -is | tr "[:upper:]" "[:lower:]") && \ 17 | CODENAME=$(lsb_release -cs) && \ 18 | echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" > /etc/apt/sources.list.d/mesosphere.list && \ 19 | apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF && \ 20 | apt-get -y update && \ 21 | apt-get -y install mesos=`apt-cache madison mesos | grep " ${MESOS_RELEASE}" | head -1 | awk '{print $3}'` && \ 22 | apt-get clean && rm -rf /var/lib/apt/lists/* 23 | 24 | # set build arg STORM_RELEASE, MIRROR with defaults values 25 | ONBUILD ARG STORM_RELEASE=0.10.1 26 | ONBUILD ARG MIRROR='' 27 | ONBUILD ARG STORM_URL='' 28 | ONBUILD ARG RELEASE='' 29 | 30 | # set runtime environment variables 31 | ENV MESOS_RELEASE ${MESOS_RELEASE:-0.28.2} 32 | ENV STORM_RELEASE ${STORM_RELEASE:-0.10.1} 33 | ENV DEBIAN_FRONTEND ${DEBIAN_FRONTEND:-noninteractive} 34 | ENV MESOS_NATIVE_JAVA_LIBRARY ${MESOS_NATIVE_JAVA_LIBRARY:-/usr/lib/libmesos.so} 35 | 36 | # Add files for storm-mesos package building 37 | COPY . /tmp 38 | 39 | # storm-mesos package building if file storm-mesos*.tgz not found 40 | ONBUILD RUN cd /tmp && \ 41 | RELEASE=`grep -1 -A 0 -B 0 '' pom.xml | head -n 1 | awk '{print $1}' | sed -e 's/.*//' | sed -e 's/<\/version>.*//'` && \ 42 | ([ -f storm-mesos-${RELEASE}-storm${STORM_RELEASE}-mesos${MESOS_RELEASE}.tgz ] || \ 43 | (apt-get -y install software-properties-common && \ 44 | add-apt-repository ppa:openjdk-r/ppa && \ 45 | apt-get -y update && \ 46 | apt-get -y install openjdk-${JAVA_PRODUCT_VERSION}-jdk maven wget curl && \ 47 | update-alternatives --install /usr/bin/java java ${JAVA_HOME%/}/bin/java 20000 && \ 48 | update-alternatives --install /usr/bin/javac javac ${JAVA_HOME%/}/bin/javac 20000 && \ 49 | STORM_RELEASE=$STORM_RELEASE MESOS_RELEASE=$MESOS_RELEASE ./bin/build-release.sh main && \ 50 | apt-get -yf autoremove openjdk-${JAVA_PRODUCT_VERSION}-jdk maven \ 51 | ) \ 52 | ) && \ 53 | mkdir -p /opt/storm && \ 54 | tar xf /tmp/storm-mesos-${RELEASE}-storm${STORM_RELEASE}-mesos${MESOS_RELEASE}.tgz -C /opt/storm --strip=1 && \ 55 | rm -rf /tmp/* ~/.m2 && \ 56 | apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 57 | 58 | WORKDIR /opt/storm 59 | -------------------------------------------------------------------------------- /storm/src/test/storm/mesos/LaunchTaskTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos; 19 | 20 | import org.apache.mesos.Protos.CommandInfo; 21 | import org.apache.mesos.Protos.ExecutorID; 22 | import org.apache.mesos.Protos.ExecutorInfo; 23 | import org.apache.mesos.Protos.FrameworkID; 24 | import org.apache.mesos.Protos.Offer; 25 | import org.apache.mesos.Protos.OfferID; 26 | import org.apache.mesos.Protos.SlaveID; 27 | import org.apache.mesos.Protos.TaskID; 28 | import org.apache.mesos.Protos.TaskInfo; 29 | import org.junit.Test; 30 | 31 | import static org.junit.Assert.assertEquals; 32 | import static org.junit.Assert.assertSame; 33 | 34 | public final class LaunchTaskTest 35 | { 36 | 37 | /** 38 | * Testing target. 39 | */ 40 | private final LaunchTask target; 41 | 42 | private final TaskInfo sampleTaskInfo; 43 | 44 | private final Offer sampleOffer; 45 | 46 | /** 47 | * Setup testing target & sample data. 48 | */ 49 | public LaunchTaskTest() 50 | { 51 | SlaveID slaveID = SlaveID.newBuilder().setValue("s1").build(); 52 | this.sampleTaskInfo = 53 | TaskInfo.newBuilder() 54 | .setName("t1").setSlaveId(slaveID) 55 | .setTaskId(TaskID.newBuilder().setValue("id2")) 56 | .setExecutor( 57 | ExecutorInfo.newBuilder() 58 | .setExecutorId(ExecutorID.newBuilder().setValue("e1")) 59 | .setCommand(CommandInfo.getDefaultInstance())) 60 | .build(); 61 | this.sampleOffer = 62 | Offer.newBuilder() 63 | .setHostname("h1").setSlaveId(slaveID) 64 | .setId(OfferID.newBuilder().setValue("id1")) 65 | .setFrameworkId(FrameworkID.newBuilder().setValue("f1").build()) 66 | .build(); 67 | this.target = new LaunchTask(sampleTaskInfo, sampleOffer); 68 | } 69 | 70 | @Test 71 | public void getTask() 72 | { 73 | assertSame( 74 | "The return value should be the same as the value passed in the argument of the constructor", 75 | sampleTaskInfo, target.getTask() 76 | ); 77 | } 78 | 79 | @Test 80 | public void getOffer() 81 | { 82 | assertSame( 83 | "The return value should be the same as the value passed in the argument of the constructor", 84 | sampleOffer, target.getOffer() 85 | ); 86 | } 87 | 88 | @Test 89 | public void objectToString() 90 | { 91 | assertEquals( 92 | "Offer: {\"offer_id\":\"id1\",\"hostname\":\"h1\"} TaskInfo: {\"task_id\":\"id2\",\"slave_id\":\"s1\",\"executor_id\":\"e1\"}", 93 | target.toString() 94 | ); 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /storm/src/test/storm/mesos/resources/AggregatedOffersTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | import org.apache.mesos.Protos.Offer; 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | import org.mockito.runners.MockitoJUnitRunner; 24 | import storm.mesos.TestUtils; 25 | 26 | import java.util.List; 27 | 28 | import static org.junit.Assert.assertEquals; 29 | import static org.junit.Assert.assertTrue; 30 | 31 | @RunWith(MockitoJUnitRunner.class) 32 | public class AggregatedOffersTest { 33 | private static final double DELTA_FOR_DOUBLE_COMPARISON = 0.0001; 34 | 35 | @Test 36 | public void testToIgnoreDynamicResources() { 37 | ScalarResource scalarResource = new ScalarResource(ResourceType.CPU); 38 | scalarResource.add(new ResourceEntries.ScalarResourceEntry(100.0), ReservationType.STATIC); 39 | scalarResource.toString(); 40 | 41 | // Note that buidOffer adds 42 | Offer offer = TestUtils.buildOffer("0-1", "h1", 0, 0); 43 | AggregatedOffers aggregatedOffers = new AggregatedOffers(offer); 44 | 45 | assertEquals(0, TestUtils.calculateAllAvailableScalarResources(aggregatedOffers, ResourceType.CPU), DELTA_FOR_DOUBLE_COMPARISON); 46 | assertEquals(0, TestUtils.calculateAllAvailableScalarResources(aggregatedOffers, ResourceType.MEM), DELTA_FOR_DOUBLE_COMPARISON); 47 | 48 | assertTrue(aggregatedOffers.getHostname().equals(offer.getHostname())); 49 | assertTrue(aggregatedOffers.getSlaveID().equals(offer.getSlaveId())); 50 | 51 | offer = TestUtils.buildOfferWithReservation("offer1", "h1", 2, 1000, 6, 1000); 52 | aggregatedOffers = new AggregatedOffers(offer); 53 | assertEquals(8, TestUtils.calculateAllAvailableScalarResources(aggregatedOffers, ResourceType.CPU), DELTA_FOR_DOUBLE_COMPARISON); 54 | assertEquals(2000, TestUtils.calculateAllAvailableScalarResources(aggregatedOffers, ResourceType.MEM), DELTA_FOR_DOUBLE_COMPARISON); 55 | assertTrue(aggregatedOffers.getHostname().equals(offer.getHostname())); 56 | assertTrue(aggregatedOffers.getSlaveID().equals(offer.getSlaveId())); 57 | 58 | offer = TestUtils.buildOfferWithPorts("offer1", "h1", 2.0, 2000, 3000, 3100); 59 | aggregatedOffers = new AggregatedOffers(offer); 60 | assertEquals(2.0, TestUtils.calculateAllAvailableScalarResources(aggregatedOffers, ResourceType.CPU), DELTA_FOR_DOUBLE_COMPARISON); 61 | assertEquals(2000, TestUtils.calculateAllAvailableScalarResources(aggregatedOffers, ResourceType.MEM),DELTA_FOR_DOUBLE_COMPARISON); 62 | List rangeResources = TestUtils.calculateAllAvailableRangeResources(aggregatedOffers, ResourceType.PORTS); 63 | assertTrue(rangeResources.size() == 101); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/LocalFileServer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos; 19 | 20 | import com.google.common.base.Optional; 21 | import org.mortbay.jetty.Connector; 22 | import org.mortbay.jetty.Handler; 23 | import org.mortbay.jetty.Server; 24 | import org.mortbay.jetty.handler.ContextHandler; 25 | import org.mortbay.jetty.handler.HandlerList; 26 | import org.mortbay.jetty.handler.ResourceHandler; 27 | import org.mortbay.jetty.nio.SelectChannelConnector; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.net.InetAddress; 32 | import java.net.URI; 33 | 34 | /** 35 | * Starts embedded Jetty server on random port to share a local directory. 36 | * Returns the full URL where files can be accessed. 37 | */ 38 | 39 | public class LocalFileServer { 40 | 41 | public static final Logger LOG = LoggerFactory.getLogger(LocalFileServer.class); 42 | private Server _server = new Server(); 43 | 44 | public static void main(String[] args) throws Exception { 45 | 46 | URI url = new LocalFileServer().serveDir("/tmp2", "/tmp", getHost(), Optional.absent()); 47 | System.out.println("************ " + url); 48 | 49 | } 50 | 51 | /** 52 | * Starts embedded Jetty server to share a local directory. 53 | * Returns the full URL where files can be retrieved. 54 | * 55 | * @param uriPath - URI Path component e.g. /config 56 | * @param filePath - Directory path to be served. Can be absolute or relative. E.g. config 57 | * @return Full URI including server, port and path of baselevel dir. Please note that the ancient Jetty 6.1 Storm uses can't be configured to return directory listings AFAIK. 58 | * @throws Exception 59 | */ 60 | public URI serveDir(String uriPath, String filePath, String host, Optional port) throws Exception { 61 | 62 | if (port.isPresent()) { 63 | LOG.info("Starting local file server on port: {}", port.get()); 64 | _server = new Server(port.get()); 65 | } else { 66 | _server = new Server(); 67 | } 68 | SelectChannelConnector connector = new SelectChannelConnector(); 69 | connector.setPort(0); 70 | _server.addConnector(connector); 71 | 72 | ResourceHandler resourceHandler = new ResourceHandler(); 73 | resourceHandler.setResourceBase(filePath); 74 | 75 | ContextHandler staticContextHandler = new ContextHandler(); 76 | staticContextHandler.setContextPath(uriPath); 77 | staticContextHandler.setHandler(resourceHandler); 78 | 79 | HandlerList handlers = new HandlerList(); 80 | handlers.setHandlers(new Handler[]{staticContextHandler}); 81 | _server.setHandler(handlers); 82 | _server.start(); 83 | 84 | // get the connector once it is init so we can get the actual port it bound to. 85 | Connector initConn = _server.getConnectors()[0]; 86 | return new URI("http", null, host, initConn.getLocalPort(), uriPath, null, null); 87 | } 88 | 89 | private static String getHost() throws Exception { 90 | return Optional.fromNullable((String) System.getenv("MESOS_NIMBUS_HOST")) 91 | .or(InetAddress.getLocalHost().getCanonicalHostName()); 92 | } 93 | 94 | public void shutdown() throws Exception { 95 | _server.stop(); 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/resources/ResourceEntries.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | public class ResourceEntries { 21 | public static final class ScalarResourceEntry implements ResourceEntry { 22 | private ReservationType reservationType; 23 | private Double value; 24 | 25 | public ReservationType getReservationType() { 26 | return reservationType; 27 | } 28 | 29 | public ScalarResourceEntry(Double value) { 30 | this.value = value; 31 | } 32 | 33 | public ScalarResourceEntry(ReservationType reservationType, Double value) { 34 | this.reservationType = reservationType; 35 | this.value = value; 36 | } 37 | 38 | public Double getValue() { 39 | return value; 40 | } 41 | 42 | @Override 43 | public ScalarResourceEntry add(ResourceEntry resourceEntry) { 44 | ScalarResourceEntry scalarResourceEntry = (ScalarResourceEntry) resourceEntry; 45 | this.value += scalarResourceEntry.getValue(); 46 | return this; 47 | } 48 | 49 | @Override 50 | public ScalarResourceEntry remove(ResourceEntry resourceEntry) { 51 | ScalarResourceEntry scalarResourceEntry = (ScalarResourceEntry) resourceEntry; 52 | this.value -= scalarResourceEntry.getValue(); 53 | return this; 54 | } 55 | } 56 | 57 | public static final class RangeResourceEntry implements ResourceEntry { 58 | private ReservationType reservationType; 59 | private Long begin; 60 | private Long end; 61 | 62 | public RangeResourceEntry(Long begin, Long end) { 63 | this.begin = begin; 64 | this.end = end; 65 | } 66 | 67 | public RangeResourceEntry(ReservationType reservationType, Long begin, Long end) { 68 | this.reservationType = reservationType; 69 | this.begin = begin; 70 | this.end = end; 71 | } 72 | 73 | public Long getBegin() { 74 | return begin; 75 | } 76 | 77 | public Long getEnd() { 78 | return end; 79 | } 80 | 81 | public ReservationType getReservationType() { 82 | return reservationType; 83 | } 84 | 85 | /** 86 | * Unused Method - Exists for the sake of completeness in terms of implementing ResourceEntry. 87 | * 88 | * Lets say, we have a range [u,v]. Using this add function, we can expand the range to [w,x] if and 89 | * only if one of the following conditions are satisfied 90 | * `w < u` 91 | * `x > v` 92 | * `w < u` and `x > v` 93 | * In case of a disjoint (u,v) and (w,x), no action is taken. 94 | */ 95 | public RangeResourceEntry add(ResourceEntry resourceEntry) { 96 | RangeResourceEntry rangeResourceEntry = (RangeResourceEntry) resourceEntry; 97 | 98 | if (rangeResourceEntry.getEnd() < rangeResourceEntry.getEnd() || 99 | rangeResourceEntry.getBegin() > this.end + 1 || rangeResourceEntry.getEnd() < this.begin - 1) { 100 | return this; 101 | } 102 | 103 | if (rangeResourceEntry.getBegin() < this.begin) { 104 | this.begin = rangeResourceEntry.getBegin(); 105 | } 106 | if (rangeResourceEntry.getEnd() > this.end) { 107 | this.end = rangeResourceEntry.getEnd(); 108 | } 109 | return this; 110 | } 111 | 112 | /** 113 | * Unused Method - Exists for the sake of completeness in terms of implementing ResourceEntry. 114 | */ 115 | public RangeResourceEntry remove(ResourceEntry resourceEntry) { 116 | RangeResourceEntry rangeResourceEntry = (RangeResourceEntry) resourceEntry; 117 | if (this.begin < rangeResourceEntry.getBegin()) { 118 | this.begin = rangeResourceEntry.getBegin(); 119 | } 120 | 121 | if (this.end > rangeResourceEntry.getEnd()) { 122 | this.end = rangeResourceEntry.getEnd(); 123 | } 124 | return this; 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /storm/src/main/storm/mesos/schedulers/SchedulerUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.schedulers; 19 | 20 | import org.apache.storm.scheduler.SupervisorDetails; 21 | import org.apache.storm.scheduler.TopologyDetails; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import storm.mesos.resources.AggregatedOffers; 26 | import storm.mesos.resources.ResourceNotAvailableException; 27 | import storm.mesos.resources.ResourceType; 28 | import storm.mesos.util.MesosCommon; 29 | 30 | import java.util.ArrayList; 31 | import java.util.Collection; 32 | import java.util.List; 33 | import java.util.Map; 34 | 35 | import static storm.mesos.resources.ResourceEntries.RangeResourceEntry; 36 | import static storm.mesos.resources.ResourceEntries.ScalarResourceEntry; 37 | 38 | public class SchedulerUtils { 39 | 40 | private static final Logger log = LoggerFactory.getLogger(SchedulerUtils.class); 41 | 42 | public static List getPorts(AggregatedOffers aggregatedOffers, int requiredCount) { 43 | List retVal = new ArrayList<>(); 44 | List resourceEntryList = aggregatedOffers.getAllAvailableResources(ResourceType.PORTS); 45 | 46 | for (RangeResourceEntry rangeResourceEntry : resourceEntryList) { 47 | Long begin = rangeResourceEntry.getBegin(); 48 | Long end = rangeResourceEntry.getEnd(); 49 | while (begin <= end && requiredCount > 0) { 50 | retVal.add(new RangeResourceEntry(begin, begin)); 51 | ++begin; 52 | --requiredCount; 53 | } 54 | if (requiredCount <= 0) { 55 | break; 56 | } 57 | } 58 | return retVal; 59 | } 60 | 61 | public static MesosWorkerSlot createMesosWorkerSlot(Map mesosStormConf, 62 | AggregatedOffers aggregatedOffers, 63 | TopologyDetails topologyDetails, 64 | boolean supervisorExists) throws ResourceNotAvailableException { 65 | 66 | double requestedWorkerCpu = MesosCommon.topologyWorkerCpu(mesosStormConf, topologyDetails); 67 | double requestedWorkerMem = MesosCommon.topologyWorkerMem(mesosStormConf, topologyDetails); 68 | 69 | requestedWorkerCpu += supervisorExists ? 0 : MesosCommon.executorCpu(mesosStormConf); 70 | requestedWorkerMem += supervisorExists ? 0 : MesosCommon.executorMem(mesosStormConf); 71 | 72 | aggregatedOffers.reserve(ResourceType.CPU, new ScalarResourceEntry(requestedWorkerCpu)); 73 | aggregatedOffers.reserve(ResourceType.MEM, new ScalarResourceEntry(requestedWorkerMem)); 74 | 75 | List ports = getPorts(aggregatedOffers, 1); 76 | if (ports.isEmpty()) { 77 | throw new ResourceNotAvailableException("No ports available to create MesosWorkerSlot."); 78 | } 79 | aggregatedOffers.reserve(ResourceType.PORTS, ports.get(0)); 80 | 81 | return new MesosWorkerSlot(aggregatedOffers.getHostname(), ports.get(0).getBegin(), topologyDetails.getId()); 82 | } 83 | 84 | /** 85 | * Check if this topology already has a supervisor running on the node where the Offer 86 | * comes from. Required to account for supervisor/mesos-executor's resource needs. 87 | * Note that there is one-and-only-one supervisor per topology per node. 88 | * 89 | * @param offerHost host that sent this Offer 90 | * @param existingSupervisors List of supervisors which already exist on the Offer's node 91 | * @param topologyId ID of topology requiring assignment 92 | * @return boolean value indicating supervisor existence 93 | */ 94 | public static boolean supervisorExists(String frameworkName, String offerHost, Collection existingSupervisors, 95 | String topologyId) { 96 | String expectedSupervisorId = MesosCommon.supervisorId(frameworkName, offerHost, topologyId); 97 | for (SupervisorDetails supervisorDetail : existingSupervisors) { 98 | if (supervisorDetail.getId().equals(expectedSupervisorId)) { 99 | return true; 100 | } 101 | } 102 | return false; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /storm/src/test/storm/mesos/resources/ResourceEntryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package storm.mesos.resources; 19 | 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.mockito.runners.MockitoJUnitRunner; 23 | 24 | import static org.junit.Assert.assertTrue; 25 | import static storm.mesos.resources.ResourceEntries.RangeResourceEntry; 26 | 27 | @RunWith(MockitoJUnitRunner.class) 28 | public class ResourceEntryTest { 29 | RangeResourceEntry existingRange; 30 | 31 | private boolean isEqual(RangeResourceEntry r1, RangeResourceEntry r2) { 32 | return r1.getBegin().equals(r2.getBegin()) && r1.getEnd().equals(r2.getEnd()); 33 | } 34 | 35 | @Test 36 | public void testDisjointAndNonAdjacentRanges() { 37 | Long beginPort = 1000l; 38 | Long endPort = 2000l; 39 | 40 | existingRange = new RangeResourceEntry(beginPort, endPort); 41 | RangeResourceEntry originalRange = new RangeResourceEntry(beginPort, endPort); 42 | 43 | RangeResourceEntry newRange = new RangeResourceEntry(beginPort - 1000, beginPort - 2); 44 | existingRange.add(newRange); 45 | assertTrue(isEqual(originalRange, existingRange)); 46 | 47 | newRange = new RangeResourceEntry(endPort + 2, endPort + 1000); 48 | existingRange.add(newRange); 49 | assertTrue(isEqual(originalRange, existingRange)); 50 | } 51 | 52 | @Test 53 | public void testDisjointAndAdjacentRanges() { 54 | Long beginPort = 1000l; 55 | Long endPort = 2000l; 56 | 57 | existingRange = new RangeResourceEntry(beginPort, endPort); 58 | 59 | RangeResourceEntry newRange = new RangeResourceEntry(endPort + 1, endPort + 1000); 60 | RangeResourceEntry expectedRange = new RangeResourceEntry(beginPort, endPort + 1000); 61 | existingRange.add(newRange); 62 | assertTrue(isEqual(expectedRange, existingRange)); 63 | 64 | existingRange = new RangeResourceEntry(beginPort, endPort); 65 | newRange = new RangeResourceEntry(beginPort - 1000, beginPort - 1); 66 | expectedRange = new RangeResourceEntry(beginPort - 1000, endPort); 67 | existingRange.add(newRange); 68 | assertTrue(isEqual(expectedRange, existingRange)); 69 | } 70 | 71 | @Test 72 | public void testOverlappingRanges() { 73 | Long beginPort = 1000l; 74 | Long endPort = 2000l; 75 | RangeResourceEntry newRange; 76 | RangeResourceEntry expectedRange; 77 | 78 | existingRange = new RangeResourceEntry(beginPort, endPort); 79 | 80 | existingRange = new RangeResourceEntry(beginPort, endPort); 81 | newRange = new RangeResourceEntry(beginPort - 500, beginPort + 500); 82 | expectedRange = new RangeResourceEntry(beginPort - 500, endPort); 83 | existingRange.add(newRange); 84 | assertTrue(isEqual(expectedRange, existingRange)); 85 | 86 | existingRange = new RangeResourceEntry(beginPort, endPort); 87 | newRange = new RangeResourceEntry(beginPort + 500, endPort + 500); 88 | expectedRange = new RangeResourceEntry(beginPort, endPort + 500); 89 | existingRange.add(newRange); 90 | assertTrue(isEqual(expectedRange, existingRange)); 91 | } 92 | 93 | @Test 94 | public void testSubSets() { 95 | Long beginPort = 1000l; 96 | Long endPort = 2000l; 97 | RangeResourceEntry newRange; 98 | RangeResourceEntry expectedRange; 99 | 100 | existingRange = new RangeResourceEntry(beginPort, endPort); 101 | 102 | newRange = new RangeResourceEntry(beginPort + 250, beginPort + 500); 103 | expectedRange = new RangeResourceEntry(beginPort, endPort); 104 | existingRange.add(newRange); 105 | assertTrue(isEqual(expectedRange, existingRange)); 106 | } 107 | 108 | @Test 109 | public void testSuperSets() { 110 | Long beginPort = 1000l; 111 | Long endPort = 2000l; 112 | RangeResourceEntry newRange; 113 | RangeResourceEntry expectedRange; 114 | 115 | existingRange = new RangeResourceEntry(beginPort, endPort); 116 | 117 | newRange = new RangeResourceEntry(beginPort - 250, endPort + 250); 118 | expectedRange = new RangeResourceEntry(beginPort - 250, endPort + 250); 119 | existingRange.add(newRange); 120 | assertTrue(isEqual(expectedRange, existingRange)); 121 | } 122 | 123 | @Test 124 | public void testInvalidRange() { 125 | Long beginPort = 1000l; 126 | Long endPort = 2000l; 127 | RangeResourceEntry newRange; 128 | RangeResourceEntry expectedRange; 129 | 130 | existingRange = new RangeResourceEntry(beginPort, endPort); 131 | 132 | newRange = new RangeResourceEntry(endPort, beginPort); 133 | expectedRange = new RangeResourceEntry(beginPort, endPort); 134 | existingRange.add(newRange); 135 | assertTrue(isEqual(expectedRange, existingRange)); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # # vi: set ft=ruby : 3 | 4 | VAGRANTFILE_API_VERSION = "2" 5 | 6 | $provision_script = <