├── dev ├── provision │ ├── .gitignore │ ├── install_compiler.sh │ ├── format_namenode.sh │ ├── start_datanode.sh │ ├── install_docker.sh │ ├── start_namenode.sh │ ├── install_default_jdk.sh │ ├── install_default_jre_headless.sh │ ├── install_java_8.sh │ ├── start_mesos_master.sh │ ├── install_mesos.sh │ ├── start_mesos_master_debian.sh │ ├── start_mesos_slave.sh │ ├── start_mesos_slave_debian.sh │ └── install_hadoop.sh ├── .gitignore ├── aws │ ├── upload_hdfs.sh │ ├── AWS_framework.json │ ├── init_framework.sh │ ├── start_framework.sh │ └── AWS_cluster.json ├── config │ ├── framework.json │ ├── cluster.json │ ├── FrameworkAndAccumulo.json │ └── AWS_framework.json ├── start_aws_framework.sh ├── README.md ├── init_framework.sh ├── upload_hdfs.sh └── start_framework.sh ├── accumulo-mesos-executor ├── src │ └── main │ │ ├── resources │ │ ├── accumulo.yaml │ │ ├── accumulo.json │ │ └── log4j.xml │ │ └── java │ │ └── aredee │ │ └── mesos │ │ └── frameworks │ │ └── accumulo │ │ └── executor │ │ ├── package-info.java │ │ └── Main.java └── pom.xml ├── accumulo-mesos-framework ├── src │ └── main │ │ ├── resources │ │ ├── webapp │ │ │ ├── swagger │ │ │ │ ├── swagger-ui.version │ │ │ │ └── dist │ │ │ │ │ ├── images │ │ │ │ │ ├── favicon.ico │ │ │ │ │ ├── throbber.gif │ │ │ │ │ ├── logo_small.png │ │ │ │ │ ├── favicon-16x16.png │ │ │ │ │ ├── favicon-32x32.png │ │ │ │ │ ├── pet_store_api.png │ │ │ │ │ ├── wordnik_api.png │ │ │ │ │ └── explorer_icons.png │ │ │ │ │ ├── fonts │ │ │ │ │ ├── droid-sans-v6-latin-700.eot │ │ │ │ │ ├── droid-sans-v6-latin-700.ttf │ │ │ │ │ ├── droid-sans-v6-latin-700.woff │ │ │ │ │ ├── droid-sans-v6-latin-700.woff2 │ │ │ │ │ ├── droid-sans-v6-latin-regular.eot │ │ │ │ │ ├── droid-sans-v6-latin-regular.ttf │ │ │ │ │ ├── droid-sans-v6-latin-regular.woff │ │ │ │ │ └── droid-sans-v6-latin-regular.woff2 │ │ │ │ │ ├── lib │ │ │ │ │ ├── jquery.slideto.min.js │ │ │ │ │ ├── jquery.wiggle.min.js │ │ │ │ │ └── jquery.ba-bbq.min.js │ │ │ │ │ ├── o2c.html │ │ │ │ │ ├── css │ │ │ │ │ ├── typography.css │ │ │ │ │ ├── reset.css │ │ │ │ │ └── style.css │ │ │ │ │ ├── lang │ │ │ │ │ ├── translator.js │ │ │ │ │ ├── ru.js │ │ │ │ │ ├── en.js │ │ │ │ │ ├── pt.js │ │ │ │ │ └── es.js │ │ │ │ │ └── index.html │ │ │ └── public │ │ │ │ └── index.html │ │ └── log4j.properties │ │ └── java │ │ └── aredee │ │ └── mesos │ │ └── frameworks │ │ └── accumulo │ │ └── framework │ │ ├── package-info.java │ │ ├── api │ │ ├── ApiException.java │ │ ├── ConfigApiService.java │ │ ├── DefaultApiService.java │ │ ├── StatusApiService.java │ │ ├── NotFoundException.java │ │ ├── EchoResource.java │ │ ├── impl │ │ │ ├── ConfigApiServiceImpl.java │ │ │ ├── DefaultApiServiceImpl.java │ │ │ ├── StatusApiServiceImpl.java │ │ │ └── ClusterApiServiceImpl.java │ │ ├── ApiOriginFilter.java │ │ ├── DefaultApi.java │ │ ├── ApiServiceFactory.java │ │ ├── StatusApi.java │ │ ├── ConfigApi.java │ │ ├── ClusterApiService.java │ │ ├── ApiResponseMessage.java │ │ ├── WebServer.java │ │ └── ClusterApi.java │ │ └── guice │ │ ├── ConfigurationModule.java │ │ └── ApiServletModule.java └── pom.xml ├── accumulo-mesos-scheduler ├── src │ ├── main │ │ └── java │ │ │ └── aredee │ │ │ └── mesos │ │ │ └── frameworks │ │ │ └── accumulo │ │ │ └── scheduler │ │ │ ├── package-info.java │ │ │ ├── launcher │ │ │ └── Launcher.java │ │ │ └── matcher │ │ │ ├── Matcher.java │ │ │ ├── MatchUtils.java │ │ │ ├── Match.java │ │ │ └── MinCpuMinRamFIFOMatcher.java │ └── test │ │ └── java │ │ └── aredee │ │ └── mesos │ │ └── frameworks │ │ └── accumulo │ │ └── scheduler │ │ └── matcher │ │ └── MinCpuMinRamFIFOMatcherTest.java └── pom.xml ├── docker ├── docker-compose.yml ├── config │ ├── framework.json-sample │ └── cluster.json-sample ├── build-framwork.sh ├── accumulo.dockerfile └── bin │ ├── install-accumulo.sh │ └── start-framework.sh ├── accumulo-mesos-common ├── src │ ├── main │ │ ├── resources │ │ │ └── accumulo-mesos.properties │ │ └── java │ │ │ └── aredee │ │ │ └── mesos │ │ │ └── frameworks │ │ │ └── accumulo │ │ │ ├── model │ │ │ ├── Monitor.java │ │ │ ├── Error.java │ │ │ ├── IdRegistry.java │ │ │ ├── ServerGroup.java │ │ │ ├── Task.java │ │ │ ├── Accumulo.java │ │ │ ├── ServerProfile.java │ │ │ └── Framework.java │ │ │ ├── configuration │ │ │ ├── Constants.java │ │ │ ├── Defaults.java │ │ │ ├── Environment.java │ │ │ └── CommandLineHandler.java │ │ │ └── initialize │ │ │ └── AccumuloInitializer.java │ └── test │ │ ├── resources │ │ ├── model │ │ │ ├── FrameworkOnly.json │ │ │ ├── AccumuloOnly.json │ │ │ └── FrameworkAndAccumulo.json │ │ └── state │ │ │ └── ZkTestFramework.json │ │ └── java │ │ └── aredee │ │ └── mesos │ │ └── frameworks │ │ └── accumulo │ │ ├── configuration │ │ └── file │ │ │ └── TestSiteXml.java │ │ ├── state │ │ └── FrameworkStateHelperTest.java │ │ └── model │ │ └── FrameworkTest.java └── pom.xml ├── accumulo-mesos-dist ├── src │ └── main │ │ └── assembly │ │ └── tarball.xml └── pom.xml ├── .gitignore ├── README.md └── Vagrantfile /dev/provision/.gitignore: -------------------------------------------------------------------------------- 1 | **/tarballs 2 | 3 | -------------------------------------------------------------------------------- /dev/.gitignore: -------------------------------------------------------------------------------- 1 | **/dist 2 | **/provision/tarballs 3 | 4 | 5 | -------------------------------------------------------------------------------- /accumulo-mesos-executor/src/main/resources/accumulo.yaml: -------------------------------------------------------------------------------- 1 | 2 | maxMemory:2048.0 -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/swagger-ui.version: -------------------------------------------------------------------------------- 1 | 2.1.2 2 | -------------------------------------------------------------------------------- /accumulo-mesos-executor/src/main/java/aredee/mesos/frameworks/accumulo/executor/package-info.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.executor; -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/package-info.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework; -------------------------------------------------------------------------------- /dev/provision/install_compiler.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | 4 | set -e 5 | 6 | 7 | apt-get -y install g++ 8 | apt-get -y install make 9 | 10 | -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/src/main/java/aredee/mesos/frameworks/accumulo/scheduler/package-info.java: -------------------------------------------------------------------------------- 1 | 2 | package aredee.mesos.frameworks.accumulo.scheduler; -------------------------------------------------------------------------------- /dev/provision/format_namenode.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | set -e 4 | 5 | # Format NameNode 6 | sudo -u hduser sh -c 'yes Y | /usr/local/hadoop/bin/hdfs namenode -format' 7 | -------------------------------------------------------------------------------- /dev/provision/start_datanode.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | set -e 4 | 5 | # Start DataNode 6 | sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh start datanode' 7 | 8 | 9 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | accumulo: 2 | build: . 3 | dockerfile: accumulo.dockerfile 4 | net: host 5 | volumes: 6 | - ./lib:/accumulo-lib:rw 7 | - ./config:/accumulo-config 8 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/resources/accumulo-mesos.properties: -------------------------------------------------------------------------------- 1 | application.version=${project.version} 2 | application.name=accumulo-mesos 3 | application.name.executor=accumulo-mesos-executor 4 | -------------------------------------------------------------------------------- /accumulo-mesos-executor/src/main/resources/accumulo.json: -------------------------------------------------------------------------------- 1 | { 2 | maxMemory:2048.0, 3 | zooKeeperDir:"/usr/local/zookeeper", 4 | "systemProperties":{"java.home":"/usr/java/jdk1.7",ACCUMULO_HOME:"/usr/local/accumulo"} 5 | } -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/favicon.ico -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/throbber.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/throbber.gif -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/logo_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/logo_small.png -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/favicon-16x16.png -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/favicon-32x32.png -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/pet_store_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/pet_store_api.png -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/wordnik_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/wordnik_api.png -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/explorer_icons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/images/explorer_icons.png -------------------------------------------------------------------------------- /dev/aws/upload_hdfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | echo "Uploading files to HDFS" 5 | hadoop fs -copyFromLocal -f /home/ubuntu/klucar/*.gz /user/klucar/. 6 | hadoop fs -copyFromLocal -f /home/ubuntu/klucar/libaccumulo.so /user/klucar/. 7 | -------------------------------------------------------------------------------- /dev/provision/install_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | set -e 4 | 5 | #Install docker 6 | echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list 7 | apt-get update 8 | apt-get -y install docker.io 9 | -------------------------------------------------------------------------------- /dev/provision/start_namenode.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | set -e 4 | 5 | # Start NameNode 6 | sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh start namenode' 7 | 8 | sudo -u hduser sh -c "/usr/local/hadoop/bin/hadoop fs -chmod 777 /" -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.eot -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.ttf -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.woff -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-700.woff2 -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.eot -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.ttf -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.woff -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aredee/accumulo-mesos/HEAD/accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/fonts/droid-sans-v6-latin-regular.woff2 -------------------------------------------------------------------------------- /dev/aws/AWS_framework.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "0.0.0.0", 3 | "httpPort": "18192", 4 | "mesosMaster": "172.31.1.11:5050", 5 | "name":"accumulo-mesos-aws-test-1", 6 | "tarballUri": "hdfs://172.31.1.11:54310/user/klucar/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 7 | "zkServers": "172.31.0.11:2181" 8 | } 9 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mesos-accumulo 6 | 7 | 8 | 9 |

Mesos-Accumulo

10 |

This is the mesos framework for Accumulo.

11 | 12 | 13 | -------------------------------------------------------------------------------- /dev/config/framework.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "172.16.0.100", 3 | "httpPort": "8192", 4 | "mesosMaster": "172.16.0.100:5050", 5 | "name":"accumulo-mesos-test-4", 6 | "id": "", 7 | "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 8 | "zkServers": "172.16.0.100:2181" 9 | } 10 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/resources/model/FrameworkOnly.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "1.1.1.1", 3 | "httpPort": "1234", 4 | "mesosMaster": "1.2.3.4:5150", 5 | "name":"accumulo-mesos-test", 6 | "id": "", 7 | "tarballUri": "hdfs://localhost:9000/data/accumulo-mesos.tar.gz", 8 | "zkServers": "server1:2181,server2:2181,server3:2181" 9 | } -------------------------------------------------------------------------------- /docker/config/framework.json-sample: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "0.0.0.0", 3 | "httpPort": "8192", 4 | "mesosMaster": "172.31.45.229:5050", 5 | "name":"accumulo-mesos-test-4", 6 | "id": "", 7 | "tarballUri": "hdfs://172.31.45.229:54310/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 8 | "zkServers": "172.31.20.165:2181" 9 | } 10 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ApiException.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | public class ApiException extends Exception{ 4 | private int code; 5 | public ApiException (int code, String msg) { 6 | super(msg); 7 | this.code = code; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ConfigApiService.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.core.Response; 4 | 5 | public abstract class ConfigApiService { 6 | 7 | public abstract Response configGet() 8 | throws NotFoundException; 9 | 10 | } 11 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/DefaultApiService.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.core.Response; 4 | 5 | public abstract class DefaultApiService { 6 | 7 | public abstract Response rootGet() 8 | throws NotFoundException; 9 | 10 | } 11 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/StatusApiService.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.core.Response; 4 | 5 | public abstract class StatusApiService { 6 | 7 | public abstract Response statusGet() 8 | throws NotFoundException; 9 | 10 | } 11 | -------------------------------------------------------------------------------- /docker/build-framwork.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd .. 3 | mvn package 4 | 5 | cd docker 6 | if [ ! -f "lib" ]; then 7 | mkdir lib 8 | fi 9 | echo "Copying libraries..." 10 | cp ../accumulo-mesos-framework/target/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar lib/ 11 | cp ../accumulo-mesos-dist/target/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz lib/ 12 | 13 | -------------------------------------------------------------------------------- /dev/provision/install_default_jdk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | 4 | PREFIX="JAVA Default JDK Provisioner:" 5 | set -e 6 | 7 | # For installing Java 8 8 | apt-get -y update 9 | apt-get -y install default-jdk 10 | 11 | if $(test -e /usr/lib/libjvm.so); then 12 | rm /usr/lib/libjvm.so 13 | fi 14 | 15 | ln -s /usr/lib/jvm/default-java/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so 16 | 17 | -------------------------------------------------------------------------------- /dev/provision/install_default_jre_headless.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | 4 | PREFIX="JAVA Default JDK Provisioner:" 5 | set -e 6 | 7 | # For installing Java 8 8 | apt-get -y update 9 | apt-get -y install default-jre-headless 10 | 11 | if $(test -e /usr/lib/libjvm.so); then 12 | rm /usr/lib/libjvm.so 13 | fi 14 | 15 | ln -s /usr/lib/jvm/default-java/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so 16 | 17 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/NotFoundException.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | public class NotFoundException extends aredee.mesos.frameworks.accumulo.framework.api.ApiException { 4 | private int code; 5 | public NotFoundException (int code, String msg) { 6 | super(code, msg); 7 | this.code = code; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | 10 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lib/jquery.slideto.min.js: -------------------------------------------------------------------------------- 1 | (function(b){b.fn.slideto=function(a){a=b.extend({slide_duration:"slow",highlight_duration:3E3,highlight:true,highlight_color:"#FFFF99"},a);return this.each(function(){obj=b(this);b("body").animate({scrollTop:obj.offset().top},a.slide_duration,function(){a.highlight&&b.ui.version&&obj.effect("highlight",{color:a.highlight_color},a.highlight_duration)})})}})(jQuery); 2 | -------------------------------------------------------------------------------- /dev/start_aws_framework.sh: -------------------------------------------------------------------------------- 1 | export ACCUMULO_HOME=/home/ubuntu/klucar/accumulo-1.7.0 2 | export ACCUMULO_CLIENT_CONF_PATH=$ACCUMULO_HOME/conf 3 | export HADOOP_PREFIX=/usr/lib/hadoop 4 | export HADOOP_CONF_DIR=/etc/hadoop 5 | export ZOOKEEPER_HOME=/etc/zookeeper 6 | 7 | java -jar /home/ubuntu/klucar/accumulo-mesos-dist-0.2.0-SNAPSHOT/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar \ 8 | -f /home/ubuntu/klucar/AWS_framework.json \ 9 | | tee $LOG 10 | 11 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/EchoResource.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | 4 | import javax.ws.rs.GET; 5 | import javax.ws.rs.Path; 6 | import javax.ws.rs.Produces; 7 | import javax.ws.rs.core.MediaType; 8 | 9 | @Path("/echo") 10 | @Produces(MediaType.TEXT_PLAIN) 11 | public class EchoResource { 12 | 13 | @GET 14 | public String echo() { 15 | return new String("Hello World!"); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/Monitor.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import java.util.ArrayList; 4 | 5 | public class Monitor extends ArrayList { 6 | 7 | @Override 8 | public String toString() { 9 | StringBuilder sb = new StringBuilder(); 10 | sb.append("class Monitor {\n"); 11 | sb.append(" " + super.toString()).append("\n"); 12 | sb.append("}\n"); 13 | return sb.toString(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dev/README.md: -------------------------------------------------------------------------------- 1 | /dev contains scripts and config used to develop and test the framework. 2 | 3 | The main Vagrant file reads scripts from the provision directory to provision 4 | a cluster running mesos, zookeeper and hdfs. 5 | 6 | The dist directory contains archives to be uploaded to hdfs (accumulo, executor, etc) 7 | 8 | The scripts here are meant to be run from inside the vagrant machines (i.e. vagrant ssh master ... cd /vagrant/dev ... do whatever) 9 | 10 | Install /etc/hosts/ of all machines running. 11 | vagrant plugin install vagrant-hostmanager 12 | vagrant hostmanager 13 | 14 | -------------------------------------------------------------------------------- /dev/init_framework.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LOG=/tmp/accumulo-framework.log 4 | 5 | export ACCUMULO_HOME=/vagrant/dev/dist/accumulo-1.7.0 6 | export ACCUMULO_CLIENT_CONF_PATH=$ACCUMULO_HOME/conf 7 | export HADOOP_PREFIX=/usr/local/hadoop 8 | export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop 9 | export ZOOKEEPER_HOME=/etc/zookeeper 10 | 11 | java -jar /vagrant/dev/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar \ 12 | -i -fc /vagrant/dev/config/framework.json -cc /vagrant/dev/config/cluster.json \ 13 | | tee $LOG 14 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/o2c.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dev/provision/install_java_8.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | 4 | PREFIX="JAVA 8 Provisioner:" 5 | set -e 6 | 7 | # For installing Java 8 8 | add-apt-repository ppa:webupd8team/java 9 | apt-get -y update 10 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections 11 | # apt-get -y install default-jdk 12 | apt-get -y install oracle-java8-installer 13 | apt-get -y install oracle-java8-set-default 14 | 15 | if $(test -e /usr/lib/libjvm.so); then 16 | rm /usr/lib/libjvm.so 17 | fi 18 | ln -s /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so 19 | 20 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/guice/ConfigurationModule.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.guice; 2 | 3 | import aredee.mesos.frameworks.accumulo.model.Framework; 4 | import com.google.inject.AbstractModule; 5 | 6 | public class ConfigurationModule extends AbstractModule { 7 | 8 | private final Framework config; 9 | 10 | public ConfigurationModule(Framework config){ 11 | this.config = config; 12 | } 13 | 14 | @Override 15 | protected void configure() { 16 | bind(Framework.class).toInstance(config); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /dev/aws/init_framework.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LOG=/tmp/accumulo-framework.log 4 | 5 | export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 6 | export ACCUMULO_HOME=/home/ubuntu/klucar/accumulo-1.7.0 7 | export ACCUMULO_CLIENT_CONF_PATH=$ACCUMULO_HOME/conf 8 | export HADOOP_PREFIX=/usr/lib/hadoop 9 | export HADOOP_CONF_DIR=/etc/hadoop 10 | export ZOOKEEPER_HOME=/etc/zookeeper 11 | 12 | java -jar /home/ubuntu/klucar/accumulo-mesos-dist-0.2.0-SNAPSHOT/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar \ 13 | -i -fc /home/ubuntu/klucar/AWS_framework.json -cc /home/ubuntu/klucar/AWS_cluster.json \ 14 | | tee $LOG 15 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/impl/ConfigApiServiceImpl.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api.impl; 2 | 3 | import aredee.mesos.frameworks.accumulo.framework.api.ConfigApiService; 4 | import aredee.mesos.frameworks.accumulo.framework.api.NotFoundException; 5 | import aredee.mesos.frameworks.accumulo.scheduler.Cluster; 6 | 7 | import javax.ws.rs.core.Response; 8 | 9 | public class ConfigApiServiceImpl extends ConfigApiService { 10 | 11 | Cluster cluster = Cluster.INSTANCE; 12 | 13 | @Override 14 | public Response configGet() throws NotFoundException { 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/impl/DefaultApiServiceImpl.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api.impl; 2 | 3 | import aredee.mesos.frameworks.accumulo.framework.api.DefaultApiService; 4 | import aredee.mesos.frameworks.accumulo.framework.api.NotFoundException; 5 | import aredee.mesos.frameworks.accumulo.scheduler.Cluster; 6 | 7 | import javax.ws.rs.core.Response; 8 | 9 | public class DefaultApiServiceImpl extends DefaultApiService { 10 | 11 | Cluster cluster = Cluster.INSTANCE; 12 | 13 | @Override 14 | public Response rootGet() throws NotFoundException { 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/impl/StatusApiServiceImpl.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api.impl; 2 | 3 | import aredee.mesos.frameworks.accumulo.framework.api.NotFoundException; 4 | import aredee.mesos.frameworks.accumulo.framework.api.StatusApiService; 5 | import aredee.mesos.frameworks.accumulo.scheduler.Cluster; 6 | 7 | import javax.ws.rs.core.Response; 8 | 9 | public class StatusApiServiceImpl extends StatusApiService { 10 | 11 | Cluster cluster = Cluster.INSTANCE; 12 | 13 | @Override 14 | public Response statusGet() throws NotFoundException { 15 | return null; 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /dev/provision/start_mesos_master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | # $1 is the master ip 4 | 5 | echo "Starting master!! $1" > startmaster.log 6 | 7 | echo "zk://$1:2181/mesos" > /etc/mesos/zk 8 | #echo "$@ master" >> /etc/hosts 9 | #echo "192.168.50.102 slave" >> /etc/hosts 10 | echo $1 | sudo tee /etc/mesos-master/ip 11 | echo $1 | sudo tee /etc/mesos-master/hostname 12 | 13 | echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc 14 | echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc 15 | echo "export PATH=$PATH:$HADOOP_HOME/bin:$JAVA_HOME/bin" >> /root/.bashrc 16 | 17 | # keep mesos slave from starting here 18 | echo manual | sudo tee /etc/init/mesos-slave.override 19 | 20 | sudo start mesos-master 21 | -------------------------------------------------------------------------------- /dev/provision/install_mesos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | PREFIX="Mesos Provisioner: " 4 | set -e 5 | 6 | echo "${PREFIX} Installing pre-reqs..." 7 | # For Mesos 8 | apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF 9 | DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') 10 | CODENAME=$(lsb_release -cs) 11 | echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list 12 | apt-get -y update 13 | 14 | apt-get -y install libcurl3 15 | apt-get -y install zookeeperd 16 | apt-get -y install aria2 17 | apt-get -y install ssh 18 | apt-get -y install rsync 19 | 20 | 21 | MESOS_VERSION="0.22.1" 22 | echo "${PREFIX}Installing mesos version: ${MESOS_VERSION}..." 23 | apt-get -y install mesos 24 | -------------------------------------------------------------------------------- /dev/provision/start_mesos_master_debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | # $1 is the master ip 4 | 5 | echo "Starting master!! $1" > startmaster.log 6 | 7 | echo "zk://$1:2181/mesos" > /etc/mesos/zk 8 | #echo "$@ master" >> /etc/hosts 9 | #echo "192.168.50.102 slave" >> /etc/hosts 10 | echo $1 | sudo tee /etc/mesos-master/ip 11 | echo $1 | sudo tee /etc/mesos-master/hostname 12 | 13 | echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc 14 | echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc 15 | echo "export PATH=$PATH:$HADOOP_HOME/bin:$JAVA_HOME/bin" >> /root/.bashrc 16 | 17 | # keep mesos slave from starting here 18 | echo manual | sudo tee /etc/init/mesos-slave.override 19 | 20 | sudo service mesos-master start 21 | -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/src/main/java/aredee/mesos/frameworks/accumulo/scheduler/launcher/Launcher.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.scheduler.launcher; 2 | 3 | import aredee.mesos.frameworks.accumulo.scheduler.matcher.Match; 4 | import org.apache.mesos.Protos; 5 | import org.apache.mesos.SchedulerDriver; 6 | 7 | /** 8 | * Interface to launch a server based on a Mesos offer. 9 | */ 10 | public interface Launcher { 11 | 12 | /** 13 | * Interface used to launch Accumulo Server tasks. 14 | * 15 | * @param driver Mesos interface to use to launch a server 16 | * @param match {@link Match} to launch. A Match must contain a server and an offer 17 | */ 18 | public Protos.TaskInfo launch(SchedulerDriver driver, Match match); 19 | } 20 | -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/src/main/java/aredee/mesos/frameworks/accumulo/scheduler/matcher/Matcher.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.scheduler.matcher; 2 | 3 | import aredee.mesos.frameworks.accumulo.model.Task; 4 | import org.apache.mesos.Protos; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Responsible for matching required servers to available offers 10 | */ 11 | public interface Matcher { 12 | /** 13 | Returns a list of matched servers and offers. If offers were not found for all servers, 14 | a Match object will be present with no offer 15 | 16 | @param tasks tasks that require resources 17 | @param offers to match servers against 18 | */ 19 | public List matchOffers(List tasks, List offers); 20 | } 21 | -------------------------------------------------------------------------------- /dev/upload_hdfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Cleaning local files" 4 | rm -rf /vagrant/dev/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT 5 | rm /vagrant/dev/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz 6 | echo "Copying archives from build, expanding" 7 | cp /vagrant/accumulo-mesos-dist/target/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz /vagrant/dev/dist/. 8 | tar xzf /vagrant/dev/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz -C /vagrant/dev/dist 9 | 10 | TEST=`hadoop fs -ls /dist` 2>&1 11 | if [ -z "$TEST" ]; then 12 | echo "Creating dist directory" 13 | hadoop fs -mkdir /dist 14 | else 15 | echo "/dist already exists... skipping" 16 | fi 17 | 18 | echo "Uploading files to HDFS" 19 | hadoop fs -copyFromLocal -f /vagrant/dev/dist/*.gz /dist/. 20 | hadoop fs -copyFromLocal -f /vagrant/dev/dist/libaccumulo.so /dist/. 21 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lib/jquery.wiggle.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | jQuery Wiggle 3 | Author: WonderGroup, Jordan Thomas 4 | URL: http://labs.wondergroup.com/demos/mini-ui/index.html 5 | License: MIT (http://en.wikipedia.org/wiki/MIT_License) 6 | */ 7 | jQuery.fn.wiggle=function(o){var d={speed:50,wiggles:3,travel:5,callback:null};var o=jQuery.extend(d,o);return this.each(function(){var cache=this;var wrap=jQuery(this).wrap('
').css("position","relative");var calls=0;for(i=1;i<=o.wiggles;i++){jQuery(this).animate({left:"-="+o.travel},o.speed).animate({left:"+="+o.travel*2},o.speed*2).animate({left:"-="+o.travel},o.speed,function(){calls++;if(jQuery(cache).parent().hasClass('wiggle-wrap')){jQuery(cache).parent().replaceWith(cache);} 8 | if(calls==o.wiggles&&jQuery.isFunction(o.callback)){o.callback();}});}});}; -------------------------------------------------------------------------------- /dev/provision/start_mesos_slave.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | # args => $1 = ip of slave, $2 = ip of mesos master, $3 = hostname of slave 4 | 5 | set -e 6 | 7 | echo "Starting slave $@" > startslave.log 8 | echo "$1" > /etc/mesos-slave/ip 9 | echo "cgroups/cpu,cgroups/mem" > /etc/mesos-slave/isolation 10 | echo "mesos" > /etc/mesos-slave/containerizers 11 | echo "/usr/local/hadoop" > /etc/mesos-slave/hadoop_home 12 | 13 | echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc 14 | echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc 15 | 16 | echo "zk://$2:2181/mesos" | sudo tee /etc/mesos/zk 17 | echo $1 | sudo tee /etc/mesos-slave/hostname 18 | echo "cpus:2;mem:2048" | sudo tee /etc/mesos-slave/resources 19 | echo manual | sudo tee /etc/init/mesos-master.override 20 | echo manual | sudo tee /etc/init/zookeeper.override 21 | 22 | 23 | start mesos-slave 24 | -------------------------------------------------------------------------------- /dev/provision/start_mesos_slave_debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | # args => $1 = ip of slave, $2 = ip of mesos master, $3 = hostname of slave 4 | 5 | set -e 6 | 7 | echo "Starting slave $@" > startslave.log 8 | echo "$1" > /etc/mesos-slave/ip 9 | echo "cgroups/cpu,cgroups/mem" > /etc/mesos-slave/isolation 10 | echo "mesos" > /etc/mesos-slave/containerizers 11 | echo "/usr/local/hadoop" > /etc/mesos-slave/hadoop_home 12 | 13 | echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc 14 | echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc 15 | 16 | echo "zk://$2:2181/mesos" | sudo tee /etc/mesos/zk 17 | echo $1 | sudo tee /etc/mesos-slave/hostname 18 | echo "cpus:2;mem:2048" | sudo tee /etc/mesos-slave/resources 19 | echo manual | sudo tee /etc/init/mesos-master.override 20 | echo manual | sudo tee /etc/init/zookeeper.override 21 | 22 | 23 | sudo service mesos-slave start 24 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/guice/ApiServletModule.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.guice; 2 | 3 | import aredee.mesos.frameworks.accumulo.framework.api.*; 4 | import com.google.inject.Singleton; 5 | import com.google.inject.servlet.ServletModule; 6 | import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; 7 | import org.eclipse.jetty.servlet.DefaultServlet; 8 | 9 | public class ApiServletModule extends ServletModule { 10 | 11 | @Override 12 | protected void configureServlets() { 13 | bind(DefaultServlet.class).in(Singleton.class); 14 | 15 | bind(EchoResource.class); 16 | bind(ClusterApi.class); 17 | bind(ConfigApi.class); 18 | bind(DefaultApi.class); 19 | bind(StatusApi.class); 20 | 21 | serve("/api/*").with(GuiceContainer.class); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /dev/start_framework.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LOG=/tmp/accumulo-framework.log 4 | 5 | export ACCUMULO_HOME=/vagrant/dev/dist/accumulo-1.7.0 6 | export ACCUMULO_CLIENT_CONF_PATH=$ACCUMULO_HOME/conf 7 | export HADOOP_PREFIX=/usr/local/hadoop 8 | export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop 9 | export ZOOKEEPER_HOME=/etc/zookeeper 10 | 11 | java -jar /vagrant/dev/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar \ 12 | -master 172.16.0.100:5050 \ 13 | -zookeepers 172.16.0.100:2181 \ 14 | -name accumulo-mesos-test-1 \ 15 | | tee $LOG 16 | 17 | 18 | # "bindAddress": "172.16.0.100", 19 | # "httpPort": "8192", 20 | # "mesosMaster": "172.16.0.100:5050", 21 | # "name":"accumulo-mesos-test", 22 | # "id": "", 23 | # "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 24 | # "zkServers": "172.16.0.100:2181" 25 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/configuration/Constants.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.configuration; 2 | 3 | import java.util.ResourceBundle; 4 | 5 | public final class Constants { 6 | 7 | // bundler used for items defined by Maven 8 | private static final ResourceBundle rb = ResourceBundle.getBundle("accumulo-mesos"); 9 | public static final String FRAMEWORK_VERSION = rb.getString("application.version"); 10 | public static final String EXE_NAME = rb.getString("application.name"); 11 | public static final String EXECUTOR_JAR = rb.getString("application.name.executor") 12 | +"-" 13 | +FRAMEWORK_VERSION 14 | +"-jar-with-dependencies" 15 | +".jar"; 16 | 17 | public static final String ACCUMULO_MESOS_DISTRO = "accumulo-mesos-dist-"+FRAMEWORK_VERSION; 18 | public static final String ACCUMULO_NATIVE_LIB = "libaccumulo.so"; 19 | } 20 | -------------------------------------------------------------------------------- /dev/aws/start_framework.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LOG=/tmp/accumulo-framework.log 4 | 5 | export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 6 | export ACCUMULO_HOME=/home/ubuntu/klucar/accumulo-1.7.0 7 | export ACCUMULO_CLIENT_CONF_PATH=$ACCUMULO_HOME/conf 8 | export HADOOP_PREFIX=/usr/lib/hadoop 9 | export HADOOP_CONF_DIR=/etc/hadoop 10 | export ZOOKEEPER_HOME=/etc/zookeeper 11 | 12 | java -jar /home/ubuntu/klucar/accumulo-mesos-dist-0.2.0-SNAPSHOT/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar \ 13 | -master 172.31.1.11:5050 \ 14 | -zookeepers 172.31.0.11:2181 \ 15 | -name $1 \ 16 | | tee $LOG 17 | 18 | 19 | # "bindAddress": "172.16.0.100", 20 | # "httpPort": "8192", 21 | # "mesosMaster": "172.16.0.100:5050", 22 | # "name":"accumulo-mesos-test", 23 | # "id": "", 24 | # "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 25 | # "zkServers": "172.16.0.100:2181" 26 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ApiOriginFilter.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.servlet.*; 4 | import javax.servlet.http.HttpServletResponse; 5 | import java.io.IOException; 6 | 7 | public class ApiOriginFilter implements Filter { 8 | @Override 9 | public void doFilter(ServletRequest request, ServletResponse response, 10 | FilterChain chain) throws IOException, ServletException { 11 | HttpServletResponse res = (HttpServletResponse) response; 12 | res.addHeader("Access-Control-Allow-Origin", "*"); 13 | res.addHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT"); 14 | res.addHeader("Access-Control-Allow-Headers", "Content-Type"); 15 | chain.doFilter(request, response); 16 | } 17 | 18 | @Override 19 | public void destroy() { 20 | } 21 | 22 | @Override 23 | public void init(FilterConfig filterConfig) throws ServletException { 24 | } 25 | } -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/src/main/java/aredee/mesos/frameworks/accumulo/scheduler/matcher/MatchUtils.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.scheduler.matcher; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * Static helper methods for handling lists of Match objects 8 | */ 9 | public class MatchUtils { 10 | // Don't allow instantiation 11 | private MatchUtils(){} 12 | 13 | /** 14 | * Iterates through list to find Match objects with no corresponding Offer 15 | * 16 | * @param matches List to check for Matches without Offers 17 | * @return List of Match objects with no Offer set 18 | */ 19 | public static List getUnmatchedServers(List matches){ 20 | List noMatch = new ArrayList<>(); 21 | for( Match match: matches){ 22 | if( !match.hasOffer() ){ 23 | noMatch.add(match); 24 | } 25 | } 26 | return noMatch; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/DefaultApi.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | 4 | import javax.ws.rs.Consumes; 5 | import javax.ws.rs.GET; 6 | import javax.ws.rs.Path; 7 | import javax.ws.rs.Produces; 8 | import javax.ws.rs.core.Response; 9 | 10 | @Path("/") 11 | @Consumes({ "application/json" }) 12 | @Produces({ "application/json", "text/html" }) 13 | //@io.swagger.annotations.Api(value = "/", description = "the API") 14 | public class DefaultApi { 15 | 16 | private final DefaultApiService delegate = ApiServiceFactory.getDefaultApi(); 17 | 18 | @GET 19 | /* 20 | @io.swagger.annotations.ApiOperation(value = "", notes = "Returns Swagger UI for this API\n", response = Void.class) 21 | @io.swagger.annotations.ApiResponses(value = { 22 | @io.swagger.annotations.ApiResponse(code = 200, message = "Success") }) 23 | */ 24 | public Response rootGet() 25 | throws NotFoundException { 26 | return delegate.rootGet(); 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ApiServiceFactory.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import aredee.mesos.frameworks.accumulo.framework.api.impl.ClusterApiServiceImpl; 4 | import aredee.mesos.frameworks.accumulo.framework.api.impl.ConfigApiServiceImpl; 5 | import aredee.mesos.frameworks.accumulo.framework.api.impl.DefaultApiServiceImpl; 6 | import aredee.mesos.frameworks.accumulo.framework.api.impl.StatusApiServiceImpl; 7 | 8 | public final class ApiServiceFactory { 9 | 10 | public static final ClusterApiService getClusterApi(){ 11 | return new ClusterApiServiceImpl(); 12 | } 13 | 14 | public static final ConfigApiService getConfigApi(){ 15 | return new ConfigApiServiceImpl(); 16 | } 17 | 18 | public static final DefaultApiService getDefaultApi(){ 19 | return new DefaultApiServiceImpl(); 20 | } 21 | 22 | 23 | public static final StatusApiService getStatusApi(){ 24 | return new StatusApiServiceImpl(); 25 | } 26 | 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/StatusApi.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.Consumes; 4 | import javax.ws.rs.GET; 5 | import javax.ws.rs.Path; 6 | import javax.ws.rs.Produces; 7 | import javax.ws.rs.core.Response; 8 | 9 | @Path("/status") 10 | @Consumes({ "application/json" }) 11 | @Produces({ "application/json", "text/html" }) 12 | //@io.swagger.annotations.Api(value = "/status", description = "the status API") 13 | public class StatusApi { 14 | 15 | private final StatusApiService delegate = ApiServiceFactory.getStatusApi(); 16 | 17 | @GET 18 | /* 19 | @io.swagger.annotations.ApiOperation(value = "", notes = "Returns Framework Status\n", response = Object.class) 20 | @io.swagger.annotations.ApiResponses(value = { 21 | @io.swagger.annotations.ApiResponse(code = 200, message = "Successful response") }) 22 | */ 23 | public Response statusGet() 24 | throws NotFoundException { 25 | return delegate.statusGet(); 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ConfigApi.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.Consumes; 4 | import javax.ws.rs.GET; 5 | import javax.ws.rs.Path; 6 | import javax.ws.rs.Produces; 7 | import javax.ws.rs.core.Response; 8 | 9 | @Path("/config") 10 | @Consumes({ "application/json" }) 11 | @Produces({ "application/json", "text/html" }) 12 | //@io.swagger.annotations.Api(value = "/config", description = "the config API") 13 | public class ConfigApi { 14 | 15 | private final ConfigApiService delegate = ApiServiceFactory.getConfigApi(); 16 | 17 | @GET 18 | /* 19 | @io.swagger.annotations.ApiOperation(value = "", notes = "Returns current Framework configuration\n", response = Framework.class) 20 | @io.swagger.annotations.ApiResponses(value = { 21 | @io.swagger.annotations.ApiResponse(code = 200, message = "Successful response") }) 22 | */ 23 | public Response configGet() 24 | throws NotFoundException { 25 | return delegate.configGet(); 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /docker/accumulo.dockerfile: -------------------------------------------------------------------------------- 1 | # Pull base image. 2 | FROM ubuntu 3 | 4 | RUN apt-get update 5 | RUN apt-get install -y software-properties-common gpgv 6 | RUN apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF 7 | #RUN add-apt-repository ppa:openjdk-r/ppa 8 | RUN apt-get update 9 | 10 | RUN \ 11 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 12 | add-apt-repository -y ppa:webupd8team/java && \ 13 | apt-get update && \ 14 | apt-get install -y oracle-java8-installer && \ 15 | rm -rf /var/lib/apt/lists/* && \ 16 | rm -rf /var/cache/oracle-jdk8-installer 17 | 18 | 19 | RUN apt-get install -y wget lsb-release 20 | RUN DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) && echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | tee /etc/apt/sources.list.d/mesosphere.list 21 | RUN apt-get -y update 22 | RUN apt-get install -y zookeeper mesos 23 | 24 | ADD bin/install-accumulo.sh / 25 | RUN chmod +x /install-accumulo.sh 26 | RUN /install-accumulo.sh 27 | 28 | ADD bin/start-framework.sh / 29 | RUN chmod +x /start-framework.sh 30 | 31 | CMD ["/start-framework.sh"] 32 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/Error.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | 6 | /** 7 | * Error message\n 8 | **/ 9 | public class Error { 10 | 11 | private Integer code = null; 12 | private String message = null; 13 | 14 | 15 | /** 16 | * Numeric error code 17 | **/ 18 | @JsonProperty("code") 19 | public Integer getCode() { 20 | return code; 21 | } 22 | public void setCode(Integer code) { 23 | this.code = code; 24 | } 25 | 26 | 27 | /** 28 | * Description of error 29 | **/ 30 | @JsonProperty("message") 31 | public String getMessage() { 32 | return message; 33 | } 34 | public void setMessage(String message) { 35 | this.message = message; 36 | } 37 | 38 | 39 | 40 | @Override 41 | public String toString() { 42 | StringBuilder sb = new StringBuilder(); 43 | sb.append("class Error {\n"); 44 | 45 | sb.append(" code: ").append(code).append("\n"); 46 | sb.append(" message: ").append(message).append("\n"); 47 | sb.append("}\n"); 48 | return sb.toString(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/IdRegistry.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.google.common.collect.Lists; 5 | 6 | import java.util.List; 7 | 8 | 9 | public class IdRegistry { 10 | 11 | public static class RegistryPair { 12 | 13 | private String name; 14 | private String id; 15 | 16 | public RegistryPair(){} 17 | 18 | public RegistryPair(String name, String id){ 19 | this.name = name; 20 | this.id = id; 21 | } 22 | 23 | @JsonProperty("id") 24 | public String getId(){ return this.id; } 25 | public void setId(String id){ this.id = id; } 26 | 27 | @JsonProperty("name") 28 | public String getName(){ return this.name; } 29 | public void setName(String name){ this.name = name; } 30 | 31 | 32 | } 33 | 34 | private List pairs = Lists.newArrayList(); 35 | 36 | public IdRegistry(){} 37 | 38 | @JsonProperty("registry") 39 | public List getRegistry(){ return this.pairs;} 40 | public void setRegistry(List pairs){ this.pairs = pairs; } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /accumulo-mesos-executor/src/main/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ClusterApiService.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.core.Response; 4 | 5 | public abstract class ClusterApiService { 6 | 7 | public abstract Response clusterKillPost() 8 | throws NotFoundException; 9 | 10 | public abstract Response clusterMasterReprovisionPost() 11 | throws NotFoundException; 12 | 13 | public abstract Response clusterMasterRestartPost() 14 | throws NotFoundException; 15 | 16 | public abstract Response clusterMonitorGet() 17 | throws NotFoundException; 18 | 19 | public abstract Response clusterStartPost() 20 | throws NotFoundException; 21 | 22 | public abstract Response clusterStopPost() 23 | throws NotFoundException; 24 | 25 | public abstract Response clusterTserverReprovisionPost(String id) 26 | throws NotFoundException; 27 | 28 | public abstract Response clusterTserverRestartPost(String id) 29 | throws NotFoundException; 30 | 31 | public abstract Response clusterTserverRollingrestartPost(Boolean master,Integer group,Boolean reprovision) 32 | throws NotFoundException; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/ServerGroup.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | 6 | public class ServerGroup { 7 | 8 | private Integer count = 1; 9 | private ServerProfile profile = null; 10 | 11 | 12 | /** 13 | * Number of instances of this profile to launch\n 14 | **/ 15 | @JsonProperty("count") 16 | public Integer getCount() { 17 | return count; 18 | } 19 | public void setCount(Integer count) { 20 | this.count = count; 21 | } 22 | 23 | 24 | /** 25 | * Server profile to launch\n 26 | **/ 27 | @JsonProperty("profile") 28 | public ServerProfile getProfile() { 29 | return profile; 30 | } 31 | public void setProfile(ServerProfile profile) { 32 | this.profile = profile; 33 | } 34 | 35 | 36 | @Override 37 | public String toString() { 38 | StringBuilder sb = new StringBuilder(); 39 | sb.append("class Server {\n"); 40 | 41 | sb.append(" count: ").append(count).append("\n"); 42 | sb.append(" profile: ").append(profile).append("\n"); 43 | sb.append("}\n"); 44 | return sb.toString(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/resources/model/AccumuloOnly.json: -------------------------------------------------------------------------------- 1 | { 2 | "instance": "testInstance", 3 | "rootUser": "jimbo", 4 | "rootPassword": "jimbopassword", 5 | "zkServers": "srvA:2181", 6 | "executorMemory": 1024, 7 | "tarballUri": "hdfs://localhost:9000/data/accumulo.tar.gz", 8 | "hdfsUri": "hdfs://localhost:9000/accumulo-mesos", 9 | "siteXml": "", 10 | "servers": [ 11 | { 12 | "count": 5, 13 | "profile":{ 14 | "name":"BasicTserver", 15 | "description":"Basic Tserver setup", 16 | "type":"tserver", 17 | "cpus":8.0, 18 | "mem":8192, 19 | "user": "accumulo" 20 | } 21 | }, 22 | { 23 | "count": 1, 24 | "profile": { 25 | "name": "BasicMaster", 26 | "description": "Basic Master setup", 27 | "type": "master", 28 | "cpus": 2.0, 29 | "mem": 2048, 30 | "user": "accumulomaster" 31 | } 32 | }, 33 | { 34 | "count": 2, 35 | "profile": { 36 | "name": "Monitor", 37 | "description": "Basic Monitor setup", 38 | "type": "monitor", 39 | "cpus": 1.0, 40 | "mem": 512, 41 | "user": "accumulomon" 42 | } 43 | }, 44 | { 45 | "count": 1, 46 | "profile": { 47 | "name": "BasicGC", 48 | "description": "Basic Garbage Collector setup", 49 | "type": "gc", 50 | "cpus": 2.0, 51 | "mem": 512, 52 | "user": "accumulogc" 53 | } 54 | } 55 | ] 56 | } -------------------------------------------------------------------------------- /dev/config/cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "instance": "TEST_10092015_3", 3 | "rootUser": "jimbo", 4 | "rootPassword": "jimbopassword", 5 | "zkServers": "172.16.0.100:2181", 6 | "executorMemory": 128, 7 | "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-1.7.0-bin.tar.gz", 8 | "hdfsUri": "hdfs://172.16.0.100:9000/accumulo-mesos", 9 | "nativeLibUri": "hdfs://172.16.0.100:9000/dist/libaccumulo.so", 10 | "siteXml": "", 11 | "servers": [ 12 | { 13 | "count": 3, 14 | "profile":{ 15 | "name":"BasicTserver", 16 | "description":"Basic Tserver setup", 17 | "type":"tserver", 18 | "cpus":1.0, 19 | "mem":1024, 20 | "user": "" 21 | } 22 | }, 23 | { 24 | "count": 1, 25 | "profile": { 26 | "name": "BasicMaster", 27 | "description": "Basic Master setup", 28 | "type": "master", 29 | "cpus": 1.0, 30 | "mem": 512, 31 | "user": "" 32 | } 33 | }, 34 | { 35 | "count": 1, 36 | "profile": { 37 | "name": "Monitor", 38 | "description": "Basic Monitor setup", 39 | "type": "monitor", 40 | "cpus": 1.0, 41 | "mem": 256, 42 | "user": "" 43 | } 44 | }, 45 | { 46 | "count": 1, 47 | "profile": { 48 | "name": "BasicGC", 49 | "description": "Basic Garbage Collector setup", 50 | "type": "gc", 51 | "cpus": 1.0, 52 | "mem": 256, 53 | "user": "" 54 | } 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /docker/config/cluster.json-sample: -------------------------------------------------------------------------------- 1 | { 2 | "instance": "TEST_10092015_3", 3 | "rootUser": "jimbo", 4 | "rootPassword": "jimbopassword", 5 | "zkServers": "172.31.20.165:2181", 6 | "executorMemory": 128, 7 | "tarballUri": "hdfs://172.31.45.229:54310/dist/accumulo-1.7.0-bin.tar.gz", 8 | "hdfsUri": "hdfs://172.31.45.229:54310/accumulo-mesos", 9 | "nativeLibUri": "hdfs://172.31.45.229:54310/dist/libaccumulo.so", 10 | "siteXml": "", 11 | "servers": [ 12 | { 13 | "count": 3, 14 | "profile":{ 15 | "name":"BasicTserver", 16 | "description":"Basic Tserver setup", 17 | "type":"tserver", 18 | "cpus":1.0, 19 | "mem":1024, 20 | "user": "" 21 | } 22 | }, 23 | { 24 | "count": 1, 25 | "profile": { 26 | "name": "BasicMaster", 27 | "description": "Basic Master setup", 28 | "type": "master", 29 | "cpus": 1.0, 30 | "mem": 512, 31 | "user": "" 32 | } 33 | }, 34 | { 35 | "count": 1, 36 | "profile": { 37 | "name": "Monitor", 38 | "description": "Basic Monitor setup", 39 | "type": "monitor", 40 | "cpus": 1.0, 41 | "mem": 256, 42 | "user": "" 43 | } 44 | }, 45 | { 46 | "count": 1, 47 | "profile": { 48 | "name": "BasicGC", 49 | "description": "Basic Garbage Collector setup", 50 | "type": "gc", 51 | "cpus": 1.0, 52 | "mem": 256, 53 | "user": "" 54 | } 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /dev/aws/AWS_cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "instance": "TEST_10092015_1", 3 | "rootUser": "jimbo", 4 | "rootPassword": "jimbopassword", 5 | "zkServers": "172.31.0.11:2181", 6 | "executorMemory": 128, 7 | "tarballUri": "hdfs://172.31.1.11:54310/user/klucar/accumulo-1.7.0-bin.tar.gz", 8 | "hdfsUri": "hdfs://172.31.1.11:54310/user/klucar/accumulo-mesos", 9 | "nativeLibUri": "hdfs://172.31.1.11:54310/user/klucar/libaccumulo.so", 10 | "siteXml": "", 11 | "servers": [ 12 | { 13 | "count": 10, 14 | "profile":{ 15 | "name":"BasicTserver", 16 | "description":"Basic Tserver setup", 17 | "type":"tserver", 18 | "cpus":2.0, 19 | "mem":12288, 20 | "user": "" 21 | } 22 | }, 23 | { 24 | "count": 1, 25 | "profile": { 26 | "name": "BasicMaster", 27 | "description": "Basic Master setup", 28 | "type": "master", 29 | "cpus": 2.0, 30 | "mem": 2048, 31 | "user": "" 32 | } 33 | }, 34 | { 35 | "count": 1, 36 | "profile": { 37 | "name": "Monitor", 38 | "description": "Basic Monitor setup", 39 | "type": "monitor", 40 | "cpus": 1.0, 41 | "mem": 1024, 42 | "user": "" 43 | } 44 | }, 45 | { 46 | "count": 1, 47 | "profile": { 48 | "name": "BasicGC", 49 | "description": "Basic Garbage Collector setup", 50 | "type": "gc", 51 | "cpus": 2.0, 52 | "mem": 2048, 53 | "user": "" 54 | } 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ApiResponseMessage.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.xml.bind.annotation.XmlTransient; 4 | 5 | @javax.xml.bind.annotation.XmlRootElement 6 | public class ApiResponseMessage { 7 | public static final int ERROR = 1; 8 | public static final int WARNING = 2; 9 | public static final int INFO = 3; 10 | public static final int OK = 4; 11 | public static final int TOO_BUSY = 5; 12 | 13 | int code; 14 | String type; 15 | String message; 16 | 17 | public ApiResponseMessage(){} 18 | 19 | public ApiResponseMessage(int code, String message){ 20 | this.code = code; 21 | switch(code){ 22 | case ERROR: 23 | setType("error"); 24 | break; 25 | case WARNING: 26 | setType("warning"); 27 | break; 28 | case INFO: 29 | setType("info"); 30 | break; 31 | case OK: 32 | setType("ok"); 33 | break; 34 | case TOO_BUSY: 35 | setType("too busy"); 36 | break; 37 | default: 38 | setType("unknown"); 39 | break; 40 | } 41 | this.message = message; 42 | } 43 | 44 | @XmlTransient 45 | public int getCode() { 46 | return code; 47 | } 48 | 49 | public void setCode(int code) { 50 | this.code = code; 51 | } 52 | 53 | public String getType() { 54 | return type; 55 | } 56 | 57 | public void setType(String type) { 58 | this.type = type; 59 | } 60 | 61 | public String getMessage() { 62 | return message; 63 | } 64 | 65 | public void setMessage(String message) { 66 | this.message = message; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/src/main/java/aredee/mesos/frameworks/accumulo/scheduler/matcher/Match.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.scheduler.matcher; 2 | 3 | import aredee.mesos.frameworks.accumulo.model.Task; 4 | import org.apache.mesos.Protos; 5 | 6 | public class Match { 7 | private Task task = null; 8 | private Protos.Offer offer = null; 9 | 10 | /** 11 | Constructs a Match object with no offer. These matches are used to pass back servers that were requested to 12 | be launched, but had no matching offers. 13 | */ 14 | public Match(Task task){ 15 | this(task, null); 16 | } 17 | 18 | /** 19 | * Constructs a match with no task. Typically used to signal extra offers were available when appropriate. 20 | * @param offer 21 | */ 22 | public Match(Protos.Offer offer){ 23 | this(null, offer); 24 | } 25 | 26 | /** 27 | * Constructs a Match between a task and an offer. 28 | */ 29 | public Match(Task task, Protos.Offer offer){ 30 | this.task = task; 31 | this.offer = offer; 32 | } 33 | 34 | public boolean hasTask(){ 35 | return this.task != null; 36 | } 37 | public boolean hasOffer(){ 38 | return this.offer != null; 39 | } 40 | public void setOffer(Protos.Offer offer){ 41 | this.offer = offer; 42 | } 43 | public void setTask(Task task){ 44 | this.task = task; 45 | } 46 | public Protos.Offer getOffer(){ 47 | return this.offer; 48 | } 49 | public Task getTask(){ 50 | return this.task; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/css/typography.css: -------------------------------------------------------------------------------- 1 | /* droid-sans-regular - latin */ 2 | @font-face { 3 | font-family: 'Droid Sans'; 4 | font-style: normal; 5 | font-weight: 400; 6 | src: url('../fonts/droid-sans-v6-latin-regular.eot'); /* IE9 Compat Modes */ 7 | src: local('Droid Sans'), local('DroidSans'), 8 | url('../fonts/droid-sans-v6-latin-regular.eot?#iefix') format('embedded-opentype'), /* IE6-IE8 */ 9 | url('../fonts/droid-sans-v6-latin-regular.woff2') format('woff2'), /* Super Modern Browsers */ 10 | url('../fonts/droid-sans-v6-latin-regular.woff') format('woff'), /* Modern Browsers */ 11 | url('../fonts/droid-sans-v6-latin-regular.ttf') format('truetype'), /* Safari, Android, iOS */ 12 | url('../fonts/droid-sans-v6-latin-regular.svg#DroidSans') format('svg'); /* Legacy iOS */ 13 | } 14 | /* droid-sans-700 - latin */ 15 | @font-face { 16 | font-family: 'Droid Sans'; 17 | font-style: normal; 18 | font-weight: 700; 19 | src: url('../fonts/droid-sans-v6-latin-700.eot'); /* IE9 Compat Modes */ 20 | src: local('Droid Sans Bold'), local('DroidSans-Bold'), 21 | url('../fonts/droid-sans-v6-latin-700.eot?#iefix') format('embedded-opentype'), /* IE6-IE8 */ 22 | url('../fonts/droid-sans-v6-latin-700.woff2') format('woff2'), /* Super Modern Browsers */ 23 | url('../fonts/droid-sans-v6-latin-700.woff') format('woff'), /* Modern Browsers */ 24 | url('../fonts/droid-sans-v6-latin-700.ttf') format('truetype'), /* Safari, Android, iOS */ 25 | url('../fonts/droid-sans-v6-latin-700.svg#DroidSans') format('svg'); /* Legacy iOS */ 26 | } 27 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lang/translator.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Translator for documentation pages. 5 | * 6 | * To enable translation you should include one of language-files in your index.html 7 | * after . 8 | * For example - 9 | * 10 | * If you wish to translate some new texsts you should do two things: 11 | * 1. Add a new phrase pair ("New Phrase": "New Translation") into your language file (for example lang/ru.js). It will be great if you add it in other language files too. 12 | * 2. Mark that text it templates this way New Phrase or . 13 | * The main thing here is attribute data-sw-translate. Only inner html, title-attribute and value-attribute are going to translate. 14 | * 15 | */ 16 | window.SwaggerTranslator = { 17 | 18 | _words:[], 19 | 20 | translate: function(sel) { 21 | var $this = this; 22 | sel = sel || '[data-sw-translate]'; 23 | 24 | $(sel).each(function() { 25 | $(this).html($this._tryTranslate($(this).html())); 26 | 27 | $(this).val($this._tryTranslate($(this).val())); 28 | $(this).attr('title', $this._tryTranslate($(this).attr('title'))); 29 | }); 30 | }, 31 | 32 | _tryTranslate: function(word) { 33 | return this._words[$.trim(word)] !== undefined ? this._words[$.trim(word)] : word; 34 | }, 35 | 36 | learn: function(wordsMap) { 37 | this._words = wordsMap; 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /accumulo-mesos-executor/src/main/java/aredee/mesos/frameworks/accumulo/executor/Main.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.executor; 2 | 3 | import aredee.mesos.frameworks.accumulo.configuration.Environment; 4 | import org.apache.mesos.ExecutorDriver; 5 | import org.apache.mesos.MesosExecutorDriver; 6 | import org.apache.mesos.Protos; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.List; 11 | 12 | public class Main { 13 | 14 | private static final Logger LOGGER = LoggerFactory.getLogger(Main.class); 15 | 16 | public static void main(String[] args){ 17 | 18 | //TODO Should executors have REST API? 19 | 20 | // TODO check for required environment variables 21 | List missingVars = Environment.getMissingVariables(Environment.REQUIRED_EXECUTOR_VARS); 22 | if( !missingVars.isEmpty() ){ 23 | LOGGER.error("Missing required environment variables {}", missingVars); 24 | System.err.println("Missing required environment variables: " + missingVars); 25 | System.exit(-1); 26 | } 27 | 28 | // start the executor process 29 | AccumuloStartExecutor executor = new AccumuloStartExecutor(); 30 | 31 | ExecutorDriver executorDriver = new MesosExecutorDriver(executor); 32 | 33 | // run() blocks until driver finishes somehow 34 | final Protos.Status runStatus = executorDriver.run(); 35 | 36 | // Ensure that the driver process terminates. 37 | LOGGER.info("Executor Driver stopped, with value " + runStatus.name()); 38 | executorDriver.stop(); 39 | 40 | LOGGER.info("Exiting executor process"); 41 | System.exit(runStatus.getNumber()); 42 | } 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/css/reset.css: -------------------------------------------------------------------------------- 1 | /* http://meyerweb.com/eric/tools/css/reset/ v2.0 | 20110126 */ 2 | html, 3 | body, 4 | div, 5 | span, 6 | applet, 7 | object, 8 | iframe, 9 | h1, 10 | h2, 11 | h3, 12 | h4, 13 | h5, 14 | h6, 15 | p, 16 | blockquote, 17 | pre, 18 | a, 19 | abbr, 20 | acronym, 21 | address, 22 | big, 23 | cite, 24 | code, 25 | del, 26 | dfn, 27 | em, 28 | img, 29 | ins, 30 | kbd, 31 | q, 32 | s, 33 | samp, 34 | small, 35 | strike, 36 | strong, 37 | sub, 38 | sup, 39 | tt, 40 | var, 41 | b, 42 | u, 43 | i, 44 | center, 45 | dl, 46 | dt, 47 | dd, 48 | ol, 49 | ul, 50 | li, 51 | fieldset, 52 | form, 53 | label, 54 | legend, 55 | table, 56 | caption, 57 | tbody, 58 | tfoot, 59 | thead, 60 | tr, 61 | th, 62 | td, 63 | article, 64 | aside, 65 | canvas, 66 | details, 67 | embed, 68 | figure, 69 | figcaption, 70 | footer, 71 | header, 72 | hgroup, 73 | menu, 74 | nav, 75 | output, 76 | ruby, 77 | section, 78 | summary, 79 | time, 80 | mark, 81 | audio, 82 | video { 83 | margin: 0; 84 | padding: 0; 85 | border: 0; 86 | font-size: 100%; 87 | font: inherit; 88 | vertical-align: baseline; 89 | } 90 | /* HTML5 display-role reset for older browsers */ 91 | article, 92 | aside, 93 | details, 94 | figcaption, 95 | figure, 96 | footer, 97 | header, 98 | hgroup, 99 | menu, 100 | nav, 101 | section { 102 | display: block; 103 | } 104 | body { 105 | line-height: 1; 106 | } 107 | ol, 108 | ul { 109 | list-style: none; 110 | } 111 | blockquote, 112 | q { 113 | quotes: none; 114 | } 115 | blockquote:before, 116 | blockquote:after, 117 | q:before, 118 | q:after { 119 | content: ''; 120 | content: none; 121 | } 122 | table { 123 | border-collapse: collapse; 124 | border-spacing: 0; 125 | } 126 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/impl/ClusterApiServiceImpl.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api.impl; 2 | 3 | import aredee.mesos.frameworks.accumulo.framework.api.ClusterApiService; 4 | import aredee.mesos.frameworks.accumulo.framework.api.NotFoundException; 5 | import aredee.mesos.frameworks.accumulo.scheduler.Cluster; 6 | 7 | import javax.ws.rs.core.Response; 8 | 9 | public class ClusterApiServiceImpl extends ClusterApiService { 10 | 11 | Cluster cluster = Cluster.INSTANCE; 12 | 13 | @Override 14 | public Response clusterKillPost() throws NotFoundException { 15 | return null; 16 | } 17 | 18 | @Override 19 | public Response clusterMasterReprovisionPost() throws NotFoundException { 20 | return null; 21 | } 22 | 23 | @Override 24 | public Response clusterMasterRestartPost() throws NotFoundException { 25 | return null; 26 | } 27 | 28 | @Override 29 | public Response clusterMonitorGet() throws NotFoundException { 30 | return null; 31 | } 32 | 33 | @Override 34 | public Response clusterStartPost() throws NotFoundException { 35 | return null; 36 | } 37 | 38 | @Override 39 | public Response clusterStopPost() throws NotFoundException { 40 | return null; 41 | } 42 | 43 | @Override 44 | public Response clusterTserverReprovisionPost(String id) throws NotFoundException { 45 | return null; 46 | } 47 | 48 | @Override 49 | public Response clusterTserverRestartPost(String id) throws NotFoundException { 50 | return null; 51 | } 52 | 53 | @Override 54 | public Response clusterTserverRollingrestartPost(Boolean master, Integer group, Boolean reprovision) throws NotFoundException { 55 | return null; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /dev/config/FrameworkAndAccumulo.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "172.16.0.100", 3 | "httpPort": "8192", 4 | "mesosMaster": "172.16.0.100:5050", 5 | "name":"accumulo-mesos-test", 6 | "id": "", 7 | "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 8 | "zkServers": "172.16.0.100:2181", 9 | "cluster":{ 10 | "instance": "TEST_09162015_1", 11 | "rootUser": "jimbo", 12 | "rootPassword": "jimbopassword", 13 | "zkServers": "172.16.0.100:2181", 14 | "executorMemory": 128, 15 | "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-1.7.0-bin.tar.gz", 16 | "hdfsUri": "hdfs://172.16.0.100:9000/accumulo-mesos", 17 | "siteXml": "", 18 | "servers": [ 19 | { 20 | "count": 3, 21 | "profile":{ 22 | "name":"BasicTserver", 23 | "description":"Basic Tserver setup", 24 | "type":"tserver", 25 | "cpus":1.0, 26 | "mem":1024, 27 | "user": "" 28 | } 29 | }, 30 | { 31 | "count": 1, 32 | "profile": { 33 | "name": "BasicMaster", 34 | "description": "Basic Master setup", 35 | "type": "master", 36 | "cpus": 1.0, 37 | "mem": 512, 38 | "user": "" 39 | } 40 | }, 41 | { 42 | "count": 1, 43 | "profile": { 44 | "name": "Monitor", 45 | "description": "Basic Monitor setup", 46 | "type": "monitor", 47 | "cpus": 1.0, 48 | "mem": 256, 49 | "user": "" 50 | } 51 | }, 52 | { 53 | "count": 1, 54 | "profile": { 55 | "name": "BasicGC", 56 | "description": "Basic Garbage Collector setup", 57 | "type": "gc", 58 | "cpus": 1.0, 59 | "mem": 256, 60 | "user": "" 61 | } 62 | } 63 | ] 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /dev/config/AWS_framework.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "0.0.0.0", 3 | "httpPort": "8192", 4 | "mesosMaster": "172.31.1.11:5050", 5 | "name":"accumulo-mesos-aws-test-102", 6 | "id": "", 7 | "tarballUri": "hdfs://172.31.1.11:54310/user/klucar/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 8 | "zkServers": "172.31.0.11:2181", 9 | "cluster":{ 10 | "instance": "TEST_09172015_102", 11 | "rootUser": "root", 12 | "rootPassword": "secret", 13 | "zkServers": "172.31.0.11:2181", 14 | "executorMemory": 512, 15 | "tarballUri": "hdfs://172.31.1.11:54310/user/klucar/accumulo-1.7.0-bin.tar.gz", 16 | "hdfsUri": "hdfs://172.31.1.11:54310/accumulo-mesos", 17 | "siteXml": "", 18 | "servers": [ 19 | { 20 | "count": 10, 21 | "profile":{ 22 | "name":"BasicTserver", 23 | "description":"Basic Tserver setup", 24 | "type":"tserver", 25 | "cpus":2.0, 26 | "mem":12288, 27 | "user": "" 28 | } 29 | }, 30 | { 31 | "count": 1, 32 | "profile": { 33 | "name": "BasicMaster", 34 | "description": "Basic Master setup", 35 | "type": "master", 36 | "cpus": 2.0, 37 | "mem": 2048, 38 | "user": "" 39 | } 40 | }, 41 | { 42 | "count": 1, 43 | "profile": { 44 | "name": "Monitor", 45 | "description": "Basic Monitor setup", 46 | "type": "monitor", 47 | "cpus": 1.0, 48 | "mem": 1024, 49 | "user": "" 50 | } 51 | }, 52 | { 53 | "count": 1, 54 | "profile": { 55 | "name": "BasicGC", 56 | "description": "Basic Garbage Collector setup", 57 | "type": "gc", 58 | "cpus": 2.0, 59 | "mem": 2048, 60 | "user": "" 61 | } 62 | } 63 | ] 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/resources/state/ZkTestFramework.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "172.16.0.100", 3 | "httpPort": "8192", 4 | "mesosMaster": "172.16.0.100:5050", 5 | "name":"accumulo-mesos-test", 6 | "id": "", 7 | "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT.tar.gz", 8 | "zkServers": "172.16.0.100:2181", 9 | "cluster":{ 10 | "instance": "TEST_09082015_1", 11 | "rootUser": "jimbo", 12 | "rootPassword": "jimbopassword", 13 | "zkServers": "172.16.0.100:2181", 14 | "executorMemory": 128, 15 | "tarballUri": "hdfs://172.16.0.100:9000/dist/accumulo-1.7.0-bin.tar.gz", 16 | "hdfsUri": "hdfs://172.16.0.100:9000/accumulo-mesos", 17 | "siteXml": "", 18 | "servers": [ 19 | { 20 | "count": 3, 21 | "profile":{ 22 | "name":"BasicTserver", 23 | "description":"Basic Tserver setup", 24 | "type":"tserver", 25 | "cpus":1.0, 26 | "mem":1024, 27 | "user": "" 28 | } 29 | }, 30 | { 31 | "count": 1, 32 | "profile": { 33 | "name": "BasicMaster", 34 | "description": "Basic Master setup", 35 | "type": "master", 36 | "cpus": 1.0, 37 | "mem": 512, 38 | "user": "" 39 | } 40 | }, 41 | { 42 | "count": 1, 43 | "profile": { 44 | "name": "Monitor", 45 | "description": "Basic Monitor setup", 46 | "type": "monitor", 47 | "cpus": 1.0, 48 | "mem": 256, 49 | "user": "" 50 | } 51 | }, 52 | { 53 | "count": 1, 54 | "profile": { 55 | "name": "BasicGC", 56 | "description": "Basic Garbage Collector setup", 57 | "type": "gc", 58 | "cpus": 1.0, 59 | "mem": 256, 60 | "user": "" 61 | } 62 | } 63 | ] 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/resources/model/FrameworkAndAccumulo.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindAddress": "1.1.1.1", 3 | "httpPort": "1234", 4 | "mesosMaster": "1.2.3.4:5150", 5 | "name":"accumulo-mesos-test", 6 | "id": "", 7 | "tarballUri": "hdfs://localhost:9000/data/accumulo-mesos.tar.gz", 8 | "zkServers": "server1:2181,server2:2181,server3:2181", 9 | "cluster":{ 10 | "instance": "testInstance", 11 | "rootUser": "jimbo", 12 | "rootPassword": "jimbopassword", 13 | "zkServers": "srvA:2181", 14 | "executorMemory": 1024, 15 | "tarballUri": "hdfs://localhost:9000/data/accumulo.tar.gz", 16 | "hdfsUri": "hdfs://localhost:9000/accumulo-mesos", 17 | "siteXml": "", 18 | "servers": [ 19 | { 20 | "count": 5, 21 | "profile":{ 22 | "name":"BasicTserver", 23 | "description":"Basic Tserver setup", 24 | "type":"tserver", 25 | "cpus":8.0, 26 | "mem":8192, 27 | "user": "accumulo" 28 | } 29 | }, 30 | { 31 | "count": 1, 32 | "profile": { 33 | "name": "BasicMaster", 34 | "description": "Basic Master setup", 35 | "type": "master", 36 | "cpus": 2.0, 37 | "mem": 2048, 38 | "user": "accumulomaster" 39 | } 40 | }, 41 | { 42 | "count": 2, 43 | "profile": { 44 | "name": "Monitor", 45 | "description": "Basic Monitor setup", 46 | "type": "monitor", 47 | "cpus": 1.0, 48 | "mem": 512, 49 | "user": "accumulomon" 50 | } 51 | }, 52 | { 53 | "count": 1, 54 | "profile": { 55 | "name": "BasicGC", 56 | "description": "Basic Garbage Collector setup", 57 | "type": "gc", 58 | "cpus": 2.0, 59 | "mem": 512, 60 | "user": "accumulogc" 61 | } 62 | } 63 | ] 64 | } 65 | } -------------------------------------------------------------------------------- /accumulo-mesos-dist/src/main/assembly/tarball.xml: -------------------------------------------------------------------------------- 1 | 12 | 14 | 15 | tar.gz 16 | 17 | bin 18 | 19 | 20 | target/tarball 21 | . 22 | 23 | 24 | ../bin/ 25 | 26 | findHome.sh 27 | startJsonAccumulo 28 | startCmdAccumulo 29 | 30 | bin/ 31 | 32 | 33 | ../conf/ 34 | 35 | default-accumulo-site.xml 36 | DefaultCluster.json 37 | 38 | conf/ 39 | 40 | 41 | . 42 | 43 | THIRD-PARTY.txt 44 | licenses/** 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/WebServer.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | 4 | import aredee.mesos.frameworks.accumulo.model.Framework; 5 | import com.google.inject.Inject; 6 | import com.google.inject.servlet.GuiceFilter; 7 | import org.apache.commons.lang3.StringUtils; 8 | import org.eclipse.jetty.server.Server; 9 | import org.eclipse.jetty.server.ServerConnector; 10 | import org.eclipse.jetty.servlet.DefaultServlet; 11 | import org.eclipse.jetty.servlet.ServletContextHandler; 12 | 13 | import java.util.EnumSet; 14 | 15 | public class WebServer { 16 | 17 | private final Framework config; 18 | private Server server; 19 | 20 | @Inject 21 | public WebServer(Framework config) { 22 | this.config = config; 23 | } 24 | 25 | public void start() throws java.lang.Exception { 26 | 27 | checkParam(this.config.getBindAddress(), "Bind address"); 28 | checkParam(this.config.getHttpPort() + "", "Http Port"); 29 | 30 | this.server = new Server(); 31 | // configure server 32 | ServerConnector http = new ServerConnector(server); 33 | http.setHost(this.config.getBindAddress()); 34 | http.setPort(this.config.getHttpPort()); 35 | server.addConnector(http); 36 | 37 | ServletContextHandler context = new ServletContextHandler(this.server, "/", ServletContextHandler.NO_SESSIONS); 38 | context.addFilter(GuiceFilter.class, "/*", EnumSet.of(javax.servlet.DispatcherType.REQUEST, javax.servlet.DispatcherType.ASYNC)); 39 | context.addServlet(DefaultServlet.class, "/*"); 40 | 41 | String staticDir = this.getClass().getClassLoader().getResource("webapp/public").toExternalForm(); 42 | context.setResourceBase(staticDir); 43 | 44 | this.server.start(); 45 | } 46 | 47 | public void stop() throws java.lang.Exception{ 48 | this.server.stop(); 49 | } 50 | 51 | private void checkParam(String value, String name) { 52 | if (StringUtils.isEmpty(value)) { 53 | throw new RuntimeException("Required parameter " + name + " is missing"); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/Task.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | import java.util.UUID; 6 | 7 | 8 | public class Task { 9 | 10 | private String slaveId = null; 11 | private String taskId = null; 12 | private ServerProfile serverProfile = null; 13 | 14 | /** 15 | **/ 16 | @JsonProperty("slaveId") 17 | public String getSlaveId() { 18 | return slaveId; 19 | } 20 | public void setSlaveId(String slaveId) { 21 | this.slaveId = slaveId; 22 | } 23 | public boolean hasSlaveId(){ 24 | if( slaveId == null ) return false; 25 | if( slaveId.isEmpty()) return false; 26 | return true; 27 | } 28 | 29 | /** 30 | **/ 31 | @JsonProperty("taskId") 32 | public String getTaskId() { 33 | return taskId; 34 | } 35 | public void setTaskId(String id) { 36 | this.taskId = id; 37 | } 38 | public boolean hasTaskId(){ 39 | if( taskId == null ) return false; 40 | if( taskId.isEmpty()) return false; 41 | return true; 42 | } 43 | 44 | 45 | /** 46 | **/ 47 | @JsonProperty("serverProfile") 48 | public ServerProfile getServerProfile() { 49 | return serverProfile; 50 | } 51 | public void setServerProfile(ServerProfile profile) { 52 | this.serverProfile = profile; 53 | } 54 | 55 | 56 | public ServerProfile.TypeEnum getType(){ 57 | return serverProfile.getType(); 58 | } 59 | 60 | public String assignTaskId(){ 61 | assert( taskId != null); 62 | assert( serverProfile != null); 63 | 64 | this.taskId = getType()+"::"+ UUID.randomUUID(); 65 | 66 | return taskId; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | StringBuilder sb = new StringBuilder(); 72 | sb.append("class Task {\n"); 73 | 74 | sb.append(" slaveId: ").append(slaveId).append("\n"); 75 | sb.append(" taskId: ").append(taskId).append("\n"); 76 | sb.append(" serverProfile: ").append(serverProfile).append("\n"); 77 | sb.append("}\n"); 78 | return sb.toString(); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lang/ru.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* jshint quotmark: double */ 4 | window.SwaggerTranslator.learn({ 5 | "Warning: Deprecated":"Ворнинг: Депрекейтед", 6 | "Implementation Notes":"Заметки", 7 | "Response Class":"Пример ответа", 8 | "Status":"Статус", 9 | "Parameters":"Параметры", 10 | "Parameter":"Параметр", 11 | "Value":"Значение", 12 | "Description":"Описание", 13 | "Parameter Type":"Тип параметра", 14 | "Data Type":"Тип данных", 15 | "HTTP Status Code":"HTTP код", 16 | "Reason":"Причина", 17 | "Response Model":"Структура ответа", 18 | "Request URL":"URL запроса", 19 | "Response Body":"Тело ответа", 20 | "Response Code":"HTTP код ответа", 21 | "Response Headers":"Заголовки ответа", 22 | "Hide Response":"Спрятать ответ", 23 | "Response Messages":"Что может прийти в ответ", 24 | "Try it out!":"Попробовать!", 25 | "Show/Hide":"Показать/Скрыть", 26 | "List Operations":"Операции кратко", 27 | "Expand Operations":"Операции подробно", 28 | "Raw":"В сыром виде", 29 | "can't parse JSON. Raw result":"Не удается распарсить ответ:", 30 | "Model Schema":"Структура", 31 | "Model":"Описание", 32 | "apply":"применить", 33 | "Username":"Имя пользователя", 34 | "Password":"Пароль", 35 | "Terms of service":"Условия использования", 36 | "Created by":"Разработано", 37 | "See more at":"Еще тут", 38 | "Contact the developer":"Связаться с разработчиком", 39 | "api version":"Версия API", 40 | "Response Content Type":"Content Type ответа", 41 | "fetching resource":"Получение ресурса", 42 | "fetching resource list":"Получение ресурсов", 43 | "Explore":"Поехали", 44 | "Show Swagger Petstore Example Apis":"Показать примеры АПИ", 45 | "Can't read from server. It may not have the appropriate access-control-origin settings.":"Не удается получить ответ от сервера. Возможно, какая-то лажа с настройками доступа", 46 | "Please specify the protocol for":"Пожалуйста, укажите протогол для", 47 | "Can't read swagger JSON from":"Не получается прочитать swagger json из", 48 | "Finished Loading Resource Information. Rendering Swagger UI":"Загрузка информации о ресурсах завершена. Рендерим", 49 | "Unable to read api":"Не удалось прочитать api", 50 | "from path":"по адресу", 51 | "server returned":"сервер сказал" 52 | }); 53 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lang/en.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* jshint quotmark: double */ 4 | window.SwaggerTranslator.learn({ 5 | "Warning: Deprecated":"Warning: Deprecated", 6 | "Implementation Notes":"Implementation Notes", 7 | "Response Class":"Response Class", 8 | "Status":"Status", 9 | "Parameters":"Parameters", 10 | "Parameter":"Parameter", 11 | "Value":"Value", 12 | "Description":"Description", 13 | "Parameter Type":"Parameter Type", 14 | "Data Type":"Data Type", 15 | "Response Messages":"Response Messages", 16 | "HTTP Status Code":"HTTP Status Code", 17 | "Reason":"Reason", 18 | "Response Model":"Response Model", 19 | "Request URL":"Request URL", 20 | "Response Body":"Response Body", 21 | "Response Code":"Response Code", 22 | "Response Headers":"Response Headers", 23 | "Hide Response":"Hide Response", 24 | "Headers":"Headers", 25 | "Try it out!":"Try it out!", 26 | "Show/Hide":"Show/Hide", 27 | "List Operations":"List Operations", 28 | "Expand Operations":"Expand Operations", 29 | "Raw":"Raw", 30 | "can't parse JSON. Raw result":"can't parse JSON. Raw result", 31 | "Model Schema":"Model Schema", 32 | "Model":"Model", 33 | "apply":"apply", 34 | "Username":"Username", 35 | "Password":"Password", 36 | "Terms of service":"Terms of service", 37 | "Created by":"Created by", 38 | "See more at":"See more at", 39 | "Contact the developer":"Contact the developer", 40 | "api version":"api version", 41 | "Response Content Type":"Response Content Type", 42 | "fetching resource":"fetching resource", 43 | "fetching resource list":"fetching resource list", 44 | "Explore":"Explore", 45 | "Show Swagger Petstore Example Apis":"Show Swagger Petstore Example Apis", 46 | "Can't read from server. It may not have the appropriate access-control-origin settings.":"Can't read from server. It may not have the appropriate access-control-origin settings.", 47 | "Please specify the protocol for":"Please specify the protocol for", 48 | "Can't read swagger JSON from":"Can't read swagger JSON from", 49 | "Finished Loading Resource Information. Rendering Swagger UI":"Finished Loading Resource Information. Rendering Swagger UI", 50 | "Unable to read api":"Unable to read api", 51 | "from path":"from path", 52 | "server returned":"server returned" 53 | }); 54 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/java/aredee/mesos/frameworks/accumulo/configuration/file/TestSiteXml.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.configuration.file; 2 | 3 | import com.google.common.base.Optional; 4 | 5 | //import org.apache.commons.configuration.XMLConfiguration; 6 | 7 | public class TestSiteXml { 8 | 9 | static final String MAX = "1024"; 10 | static final String MIN = "512"; 11 | static final String MAXO = "1024.0"; 12 | static final String MINO = "512.0"; 13 | 14 | static final String JSON = "{'minMemory':'512','maxMemory':'1024'}"; 15 | static final int MAX_TRIES = 10000000; 16 | 17 | // TODO get test working 18 | // @SuppressWarnings("unchecked") 19 | // @Test 20 | public void testXmlConfiguration() { 21 | 22 | Optional value; 23 | /* 24 | try { 25 | AccumuloSiteXml xmlSite = new AccumuloSiteXml(); 26 | System.out.println(xmlSite.toXml()); 27 | 28 | value = xmlSite.getPassword(); 29 | assertTrue(value.isPresent()); 30 | assertTrue(value.get().equalsIgnoreCase("DEFAULT")); 31 | 32 | xmlSite.setPassword("newpassword"); 33 | System.out.println(xmlSite.toXml()); 34 | value = xmlSite.getPassword(); 35 | assertTrue(value.isPresent()); 36 | assertTrue(value.get().equalsIgnoreCase("newpassword")); 37 | 38 | value = xmlSite.getPropertyValue("instance.volumes"); 39 | assertTrue(value.isPresent()); 40 | 41 | value = xmlSite.getPropertyValue("xyz.volumes"); 42 | assertTrue(!value.isPresent()); 43 | 44 | xmlSite.setPropertyValue("BLAH", "BLAHBLAH"); 45 | 46 | System.out.println(xmlSite.toXml()); 47 | 48 | value = xmlSite.getPropertyValue("BLAH"); 49 | assertTrue(value.isPresent()); 50 | 51 | String xml = xmlSite.toString(); 52 | 53 | AccumuloSiteXml xmlSite2 = new AccumuloSiteXml(new ByteArrayInputStream(xml.getBytes())); 54 | File xmlFile = new File("./MyNewSiteFile.xml"); 55 | xmlSite2.writeSiteFile(xmlFile); 56 | assertTrue(xmlFile.exists()); 57 | 58 | 59 | } catch (Exception e) { 60 | // TODO Auto-generated catch block 61 | e.printStackTrace(); 62 | fail(); 63 | } 64 | */ 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lang/pt.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* jshint quotmark: double */ 4 | window.SwaggerTranslator.learn({ 5 | "Warning: Deprecated":"Aviso: Depreciado", 6 | "Implementation Notes":"Notas de Implementação", 7 | "Response Class":"Classe de resposta", 8 | "Status":"Status", 9 | "Parameters":"Parâmetros", 10 | "Parameter":"Parâmetro", 11 | "Value":"Valor", 12 | "Description":"Descrição", 13 | "Parameter Type":"Tipo de parâmetro", 14 | "Data Type":"Tipo de dados", 15 | "Response Messages":"Mensagens de resposta", 16 | "HTTP Status Code":"Código de status HTTP", 17 | "Reason":"Razão", 18 | "Response Model":"Modelo resposta", 19 | "Request URL":"URL requisição", 20 | "Response Body":"Corpo da resposta", 21 | "Response Code":"Código da resposta", 22 | "Response Headers":"Cabeçalho da resposta", 23 | "Headers":"Cabeçalhos", 24 | "Hide Response":"Esconder resposta", 25 | "Try it out!":"Tente agora!", 26 | "Show/Hide":"Mostrar/Esconder", 27 | "List Operations":"Listar operações", 28 | "Expand Operations":"Expandir operações", 29 | "Raw":"Cru", 30 | "can't parse JSON. Raw result":"Falha ao analisar JSON. Resulto cru", 31 | "Model Schema":"Modelo esquema", 32 | "Model":"Modelo", 33 | "apply":"Aplicar", 34 | "Username":"Usuário", 35 | "Password":"Senha", 36 | "Terms of service":"Termos do serviço", 37 | "Created by":"Criado por", 38 | "See more at":"Veja mais em", 39 | "Contact the developer":"Contate o desenvolvedor", 40 | "api version":"Versão api", 41 | "Response Content Type":"Tipo de conteúdo da resposta", 42 | "fetching resource":"busca recurso", 43 | "fetching resource list":"buscando lista de recursos", 44 | "Explore":"Explorar", 45 | "Show Swagger Petstore Example Apis":"Show Swagger Petstore Example Apis", 46 | "Can't read from server. It may not have the appropriate access-control-origin settings.":"Não é possível ler do servidor. Pode não ter as apropriadas configurações access-control-origin", 47 | "Please specify the protocol for":"Por favor especifique o protocolo", 48 | "Can't read swagger JSON from":"Não é possível ler o JSON Swagger de", 49 | "Finished Loading Resource Information. Rendering Swagger UI":"Carregar informação de recurso finalizada. Renderizando Swagger UI", 50 | "Unable to read api":"Não foi possível ler api", 51 | "from path":"do caminho", 52 | "server returned":"servidor retornou" 53 | }); 54 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/java/aredee/mesos/frameworks/accumulo/state/FrameworkStateHelperTest.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.state; 2 | 3 | import aredee.mesos.frameworks.accumulo.model.Framework; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.apache.commons.io.IOUtils; 6 | import org.apache.mesos.state.InMemoryState; 7 | import org.junit.After; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | import java.io.File; 12 | import java.io.FileInputStream; 13 | import java.io.IOException; 14 | import java.net.URL; 15 | import java.util.Map; 16 | 17 | import static org.junit.Assert.assertFalse; 18 | import static org.junit.Assert.assertTrue; 19 | 20 | public class FrameworkStateHelperTest { 21 | 22 | FrameworkStateHelper stateHelper; 23 | final String FID = "test-framework-uuid"; 24 | 25 | @Before 26 | public void setUp() throws Exception { 27 | stateHelper = new FrameworkStateHelper(new InMemoryState()); 28 | } 29 | 30 | @After 31 | public void tearDown() throws Exception { 32 | } 33 | 34 | 35 | //@Test 36 | public void testHasRegisteredFrameworks() throws Exception { 37 | assertFalse(stateHelper.hasRegisteredFrameworks()); 38 | } 39 | 40 | //@Test 41 | public void testGetFrameworkIdMap() throws Exception { 42 | stateHelper.getFrameworkIdMap(); 43 | } 44 | 45 | //@Test 46 | public void testGetFrameworkNameMap() throws Exception { 47 | Map frameworks = stateHelper.getFrameworkNameMap(); 48 | assertTrue(frameworks.isEmpty()); 49 | } 50 | 51 | @Test 52 | public void testSaveFrameworkConfig() throws Exception { 53 | ObjectMapper mapper = new ObjectMapper(); 54 | byte[] jsonBytes = readJsonResource("/state/ZkTestFramework.json"); 55 | Framework framework = mapper.readValue(jsonBytes, Framework.class); 56 | framework.setId(FID); 57 | 58 | stateHelper.saveFrameworkConfig(framework); 59 | 60 | Framework fromState = stateHelper.getFrameworkConfig(FID); 61 | 62 | assertTrue(framework.toString().equals(fromState.toString())); 63 | } 64 | 65 | //@Test 66 | public void testGetFrameworkConfig() throws Exception { 67 | 68 | } 69 | 70 | private byte[] readJsonResource(String resource) throws IOException { 71 | URL url = this.getClass().getResource(resource); 72 | File jsonFile = new File(url.getFile()); 73 | byte [] jsonBytes = IOUtils.toByteArray(new FileInputStream(jsonFile)); 74 | return jsonBytes; 75 | } 76 | } -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/configuration/Defaults.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.configuration; 2 | 3 | 4 | import java.util.ResourceBundle; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | public final class Defaults { 8 | 9 | private static final ResourceBundle rb = ResourceBundle.getBundle("accumulo-mesos"); 10 | 11 | // Default Port to bind http interface to 12 | public final static int HTTP_PORT = 18120; 13 | 14 | // Bind to all interfaces by default 15 | public final static String BIND_ADDRESS = "0.0.0.0"; 16 | 17 | // Look for mesos master on local machine 18 | public final static String MESOS_MASTER = "10.0.2.15:5050"; 19 | 20 | // Framework name 21 | public final static String FRAMEWORK_NAME = "Accumulo-Mesos"; 22 | 23 | // Zookeeper server 24 | public final static String ZK_SERVERS = "127.0.0.1:2181"; 25 | // Zookeeper connection timeout 26 | public final static long ZOOKEEPER_TIMEOUT = 3l; 27 | public final static TimeUnit ZK_TIMEOUT_UNIT = TimeUnit.SECONDS; 28 | 29 | // I spent way too much time figuring out this needs a leading / so don't be me 30 | public final static String ZK_STATE_ZNODE = "/accumulo-mesos"; 31 | 32 | // Memory settings are MB 33 | public final static int MIN_MASTER_MEM = 512; 34 | public final static int MIN_TSERVER_MEM = 512; 35 | public final static int MIN_GC_MEM = 128; 36 | public final static int MIN_MONITOR_MEM = 128; 37 | 38 | public final static String DEFAULT_8G_MAX_MEMORY = "8192"; 39 | public final static String DEFAULT_2G_MAX_MEMORY = "2048"; 40 | public final static String DEFAULT_1G_MAX_MEMORY = "1024"; 41 | 42 | // Minimum CPU settings 43 | public final static int MIN_MASTER_CPUS = 1; 44 | public final static int MIN_TSERVER_CPUS = 1; 45 | public final static int MIN_GC_CPUS = 1; 46 | public final static int MIN_MONITOR_CPUS = 1; 47 | 48 | // Minimum number of tservers to launch 49 | public final static int MIN_TSERVERS = 3; 50 | 51 | public final static String ROOT_USER = "root"; 52 | public final static String ROOT_PASSWORD = "secret"; 53 | 54 | public final static double MAX_EXECUTOR_MEM = 128; 55 | public final static double MIN_EXECUTOR_MEM = 16; 56 | // CPUS to be consumed by executor outside of server process 57 | public final static double EXECUTOR_CPUS = 0.25; 58 | 59 | // Default location of the accumulo-site.xml file 60 | public final static String ACCUMULO_SITE_URI = "file:./conf/default-accumulo-site.xml"; 61 | 62 | 63 | } 64 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lang/es.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* jshint quotmark: double */ 4 | window.SwaggerTranslator.learn({ 5 | "Warning: Deprecated":"Advertencia: Obsoleto", 6 | "Implementation Notes":"Notas de implementación", 7 | "Response Class":"Clase de la Respuesta", 8 | "Status":"Status", 9 | "Parameters":"Parámetros", 10 | "Parameter":"Parámetro", 11 | "Value":"Valor", 12 | "Description":"Descripción", 13 | "Parameter Type":"Tipo del Parámetro", 14 | "Data Type":"Tipo del Dato", 15 | "Response Messages":"Mensajes de la Respuesta", 16 | "HTTP Status Code":"Código de Status HTTP", 17 | "Reason":"Razón", 18 | "Response Model":"Modelo de la Respuesta", 19 | "Request URL":"URL de la Solicitud", 20 | "Response Body":"Cuerpo de la Respuesta", 21 | "Response Code":"Código de la Respuesta", 22 | "Response Headers":"Encabezados de la Respuesta", 23 | "Hide Response":"Ocultar Respuesta", 24 | "Try it out!":"Pruébalo!", 25 | "Show/Hide":"Mostrar/Ocultar", 26 | "List Operations":"Listar Operaciones", 27 | "Expand Operations":"Expandir Operaciones", 28 | "Raw":"Crudo", 29 | "can't parse JSON. Raw result":"no puede parsear el JSON. Resultado crudo", 30 | "Model Schema":"Esquema del Modelo", 31 | "Model":"Modelo", 32 | "apply":"aplicar", 33 | "Username":"Nombre de usuario", 34 | "Password":"Contraseña", 35 | "Terms of service":"Términos de Servicio", 36 | "Created by":"Creado por", 37 | "See more at":"Ver más en", 38 | "Contact the developer":"Contactar al desarrollador", 39 | "api version":"versión de la api", 40 | "Response Content Type":"Tipo de Contenido (Content Type) de la Respuesta", 41 | "fetching resource":"buscando recurso", 42 | "fetching resource list":"buscando lista del recurso", 43 | "Explore":"Explorar", 44 | "Show Swagger Petstore Example Apis":"Mostrar Api Ejemplo de Swagger Petstore", 45 | "Can't read from server. It may not have the appropriate access-control-origin settings.":"No se puede leer del servidor. Tal vez no tiene la configuración de control de acceso de origen (access-control-origin) apropiado.", 46 | "Please specify the protocol for":"Por favor, especificar el protocola para", 47 | "Can't read swagger JSON from":"No se puede leer el JSON de swagger desde", 48 | "Finished Loading Resource Information. Rendering Swagger UI":"Finalizada la carga del recurso de Información. Mostrando Swagger UI", 49 | "Unable to read api":"No se puede leer la api", 50 | "from path":"desde ruta", 51 | "server returned":"el servidor retornó" 52 | }); 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.vagrant/ 2 | **/logs/ 3 | 4 | # Compiled Licenses 5 | **/licenses/ 6 | **/THIRD-PARTY.txt 7 | 8 | # JAVA 9 | *.class 10 | 11 | # Mobile Tools for Java (J2ME) 12 | .mtj.tmp/ 13 | 14 | # Package Files # 15 | *.jar 16 | *.war 17 | *.ear 18 | 19 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 20 | hs_err_pid* 21 | 22 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 23 | 24 | *.iml 25 | 26 | ## Directory-based project format: 27 | .idea/ 28 | # if you remove the above rule, at least ignore the following: 29 | 30 | # User-specific stuff: 31 | # .idea/workspace.xml 32 | # .idea/tasks.xml 33 | # .idea/dictionaries 34 | 35 | # Sensitive or high-churn files: 36 | # .idea/dataSources.ids 37 | # .idea/dataSources.xml 38 | # .idea/sqlDataSources.xml 39 | # .idea/dynamic.xml 40 | # .idea/uiDesigner.xml 41 | 42 | # Gradle: 43 | # .idea/gradle.xml 44 | # .idea/libraries 45 | 46 | # Mongo Explorer plugin: 47 | # .idea/mongoSettings.xml 48 | 49 | ## File-based project format: 50 | *.ipr 51 | *.iws 52 | 53 | ## Plugin-specific files: 54 | 55 | # IntelliJ 56 | /out/ 57 | 58 | # mpeltonen/sbt-idea plugin 59 | .idea_modules/ 60 | 61 | # JIRA plugin 62 | atlassian-ide-plugin.xml 63 | 64 | # Crashlytics plugin (for Android Studio and IntelliJ) 65 | com_crashlytics_export_strings.xml 66 | crashlytics.properties 67 | crashlytics-build.properties 68 | 69 | *.pydevproject 70 | .metadata 71 | .gradle 72 | tmp/ 73 | *.tmp 74 | *.bak 75 | *.swp 76 | *~.nib 77 | local.properties 78 | .settings/ 79 | .loadpath 80 | 81 | # Eclipse Core 82 | .project 83 | 84 | # External tool builders 85 | .externalToolBuilders/ 86 | 87 | # Locally stored "Eclipse launch configurations" 88 | *.launch 89 | 90 | # CDT-specific 91 | .cproject 92 | 93 | # JDT-specific (Eclipse Java Development Tools) 94 | .classpath 95 | 96 | # PDT-specific 97 | .buildpath 98 | 99 | # sbteclipse plugin 100 | .target 101 | 102 | # TeXlipse plugin 103 | .texlipse 104 | 105 | #VIM 106 | [._]*.s[a-w][a-z] 107 | [._]s[a-w][a-z] 108 | *.un~ 109 | Session.vim 110 | .netrwhist 111 | *~ 112 | 113 | .DS_Store 114 | .AppleDouble 115 | .LSOverride 116 | 117 | # Icon must end with two \r 118 | Icon 119 | 120 | 121 | # Thumbnails 122 | ._* 123 | 124 | # Files that might appear in the root of a volume 125 | .DocumentRevisions-V100 126 | .fseventsd 127 | .Spotlight-V100 128 | .TemporaryItems 129 | .Trashes 130 | .VolumeIcon.icns 131 | 132 | # Directories potentially created on remote AFP share 133 | .AppleDB 134 | .AppleDesktop 135 | Network Trash Folder 136 | Temporary Items 137 | .apdisk 138 | 139 | # Maven 140 | target/ 141 | pom.xml.tag 142 | pom.xml.releaseBackup 143 | pom.xml.versionsBackup 144 | pom.xml.next 145 | release.properties 146 | dependency-reduced-pom.xml 147 | buildNumber.properties 148 | .mvn/timing.properties 149 | 150 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Accumulo Mesos Framework 2 | ========================= 3 | Initialize and run Accumulo clusters as a Mesos framework. 4 | 5 | ------------ 6 | 7 | **DISCLAIMER** 8 | _This is a very early version of accumulo-mesos framework. This 9 | document, code behavior, and anything else may change without notice and/or break older installations._ 10 | 11 | ------------ 12 | 13 | # Design 14 | The accumulo-mesos framework launches accumulo server processes on mesos client machines using 15 | the `$ACCUMULO_HOME/bin/accumulo ` script. It automatically configures Java and Accumulo 16 | memory settings based on the mesos offer. The framework doesn't depend on Accumulo directly so 17 | it should be able to support the more recent versions of Accumulo. It is tested with 1.7.0 18 | 19 | # Current Status 20 | 21 | ### Implemented 22 | * Framework no longer depends on accumulo! Accumulo is uploaded to HDFS. There are assumptions about 23 | where somethings will be within the accumulo tarball when extracted, but this has been stable. 24 | * Accumulo init is separate step from running the framework. Currently this requires a local copy of 25 | Accumulo somewhere. 26 | 27 | ### Near Term Tasks 28 | * Docker? 29 | * Run accumulo init from a mesos client just like the accumulo servers. 30 | * Flesh out the framework webservice. 31 | * Reconnect framework to a running cluster. 32 | 33 | # Running the Framework 34 | First you have to upload artifacts to HDFS (accumulo tarball, framework tarball, native libs .so) 35 | and copy that into your config JSON structures. (see `dev/config`) 36 | 37 | Running the framework is then a two step process. First you must initialize the framework. 38 | See `dev/init_framework.sh` Then you can run the framework see `dev\start_framework.sh` 39 | 40 | 41 | ``` 42 | java -jar /vagrant/dev/dist/accumulo-mesos-dist-0.2.0-SNAPSHOT/accumulo-mesos-framework-0.2.0-SNAPSHOT-jar-with-dependencies.jar -h 43 | usage: accumulo-mesos [-b ] [-cc ] [-fc ] [-h] [-i] [-m 44 | ] [-n ] [-P ] [-t ] [-v] [-z ] 45 | -b,--bind-address IP address of interface to bind HTTP interface 46 | to 47 | -cc,--cluster JSON file containing cluster configuration 48 | -fc,--framework JSON file of entire framework configuration 49 | -h,--help Print this message and exit 50 | -i,--init If present, initialize new Accumulo instance 51 | -m,--master Location of mesos master to connect to 52 | -n,--name Name of this mesos framework 53 | -P,--port Port number to serve HTTP interface 54 | -t,--tarball URI of framework tarball 55 | -v,--version Show version number 56 | -z,--zookeepers List of Zookeeper servers 57 | ``` 58 | 59 | ## Configuration 60 | See config examples in `dev/config` 61 | 62 | ## Testing 63 | A multi-vm Vagrantfile is provided along with many provisioning scripts to setup 64 | the VMs for testing the framework. See `/dev` directory for more info. 65 | 66 | # Thanks 67 | Thanks to the cassandra-mesos project. I stole a lot of the project setup and framework design ideas from there. 68 | -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | aredee.mesos.frameworks.accumulo 7 | accumulo-mesos-parent 8 | ${global.version} 9 | 10 | 4.0.0 11 | 12 | accumulo-mesos-scheduler 13 | 14 | 15 | The Accumulo-on-Mesos framework to deploy Apache Accumulo to Apache Mesos. 16 | This artifact contains the Mesos scheduler for the Accumulo-on-Mesos framework. 17 | 18 | 19 | 20 | 21 | Apache 2 22 | http://www.apache.org/licenses/LICENSE-2.0.txt 23 | repo 24 | Apache License Version 2.0 25 | 26 | 27 | 28 | Apache Mesos Accumulo framework 29 | http://mesos.apache.org/ 30 | 31 | 32 | 33 | 34 | ossrh 35 | https://oss.sonatype.org/content/repositories/snapshots 36 | 37 | 38 | ossrh 39 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 40 | 41 | 42 | 43 | scm:git:https://github.com/aredee/accumulo-mesos.git 44 | scm:git:https://github.com/aredee/accumulo-mesos.git 45 | https://github.com/aredee/accumulo-mesos 46 | 47 | 48 | Github 49 | https://github.com/aredee/accumulo-mesos/issues 50 | 51 | 52 | 53 | 54 | aredee.mesos.frameworks.accumulo 55 | accumulo-mesos-common 56 | ${project.version} 57 | 58 | 59 | org.apache.mesos 60 | mesos 61 | 62 | 63 | org.slf4j 64 | slf4j-api 65 | 66 | 67 | org.slf4j 68 | ${slf4j.binding} 69 | 70 | 71 | com.google.guava 72 | guava 73 | 74 | 75 | junit 76 | junit 77 | test 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /docker/bin/install-accumulo.sh: -------------------------------------------------------------------------------- 1 | ## 2 | # YARN installer script for Apache Myriad Deployment 3 | ## 4 | HADOOP_VER="2.7.1" 5 | HADOOP_TARBALL_URL=http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz 6 | ACCUMULO_TARBALL_URL=http://download.nextag.com/apache/accumulo/1.7.0/accumulo-1.7.0-bin.tar.gz 7 | 8 | 9 | echo "Installing Yarn...." 10 | if [ ! -z "$1" ];then 11 | HADOOP_TARBALL_URL=$1 12 | echo "Deleting previous hadoop home" 13 | rm -rf ${HADOOP_HOME} 14 | fi 15 | 16 | # Download the tarball 17 | wget -O /opt/hadoop.tgz ${HADOOP_TARBALL_URL} 18 | HADOOP_BASENAME=`basename ${HADOOP_TARBALL_URL} .tar.gz` 19 | 20 | # Put in env defaults if they are missing 21 | export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'} 22 | export HADOOP_USER=${HADOOP_USER:='accumulo'} 23 | export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'} 24 | export USER_UID=${USER_UID:='113'} 25 | export GROUP_UID=${GROUP_GID:='112'} 26 | 27 | # Add hduser user 28 | groupadd $HADOOP_GROUP -g ${GROUP_UID} 29 | useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash 30 | #mkdir /home/${HADOOP_USER} 31 | chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER} 32 | 33 | # Extract Hadoop 34 | tar vxzf /opt/hadoop.tgz -C /tmp 35 | #mv /tmp/hadoop-${HADOOP_VER} ${HADOOP_HOME} 36 | echo "Moving /tmp/hadoop-${HADOOP_BASENAME} to ${HADOOP_HOME}" 37 | mv /tmp/${HADOOP_BASENAME} ${HADOOP_HOME} 38 | ls -lath ${HADOOP_HOME} 39 | 40 | mkdir /home/$HADOOP_USER 41 | chown -R ${HADOOP_USER}:${HADOOP_GROUP} ${HADOOP_HOME} 42 | 43 | # Init bashrc with hadoop env variables 44 | sh -c 'echo export JAVA_HOME=/usr >> /home/${HADOOP_USER}/.bashrc' 45 | sh -c 'echo export HADOOP_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc' 46 | sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/bin >> /home/${HADOOP_USER}/.bashrc' 47 | sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/sbin >> /home/${HADOOP_USER}/.bashrc' 48 | sh -c 'echo export HADOOP_MAPRED_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc' 49 | sh -c 'echo export HADOOP_COMMON_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc' 50 | sh -c 'echo export HADOOP_HDFS_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc' 51 | sh -c 'echo export YARN_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc' 52 | sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_HOME\}/lib/native >> /home/${HADOOP_USER}/.bashrc' 53 | sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\${HADOOP_HOME}/lib\" >> /home/${HADOOP_USER}/.bashrc' 54 | 55 | # Link Mesos Libraries 56 | touch ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh 57 | echo "export JAVA_HOME=/usr" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh 58 | echo "export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh 59 | 60 | # Ensure the hadoop-env is executable 61 | chmod +x ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh 62 | 63 | 64 | 65 | # install accumulo 66 | ACCUMULO_HOME=/opt/accumulo 67 | 68 | echo "Installing accumulo..." 69 | if [ ! -z "$1" ];then 70 | ACCUMULO_TARBALL_URL=$1 71 | echo "Deleting previous accumulo home" 72 | rm -rf ${ACCUMULO_HOME} 73 | fi 74 | 75 | wget -O /opt/accumulo.tgz ${ACCUMULO_TARBALL_URL} 76 | # Extract Accumulo 77 | tar vxzf /opt/accumulo.tgz -C /tmp 78 | #mv /tmp/accumulo-${HADOOP_VER} ${ACCUMULO_HOME} 79 | echo "Moving /tmp/hadoop-${ACCUMULO_BASENAME} to ${ACCUMULO_HOME}" 80 | mv /tmp/${ACCUMULO_BASENAME} ${ACCUMULO_HOME} 81 | ls -lath ${ACCUMULO_HOME} 82 | chown -R ${HADOOP_USER}:${HADOOP_GROUP} ${ACCUMULO_HOME} 83 | -------------------------------------------------------------------------------- /accumulo-mesos-scheduler/src/main/java/aredee/mesos/frameworks/accumulo/scheduler/matcher/MinCpuMinRamFIFOMatcher.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.scheduler.matcher; 2 | 3 | import aredee.mesos.frameworks.accumulo.configuration.Defaults; 4 | import aredee.mesos.frameworks.accumulo.model.Accumulo; 5 | import aredee.mesos.frameworks.accumulo.model.ServerProfile; 6 | import aredee.mesos.frameworks.accumulo.model.Task; 7 | import com.google.common.collect.Lists; 8 | import org.apache.mesos.Protos; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.List; 13 | 14 | /** 15 | * Matcher that finds an offer that meets minimum cpu and ram requirements for the server on a first come first 16 | * served basis. 17 | * 18 | */ 19 | public class MinCpuMinRamFIFOMatcher implements Matcher { 20 | 21 | private static final Logger LOGGER = LoggerFactory.getLogger(MinCpuMinRamFIFOMatcher.class); 22 | 23 | private Accumulo config; 24 | 25 | public MinCpuMinRamFIFOMatcher(Accumulo config){ 26 | this.config = config; 27 | } 28 | 29 | /** 30 | * Returns a list of matched servers and offers. If offers were not found for all servers, 31 | * a Match object will be present with no offer 32 | * 33 | * @param tasks 34 | * @param offers 35 | * 36 | */ 37 | @Override 38 | public List matchOffers(List tasks, List offers) { 39 | 40 | LOGGER.info("Matching {} tasks to {} offers", tasks.size(), offers.size()); 41 | 42 | List matches = Lists.newArrayListWithCapacity(tasks.size()); 43 | 44 | for(int tt = 0; tt < tasks.size(); tt++){ 45 | Task task = tasks.get(tt); 46 | for( int oo = 0; (oo < offers.size()) ; oo++){ 47 | Protos.Offer offer = offers.get(oo); 48 | LOGGER.info("Checking offer id {} for task {}", offer.getId().getValue(), task.getType().getServerKeyword()); 49 | if( offerMatchesTask(task, offer)){ 50 | // create a match 51 | Match match = new Match(task, offer); 52 | matches.add(match); 53 | offers.remove(offer); 54 | 55 | LOGGER.info("Found match! task {} offer {}", task.getType().getServerKeyword(), offer.getId().getValue()); 56 | break; 57 | } 58 | } 59 | } 60 | 61 | return matches; 62 | } 63 | 64 | private boolean offerMatchesTask(Task task, Protos.Offer offer){ 65 | double offerCpus = -1; 66 | double offerMem = -1; 67 | 68 | // Get offer resources 69 | for( Protos.Resource resource : offer.getResourcesList()){ 70 | if (resource.getName().equalsIgnoreCase("cpus")) { 71 | offerCpus = resource.hasScalar() ? resource.getScalar().getValue() : 0.0; 72 | } else if (resource.getName().equalsIgnoreCase("mem")) { 73 | offerMem = resource.hasScalar() ? resource.getScalar().getValue() : 0.0; 74 | } 75 | } 76 | // TODO Have to take into account the executor resources because MESOS will. 77 | 78 | // Get profile resources 79 | ServerProfile profile = task.getServerProfile(); 80 | double profileCpus = profile.getCpus().doubleValue() + Defaults.EXECUTOR_CPUS; 81 | double profileMem = profile.getMemory().doubleValue() + config.getExecutorMemory(); 82 | 83 | boolean offerMatches = cpusAndMemAreAdequate(offerCpus, offerMem, profileCpus, profileMem); 84 | 85 | return offerMatches; 86 | } 87 | 88 | private boolean cpusAndMemAreAdequate(double offerCpu, double offerMem, double serverCpu, double serverMem) { 89 | return ( offerCpu >= serverCpu ) && ( offerMem >= serverMem ); 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/configuration/Environment.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.configuration; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | import java.util.Set; 7 | 8 | 9 | public class Environment { 10 | 11 | public static final String JAVA_HOME = "JAVA_HOME"; 12 | 13 | public static final String CLASSPATH = "CLASSPATH"; 14 | 15 | public static final String HADOOP_PREFIX = "HADOOP_PREFIX"; 16 | public static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR"; 17 | public static final String HADOOP_HOME = "HADOOP_HOME"; 18 | 19 | public static final String ACCUMULO_HOME = "ACCUMULO_HOME"; 20 | public static final String ACCUMULO_LOG_DIR = "ACCUMULO_LOG_DIR"; 21 | public static final String ACCUMULO_CLIENT_CONF_PATH = "ACCUMULO_CLIENT_CONF_PATH"; 22 | public static final String ACCUMULO_CONF_DIR = "ACCUMULO_CONF_DIR"; 23 | public static final String ACCUMULO_WALOG = "ACCUMULO_WALOG"; 24 | 25 | public static final String LD_LIBRARY_PATH = "LD_LIBRARY_PATH"; 26 | public static final String DYLD_LIBRARY_PATH = "DYLD_LIBRARY_PATH"; 27 | 28 | // List of paths separated by commas 29 | public static final String NATIVE_LIB_PATHS = "NATIVE_LIB_PATHS"; 30 | 31 | public static final String ZOOKEEPER_HOME = "ZOOKEEPER_HOME"; 32 | 33 | public static final String MESOS_DIRECTORY = "MESOS_DIRECTORY"; 34 | 35 | public static final List REQUIRED_FRAMEWORK_VARS = Arrays.asList( 36 | JAVA_HOME, 37 | ACCUMULO_HOME, 38 | HADOOP_PREFIX, 39 | HADOOP_CONF_DIR, 40 | ZOOKEEPER_HOME); 41 | 42 | public static final List REQUIRED_EXECUTOR_VARS = Arrays.asList( 43 | JAVA_HOME, 44 | ACCUMULO_HOME, 45 | HADOOP_PREFIX, 46 | ZOOKEEPER_HOME, 47 | ACCUMULO_LOG_DIR); 48 | 49 | 50 | /* 51 | Optional Accumulo vars for executor 52 | */ 53 | public static final String ACCUMULO_XTRAJARS = "ACCUMULO_XTRAJARS"; 54 | public static final String ACCUMULO_GENERAL_OPTS = "ACCUMULO_GENERAL_OPTS"; 55 | public static final String ACCUMULO_JAAS_CONF = "ACCUMULO_JAAS_CONF"; //-Djava.security.auth.login.config=${ACCUMULO_JAAS_CONF} 56 | public static final String ACCUMULO_KRB5_CONF = "ACCUMULO_KRB5_CONF"; //-Djava.security.krb5.conf=${ACCUMULO_KRB5_CONF} 57 | 58 | public static final String ACCUMULO_MASTER_OPTS = "ACCUMULO_MASTER_OPTS"; 59 | public static final String ACCUMULO_GC_OPTS = "ACCUMULO_GC_OPTS"; 60 | public static final String ACCUMULO_TSERVER_OPTS = "ACCUMULO_TSERVER_OPTS"; 61 | public static final String ACCUMULO_MONITOR_OPTS = "ACCUMULO_MONITOR_OPTS"; 62 | public static final String ACCUMULO_LOGGER_OPTS = "ACCUMULO_LOGGER_OPTS"; 63 | public static final String ACCUMULO_OTHER_OPTS = "ACCUMULO_OTHER_OPTS"; 64 | public static final String ACCUMULO_KILL_CMD = "ACCUMULO_KILL_CMD"; 65 | 66 | // generated 67 | public static final String START_JAR = "START_JAR"; 68 | public static final String LIB_PATH = "LIB_PATH"; 69 | /* 70 | if [ -e "${HADOOP_PREFIX}/lib/native/libhadoop.so" ]; then 71 | LIB_PATH="${HADOOP_PREFIX}/lib/native" 72 | LD_LIBRARY_PATH="${LIB_PATH}:${LD_LIBRARY_PATH}" # For Linux 73 | DYLD_LIBRARY_PATH="${LIB_PATH}:${DYLD_LIBRARY_PATH}" # For Mac 74 | fi 75 | */ 76 | 77 | 78 | public static List getMissingVariables(List vars) { 79 | List missingVariables = new ArrayList<>(); 80 | Set envKeys = System.getenv().keySet(); 81 | for(String var : vars ) { 82 | if(!envKeys.contains(var)){ 83 | missingVariables.add(var); 84 | } 85 | } 86 | return missingVariables; 87 | } 88 | 89 | public static String get(String var){ 90 | return System.getenv(var); 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/lib/jquery.ba-bbq.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | * jQuery BBQ: Back Button & Query Library - v1.2.1 - 2/17/2010 3 | * http://benalman.com/projects/jquery-bbq-plugin/ 4 | * 5 | * Copyright (c) 2010 "Cowboy" Ben Alman 6 | * Dual licensed under the MIT and GPL licenses. 7 | * http://benalman.com/about/license/ 8 | */ 9 | (function($,p){var i,m=Array.prototype.slice,r=decodeURIComponent,a=$.param,c,l,v,b=$.bbq=$.bbq||{},q,u,j,e=$.event.special,d="hashchange",A="querystring",D="fragment",y="elemUrlAttr",g="location",k="href",t="src",x=/^.*\?|#.*$/g,w=/^.*\#/,h,C={};function E(F){return typeof F==="string"}function B(G){var F=m.call(arguments,1);return function(){return G.apply(this,F.concat(m.call(arguments)))}}function n(F){return F.replace(/^[^#]*#?(.*)$/,"$1")}function o(F){return F.replace(/(?:^[^?#]*\?([^#]*).*$)?.*/,"$1")}function f(H,M,F,I,G){var O,L,K,N,J;if(I!==i){K=F.match(H?/^([^#]*)\#?(.*)$/:/^([^#?]*)\??([^#]*)(#?.*)/);J=K[3]||"";if(G===2&&E(I)){L=I.replace(H?w:x,"")}else{N=l(K[2]);I=E(I)?l[H?D:A](I):I;L=G===2?I:G===1?$.extend({},I,N):$.extend({},N,I);L=a(L);if(H){L=L.replace(h,r)}}O=K[1]+(H?"#":L||!K[1]?"?":"")+L+J}else{O=M(F!==i?F:p[g][k])}return O}a[A]=B(f,0,o);a[D]=c=B(f,1,n);c.noEscape=function(G){G=G||"";var F=$.map(G.split(""),encodeURIComponent);h=new RegExp(F.join("|"),"g")};c.noEscape(",/");$.deparam=l=function(I,F){var H={},G={"true":!0,"false":!1,"null":null};$.each(I.replace(/\+/g," ").split("&"),function(L,Q){var K=Q.split("="),P=r(K[0]),J,O=H,M=0,R=P.split("]["),N=R.length-1;if(/\[/.test(R[0])&&/\]$/.test(R[N])){R[N]=R[N].replace(/\]$/,"");R=R.shift().split("[").concat(R);N=R.length-1}else{N=0}if(K.length===2){J=r(K[1]);if(F){J=J&&!isNaN(J)?+J:J==="undefined"?i:G[J]!==i?G[J]:J}if(N){for(;M<=N;M++){P=R[M]===""?O.length:R[M];O=O[P]=M').hide().insertAfter("body")[0].contentWindow;q=function(){return a(n.document[c][l])};o=function(u,s){if(u!==s){var t=n.document;t.open().close();t[c].hash="#"+u}};o(a())}}m.start=function(){if(r){return}var t=a();o||p();(function s(){var v=a(),u=q(t);if(v!==t){o(t=v,u);$(i).trigger(d)}else{if(u!==t){i[c][l]=i[c][l].replace(/#.*/,"")+"#"+u}}r=setTimeout(s,$[d+"Delay"])})()};m.stop=function(){if(!n){r&&clearTimeout(r);r=0}};return m})()})(jQuery,this); -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # # vi: set ft=ruby : 3 | 4 | VAGRANTFILE_API_VERSION = "2" 5 | 6 | NUM_SLAVES = 6 7 | 8 | # re-write /etc/hosts because ubuntu does 127.0.1.1 stuff that borks Hadoop 9 | $host_script = < 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 102 | 103 | 104 | 105 | 115 | 116 |
 
117 |
118 | 119 | 120 | -------------------------------------------------------------------------------- /accumulo-mesos-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | aredee.mesos.frameworks.accumulo 7 | accumulo-mesos-parent 8 | ${global.version} 9 | 10 | 4.0.0 11 | 12 | accumulo-mesos-common 13 | 14 | 15 | The Accumulo-on-Mesos framework to deploy Apache Accumulo to Apache Mesos. 16 | This artifact contains the protobuf generated code. 17 | 18 | 19 | 20 | 21 | Apache 2 22 | http://www.apache.org/licenses/LICENSE-2.0.txt 23 | repo 24 | Apache License Version 2.0 25 | 26 | 27 | 28 | Apache Mesos Accumulo framework 29 | http://mesos.apache.org/ 30 | 31 | 32 | 33 | 34 | ossrh 35 | https://oss.sonatype.org/content/repositories/snapshots 36 | 37 | 38 | ossrh 39 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 40 | 41 | 42 | 43 | scm:git:https://github.com/aredee/accumulo-mesos.git 44 | scm:git:https://github.com/aredee/accumulo-mesos.git 45 | https://github.com/aredee/accumulo-mesos 46 | 47 | 48 | Github 49 | https://github.com/aredee/accumulo-mesos/issues 50 | 51 | 52 | 53 | 54 | com.google.protobuf 55 | protobuf-java 56 | 57 | 58 | com.google.guava 59 | guava 60 | 61 | 62 | commons-cli 63 | commons-cli 64 | 65 | 66 | org.apache.mesos 67 | mesos 68 | 69 | 70 | org.slf4j 71 | slf4j-api 72 | 73 | 74 | org.slf4j 75 | ${slf4j.binding} 76 | 77 | 78 | org.apache.commons 79 | commons-vfs2 80 | 81 | 82 | org.apache.commons 83 | commons-lang3 84 | 85 | 86 | commons-io 87 | commons-io 88 | 89 | 90 | javax.validation 91 | validation-api 92 | 93 | 94 | com.fasterxml.jackson.core 95 | jackson-core 96 | 97 | 98 | com.fasterxml.jackson.core 99 | jackson-annotations 100 | 101 | 102 | com.fasterxml.jackson.core 103 | jackson-databind 104 | 105 | 106 | com.fasterxml.jackson.jaxrs 107 | jackson-jaxrs-base 108 | 109 | 110 | com.fasterxml.jackson.jaxrs 111 | jackson-jaxrs-json-provider 112 | 113 | 114 | junit 115 | junit 116 | test 117 | 118 | 119 | 120 | 121 | 122 | 123 | src/main/resources 124 | true 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/ServerProfile.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.google.common.collect.Lists; 5 | 6 | import java.math.BigDecimal; 7 | import java.util.List; 8 | 9 | 10 | public class ServerProfile { 11 | 12 | public enum TypeEnum { 13 | master("master"), 14 | tserver("tserver"), 15 | gc("gc"), 16 | tracer("tracer"), 17 | monitor("monitor"), 18 | init("init"), 19 | shell("shell"); 20 | 21 | private final String keyword; 22 | 23 | TypeEnum(String keyword){ 24 | this.keyword = keyword; 25 | } 26 | 27 | public String getServerKeyword(){ return keyword;} 28 | } 29 | 30 | private String name = null; 31 | private String description = null; 32 | private TypeEnum type = null; 33 | private Integer memory = 128; 34 | private BigDecimal cpus = null; 35 | private String launcher = null; 36 | private String user = null; 37 | private String siteXml = null; 38 | private List serverKeywordArgs = null; 39 | private boolean useNativeMaps = false; 40 | 41 | /** 42 | * A short name for this profile\n 43 | **/ 44 | @JsonProperty("name") 45 | public String getName() { 46 | return name; 47 | } 48 | public void setName(String name) { 49 | this.name = name; 50 | } 51 | 52 | 53 | /** 54 | * Description of this profile\n 55 | **/ 56 | @JsonProperty("description") 57 | public String getDescription() { 58 | return description; 59 | } 60 | public void setDescription(String description) { 61 | this.description = description; 62 | } 63 | 64 | /** 65 | * Accumulo server type\n 66 | **/ 67 | @JsonProperty("type") 68 | public TypeEnum getType() { 69 | return type; 70 | } 71 | public void setType(TypeEnum type) { 72 | this.type = type; 73 | } 74 | 75 | 76 | /** 77 | * Memory to allocate to this server in MB. -Xmx yyyyM\n 78 | **/ 79 | @JsonProperty("mem") 80 | public Integer getMemory() { 81 | return memory; 82 | } 83 | public void setMemory(Integer memory) { 84 | this.memory = memory; 85 | } 86 | 87 | 88 | /** 89 | * Number of cpus to allocate to this server\n 90 | **/ 91 | @JsonProperty("cpus") 92 | public BigDecimal getCpus() { 93 | return cpus; 94 | } 95 | public void setCpus(BigDecimal cpus) { 96 | this.cpus = cpus; 97 | } 98 | 99 | 100 | /** 101 | * Fully qualified class name of launcher class to launch with.\n 102 | **/ 103 | //TODO implement defining launcher as config 104 | @JsonProperty("launcher") 105 | public String getLauncher() { 106 | return launcher; 107 | } 108 | public void setLauncher(String launcher) { 109 | this.launcher = launcher; 110 | } 111 | 112 | 113 | /** 114 | * System user name to run server processes as.\n 115 | **/ 116 | @JsonProperty("user") 117 | public String getUser() { 118 | return user; 119 | } 120 | public void setUser(String user) { 121 | this.user = user; 122 | } 123 | public boolean hasUser(){ 124 | boolean hasUser = false; 125 | if( user != null ){ 126 | hasUser = !user.isEmpty(); 127 | } 128 | return hasUser; 129 | } 130 | 131 | @JsonProperty("siteXml") 132 | public String getSiteXml() { return siteXml; } 133 | public void setSiteXml(String siteXml) { this.siteXml = siteXml; } 134 | 135 | @JsonProperty("serverKeywordArgs") 136 | public List getServerKeywordArgs(){ 137 | if( serverKeywordArgs == null ){ 138 | serverKeywordArgs = Lists.newArrayList(); 139 | } 140 | return this.serverKeywordArgs; 141 | } 142 | public void setServerKeywordArgs(List serverKeywordArgs){ 143 | this.serverKeywordArgs = serverKeywordArgs; 144 | } 145 | 146 | @JsonProperty("useNativeMaps") 147 | public boolean getUseNativeMaps(){ return useNativeMaps; } 148 | public void setUseNativemaps(boolean useNativeMaps){ 149 | this.useNativeMaps = useNativeMaps; 150 | } 151 | 152 | 153 | @Override 154 | public String toString() { 155 | StringBuilder sb = new StringBuilder(); 156 | sb.append("class ServerProfile {\n"); 157 | 158 | sb.append(" name: ").append(name).append("\n"); 159 | sb.append(" description: ").append(description).append("\n"); 160 | sb.append(" type: ").append(type).append("\n"); 161 | sb.append(" memory: ").append(memory).append("\n"); 162 | sb.append(" cpus: ").append(cpus).append("\n"); 163 | sb.append(" launcher: ").append(launcher).append("\n"); 164 | sb.append(" user: ").append(user).append("\n"); 165 | sb.append(" siteXml: ").append(siteXml).append("\n"); 166 | sb.append("}\n"); 167 | return sb.toString(); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/initialize/AccumuloInitializer.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.initialize; 2 | 3 | import aredee.mesos.frameworks.accumulo.configuration.Constants; 4 | import aredee.mesos.frameworks.accumulo.configuration.Environment; 5 | import aredee.mesos.frameworks.accumulo.model.Accumulo; 6 | import aredee.mesos.frameworks.accumulo.model.ServerProfile; 7 | import aredee.mesos.frameworks.accumulo.process.AccumuloProcessFactory; 8 | import org.apache.commons.io.IOUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.File; 13 | import java.io.FileOutputStream; 14 | import java.io.IOException; 15 | import java.io.OutputStream; 16 | import java.nio.file.CopyOption; 17 | import java.nio.file.Files; 18 | import java.nio.file.StandardCopyOption; 19 | import java.util.LinkedList; 20 | 21 | public class AccumuloInitializer { 22 | private static final Logger LOGGER = LoggerFactory.getLogger(AccumuloInitializer.class); 23 | 24 | private Accumulo config; 25 | private String accumuloHome; 26 | 27 | public AccumuloInitializer(Accumulo config) { 28 | this.config = config; 29 | this.accumuloHome = Environment.get(Environment.ACCUMULO_HOME); 30 | } 31 | 32 | /** 33 | * Write the accumulo site file and initialize accumulo. 34 | * 35 | * The system property Environment.ACCUMULO_HOME and HADOOP_CONF_DIR must be set and config must have 36 | * the accumulo instance name, root password, along with the accumulo directories set. 37 | * 38 | * @return accumulo instance name 39 | */ 40 | public int initialize() throws IOException { 41 | 42 | // run accumulo init procedure 43 | LOGGER.info("Writing accumulo-site.xml"); 44 | 45 | AccumuloSiteXml siteXml = new AccumuloSiteXml(this.config); 46 | siteXml.initializeFromScheduler(AccumuloSiteXml.getEmptySiteXml()); 47 | 48 | writeAccumuloSiteFile(accumuloHome, siteXml); 49 | config.setSiteXml(siteXml.toString()); 50 | 51 | // accumulo-env.sh 52 | copyAccumuloEnvFile(accumuloHome); // IOException 53 | 54 | LinkedList initArgs = new LinkedList<>(); 55 | initArgs.add("--instance-name"); 56 | initArgs.add(config.getInstance()); 57 | initArgs.add("--password"); 58 | initArgs.add(config.getRootPassword()); 59 | initArgs.add("--clear-instance-name"); 60 | 61 | AccumuloProcessFactory processFactory = new AccumuloProcessFactory(); 62 | 63 | Process initProcess; 64 | int status = 0; 65 | try { 66 | initProcess = processFactory.exec(ServerProfile.TypeEnum.init.getServerKeyword(), 67 | initArgs.toArray(new String[initArgs.size()])); 68 | initProcess.waitFor(); 69 | status = initProcess.exitValue(); 70 | } catch (Exception ioe) { 71 | LOGGER.error("IOException while trying to initialize Accumulo", ioe); 72 | status = -1; 73 | } 74 | return status; 75 | } 76 | 77 | public static void writeAccumuloSiteFile(String accumuloHomeDir, AccumuloSiteXml siteXml) { 78 | LOGGER.info("ACCUMULO HOME? " + accumuloHomeDir); 79 | try { 80 | 81 | File accumuloSiteFile = new File(accumuloHomeDir + File.separator + 82 | "conf" + File.separator + "accumulo-site.xml"); 83 | 84 | LOGGER.info("Writing accumulo-site.xml to {}", accumuloSiteFile.getAbsolutePath()); 85 | 86 | OutputStream siteFile = new FileOutputStream(accumuloSiteFile); 87 | IOUtils.write(siteXml.toXml(), siteFile); 88 | IOUtils.closeQuietly(siteFile); 89 | 90 | } catch (Exception e) { 91 | logErrorAndDie("Error Creating accumulo-site.xml\n",e); 92 | } 93 | } 94 | 95 | /** 96 | * This is required because the bin/accumulo script complains without it. The framework should be setting 97 | * all the environment variables that this sets, and the script uses them if already set. 98 | * 99 | * @param accumuloHomeDir 100 | * @throws IOException 101 | */ 102 | public static void copyAccumuloEnvFile(String accumuloHomeDir) throws IOException { 103 | File inputFile = new File(accumuloHomeDir+File.separator+"conf/examples/1GB/native-standalone/accumulo-env.sh"); 104 | File destFile = new File(accumuloHomeDir+File.separator+"conf/accumulo-env.sh"); 105 | Files.copy(inputFile.toPath(), destFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 106 | } 107 | 108 | /** 109 | * Copy native maps from mesosDirectory to accumuloHome/lib/native if the maps exist 110 | * 111 | * @param mesosDirectory 112 | * @param accumuloHome 113 | * @throws IOException 114 | */ 115 | public static void copyAccumuloNativeMaps( String mesosDirectory, String accumuloHome) throws IOException { 116 | File inputFile = new File(mesosDirectory+File.separator+ Constants.ACCUMULO_NATIVE_LIB); 117 | File destFile = new File(accumuloHome+File.separator+"lib/native"); 118 | if( inputFile.exists() ) { 119 | Files.copy(inputFile.toPath(), destFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 120 | } 121 | } 122 | 123 | 124 | private static void logErrorAndDie(String message, Exception e){ 125 | LOGGER.error(message, e); 126 | throw new RuntimeException(message, e); 127 | } 128 | 129 | 130 | } 131 | -------------------------------------------------------------------------------- /dev/provision/install_hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -v 2 | 3 | # $1 is ip of namenode, $2 is ip of slave 4 | 5 | set -e 6 | 7 | HADOOP_VER=2.5.2 8 | 9 | apt-get update 10 | 11 | apt-get install -y openssh-server 12 | apt-get install -y tar 13 | apt-get install -y gzip 14 | 15 | # Add hduser user and hadoop group 16 | 17 | if [ `/bin/egrep -i "^hadoop:" /etc/group` ]; then 18 | echo "Group hadoop already exists" 19 | else 20 | echo "Adding hadoop group" 21 | addgroup hadoop 22 | fi 23 | 24 | 25 | if [ `/bin/egrep -i "^hduser:" /etc/passwd` ]; then 26 | echo "User hduser already exists" 27 | else 28 | echo "creating hduser in group hadoop" 29 | adduser --ingroup hadoop --disabled-password --gecos "" --home /home/hduser hduser 30 | adduser hduser sudo 31 | fi 32 | 33 | 34 | # Setup password-less auth 35 | sudo -u hduser sh -c "mkdir /home/hduser/.ssh" 36 | sudo -u hduser sh -c "chmod 700 /home/hduser/.ssh" 37 | sudo -u hduser sh -c "yes | ssh-keygen -t rsa -N '' -f /home/hduser/.ssh/id_rsa" 38 | sudo -u hduser sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys' 39 | sudo -u hduser sh -c "ssh-keyscan -H $1 >> /home/hduser/.ssh/known_hosts" 40 | sudo -u hduser sh -c "ssh-keyscan -H localhost >> /home/hduser/.ssh/known_hosts" 41 | sudo -u hduser sh -c "ssh-keyscan -H $2 >> /home/hduser/.ssh/known_hosts" 42 | 43 | # Download Hadoop 44 | # I've decided to provide this under the vagrant directory 45 | #cd ~ 46 | #if [ ! -f hadoop-${HADOOP_VER}.tar.gz ]; then 47 | # wget http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz 48 | #fi 49 | 50 | sudo tar ixzf /vagrant/dev/provision/tarballs/hadoop-${HADOOP_VER}.tar.gz -C /usr/local 51 | cd /usr/local 52 | rm -rf hadoop 53 | sudo mv -f hadoop-${HADOOP_VER} hadoop 54 | sudo chown -R hduser:hadoop hadoop 55 | 56 | # Init bashrc with hadoop env variables 57 | sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /home/hduser/.bashrc' 58 | sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/hduser/.bashrc' 59 | sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/hduser/.bashrc' 60 | sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/hduser/.bashrc' 61 | sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' 62 | sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' 63 | sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' 64 | sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' 65 | sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >> /home/hduser/.bashrc' 66 | sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >> /home/hduser/.bashrc' 67 | # hit the vagrant user with the same thing 68 | sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /home/vagrant/.bashrc' 69 | sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/vagrant/.bashrc' 70 | sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/vagrant/.bashrc' 71 | sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/vagrant/.bashrc' 72 | sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' 73 | sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' 74 | sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' 75 | sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' 76 | sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >> /home/vagrant/.bashrc' 77 | sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >> /home/vagrant/.bashrc' 78 | 79 | 80 | # Modify JAVA_HOME in hadoop-env 81 | cd /usr/local/hadoop/etc/hadoop 82 | sudo -u hduser sed -i.bak s=\${JAVA_HOME}=//usr/lib/jvm/java-7-openjdk-amd64/=g hadoop-env.sh 83 | pwd 84 | 85 | /usr/local/hadoop/bin/hadoop version 86 | 87 | # Update configuration 88 | #sudo -u hduser sed -i.bak 's==\\fs\.default\.name\\hdfs://localhost:9000\\=g' core-site.xml 89 | sudo -u hduser sed -i.bak 's==\\fs\.default\.name\\hdfs://'"$1"':9000\\=g' core-site.xml 90 | sudo -u hduser sed -i.bak 's==\\yarn\.nodemanager\.aux-services\mapreduce_shuffle\\\yarn.nodemanager.aux-services.mapreduce.shuffle.class\org\.apache\.hadoop\.mapred\.ShuffleHandler\=g' yarn-site.xml 91 | 92 | sudo -u hduser cp mapred-site.xml.template mapred-site.xml 93 | sudo -u hduser sed -i.bak 's==\\mapreduce\.framework\.name\yarn\=g' mapred-site.xml 94 | 95 | cd ~ 96 | sudo -u hduser sh -c 'mkdir -p ~hduser/mydata/hdfs/namenode' 97 | sudo -u hduser sh -c 'mkdir -p ~hduser/mydata/hdfs/datanode' 98 | sudo chown -R hduser:hadoop ~hduser/mydata 99 | 100 | cd /usr/local/hadoop/etc/hadoop 101 | sudo -u hduser sed -i.bak 's==\\dfs\.replication\1\\\\dfs\.namenode\.name\.dir\file:/home/hduser/mydata/hdfs/namenode\\\dfs\.datanode\.data\.dir\file:/home/hduser/mydata/hdfs/datanode\\\dfs\.namenode\.datanode\.registration\.ip-hostname-check\false\=g' hdfs-site.xml 102 | 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/model/Framework.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import aredee.mesos.frameworks.accumulo.configuration.Defaults; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.google.common.base.MoreObjects; 6 | import com.google.common.base.Preconditions; 7 | import com.google.common.base.Strings; 8 | 9 | import java.util.Objects; 10 | 11 | 12 | public class Framework { 13 | 14 | private String bindAddress = Defaults.BIND_ADDRESS; 15 | private Integer httpPort = Defaults.HTTP_PORT; 16 | private String mesosMaster = Defaults.MESOS_MASTER; 17 | private String name; 18 | private String id; 19 | private String tarballUri; 20 | private String dockerImage; 21 | private String zkServers = Defaults.ZK_SERVERS; 22 | private Accumulo cluster = null; 23 | 24 | 25 | /** 26 | * IP address to bind framework webserver to. 27 | **/ 28 | @JsonProperty("bindAddress") 29 | public String getBindAddress() { 30 | return bindAddress; 31 | } 32 | public void setBindAddress(String bindAddress) { 33 | this.bindAddress = bindAddress; 34 | } 35 | 36 | /** 37 | * Port to bind framework webserver to. 38 | **/ 39 | @JsonProperty("httpPort") 40 | public Integer getHttpPort() { 41 | return httpPort; 42 | } 43 | public void setHttpPort(Integer httpPort) { 44 | this.httpPort = httpPort; 45 | } 46 | 47 | /** 48 | * IP and port of Mesos Master node to register with. 49 | **/ 50 | @JsonProperty("mesosMaster") 51 | public String getMesosMaster() { 52 | return mesosMaster; 53 | } 54 | public void setMesosMaster(String mesosMaster) { 55 | this.mesosMaster = mesosMaster; 56 | } 57 | 58 | 59 | /** 60 | * Descriptive name for this framework. 61 | **/ 62 | @JsonProperty("name") 63 | public String getName() { 64 | return name; 65 | } 66 | public void setName(String name) { 67 | this.name = name; 68 | } 69 | public boolean hasName(){ return !Strings.isNullOrEmpty(name); } 70 | 71 | /** 72 | * Unique ID that references this framework 73 | **/ 74 | @JsonProperty("id") 75 | public String getId() { 76 | return id; 77 | } 78 | public void setId(String id) { 79 | this.id = id; 80 | } 81 | public boolean hasId(){ return !Strings.isNullOrEmpty(id); } 82 | 83 | /** 84 | * URI of tarball containing framework distribution 85 | **/ 86 | @JsonProperty("tarballUri") 87 | public String getTarballUri() { 88 | return tarballUri; 89 | } 90 | public void setTarballUri(String tarballUri) { 91 | this.tarballUri = tarballUri; 92 | } 93 | public boolean hasTarballUri(){ return !Strings.isNullOrEmpty(tarballUri); } 94 | 95 | /** 96 | * Name of docker container to use for deployment 97 | * 98 | * @return 99 | */ 100 | @JsonProperty("dockerInfo") 101 | public String getDockerImage() { return dockerImage; } 102 | public void setDockerImage(String dockerImage){ this.dockerImage = dockerImage; } 103 | public boolean hasDockerImage() { return !Strings.isNullOrEmpty(dockerImage); } 104 | 105 | /** 106 | * List of Zookeeper servers to store Framework state on. Does not have\nto be the same ZK servers used for the Accumulo cluster 107 | **/ 108 | @JsonProperty("zkServers") 109 | public String getZkServers() { 110 | return zkServers; 111 | } 112 | public void setZkServers(String zkServers) { 113 | this.zkServers = zkServers; 114 | } 115 | 116 | 117 | /** 118 | * Accumulo cluster definition 119 | **/ 120 | @JsonProperty("cluster") 121 | public Accumulo getCluster() { 122 | return cluster; 123 | } 124 | public void setCluster(Accumulo cluster) { 125 | this.cluster = cluster; 126 | } 127 | public boolean hasCluster(){ 128 | return isNotNullEmpty(cluster); 129 | } 130 | 131 | private static boolean isNotNullEmpty(Object thing){ 132 | boolean hasThing = false; 133 | if( thing != null ){ 134 | hasThing = true; 135 | if( thing instanceof String){ 136 | if( ((String) thing).trim().isEmpty() ){ 137 | hasThing = false; 138 | } 139 | } 140 | } 141 | return hasThing; 142 | } 143 | 144 | public void merge(Framework other){ 145 | if(!this.hasName() && other.hasName()) this.setName(other.getName()); 146 | if(!this.hasId() && other.hasId()) this.setId(other.getId()); 147 | if(!this.hasDockerImage() && other.hasDockerImage()) this.setDockerImage(other.getDockerImage()); 148 | if(!this.hasTarballUri() && other.hasTarballUri()) this.setTarballUri(other.getTarballUri()); 149 | if(!this.hasCluster() && other.hasCluster()) this.setCluster(other.getCluster()); 150 | } 151 | 152 | @Override 153 | public String toString() { 154 | 155 | MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); 156 | helper 157 | .add("bindAddress", bindAddress) 158 | .add("httpPort", httpPort) 159 | .add("dockerImage", this.dockerImage) 160 | .add("id", this.id) 161 | .add("mesosMaster", this.mesosMaster) 162 | .add("name", this.name) 163 | .add("tarballUri", this.tarballUri) 164 | .add("zkServers", this.zkServers); 165 | if( this.hasCluster() ){ 166 | helper.add("cluster", this.cluster.toString()); 167 | } else { 168 | helper.add("cluster", ""); 169 | } 170 | 171 | return helper.toString(); 172 | 173 | } 174 | 175 | @Override 176 | public int hashCode(){ 177 | return Objects.hash(bindAddress, 178 | httpPort, 179 | cluster, 180 | dockerImage, 181 | id, 182 | mesosMaster, 183 | name, 184 | tarballUri, 185 | zkServers); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/resources/webapp/swagger/dist/css/style.css: -------------------------------------------------------------------------------- 1 | .swagger-section #header a#logo { 2 | font-size: 1.5em; 3 | font-weight: bold; 4 | text-decoration: none; 5 | background: transparent url(../images/logo.png) no-repeat left center; 6 | padding: 20px 0 20px 40px; 7 | } 8 | #text-head { 9 | font-size: 80px; 10 | font-family: 'Roboto', sans-serif; 11 | color: #ffffff; 12 | float: right; 13 | margin-right: 20%; 14 | } 15 | .navbar-fixed-top .navbar-nav { 16 | height: auto; 17 | } 18 | .navbar-fixed-top .navbar-brand { 19 | height: auto; 20 | } 21 | .navbar-header { 22 | height: auto; 23 | } 24 | .navbar-inverse { 25 | background-color: #000; 26 | border-color: #000; 27 | } 28 | #navbar-brand { 29 | margin-left: 20%; 30 | } 31 | .navtext { 32 | font-size: 10px; 33 | } 34 | .h1, 35 | h1 { 36 | font-size: 60px; 37 | } 38 | .navbar-default .navbar-header .navbar-brand { 39 | color: #a2dfee; 40 | } 41 | /* tag titles */ 42 | .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a { 43 | color: #393939; 44 | font-family: 'Arvo', serif; 45 | font-size: 1.5em; 46 | } 47 | .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a:hover { 48 | color: black; 49 | } 50 | .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 { 51 | color: #525252; 52 | padding-left: 0px; 53 | display: block; 54 | clear: none; 55 | float: left; 56 | font-family: 'Arvo', serif; 57 | font-weight: bold; 58 | } 59 | .navbar-default .navbar-collapse, 60 | .navbar-default .navbar-form { 61 | border-color: #0A0A0A; 62 | } 63 | .container1 { 64 | width: 1500px; 65 | margin: auto; 66 | margin-top: 0; 67 | background-image: url('../images/shield.png'); 68 | background-repeat: no-repeat; 69 | background-position: -40px -20px; 70 | margin-bottom: 210px; 71 | } 72 | .container-inner { 73 | width: 1200px; 74 | margin: auto; 75 | background-color: rgba(223, 227, 228, 0.75); 76 | padding-bottom: 40px; 77 | padding-top: 40px; 78 | border-radius: 15px; 79 | } 80 | .header-content { 81 | padding: 0; 82 | width: 1000px; 83 | } 84 | .title1 { 85 | font-size: 80px; 86 | font-family: 'Vollkorn', serif; 87 | color: #404040; 88 | text-align: center; 89 | padding-top: 40px; 90 | padding-bottom: 100px; 91 | } 92 | #icon { 93 | margin-top: -18px; 94 | } 95 | .subtext { 96 | font-size: 25px; 97 | font-style: italic; 98 | color: #08b; 99 | text-align: right; 100 | padding-right: 250px; 101 | } 102 | .bg-primary { 103 | background-color: #00468b; 104 | } 105 | .navbar-default .nav > li > a, 106 | .navbar-default .nav > li > a:focus { 107 | color: #08b; 108 | } 109 | .navbar-default .nav > li > a, 110 | .navbar-default .nav > li > a:hover { 111 | color: #08b; 112 | } 113 | .navbar-default .nav > li > a, 114 | .navbar-default .nav > li > a:focus:hover { 115 | color: #08b; 116 | } 117 | .text-faded { 118 | font-size: 25px; 119 | font-family: 'Vollkorn', serif; 120 | } 121 | .section-heading { 122 | font-family: 'Vollkorn', serif; 123 | font-size: 45px; 124 | padding-bottom: 10px; 125 | } 126 | hr { 127 | border-color: #00468b; 128 | padding-bottom: 10px; 129 | } 130 | .description { 131 | margin-top: 20px; 132 | padding-bottom: 200px; 133 | } 134 | .description li { 135 | font-family: 'Vollkorn', serif; 136 | font-size: 25px; 137 | color: #525252; 138 | margin-left: 28%; 139 | padding-top: 5px; 140 | } 141 | .gap { 142 | margin-top: 200px; 143 | } 144 | .troubleshootingtext { 145 | color: rgba(255, 255, 255, 0.7); 146 | padding-left: 30%; 147 | } 148 | .troubleshootingtext li { 149 | list-style-type: circle; 150 | font-size: 25px; 151 | padding-bottom: 5px; 152 | } 153 | .overlay { 154 | position: absolute; 155 | top: 0; 156 | left: 0; 157 | width: 100%; 158 | height: 100%; 159 | z-index: 1000; 160 | } 161 | .block.response_body.json:hover { 162 | cursor: pointer; 163 | } 164 | .backdrop { 165 | color: blue; 166 | } 167 | #myModal { 168 | height: 100%; 169 | } 170 | .modal-backdrop { 171 | bottom: 0; 172 | position: fixed; 173 | } 174 | .curl { 175 | padding: 10px; 176 | font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; 177 | font-size: 0.9em; 178 | max-height: 400px; 179 | margin-top: 5px; 180 | overflow-y: auto; 181 | background-color: #fcf6db; 182 | border: 1px solid #e5e0c6; 183 | border-radius: 4px; 184 | } 185 | .curl_title { 186 | font-size: 1.1em; 187 | margin: 0; 188 | padding: 15px 0 5px; 189 | font-family: 'Open Sans', 'Helvetica Neue', Arial, sans-serif; 190 | font-weight: 500; 191 | line-height: 1.1; 192 | } 193 | .footer { 194 | display: none; 195 | } 196 | .swagger-section .swagger-ui-wrap h2 { 197 | padding: 0; 198 | } 199 | h2 { 200 | margin: 0; 201 | margin-bottom: 5px; 202 | } 203 | .markdown p { 204 | font-size: 15px; 205 | font-family: 'Arvo', serif; 206 | } 207 | .swagger-section .swagger-ui-wrap .code { 208 | font-size: 15px; 209 | font-family: 'Arvo', serif; 210 | } 211 | .swagger-section .swagger-ui-wrap b { 212 | font-family: 'Arvo', serif; 213 | } 214 | #signin:hover { 215 | cursor: pointer; 216 | } 217 | .dropdown-menu { 218 | padding: 15px; 219 | } 220 | .navbar-right .dropdown-menu { 221 | left: 0; 222 | right: auto; 223 | } 224 | #signinbutton { 225 | width: 100%; 226 | height: 32px; 227 | font-size: 13px; 228 | font-weight: bold; 229 | color: #08b; 230 | } 231 | .navbar-default .nav > li .details { 232 | color: #000000; 233 | text-transform: none; 234 | font-size: 15px; 235 | font-weight: normal; 236 | font-family: 'Open Sans', sans-serif; 237 | font-style: italic; 238 | line-height: 20px; 239 | top: -2px; 240 | } 241 | .navbar-default .nav > li .details:hover { 242 | color: black; 243 | } 244 | #signout { 245 | width: 100%; 246 | height: 32px; 247 | font-size: 13px; 248 | font-weight: bold; 249 | color: #08b; 250 | } 251 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/test/java/aredee/mesos/frameworks/accumulo/model/FrameworkTest.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.model; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.commons.io.IOUtils; 5 | import org.junit.Test; 6 | 7 | import java.io.File; 8 | import java.io.FileInputStream; 9 | import java.io.IOException; 10 | import java.net.URL; 11 | import java.util.List; 12 | 13 | import static org.junit.Assert.*; 14 | 15 | public class FrameworkTest { 16 | 17 | private static final ObjectMapper mapper = new ObjectMapper(); 18 | 19 | 20 | 21 | @Test 22 | public void testFrameworkOnly() throws Exception { 23 | byte[] jsonBytes = readJsonResource("/model/FrameworkOnly.json"); 24 | Framework framework = mapper.readValue(jsonBytes, Framework.class); 25 | 26 | assertFalse(framework.hasCluster()); 27 | assertFalse(framework.hasId()); 28 | assertTrue(framework.hasName()); 29 | assertTrue(framework.hasTarballUri()); 30 | 31 | verifyFramework(framework); 32 | } 33 | 34 | @Test 35 | public void testAccumuloOnly() throws Exception { 36 | byte[] jsonBytes = readJsonResource("/model/AccumuloOnly.json"); 37 | Accumulo cluster = mapper.readValue(jsonBytes, Accumulo.class); 38 | 39 | assertFalse(cluster.hasSiteXml()); 40 | 41 | verifyCluster(cluster); 42 | 43 | } 44 | 45 | @Test 46 | public void testFrameworkAndAccumulo() throws Exception { 47 | byte[] jsonBytes = readJsonResource("/model/FrameworkAndAccumulo.json"); 48 | Framework framework = mapper.readValue(jsonBytes, Framework.class); 49 | 50 | assertTrue(framework.hasCluster()); 51 | 52 | verifyFramework(framework); 53 | verifyCluster(framework.getCluster()); 54 | 55 | } 56 | 57 | private void verifyFramework(Framework framework){ 58 | assertEquals("1.1.1.1", framework.getBindAddress()); 59 | assertEquals(new Integer(1234), framework.getHttpPort()); 60 | assertEquals("1.2.3.4:5150", framework.getMesosMaster()); 61 | assertEquals("accumulo-mesos-test", framework.getName()); 62 | assertEquals("hdfs://localhost:9000/data/accumulo-mesos.tar.gz", framework.getTarballUri()); 63 | assertEquals("server1:2181,server2:2181,server3:2181", framework.getZkServers()); 64 | } 65 | 66 | private void verifyCluster(Accumulo cluster){ 67 | assertEquals("testInstance", cluster.getInstance()); 68 | assertEquals("jimbo", cluster.getRootUser()); 69 | assertEquals("jimbopassword", cluster.getRootPassword()); 70 | assertEquals("srvA:2181", cluster.getZkServers()); 71 | assertEquals(new Integer(1024), cluster.getExecutorMemory()); 72 | assertEquals("hdfs://localhost:9000/data/accumulo.tar.gz", cluster.getTarballUri()); 73 | assertEquals("hdfs://localhost:9000/accumulo-mesos", cluster.getHdfsUri()); 74 | assertEquals(4, cluster.getServerGroups().size()); 75 | 76 | verifyServerGroup(getServerGroupByProfileType(cluster.getServerGroups(), "tserver"), 77 | 5, 78 | "BasicTserver", 79 | "Basic Tserver setup", 80 | "tserver", 81 | 8.0, 82 | 8192, 83 | "accumulo" 84 | ); 85 | 86 | verifyServerGroup(getServerGroupByProfileType(cluster.getServerGroups(), "master"), 87 | 1, 88 | "BasicMaster", 89 | "Basic Master setup", 90 | "master", 91 | 2.0, 92 | 2048, 93 | "accumulomaster" 94 | ); 95 | 96 | verifyServerGroup(getServerGroupByProfileType(cluster.getServerGroups(), "gc"), 97 | 1, 98 | "BasicGC", 99 | "Basic Garbage Collector setup", 100 | "gc", 101 | 2.0, 102 | 512, 103 | "accumulogc" 104 | ); 105 | 106 | verifyServerGroup(getServerGroupByProfileType(cluster.getServerGroups(), "monitor"), 107 | 2, 108 | "Monitor", 109 | "Basic Monitor setup", 110 | "monitor", 111 | 1.0, 112 | 512, 113 | "accumulomon" 114 | ); 115 | 116 | } 117 | 118 | private void verifyServerGroup(ServerGroup group, 119 | int count, 120 | String name, 121 | String description, 122 | String type, 123 | double cpus, 124 | int mem, 125 | String user 126 | ){ 127 | 128 | assertEquals(new Integer(count), group.getCount()); 129 | ServerProfile profile = group.getProfile(); 130 | assertEquals(name, profile.getName()); 131 | assertEquals(description, profile.getDescription()); 132 | assertEquals(type, profile.getType().name()); 133 | assertTrue( cpus == profile.getCpus().doubleValue() ); 134 | assertEquals(new Integer(mem), profile.getMemory()); 135 | assertEquals(user, profile.getUser()); 136 | } 137 | 138 | 139 | private ServerGroup getServerGroupByProfileType(List groups, String type){ 140 | for( ServerGroup group : groups ){ 141 | String profileType = group.getProfile().getType().name(); 142 | if( profileType.equals(type)){ 143 | return group; 144 | } 145 | } 146 | return null; 147 | } 148 | 149 | private byte[] readJsonResource(String resource) throws IOException { 150 | URL url = this.getClass().getResource(resource); 151 | File jsonFile = new File(url.getFile()); 152 | byte [] jsonBytes = IOUtils.toByteArray(new FileInputStream(jsonFile)); 153 | return jsonBytes; 154 | } 155 | } -------------------------------------------------------------------------------- /accumulo-mesos-dist/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | aredee.mesos.frameworks.accumulo 7 | accumulo-mesos-parent 8 | ${global.version} 9 | ../pom.xml 10 | 11 | 4.0.0 12 | pom 13 | 14 | accumulo-mesos-dist 15 | 16 | 17 | The Accumulo-on-Mesos framework to deploy Apache Accumulo to Apache Mesos. 18 | This artifact contains a distributable tarball. 19 | 20 | 21 | 22 | 23 | Apache 2 24 | http://www.apache.org/licenses/LICENSE-2.0.txt 25 | repo 26 | Apache License Version 2.0 27 | 28 | 29 | 30 | Apache Mesos accumulo framework 31 | http://mesos.apache.org/ 32 | 33 | 34 | 35 | 36 | ossrh 37 | https://oss.sonatype.org/content/repositories/snapshots 38 | 39 | 40 | ossrh 41 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 42 | 43 | 44 | 45 | scm:git:https://github.com/aredee/accumulo-mesos.git 46 | scm:git:https://github.com/aredee/accumulo-mesos.git 47 | https://github.com/aredee/accumulo-mesos 48 | 49 | 50 | Github 51 | https://github.com/aredee/accumulo-mesos/issues 52 | 53 | 54 | 55 | 56 | aredee.mesos.frameworks.accumulo 57 | accumulo-mesos-executor 58 | jar-with-dependencies 59 | ${project.parent.version} 60 | 61 | 62 | aredee.mesos.frameworks.accumulo 63 | accumulo-mesos-framework 64 | jar-with-dependencies 65 | ${project.parent.version} 66 | 67 | 68 | 69 | 70 | 71 | 72 | maven-dependency-plugin 73 | 74 | 75 | tarball 76 | 77 | copy 78 | 79 | 80 | ${project.build.directory}/tarball 81 | 82 | 83 | aredee.mesos.frameworks.accumulo 84 | accumulo-mesos-framework 85 | jar-with-dependencies 86 | 87 | 88 | aredee.mesos.frameworks.accumulo 89 | accumulo-mesos-executor 90 | jar-with-dependencies 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | maven-assembly-plugin 99 | 100 | 101 | 102 | single 103 | 104 | package 105 | 106 | 107 | src/main/assembly/tarball.xml 108 | 109 | false 110 | false 111 | gnu 112 | 113 | 114 | 115 | 116 | 117 | 118 | maven-source-plugin 119 | 120 | 121 | attach-sources 122 | none 123 | 124 | 125 | 126 | 127 | maven-javadoc-plugin 128 | 129 | 130 | javadoc 131 | none 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | aredee.mesos.frameworks.accumulo 7 | accumulo-mesos-parent 8 | ${global.version} 9 | 10 | 4.0.0 11 | 12 | accumulo-mesos-framework 13 | 14 | 15 | The Accumulo-on-Mesos framework to deploy Apache accumulo to Apache Mesos. 16 | This artifact contains executable framework and scheduler. 17 | 18 | 19 | 20 | 21 | Apache 2 22 | http://www.apache.org/licenses/LICENSE-2.0.txt 23 | repo 24 | Apache License Version 2.0 25 | 26 | 27 | 28 | Apache Mesos Accumulo framework 29 | http://mesos.apache.org/ 30 | 31 | 32 | 33 | 34 | ossrh 35 | https://oss.sonatype.org/content/repositories/snapshots 36 | 37 | 38 | ossrh 39 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 40 | 41 | 42 | 43 | scm:git:https://github.com/aredee/accumulo-mesos.git 44 | scm:git:https://github.com/aredee/accumulo-mesos.git 45 | https://github.com/aredee/accumulo-mesos 46 | 47 | 48 | Github 49 | https://github.com/aredee/accumulo-mesos/issues 50 | 51 | 52 | 53 | 54 | aredee.mesos.frameworks.accumulo 55 | accumulo-mesos-scheduler 56 | ${project.parent.version} 57 | 58 | 59 | aredee.mesos.frameworks.accumulo 60 | accumulo-mesos-common 61 | ${project.parent.version} 62 | 63 | 64 | org.apache.mesos 65 | mesos 66 | 67 | 68 | commons-cli 69 | commons-cli 70 | 71 | 72 | org.eclipse.jetty 73 | jetty-server 74 | 75 | 76 | org.eclipse.jetty 77 | jetty-servlet 78 | 79 | 80 | org.eclipse.jetty 81 | jetty-servlets 82 | 83 | 84 | javax.servlet 85 | javax.servlet-api 86 | 87 | 88 | com.sun.jersey 89 | jersey-server 90 | 91 | 92 | com.sun.jersey 93 | jersey-servlet 94 | 95 | 96 | com.sun.jersey.contribs 97 | jersey-guice 98 | 99 | 100 | com.google.inject 101 | guice 102 | 103 | 104 | com.google.inject.extensions 105 | guice-servlet 106 | 107 | 108 | org.slf4j 109 | slf4j-api 110 | 111 | 112 | org.slf4j 113 | ${slf4j.binding} 114 | 115 | 116 | junit 117 | junit 118 | test 119 | 120 | 121 | 122 | 123 | 124 | 125 | maven-assembly-plugin 126 | 127 | 128 | jar-with-dependencies 129 | 130 | 131 | 132 | aredee.mesos.frameworks.accumulo.framework.Main 133 | 134 | 135 | 136 | 137 | 138 | make-assembly 139 | package 140 | 141 | single 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /accumulo-mesos-framework/src/main/java/aredee/mesos/frameworks/accumulo/framework/api/ClusterApi.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.framework.api; 2 | 3 | import javax.ws.rs.*; 4 | import javax.ws.rs.core.Response; 5 | 6 | @Path("/cluster") 7 | @Consumes({ "application/json" }) 8 | @Produces({ "application/json", "text/html" }) 9 | //@io.swagger.annotations.Api(value = "/cluster", description = "the cluster API") 10 | public class ClusterApi { 11 | 12 | private final ClusterApiService delegate = ApiServiceFactory.getClusterApi(); 13 | 14 | @POST 15 | @Path("/kill") 16 | /* 17 | @io.swagger.annotations.ApiOperation(value = "", notes = "Kill all running processes using Mesos\n", response = Void.class) 18 | @io.swagger.annotations.ApiResponses(value = { 19 | @io.swagger.annotations.ApiResponse(code = 202, message = "Kill accepted"), 20 | 21 | @io.swagger.annotations.ApiResponse(code = 409, message = "Cluster not running") }) 22 | 23 | */ 24 | public Response clusterKillPost() 25 | throws NotFoundException { 26 | return delegate.clusterKillPost(); 27 | } 28 | 29 | @POST 30 | @Path("/master/reprovision") 31 | /* 32 | @io.swagger.annotations.ApiOperation(value = "", notes = "Restarts Accumulo master server using a new Mesos Resource\n", response = Void.class) 33 | @io.swagger.annotations.ApiResponses(value = { 34 | @io.swagger.annotations.ApiResponse(code = 202, message = "Reprovision accepted"), 35 | 36 | @io.swagger.annotations.ApiResponse(code = 409, message = "Master not running") }) 37 | */ 38 | public Response clusterMasterReprovisionPost() 39 | throws NotFoundException { 40 | return delegate.clusterMasterReprovisionPost(); 41 | } 42 | 43 | @POST 44 | @Path("/master/restart") 45 | 46 | /* 47 | @io.swagger.annotations.ApiOperation(value = "", notes = "Restarts Accumulo master server\n", response = Void.class) 48 | @io.swagger.annotations.ApiResponses(value = { 49 | @io.swagger.annotations.ApiResponse(code = 202, message = "Restart accepted"), 50 | 51 | @io.swagger.annotations.ApiResponse(code = 409, message = "Master not running") }) 52 | */ 53 | public Response clusterMasterRestartPost() 54 | throws NotFoundException { 55 | return delegate.clusterMasterRestartPost(); 56 | } 57 | 58 | @GET 59 | @Path("/monitor") 60 | /* 61 | @io.swagger.annotations.ApiOperation(value = "", notes = "Get location(s) of Accumulo Monitor server\n", response = Monitor.class) 62 | @io.swagger.annotations.ApiResponses(value = { 63 | @io.swagger.annotations.ApiResponse(code = 200, message = "Monitor found") }) 64 | */ 65 | public Response clusterMonitorGet() 66 | throws NotFoundException { 67 | return delegate.clusterMonitorGet(); 68 | } 69 | 70 | @POST 71 | @Path("/start") 72 | /* 73 | @io.swagger.annotations.ApiOperation(value = "", notes = "Start configured Accumulo cluster\n", response = Void.class) 74 | @io.swagger.annotations.ApiResponses(value = { 75 | @io.swagger.annotations.ApiResponse(code = 202, message = "Start accepted"), 76 | 77 | @io.swagger.annotations.ApiResponse(code = 409, message = "Cluster already running") }) 78 | */ 79 | public Response clusterStartPost() 80 | throws NotFoundException { 81 | return delegate.clusterStartPost(); 82 | } 83 | 84 | @POST 85 | @Path("/stop") 86 | /* 87 | @io.swagger.annotations.ApiOperation(value = "", notes = "Stop running Accumulo cluster using Accumulo\n", response = Void.class) 88 | @io.swagger.annotations.ApiResponses(value = { 89 | @io.swagger.annotations.ApiResponse(code = 202, message = "Stop accepted"), 90 | 91 | @io.swagger.annotations.ApiResponse(code = 409, message = "Cluster not running") }) 92 | */ 93 | public Response clusterStopPost() 94 | throws NotFoundException { 95 | return delegate.clusterStopPost(); 96 | } 97 | 98 | @POST 99 | @Path("/tserver/reprovision") 100 | /* 101 | @io.swagger.annotations.ApiOperation(value = "", notes = "Reprovisions an Accumulo Tserver using a new Mesos Resource\n", response = Void.class) 102 | @io.swagger.annotations.ApiResponses(value = { 103 | @io.swagger.annotations.ApiResponse(code = 202, message = "Reprovision accepted"), 104 | 105 | @io.swagger.annotations.ApiResponse(code = 404, message = "tserver does not exist"), 106 | 107 | @io.swagger.annotations.ApiResponse(code = 409, message = "tserver exists but is not running") }) 108 | 109 | @ApiParam(value = "ID of the tserver",required=true) 110 | */ 111 | public Response clusterTserverReprovisionPost( @QueryParam("id") String id) 112 | throws NotFoundException { 113 | return delegate.clusterTserverReprovisionPost(id); 114 | } 115 | 116 | @POST 117 | @Path("/tserver/restart") 118 | /* 119 | @io.swagger.annotations.ApiOperation(value = "", notes = "Restarts an Accumulo Tserver\n", response = Void.class) 120 | @io.swagger.annotations.ApiResponses(value = { 121 | @io.swagger.annotations.ApiResponse(code = 202, message = "Reprovision accepted"), 122 | 123 | @io.swagger.annotations.ApiResponse(code = 404, message = "tserver does not exist"), 124 | 125 | @io.swagger.annotations.ApiResponse(code = 409, message = "tserver exists but is not running") }) 126 | 127 | @ApiParam(value = "ID of the tserver",required=true) 128 | */ 129 | public Response clusterTserverRestartPost(@QueryParam("id") String id) 130 | throws NotFoundException { 131 | return delegate.clusterTserverRestartPost(id); 132 | } 133 | 134 | @POST 135 | @Path("/tserver/rollingrestart") 136 | /* 137 | @io.swagger.annotations.ApiOperation(value = "", notes = "Perform a rolling restart of all cluster servers.\n", response = Void.class) 138 | @io.swagger.annotations.ApiResponses(value = { 139 | @io.swagger.annotations.ApiResponse(code = 202, message = "Rolling restart accepted") }) 140 | 141 | @ApiParam(value = "Include master server in restart", defaultValue="false") 142 | @ApiParam(value = "Number of servers to restart at once.", allowableValues="{}", defaultValue="1") 143 | @ApiParam(value = "Reprovision servers on new Mesos Resources", defaultValue="false") 144 | 145 | */ 146 | public Response clusterTserverRollingrestartPost( @QueryParam("master") Boolean master, 147 | @QueryParam("group") Integer group, 148 | @QueryParam("reprovision") Boolean reprovision) 149 | throws NotFoundException { 150 | return delegate.clusterTserverRollingrestartPost(master,group,reprovision); 151 | } 152 | } 153 | 154 | -------------------------------------------------------------------------------- /accumulo-mesos-common/src/main/java/aredee/mesos/frameworks/accumulo/configuration/CommandLineHandler.java: -------------------------------------------------------------------------------- 1 | package aredee.mesos.frameworks.accumulo.configuration; 2 | 3 | import aredee.mesos.frameworks.accumulo.model.Accumulo; 4 | import aredee.mesos.frameworks.accumulo.model.Framework; 5 | import com.fasterxml.jackson.core.JsonParseException; 6 | import org.apache.commons.cli.CommandLine; 7 | import org.apache.commons.cli.GnuParser; 8 | import org.apache.commons.cli.HelpFormatter; 9 | import org.apache.commons.cli.Options; 10 | import org.apache.commons.cli.ParseException; 11 | import org.apache.commons.cli.Parser; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import com.fasterxml.jackson.databind.ObjectMapper; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | 19 | 20 | public class CommandLineHandler { 21 | private static final Logger LOGGER = LoggerFactory.getLogger(CommandLineHandler.class); 22 | 23 | private static final String OPTION_HELP = "h"; 24 | private static final String OPTION_VERSION = "v"; 25 | private static final String OPTION_PORT = "P"; 26 | private static final String OPTION_BIND = "b"; 27 | private static final String OPTION_MASTER = "m"; 28 | private static final String OPTION_NAME = "n"; 29 | private static final String OPTION_ZOOKEEPERS = "z"; 30 | private static final String OPTION_TARBALL = "t"; 31 | private static final String OPTION_FRAMEWORK = "fc"; 32 | private static final String OPTION_CLUSTER = "cc"; 33 | private static final String OPTION_INITIALIZE = "i"; 34 | 35 | private static final Options options; 36 | static { 37 | options = new Options(); 38 | options.addOption(OPTION_HELP, "help", false, "Print this message and exit"); 39 | options.addOption(OPTION_VERSION, "version", false, "Show version number"); 40 | options.addOption(OPTION_PORT, "port", true, "Port number to serve HTTP interface"); 41 | options.addOption(OPTION_BIND, "bind-address", true, "IP address of interface to bind HTTP interface to"); 42 | options.addOption(OPTION_MASTER, "master", true, "Location of mesos master to connect to"); 43 | options.addOption(OPTION_NAME, "name", true, "Name of this mesos framework"); 44 | options.addOption(OPTION_ZOOKEEPERS, "zookeepers", true, "List of Zookeeper servers"); 45 | options.addOption(OPTION_TARBALL, "tarball", true, "URI of framework tarball"); 46 | options.addOption(OPTION_FRAMEWORK, "framework", true, "JSON file of entire framework configuration"); 47 | options.addOption(OPTION_CLUSTER, "cluster", true, "JSON file containing cluster configuration"); 48 | options.addOption(OPTION_INITIALIZE, "init", false, "If present, initialize new Accumulo instance"); 49 | } 50 | 51 | private static final Parser parser = new GnuParser(); 52 | private final CommandLine cmdLine; 53 | 54 | public CommandLineHandler(CommandLine cmdLine){ 55 | this.cmdLine = cmdLine; 56 | } 57 | 58 | public CommandLineHandler(String args[]){ 59 | this(CommandLineHandler.parseArgs(args)); 60 | } 61 | 62 | /** 63 | * Parses command line arguments for configuration information. 64 | * 65 | * Explicit options override anything in JSON. For example if JSON specifies 66 | * port as 8080 and commandline has -P 5150, the port will be 5150. The returned 67 | * object might not contain enough information to initialize and run a cluster. 68 | * This is because a recovery re-launch of the framework should just be able to 69 | * read the configuration from the Mesos store. 70 | * 71 | * @return Framework configuration 72 | * 73 | */ 74 | public Framework getFrameworkDefinition() throws IOException { 75 | 76 | ObjectMapper mapper = new ObjectMapper(); 77 | 78 | Framework config = null; 79 | // Read full config if available 80 | if( cmdLine.hasOption(OPTION_FRAMEWORK)){ 81 | File frameworkFile = new File(cmdLine.getOptionValue(OPTION_FRAMEWORK)); 82 | LOGGER.info("Reading Framework Configuration from: {}", frameworkFile.getAbsolutePath()); 83 | config = mapper.readValue(frameworkFile, Framework.class); 84 | } else { 85 | config = new Framework(); 86 | } 87 | 88 | // Override config with explicit command line options. 89 | if( cmdLine.hasOption(OPTION_PORT) ){ 90 | config.setHttpPort(Integer.parseInt(cmdLine.getOptionValue(OPTION_PORT))); 91 | } 92 | 93 | if( cmdLine.hasOption(OPTION_BIND) ){ 94 | config.setBindAddress(cmdLine.getOptionValue(OPTION_BIND)); 95 | } 96 | 97 | if ( cmdLine.hasOption(OPTION_MASTER) ){ 98 | config.setMesosMaster(cmdLine.getOptionValue(OPTION_MASTER)); 99 | } 100 | 101 | if ( cmdLine.hasOption(OPTION_NAME) ){ 102 | config.setName(cmdLine.getOptionValue(OPTION_NAME)); 103 | } 104 | 105 | if (cmdLine.hasOption(OPTION_ZOOKEEPERS) ){ 106 | config.setZkServers(cmdLine.getOptionValue(OPTION_ZOOKEEPERS)); 107 | } 108 | 109 | if (cmdLine.hasOption(OPTION_TARBALL)){ 110 | config.setTarballUri(cmdLine.getOptionValue(OPTION_TARBALL)); 111 | } 112 | 113 | // If there's an Accumulo cluster JSON provided, override. 114 | if (cmdLine.hasOption(OPTION_CLUSTER)) { 115 | File clusterFile = new File(cmdLine.getOptionValue(OPTION_CLUSTER)); 116 | LOGGER.info("Loading cluster configuration {}", clusterFile.getAbsolutePath()); 117 | Accumulo clusterConfig = mapper.readValue(clusterFile, Accumulo.class); 118 | config.setCluster(clusterConfig); 119 | } 120 | 121 | LOGGER.debug("Configuration after command line handling: " + config.toString()); 122 | 123 | return config; 124 | } 125 | 126 | public boolean isInitializeRequest(){ return cmdLine.hasOption(OPTION_INITIALIZE); } 127 | 128 | private static CommandLine parseArgs(String args[]) { 129 | CommandLine cmdLine = null; 130 | try { 131 | cmdLine = parser.parse(options, args); 132 | } catch (ParseException e) { 133 | System.err.println(e); 134 | } 135 | return cmdLine; 136 | } 137 | 138 | // print help or version information. 139 | public boolean checkHelpOrVersion() { 140 | 141 | // Handle exiting options 142 | if (cmdLine.hasOption('h')){ 143 | printHelp(); 144 | return true; 145 | } 146 | if (cmdLine.hasOption('v')){ 147 | printVersion(); 148 | return true; 149 | } 150 | return false; 151 | } 152 | 153 | private void printVersion(){ 154 | System.out.format("%s : %s\n", Constants.EXE_NAME, Constants.FRAMEWORK_VERSION); 155 | } 156 | 157 | private void printHelp(){ 158 | HelpFormatter formatter = new HelpFormatter(); 159 | formatter.printHelp( Constants.EXE_NAME, options, true ); 160 | } 161 | 162 | } 163 | --------------------------------------------------------------------------------