├── .gitignore ├── Dockerfile ├── README.md ├── ansible ├── ansible.cfg ├── bootstrap-cluster.yml ├── install-agents.yml ├── install-masters.yml ├── install-mesos-dns.yml ├── install-registry.yml ├── inventory │ └── dynamic_inventory ├── provision.yml ├── restart-agent-services.yml ├── restart-master-services.yml ├── restart-services.yml ├── roles │ ├── agent │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ ├── mesos-master.override │ │ │ └── zookeeper.override │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── containerizers.j2 │ │ │ ├── executor_registration_timeout.j2 │ │ │ ├── hostname.j2 │ │ │ ├── ip.j2 │ │ │ ├── mesos-slave.service.j2 │ │ │ └── zk.j2 │ │ └── vars │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ ├── common │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── CoreOS │ │ │ │ ├── 30-increase-ulimit.conf.j2 │ │ │ │ ├── 50-insecure-registry.conf.j2 │ │ │ │ └── hosts.j2 │ │ │ └── Debian │ │ │ │ └── hosts.j2 │ │ └── vars │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ └── RedHat.yml │ ├── coreos-bootstrap │ │ ├── .editorconfig │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── files │ │ │ ├── bootstrap.sh │ │ │ ├── get-pip.py │ │ │ └── runner │ │ ├── meta │ │ │ ├── .galaxy_install_info │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── tests │ │ │ └── test.yml │ ├── docker │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ ├── .galaxy_install_info │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ ├── init.yml │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── docker-defaults.j2 │ │ │ ├── docker-init.j2 │ │ │ ├── docker-repo.j2 │ │ │ ├── docker-service.j2 │ │ │ ├── docker-sysconfig.j2 │ │ │ └── docker.socket.j2 │ │ └── vars │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ ├── master │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ └── mesos-slave.override │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── marathon │ │ │ │ ├── hostname.j2 │ │ │ │ ├── marathon.service.j2 │ │ │ │ └── marathon_env.j2 │ │ │ ├── master │ │ │ │ ├── hostname.j2 │ │ │ │ ├── ip.j2 │ │ │ │ ├── mesos-master.service.j2 │ │ │ │ ├── quorum.j2 │ │ │ │ └── zk.j2 │ │ │ └── zookeeper │ │ │ │ ├── myid.j2 │ │ │ │ ├── zoo.cfg.j2 │ │ │ │ └── zookeeper.service.j2 │ │ └── vars │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ ├── mesos-dns │ │ ├── tasks │ │ │ ├── CoreOS.yml │ │ │ ├── Debian.yml │ │ │ ├── RedHat.yml │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── mesosdns.json.j2 │ │ │ └── resolv.conf.j2 │ │ └── vars │ │ │ └── main.yml │ └── registry │ │ ├── defaults │ │ └── main.yml │ │ ├── tasks │ │ ├── CoreOS.yml │ │ ├── Debian.yml │ │ ├── RedHat.yml │ │ └── main.yml │ │ ├── templates │ │ ├── pull_and_push.sh.j2 │ │ └── registry.service.j2 │ │ └── vars │ │ ├── CoreOS.yml │ │ ├── Debian.yml │ │ ├── RedHat.yml │ │ └── main.yml └── ssh.config ├── bin └── mesosctl.js ├── config └── docker.yml ├── lib ├── mesosCtl.js └── schema │ ├── marathon_app.json │ ├── marathon_deployment.json │ ├── marathon_group.json │ └── mesosctl_config.json ├── modules ├── cluster.js ├── config.js ├── installed │ └── .gitkeep ├── marathon.js ├── package.js ├── repository.js └── task.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Node.js modules 2 | node_modules/ 3 | 4 | # Internal tests 5 | test/ 6 | 7 | # Temporary files 8 | temp/ 9 | 10 | # Ansible retry files 11 | *.retry 12 | 13 | # WebStorm 14 | .idea/ 15 | 16 | .DS_Store 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM williamyeh/ansible:alpine3 2 | 3 | MAINTAINER tobilg@gmail.com 4 | 5 | # Node.js installation taken from https://github.com/mhart/alpine-node/blob/master/Dockerfile 6 | ENV VERSION=v4.5.0 NPM_VERSION=2 7 | 8 | ENV CONFIG_FLAGS="--fully-static" DEL_PKGS="libgcc libstdc++" RM_DIRS=/usr/include 9 | 10 | RUN apk add --no-cache curl make gcc g++ python linux-headers paxctl libgcc libstdc++ gnupg && \ 11 | gpg --keyserver pool.sks-keyservers.net --recv-keys 9554F04D7259F04124DE6B476D5A82AC7E37093B && \ 12 | gpg --keyserver pool.sks-keyservers.net --recv-keys 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 && \ 13 | gpg --keyserver pool.sks-keyservers.net --recv-keys 0034A06D9D9B0064CE8ADF6BF1747F4AD2306D93 && \ 14 | gpg --keyserver pool.sks-keyservers.net --recv-keys FD3A5288F042B6850C66B31F09FE44734EB7990E && \ 15 | gpg --keyserver pool.sks-keyservers.net --recv-keys 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 && \ 16 | gpg --keyserver pool.sks-keyservers.net --recv-keys DD8F2338BAE7501E3DD5AC78C273792F7D83545D && \ 17 | gpg --keyserver pool.sks-keyservers.net --recv-keys C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 && \ 18 | gpg --keyserver pool.sks-keyservers.net --recv-keys B9AE9905FFD7803F25714661B63B535A4C206CA9 && \ 19 | curl -o node-${VERSION}.tar.gz -sSL https://nodejs.org/dist/${VERSION}/node-${VERSION}.tar.gz && \ 20 | curl -o SHASUMS256.txt.asc -sSL https://nodejs.org/dist/${VERSION}/SHASUMS256.txt.asc && \ 21 | gpg --verify SHASUMS256.txt.asc && \ 22 | grep node-${VERSION}.tar.gz SHASUMS256.txt.asc | sha256sum -c - && \ 23 | tar -zxf node-${VERSION}.tar.gz && \ 24 | cd node-${VERSION} && \ 25 | export GYP_DEFINES="linux_use_gold_flags=0" && \ 26 | ./configure --prefix=/usr ${CONFIG_FLAGS} && \ 27 | NPROC=$(grep -c ^processor /proc/cpuinfo 2>/dev/null || 1) && \ 28 | make -j${NPROC} -C out mksnapshot BUILDTYPE=Release && \ 29 | paxctl -cm out/Release/mksnapshot && \ 30 | make -j${NPROC} && \ 31 | make install && \ 32 | paxctl -cm /usr/bin/node && \ 33 | cd / && \ 34 | if [ -x /usr/bin/npm ]; then \ 35 | npm install -g npm@${NPM_VERSION} && \ 36 | find /usr/lib/node_modules/npm -name test -o -name .bin -type d | xargs rm -rf; \ 37 | fi && \ 38 | apk del curl make gcc g++ python linux-headers paxctl gnupg ${DEL_PKGS} && \ 39 | rm -rf /etc/ssl /node-${VERSION}.tar.gz /SHASUMS256.txt.asc /node-${VERSION} ${RM_DIRS} \ 40 | /usr/share/man /tmp/* /var/cache/apk/* /root/.npm /root/.node-gyp /root/.gnupg \ 41 | /usr/lib/node_modules/npm/man /usr/lib/node_modules/npm/doc /usr/lib/node_modules/npm/html 42 | 43 | RUN npm install -g mesosctl@0.1.10 44 | 45 | CMD [ "mesosctl", "--version" ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mesosctl 2 | 3 | [![Package version](https://img.shields.io/npm/v/mesosctl.svg)](https://www.npmjs.com/package/mesosctl) [![Package downloads](https://img.shields.io/npm/dt/mesosctl.svg)](https://www.npmjs.com/package/mesosctl) [![Package license](https://img.shields.io/npm/l/mesosctl.svg)](https://www.npmjs.com/package/mesosctl) [![Run on Katacoda](https://img.shields.io/badge/Run-on%20Katacoda-00b7ef.svg)](https://www.katacoda.com/mesoshq/scenarios/1) [![Slack Status](https://mesosctl.herokuapp.com/badge.svg)](https://mesosctl.herokuapp.com) 4 | 5 | A command-line tool to dynamically provision and manage Mesos clusters and their applications. Try it live with [Katacoda](https://www.katacoda.com/mesoshq/scenarios/1)! 6 | 7 | ## Motivation 8 | 9 | When `mesosctl` was started as a project, Mesosphere's [DC/OS](http://www.dcos.io) was only available for local installations for paying customers. 10 | So, with the creation of `mesosctl` the goal was to provide a simple and reliable way of creating and running Mesos clusters on own infrastructure. 11 | In the meantime, DC/OS has been open-sourced, but there assumingly is room for another, although maybe simpler (both in installation, usage and functionality) tool besides DC/OS. 12 | That's why we decided to finish and open-source `mesosctl` as well. 13 | 14 | You might think of `mesosctl` as a mixture of [DC/OS](http://www.dcos.io) and [minimesos](https://github.com/ContainerSolutions/minimesos), providing not as much sophistication as DC/OS, but is also able to run universe packages. 15 | 16 | ## Concepts 17 | 18 | `mesosctl` is running a "sub-shell" application (written as a Node.js module using [vorpal.js](https://github.com/dthree/vorpal)), meaning it starts a process which keeps the state of the configurations in memory. Unless you `exit` the `mesosctl` shell, you can directly interact with you cluster. 19 | 20 | It uses [Ansible](http://docs.ansible.com/ansible/intro_installation.html) to run the OS-individal tasks to set up a Mesos cluster. This is based on configuration files which are stored as YAML (see [Usage](#usage)). 21 | 22 | After the cluster has been set up, `mesosctl` uses the APIs of the Mesos Masters/Agents, as well as the Marathon API to provide the interactive functionality. 23 | 24 | ## Status 25 | 26 | The current version is `0.1.10`, which is an early release. We consider it as `alpha` and a developer preview. 27 | 28 | ## Installation 29 | 30 | The preferred ways to run `mesosctl` is either via local NPM installation, or via the [mesoshq/mesosctl](https://hub.docker.com/r/mesoshq/mesosctl) Docker image. Currently, we only 31 | provide a command-line installation. A GUI installer will follow in the future. 32 | 33 | ### mesosctl CLI installation 34 | 35 | **NPM** 36 | 37 | You can install the NPM package globally by running 38 | 39 | npm install mesosctl -g 40 | 41 | When installing via the NPM package, `mesosctl` expects the following tools to by present on your system: 42 | 43 | * [Node.js](https://nodejs.org/en/download/) >= 4 and [NPM](https://www.npmjs.com/) 44 | * [Ansible](http://docs.ansible.com/ansible/intro_installation.html) >= 2.1.1.0 45 | 46 | **Docker** 47 | 48 | You can start the Docker image by running 49 | 50 | docker run --net=host -it mesoshq/mesosctl mesosctl 51 | 52 | If you want to use pre-existing local configurations, please map the relevant folder as a volume like this: 53 | 54 | -v /path/to/local/config:/opt/mesosctl/config 55 | 56 | Also, make sure that you host's network settings are configured in a way that the Docker daemon networking can "see" the Vagrant networks. 57 | 58 | You can also use a Bash wrapper script, which you should place somewhere in the `PATH`, e.g. save the script somewhere and create a symlink to `/usr/local/bin`: 59 | 60 | ```bash 61 | #!/bin/bash 62 | 63 | # Create .mesosctl folder in user's home directory is it doesn't exist 64 | mkdir -p ~/.mesosctl 65 | 66 | # Run Docker image and map the local configuration folder 67 | docker run --net=host -it -e MESOSCTL_CONFIGURATION_BASE_PATH=/config -v ~/.mesosctl:/config:rw mesoshq/mesosctl mesosctl 68 | 69 | ``` 70 | 71 | ### Local / Vagrant cluster installation 72 | 73 | For testing and local development, one of the provided Vagrant cluster configurations can be used: 74 | 75 | * [mesoshq/vagrant-cluster-coreos](https://github.com/mesoshq/vagrant-cluster-coreos) (using CoreOS stable latest) 76 | * [mesoshq/vagrant-cluster-ubuntu](https://github.com/mesoshq/vagrant-cluster-ubuntu) (using Ubuntu 14.04 latest) 77 | * [mesoshq/vagrant-cluster-centos](https://github.com/mesoshq/vagrant-cluster-centos) (using CentOS 7 latest) 78 | 79 | Obviously, you'll need a working [Vagrant](https://www.vagrantup.com/downloads.html) installation (>= v1.8, with VirtualBox). 80 | 81 | ### Remote installation 82 | 83 | You can use `mesosctl` to install Mesos clusters on freshly installed Linux machines. It currently supports the following OS families/flavors: 84 | 85 | * CoreOS 86 | * Debian 87 | * Debian Jessie 88 | * Ubuntu Xenial 89 | * Ubuntu Vivid 90 | * Ubuntu Trusty 91 | * RedHat 92 | * CentOS 7 93 | * CentOS 6 94 | * RHEL 7 95 | * RHEL 6 96 | * Fedora 97 | 98 | The requirements on the remote machines which should be utilized by `mesosctl` are the following: 99 | 100 | * A common user with sufficient permission to install packages and run services, as well as the belonging SSH key of this user to be able to connect via SSH. This SSH key needs to be present on the machine you're running `mesosctl` on. 101 | * A Python version >= 2.4 (but if you are running less than Python 2.5 on the remotes, you will also need `python-simplejson`) 102 | * Network connectivity to see the other hosts of the cluster 103 | * Internet access (installer packages and Docker images are downloaded during installation) 104 | 105 | The remote machines need to be reachable from the host you're running `mesosctl` on, otherwise the installation will fail. 106 | 107 | #### Using cloud providers 108 | 109 | Currently, either the direct access via SSH is supported, or via a preconfigured and running "SSH over VPN" connection. 110 | 111 | See the following links on how to configure SSH access for cloud-hosted nodes: 112 | 113 | * [AWS](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstancesLinux.html) 114 | * [GCE](https://cloud.google.com/compute/docs/instances/connecting-to-instance) 115 | * [DigitalOcean](https://www.digitalocean.com/community/tutorials/how-to-use-ssh-keys-with-digitalocean-droplets) 116 | * [Azure](https://azure.microsoft.com/de-de/documentation/articles/virtual-machines-linux-ssh-from-linux/) 117 | 118 | ## Usage 119 | 120 | Once you have installed `mesosctl` either as global NPM module or as Docker container (having the wrapper script in the path), you can use 121 | 122 | ```bash 123 | $ mesosctl 124 | ``` 125 | 126 | to open the `mesosctl` sub-shell application. 127 | 128 | ### Create a configuration file 129 | 130 | Configuration files for `mesosctl` are YAML files. To be able to provision a Mesos cluster, you'll need to specify the following details: 131 | 132 | * `cluster_name`: The desired cluster name. Must be a `string` containg no spaces. 133 | * `os_family`: The OS family. Must be one of `CoreOS`, `Debian` or `RedHat` (see [Remote installation](#remote-installation)). 134 | * `ssh_key_path`: The (absolute) path to the SSH key which provides access to all hosts. This must be the same for all hosts. 135 | * `ssh_user`: The username belonging to the SSH key specified above. This must be the same for all hosts. 136 | * `ssh_port`: The SSH port to access all hosts. This must be the same for all hosts. 137 | * `dns_servers`: The list of external DNS resolvers. 138 | * `masters`: The list of IP addresses which should be provisioned as Mesos masters. 139 | * `agents`: The list of IP addresses which should be provisioned as Mesos agents. 140 | * `registry`: The IP address which should be provisioned as private Docker registry. 141 | 142 | An example configuration file is the following: 143 | 144 | ```bash 145 | cluster_name: my_first_mesos_cluster 146 | os_family: Debian 147 | ssh_key_path: /Users/developer/.vagrant.d/insecure_private_key 148 | ssh_port: 22 149 | ssh_user: vagrant 150 | dns_servers: 151 | - 8.8.8.8 152 | - 8.8.4.4 153 | masters: 154 | - 172.17.10.101 155 | - 172.17.10.102 156 | - 172.17.10.103 157 | agents: 158 | - 172.17.10.101 159 | - 172.17.10.102 160 | - 172.17.10.103 161 | registry: 162 | - 172.17.10.101 163 | ``` 164 | 165 | 166 | ### Using pre-existing configurations 167 | 168 | If you have a pre-existing configuration file (e.g. when you use a Vagrant cluster setup), you can use the following command to load it: 169 | 170 | ```bash 171 | mesosctl $ config load /path/to/mesosctl.yml 172 | ``` 173 | 174 | ### Create a configuration via `mesosctl` 175 | 176 | Have a look at the [config set](#configuration) commands to create the cluster configuration manually. The list of [mandatory fields](#create-a-configuration-file) applies. 177 | 178 | ### Cluster provisioning 179 | 180 | Once you have a configuation, you can run 181 | 182 | ```bash 183 | mesosctl $ config validate 184 | --> The current configuration is valid! 185 | ``` 186 | 187 | If the configuration is valid, then you can start with the provisioning of the cluster: 188 | 189 | ```bash 190 | mesosctl $ cluster provision --verbose 191 | ``` 192 | 193 | This may take a while, because `mesosctl` will trigger the installation of all required software on the configured cluster hosts. If you used one of the Vagrant sample clusters (see above), once the provisioning of the cluster is finished, you'll have a 3 node cluster, where each node is running the Mesos Master, the Mesos Agent and a Marathon instance. The respective IP addresses can also be found in the Vagrant project's docs. 194 | 195 | You can check the cluster status via `mesosctl` like this: 196 | 197 | ```bash 198 | mesosctl $ cluster status 199 | 200 | Cluster 'mesos_cluster_ubuntu' utilization: 201 | 202 | CPU [== ] 10% 203 | Memory [= ] 6% 204 | Disk [ ] 0% 205 | Ports [ ] 0% 206 | ``` 207 | 208 | You can also retrieve the current leader's address, which you can use in a browser to get the Mesos Master's UI: 209 | 210 | ```bash 211 | mesosctl $ cluster get leader 212 | --> Current leading Master's address is 172.17.10.103:5050 213 | ``` 214 | 215 | ## Command reference 216 | 217 | You can use the following commands with `mesosctl`: 218 | 219 | ### Configuration 220 | 221 | You can create, update and load existing configurations. 222 | 223 | ``` 224 | config create Creates a configuration 225 | config load [pathToConfig] Loads an existing configuration, either from specified path or from a selection of existing configurations 226 | config show Displays the current configuration 227 | config validate Validates the current configuration 228 | config get clustername Gets the cluster name 229 | config set clustername Defines the cluster name 230 | config set os Defines the OS type 231 | config get os Gets the OS type 232 | config set ssh.keypath Defines the path to the SSH key for accessing the hosts 233 | config get ssh.keypath Gets the path to the SSH key for accessing the hosts 234 | config set ssh.user Defines the user name for the SSH key for accessing the hosts 235 | config get ssh.user Gets the user name for the SSH key for accessing the hosts 236 | config set ssh.port Defines the port for the SSH connection for accessing the hosts 237 | config get ssh.port Gets the port for the SSH connection for accessing the hosts 238 | config set admin.user Defines the admin user name 239 | config get admin.user Gets the admin user name 240 | config set admin.password Defines the admin password 241 | config set dns.servers [dnsServer...] Defines the DNS nameservers 242 | config add dns.servers [dnsServer...] Adds IP address(es) to the DNS nameserver list 243 | config remove dns.servers [dnsServer...] Remove IP address(es) from the DNS nameserver list 244 | config set masters [masterServer...] Defines the Mesos Master servers 245 | config add masters [masterServer...] Adds IP address(es) to the Mesos Master server list 246 | config remove masters [masterServer...] Remove IP address(es) from the Mesos Master servers list 247 | config set agents [agentServer...] Defines the Mesos Agent servers 248 | config add agents [agentServer...] Adds IP address(es) to the Mesos Agent server list 249 | config remove agents [agentServer...] Remove IP address(es) from the Mesos Agent servers list 250 | config set registry Defines the private Docker Registry server 251 | config get registry Gets the private Docker Registry server IP address 252 | ``` 253 | 254 | ### Cluster 255 | 256 | You can provison a cluster based on the before defined configuration, and get status information as well as connect to nodes via SSH. 257 | 258 | ``` 259 | cluster provision [options] Provisions the cluster based on the current configuration 260 | cluster status Display the cluster status 261 | cluster status agent Display the Mesos agent status and utilization 262 | cluster ssh [command] Issue a SSH command on the remote host. The command must be wrapped in double quotation marks, like `"ls -la"`. 263 | cluster get leader Returns the currently leading Mesos Master's IP address 264 | cluster restart services Restarts the ZooKeeper, Master, Agent and Marathon services on both Masters and Agents 265 | ``` 266 | 267 | ### Repository 268 | 269 | You can use the standard DC/OS (Mesosphere) universe repository. 270 | 271 | ``` 272 | repository install Download and installs the current DC/OS Universe repository 273 | repository update Updates the local DC/OS Universe repository with the remote 274 | repository check Checks if the DC/OS Universe repository is installed locally 275 | ``` 276 | 277 | ### Packages 278 | 279 | You (un)install packages from the DC/OS universe repository. Please be aware that `mesosctl` cannot programatically check whether the actual configuration supports the chosen package from a resource perspective (number of nodes, available cpus and memory), because this information is currently not exposed in the package definitions. 280 | 281 | ``` 282 | package install [options] Installs a package 283 | package describe [options] Displays information about a package 284 | package uninstall Uninstalls a package 285 | package search Searches for packages with specific string 286 | ``` 287 | 288 | ### Marathon applications and groups 289 | 290 | You can start and manage applications/groups via Marathon. 291 | 292 | ``` 293 | marathon info list Shows information about the running Marathon instance 294 | marathon app list Lists all running apps 295 | marathon app remove Removes a specific app (i.e. stops the app) 296 | marathon app restart Restarts a specific app 297 | marathon app show Show the configuration details of a specific app 298 | marathon app start Starts an app with a specific configuration 299 | marathon app update [properties...] Updates a running specific app 300 | marathon app version list Display the version list for a specific app 301 | marathon app scale Scales (up od down) a specific app 302 | marathon deployment list Lists all current deployments 303 | marathon deployment rollback Triggers a rollback of a specific deployment 304 | marathon deployment remove Removes/stops a specific deployment 305 | marathon group add Adds a new group 306 | marathon group list Lists all current groups 307 | marathon group scale Scales (up or down) a specifc group 308 | marathon group show Show details about a specific group 309 | marathon group remove Remove/stop a specific group 310 | marathon task list Show all running tasks 311 | marathon task show Show details about a specify running task 312 | ``` 313 | 314 | ### Tasks 315 | 316 | You can introspect running tasks from the leading Mesos master. 317 | 318 | ``` 319 | task list [options] Lists all task in the cluster 320 | task show [options] Retrieves information about a task 321 | task log [options] [file] Print the task log. By default, the 10 most recent task logs from stdout are printed. 322 | task ls [path] Print the list of files in the Mesos task sandbox 323 | ``` 324 | 325 | ## Thanks 326 | 327 | A special "thank you" goes to [Ben Hall](https://twitter.com/ben_hall) and [Team Katacoda](https://www.katacoda.com) for providing an interactive environment for testing `mesosctl`, for free! -------------------------------------------------------------------------------- /ansible/ansible.cfg: -------------------------------------------------------------------------------- 1 | [ssh_connection] 2 | ssh_args = -o ControlMaster=auto -o ControlPersist=60s -F ssh.config 3 | 4 | [defaults] 5 | hostfile = inventory 6 | 7 | ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by mesosctl (https://mesoshq.github.io/mesosctl) 8 | -------------------------------------------------------------------------------- /ansible/bootstrap-cluster.yml: -------------------------------------------------------------------------------- 1 | - name: Bootstrap cluster 2 | hosts: all 3 | gather_facts: False 4 | roles: 5 | - role: coreos-bootstrap 6 | when: os_family == "CoreOS" 7 | - role: docker 8 | become: yes 9 | become_user: root 10 | when: os_family != "CoreOS" 11 | - role: common 12 | -------------------------------------------------------------------------------- /ansible/install-agents.yml: -------------------------------------------------------------------------------- 1 | - name: Install Mesos agents 2 | hosts: agents 3 | gather_facts: true 4 | become: yes 5 | become_user: root 6 | roles: 7 | - role: agent -------------------------------------------------------------------------------- /ansible/install-masters.yml: -------------------------------------------------------------------------------- 1 | # This playbook will install then Mesos Masters (including Zookeepers) and Marathons on the 'masters' hosts for Debian and RedHat based installations, 2 | # and the Docker images for Mesos Masters, Zookeepers and Marathons for CoreOS 3 | - name: Install Mesos masters, Marathon and ZooKeeper 4 | hosts: masters 5 | gather_facts: true 6 | become: yes 7 | become_user: root 8 | roles: 9 | - role: master -------------------------------------------------------------------------------- /ansible/install-mesos-dns.yml: -------------------------------------------------------------------------------- 1 | - name: Install Mesos DNS 2 | hosts: agents 3 | gather_facts: true 4 | become: yes 5 | become_user: root 6 | roles: 7 | - role: mesos-dns -------------------------------------------------------------------------------- /ansible/install-registry.yml: -------------------------------------------------------------------------------- 1 | - name: Install Docker registry 2 | hosts: registry 3 | gather_facts: true 4 | become: yes 5 | become_user: root 6 | roles: 7 | - role: registry -------------------------------------------------------------------------------- /ansible/inventory/dynamic_inventory: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var path = require("path"); 3 | var mesosCtl = require("../../lib/mesosCtl")(); 4 | 5 | var configuration = mesosCtl.functions.deserializeYaml(process.env.MESOSCTL_CONFIGURATION_PATH); 6 | var dockerImages = mesosCtl.functions.deserializeYaml(path.join(__dirname, "../../", "config/docker.yml")); 7 | 8 | function toHostName (ipAddress, prefix) { 9 | var octetArray = ipAddress.split("\."); 10 | return prefix+"-" + octetArray[2] + "" + octetArray[3] 11 | } 12 | 13 | function toHostArray (ipArray, hostMap) { 14 | var resultArray = []; 15 | ipArray.forEach(function (ipAddress) { 16 | resultArray.push(hostMap[ipAddress]); 17 | }); 18 | return resultArray; 19 | } 20 | 21 | function toHostObj (hostArray, prefix) { 22 | var hostObj = {}; 23 | hostArray.forEach(function (ipAddress) { 24 | hostObj[toHostName(ipAddress, prefix)] = { 25 | "ansible_ssh_host": ipAddress, 26 | "ansible_ssh_port": configuration.ssh_port, 27 | "ansible_ssh_user": configuration.ssh_user, 28 | "ansible_ssh_private_key_file": configuration.ssh_key_path 29 | } 30 | }); 31 | return hostObj; 32 | } 33 | 34 | function toIPHostMap (ipArray, prefix) { 35 | var map = {}; 36 | ipArray.forEach(function (ipAddress) { 37 | map[ipAddress] = toHostName(ipAddress, prefix); 38 | }); 39 | return map; 40 | } 41 | 42 | function getZooKeeperClusterConnection (ipArray) { 43 | return ipArray.join(":2888:3888,") + ":2888:3888"; 44 | } 45 | 46 | function getZooKeeperConnection (ipArray) { 47 | return "zk://" + ipArray.join(":2181,") + ":2181"; 48 | } 49 | 50 | var hostPrefix = "host"; 51 | var uniqueIPs = mesosCtl.functions.getUniqueItemArray(configuration.masters.concat(configuration.agents, configuration.registry)); 52 | var hosts = toHostObj(uniqueIPs, hostPrefix); 53 | var ipHostMap = toIPHostMap(uniqueIPs, hostPrefix); 54 | 55 | var list = { 56 | "all": { 57 | "vars": { 58 | "registry": configuration.registry[0] + ":5000", 59 | "registry_config": { 60 | "hostname": "registry", 61 | "port": 5000, 62 | "ip": configuration.registry[0] 63 | }, 64 | "os_family": configuration.os_family, 65 | "docker": dockerImages, 66 | "zookeeper": { 67 | "cluster_connection": getZooKeeperClusterConnection(configuration.masters), 68 | "base_connection": getZooKeeperConnection(configuration.masters), 69 | "pure_connection": getZooKeeperConnection(configuration.masters).replace("zk://", ""), 70 | "mesos_connection": getZooKeeperConnection(configuration.masters) + "/mesos", 71 | "marathon_connection": getZooKeeperConnection(configuration.masters) + "/marathon" 72 | }, 73 | "dns_servers": configuration.dns_servers 74 | } 75 | }, 76 | "masters": { 77 | "hosts": toHostArray(configuration.masters, ipHostMap), 78 | "vars": { 79 | "cluster_name": configuration.cluster_name 80 | } 81 | }, 82 | "agents": { 83 | "hosts": toHostArray(configuration.agents, ipHostMap) 84 | }, 85 | "registry": { 86 | "hosts": toHostArray(configuration.registry, ipHostMap) 87 | }, 88 | "_meta": { 89 | "hostvars": hosts 90 | } 91 | }; 92 | 93 | // Add specific Python interpreter variable if OS family is CoreOS 94 | if (configuration.os_family === "CoreOS") { 95 | list.all.vars["ansible_python_interpreter"] = "/home/core/bin/python" 96 | } 97 | 98 | var args = process.argv; 99 | var command = args[2]; 100 | 101 | if (command === "--list") { 102 | console.log(JSON.stringify(list)); 103 | } 104 | 105 | if (command.indexOf("--host=") > -1) { 106 | var hostName = command.replace("--host=", ""); 107 | console.log(JSON.stringify(hosts[hostName])); 108 | } -------------------------------------------------------------------------------- /ansible/provision.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: provision.yml 3 | # Provisions the Mesos cluster on bare CoreOS/Debian/CentOS machines 4 | # and installs the local Docker registry, Marathon and Mesos DNS 5 | - include: bootstrap-cluster.yml 6 | - include: install-registry.yml 7 | - include: install-masters.yml 8 | - include: install-agents.yml 9 | - include: install-mesos-dns.yml -------------------------------------------------------------------------------- /ansible/restart-agent-services.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart Mesos agent services 3 | hosts: agents 4 | gather_facts: true 5 | become: yes 6 | become_user: root 7 | tasks: 8 | - name: Restart Mesos Agent 9 | service: name=mesos-slave state=restarted 10 | -------------------------------------------------------------------------------- /ansible/restart-master-services.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart Mesos master, Marathon and ZooKeeper services 3 | hosts: masters 4 | gather_facts: true 5 | become: yes 6 | become_user: root 7 | tasks: 8 | - name: Restart ZooKeeper 9 | service: name=zookeeper state=restarted 10 | 11 | - name: Restart Mesos Master 12 | service: name=mesos-master state=restarted 13 | 14 | - name: Restart Marathon 15 | service: name=marathon state=restarted -------------------------------------------------------------------------------- /ansible/restart-services.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: restart-all.yml 3 | # * Restarts the Mesos Master, ZooKeeper and Marathon services on Master nodes 4 | # * Restart the Mesos Agent service on Agent nodes 5 | - include: restart-master-services.yml 6 | - include: restart-agent-services.yml -------------------------------------------------------------------------------- /ansible/roles/agent/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | mesos_containerizers: "docker,mesos" 3 | mesos_executor_timeout: "5mins" 4 | executor_shutdown_grace_period: "60secs" 5 | docker_stop_timeout: "50secs" 6 | -------------------------------------------------------------------------------- /ansible/roles/agent/files/mesos-master.override: -------------------------------------------------------------------------------- 1 | manual -------------------------------------------------------------------------------- /ansible/roles/agent/files/zookeeper.override: -------------------------------------------------------------------------------- 1 | manual -------------------------------------------------------------------------------- /ansible/roles/agent/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart mesos-slave 3 | service: name=mesos-slave state=restarted 4 | -------------------------------------------------------------------------------- /ansible/roles/agent/tasks/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/slave/tasks/CoreOS.yml 3 | 4 | - name: Pull Mesos Slave container 5 | raw: docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.agent | basename }} 6 | 7 | - name: Create /var/tmp/mesos/slave folder 8 | become: yes 9 | become_user: root 10 | file: 11 | path: /var/tmp/mesos/slave 12 | state: directory 13 | 14 | - name: Create /var/log/mesos/slave folder 15 | become: yes 16 | become_user: root 17 | file: 18 | path: /var/log/mesos/slave 19 | state: directory 20 | 21 | - name: Make sure Mesos Slave is configured 22 | template: src=mesos-slave.service.j2 dest=/etc/systemd/system/mesos-slave.service 23 | 24 | - name: Ensure systemd is reloaded if mesos-slave.service has changed 25 | shell: systemctl daemon-reload 26 | 27 | - name: (Re)start mesos-slave.service 28 | shell: systemctl restart mesos-slave.service 29 | 30 | - name: Make sure Mesos Slave is running and enabled 31 | service: name=mesos-slave.service state=running enabled=yes 32 | -------------------------------------------------------------------------------- /ansible/roles/agent/tasks/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add apt key for mesos 3 | apt_key: keyserver=keyserver.ubuntu.com id=E56151BF 4 | 5 | - name: Set DISTRO variable 6 | shell: "lsb_release -is | tr '[:upper:]' '[:lower:]'" 7 | register: distro_var 8 | 9 | - name: Set CODENAME variable 10 | shell: "lsb_release -cs" 11 | register: codename_var 12 | 13 | - name: Add apt repository for mesos 14 | apt_repository: repo="deb http://repos.mesosphere.com/{{ distro_var.stdout }} {{ codename_var.stdout }} main" update_cache=yes 15 | 16 | - name: Install mesos 17 | apt: name=mesos update_cache=yes 18 | 19 | - name: Stop zookeeper 20 | service: name=zookeeper state=stopped 21 | when: "'masters' not in group_names" 22 | 23 | - name: Disable zookeeper 24 | copy: src=zookeeper.override dest=/etc/init/zookeeper.override 25 | when: "'masters' not in group_names" 26 | 27 | - name: Stop mesos-master 28 | service: name=mesos-master state=stopped 29 | when: "'masters' not in group_names" 30 | 31 | - name: Disable mesos-master 32 | copy: src=mesos-master.override dest=/etc/init/mesos-master.override 33 | when: "'masters' not in group_names" 34 | 35 | - name: Set zookeeper address for mesos 36 | template: src=zk.j2 dest=/etc/mesos/zk 37 | notify: 38 | - Restart mesos-slave 39 | 40 | - name: Set hostname 41 | template: src=hostname.j2 dest=/etc/mesos-slave/hostname 42 | notify: 43 | - Restart mesos-slave 44 | 45 | - name: Set IP 46 | template: src=ip.j2 dest=/etc/mesos-slave/ip 47 | notify: 48 | - Restart mesos-slave 49 | 50 | - name: Mesos containerizers 51 | template: src=containerizers.j2 dest=/etc/mesos-slave/containerizers 52 | notify: 53 | - Restart mesos-slave 54 | 55 | - name: Mesos executor_registration_timeout 56 | template: src=executor_registration_timeout.j2 dest=/etc/mesos-slave/executor_registration_timeout 57 | notify: 58 | - Restart mesos-slave 59 | 60 | - name: Set LIBPROCESS_IP env 61 | lineinfile: 62 | dest: /etc/default/mesos 63 | regexp: "^LIBPROCESS_IP" 64 | line: "LIBPROCESS_IP={{ ansible_ssh_host }}" 65 | notify: 66 | - Restart mesos-slave 67 | when: "'masters' not in group_names" 68 | 69 | - name: Set LIBPROCESS_ADVERTISE_IP env 70 | lineinfile: 71 | dest: /etc/default/mesos 72 | regexp: "^LIBPROCESS_ADVERTISE_IP" 73 | line: "LIBPROCESS_ADVERTISE_IP={{ ansible_ssh_host }}" 74 | notify: 75 | - Restart mesos-slave 76 | when: "'masters' not in group_names" 77 | 78 | - name: Start mesos-slave 79 | service: name=mesos-slave state=started 80 | -------------------------------------------------------------------------------- /ansible/roles/agent/tasks/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add the repository 3 | yum: name=http://repos.mesosphere.com/el/7/noarch/RPMS/mesosphere-el-repo-7-1.noarch.rpm 4 | 5 | - name: Install mesos 6 | yum: name=mesos update_cache=yes 7 | 8 | - name: Stop mesos-master 9 | service: name=mesos-master state=stopped 10 | when: "'masters' not in group_names" 11 | 12 | - name: Disable mesos-master 13 | command: systemctl disable mesos-master.service 14 | when: "'masters' not in group_names" 15 | 16 | - name: Set zookeeper address for mesos 17 | template: src=zk.j2 dest=/etc/mesos/zk 18 | notify: 19 | - Restart mesos-slave 20 | 21 | - name: Set hostname 22 | template: src=hostname.j2 dest=/etc/mesos-slave/hostname 23 | notify: 24 | - Restart mesos-slave 25 | 26 | - name: Set IP 27 | template: src=ip.j2 dest=/etc/mesos-slave/ip 28 | notify: 29 | - Restart mesos-slave 30 | 31 | - name: Mesos containerizers 32 | template: src=containerizers.j2 dest=/etc/mesos-slave/containerizers 33 | notify: 34 | - Restart mesos-slave 35 | 36 | - name: Mesos executor_registration_timeout 37 | template: src=executor_registration_timeout.j2 dest=/etc/mesos-slave/executor_registration_timeout 38 | notify: 39 | - Restart mesos-slave 40 | 41 | - name: Set LIBPROCESS_IP env 42 | lineinfile: 43 | dest: /etc/default/mesos 44 | regexp: "^LIBPROCESS_IP" 45 | line: "LIBPROCESS_IP={{ ansible_ssh_host }}" 46 | notify: 47 | - Restart mesos-slave 48 | when: "'masters' not in group_names" 49 | 50 | - name: Set LIBPROCESS_ADVERTISE_IP env 51 | lineinfile: 52 | dest: /etc/default/mesos 53 | regexp: "^LIBPROCESS_ADVERTISE_IP" 54 | line: "LIBPROCESS_ADVERTISE_IP={{ ansible_ssh_host }}" 55 | notify: 56 | - Restart mesos-slave 57 | when: "'masters' not in group_names" 58 | 59 | - name: Start mesos-slave 60 | service: name=mesos-slave state=started -------------------------------------------------------------------------------- /ansible/roles/agent/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: "{{ansible_os_family}}.yml" 3 | 4 | - include: CoreOS.yml 5 | when: ansible_os_family == "CoreOS" 6 | 7 | - include: Debian.yml 8 | when: ansible_os_family == "Debian" 9 | 10 | - include: RedHat.yml 11 | when: ansible_os_family == "RedHat" 12 | -------------------------------------------------------------------------------- /ansible/roles/agent/templates/containerizers.j2: -------------------------------------------------------------------------------- 1 | {{ mesos_containerizers }} -------------------------------------------------------------------------------- /ansible/roles/agent/templates/executor_registration_timeout.j2: -------------------------------------------------------------------------------- 1 | {{ mesos_executor_timeout }} -------------------------------------------------------------------------------- /ansible/roles/agent/templates/hostname.j2: -------------------------------------------------------------------------------- 1 | {{ ansible_ssh_host }} 2 | -------------------------------------------------------------------------------- /ansible/roles/agent/templates/ip.j2: -------------------------------------------------------------------------------- 1 | {{ ansible_ssh_host }} -------------------------------------------------------------------------------- /ansible/roles/agent/templates/mesos-slave.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Mesos Slave 3 | After=mesos-master.service 4 | Requires=docker.service 5 | 6 | [Service] 7 | Restart=on-failure 8 | RestartSec=20 9 | TimeoutStartSec=0 10 | Environment=LIBPROCESS_IP={{ ansible_ssh_host }} 11 | ExecStartPre=-/usr/bin/docker kill mesos_slave 12 | ExecStartPre=-/usr/bin/docker rm mesos_slave 13 | ExecStartPre=/usr/bin/docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.agent | basename }} 14 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 15 | --name=mesos_slave \ 16 | --net=host \ 17 | --privileged \ 18 | -v /var/tmp/mesos/slave:/var/tmp/mesos \ 19 | -v /var/log/mesos/slave:/var/log/mesos \ 20 | -v /sys/fs/cgroup/:/cgroup \ 21 | -v /sys:/sys \ 22 | -v /usr/bin/docker:/usr/bin/docker:ro \ 23 | -v /var/run/docker.sock:/var/run/docker.sock \ 24 | -v /lib64/libdevmapper.so.1.02:/lib/libdevmapper.so.1.02:ro \ 25 | -v /lib64/libsystemd.so.0:/lib/libsystemd.so.0:ro \ 26 | -v /lib64/libgcrypt.so.20:/lib/libgcrypt.so.20:ro \ 27 | -p 5051:5051 \ 28 | -e MESOS_IP={{ ansible_ssh_host }} \ 29 | -e MESOS_HOSTNAME={{ ansible_ssh_host }} \ 30 | -e MESOS_CONTAINERIZERS={{ mesos_containerizers }} \ 31 | -e MESOS_MASTER={{ zookeeper.mesos_connection }} \ 32 | -e MESOS_SWITCH_USER=0 \ 33 | -e MESOS_LOG_DIR=/var/log/mesos \ 34 | -e MESOS_WORK_DIR=/var/tmp/mesos \ 35 | -e MESOS_EXECUTOR_REGISTRATION_TIMEOUT={{ mesos_executor_timeout }} \ 36 | {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.agent | basename }} \ 37 | --executor_shutdown_grace_period={{ executor_shutdown_grace_period }} \ 38 | --docker_stop_timeout={{ docker_stop_timeout }}" 39 | ExecStop=/usr/bin/docker stop mesos_slave 40 | 41 | [Install] 42 | WantedBy=multi-user.target 43 | -------------------------------------------------------------------------------- /ansible/roles/agent/templates/zk.j2: -------------------------------------------------------------------------------- 1 | {{ zookeeper.mesos_connection }} 2 | -------------------------------------------------------------------------------- /ansible/roles/agent/vars/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/agent/vars/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /ansible/roles/agent/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | os_version: "{{ ansible_lsb.release if ansible_lsb is defined else ansible_distribution_version }}" 3 | os_version_major: "{{ os_version | regex_replace('^([0-9]+)[^0-9]*.*', '\\\\1') }}" 4 | 5 | mesosphere_releases: 6 | '6': 'mesosphere-el-repo-6-3.noarch.rpm' 7 | '7': 'mesosphere-el-repo-7-1.noarch.rpm' -------------------------------------------------------------------------------- /ansible/roles/agent/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for ansible-slave 3 | -------------------------------------------------------------------------------- /ansible/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/common/tasks/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install docker-py 4 | pip: name=docker-py executable=/home/core/pypy/bin/pip 5 | 6 | - name: Install httplib2 7 | pip: name=httplib2 executable=/home/core/pypy/bin/pip 8 | 9 | - name: Set /etc/hosts to enable local hostname resolution (for Mesos DNS and Marathon) 10 | become: yes 11 | become_user: root 12 | template: src=CoreOS/hosts.j2 dest=/etc/hosts 13 | 14 | - name: Create /etc/systemd/system/docker.service.d folder 15 | become: yes 16 | become_user: root 17 | file: 18 | path: /etc/systemd/system/docker.service.d/ 19 | state: directory 20 | 21 | - name: Increase ulimit 22 | become: yes 23 | become_user: root 24 | template: src="{{ansible_os_family}}/30-increase-ulimit.conf.j2" dest=/etc/systemd/system/docker.service.d/30-increase-ulimit.conf 25 | 26 | - name: Allow insecure registry 27 | become: yes 28 | become_user: root 29 | template: src="{{ansible_os_family}}/50-insecure-registry.conf.j2" dest=/etc/systemd/system/docker.service.d/50-insecure-registry.conf 30 | 31 | - name: Ensure systemd is reloaded after 30-increase-ulimit.conf and 50-insecure-registry.conf have been added 32 | become: yes 33 | become_user: root 34 | shell: systemctl daemon-reload 35 | 36 | - name: Restart Docker service 37 | become: yes 38 | become_user: root 39 | raw: systemctl restart docker.service -------------------------------------------------------------------------------- /ansible/roles/common/tasks/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install python pip 3 | become: yes 4 | become_user: root 5 | apt: name=python-pip 6 | 7 | - name: Install docker-py 8 | become: yes 9 | become_user: root 10 | pip: name=docker-py 11 | 12 | - name: Install httplib2 13 | become: yes 14 | become_user: root 15 | pip: name=httplib2 16 | 17 | - name: Install Mesos dependencies 18 | become: yes 19 | become_user: root 20 | apt: name={{item}} update_cache=yes 21 | with_items: 22 | - unzip 23 | 24 | - name: Disable ipv6 25 | become: yes 26 | become_user: root 27 | shell: sysctl -w net.ipv6.conf.all.disable_ipv6=1 && sysctl -w net.ipv6.conf.default.disable_ipv6=1 28 | 29 | - name: Set /etc/hosts to enable local hostname resolution (for Mesos DNS and Marathon) 30 | become: yes 31 | become_user: root 32 | template: src=Debian/hosts.j2 dest=/etc/hosts 33 | -------------------------------------------------------------------------------- /ansible/roles/common/tasks/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Add the repository for pip 4 | become: yes 5 | become_user: root 6 | yum: 7 | name: "epel-release" 8 | update_cache: yes 9 | 10 | - name: Install pip 11 | become: yes 12 | become_user: root 13 | yum: 14 | name: "python-pip" 15 | update_cache: yes 16 | 17 | - name: Install docker-py 18 | become: yes 19 | become_user: root 20 | pip: name=docker-py 21 | 22 | - name: Install httplib2 23 | become: yes 24 | become_user: root 25 | pip: name=httplib2 26 | 27 | - name: Install Mesos dependencies 28 | become: yes 29 | become_user: root 30 | yum: 31 | name: "{{item}}" 32 | update_cache: yes 33 | with_items: 34 | - tar 35 | - xz 36 | - unzip 37 | - curl 38 | 39 | - name: Disable ipv6 40 | become: yes 41 | become_user: root 42 | shell: sysctl -w net.ipv6.conf.all.disable_ipv6=1 && sysctl -w net.ipv6.conf.default.disable_ipv6=1 43 | 44 | - name: Add hostname entry to match private IP to the given hostname and the IP for the local Docker registry 45 | become: yes 46 | become_user: root 47 | lineinfile: 48 | dest: /etc/hosts 49 | insertafter: EOF 50 | line: "{{ansible_ssh_host}} {{ansible_hostname}}\n{{ registry_config.ip }} {{ registry_config.hostname }}" -------------------------------------------------------------------------------- /ansible/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Gathering facts 4 | setup: 5 | 6 | - include_vars: "{{ansible_os_family}}.yml" 7 | 8 | - include: Debian.yml 9 | when: ansible_os_family == "Debian" 10 | 11 | - include: RedHat.yml 12 | when: ansible_os_family == "RedHat" 13 | 14 | - include: CoreOS.yml 15 | when: ansible_os_family == "CoreOS" 16 | -------------------------------------------------------------------------------- /ansible/roles/common/templates/CoreOS/30-increase-ulimit.conf.j2: -------------------------------------------------------------------------------- 1 | [Service] 2 | LimitMEMLOCK=infinity 3 | LimitNOFILE=infinity -------------------------------------------------------------------------------- /ansible/roles/common/templates/CoreOS/50-insecure-registry.conf.j2: -------------------------------------------------------------------------------- 1 | [Service] 2 | Environment='DOCKER_OPTS=--insecure-registry="{{ registry_config.hostname }}:{{ registry_config.port }}"' -------------------------------------------------------------------------------- /ansible/roles/common/templates/CoreOS/hosts.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 127.0.0.1 localhost 3 | 4 | {{ ansible_ssh_host }} {{ansible_fqdn}} {{ansible_hostname}} 5 | {{ registry_config.ip }} {{ registry_config.hostname }} 6 | -------------------------------------------------------------------------------- /ansible/roles/common/templates/Debian/hosts.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 127.0.0.1 localhost localhost.localdomain 3 | 4 | {{ ansible_ssh_host }} {{ansible_fqdn}} {{ansible_hostname}} 5 | {{ registry_config.ip }} {{ registry_config.hostname }} 6 | -------------------------------------------------------------------------------- /ansible/roles/common/vars/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/common/vars/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /ansible/roles/common/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | os_version: "{{ ansible_lsb.release if ansible_lsb is defined else ansible_distribution_version }}" 3 | os_version_major: " 4 | {%- if ansible_distribution_major_version is defined -%} 5 | {{- ansible_distribution_major_version -}} 6 | {%- elif '.' in os_version %} 7 | {{- os_version[:os_version.index('.')] -}} 8 | {%- else -%} 9 | {{- os_version -}} 10 | {%- endif -%} 11 | " 12 | docker_yum_repo_uris: 13 | Amazon: "centos/{{ 6 if '2014' in os_version or ('2015' in os_version and os_version[os_version.index('.'):] | int <= 3) else 7 }}" 14 | CentOS: "centos/{{ os_version_major }}" 15 | Fedora: "fedora/{{ os_version_major }}" 16 | OracleLinux: "oraclelinux/{{ os_version_major }}" -------------------------------------------------------------------------------- /ansible/roles/coreos-bootstrap/.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | indent_style = space 11 | indent_size = 2 12 | -------------------------------------------------------------------------------- /ansible/roles/coreos-bootstrap/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | env: 6 | - SITE=test.yml 7 | 8 | before_install: 9 | - sudo apt-get update -qq 10 | - sudo apt-get install -y curl 11 | 12 | install: 13 | - pip install ansible 14 | 15 | # Add ansible.cfg to pick up roles path. 16 | - "printf '[defaults]\nroles_path = ../' > ansible.cfg" 17 | 18 | script: 19 | - "ansible-playbook -i tests/inventory tests/$SITE --syntax-check" 20 | -------------------------------------------------------------------------------- /ansible/roles/coreos-bootstrap/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Roman Shtylman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ansible/roles/coreos-bootstrap/README.md: -------------------------------------------------------------------------------- 1 | # coreos-bootstrap 2 | 3 | In order to effectively run ansible, the target machine needs to have a python interpreter. Coreos machines are minimal and do not ship with any version of python. To get around this limitation we can install [pypy](http://pypy.org/), a lightweight python interpreter. The coreos-bootstrap role will install pypy for us and we will update our inventory file to use the installed python interpreter. 4 | 5 | # install 6 | 7 | ``` 8 | ansible-galaxy install defunctzombie.coreos-bootstrap 9 | ``` 10 | 11 | # Configure your project 12 | 13 | Unlike a typical role, you need to configure Ansible to use an alternative python interpreter for coreos hosts. This can be done by adding a `coreos` group to your inventory file and setting the group's vars to use the new python interpreter. This way, you can use ansible to manage CoreOS and non-CoreOS hosts. Simply put every host that has CoreOS into the `coreos` inventory group and it will automatically use the specified python interpreter. 14 | ``` 15 | [coreos] 16 | host-01 17 | host-02 18 | 19 | [coreos:vars] 20 | ansible_ssh_user=core 21 | ansible_python_interpreter=/home/core/bin/python 22 | ``` 23 | 24 | This will configure ansible to use the python interpreter at `/home/core/bin/python` which will be created by the coreos-bootstrap role. 25 | 26 | ## Bootstrap Playbook 27 | 28 | Now you can simply add the following to your playbook file and include it in your `site.yml` so that it runs on all hosts in the coreos group. 29 | 30 | ```yaml 31 | - hosts: coreos 32 | gather_facts: False 33 | roles: 34 | - defunctzombie.coreos-bootstrap 35 | ``` 36 | 37 | Make sure that `gather_facts` is set to false, otherwise ansible will try to first gather system facts using python which is not yet installed! 38 | 39 | ## Example Playbook 40 | 41 | After bootstrap, you can use ansible as usual to manage system services, install python modules (via pip), and run containers. Below is a basic example that starts the `etcd` service, installs the `docker-py` module and then uses the ansible `docker` module to pull and start a basic nginx container. 42 | 43 | ```yaml 44 | - name: Nginx Example 45 | hosts: web 46 | sudo: true 47 | tasks: 48 | - name: Start etcd 49 | service: name=etcd.service state=started 50 | 51 | - name: Install docker-py 52 | pip: name=docker-py 53 | 54 | - name: pull container 55 | raw: docker pull nginx:1.7.1 56 | 57 | - name: launch nginx container 58 | docker: 59 | image="nginx:1.7.1" 60 | name="example-nginx" 61 | ports="8080:80" 62 | state=running 63 | ``` 64 | 65 | # License 66 | MIT 67 | -------------------------------------------------------------------------------- /ansible/roles/coreos-bootstrap/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | set -e 4 | 5 | cd 6 | 7 | if [[ -e $HOME/.bootstrapped ]]; then 8 | exit 0 9 | fi 10 | 11 | PYPY_VERSION=5.1.0 12 | 13 | if [[ -e $HOME/pypy-$PYPY_VERSION-linux64.tar.bz2 ]]; then 14 | tar -xjf $HOME/pypy-$PYPY_VERSION-linux64.tar.bz2 15 | rm -rf $HOME/pypy-$PYPY_VERSION-linux64.tar.bz2 16 | else 17 | wget -O - https://bitbucket.org/pypy/pypy/downloads/pypy-$PYPY_VERSION-linux64.tar.bz2 |tar -xjf - 18 | fi 19 | 20 | mv -n pypy-$PYPY_VERSION-linux64 pypy 21 | 22 | ## library fixup 23 | mkdir -p pypy/lib 24 | ln -snf /lib64/libncurses.so.5.9 $HOME/pypy/lib/libtinfo.so.5 25 | 26 | mkdir -p $HOME/bin 27 | 28 | cat > $HOME/bin/python <=') else '-d' }}" 10 | 11 | # Set the local registry as insecure registry 12 | docker_opts: '"--insecure-registry {{ registry_config.hostname }}:{{ registry_config.port }}"' 13 | 14 | # Other parameters 15 | docker_create_group: true 16 | docker_service_start_timeout: '' # use system default value 17 | 18 | # Expose Docker socket? 19 | docker_listen_tcp: False 20 | docker_listen_port: 4243 21 | -------------------------------------------------------------------------------- /ansible/roles/docker/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart Docker 3 | service: name=docker state=restarted enabled=yes sleep=5 4 | 5 | - name: Reload systemd 6 | command: systemctl daemon-reload 7 | -------------------------------------------------------------------------------- /ansible/roles/docker/meta/.galaxy_install_info: -------------------------------------------------------------------------------- 1 | {install_date: 'Fri Apr 8 12:14:11 2016', version: master} 2 | -------------------------------------------------------------------------------- /ansible/roles/docker/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Jason Giedymin 4 | description: Ansible Docker Playbook Role 5 | company: http://jasongiedymin.com 6 | license: Apache 2 7 | min_ansible_version: 1.2 8 | # 9 | # Below are all platforms currently available. Just uncomment 10 | # the ones that apply to your role. If you don't see your 11 | # platform on this list, let us know and we'll get it added! 12 | # 13 | platforms: 14 | #- name: EL 15 | # versions: 16 | # - all 17 | # - 5 18 | # - 6 19 | #- name: GenericUNIX 20 | # versions: 21 | # - all 22 | # - any 23 | #- name: Fedora 24 | # versions: 25 | # - all 26 | # - 16 27 | # - 17 28 | # - 18 29 | # - 19 30 | # - 20 31 | #- name: opensuse 32 | # versions: 33 | # - all 34 | # - 12.1 35 | # - 12.2 36 | # - 12.3 37 | # - 13.1 38 | # - 13.2 39 | #- name: GenericBSD 40 | # versions: 41 | # - all 42 | # - any 43 | #- name: FreeBSD 44 | # versions: 45 | # - all 46 | # - 8.0 47 | # - 8.1 48 | # - 8.2 49 | # - 8.3 50 | # - 8.4 51 | # - 9.0 52 | # - 9.1 53 | # - 9.1 54 | # - 9.2 55 | - name: Ubuntu 56 | versions: 57 | - all 58 | - lucid 59 | - maverick 60 | - natty 61 | - oneiric 62 | - precise 63 | - quantal 64 | - raring 65 | - saucy 66 | - trusty 67 | #- name: SLES 68 | # versions: 69 | # - all 70 | # - 10SP3 71 | # - 10SP4 72 | # - 11 73 | # - 11SP1 74 | # - 11SP2 75 | # - 11SP3 76 | #- name: GenericLinux 77 | # versions: 78 | # - all 79 | # - any 80 | - name: Debian 81 | versions: 82 | - all 83 | - etch 84 | - lenny 85 | - squeeze 86 | - wheezy 87 | 88 | # Below are all categories currently available. Just as with 89 | # the platforms above, uncomment those that apply to your role. 90 | # 91 | categories: 92 | - cloud 93 | #- cloud:ec2 94 | #- cloud:gce 95 | #- cloud:rax 96 | #- database 97 | #- database:nosql 98 | #- database:sql 99 | #- development 100 | #- monitoring 101 | #- networking 102 | #- packaging 103 | - system 104 | #- web 105 | - platform 106 | dependencies: [] 107 | # List your role dependencies here, one per line. Only 108 | # dependencies available via galaxy should be listed here. 109 | # Be sure to remove the '[]' above if you add dependencies 110 | # to this list. 111 | 112 | -------------------------------------------------------------------------------- /ansible/roles/docker/tasks/Debian.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Add specific key 3 | apt_key: keyserver={{docker_repo_key_server}} id={{docker_repo_key}} 4 | 5 | - name: Add docker repo 6 | apt_repository: repo='deb {{docker_repo}} ubuntu-{{ansible_distribution_release}} main' state=present 7 | 8 | - name: Update apt 9 | apt: update_cache=yes 10 | 11 | - name: Purge the old repo if it exists 12 | apt: name=lxc-docker* purge=yes 13 | ignore_errors: yes 14 | 15 | - name: Verify that apt is pulling from the right repository 16 | command: apt-cache policy docker-engine 17 | 18 | - name: Install the recommended package 19 | apt: name=linux-image-extra-{{ansible_kernel}} state=present 20 | when: docker_install_recommended_package 21 | 22 | - name: Install Docker 23 | apt: name=docker-engine state=present update_cache=yes 24 | 25 | - name: Change ufw forward policy to ACCEPT 26 | command: sed -i 's/DEFAULT_FORWARD_POLICY="DROP"/DEFAULT_FORWARD_POLICY="ACCEPT"/g' /etc/default/ufw 27 | when: docker_listen_tcp == True 28 | 29 | - name: Reload ufw 30 | command: ufw reload 31 | when: docker_listen_tcp == True 32 | 33 | - name: Allow incoming tcp traffic on {{docker_listen_port}} 34 | command: ufw allow {{docker_listen_port}}/tcp 35 | when: docker_listen_tcp == True 36 | -------------------------------------------------------------------------------- /ansible/roles/docker/tasks/RedHat.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Install Docker Repository 3 | template: src=docker-repo.j2 dest=/etc/yum.repos.d/docker.repo 4 | 5 | - name: Install Docker Engine 6 | yum: name=docker-engine state=present 7 | 8 | # Fix for Docker 1.12.0, see https://github.com/docker/docker/issues/25098 9 | - name: Create docker.socket file manually 10 | template: src=docker.socket.j2 dest=/usr/lib/systemd/system/docker.socket 11 | notify: 12 | - Reload systemd 13 | - Restart Docker 14 | -------------------------------------------------------------------------------- /ansible/roles/docker/tasks/init.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Check if /etc/init exists 3 | stat: path=/etc/init/ 4 | register: etc_init 5 | 6 | - name: Docker upstart default config file 7 | template: src=docker-defaults.j2 dest=/etc/default/docker 8 | when: etc_init.stat.exists == true 9 | notify: 10 | - Restart Docker 11 | 12 | - name: Docker init file 13 | template: src=docker-init.j2 dest=/etc/init/docker.conf 14 | when: etc_init.stat.exists == true 15 | notify: 16 | - Restart Docker 17 | 18 | - name: Check if systemd exists 19 | stat: path=/usr/lib/systemd/system/ 20 | register: systemd_check 21 | 22 | - name: Docker systemd default config file 23 | template: src=docker-sysconfig.j2 dest=/etc/sysconfig/docker 24 | when: systemd_check.stat.exists == true 25 | notify: 26 | - Reload systemd 27 | - Restart Docker 28 | 29 | - name: Docker systemd file 30 | template: src=docker-service.j2 dest=/lib/systemd/system/docker.service 31 | when: systemd_check.stat.exists == true 32 | notify: 33 | - Reload systemd 34 | - Restart Docker 35 | -------------------------------------------------------------------------------- /ansible/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Gathering facts 4 | setup: 5 | 6 | - include_vars: "{{ ansible_os_family }}.yml" 7 | 8 | - name: Create docker group 9 | group: name=docker state=present system=yes 10 | when: docker_create_group 11 | tags: 12 | - config 13 | 14 | - name: Create the docker group and add your user 15 | command: usermod -aG docker {{ansible_ssh_user}} 16 | when: docker_create_group 17 | tags: 18 | - config 19 | 20 | - include: Debian.yml 21 | when: ansible_os_family == "Debian" 22 | 23 | - include: RedHat.yml 24 | when: ansible_os_family == "RedHat" 25 | 26 | - include: init.yml 27 | -------------------------------------------------------------------------------- /ansible/roles/docker/templates/docker-defaults.j2: -------------------------------------------------------------------------------- 1 | # Generated by Ansible for {{ansible_fqdn}} 2 | 3 | DOCKER_OPTS={{docker_opts}} 4 | -------------------------------------------------------------------------------- /ansible/roles/docker/templates/docker-init.j2: -------------------------------------------------------------------------------- 1 | # Generated by Ansible for {{ansible_fqdn}} 2 | description "Docker daemon" 3 | 4 | start on filesystem 5 | stop on runlevel [!2345] 6 | 7 | respawn 8 | 9 | script 10 | [ ! -f /etc/default/docker ] || . /etc/default/docker 11 | /usr/bin/docker {{ docker_daemon_opts }} $DOCKER_OPTS 12 | end script -------------------------------------------------------------------------------- /ansible/roles/docker/templates/docker-repo.j2: -------------------------------------------------------------------------------- 1 | [dockerrepo] 2 | name=Docker Repository 3 | baseurl={{ docker_yum_repo }} 4 | enabled=1 5 | gpgcheck=1 6 | gpgkey={{ docker_yum_repo_gpg }} 7 | -------------------------------------------------------------------------------- /ansible/roles/docker/templates/docker-service.j2: -------------------------------------------------------------------------------- 1 | # {{ansible_managed}} 2 | 3 | [Unit] 4 | Description=Docker Application Container Engine 5 | Documentation=https://docs.docker.com 6 | After=network.target docker.socket 7 | Requires=docker.socket 8 | 9 | [Service] 10 | Type=notify 11 | EnvironmentFile=-/etc/sysconfig/docker 12 | ExecStart=/usr/bin/docker daemon -H fd:// $DOCKER_OPTS 13 | {% if docker_service_start_timeout != '' %} 14 | TimeoutStartSec={{ docker_service_start_timeout }} 15 | {% endif %} 16 | MountFlags=slave 17 | LimitNOFILE=1048576 18 | LimitNPROC=1048576 19 | LimitCORE=infinity 20 | 21 | [Install] 22 | WantedBy=multi-user.target 23 | Also=docker.socket 24 | -------------------------------------------------------------------------------- /ansible/roles/docker/templates/docker-sysconfig.j2: -------------------------------------------------------------------------------- 1 | # {{ansible_managed}} 2 | 3 | DOCKER_OPTS={{docker_opts}} 4 | 5 | # CentOS 6, RHEL 6 6 | other_args={{docker_opts}} 7 | -------------------------------------------------------------------------------- /ansible/roles/docker/templates/docker.socket.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Docker Socket for the API 3 | PartOf=docker.service 4 | 5 | [Socket] 6 | ListenStream=/var/run/docker.sock 7 | SocketMode=0660 8 | SocketUser=root 9 | SocketGroup=docker 10 | 11 | [Install] 12 | WantedBy=sockets.target -------------------------------------------------------------------------------- /ansible/roles/docker/vars/Debian.yml: -------------------------------------------------------------------------------- 1 | # replace with gist variant 2 | docker_repo_key_server: "hkp://p80.pool.sks-keyservers.net:80" 3 | docker_repo_key: "58118E89F3A912897C070ADBF76221572C52609D" 4 | docker_repo: "https://apt.dockerproject.org/repo" 5 | docker_install_recommended_package: True 6 | -------------------------------------------------------------------------------- /ansible/roles/docker/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | os_version: "{{ ansible_lsb.release if ansible_lsb is defined else ansible_distribution_version }}" 2 | 3 | -------------------------------------------------------------------------------- /ansible/roles/docker/vars/main.yml: -------------------------------------------------------------------------------- 1 | docker_playbook_version: "0.1.3" 2 | -------------------------------------------------------------------------------- /ansible/roles/master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | mesosphere_yum_repo: "http://repos.mesosphere.com/el/{{ os_version_major }}/noarch/RPMS/{{ mesosphere_releases[ansible_distribution_major_version] }}" -------------------------------------------------------------------------------- /ansible/roles/master/files/mesos-slave.override: -------------------------------------------------------------------------------- 1 | manual -------------------------------------------------------------------------------- /ansible/roles/master/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart zookeeper 3 | service: name=zookeeper state=restarted 4 | 5 | - name: Restart mesos-master 6 | service: name=mesos-master state=restarted 7 | 8 | - name: Restart marathon 9 | service: name=marathon state=restarted -------------------------------------------------------------------------------- /ansible/roles/master/tasks/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Install ZooKeeper 4 | - name: Pull Zookeeper container 5 | raw: docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.zookeeper | basename }} 6 | 7 | - name: Make sure Zookeeper is configured 8 | template: src=zookeeper/zookeeper.service.j2 dest=/etc/systemd/system/zookeeper.service 9 | 10 | - name: Ensure systemd is reloaded if zookeeper.service has changed 11 | shell: systemctl daemon-reload 12 | 13 | - name: (Re)start zookeeper.service 14 | shell: systemctl restart zookeeper.service 15 | 16 | - name: Make sure Zookeeper is running and enabled 17 | service: name=zookeeper.service state=running enabled=yes 18 | 19 | # Install Mesos Master 20 | - name: Pull Mesos Master container 21 | raw: docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.master | basename }} 22 | 23 | - name: Create folder /var/tmp/mesos/master for Mesos master on host 24 | file: 25 | path: /var/tmp/mesos/master 26 | state: directory 27 | 28 | - name: Create folder /var/log/mesos/master for Mesos master on host 29 | file: 30 | path: /var/log/mesos/master 31 | state: directory 32 | 33 | - name: Make sure Mesos Master is configured 34 | template: src=master/mesos-master.service.j2 dest=/etc/systemd/system/mesos-master.service 35 | 36 | - name: Ensure systemd is reloaded if mesos-master.service has changed 37 | shell: systemctl daemon-reload 38 | 39 | - name: (Re)start mesos-master.service 40 | shell: systemctl restart mesos-master.service 41 | 42 | - name: Make sure Mesos Master is running and enabled 43 | service: name=mesos-master.service state=running enabled=yes 44 | 45 | # Install Marathon 46 | - name: Pull Marathon container 47 | raw: docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.marathon | basename }} 48 | 49 | - name: Make sure Marathon is configured 50 | template: src=marathon/marathon.service.j2 dest=/etc/systemd/system/marathon.service 51 | 52 | - name: Ensure systemd is reloaded if marathon.service has changed 53 | shell: systemctl daemon-reload 54 | 55 | - name: (Re)start marathon.service 56 | shell: systemctl restart marathon.service 57 | 58 | - name: Make sure Marathon is running and enabled 59 | service: name=marathon.service state=running enabled=yes -------------------------------------------------------------------------------- /ansible/roles/master/tasks/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Install Mesos, ZooKeeper & Marathon 4 | - name: Add apt key for mesos 5 | become: yes 6 | become_user: root 7 | apt_key: keyserver=keyserver.ubuntu.com id=E56151BF 8 | 9 | - name: Set DISTRO variable 10 | become: yes 11 | become_user: root 12 | shell: "lsb_release -is | tr '[:upper:]' '[:lower:]'" 13 | register: distro_var 14 | 15 | - name: Set CODENAME variable 16 | become: yes 17 | become_user: root 18 | shell: "lsb_release -cs" 19 | register: codename_var 20 | 21 | - name: Add apt repository for mesos 22 | apt_repository: repo="deb http://repos.mesosphere.com/{{ distro_var.stdout }} {{ codename_var.stdout }} main" update_cache=yes 23 | 24 | - name: Add openjdk repository 25 | apt_repository: repo="ppa:openjdk-r/ppa" 26 | when: ansible_lsb.id == 'Ubuntu' 27 | 28 | - name: Install mesos 29 | apt: name={{item}} update_cache=yes 30 | with_items: 31 | - mesos 32 | - openjdk-8-jre-headless 33 | - marathon 34 | 35 | - name: Set zookeeper ID 36 | template: src=zookeeper/myid.j2 dest=/etc/zookeeper/conf/myid 37 | notify: 38 | - Restart zookeeper 39 | 40 | - name: Append zookeeper IP addresses 41 | template: src=zookeeper/zoo.cfg.j2 dest=/etc/zookeeper/conf/zoo.cfg 42 | notify: 43 | - Restart zookeeper 44 | 45 | - name: Set Zookeeper address for Mesos 46 | template: src=master/zk.j2 dest=/etc/mesos/zk 47 | notify: 48 | - Restart mesos-master 49 | - Restart marathon 50 | 51 | - name: Set quorum 52 | template: src=master/quorum.j2 dest=/etc/mesos-master/quorum 53 | notify: 54 | - Restart mesos-master 55 | 56 | - name: Ensure /etc/marathon/conf directory 57 | file: path=/etc/marathon/conf state=directory 58 | 59 | - name: Set LIBPROCESS_IP for Marathon in /etc/environment 60 | lineinfile: 61 | dest: /etc/environment 62 | regexp: "^LIBPROCESS_IP" 63 | line: "LIBPROCESS_IP={{ ansible_ssh_host }}\nLIBPROCESS_PORT=9090" 64 | notify: 65 | - Restart marathon 66 | 67 | #- name: Set LIBPROCESS_IP for Marathon in the init.d script 68 | # lineinfile: 69 | # dest: /etc/init.d/marathon 70 | # regexp: "^PID" 71 | # insertafter: "^#PID" 72 | # line: "PID=/var/run/marathon.pid\nLIBPROCESS_IP={{ ansible_ssh_host }}\nLIBPROCESS_PORT=9090" 73 | # notify: 74 | # - Restart marathon 75 | 76 | - name: Set mesos-master ip 77 | template: src=master/ip.j2 dest=/etc/mesos-master/ip 78 | notify: 79 | - Restart mesos-master 80 | 81 | - name: Set mesos-master hostname 82 | template: src=master/hostname.j2 dest=/etc/mesos-master/hostname 83 | notify: 84 | - Restart mesos-master 85 | 86 | - name: Set marathon hostname 87 | template: src=marathon/hostname.j2 dest=/etc/marathon/conf/hostname 88 | notify: 89 | - Restart marathon 90 | 91 | - name: Stop mesos-slave 92 | service: name=mesos-slave state=stopped 93 | when: "'slaves' not in group_names" 94 | 95 | - name: Disable mesos-slave 96 | copy: src=mesos-slave.override dest=/etc/init/mesos-slave.override 97 | when: "'slaves' not in group_names" 98 | 99 | - name: Set LIBPROCESS_IP env 100 | lineinfile: 101 | dest: /etc/default/mesos 102 | regexp: "^LIBPROCESS_IP" 103 | line: "LIBPROCESS_IP={{ ansible_ssh_host }}" 104 | notify: 105 | - Restart mesos-master 106 | - Restart marathon 107 | 108 | - name: Set LIBPROCESS_ADVERTISE_IP env 109 | lineinfile: 110 | dest: /etc/default/mesos 111 | regexp: "^LIBPROCESS_ADVERTISE_IP" 112 | line: "LIBPROCESS_ADVERTISE_IP={{ ansible_ssh_host }}" 113 | notify: 114 | - Restart mesos-master 115 | - Restart marathon 116 | 117 | - name: Set cluster name 118 | lineinfile: 119 | dest: /etc/default/mesos-master 120 | regexp: "^CLUSTER" 121 | line: "CLUSTER={{ cluster_name }}" 122 | notify: 123 | - Restart mesos-master 124 | - Restart marathon 125 | 126 | # This is used to overcome a probable bug: http://stackoverflow.com/questions/31858937/transport-endpoint-not-connected-mesos-slave-master 127 | - name: Set quorum 128 | lineinfile: 129 | dest: /etc/default/mesos-master 130 | regexp: "^MESOS_QUORUM" 131 | line: "MESOS_QUORUM=`cat /etc/mesos-master/quorum`" 132 | notify: 133 | - Restart mesos-master 134 | - Restart marathon 135 | 136 | - name: Start services 137 | service: name={{item}} state=started 138 | with_items: 139 | - zookeeper 140 | - mesos-master 141 | - marathon -------------------------------------------------------------------------------- /ansible/roles/master/tasks/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add the repository 3 | yum: name={{ mesosphere_yum_repo }} state=present 4 | 5 | - name: Install mesos and marathon 6 | yum: name={{item}} update_cache=yes 7 | with_items: 8 | - mesos 9 | - marathon 10 | 11 | - name: Install zookeeper 12 | yum: name={{ zookeeper_pkg_names[os_version_major] }} 13 | 14 | - name: Ensure /var/log/zookeeper directory 15 | file: path=/var/log/zookeeper state=directory 16 | 17 | - name: Set zookeeper ID (/etc/zookeeper/conf) 18 | template: src=zookeeper/myid.j2 dest=/etc/zookeeper/conf/myid 19 | notify: 20 | - Restart zookeeper 21 | 22 | - name: Set zookeeper ID (/var/lib/zookeeper) 23 | template: src=zookeeper/myid.j2 dest=/var/lib/zookeeper/myid 24 | notify: 25 | - Restart zookeeper 26 | 27 | - name: Append zookeeper IP addresses 28 | template: src=zookeeper/zoo.cfg.j2 dest=/etc/zookeeper/conf/zoo.cfg 29 | notify: 30 | - Restart zookeeper 31 | 32 | - name: Set zookeeper address for mesos 33 | template: src=master/zk.j2 dest=/etc/mesos/zk 34 | notify: 35 | - Restart mesos-master 36 | - Restart marathon 37 | 38 | - name: Set quorum 39 | template: src=master/quorum.j2 dest=/etc/mesos-master/quorum 40 | notify: 41 | - Restart mesos-master 42 | 43 | - name: Set mesos-master hostname 44 | template: src=master/hostname.j2 dest=/etc/mesos-master/hostname 45 | notify: 46 | - Restart mesos-master 47 | 48 | - name: Set mesos-master ip 49 | template: src=master/ip.j2 dest=/etc/mesos-master/ip 50 | notify: 51 | - Restart mesos-master 52 | 53 | - name: Ensure /etc/marathon/conf directory 54 | file: path=/etc/marathon/conf state=directory 55 | 56 | - name: Set marathon hostname 57 | template: src=marathon/hostname.j2 dest=/etc/marathon/conf/hostname 58 | notify: 59 | - Restart marathon 60 | 61 | - name: Set the LIBPROCESS_IP environment variable for Marathon 62 | template: src=marathon/marathon_env.j2 dest=/etc/sysconfig/marathon 63 | notify: 64 | - Restart marathon 65 | 66 | - name: Stop mesos-slave 67 | service: name=mesos-slave state=stopped 68 | 69 | - name: Disable mesos-slave for version 7 70 | command: systemctl disable mesos-slave.service 71 | 72 | - name: Set LIBPROCESS_IP env 73 | lineinfile: 74 | dest: /etc/default/mesos 75 | regexp: "^LIBPROCESS_IP" 76 | line: "LIBPROCESS_IP={{ ansible_ssh_host }}" 77 | notify: 78 | - Restart mesos-master 79 | 80 | - name: Set LIBPROCESS_ADVERTISE_IP env 81 | lineinfile: 82 | dest: /etc/default/mesos 83 | regexp: "^LIBPROCESS_ADVERTISE_IP" 84 | line: "LIBPROCESS_ADVERTISE_IP={{ ansible_ssh_host }}" 85 | notify: 86 | - Restart mesos-master 87 | 88 | # This is used to overcome a probable bug: http://stackoverflow.com/questions/31858937/transport-endpoint-not-connected-mesos-slave-master 89 | - name: Set quorum 90 | lineinfile: 91 | dest: /etc/default/mesos-master 92 | regexp: "^MESOS_QUORUM" 93 | line: "MESOS_QUORUM=`cat /etc/mesos-master/quorum`" 94 | notify: 95 | - Restart mesos-master 96 | 97 | - name: Start services 98 | service: name={{item}} state=started 99 | with_items: 100 | - zookeeper 101 | - mesos-master 102 | - marathon 103 | -------------------------------------------------------------------------------- /ansible/roles/master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/mesos-master/tasks/main.yml 3 | 4 | - include_vars: "{{ansible_os_family}}.yml" 5 | 6 | - include: Debian.yml 7 | when: ansible_os_family == "Debian" 8 | 9 | - include: RedHat.yml 10 | when: ansible_os_family == "RedHat" 11 | 12 | - include: CoreOS.yml 13 | when: ansible_os_family == "CoreOS" 14 | -------------------------------------------------------------------------------- /ansible/roles/master/templates/marathon/hostname.j2: -------------------------------------------------------------------------------- 1 | {{ ansible_ssh_host }} 2 | -------------------------------------------------------------------------------- /ansible/roles/master/templates/marathon/marathon.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Marathon 3 | After=mesos-slave.service 4 | Requires=docker.service 5 | 6 | [Service] 7 | Restart=on-failure 8 | RestartSec=20 9 | TimeoutStartSec=0 10 | Environment=LIBPROCESS_IP={{ ansible_ssh_host }} 11 | ExecStartPre=-/usr/bin/docker kill marathon 12 | ExecStartPre=-/usr/bin/docker rm marathon 13 | ExecStartPre=/usr/bin/docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.marathon | basename }} 14 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 15 | --name marathon \ 16 | --net=host \ 17 | -p 8080:8080 \ 18 | -p 9090:9090 \ 19 | {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.marathon | basename }} \ 20 | --master {{ zookeeper.mesos_connection }} \ 21 | --zk {{ zookeeper.marathon_connection }} \ 22 | --checkpoint \ 23 | --hostname {{ ansible_ssh_host }} \ 24 | --event_subscriber http_callback" 25 | ExecStop=/usr/bin/docker stop marathon 26 | 27 | [Install] 28 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /ansible/roles/master/templates/marathon/marathon_env.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | LIBPROCESS_IP={{ ansible_ssh_host }} 4 | #LIBPROCESS_PORT=9090 -------------------------------------------------------------------------------- /ansible/roles/master/templates/master/hostname.j2: -------------------------------------------------------------------------------- 1 | {{ ansible_ssh_host }} 2 | -------------------------------------------------------------------------------- /ansible/roles/master/templates/master/ip.j2: -------------------------------------------------------------------------------- 1 | {{ ansible_ssh_host }} -------------------------------------------------------------------------------- /ansible/roles/master/templates/master/mesos-master.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Mesos Master 3 | After=zookeeper.service 4 | Requires=docker.service 5 | 6 | [Service] 7 | Restart=on-failure 8 | RestartSec=20 9 | TimeoutStartSec=0 10 | ExecStartPre=-/usr/bin/docker kill mesos_master 11 | ExecStartPre=-/usr/bin/docker rm mesos_master 12 | ExecStartPre=/usr/bin/docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.master | basename }} 13 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 14 | --name=mesos_master \ 15 | --net=host \ 16 | -v /var/tmp/mesos/master:/var/tmp/mesos \ 17 | -v /var/log/mesos/master:/var/log/mesos \ 18 | -e MESOS_IP={{ ansible_ssh_host }} \ 19 | -e MESOS_HOSTNAME={{ ansible_ssh_host }} \ 20 | -e MESOS_CLUSTER=mesos-cluster \ 21 | -e MESOS_ZK={{ zookeeper.mesos_connection }} \ 22 | -e MESOS_LOG_DIR=/var/log/mesos \ 23 | -e MESOS_WORK_DIR=/var/tmp/mesos \ 24 | -e MESOS_QUORUM={{ ((groups['masters'] | length)/2 ) | round(0, 'ceil') | int }} \ 25 | -e MESOS_ROLES=internal,external \ 26 | {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.master | basename }}" 27 | ExecStop=/usr/bin/docker stop mesos_master 28 | 29 | [Install] 30 | WantedBy=multi-user.target 31 | -------------------------------------------------------------------------------- /ansible/roles/master/templates/master/quorum.j2: -------------------------------------------------------------------------------- 1 | {{ ((groups.masters | length) // 2) + 1 }} 2 | -------------------------------------------------------------------------------- /ansible/roles/master/templates/master/zk.j2: -------------------------------------------------------------------------------- 1 | {{ zookeeper.mesos_connection }} -------------------------------------------------------------------------------- /ansible/roles/master/templates/zookeeper/myid.j2: -------------------------------------------------------------------------------- 1 | {{ (groups.masters.index(inventory_hostname)+1) }} -------------------------------------------------------------------------------- /ansible/roles/master/templates/zookeeper/zoo.cfg.j2: -------------------------------------------------------------------------------- 1 | tickTime=2000 2 | dataDir=/var/lib/zookeeper 3 | dataLogDir=/var/log/zookeeper 4 | clientPort=2181 5 | initLimit=90 6 | syncLimit=30 7 | 8 | {% for server in groups.masters %} 9 | server.{{ loop.index }}={{hostvars[server]['ansible_ssh_host']}}:2888:3888 10 | {% endfor %} 11 | -------------------------------------------------------------------------------- /ansible/roles/master/templates/zookeeper/zookeeper.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Zookeeper 3 | After=docker.service 4 | Requires=docker.service 5 | 6 | [Service] 7 | Restart=on-failure 8 | RestartSec=20 9 | TimeoutStartSec=0 10 | ExecStartPre=-/usr/bin/docker kill zookeeper 11 | ExecStartPre=-/usr/bin/docker rm zookeeper 12 | ExecStartPre=/usr/bin/docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.zookeeper | basename }} 13 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 14 | --net=host \ 15 | --name=zookeeper \ 16 | -e ZOOKEEPER_HOSTS={{ zookeeper.cluster_connection }} \ 17 | -e LOCAL_ZK_IP={{ ansible_ssh_host }} \ 18 | -p 2181:2181 \ 19 | -p 2888:2888 \ 20 | -p 3888:3888 \ 21 | {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.zookeeper | basename }}" 22 | ExecStop=/usr/bin/docker stop zookeeper 23 | 24 | [Install] 25 | WantedBy=multi-user.target 26 | -------------------------------------------------------------------------------- /ansible/roles/master/vars/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/master/vars/Debian.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/master/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | os_version: "{{ ansible_lsb.release if ansible_lsb is defined else ansible_distribution_version }}" 2 | os_version_major: "{{ ansible_distribution_major_version }}" 3 | 4 | mesosphere_releases: 5 | '7': 'mesosphere-el-repo-7-1.noarch.rpm' 6 | '6': 'mesosphere-el-repo-6-2.noarch.rpm' 7 | 8 | zookeeper_pkg_names: 9 | '7': 'mesosphere-zookeeper' 10 | '6': 'mesosphere-zookeeper' -------------------------------------------------------------------------------- /ansible/roles/master/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/tasks/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Copy resolv.conf file 4 | become: yes 5 | become_user: root 6 | template: src=resolv.conf.j2 dest=/etc/resolv.conf -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/tasks/Debian.yml: -------------------------------------------------------------------------------- 1 | - name: Add nameservers to /etc/resolv.conf 2 | lineinfile: 3 | dest=/etc/resolvconf/resolv.conf.d/head 4 | insertbefore=BOF 5 | line='nameserver {{ansible_ssh_host}}\n{% for server in dns_servers %}nameserver {{server}}\n{% endfor %}' 6 | 7 | - name: Reload resolvconf 8 | shell: resolvconf -u -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/tasks/RedHat.yml: -------------------------------------------------------------------------------- 1 | - name: Add the nameservers to /etc/resolv.conf 2 | lineinfile: 3 | dest=/etc/resolv.conf 4 | insertbefore=BOF 5 | line='nameserver {{ansible_ssh_host}}\n{% for server in dns_servers %}nameserver {{server}}\n{% endfor %}' -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/mesos-dns/tasks/main.yml 3 | 4 | - name: Pull Mesos DNS image 5 | raw: docker pull {{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.mesosdns | basename }} 6 | 7 | - include: CoreOS.yml 8 | when: ansible_os_family == "CoreOS" 9 | 10 | - include: Debian.yml 11 | when: ansible_os_family == "Debian" 12 | 13 | - include: RedHat.yml 14 | when: ansible_os_family == "RedHat" 15 | 16 | - name: Wait for Marathon to come up 17 | run_once: true 18 | wait_for: host="{{ hostvars[groups['masters'][0]]['ansible_ssh_host'] }}" port=8080 state=started delay=1 timeout=15 19 | 20 | - name: Post Mesos DNS application to Marathon 21 | run_once: true 22 | uri: 23 | url: "http://{{ hostvars[groups['masters'][0]]['ansible_ssh_host'] }}:8080/v2/apps" 24 | method: POST 25 | body: "{{ lookup('template','roles/mesos-dns/templates/mesosdns.json.j2') }}" 26 | status_code: 201 27 | body_format: json 28 | register: status 29 | failed_when: "'OK' not in status.msg" 30 | -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/templates/mesosdns.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "id": "mesos-dns", 3 | "env": { 4 | "MESOS_ZK": "{{ zookeeper.mesos_connection }}", 5 | "MESOS_DNS_EXTERNAL_SERVERS": "{{ dns_servers | join(',') }}", 6 | "MESOS_DNS_REFRESH": "{{ settings.refresh_timeout }}", 7 | "MESOS_DNS_HTTP_ENABLED": "true", 8 | "MESOS_IP_SOURCES": "mesos,host" 9 | }, 10 | "container": { 11 | "docker": { 12 | "image": "{{ registry_config.hostname }}:{{ registry_config.port }}/{{ docker.images.mesosdns | basename }}", 13 | "network": "HOST" 14 | }, 15 | "type": "DOCKER" 16 | }, 17 | "cpus": {{ resources.cpus }}, 18 | "mem": {{ resources.mem }}, 19 | "instances": {{ groups.agents | length }}, 20 | "constraints": [["hostname", "UNIQUE"]], 21 | "healthChecks": [ 22 | { 23 | "path": "/v1/version", 24 | "protocol": "HTTP", 25 | "gracePeriodSeconds": 30, 26 | "intervalSeconds": 10, 27 | "timeoutSeconds": 20, 28 | "maxConsecutiveFailures": 3, 29 | "ignoreHttp1xx": false, 30 | "port": 8123 31 | } 32 | ] 33 | } -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | nameserver {{ansible_ssh_host}} 2 | {% for server in dns_servers %} 3 | nameserver {{server}} 4 | {% endfor %} 5 | -------------------------------------------------------------------------------- /ansible/roles/mesos-dns/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | resources: 3 | cpus: 0.2 4 | mem: 128 5 | settings: 6 | refresh_timeout: 10 7 | -------------------------------------------------------------------------------- /ansible/roles/registry/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/registry/tasks/CoreOS.yml: -------------------------------------------------------------------------------- 1 | - name: Copy registry.service file 2 | become: yes 3 | become_user: root 4 | template: src=registry.service.j2 dest=/etc/systemd/system/registry.service 5 | register: result 6 | 7 | - name: Ensure systemd is reloaded if registry.service has been added 8 | become: yes 9 | become_user: root 10 | shell: systemctl daemon-reload 11 | when: result|changed 12 | 13 | - name: Restart Docker Registry service 14 | become: yes 15 | become_user: root 16 | shell: systemctl restart registry.service -------------------------------------------------------------------------------- /ansible/roles/registry/tasks/Debian.yml: -------------------------------------------------------------------------------- 1 | - name: Run Docker image for Registry 2 | docker: 3 | name: registry 4 | image: "{{ docker.images.registry }}" 5 | state: reloaded 6 | restart_policy: always 7 | net: host 8 | volumes: 9 | - "/opt/registry/data:/var/lib/registry:rw" 10 | - "/opt/registry/certs:/certs:ro" 11 | ports: 12 | - "5000:5000" 13 | env: 14 | REGISTRY_HTTP_TLS_CERTIFICATE: "/certs/registry.crt" 15 | REGISTRY_HTTP_TLS_KEY: "/certs/registry.key" -------------------------------------------------------------------------------- /ansible/roles/registry/tasks/RedHat.yml: -------------------------------------------------------------------------------- 1 | - name: Run Docker image for Registry 2 | docker: 3 | name: registry 4 | image: "{{ docker.images.registry }}" 5 | state: reloaded 6 | restart_policy: always 7 | net: host 8 | volumes: 9 | - "/opt/registry/data:/var/lib/registry:rw" 10 | - "/opt/registry/certs:/certs:ro" 11 | ports: 12 | - "5000:5000" 13 | env: 14 | REGISTRY_HTTP_TLS_CERTIFICATE: "/certs/registry.crt" 15 | REGISTRY_HTTP_TLS_KEY: "/certs/registry.key" -------------------------------------------------------------------------------- /ansible/roles/registry/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/registry/tasks/main.yml 3 | - include_vars: "{{ansible_os_family}}.yml" 4 | 5 | - name: Pull Registry container 6 | shell: docker pull {{ docker.images.registry }} 7 | 8 | - name: Create data directory on host 9 | file: 10 | path: /opt/registry/data 11 | state: directory 12 | 13 | - name: Create certs directory on host 14 | file: 15 | path: /opt/registry/certs 16 | state: directory 17 | 18 | - name: Prepare for IP SANs for SSL cert 19 | shell: echo "subjectAltName = IP:{{ registry_config.ip }}" > /opt/registry/certs/extfile.cnf 20 | 21 | - name: Create self-signed SSL cert 22 | command: 'openssl req -new -nodes -x509 -subj "/C=DE/ST=Hamburg/L=Hamburg/O=IT/CN={{registry_config.hostname}}" -days 3650 -extensions /opt/registry/certs/extfile.cnf -keyout /opt/registry/certs/registry.key -out /opt/registry/certs/registry.crt -extensions v3_ca creates=/opt/registry/certs/registry.crt' 23 | 24 | - include: Debian.yml 25 | when: ansible_os_family == "Debian" 26 | 27 | - include: RedHat.yml 28 | when: ansible_os_family == "RedHat" 29 | 30 | - include: CoreOS.yml 31 | when: ansible_os_family == "CoreOS" 32 | 33 | - name: Copy pull_and_push.sh file 34 | template: src=pull_and_push.sh.j2 dest=/tmp/pull_and_push.sh mode=0755 35 | 36 | - name: Pull all required Docker images and push to local registry 37 | shell: /tmp/pull_and_push.sh -------------------------------------------------------------------------------- /ansible/roles/registry/templates/pull_and_push.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Pull, tag and push images 4 | {% for key, value in docker.images.iteritems() %} 5 | {% if key != "registry" %} 6 | docker pull {{ value }} 7 | docker tag {{ value }} {{ registry_config.hostname }}:{{ registry_config.port }}/{{ value | basename }} 8 | docker push {{ registry_config.hostname }}:{{ registry_config.port }}/{{ value | basename }} 9 | {% endif %} 10 | {% endfor %} 11 | 12 | # Remove original images 13 | {% for key, value in docker.images.iteritems() %} 14 | {% if key != "registry" %} 15 | docker rmi {{ value }} 16 | {% endif %} 17 | {% endfor %} -------------------------------------------------------------------------------- /ansible/roles/registry/templates/registry.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Docker Registry 3 | After=docker.service 4 | Requires=docker.service 5 | 6 | [Service] 7 | Restart=on-failure 8 | RestartSec=20 9 | TimeoutStartSec=0 10 | ExecStartPre=-/usr/bin/docker kill registry 11 | ExecStartPre=-/usr/bin/docker rm registry 12 | ExecStartPre=/usr/bin/docker pull {{ docker.images.registry }} 13 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 14 | --net=host \ 15 | --name=registry \ 16 | -p 5000:5000 \ 17 | -v /opt/registry/data:/var/lib/registry \ 18 | -v /opt/registry/certs:/certs \ 19 | -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \ 20 | -e REGISTRY_HTTP_TLS_KEY=/certs/registry.key \ 21 | {{ docker.images.registry }}" 22 | ExecStop=/usr/bin/docker stop registry 23 | 24 | [Install] 25 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /ansible/roles/registry/vars/CoreOS.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/registry/vars/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Overwrite variable from "all" 3 | docker: 4 | images: 5 | registry: registry:2.4 6 | mesosdns: tobilg/mesos-dns:v0.5.2 -------------------------------------------------------------------------------- /ansible/roles/registry/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Overwrite variable from "all" 3 | docker: 4 | images: 5 | registry: registry:2.4 6 | mesosdns: tobilg/mesos-dns:v0.5.2 -------------------------------------------------------------------------------- /ansible/roles/registry/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /ansible/ssh.config: -------------------------------------------------------------------------------- 1 | Host * 2 | UserKnownHostsFile /dev/null 3 | StrictHostKeyChecking no 4 | PasswordAuthentication no 5 | IdentitiesOnly yes 6 | LogLevel FATAL -------------------------------------------------------------------------------- /bin/mesosctl.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | var vorpal = require('vorpal')(), 3 | mesosCtl = require("../lib/mesosCtl")(); 4 | 5 | vorpal = require("../modules/config")(vorpal, mesosCtl); 6 | vorpal = require("../modules/cluster")(vorpal, mesosCtl); 7 | vorpal = require("../modules/package")(vorpal, mesosCtl); 8 | vorpal = require("../modules/marathon")(vorpal, mesosCtl); 9 | vorpal = require("../modules/repository")(vorpal, mesosCtl); 10 | vorpal = require("../modules/task")(vorpal, mesosCtl); 11 | 12 | process.on('uncaughtException', function (error) { 13 | console.log("Caught exception: "); 14 | console.log(error.stack); 15 | }); 16 | 17 | vorpal 18 | .delimiter('mesosctl $ ') 19 | .show(); 20 | -------------------------------------------------------------------------------- /config/docker.yml: -------------------------------------------------------------------------------- 1 | images: 2 | registry: registry:2.4 3 | zookeeper: tobilg/zookeeper:3.4.6 4 | master: mesosphere/mesos-master:1.0.1-2.0.93.ubuntu1404 5 | agent: mesosphere/mesos-slave:1.0.1-2.0.93.ubuntu1404 6 | marathon: mesosphere/marathon:v1.3.0 7 | mesosdns: tobilg/mesos-dns:v0.5.2 8 | 9 | # For Debian/RedHat also configure 10 | # roles/registry/vars/Debian.yml 11 | # roles/registry/vars/RedHat.yml 12 | # when updating Docker images! -------------------------------------------------------------------------------- /lib/mesosCtl.js: -------------------------------------------------------------------------------- 1 | var os = require('os'), 2 | path = require('path'), 3 | yaml = require('js-yaml'), 4 | fs = require('fs'), 5 | unzip = require("unzip"), 6 | request = require("request"), 7 | MesosDNSAgent = require("mesosdns-http-agent"), 8 | Ajv = require("ajv"), 9 | lunr = require("lunr"); 10 | 11 | var ajv = Ajv({ loadSchema: function (uri, callback) { 12 | request({ url: uri, json:true, method: "GET" }, function(err, res, body) { 13 | if (err || res.statusCode >= 400) 14 | callback(err || new Error('Loading error: ' + res.statusCode)); 15 | else 16 | callback(null, body); 17 | }); 18 | }}); 19 | 20 | module.exports = function () { 21 | 22 | var mesosCtl = { 23 | options: { 24 | os: { 25 | allowed: ["CoreOS", "Ubuntu Xenial", "Ubuntu Vivid", "Ubuntu Trusty", "Centos 7", "Centos 6", "RedHat Enterprise Linux 7", "RedHat Enterprise Linux 6", "Debian Jessie"], 26 | families: { 27 | "CoreOS": ["CoreOS"], 28 | "Debian": ["Ubuntu Xenial", "Ubuntu Vivid", "Ubuntu Trusty", "Debian Jessie"], 29 | "RedHat": ["Centos 7", "Centos 6", "RedHat Enterprise Linux 7", "RedHat Enterprise Linux 6"] 30 | } 31 | }, 32 | configStoragePath: process.env.MESOSCTL_CONFIGURATION_BASE_PATH || os.homedir() + "/.mesosctl", 33 | currentFile: ".current", 34 | repository: { 35 | version: "version-2.x", 36 | archive: "https://github.com/mesosphere/universe/archive/%%VERSION%%.zip", 37 | relativePath: "/repository/universe-%%VERSION%%", 38 | relativeIndexPath: "/repository/universe-%%VERSION%%/repo/meta/index.json" 39 | }, 40 | marathonBaseUrl: "http://leader.mesos:8080", 41 | masterBaseUrl: "http://leader.mesos:5050" 42 | }, 43 | hasValidConfig: false, 44 | currentConfigPath: null, 45 | packages: { 46 | index: lunr(function () { 47 | this.field('tags', {boost: 10}); 48 | this.field('description'); 49 | this.field('name'); 50 | this.ref('name') 51 | }), 52 | map: {}, 53 | list: [], 54 | loaded: false 55 | }, 56 | config: {}, 57 | functions: { 58 | loadPackages: function (repositoryData) { 59 | 60 | repositoryData.packages.forEach(function (pkg){ 61 | 62 | // Index package in lunr for package search 63 | mesosCtl.packages.index.add({ 64 | name: pkg.name, 65 | description: pkg.description, 66 | tags: pkg.tags 67 | }); 68 | 69 | // Add package to list 70 | mesosCtl.packages.list.push(pkg.name); 71 | 72 | // Add to map 73 | mesosCtl.packages.map[pkg.name] = pkg; 74 | 75 | }); 76 | 77 | }, 78 | initPackageIndex: function () { 79 | if (mesosCtl.functions.checkRepository()) { 80 | 81 | // Reset other properties 82 | mesosCtl.packages.list.length = 0; 83 | mesosCtl.packages.map = {}; 84 | mesosCtl.packages.loaded = false; 85 | 86 | // Load local package index 87 | mesosCtl.functions.loadPackages(mesosCtl.functions.getLocalRepositoryIndex()); 88 | 89 | // Mark as loaded 90 | mesosCtl.packages.loaded = true; 91 | 92 | } 93 | }, 94 | serializeConfiguration: function (path) { 95 | if (path) { 96 | return mesosCtl.functions.serializeYaml(path, mesosCtl.config); 97 | } else { 98 | return mesosCtl.functions.serializeYaml(mesosCtl.functions.getLocalConfigPath(mesosCtl.config.cluster_name), mesosCtl.config); 99 | } 100 | }, 101 | deserializeConfiguration: function (configurationName) { 102 | return mesosCtl.functions.deserializeYaml(mesosCtl.functions.getLocalConfigPath(configurationName)); 103 | }, 104 | serializeYaml: function (filePath, data) { 105 | try { 106 | fs.writeFileSync(filePath, yaml.safeDump(data), 'utf8'); 107 | return { error: null }; 108 | } catch (e) { 109 | return { error: e }; 110 | } 111 | }, 112 | deserializeYaml: function (filePath) { 113 | try { 114 | return yaml.safeLoad(fs.readFileSync(filePath, 'utf8')); 115 | } catch (e) { 116 | return { error: e }; 117 | } 118 | }, 119 | toYAML: function (data, callback) { 120 | try { 121 | callback(null, yaml.safeDump(data)); 122 | } catch (error) { 123 | callback(error, null); 124 | } 125 | }, 126 | getLocalConfigPath: function (configurationName) { 127 | return mesosCtl.options.configStoragePath + "/" + configurationName + ".yml" 128 | }, 129 | ensureConfigStoragePath: function () { 130 | fs.stat(mesosCtl.options.configStoragePath, function (err, stat) { 131 | if (err) { 132 | fs.mkdirSync(mesosCtl.options.configStoragePath); 133 | } 134 | }); 135 | }, 136 | checkIfConfigurationExists: function (configurationPath) { 137 | try { 138 | var stats = fs.statSync(configurationPath); 139 | return true; 140 | } catch (e) { 141 | return false; 142 | } 143 | }, 144 | loadAndValidateConfiguration: function (configPath, mesosCtl, callback) { 145 | 146 | // Assign loaded config 147 | var config = mesosCtl.functions.deserializeYaml(configPath); 148 | 149 | // Check for errors 150 | if (!config.hasOwnProperty("error")) { 151 | 152 | // Validate provided config via according JSON schema 153 | mesosCtl.functions.isValidSchema("config", config, function (error, isValid) { 154 | 155 | if (error) { 156 | 157 | console.log("An error occurred: " + JSON.stringify(error)); 158 | callback(); 159 | 160 | } else if (isValid) { 161 | 162 | // Check if configuration contains an OS family property, if not, set it 163 | if (!config.hasOwnProperty("os_family")) { 164 | 165 | // Match OS family and set ENV variable for Ansible 166 | Object.getOwnPropertyNames(mesosCtl.options.os.families).forEach(function (family) { 167 | if (mesosCtl.options.os.families[family].indexOf(config.os) > -1) { 168 | config.os_family = family; 169 | } 170 | }); 171 | 172 | } 173 | 174 | // Set the config 175 | mesosCtl.config = config; 176 | 177 | // Set hasValidConfig property to true 178 | mesosCtl.hasValidConfig = true; 179 | 180 | // Set currentConfigPath 181 | mesosCtl.currentConfigPath = configPath; 182 | 183 | console.log("--> The given configuration was successfully loaded!"); 184 | 185 | callback(); 186 | 187 | } else { 188 | 189 | console.log("An error occurred: The provided configuration is not valid!"); 190 | callback(); 191 | 192 | } 193 | 194 | }); 195 | 196 | } else { 197 | self.log("An error occurred: " + config.error); 198 | callback(); 199 | } 200 | 201 | }, 202 | listConfigurations: function () { 203 | var configurations = []; 204 | fs.readdirSync(mesosCtl.options.configStoragePath).forEach(function (fileName) { 205 | var filePath = path.join(mesosCtl.options.configStoragePath, fileName); 206 | var stat = fs.statSync(filePath); 207 | if (stat.isFile() && fileName.indexOf(".yml") > -1) { 208 | configurations.push(fileName.replace(".yml", "")); 209 | } 210 | }); 211 | return configurations; 212 | }, 213 | isValidSchema: function (type, data, callback) { 214 | 215 | var allowedValidations = ["app", "group", "deployment", "config"]; 216 | 217 | // Check if type is allowed 218 | if (allowedValidations.indexOf(type) > -1) { 219 | 220 | var fileName = ""; 221 | 222 | if (type === "config") { 223 | fileName = "mesosctl_" + type + ".json"; 224 | } else { 225 | fileName = "marathon_" + type + ".json"; 226 | } 227 | 228 | var schemaPath = path.join(__dirname, "schema", fileName); 229 | 230 | // Load schema json 231 | var schema = JSON.parse(fs.readFileSync(schemaPath)); 232 | 233 | ajv.compileAsync(schema, function (err, validate) { 234 | 235 | if (err) callback(err, null); 236 | var valid = validate(data); 237 | 238 | if (!valid) { 239 | callback(validate.errors, valid); 240 | } else { 241 | callback(null, valid); 242 | } 243 | 244 | }); 245 | 246 | } else { 247 | callback("The schema type to check is not in the list of allowed schema type (" + allowedValidations.join(",") + ")", false); 248 | } 249 | 250 | }, 251 | isValidSchemaDynamic: function (schemaData, data, callback) { 252 | 253 | ajv.compileAsync(schemaData, function (err, validate) { 254 | 255 | if (err) callback(err, null); 256 | var valid = validate(data); 257 | 258 | if (!valid) { 259 | callback(validate.errors, valid); 260 | } else { 261 | callback(null, valid); 262 | } 263 | 264 | }); 265 | 266 | }, 267 | getConfigurationParameters: function (callback) { 268 | try { 269 | var configSchema = JSON.parse(fs.readFileSync(path.join(__dirname, "schema", "mesosctl_config.json"), 'utf8')); 270 | 271 | var configObj = { 272 | required: {}, 273 | optional: {} 274 | }; 275 | 276 | Object.getOwnPropertyNames(configSchema.properties).forEach(function (property) { 277 | if (configSchema.required.indexOf(property) > -1) { 278 | configObj.required[property] = configSchema.properties[property]; 279 | } else { 280 | if (property !== "provisioned" && property !== "os_family") { 281 | configObj.optional[property] = configSchema.properties[property]; 282 | } 283 | } 284 | }); 285 | 286 | callback(null, configObj); 287 | } catch (error) { 288 | callback(error, null); 289 | } 290 | }, 291 | checkIfValidIP4Address: function(address) { 292 | var regex = /^(?!0)(?!.*\.$)((1?\d?\d|25[0-5]|2[0-4]\d)(\.|$)){4}$/; 293 | return regex.test(address); 294 | }, 295 | getValidIPAddresses: function(ipAddresses) { 296 | var addresses = []; 297 | // Check for valid IP addresses 298 | if (ipAddresses.length > 0) { 299 | ipAddresses.forEach(function(address) { 300 | if (mesosCtl.functions.checkIfValidIP4Address(address)) { 301 | addresses.push(address); 302 | } 303 | }); 304 | } 305 | return addresses; 306 | }, 307 | getUniqueItemArray: function (inputArray) { 308 | var temp = []; 309 | inputArray.forEach(function(item) { 310 | if (temp.indexOf(item) === -1) { 311 | temp.push(item); 312 | } 313 | }); 314 | return temp; 315 | }, 316 | removeItemFromArray: function (arr, item) { 317 | for(var i = arr.length; i--;) { 318 | if(arr[i] === item) { 319 | arr.splice(i, 1); 320 | } 321 | } 322 | return arr; 323 | }, 324 | uniqueConcat: function (inputArray, toAddArray) { 325 | toAddArray.forEach(function(item) { 326 | // Only add address if not already present 327 | if (inputArray.indexOf(item) === -1) { 328 | inputArray.push(item); 329 | } 330 | }); 331 | return inputArray; 332 | }, 333 | removeAddresses: function (inputArray, toRemoveArray) { 334 | // Copy of original addresses 335 | var originalAddresses = inputArray.slice(0); 336 | 337 | // Remove adresses 338 | toRemoveArray.forEach(function(address) { 339 | originalAddresses = mesosCtl.functions.removeItemFromArray(originalAddresses, address); 340 | }); 341 | 342 | return originalAddresses; 343 | }, 344 | downloadRepository: function () { 345 | request.get(mesosCtl.options.repository.archive.replace("%%VERSION%%", mesosCtl.options.repository.version)).pipe(unzip.Extract({ path: mesosCtl.options.configStoragePath+"/repository" })); 346 | }, 347 | checkRepository: function () { 348 | try { 349 | var stats = fs.statSync(path.join(mesosCtl.options.configStoragePath, mesosCtl.options.repository.relativePath.replace("%%VERSION%%", mesosCtl.options.repository.version))); 350 | return true; 351 | } catch (e) { 352 | return false; 353 | } 354 | }, 355 | removeRepository: function () { 356 | if (mesosCtl.functions.checkRepository()) { 357 | // TODO: Implement secure recursive directory deletion (use rimraf) 358 | } 359 | }, 360 | getLocalRepositoryIndex: function () { 361 | return JSON.parse(fs.readFileSync(path.join(mesosCtl.options.configStoragePath, mesosCtl.options.repository.relativeIndexPath.replace("%%VERSION%%", mesosCtl.options.repository.version)), 'utf8')); 362 | }, 363 | getPackageFile: function (packageName, packageVersion, fileType) { 364 | var pathToFile = path.join(mesosCtl.options.configStoragePath, mesosCtl.options.repository.relativePath.replace("%%VERSION%%", mesosCtl.options.repository.version) + "/repo/packages/" + packageName.substring(0, 1).toUpperCase() + "/" + packageName + "/" + packageVersion.toString()); 365 | var jsonFiles = ["command", "config", "resource", "package"]; 366 | 367 | if (jsonFiles.indexOf(fileType.toLowerCase()) > -1) { 368 | return JSON.parse(fs.readFileSync(path.join(pathToFile, "/" + fileType.toLowerCase() + ".json"), 'utf8')); 369 | } else if (fileType.toLowerCase() === "marathon") { 370 | return fs.readFileSync(path.join(pathToFile, "/marathon.json.mustache"), 'utf8'); 371 | } 372 | }, 373 | getAgents: function () { 374 | return mesosCtl.config.agents || []; 375 | }, 376 | getLeader: function (callback) { 377 | var options = { 378 | agentClass: MesosDNSAgent, 379 | agentOptions: { 380 | "dnsServers": mesosCtl.functions.getAgents(), 381 | "mesosTLD": ".mesos" 382 | }, 383 | method: "GET", 384 | json: true, 385 | url: "http://leader.mesos:5050/state" 386 | }; 387 | request(options, function(error, response, body) { 388 | if (error || !response) { 389 | callback(error, null); 390 | } 391 | if (response.statusCode < 400 && !error) { 392 | 393 | var leaderUrl = body.leader.replace("master@", ""); 394 | 395 | callback(null, leaderUrl); 396 | } else { 397 | callback("There was a problem in the execution of this command", body); 398 | } 399 | }); 400 | }, 401 | leftPad: function (str, len, char) { 402 | str = String(str); 403 | var i = -1; 404 | if (!char && char !== 0) char = " "; 405 | len = len - str.length; 406 | while (++i < len) { 407 | str = char + str; 408 | } 409 | return str; 410 | }, 411 | rightPad: function (str, len, char) { 412 | if (! str || ! char || str.length >= len) { 413 | return str; 414 | } 415 | var max = (len - str.length)/char.length; 416 | for (var i = 0; i < max; i++) { 417 | str += char; 418 | } 419 | return str; 420 | } 421 | } 422 | }; 423 | 424 | return mesosCtl; 425 | 426 | }; -------------------------------------------------------------------------------- /lib/schema/marathon_deployment.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "id": "https://raw.githubusercontent.com/mesosphere/marathon/master/docs/docs/rest-api/public/api/v2/schema/Group.json", 4 | "additionalProperties": false, 5 | "definitions": { 6 | "pathType": { 7 | "type": "string", 8 | "pattern": "^(\\/?((\\.{2})|([a-z0-9][a-z0-9\\-.]*[a-z0-9]+)|([a-z0-9]*))($|\\/))+$", 9 | "minLength": 1 10 | } 11 | }, 12 | "type": "object", 13 | "properties": { 14 | "id": { 15 | "type": "string", 16 | "$ref": "#/definitions/pathType", 17 | "description": "Unique identifier for the app consisting of a series of names separated by slashes. Each name must be at least 1 character and may only contain digits (`0-9`), dashes (`-`), dots (`.`), and lowercase letters (`a-z`). The name may not begin or end with a dash." 18 | }, 19 | "apps": { 20 | "type": "array", 21 | "description": "The list of AppDefinitions in this group. See AppDefinition.json for the schema.", 22 | "items": { 23 | "$ref": "./AppDefinition.json" 24 | } 25 | }, 26 | "groups": { 27 | "type": "array", 28 | "description": "Groups can build a tree. Each group can contain sub-groups. The sub-groups are defined here.", 29 | "items": { 30 | "$ref": "#" 31 | } 32 | }, 33 | "dependencies": { 34 | "type": "array", 35 | "description": "A list of services upon which this application depends. An order is derived from the dependencies for performing start/stop and upgrade of the application. For example, an application /a relies on the services /b which itself relies on /c. To start all 3 applications, first /c is started than /b than /a.", 36 | "items": { 37 | "$ref": "#/definitions/pathType" 38 | } 39 | }, 40 | "version": { 41 | "type": "string", 42 | "description": "The version of this definition.", 43 | "format": "date-time" 44 | } 45 | }, 46 | "required": [ 47 | "id" 48 | ] 49 | } -------------------------------------------------------------------------------- /lib/schema/marathon_group.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "id": "https://raw.githubusercontent.com/mesosphere/marathon/master/docs/docs/rest-api/public/api/v2/schema/Group.json", 4 | "additionalProperties": false, 5 | "definitions": { 6 | "pathType": { 7 | "type": "string", 8 | "pattern": "^(\\/?((\\.{2})|([a-z0-9][a-z0-9\\-.]*[a-z0-9]+)|([a-z0-9]*))($|\\/))+$", 9 | "minLength": 1 10 | } 11 | }, 12 | "type": "object", 13 | "properties": { 14 | "id": { 15 | "type": "string", 16 | "$ref": "#/definitions/pathType", 17 | "description": "Unique identifier for the app consisting of a series of names separated by slashes. Each name must be at least 1 character and may only contain digits (`0-9`), dashes (`-`), dots (`.`), and lowercase letters (`a-z`). The name may not begin or end with a dash." 18 | }, 19 | "apps": { 20 | "type": "array", 21 | "description": "The list of AppDefinitions in this group. See AppDefinition.json for the schema.", 22 | "items": { 23 | "$ref": "https://raw.githubusercontent.com/mesosphere/marathon/master/docs/docs/rest-api/public/api/v2/schema/AppDefinition.json" 24 | } 25 | }, 26 | "groups": { 27 | "type": "array", 28 | "description": "Groups can build a tree. Each group can contain sub-groups. The sub-groups are defined here.", 29 | "items": { 30 | "$ref": "#" 31 | } 32 | }, 33 | "dependencies": { 34 | "type": "array", 35 | "description": "A list of services upon which this application depends. An order is derived from the dependencies for performing start/stop and upgrade of the application. For example, an application /a relies on the services /b which itself relies on /c. To start all 3 applications, first /c is started than /b than /a.", 36 | "items": { 37 | "$ref": "#/definitions/pathType" 38 | } 39 | }, 40 | "version": { 41 | "type": "string", 42 | "description": "The version of this definition.", 43 | "format": "date-time" 44 | } 45 | }, 46 | "required": [ 47 | "id" 48 | ] 49 | } -------------------------------------------------------------------------------- /lib/schema/mesosctl_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "title": "mesosctl configuration", 4 | "description": "A configuration file for a cluster used by mesosctl", 5 | "type": "object", 6 | "properties": { 7 | "cluster_name": { 8 | "description": "The cluster name", 9 | "type": "string", 10 | "default": "mesos-cluster" 11 | }, 12 | "os": { 13 | "description": "Operation system name / code", 14 | "type": "string", 15 | "enum": ["CoreOS", "Ubuntu Xenial", "Ubuntu Vivid", "Ubuntu Trusty", "Centos 7", "Centos 6", "RedHat Enterprise Linux 7", "RedHat Enterprise Linux 6", "Debian Jessie"] 16 | }, 17 | "os_family": { 18 | "description": "Operation system family name / code", 19 | "type": "string", 20 | "enum": ["CoreOS", "RedHat", "Debian"] 21 | }, 22 | "ssh_user": { 23 | "description": "SSH user name", 24 | "type": "string" 25 | }, 26 | "ssh_port": { 27 | "description": "SSH server port", 28 | "type": "integer" 29 | }, 30 | "ssh_key_path": { 31 | "description": "Full path to the SSH key to use", 32 | "type": "string" 33 | }, 34 | "admin_user": { 35 | "description": "Admin user name", 36 | "type": "string", 37 | "default": "admin" 38 | }, 39 | "admin_password_hash": { 40 | "description": "Password hash for the admin user", 41 | "type": "string" 42 | }, 43 | "log_directory": { 44 | "description": "Directory to log the mesosctl output to", 45 | "type": "string", 46 | "default": "~/.mesosctl/logs" 47 | }, 48 | "dns_servers": { 49 | "description": "List of external DNS servers (IP adresses)", 50 | "type": "array", 51 | "items": { 52 | "type": "string" 53 | }, 54 | "minItems": 1, 55 | "uniqueItems": true 56 | }, 57 | "masters": { 58 | "type": "array", 59 | "description": "List of Mesos Master nodes (IP adresses)", 60 | "items": { 61 | "type": "string" 62 | }, 63 | "minItems": 1, 64 | "uniqueItems": true 65 | }, 66 | "agents": { 67 | "type": "array", 68 | "description": "List of Mesos Agent nodes (IP adresses)", 69 | "items": { 70 | "type": "string" 71 | }, 72 | "minItems": 1, 73 | "uniqueItems": true 74 | }, 75 | "registry": { 76 | "type": "array", 77 | "description": "Docker registry host (IP adress)", 78 | "items": { 79 | "type": "string" 80 | }, 81 | "minItems": 1, 82 | "maxItems": 1, 83 | "uniqueItems": true 84 | }, 85 | "provisioned": { 86 | "type": "boolean", 87 | "default": false, 88 | "description": "Indicates whether the configuration has been provisioned or not" 89 | } 90 | }, 91 | "required": ["cluster_name", "os_family", "ssh_key_path", "ssh_port", "ssh_user", "dns_servers", "masters", "agents", "registry"] 92 | } -------------------------------------------------------------------------------- /modules/cluster.js: -------------------------------------------------------------------------------- 1 | var Ansible = require('node-ansible'), 2 | path = require("path"), 3 | os = require("os"), 4 | MesosDNSAgent = require("mesosdns-http-agent"), 5 | request = require("request"), 6 | ProgressBar = require("progress"), 7 | chalk = require("chalk"), 8 | sshClient = require('ssh2').Client; 9 | 10 | function extractPorts(portString) { 11 | portString = portString.replace("[", "").replace("]", "").toString(); 12 | var portRanges = []; 13 | if (portString.indexOf(", ") > -1) { 14 | portRanges = portString.split(", "); 15 | } else { 16 | var p = "" + portString.toString(); 17 | portRanges.push(String(p)); 18 | } 19 | 20 | var portCount = 0; 21 | 22 | portRanges.forEach(function (portRange) { 23 | var temp = portRange.split("-"); 24 | var start = temp[0], 25 | end = temp[1]; 26 | if (start === end) { 27 | portCount++; 28 | } else { 29 | portCount += (end-start); 30 | } 31 | }); 32 | return portCount; 33 | } 34 | 35 | function renderUtilization(vorpal, utilization, header) { 36 | 37 | vorpal.log(os.EOL + header + os.EOL); 38 | 39 | // TODO: Find a way to use the vorpal output stream for logging via the `stream` option 40 | 41 | var cpuBar = new ProgressBar(' CPU [:bar] :percent\n', { 42 | complete: '=', 43 | incomplete: ' ', 44 | width: 20, 45 | total: 1 46 | }); 47 | cpuBar.tick(utilization.cpus); 48 | 49 | var memoryBar = new ProgressBar(' Memory [:bar] :percent\n', { 50 | complete: '=', 51 | incomplete: ' ', 52 | width: 20, 53 | total: 1 54 | }); 55 | memoryBar.tick(utilization.memory); 56 | 57 | var diskBar = new ProgressBar(' Disk [:bar] :percent\n', { 58 | complete: '=', 59 | incomplete: ' ', 60 | width: 20, 61 | total: 1 62 | }); 63 | diskBar.tick(utilization.disk); 64 | 65 | var portsBar = new ProgressBar(' Ports [:bar] :percent\n', { 66 | complete: '=', 67 | incomplete: ' ', 68 | width: 20, 69 | total: 1 70 | }); 71 | portsBar.tick(utilization.ports); 72 | 73 | vorpal.log(""); 74 | 75 | } 76 | 77 | function getUtilizationStats(dnsServers, cb) { 78 | 79 | var options = { 80 | agentClass: MesosDNSAgent, 81 | agentOptions: { 82 | "dnsServers": dnsServers, 83 | "mesosTLD": ".mesos" 84 | }, 85 | url: "http://leader.mesos:5050/state-summary" 86 | }; 87 | 88 | try { 89 | request(options, function(error, res, body) { 90 | if (error) { 91 | cb(error, null); 92 | } else { 93 | 94 | var response = JSON.parse(body); 95 | 96 | var overallResources = { 97 | cpus: 0, 98 | memory: 0, 99 | disk: 0, 100 | ports: 0 101 | }; 102 | 103 | var overallUsedResources = { 104 | cpus: 0, 105 | memory: 0, 106 | disk: 0, 107 | ports: 0 108 | }; 109 | 110 | var agentMap = {}; 111 | 112 | response.slaves.forEach(function (slave) { 113 | 114 | // Add to overall resources 115 | overallResources.cpus += slave.resources.cpus; 116 | overallResources.memory += slave.resources.mem; 117 | overallResources.disk += slave.resources.disk; 118 | overallResources.ports += extractPorts(slave.resources.ports); 119 | 120 | // Add to used resources 121 | overallUsedResources.cpus += slave.used_resources.cpus; 122 | overallUsedResources.memory += slave.used_resources.mem; 123 | overallUsedResources.disk += slave.used_resources.disk; 124 | overallUsedResources.ports += extractPorts(slave.used_resources.ports); 125 | 126 | // Add info per agent 127 | agentMap[slave.hostname] = { 128 | utilization: { 129 | cpus: parseFloat((slave.used_resources.cpus/slave.resources.cpus).toFixed(4)), 130 | memory: parseFloat((slave.used_resources.mem/slave.resources.mem).toFixed(4)), 131 | disk: parseFloat((slave.used_resources.disk/slave.resources.disk).toFixed(4)), 132 | ports: parseFloat((extractPorts(slave.used_resources.ports)/extractPorts(slave.resources.ports)).toFixed(4)) 133 | } 134 | }; 135 | 136 | }); 137 | 138 | var utilization = { 139 | cluster: { 140 | utilization: { 141 | cpus: parseFloat((overallUsedResources.cpus/overallResources.cpus).toFixed(4)), 142 | memory: parseFloat((overallUsedResources.memory/overallResources.memory).toFixed(4)), 143 | disk: parseFloat((overallUsedResources.disk/overallResources.disk).toFixed(4)), 144 | ports: parseFloat((overallUsedResources.ports/overallResources.ports).toFixed(4)) 145 | }, 146 | resources: { 147 | cpus: overallResources.cpus, 148 | memory: overallResources.memory, 149 | disk: overallResources.disk, 150 | ports: overallResources.ports 151 | }, 152 | usedResources: { 153 | cpus: overallUsedResources.cpus, 154 | memory: overallUsedResources.memory, 155 | disk: overallUsedResources.disk, 156 | ports: overallUsedResources.ports 157 | }, 158 | unusedResources: { 159 | cpus: overallResources.cpus-overallUsedResources.cpus, 160 | memory: overallResources.memory-overallUsedResources.memory, 161 | disk: overallResources.disk-overallUsedResources.disk, 162 | ports: overallResources.ports-overallUsedResources.ports 163 | } 164 | }, 165 | agents: agentMap 166 | }; 167 | 168 | cb(null, utilization); 169 | 170 | } 171 | 172 | }); 173 | } catch (error) { 174 | cb(error, null); 175 | } 176 | 177 | 178 | } 179 | 180 | function provision (mesosCtl, options, callback) { 181 | 182 | // Set Ansible working directory 183 | var ansibleWorkDir = path.join(__dirname, "../", "ansible"); 184 | 185 | // Set MESOSCTL_CONFIGURATION_PATH environment variable for the dynamic inventory 186 | process.env.MESOSCTL_CONFIGURATION_PATH = mesosCtl.currentConfigPath; 187 | 188 | var taskMatcher = /TASK \[(.*)\]/; 189 | var playMatcher = /PLAY \[(.*)\]/; 190 | var storedOutput = ""; 191 | var checkOutput = false; 192 | var playbook = new Ansible.Playbook().playbook(ansibleWorkDir + "/provision"); 193 | 194 | // Log the proceeding installation steps 195 | playbook.on('stdout', function(data) { 196 | var output = data.toString(); 197 | var foundPlay = output.match(playMatcher); 198 | var foundTask = output.match(taskMatcher); 199 | var foundInclude = (output.match(/include/) === null ? false : true); 200 | var foundSkipped = (output.match(/skipping/) === null ? false : true); // If skipped, don't show! 201 | 202 | // Log playbook starts 203 | if (foundPlay && foundPlay.length === 2) { 204 | console.log("---------------------------------------------------------------------"); 205 | console.log("Starting play " + foundPlay[1]); 206 | console.log("---------------------------------------------------------------------"); 207 | } 208 | 209 | if (checkOutput) { 210 | if (!foundSkipped && options && options.verbose) { 211 | console.log("Starting task " + storedOutput); 212 | } 213 | checkOutput = false; 214 | } else { 215 | if (foundTask && foundTask.length === 2 && !foundInclude) { 216 | storedOutput = foundTask[1]; 217 | checkOutput = true; 218 | } 219 | } 220 | 221 | }); 222 | 223 | playbook.exec({ cwd: ansibleWorkDir }).then(function(successResult) { 224 | 225 | // Set as provisioned 226 | mesosCtl.config.provisioned = true; 227 | 228 | // Serialize the configuration 229 | mesosCtl.functions.serializeConfiguration(mesosCtl.currentConfigPath); 230 | 231 | callback(); 232 | 233 | }, function(error) { 234 | 235 | var fatalGlobalMatcher = /(fatal:.*)/g; 236 | var fatalRealMatcher = /^(fatal)(?!.*?lxc-docker).*$/g; 237 | var globalErrors = error.toString().match(fatalGlobalMatcher); 238 | var realErrors = []; 239 | 240 | // Check if the global error signatures found match the "real" error signatures (omit ignored fatal messages) 241 | if (globalErrors && globalErrors.length > 0) { 242 | globalErrors.forEach(function (globalError) { 243 | var found = globalError.match(fatalRealMatcher); 244 | if (found && found.length > 0) { 245 | realErrors.push(globalError); 246 | } 247 | }); 248 | } 249 | 250 | console.log("---------------------------------------------------------------------"); 251 | console.log("Error(s) occurred:"); 252 | console.log("---------------------------------------------------------------------"); 253 | if (realErrors && realErrors.length > 0) { 254 | console.log(realErrors.join("\n")); 255 | } else { 256 | console.log(error.toString()); 257 | } 258 | 259 | callback(); 260 | 261 | }); 262 | 263 | } 264 | 265 | function restartServices (mesosCtl, options, callback) { 266 | 267 | // Set Ansible working directory 268 | var ansibleWorkDir = path.join(__dirname, "../", "ansible"); 269 | 270 | // Set MESOSCTL_CONFIGURATION_PATH environment variable for the dynamic inventory 271 | process.env.MESOSCTL_CONFIGURATION_PATH = mesosCtl.currentConfigPath; 272 | 273 | var taskMatcher = /TASK \[(.*)\]/; 274 | var playMatcher = /PLAY \[(.*)\]/; 275 | var storedOutput = ""; 276 | var checkOutput = false; 277 | var playbook = new Ansible.Playbook().playbook(ansibleWorkDir + "/restart-services"); 278 | 279 | // Log the proceeding installation steps 280 | playbook.on('stdout', function(data) { 281 | var output = data.toString(); 282 | var foundPlay = output.match(playMatcher); 283 | var foundTask = output.match(taskMatcher); 284 | var foundInclude = (output.match(/include/) === null ? false : true); 285 | var foundSkipped = (output.match(/skipping/) === null ? false : true); // If skipped, don't show! 286 | 287 | // Log playbook starts 288 | if (foundPlay && foundPlay.length === 2) { 289 | console.log("---------------------------------------------------------------------"); 290 | console.log("Starting play " + foundPlay[1]); 291 | console.log("---------------------------------------------------------------------"); 292 | } 293 | 294 | if (checkOutput) { 295 | if (!foundSkipped && options && options.verbose) { 296 | console.log("Starting task " + storedOutput); 297 | } 298 | checkOutput = false; 299 | } else { 300 | if (foundTask && foundTask.length === 2 && !foundInclude) { 301 | storedOutput = foundTask[1]; 302 | checkOutput = true; 303 | } 304 | } 305 | 306 | }); 307 | 308 | playbook.exec({ cwd: ansibleWorkDir }).then(function(successResult) { 309 | 310 | callback(); 311 | 312 | }, function(error) { 313 | 314 | var fatalGlobalMatcher = /(fatal:.*)/g; 315 | var fatalRealMatcher = /^(fatal)(?!.*?lxc-docker).*$/g; 316 | var globalErrors = error.toString().match(fatalGlobalMatcher); 317 | var realErrors = []; 318 | 319 | // Check if the global error signatures found match the "real" error signatures (omit ignored fatal messages) 320 | if (globalErrors && globalErrors.length > 0) { 321 | globalErrors.forEach(function (globalError) { 322 | var found = globalError.match(fatalRealMatcher); 323 | if (found && found.length > 0) { 324 | realErrors.push(globalError); 325 | } 326 | }); 327 | } 328 | 329 | console.log("---------------------------------------------------------------------"); 330 | console.log("Error(s) occurred:"); 331 | console.log("---------------------------------------------------------------------"); 332 | if (realErrors && realErrors.length > 0) { 333 | console.log(realErrors.join("\n")); 334 | } else { 335 | console.log(error.toString()); 336 | } 337 | 338 | callback(); 339 | 340 | }); 341 | 342 | } 343 | 344 | module.exports = function(vorpal, mesosCtl) { 345 | 346 | vorpal 347 | .command('cluster restart services', 'Restarts the Mesos Master, Agent, ZooKeeper and Marathon services') 348 | .option("--verbose", "Set the verbose logging of the provision process") 349 | .action(function (args, callback) { 350 | 351 | var self = this; 352 | 353 | // Check if the configuration has been provisioned and if it has a valid configuration 354 | if (!mesosCtl.hasValidConfig || !mesosCtl.currentConfigPath) { 355 | 356 | self.log("--> Currently there is no valid configuration loaded! Therefore, the cluster cannot be provisioned!"); 357 | callback(); 358 | 359 | } else { 360 | 361 | // Trigger service restart 362 | restartServices(mesosCtl, args.options, callback); 363 | 364 | } 365 | 366 | }); 367 | 368 | vorpal 369 | .command('cluster provision', 'Provisions the cluster based on the current configuration') 370 | .option("--verbose", "Set the verbose logging of the provision process") 371 | .action(function (args, callback) { 372 | 373 | var self = this; 374 | 375 | // Check if the configuration has been provisioned and if it has a valid configuration 376 | if (!mesosCtl.hasValidConfig || !mesosCtl.currentConfigPath) { 377 | 378 | self.log("--> Currently there is no valid configuration loaded! Therefore, the cluster cannot be provisioned!"); 379 | callback(); 380 | 381 | } else if (mesosCtl.config.provisioned) { 382 | 383 | self.prompt({ 384 | type: 'confirm', 385 | name: 'reprovision', 386 | default: false, 387 | message: 'This configuration has already been provisioned. \nRe-provisioning can have effects on the cluster as well as all possibly running applications. Continue ' 388 | }, function (result) { 389 | 390 | // Check if the cluster should be re-provisioned 391 | if (result.reprovision) { 392 | 393 | // Re-provision 394 | provision(mesosCtl, args.options, callback); 395 | 396 | } else { 397 | 398 | self.log("--> The cluster will not be re-provisioned."); 399 | callback(); 400 | 401 | } 402 | 403 | }); 404 | 405 | } else { 406 | 407 | // Provision 408 | provision(mesosCtl, args.options, callback); 409 | 410 | } 411 | 412 | }); 413 | 414 | vorpal 415 | .command('cluster status', 'Display the cluster status') 416 | .action(function (args, callback) { 417 | var self = this; 418 | 419 | // Check if the configuration has been provisioned and if it has a valid configuration 420 | if (!mesosCtl.hasValidConfig || !mesosCtl.currentConfigPath) { 421 | 422 | self.log("--> Currently there is no valid configuration loaded!"); 423 | callback(); 424 | 425 | } else if (!mesosCtl.config.provisioned) { 426 | 427 | self.log("--> The loaded configuration was not provisioned yet!"); 428 | callback(); 429 | 430 | } else { 431 | 432 | // Get the statistics 433 | getUtilizationStats(mesosCtl.functions.getAgents(), function (error, utilization) { 434 | 435 | if (error) { 436 | self.log(chalk.red(error)); 437 | callback(); 438 | } else { 439 | // Render the stats 440 | renderUtilization(vorpal, utilization.cluster.utilization, "Cluster '" + mesosCtl.config.cluster_name + "' utilization:"); 441 | 442 | // Trigger callback 443 | callback(); 444 | } 445 | 446 | }); 447 | 448 | } 449 | 450 | }); 451 | 452 | vorpal 453 | .command('cluster status agent ', 'Display the Mesos agent status and utilization') 454 | .autocomplete({ 455 | data: function () { 456 | if (!mesosCtl.config.masters || !mesosCtl.config.agents || !mesosCtl.config.registry) { 457 | return []; 458 | } else { 459 | return mesosCtl.functions.getUniqueItemArray(mesosCtl.config.masters.concat(mesosCtl.config.agents, mesosCtl.config.registry)).sort(); 460 | } 461 | } 462 | }) 463 | .action(function (args, callback) { 464 | var self = this; 465 | 466 | if (mesosCtl.config.agents.indexOf(args.agentIPAddress) > -1) { 467 | 468 | // Get the statistics 469 | getUtilizationStats(mesosCtl.config.agents, function (error, utilization) { 470 | 471 | if (error) { 472 | self.log(chalk.bgRed(error)); 473 | } 474 | 475 | // Render the stats 476 | renderUtilization(utilization.agents[args.agentIPAddress].utilization, "Agent " + args.agentIPAddress + " utilization:"); 477 | 478 | // Trigger callback 479 | callback(); 480 | 481 | }); 482 | 483 | } else { 484 | self.log(chalk.red("--> The Mesos agent IP address couldn't be found in the list of configured agents!")); 485 | callback(); 486 | } 487 | 488 | }); 489 | 490 | vorpal 491 | .command('cluster ssh ', 'Issue a SSH command on the remote host') 492 | .autocomplete({ 493 | data: function () { 494 | if (!mesosCtl.config.masters || !mesosCtl.config.agents || !mesosCtl.config.registry) { 495 | return []; 496 | } else { 497 | return mesosCtl.functions.getUniqueItemArray(mesosCtl.config.masters.concat(mesosCtl.config.agents, mesosCtl.config.registry)).sort(); 498 | } 499 | } 500 | }) 501 | .action(function (args, callback) { 502 | var self = this; 503 | 504 | // Check if the configuration has been provisioned and if it has a valid configuration 505 | if (!mesosCtl.hasValidConfig || !mesosCtl.currentConfigPath) { 506 | 507 | self.log("--> Currently there is no valid configuration loaded! Therefore, the command cannot be executed!"); 508 | callback(); 509 | 510 | } else { 511 | 512 | var conn = new sshClient(); 513 | 514 | conn.on('error', function (error) { 515 | self.log("An error occurred: " + JSON.stringify(error)); 516 | callback(); 517 | }).on('ready', function() { 518 | conn.exec(args.command, function(error, stream) { 519 | if (error) { 520 | self.log("--> An error occurred: " + JSON.stringify(error)); 521 | } 522 | stream.on('close', function(code, signal) { 523 | conn.end(); 524 | callback(); 525 | }).on('data', function(data) { 526 | self.log(data.toString()); 527 | }).stderr.on('data', function(data) { 528 | self.log(data.toString()); 529 | }); 530 | }); 531 | }).connect({ 532 | host: args.ipAddress, 533 | port: mesosCtl.config.ssh_port, 534 | username: mesosCtl.config.ssh_user, 535 | privateKey: require('fs').readFileSync(mesosCtl.config.ssh_key_path) 536 | }); 537 | 538 | } 539 | 540 | }); 541 | 542 | vorpal 543 | .command("cluster get leader", "Returns the currently leading Mesos Master's address") 544 | .action(function (args, callback) { 545 | var self = this; 546 | 547 | // Check if the configuration has been provisioned and if it has a valid configuration 548 | if (!mesosCtl.hasValidConfig || !mesosCtl.currentConfigPath) { 549 | 550 | self.log("--> Currently there is no valid configuration loaded! Therefore, the command cannot be executed!"); 551 | callback(); 552 | 553 | } else { 554 | mesosCtl.functions.getLeader(function (error, leaderIP) { 555 | if (error) { 556 | self.log("--> An error occured: " + JSON.stringify(error)); 557 | callback(); 558 | } else { 559 | self.log("--> Current leading Master's address is " + leaderIP); 560 | callback(); 561 | } 562 | }) 563 | } 564 | }); 565 | 566 | return vorpal; 567 | 568 | }; -------------------------------------------------------------------------------- /modules/installed/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mesoshq/mesosctl/c03b0dcf4fe5b3e3b00d586d2369754fc051cee0/modules/installed/.gitkeep -------------------------------------------------------------------------------- /modules/package.js: -------------------------------------------------------------------------------- 1 | var request = require("request"), 2 | MesosDNSAgent = require("mesosdns-http-agent"), 3 | Mustache = require("mustache"), 4 | chalk = require('chalk'); 5 | 6 | // Overwrite Mustache.js HTML escaping 7 | Mustache.escape = function (value) { 8 | return value; 9 | }; 10 | 11 | function getDefaultConfiguration (config) { 12 | 13 | var defaultConfigProperties = {}; 14 | 15 | Object.getOwnPropertyNames(config.properties).forEach(function (propertyType) { 16 | 17 | // Iterate over all properties of propertyType 18 | Object.getOwnPropertyNames(config.properties[propertyType].properties).forEach(function (property) { 19 | 20 | // Create subobject if not yet present 21 | if (!defaultConfigProperties[propertyType]) { 22 | defaultConfigProperties[propertyType] = {}; 23 | } 24 | 25 | if (config.properties[propertyType].properties[property].properties) { 26 | 27 | Object.getOwnPropertyNames(config.properties[propertyType].properties[property].properties).forEach(function (subProperty) { 28 | 29 | // Create subobject if not yet present 30 | if (!defaultConfigProperties[propertyType][property]) { 31 | defaultConfigProperties[propertyType][property] = {}; 32 | } 33 | 34 | var value = config.properties[propertyType].properties[property].properties[subProperty].default; 35 | 36 | // Fix the Mustache.js rendering of empty arrays upfront -> cast to string 37 | if (Array.isArray(value) && value.length === 0) { 38 | value = "[]"; 39 | } 40 | defaultConfigProperties[propertyType][property][subProperty] = value; 41 | 42 | }); 43 | 44 | } else { 45 | 46 | var value = config.properties[propertyType].properties[property].default; 47 | 48 | // Fix the Mustache.js rendering of empty arrays upfront -> cast to string 49 | if (Array.isArray(value) && value.length === 0) { 50 | value = "[]"; 51 | } 52 | 53 | defaultConfigProperties[propertyType][property] = value; 54 | 55 | } 56 | 57 | }); 58 | 59 | }); 60 | 61 | return defaultConfigProperties; 62 | 63 | } 64 | 65 | function installPackage (payload, mesosCtl, cb) { 66 | var options = { 67 | agentClass: MesosDNSAgent, 68 | agentOptions: { 69 | "dnsServers": mesosCtl.functions.getAgents(), 70 | "mesosTLD": ".mesos" 71 | }, 72 | method: "POST", 73 | url: mesosCtl.options.marathonBaseUrl + "/v2/apps", 74 | json: true, 75 | body: payload 76 | }; 77 | 78 | request(options, function(error, response, body) { 79 | if (error || !response) { 80 | cb((error || "An error occurred"), false); 81 | } 82 | if (response && response.statusCode == 201 && !error) { 83 | cb(null, true); 84 | } else { 85 | cb("There was a problem installing the package", false); 86 | } 87 | }); 88 | 89 | } 90 | 91 | function doRequest (requestOptionsObj, dnsServers, callback) { 92 | 93 | var options = { 94 | agentClass: MesosDNSAgent, 95 | agentOptions: { 96 | "dnsServers": dnsServers, 97 | "mesosTLD": ".mesos" 98 | }, 99 | json: true 100 | }; 101 | 102 | Object.getOwnPropertyNames(requestOptionsObj).forEach(function (property) { 103 | options[property] = requestOptionsObj[property]; 104 | }); 105 | 106 | request(options, function(error, response, body) { 107 | if (error || !response) { 108 | callback(error, null); 109 | } 110 | if (response.statusCode < 400 && !error) { 111 | callback(null, body); 112 | } else { 113 | callback("There was a problem in the execution of this command: " + (error ? error : ""), body); 114 | } 115 | }); 116 | 117 | } 118 | 119 | function handleError (error, data) { 120 | console.log("--> " + error + (data && data.message ? data.message + (data.details && data.details[0] && data.details[0].errors && data.details[0].errors[0] ? " (Details: " + data.details[0].errors[0] +")" : "") : "")); 121 | } 122 | 123 | module.exports = function(vorpal, mesosCtl) { 124 | 125 | // Init package index if repository is installed 126 | mesosCtl.functions.initPackageIndex(); 127 | 128 | vorpal 129 | .command('package install ', 'Installs a package') 130 | .option("--config ", "The absolute path to the package's configuration") 131 | .autocomplete(mesosCtl.packages.list) 132 | .action(function(args, callback) { 133 | var self = this; 134 | 135 | if (!mesosCtl.functions.checkRepository()) { 136 | self.log("--> The DC/OS Universe repository has not yet been installed locally! Please run 'repository download'."); 137 | callback(); 138 | } else { 139 | 140 | self.prompt({ 141 | type: 'list', 142 | name: 'packageVersion', 143 | message: 'Please select a version to install: ', 144 | choices: function () { 145 | var versions = []; 146 | Object.getOwnPropertyNames(mesosCtl.packages.map[args.packageName].versions).forEach(function (version) { 147 | versions.push(version); 148 | }); 149 | return versions; 150 | } 151 | }, function(versionResult) { 152 | 153 | // Load the config file 154 | var config = mesosCtl.functions.getPackageFile(args.packageName, mesosCtl.packages.map[args.packageName].versions[versionResult.packageVersion], "config"); 155 | 156 | // Load the package info file 157 | var package = mesosCtl.functions.getPackageFile(args.packageName, mesosCtl.packages.map[args.packageName].versions[versionResult.packageVersion], "package"); 158 | 159 | // Load the Mustache template for Marathon 160 | var marathonTemplate = mesosCtl.functions.getPackageFile(args.packageName, mesosCtl.packages.map[args.packageName].versions[versionResult.packageVersion], "marathon"); 161 | 162 | // View placeholder 163 | var view = {}; 164 | 165 | // Check if custom config has been provided 166 | if (args.options && args.options.config) { 167 | // Check if provided configuration path exists 168 | if (mesosCtl.functions.checkIfConfigurationExists(args.options.config)) { 169 | 170 | var customConfig = JSON.parse(require("fs").readFileSync(args.options.config, "utf8").toString()); 171 | 172 | mesosCtl.functions.isValidSchemaDynamic(config, customConfig, function (error, isValid) { 173 | 174 | if (error) { 175 | self.log("--> An error occurred: " + JSON.stringify(error)); 176 | callback(); 177 | } else { 178 | if (isValid) { 179 | // Set custom configuration as "view" 180 | view = customConfig; 181 | 182 | // Show preInstall Notes if they exist 183 | if (package.preInstallNotes) { 184 | self.log(package.preInstallNotes); 185 | } 186 | 187 | // Add resource information to default configuration 188 | view.resource = mesosCtl.functions.getPackageFile(args.packageName, mesosCtl.packages.map[args.packageName].versions[versionResult.packageVersion], "resource"); 189 | 190 | self.log(JSON.stringify(view)); 191 | self.log(Mustache.render(marathonTemplate, view)); 192 | 193 | // Parse Marathon app template 194 | var payload = JSON.parse(Mustache.render(marathonTemplate, view)); 195 | 196 | // Run package installation 197 | installPackage(payload, mesosCtl, function (error, installationOk) { 198 | if (installationOk) { 199 | // Show postInstall Notes if they exist 200 | if (package.postInstallNotes) { 201 | self.log(package.postInstallNotes + "\n"); 202 | } else { 203 | self.log("--> The package " + args.packageName + " was installed sucessfully!"); 204 | } 205 | 206 | // Check if installedPackages property exists 207 | if (!mesosCtl.config.installedPackages) { 208 | mesosCtl.config.installedPackages = []; 209 | } 210 | 211 | // Store package in installed packages 212 | mesosCtl.config.installedPackages.push(args.packageName); 213 | } else { 214 | self.log("--> The package " + args.packageName + " wasn't installed sucessfully!"); 215 | self.log("--> " + error); 216 | } 217 | 218 | callback(); 219 | 220 | }); 221 | 222 | } else { 223 | self.log("--> The provided configuration is invalid!"); 224 | } 225 | } 226 | 227 | }); 228 | 229 | } else { 230 | self.log("--> The provided path '" + args.options.pathToConfig + "' doesn't exist!"); 231 | callback(); 232 | } 233 | } else { 234 | // Get default configuration 235 | view = getDefaultConfiguration(config); 236 | 237 | // Show preInstall Notes if they exist 238 | if (package.preInstallNotes) { 239 | self.log(package.preInstallNotes); 240 | } 241 | 242 | // Add resource information to default configuration 243 | view.resource = mesosCtl.functions.getPackageFile(args.packageName, mesosCtl.packages.map[args.packageName].versions[versionResult.packageVersion], "resource"); 244 | 245 | // Parse Marathon app template 246 | var payload = JSON.parse(Mustache.render(marathonTemplate, view)); 247 | 248 | // Run package installation 249 | installPackage(payload, mesosCtl, function (error, installationOk) { 250 | if (installationOk) { 251 | // Show postInstall Notes if they exist 252 | if (package.postInstallNotes) { 253 | self.log(package.postInstallNotes + "\n"); 254 | } else { 255 | self.log("--> The package " + args.packageName + " was installed sucessfully!"); 256 | } 257 | 258 | // Check if installedPackages property exists 259 | if (!mesosCtl.config.installedPackages) { 260 | mesosCtl.config.installedPackages = []; 261 | } 262 | 263 | // Store package in installed packages 264 | mesosCtl.config.installedPackages.push(args.packageName); 265 | } else { 266 | self.log("--> The package " + args.packageName + " wasn't installed sucessfully!"); 267 | self.log("--> " + error); 268 | } 269 | 270 | callback(); 271 | 272 | }); 273 | 274 | } 275 | 276 | }); 277 | 278 | 279 | } 280 | 281 | }); 282 | 283 | vorpal 284 | .command('package describe ', 'Displays information about a package') 285 | .autocomplete(mesosCtl.packages.list) 286 | .option("--options ", "Use the package installation options provides by a specific file") 287 | .option("--package-versions", "Show the available package versions") 288 | //.option("--render", "Populate the package's templates with values from package's config.json and potentially a user-supplied configuration") 289 | .action(function(args, callback) { 290 | 291 | var self = this; 292 | 293 | if (!mesosCtl.functions.checkRepository()) { 294 | self.log("--> The DC/OS Universe repository has not yet been installed locally! Please run 'repository install'."); 295 | callback(); 296 | } else { 297 | 298 | if (args.options.hasOwnProperty("package-versions")) { 299 | var header = "The package '" + args.packageName + "' currently has the following versions:"; 300 | self.log(header); 301 | self.log(mesosCtl.functions.rightPad("-", header.length, "-")); 302 | self.log(Object.getOwnPropertyNames(mesosCtl.packages.map[args.packageName].versions).join("\n")); 303 | } else if (args.options.hasOwnProperty("render")) { 304 | // TODO: Implement! 305 | } else { 306 | var header = "Package '" + args.packageName + "':"; 307 | self.log(chalk.green.bold(header)); 308 | self.log(mesosCtl.functions.rightPad("-", header.length, "-")); 309 | self.log(chalk.bold(mesosCtl.functions.rightPad("Description:", 17, " ")) + mesosCtl.packages.map[args.packageName].description); 310 | self.log(chalk.bold(mesosCtl.functions.rightPad("Current version:", 17, " ")) + mesosCtl.packages.map[args.packageName].currentVersion); 311 | self.log(chalk.bold(mesosCtl.functions.rightPad("Tags:", 17, " ")) + mesosCtl.packages.map[args.packageName].tags.join(", ")); 312 | } 313 | 314 | callback(); 315 | 316 | } 317 | 318 | }); 319 | 320 | vorpal 321 | .command('package uninstall ', 'Uninstalls a package') 322 | .autocomplete(mesosCtl.packages.list) 323 | .action(function(args, callback) { 324 | 325 | var self = this; 326 | 327 | if (!mesosCtl.functions.checkRepository()) { 328 | self.log("--> The DC/OS Universe repository has not yet been installed locally! Please run 'repository install'."); 329 | callback(); 330 | } else if (mesosCtl.packages.list.indexOf(args.packageName) === -1) { 331 | self.log("--> An error occurred: The given package name '" + args.packageName + "' cannot be found in the repository. Please check whether the name is correct!"); 332 | callback(); 333 | } else { 334 | 335 | // Load the package info file 336 | var package = mesosCtl.functions.getPackageFile(args.packageName, mesosCtl.packages.map[args.packageName].versions[mesosCtl.packages.map[args.packageName].currentVersion], "package"); 337 | 338 | // Determine if the package contains a framework, or is just as Marathon app 339 | var isFramework = (package && package.hasOwnProperty("framework") ? package["framework"] : false); 340 | 341 | var appRequest = { 342 | url: mesosCtl.options.marathonBaseUrl + "/v2/apps/" + args.packageName, 343 | method: "GET" 344 | }; 345 | 346 | // Check if app exists 347 | doRequest(appRequest, mesosCtl.functions.getAgents(), function (error, appResponse) { 348 | if (error) { 349 | handleError(error, appResponse); 350 | callback(); 351 | } else { 352 | 353 | var appDeleteRequest = { 354 | url: mesosCtl.options.marathonBaseUrl + "/v2/apps/" + args.packageName, 355 | method: "DELETE" 356 | }; 357 | 358 | // If it exists, delete from Marathon 359 | doRequest(appDeleteRequest, mesosCtl.functions.getAgents(), function (error, appDeleteResponse) { 360 | if (error) { 361 | handleError(error, appDeleteResponse); 362 | callback(); 363 | } else { 364 | 365 | self.log("--> The Marathon app of package '" + args.packageName + "' was deleted!"); 366 | 367 | // Check if the package is a framework as well 368 | if (isFramework) { 369 | 370 | var frameworkRequest = { 371 | url: mesosCtl.options.masterBaseUrl + "/frameworks", 372 | method: "GET" 373 | }; 374 | 375 | // Check is framework exists 376 | doRequest(frameworkRequest, mesosCtl.functions.getAgents(), function (error, frameworkResponse) { 377 | if (error) { 378 | handleError(error, frameworkResponse); 379 | callback(); 380 | } else { 381 | 382 | var frameworkId = "", 383 | found = false; 384 | 385 | // Check if package name can be found among the active frameworks 386 | frameworkResponse.frameworks.forEach(function (framework) { 387 | if (framework.name === args.packageName && framework.active) { 388 | frameworkId = framework.id; 389 | found = true; 390 | } 391 | }); 392 | 393 | // If found, delete framework 394 | if (found) { 395 | 396 | var frameworkDeleteRequest = { 397 | url: mesosCtl.options.masterBaseUrl + "/master/teardown", 398 | method: "POST", 399 | form: { 400 | frameworkId: frameworkId 401 | } 402 | }; 403 | 404 | // Delete framework 405 | doRequest(frameworkDeleteRequest, mesosCtl.functions.getAgents(), function (error, frameworkDeleteResponse) { 406 | if (error) { 407 | handleError(error, frameworkDeleteResponse); 408 | callback(); 409 | } else { 410 | self.log("--> The framework of package '" + args.packageName + "' was deleted!"); 411 | self.log("--> The package '" + args.packageName + "' was uninstalled successfully!"); 412 | callback(); 413 | } 414 | }); 415 | 416 | } else { 417 | self.log("--> The package '" + args.packageName + "' was uninstalled successfully!"); 418 | callback(); 419 | } 420 | 421 | } 422 | }); 423 | 424 | } else { 425 | self.log("--> The package '" + args.packageName + "' was uninstalled successfully!"); 426 | callback(); 427 | } 428 | 429 | } 430 | }); 431 | 432 | } 433 | }); 434 | 435 | } 436 | 437 | }); 438 | 439 | vorpal 440 | .command('package search ', 'Searches for packages with specific string') 441 | .action(function(args, callback) { 442 | 443 | var self = this; 444 | 445 | if (!mesosCtl.functions.checkRepository()) { 446 | self.log("--> The DC/OS Universe repository has not yet been installed locally! Please run 'repository install'."); 447 | callback(); 448 | } else { 449 | 450 | var searchResults = mesosCtl.packages.index.search(args.searchString); 451 | var searchResponse = []; 452 | var searchResponseIndex = 1; 453 | 454 | var header = "Found " + searchResults.length + " potential matches:"; 455 | self.log(chalk.bold(header)); 456 | self.log(mesosCtl.functions.rightPad("-", header.length, "-")); 457 | 458 | searchResults.forEach(function (result) { 459 | searchResponse.push(chalk.green.bold(result.ref)); 460 | searchResponse.push(mesosCtl.packages.map[result.ref].description.replace(/\n/g, "")); 461 | searchResponse.push(""); 462 | searchResponseIndex++; 463 | }); 464 | 465 | self.log(searchResponse.join("\n")); 466 | 467 | callback(); 468 | 469 | } 470 | 471 | }); 472 | 473 | return vorpal; 474 | 475 | }; -------------------------------------------------------------------------------- /modules/repository.js: -------------------------------------------------------------------------------- 1 | var path = require("path"), 2 | fs = require("fs"); 3 | 4 | module.exports = function(vorpal, mesosCtl) { 5 | 6 | vorpal 7 | .command("repository install", "Download and installs the current DC/OS Universe repository") 8 | .action(function(args, callback) { 9 | 10 | var self = this; 11 | 12 | if (!mesosCtl.functions.checkRepository()) { 13 | self.log("--> Installing the DC/OS repository locally!"); 14 | mesosCtl.functions.downloadRepository(); 15 | self.log("--> Initializing the local package index!"); 16 | mesosCtl.functions.initPackageIndex(); 17 | self.log("--> Done!"); 18 | callback(); 19 | } else { 20 | self.log("--> The DC/OS Universe repository has already been downloaded. You can update it by running 'repository update'."); 21 | callback(); 22 | } 23 | 24 | }); 25 | 26 | vorpal 27 | .command("repository update", "Updates the local DC/OS Universe repository with the remote") 28 | .action(function(args, callback) { 29 | 30 | var self = this; 31 | 32 | self.log("--> Downloading the latest DC/OS repository!"); 33 | mesosCtl.functions.downloadRepository(); 34 | self.log("--> Updating the local DC/OS repository and package index!"); 35 | mesosCtl.functions.initPackageIndex(); 36 | self.log("--> Done!"); 37 | 38 | callback(); 39 | 40 | }); 41 | 42 | vorpal 43 | .command("repository check", "Checks if the DC/OS Universe repository is installed locally") 44 | .action(function(args, callback) { 45 | 46 | var self = this; 47 | 48 | if (!mesosCtl.functions.checkRepository()) { 49 | self.log("--> The DC/OS Universe repository has not yet been downloaded. Run 'repository install' to retrieve the current version."); 50 | callback(); 51 | } else { 52 | self.log("--> The DC/OS Universe repository has been downloaded. Packages can be installed via 'package install'."); 53 | callback(); 54 | } 55 | 56 | }); 57 | 58 | return vorpal; 59 | 60 | }; -------------------------------------------------------------------------------- /modules/task.js: -------------------------------------------------------------------------------- 1 | var request = require("request"), 2 | AsciiTable = require('ascii-table'), 3 | MesosDNSAgent = require("mesosdns-http-agent"); 4 | 5 | function getSlaveInfo(slaveBaseUrl, cb) { 6 | 7 | var options = { 8 | url: slaveBaseUrl + "/state", 9 | method: "GET", 10 | json: true 11 | }; 12 | 13 | request(options, function (error, response, body) { 14 | if (error) { 15 | cb(error, null); 16 | } else { 17 | cb(null, body); 18 | } 19 | }); 20 | 21 | } 22 | 23 | function getFileList(slaveBaseUrl, path, mesosCtl, cb) { 24 | 25 | var options = { 26 | url: slaveBaseUrl + "/files/browse?path=" + encodeURIComponent(path), 27 | method: "GET", 28 | json: true 29 | }; 30 | 31 | var months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; 32 | 33 | request(options, function (error, response, filesArray) { 34 | if (error) { 35 | cb(error, null); 36 | } else { 37 | 38 | if (response.statusCode === 404) { 39 | cb("The requested path couldn't be found for this executor!", null); 40 | } else if (response.statusCode >= 200 && response.statusCode < 400) { 41 | var resultArray = [], 42 | maxLengths = {}; 43 | 44 | // Get the max lengths 45 | filesArray.forEach(function (fileObj) { 46 | var fileNameArray = fileObj.path.split("\/"), 47 | fileDate = new Date(fileObj.mtime*1000); 48 | fileObj.fileName = fileNameArray[fileNameArray.length-1]; 49 | fileObj.dateString = months[fileDate.getMonth()] + " " + fileDate.getDate() + " " + fileDate.getFullYear() + " " + (fileDate.getHours() < 10 ? "0" + fileDate.getHours() : fileDate.getHours()) + ":" + (fileDate.getMinutes() < 10 ? "0" + fileDate.getMinutes() : fileDate.getMinutes()); 50 | Object.getOwnPropertyNames(fileObj).forEach(function (property) { 51 | if (!maxLengths.hasOwnProperty(property)) { 52 | maxLengths[property] = fileObj[property].toString().length; 53 | } else { 54 | if (maxLengths[property] < fileObj[property].toString().length) { 55 | maxLengths[property] = fileObj[property].toString().length; 56 | } 57 | } 58 | }); 59 | }); 60 | 61 | // Format entries 62 | filesArray.forEach(function (fileObj) { 63 | resultArray.push(fileObj.mode + mesosCtl.functions.leftPad(fileObj.uid, maxLengths.uid+2, " ") + mesosCtl.functions.leftPad(fileObj.gid, maxLengths.gid+2, " ") + mesosCtl.functions.leftPad(fileObj.size, maxLengths.size+2, " ") + mesosCtl.functions.leftPad(fileObj.dateString, maxLengths.dateString+2, " ") + " " + fileObj.fileName); 64 | }); 65 | 66 | cb(null, resultArray); 67 | } else { 68 | cb("An unknown error occurred!", null); 69 | } 70 | 71 | } 72 | }); 73 | 74 | } 75 | 76 | function getFileContents(slaveBaseUrl, path, lines, offset, cb) { 77 | 78 | var options = { 79 | url: slaveBaseUrl + "/files/read?path=" + encodeURIComponent(path) + (offset !== null ? "&offset=" + offset : ""), // + (lines ? "&length=" + lines : "") + 80 | method: "GET", 81 | json: true 82 | }; 83 | 84 | request(options, function (error, response, body) { 85 | if (error) { 86 | cb(error, null); 87 | } else { 88 | 89 | if (response.statusCode === 404) { 90 | cb("The requested file couldn't be found for this executor!", null); 91 | } else if (response.statusCode >= 200 && response.statusCode < 400) { 92 | var data = body.data; 93 | var linesArray = data.split("\n"); 94 | 95 | // Check if it's an array 96 | if (Array.isArray(linesArray)) { 97 | if (linesArray.length <= lines) { 98 | cb(null, linesArray); 99 | } else { 100 | cb(null, linesArray.slice((parseInt(lines)+1)*-1)); 101 | } 102 | 103 | } else { 104 | cb(null, linesArray); 105 | } 106 | } else { 107 | cb("An unknown error occurred!", null); 108 | } 109 | 110 | } 111 | }); 112 | 113 | } 114 | 115 | function getTasks(dnsServers, includeCompletedTasks, searchString, cb) { 116 | 117 | var options = { 118 | agentClass: MesosDNSAgent, 119 | agentOptions: { 120 | "dnsServers": dnsServers, 121 | "mesosTLD": ".mesos" 122 | }, 123 | url: "http://leader.mesos:5050/state" 124 | }; 125 | 126 | try { 127 | request(options, function (error, response, body) { 128 | if (error) { 129 | cb(error, null); 130 | } else { 131 | var data = JSON.parse(body); 132 | var taskMap = {}; 133 | var taskObj = { 134 | slaveMap: {}, 135 | frameworkMap: {}, 136 | taskMap: {} 137 | }; 138 | 139 | // Get slaveMap for faster references 140 | if (data.slaves && data.slaves.length > 0) { 141 | data.slaves.forEach(function (slave) { 142 | var info = slave.pid.split("@"); 143 | taskObj.slaveMap[slave.id] = { 144 | name: info[0], 145 | host: slave.hostname, 146 | port: info[1].split(":")[1] 147 | } 148 | }); 149 | } 150 | 151 | // Create task array 152 | if (data.frameworks && data.frameworks.length > 0) { 153 | data.frameworks.forEach(function (framework) { 154 | taskObj.frameworkMap[framework.id] = framework.name; 155 | framework.tasks.forEach(function (task) { 156 | delete task.statuses; 157 | delete task.discovery; 158 | delete task.container; 159 | taskMap[task.id] = task; 160 | }); 161 | if (includeCompletedTasks) { 162 | framework.completed_tasks.forEach(function (task) { 163 | delete task.statuses; 164 | delete task.discovery; 165 | delete task.container; 166 | taskMap[task.id] = task; 167 | }); 168 | } 169 | }); 170 | } 171 | 172 | // Filter for search string 173 | Object.getOwnPropertyNames(taskMap).forEach(function (taskId) { 174 | var task = taskMap[taskId]; 175 | // Only filter is searchString is actually not null 176 | if (searchString) { 177 | var searchObj = new RegExp(searchString, "gi"); 178 | if (searchObj.test(task.id)) { 179 | taskObj.taskMap[taskId] = task; 180 | } 181 | } else { 182 | taskObj.taskMap[taskId] = task; 183 | } 184 | 185 | }); 186 | 187 | cb(null, taskObj); 188 | } 189 | 190 | }); 191 | } catch (error) { 192 | cb(error, null); 193 | } 194 | 195 | } 196 | 197 | module.exports = function(vorpal, mesosCtl) { 198 | 199 | vorpal 200 | .command("task list", "Lists all task in the cluster") 201 | .option("--completed", "Print completed and in-progress tasks") 202 | .option("--json", "Print JSON-formatted list of tasks") 203 | .action(function(args, callback) { 204 | 205 | var self = this, 206 | includeCompletedTasks = (args.options.completed ? true : false); 207 | 208 | getTasks(mesosCtl.functions.getAgents(), includeCompletedTasks, null, function (error, tasksObj) { 209 | 210 | if (error) { 211 | 212 | self.log("--> An error occurred: " + error); 213 | callback(); 214 | 215 | } else { 216 | 217 | if (args.options.json) { 218 | 219 | Object.getOwnPropertyNames(tasksObj.taskMap).forEach(function (taskId) { 220 | self.log("--> Found task " + taskId); 221 | self.log(JSON.stringify(tasksObj.taskMap[taskId])); 222 | }); 223 | 224 | callback(); 225 | 226 | } else { 227 | 228 | var table = new AsciiTable(); 229 | table.setHeading("Task ID", "Task Name", "Framework Name", "Slave Name", "CPUs", "Memory", "Disk", "Port(s)", "State"); 230 | 231 | Object.getOwnPropertyNames(tasksObj.taskMap).forEach(function (taskId) { 232 | var task = tasksObj.taskMap[taskId]; 233 | table.addRow(task.id, task.name, tasksObj.frameworkMap[task.framework_id], tasksObj.slaveMap[task.slave_id].name+"@"+tasksObj.slaveMap[task.slave_id].host+":"+tasksObj.slaveMap[task.slave_id].port, task.resources.cpus, task.resources.mem, task.resources.disk, task.resources.ports.replace("[", "").replace("]", ""), task.state); 234 | }); 235 | 236 | self.log(table.toString()); 237 | 238 | callback(); 239 | 240 | } 241 | 242 | } 243 | 244 | }); 245 | 246 | }); 247 | 248 | vorpal 249 | .command("task show ", "Retrieves information about a task") 250 | .option("--completed", "Print completed and in-progress tasks") 251 | .option("--json", "Print JSON-formatted list of tasks") 252 | .action(function(args, callback) { 253 | 254 | var self = this, 255 | includeCompletedTasks = (args.options.completed ? true : false); 256 | 257 | getTasks(mesosCtl.functions.getAgents(), includeCompletedTasks, args.task, function (error, tasksObj) { 258 | 259 | if (error) { 260 | 261 | self.log("--> An error occurred: " + error); 262 | callback(); 263 | 264 | } else { 265 | 266 | if (args.options.json) { 267 | 268 | Object.getOwnPropertyNames(tasksObj.taskMap).forEach(function (taskId) { 269 | self.log("--> Found task " + taskId); 270 | self.log(JSON.stringify(tasksObj.taskMap[taskId])); 271 | }); 272 | 273 | callback(); 274 | 275 | } else { 276 | 277 | var table = new AsciiTable(); 278 | table.setHeading("Task ID", "Task Name", "Framework Name", "Slave Name", "CPUs", "Memory", "Disk", "Port(s)", "State"); 279 | 280 | Object.getOwnPropertyNames(tasksObj.taskMap).forEach(function (taskId) { 281 | var task = tasksObj.taskMap[taskId]; 282 | table.addRow(task.id, task.name, tasksObj.frameworkMap[task.framework_id], tasksObj.slaveMap[task.slave_id].name+"@"+tasksObj.slaveMap[task.slave_id].host+":"+tasksObj.slaveMap[task.slave_id].port, task.resources.cpus, task.resources.mem, task.resources.disk, task.resources.ports.replace("[", "").replace("]", ""), task.state); 283 | }); 284 | 285 | self.log(table.toString()); 286 | 287 | callback(); 288 | 289 | } 290 | 291 | } 292 | 293 | }); 294 | 295 | }); 296 | 297 | vorpal 298 | .command("task log [file]", "Print the task log. By default, the 10 most recent task logs from stdout are printed.") 299 | .option("--completed", "Print completed and in-progress tasks") 300 | .option("--json", "Print JSON-formatted list of tasks") 301 | .option("--lines ", "Print the last N lines. The default is 10 lines.") 302 | .action(function(args, callback) { 303 | 304 | var self = this, 305 | includeCompletedTasks = (args.options.completed ? true : false), 306 | taskId = args.taskId.trim(), 307 | fileName = (args.file ? args.file : "stdout"), 308 | lines = (args.options && args.options.lines ? parseInt(args.options.lines) : 10); 309 | 310 | // Get the tasks 311 | getTasks(mesosCtl.functions.getAgents(), includeCompletedTasks, null, function (error, tasksObj) { 312 | 313 | if (error) { 314 | 315 | self.log("--> An error occurred: " + error); 316 | callback(); 317 | 318 | } else { 319 | 320 | // Check if we have a matching taskId 321 | if (Object.getOwnPropertyNames(tasksObj.taskMap).indexOf(taskId) === -1) { 322 | self.log("--> The task with the id " + args.taskId + " couldn't be found!"); 323 | callback(); 324 | } else { 325 | 326 | var slaveId = tasksObj.taskMap[taskId].slave_id; 327 | var frameworkId = tasksObj.taskMap[taskId].framework_id; 328 | var slaveBaseUrl = "http://" + tasksObj.slaveMap[slaveId].host + ":" + tasksObj.slaveMap[slaveId].port; 329 | 330 | // Get slave info 331 | getSlaveInfo(slaveBaseUrl + "/" + tasksObj.slaveMap[slaveId].name, function (error, slaveInfoObj) { 332 | 333 | if (error) { 334 | 335 | self.log("--> An error occurred: " + error); 336 | callback(); 337 | 338 | } else { 339 | 340 | // Iterate over frameworks 341 | slaveInfoObj.frameworks.forEach(function (frameworkObj) { 342 | 343 | // Match the frameworkId 344 | if (frameworkObj.id === frameworkId) { 345 | 346 | // Iterate over the executors 347 | frameworkObj.executors.forEach(function (executor) { 348 | 349 | // Match the taskId 350 | if (executor.source === taskId) { 351 | 352 | // Get the file contents if we have a match 353 | getFileContents(slaveBaseUrl, executor.directory + "/" + fileName, lines, 0, function (error, fileContentArray) { 354 | 355 | if (error) { 356 | 357 | self.log("--> An error occurred: " + error); 358 | callback(); 359 | 360 | } else { 361 | 362 | self.log("--> Displaying the contents of file '" + fileName + "':"); 363 | self.log(fileContentArray.join("\n")); 364 | callback(); 365 | 366 | } 367 | 368 | }); 369 | 370 | } 371 | 372 | }); 373 | } 374 | 375 | }); 376 | 377 | } 378 | 379 | }); 380 | 381 | } 382 | 383 | } 384 | 385 | }); 386 | 387 | }); 388 | 389 | vorpal 390 | .command("task ls [path]", "Print the list of files in the Mesos task sandbox") 391 | .action(function(args, callback) { 392 | 393 | var self = this, 394 | taskId = args.taskId; 395 | 396 | // Get the tasks 397 | getTasks(mesosCtl.functions.getAgents(), false, null, function (error, tasksObj) { 398 | 399 | if (error) { 400 | 401 | self.log("An error occurred: " + error); 402 | callback(); 403 | 404 | } else { 405 | 406 | // Check if we have a matching taskId 407 | if (Object.getOwnPropertyNames(tasksObj.taskMap).indexOf(taskId) === -1) { 408 | self.log("The task with the id " + args.taskId + " couldn't be found!"); 409 | callback(); 410 | } else { 411 | 412 | var slaveId = tasksObj.taskMap[taskId].slave_id; 413 | var frameworkId = tasksObj.taskMap[taskId].framework_id; 414 | var slaveBaseUrl = "http://" + tasksObj.slaveMap[slaveId].host + ":" + tasksObj.slaveMap[slaveId].port; 415 | 416 | // Get slave info 417 | getSlaveInfo(slaveBaseUrl + "/" + tasksObj.slaveMap[slaveId].name, function (error, slaveInfoObj) { 418 | 419 | if (error) { 420 | 421 | self.log("An error occurred: " + error); 422 | callback(); 423 | 424 | } else { 425 | 426 | // Iterate over frameworks 427 | slaveInfoObj.frameworks.forEach(function (frameworkObj) { 428 | 429 | // Match the frameworkId 430 | if (frameworkObj.id === frameworkId) { 431 | 432 | // Iterate over the executors 433 | frameworkObj.executors.forEach(function (executor) { 434 | 435 | // Match the taskId 436 | if (executor.source === taskId) { 437 | 438 | // Get the file contents if we have a match 439 | getFileList(slaveBaseUrl, executor.directory, mesosCtl, function (error, filesArray) { 440 | 441 | if (error) { 442 | 443 | self.log("An error occurred: " + error); 444 | callback(); 445 | 446 | } else { 447 | 448 | //self.log("Displaying the contents of file '" + fileName + "':"); 449 | self.log(filesArray.join("\n")); 450 | callback(); 451 | 452 | } 453 | 454 | }); 455 | 456 | } 457 | 458 | }); 459 | } 460 | 461 | }); 462 | 463 | } 464 | 465 | }); 466 | 467 | } 468 | 469 | } 470 | 471 | }); 472 | 473 | }); 474 | 475 | return vorpal; 476 | 477 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mesosctl", 3 | "version": "0.1.10", 4 | "description": "A command-line tool to dynamically provision and manage Mesos clusters and their applications", 5 | "main": "bin/mesosctl.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [ 10 | "mesos", 11 | "cli", 12 | "dcos", 13 | "mesosctl", 14 | "cluster", 15 | "ansible", 16 | "devops", 17 | "apps", 18 | "automation", 19 | "paas" 20 | ], 21 | "author": { 22 | "name": "TobiLG", 23 | "email": "tobilg@gmail.com", 24 | "url": "https://github.com/tobilg" 25 | }, 26 | "bin": { 27 | "mesosctl": "bin/mesosctl.js" 28 | }, 29 | "license": "MIT", 30 | "dependencies": { 31 | "ajv": "^4.6.0", 32 | "ascii-table": "0.0.9", 33 | "async": "^2.0.1", 34 | "chalk": "^1.1.3", 35 | "lunr": "^0.7.0", 36 | "js-yaml": "^3.6.1", 37 | "mesosdns-http-agent": "^0.2.2", 38 | "mustache": "^2.2.1", 39 | "node-ansible": "^0.5.5", 40 | "password-hash": "^1.2.2", 41 | "progress": "^1.1.8", 42 | "request": "^2.74.0", 43 | "ssh2": "^0.5.1", 44 | "unzip": "^0.1.11", 45 | "vorpal": "^1.11.4" 46 | }, 47 | "repository": { 48 | "type": "git", 49 | "url": "https://github.com/mesoshq/mesosctl.git" 50 | }, 51 | "bugs": { 52 | "url": "https://github.com/mesoshq/mesosctl/issues" 53 | } 54 | } 55 | --------------------------------------------------------------------------------