├── scripts
    ├── node-attach
    ├── kibana-attach
    ├── logstash-attach
    ├── node-restart
    ├── kibana-restart
    ├── logstash-restart
    ├── kibana-status
    ├── node-status
    ├── logstash-status
    ├── kibana-stop
    ├── node-stop
    ├── logstash-stop
    ├── kibana-start
    ├── logstash-start
    └── node-start
├── .gitignore
├── lib
    ├── kibana-script.rb
    ├── logstash-script.rb
    ├── elasticsearch-script.rb
    ├── upgrade-kibana.sh
    ├── upgrade-logstash.sh
    ├── upgrade-es.sh
    └── elasticsearch-module.rb
├── conf
    ├── logstash.conf.erb
    └── elasticsearch.yml.erb
├── Vagrantfile
└── README.md


/scripts/node-attach:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | screen -rx elastic
4 | 


--------------------------------------------------------------------------------
/scripts/kibana-attach:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | screen -rx kibana
4 | 


--------------------------------------------------------------------------------
/scripts/logstash-attach:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | screen -rx logstash
4 | 


--------------------------------------------------------------------------------
/scripts/node-restart:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | node-stop
4 | node-start
5 | 


--------------------------------------------------------------------------------
/scripts/kibana-restart:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | kibana-stop
4 | kibana-start
5 | 


--------------------------------------------------------------------------------
/scripts/logstash-restart:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | logstash-stop
4 | logstash-start
5 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .vagrant/
2 | conf/*.yml
3 | conf/*.conf
4 | logs/
5 | data/
6 | *.rpm
7 | *.gz
8 | 


--------------------------------------------------------------------------------
/scripts/kibana-status:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep kibana | sed 's/\s/ /g')
 4 | 
 5 | if [ -z "$INSTANCE" ]
 6 | then
 7 |     echo "No running instance"
 8 | else
 9 |     echo $INSTANCE
10 | fi
11 | 


--------------------------------------------------------------------------------
/scripts/node-status:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep elastic | sed 's/\s/ /g')
 4 | 
 5 | if [ -z "$INSTANCE" ]
 6 | then
 7 |     echo "No running instance"
 8 | else
 9 |     echo $INSTANCE
10 | fi
11 | 


--------------------------------------------------------------------------------
/scripts/logstash-status:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep logstash | sed 's/\s/ /g')
 4 | 
 5 | if [ -z "$INSTANCE" ]
 6 | then
 7 |     echo "No running instance"
 8 | else
 9 |     echo $INSTANCE
10 | fi
11 | 


--------------------------------------------------------------------------------
/scripts/kibana-stop:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep kibana | sed 's/\s/ /g')
 4 | 
 5 | if [ -n "$INSTANCE" ]
 6 | then
 7 |     screen -X -S kibana quit
 8 |     echo "Killed $INSTANCE"
 9 | else
10 |     echo "No running instance"
11 | fi
12 | 


--------------------------------------------------------------------------------
/scripts/node-stop:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep elastic | sed 's/\s/ /g')
 4 | 
 5 | if [ -n "$INSTANCE" ]
 6 | then
 7 |     screen -X -S elastic quit
 8 |     echo "Killed $INSTANCE"
 9 | else
10 |     echo "No running instance"
11 | fi
12 | 


--------------------------------------------------------------------------------
/scripts/logstash-stop:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep logstash | sed 's/\s/ /g')
 4 | 
 5 | if [ -n "$INSTANCE" ]
 6 | then
 7 |     screen -X -S logstash quit
 8 |     echo "Killed $INSTANCE"
 9 | else
10 |     echo "No running instance"
11 | fi
12 | 


--------------------------------------------------------------------------------
/lib/kibana-script.rb:
--------------------------------------------------------------------------------
 1 | @kibana_start_inline_script = <<SCRIPT
 2 | if ! cat /etc/profile | grep -q vagrant
 3 | then
 4 |     cat <<EOT >> /etc/profile.d/vagrant-elasticsearch-kibana.sh
 5 | 
 6 | export VM_NAME=%s
 7 | export VM_NODE_NAME=%s
 8 | export VM_NODE_IP=%s
 9 | export PATH=/vagrant/scripts:/home/vagrant/kibana/bin:\\$PATH
10 | EOT
11 | 
12 |     sed 's#^.*secure_path="\\(.*\\)"$#Defaults secure_path="\\1:/vagrant/scripts:/home/vagrant/kibana/bin"#' -i /etc/sudoers
13 |     echo 'Defaults env_keep = "VM_NAME"' >> /etc/sudoers
14 | 
15 |     source /etc/profile
16 | fi
17 | 
18 | screen -li | grep -q kibana || kibana-start
19 | SCRIPT
20 | 


--------------------------------------------------------------------------------
/lib/logstash-script.rb:
--------------------------------------------------------------------------------
 1 | @logstash_start_inline_script = <<SCRIPT
 2 | if ! cat /etc/profile | grep -q vagrant
 3 | then
 4 |     cat <<EOT >> /etc/profile.d/vagrant-elasticsearch-logstash.sh
 5 | 
 6 | export VM_NAME=%s
 7 | export VM_NODE_NAME=%s
 8 | export VM_NODE_IP=%s
 9 | export PATH=/vagrant/scripts:/home/vagrant/logstash/bin:\\$PATH
10 | EOT
11 | 
12 |     sed 's#^.*secure_path="\\(.*\\)"$#Defaults secure_path="\\1:/vagrant/scripts:/home/vagrant/logstash/bin"#' -i /etc/sudoers
13 |     echo 'Defaults env_keep = "VM_NAME"' >> /etc/sudoers
14 | 
15 |     source /etc/profile
16 | fi
17 | 
18 | screen -li | grep -q logstash || logstash-start
19 | SCRIPT
20 | 


--------------------------------------------------------------------------------
/lib/elasticsearch-script.rb:
--------------------------------------------------------------------------------
 1 | @node_start_inline_script = <<SCRIPT
 2 | if ! cat /etc/profile | grep -q vagrant
 3 | then
 4 |     cat <<EOT >> /etc/profile.d/vagrant-elasticsearch-cluster.sh
 5 | 
 6 | export VM_NAME=%s
 7 | export VM_NODE_NAME=%s
 8 | export VM_NODE_IP=%s
 9 | export PATH=/vagrant/scripts:/home/vagrant/elasticsearch/bin:\\$PATH
10 | EOT
11 | 
12 |     sed 's#^.*secure_path="\\(.*\\)"$#Defaults secure_path="\\1:/vagrant/scripts:/home/vagrant/elasticsearch/bin"#' -i /etc/sudoers
13 |     echo 'Defaults env_keep = "VM_NAME"' >> /etc/sudoers
14 | 
15 |     source /etc/profile
16 | fi
17 | 
18 | screen -li | grep -q elastic || node-start
19 | SCRIPT
20 | 


--------------------------------------------------------------------------------
/scripts/kibana-start:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep kibana | sed 's/\s/ /g')
 4 | 
 5 | if [ -n "$INSTANCE" ]
 6 | then
 7 |     echo "Already running: $INSTANCE"
 8 | else
 9 |     mkdir -p /vagrant/logs
10 | 
11 |     screen -S kibana -d -m bash -l -c "su - vagrant -c \"kibana serve > /vagrant/logs/kibana-$VM_NAME.log 2>&1\""
12 |     echo "-----------------------------------------------------------------------------------------------------------"
13 |     echo " => Started $VM_NAME - $VM_NODE_NAME - $VM_NODE_IP: $(screen -li | grep kibana | sed 's/\s/ /g')"
14 |     echo "-----------------------------------------------------------------------------------------------------------"
15 | fi
16 | 


--------------------------------------------------------------------------------
/scripts/logstash-start:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep logstash | sed 's/\s/ /g')
 4 | 
 5 | if [ -n "$INSTANCE" ]
 6 | then
 7 |     echo "Already running: $INSTANCE"
 8 | else
 9 |     mkdir -p /vagrant/logs
10 | 
11 |     screen -S logstash -d -m bash -l -c "su - vagrant -c \"LS_JAVA_OPTS='-Djava.net.preferIPv4Stack=true' logstash -f /vagrant/conf/logstash-${VM_NAME}.conf > /vagrant/logs/logstash-${VM_NAME}.log 2>&1\""
12 |     echo "-----------------------------------------------------------------------------------------------------------"
13 |     echo " => Started $VM_NAME - $VM_NODE_NAME - $VM_NODE_IP: $(screen -li | grep logstash | sed 's/\s/ /g')"
14 |     echo "-----------------------------------------------------------------------------------------------------------"
15 | fi
16 | 


--------------------------------------------------------------------------------
/conf/logstash.conf.erb:
--------------------------------------------------------------------------------
 1 | input { 
 2 |   tcp {
 3 |     port => 5514
 4 |     type => syslog
 5 |   }
 6 |   udp {
 7 |     port => 5514
 8 |     type => syslog
 9 |   }
10 | }
11 | 
12 | filter {
13 |   if [type] == "syslog" {
14 |     grok {
15 |       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
16 |       add_field => [ "received_at", "%{@timestamp}" ]
17 |       add_field => [ "received_from", "%{host}" ]
18 |     }
19 |     syslog_pri { }
20 |     date {
21 |       match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
22 |     }
23 |   }
24 | }
25 | 
26 | output {
27 |     elasticsearch {
28 |         hosts => [ "<%= @node01_ip.strip %>:9200" ]
29 |         sniffing => true
30 |     }
31 | }
32 | 


--------------------------------------------------------------------------------
/scripts/node-start:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | INSTANCE=$(screen -li | grep elastic | sed 's/\s/ /g')
 4 | 
 5 | if [ -n "$INSTANCE" ]
 6 | then
 7 |     echo "Already running: $INSTANCE"
 8 | else
 9 |     mkdir -p /vagrant/logs
10 | 
11 |     rm -f elasticsearch/config/elasticsearch.yml
12 |     cp /vagrant/conf/elasticsearch-$VM_NAME.yml elasticsearch/config/elasticsearch.yml
13 |     chown vagrant: elasticsearch/config/elasticsearch.yml
14 | 
15 |     screen -S elastic -d -m bash -l -c "su - vagrant -c \"ES_JAVA_OPTS='-Djava.net.preferIPv4Stack=true'  elasticsearch > /vagrant/logs/elasticsearch-$VM_NAME.log 2>&1\""
16 |     echo "-----------------------------------------------------------------------------------------------------------"
17 |     echo " => Started $VM_NAME - $VM_NODE_NAME - $VM_NODE_IP: $(screen -li | grep elastic | sed 's/\s/ /g')"
18 |     echo "-----------------------------------------------------------------------------------------------------------"
19 | fi
20 | 


--------------------------------------------------------------------------------
/lib/upgrade-kibana.sh:
--------------------------------------------------------------------------------
 1 | yum -q -y install screen
 2 | yum -q -y install wget
 3 | 
 4 | # Setting ES version to install
 5 | KIBANA_VERSION="kibana-4.6.1-linux-x86_64"
 6 | KIBANA_PLUGIN_INSTALL_CMD="kibana/bin/kibana plugin --install"
 7 | 
 8 | # Removing all previous potentially installed version
 9 | rm -rf kibana
10 | rm -rf kibana-*
11 | 
12 | # Downloading the version to install
13 | if [ ! -f "/vagrant/$KIBANA_VERSION.tar.gz" ]; then
14 |     wget -q https://download.elastic.co/kibana/kibana/${KIBANA_VERSION}.tar.gz
15 |     tar -zxf $KIBANA_VERSION.tar.gz
16 |     rm -rf $KIBANA_VERSION.tar.gz
17 | else
18 |     tar -zxf /vagrant/$KIBANA_VERSION.tar.gz
19 | fi
20 | 
21 | # Renaming extracted folder to a generic name to avoid changing commands 
22 | mv $KIBANA_VERSION kibana
23 | ${KIBANA_PLUGIN_INSTALL_CMD} elastic/sense
24 | 
25 | chown -R vagrant: kibana
26 | 
27 | firewall-cmd --zone=public --add-port=9200/tcp --permanent
28 | firewall-cmd --zone=public --add-port=9300/tcp --permanent
29 | firewall-cmd --zone=public --add-port=5601/tcp --permanent
30 | systemctl stop firewalld
31 | systemctl start firewalld
32 | 


--------------------------------------------------------------------------------
/lib/upgrade-logstash.sh:
--------------------------------------------------------------------------------
 1 | yum -q -y install screen
 2 | yum -q -y install wget
 3 | 
 4 | # Install JAVA
 5 | if [ ! -f "/vagrant/jdk-8u101-linux-x64.rpm" ]; then
 6 |     curl --silent -L --cookie "oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u101-b13/jdk-8u101-linux-x64.rpm -o /vagrant/jdk-8u101-linux-x64.rpm --retry 999 --retry-max-time 0 -C -
 7 | fi
 8 | yum -q -y localinstall /vagrant/jdk-8u101-linux-x64.rpm
 9 | 
10 | # Setting ES version to install
11 | LOGSTASH_VERSION="logstash-2.4.0"
12 | 
13 | # Removing all previous potentially installed version
14 | rm -rf logstash
15 | rm -rf logstash-*
16 | 
17 | # Downloading the version to install
18 | if [ ! -f "/vagrant/$LOGSTASH_VERSION.tar.gz" ]; then
19 |     wget -q https://download.elastic.co/logstash/logstash/${LOGSTASH_VERSION}.tar.gz
20 |     tar -zxf $LOGSTASH_VERSION.tar.gz
21 |     rm -rf $LOGSTASH_VERSION.tar.gz
22 | else
23 |     tar -zxf /vagrant/$LOGSTASH_VERSION.tar.gz
24 | fi
25 | 
26 | # Renaming extracted folder to a generic name to avoid changing commands 
27 | mv $LOGSTASH_VERSION logstash
28 | 
29 | chown -R vagrant: logstash
30 | 
31 | firewall-cmd --zone=public --add-port=5514/tcp --permanent
32 | firewall-cmd --zone=public --add-port=5514/udp --permanent
33 | systemctl stop firewalld
34 | systemctl start firewalld
35 | 


--------------------------------------------------------------------------------
/lib/upgrade-es.sh:
--------------------------------------------------------------------------------
 1 | yum -q -y install screen
 2 | yum -q -y install wget
 3 | 
 4 | # Install JAVA
 5 | if [ ! -f "/vagrant/jdk-8u101-linux-x64.rpm" ]; then
 6 |     curl --silent -L --cookie "oraclelicense=accept-securebackup-cookie" \
 7 |       http://download.oracle.com/otn-pub/java/jdk/8u101-b13/jdk-8u101-linux-x64.rpm \
 8 |       -o /vagrant/jdk-8u101-linux-x64.rpm --retry 999 --retry-max-time 0 -C -
 9 | fi
10 | yum -q -y localinstall /vagrant/jdk-8u101-linux-x64.rpm
11 | 
12 | # Setting ES version to install
13 | ES_VERSION="2.4.1"
14 | ES_TGZ="elasticsearch-${ES_VERSION}.tar.gz"
15 | ES_DIR="elasticsearch-${ES_VERSION}"
16 | ES_VERSION_URL="https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${ES_VERSION}/${ES_TGZ}"
17 | ES_PLUGIN_INSTALL_CMD="elasticsearch/bin/plugin install"
18 | 
19 | # Removing all previous potentially installed version
20 | rm -rf elasticsearch
21 | rm -rf elasticsearch-*
22 | 
23 | # Downloading the version to install
24 | if [ ! -f "/vagrant/$ES_TGZ" ]; then
25 |     wget -q $ES_VERSION_URL
26 |     tar -zxf $ES_TGZ
27 |     rm -rf $ES_TGZ
28 | else
29 |     tar -zxf /vagrant/$ES_TGZ
30 | fi
31 | 
32 | # Renaming extracted folder to a generic name to avoid changing ES commands (elasticsearch/bin/...)
33 | mv $ES_DIR elasticsearch
34 | 
35 | #############
36 | # Plugins
37 | 
38 | ESPLUGINS=()
39 | 
40 | ESPLUGINS+=(analysis-icu)
41 | #ESPLUGINS+=(mapper-attachments)
42 | 
43 | #Monitoring
44 | ESPLUGINS+=(mobz/elasticsearch-head)
45 | ESPLUGINS+=(lmenezes/elasticsearch-kopf)
46 | ESPLUGINS+=(karmi/elasticsearch-paramedic)
47 | ESPLUGINS+=(royrusso/elasticsearch-HQ)
48 | 
49 | #ESPLUGINS+=(lang-javascript)
50 | 
51 | for P in ${ESPLUGINS[*]}
52 | do
53 | 	${ES_PLUGIN_INSTALL_CMD} $P
54 | done
55 | 
56 | chown -R vagrant: elasticsearch
57 | 
58 | firewall-cmd --zone=public --add-port=9200/tcp --permanent
59 | firewall-cmd --zone=public --add-port=9300/tcp --permanent
60 | systemctl stop firewalld
61 | systemctl start firewalld
62 | 


--------------------------------------------------------------------------------
/conf/elasticsearch.yml.erb:
--------------------------------------------------------------------------------
  1 | # ======================== Elasticsearch Configuration =========================
  2 | #
  3 | # NOTE: Elasticsearch comes with reasonable defaults for most settings.
  4 | #       Before you set out to tweak and tune the configuration, make sure you
  5 | #       understand what are you trying to accomplish and the consequences.
  6 | #
  7 | # The primary way of configuring a node is via this file. This template lists
  8 | # the most important settings you may want to configure for a production cluster.
  9 | #
 10 | # Please see the documentation for further information on configuration options:
 11 | # <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
 12 | #
 13 | # ---------------------------------- Cluster -----------------------------------
 14 | #
 15 | # Use a descriptive name for your cluster:
 16 | #
 17 | cluster.name: "<%= @cluster_name.strip %>"
 18 | #
 19 | # ------------------------------------ Node ------------------------------------
 20 | #
 21 | # Use a descriptive name for the node:
 22 | #
 23 | node.name: "<%= @node_name.strip %>"
 24 | # Every node can be configured to allow or deny being eligible as the master,
 25 | # and to allow or deny to store the data.
 26 | #
 27 | # Allow this node to be eligible as a master node (enabled by default):
 28 | #
 29 | node.master: <%= @node_master %>
 30 | #
 31 | # Allow this node to store data (enabled by default):
 32 | #
 33 | node.data: <%= @node_data %>
 34 | 
 35 | #
 36 | # Add custom attributes to the node:
 37 | #
 38 | # node.rack: r1
 39 | #
 40 | index.number_of_shards: 3
 41 | index.number_of_replicas: 2
 42 | # ----------------------------------- Paths ------------------------------------
 43 | #
 44 | # Path to directory where to store the data (separate multiple locations by comma):
 45 | #
 46 | path.data: /vagrant/data/<%= @vm_name.strip %>
 47 | #
 48 | # Path to log files:
 49 | #
 50 | path.logs: /vagrant/logs/<%= @vm_name.strip %>
 51 | #
 52 | # ----------------------------------- Memory -----------------------------------
 53 | #
 54 | # Lock the memory on startup:
 55 | #
 56 | # bootstrap.mlockall: true
 57 | #
 58 | # Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
 59 | # available on the system and that the owner of the process is allowed to use this limit.
 60 | #
 61 | # Elasticsearch performs poorly when the system is swapping the memory.
 62 | #
 63 | # ---------------------------------- Network -----------------------------------
 64 | #
 65 | # Set the bind adress to a specific IP (IPv4 or IPv6):
 66 | #
 67 | network.bind_host: 0.0.0.0
 68 | network.publish_host: <%= @node_ip.strip %>
 69 | #
 70 | # Set a custom port for HTTP:
 71 | #
 72 | # http.port: 9200
 73 | #
 74 | # For more information, see the documentation at:
 75 | # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
 76 | #
 77 | # ---------------------------------- Gateway -----------------------------------
 78 | #
 79 | # Block initial recovery after a full cluster restart until N nodes are started:
 80 | #
 81 | # gateway.recover_after_nodes: 3
 82 | #
 83 | # For more information, see the documentation at:
 84 | # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
 85 | #
 86 | # --------------------------------- Discovery ----------------------------------
 87 | #
 88 | # Elasticsearch nodes will find each other via unicast, by default.
 89 | #
 90 | # Pass an initial list of hosts to perform discovery when new node is started:
 91 | # The default list of hosts is ["127.0.0.1", "[::1]"]
 92 | #
 93 | discovery.zen.ping.unicast.hosts: ["<%= @node01_ip.strip %>" ]
 94 | #
 95 | # Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
 96 | #
 97 | # discovery.zen.minimum_master_nodes: 3
 98 | #
 99 | # For more information, see the documentation at:
100 | # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
101 | #
102 | # ---------------------------------- Various -----------------------------------
103 | #
104 | # Disable starting multiple nodes on a single system:
105 | #
106 | # node.max_local_storage_nodes: 1
107 | #
108 | # Require explicit names when deleting indices:
109 | #
110 | # action.destructive_requires_name: true
111 | 
112 | # Disk threshold allocation can perform poorly on vagrant when the user doesn't
113 | # have much disk space
114 | cluster.routing.allocation.disk.threshold_enabled: false
115 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
  1 | # -*- mode: ruby -*-
  2 | # vi: set ft=ruby :
  3 | require 'erb'
  4 | require_relative 'lib/elasticsearch-module.rb'
  5 | require_relative 'lib/elasticsearch-script.rb'
  6 | require_relative 'lib/kibana-script.rb'
  7 | require_relative 'lib/logstash-script.rb'
  8 | 
  9 | utils = Vagrant::ElastiSearchCluster::Util.new
 10 | 
 11 | Vagrant.configure("2") do |config|
 12 | 
 13 |     utils.manage_and_print_config
 14 | 
 15 |     nodes_number = utils.get_cluster_info 'cluster_count'
 16 |     nodes_number = nodes_number.to_i
 17 | 
 18 |     cluster_ram = utils.get_cluster_info 'cluster_ram'
 19 |     cluster_ram = cluster_ram.to_i
 20 | 
 21 |     cluster_cpu = utils.get_cluster_info 'cluster_cpu'
 22 |     cluster_cpu = cluster_cpu.to_i
 23 | 
 24 |     config.vm.box = 'bento/centos-7.1'
 25 |     #config.vm.box = 'bhaskarvk/centos7-x86_64'
 26 | 
 27 |     # Workaround for SSH authentication failure in Vagrant 1.8.5
 28 |     config.ssh.insert_key = false
 29 | 
 30 |     # Virtualbox
 31 |     config.vm.provider 'virtualbox' do |vbox, override|
 32 |         override.vm.synced_folder ".", "/vagrant", :id => "vagrant-root",
 33 |             :mount_options => ['dmode=777', 'fmode=777']
 34 |         vbox.customize ['modifyvm', :id, '--memory', cluster_ram]
 35 |         vbox.customize ['modifyvm', :id, '--cpus', cluster_cpu]
 36 |         vbox.gui = false
 37 |     end
 38 | 
 39 |     # Parallels
 40 |     config.vm.provider "parallels" do |v, override|
 41 |         override.vm.box = "parallels/centos-7.1"
 42 |         #v.update_guest_tools = true
 43 |         v.optimize_power_consumption = false
 44 |         v.memory = cluster_ram
 45 |         v.cpus = cluster_cpu
 46 |     end
 47 | 
 48 |     # VMWare
 49 |     ["vmware_fusion", "vmware_workstation"].each do |vmware|
 50 |         config.vm.provider vmware do |v|
 51 |             v.vmx["memsize"] = cluster_ram
 52 |             v.vmx["numvcpus"] = cluster_cpu
 53 |             v.gui = false
 54 |         end
 55 |     end
 56 | 
 57 |     # ES Nodes
 58 |     (1..nodes_number).each do |index|
 59 |         name = utils.get_vm_name index
 60 |         node_name = utils.get_node_name index
 61 |         ip = utils.get_vm_ip index
 62 |         primary = (index.eql? 1)
 63 | 
 64 |         utils.build_config index
 65 | 
 66 |         config.vm.define :"#{name}", primary: primary do |node|
 67 |             node.vm.hostname = "#{name}.es.dev"
 68 |             node.vm.network 'private_network', ip: ip, auto_config: true
 69 |             node.vm.provision 'shell', path: './lib/upgrade-es.sh'
 70 |             node.vm.provision 'shell', inline: @node_start_inline_script % [name, node_name, ip],
 71 |                 run: 'always'
 72 |             node.vm.network "forwarded_port", guest: 9200, host: 9200+index
 73 |             node.vm.network "forwarded_port", guest: 9300, host: 9300+index
 74 |             node.vm.provider "parallels" do |v|
 75 |                 v.name = "elasticsearch-#{name}"
 76 |             end
 77 |             node.vm.provider "virtualbox" do |v|
 78 |                 v.name = "elasticsearch-#{name}"
 79 |             end
 80 |             ["vmware_fusion", "vmware_workstation"].each do |vmware|
 81 |                 node.vm.provider vmware do |v|
 82 |                     v.vmx["displayname"] =  "elasticsearch-#{name}"
 83 |                 end
 84 |             end
 85 |         end
 86 |     end
 87 | 
 88 |     # Kibana + Client Node
 89 |     config.vm.define :"kibana" do |node|
 90 |         name = utils.get_kibana_vm_name
 91 |         node_name = utils.get_kibana_node_name
 92 |         ip = utils.get_kibana_vm_ip
 93 |         utils.build_kibana_config
 94 |         node.vm.hostname = "#{node_name}.es.dev"
 95 |         node.vm.network 'private_network', ip: ip, auto_config: true
 96 |         node.vm.provision 'shell', path: './lib/upgrade-es.sh'
 97 |         node.vm.provision 'shell', path: './lib/upgrade-kibana.sh'
 98 |         node.vm.provision 'shell', inline: @node_start_inline_script % [name, node_name, ip],
 99 |             run: 'always'
100 |         node.vm.provision 'shell', inline: @kibana_start_inline_script % [name, node_name, ip],
101 |             run: 'always'
102 |         node.vm.network "forwarded_port", guest: 9200, host: 9200
103 |         node.vm.network "forwarded_port", guest: 9300, host: 9300
104 |         node.vm.network "forwarded_port", guest: 5601, host: 5601
105 |         node.vm.provider "parallels" do |v|
106 |             v.name = "elasticsearch-#{name}"
107 |         end
108 |         node.vm.provider "virtualbox" do |v|
109 |             v.name = "elasticsearch-#{name}"
110 |         end
111 |         ["vmware_fusion", "vmware_workstation"].each do |vmware|
112 |             node.vm.provider vmware do |v|
113 |                 v.vmx["displayname"] =  "elasticsearch-#{name}"
114 |             end
115 |         end
116 |     end
117 | 
118 |     # logstash
119 |     config.vm.define :"logstash" do |node|
120 |         name = utils.get_logstash_vm_name
121 |         node_name = utils.get_logstash_node_name
122 |         ip = utils.get_logstash_vm_ip
123 |         utils.build_logstash_config
124 |         node.vm.hostname = "#{node_name}.es.dev"
125 |         node.vm.network 'private_network', ip: ip, auto_config: true
126 |         node.vm.provision 'shell', path: './lib/upgrade-logstash.sh'
127 |         node.vm.provision 'shell', inline: @logstash_start_inline_script % [name, node_name, ip],
128 |             run: 'always'
129 |         node.vm.network "forwarded_port", guest: 5514, host: 5514, protocol: 'tcp'
130 |         node.vm.network "forwarded_port", guest: 5514, host: 5514, protocol: 'udp'
131 |         node.vm.provider "parallels" do |v|
132 |             v.name = "elasticsearch-#{name}"
133 |         end
134 |         node.vm.provider "virtualbox" do |v|
135 |             v.name = "elasticsearch-#{name}"
136 |         end
137 |         ["vmware_fusion", "vmware_workstation"].each do |vmware|
138 |             node.vm.provider vmware do |v|
139 |                 v.vmx["displayname"] =  "elasticsearch-#{name}"
140 |             end
141 |         end
142 |     end
143 |     utils.logger.info "----------------------------------------------------------"
144 | end
145 | 


--------------------------------------------------------------------------------
/lib/elasticsearch-module.rb:
--------------------------------------------------------------------------------
  1 | module Vagrant
  2 |     module ElastiSearchCluster
  3 |         class Util
  4 |             attr_accessor :logger
  5 | 
  6 |             def get_dot_vagrant
  7 |                 File.join(File.dirname(__FILE__),'..','.vagrant')
  8 |             end
  9 | 
 10 |             def get_conf_dir
 11 |                 File.join(File.dirname(__FILE__),'..','conf')
 12 |             end
 13 | 
 14 |             def initialize
 15 |                 @params = [
 16 |                     'cluster_name' => ['CLUSTER_NAME', 'cluster_name', 'dev-es-cluster'],
 17 |                     'cluster_ip' => ['CLUSTER_IP_PATTERN', 'cluster_ip', '10.1.1.%d'],
 18 |                     'cluster_count' => ['CLUSTER_COUNT', 'cluster_count', 3],
 19 |                     'cluster_ram' => ['CLUSTER_RAM', 'cluster_ram', 1024],
 20 |                     'cluster_cpu' => ['CLUSTER_CPU', 'cluster_cpu', 1],
 21 |                 ]
 22 | 
 23 |                 @names = %w(thor zeus isis shifu baal)
 24 |                 @logger = Vagrant::UI::Colored.new
 25 |                 @logger.opts[:color] = :white
 26 |             end
 27 | 
 28 |             def get_vm_name(index)
 29 |                 "vm#{index}"
 30 |             end
 31 | 
 32 |             def get_vm_ip(index)
 33 |                 ip = get_cluster_info 'cluster_ip'
 34 |                 ip.strip % (10 + index)
 35 |             end
 36 | 
 37 |             def get_node_name(index)
 38 |                 @names[index - 1]
 39 |             end
 40 | 
 41 |             def get_kibana_vm_name
 42 |                 "vm250"
 43 |             end
 44 | 
 45 |             def get_kibana_vm_ip
 46 |                 ip = get_cluster_info 'cluster_ip'
 47 |                 ip.strip % 250
 48 |             end
 49 | 
 50 |             def get_kibana_node_name
 51 |                 "kibana"
 52 |             end
 53 | 
 54 |             def get_logstash_vm_name
 55 |                 "vm251"
 56 |             end
 57 | 
 58 |             def get_logstash_vm_ip
 59 |                 ip = get_cluster_info 'cluster_ip'
 60 |                 ip.strip % 251
 61 |             end
 62 | 
 63 |             def get_logstash_node_name
 64 |                 "logstash"
 65 |             end
 66 | 
 67 |             def get_cluster_info(index)
 68 |                 return ENV[@params[0][index][0]] if ENV[@params[0][index][0]]
 69 |                 dot_vagrant = get_dot_vagrant()
 70 |                 return (File.read "#{dot_vagrant}/#{@params[0][index][1]}") if File.exist? "#{dot_vagrant}/#{@params[0][index][1]}"
 71 |                 "#{@params[0][index][2]}"
 72 |             end
 73 | 
 74 |             def save_cluster_info(index, value)
 75 |                 dot_vagrant = get_dot_vagrant()
 76 |                 Dir.mkdir("#{dot_vagrant}") unless Dir.exist?("#{dot_vagrant}")
 77 |                 File.open("#{dot_vagrant}/#{@params[0][index][1]}", 'w') do |file|
 78 |                     file.puts value.to_s
 79 |                 end
 80 |             end
 81 | 
 82 |             def get_config_template
 83 |                 conf_dir = get_conf_dir()
 84 |                 config_file = File.open("#{conf_dir}/elasticsearch.yml.erb", 'r')
 85 |                 ERB.new(config_file.read)
 86 |             end
 87 | 
 88 |             def get_logstash_config_template
 89 |                 conf_dir = get_conf_dir()
 90 |                 config_file = File.open("#{conf_dir}/logstash.conf.erb", 'r')
 91 |                 ERB.new(config_file.read)
 92 |             end
 93 | 
 94 |             def build_config(index)
 95 |                 vm = get_vm_name index
 96 |                 conf_dir = get_conf_dir()
 97 |                 conf_file_format = "#{conf_dir}/elasticsearch-#{vm}.yml"
 98 | 
 99 |                 File.open(conf_file_format, 'w') do |file|
100 |                     @vm_name = vm
101 |                     @node01_ip = get_vm_ip 1
102 |                     @node_ip = get_vm_ip index
103 |                     @node_name = get_node_name index
104 |                     @node_master = true
105 |                     @node_data = true
106 |                     @cluster_ip = get_cluster_info 'cluster_ip'
107 |                     @cluster_name = get_cluster_info 'cluster_name'
108 | 
109 |                     @logger.info "Building configuration for #{vm}"
110 |                     file.puts self.get_config_template.result(binding)
111 |                 end unless File.exist? conf_file_format
112 |             end
113 | 
114 |             def build_kibana_config
115 |                 vm = get_kibana_vm_name
116 |                 conf_dir = get_conf_dir()
117 |                 conf_file_format = "#{conf_dir}/elasticsearch-#{vm}.yml"
118 | 
119 |                 File.open(conf_file_format, 'w') do |file|
120 |                     @vm_name = vm
121 |                     @node_ip = get_kibana_vm_ip
122 |                     @node_name = get_kibana_node_name
123 |                     @node_master = false
124 |                     @node_data = false
125 |                     @cluster_ip = get_cluster_info 'cluster_ip'
126 |                     @cluster_name = get_cluster_info 'cluster_name'
127 | 
128 |                     @logger.info "Building configuration for #{vm}"
129 |                     file.puts self.get_config_template.result(binding)
130 |                 end unless File.exist? conf_file_format
131 |             end
132 | 
133 |             def build_logstash_config
134 |                 vm = get_logstash_vm_name
135 |                 conf_dir = get_conf_dir()
136 |                 conf_file_format = "#{conf_dir}/logstash-#{vm}.conf"
137 | 
138 |                 File.open(conf_file_format, 'w') do |file|
139 |                     @node01_ip = get_vm_ip 1
140 |                     @cluster_name = get_cluster_info 'cluster_name'
141 |                     @logger.info "Building configuration for #{vm}"
142 |                     file.puts self.get_logstash_config_template.result(binding)
143 |                 end unless File.exist? conf_file_format
144 |             end
145 | 
146 |             def manage_and_print_config
147 |                 self.logger.info "----------------------------------------------------------"
148 |                 self.logger.info "          Your ES cluster configurations"
149 |                 self.logger.info "----------------------------------------------------------"
150 | 
151 |                 # Building and showing CLUSTER NAME information
152 |                 index = 'cluster_name'
153 |                 cluster_name = self.get_cluster_info index
154 |                 self.logger.info "Cluster Name: #{cluster_name.strip}"
155 |                 self.save_cluster_info index, cluster_name
156 | 
157 |                 # Building and showing CLUSTER COUNT information
158 |                 index = 'cluster_count'
159 |                 nodes_number = self.get_cluster_info index
160 |                 self.logger.info "Cluster size: #{nodes_number.strip}"
161 |                 self.save_cluster_info index, nodes_number
162 | 
163 |                 # Building and showing CLUSTER IP PATTERN information
164 |                 index = 'cluster_ip'
165 |                 cluster_network_ip = self.get_cluster_info index
166 |                 self.logger.info "Cluster network IP: #{cluster_network_ip.strip % 0}"
167 |                 self.save_cluster_info index, cluster_network_ip
168 | 
169 |                 # Building and showing CLUSTER RAM information
170 |                 index = 'cluster_ram'
171 |                 cluster_ram = self.get_cluster_info index
172 |                 self.logger.info "Cluster RAM (for each node): #{cluster_ram.strip}"
173 |                 self.save_cluster_info index, cluster_ram
174 |                 #
175 |                 # Building and showing CLUSTER RAM information
176 |                 index = 'cluster_cpu'
177 |                 cluster_cpu = self.get_cluster_info index
178 |                 self.logger.info "Cluster CPU (for each node): #{cluster_cpu.strip}"
179 |                 self.save_cluster_info index, cluster_cpu
180 | 
181 |                 self.logger.info "----------------------------------------------------------"
182 |             end
183 |         end
184 |     end
185 | end
186 | 
187 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | Vagrant ELK (Elasticsearch + Logstash + Kibana) Cluster
  2 | =============================
  3 | 
  4 | **For ES 2.0 and above**
  5 | 
  6 | Create an ELK Stack with a single bash command in Vmware, Parallels, or VirtualBox :
  7 | 
  8 | ```bash
  9 | vagrant up --no-parallel --provider <virtualbox|parallels|vmware_fusion|vmware_workstation>
 10 | ```
 11 | <div style='color:red'>**Read Pre Requisites below.**</div>**I would also highly recommend that you read this file in its entirety at least once.**<br/><br/>
 12 | 
 13 | ---
 14 | 
 15 | 
 16 | **Software versions information**
 17 | 
 18 | | Software              | Version     | Description                        |
 19 | | --------------------------------- | ----------- | ----------------------------------------- |
 20 | | CentOS|7.1| Guest OS <br/> VMWare and Virtual box :[chef/centos-7.1](https://atlas.hashicorp.com/chef/boxes/centos-7.1) <br/> & parallels : [parallels/centos-7.1](https://atlas.hashicorp.com/parallels/boxes/centos-7.1) |
 21 | | Java (oracle)              | 1.8.0_101    |     |
 22 | | ElasticSearch                     | 2.4.1       | [Reference Guide](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) / [Definitive Guide](https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html) |
 23 | | Kibana | 4.6.1 | [Reference Guide](https://www.elastic.co/guide/en/kibana/current/index.html)|
 24 | | LogStash | 2.4.0 | [Reference Guide](https://www.elastic.co/guide/en/logstash/current/index.html)|
 25 | 
 26 | **Cluster Details**
 27 | 
 28 | _Default Cluster Name_ : **es-dev-cluster**
 29 | 
 30 | _Default Network Setup_: **Private Network 10.1.1.0/24**
 31 | 
 32 | _Default CPU Cores Per Node_ : **1**
 33 | 
 34 | _Default RAM Per Node_ : **1024MB**
 35 | 
 36 | _ES Endpoint URL_ : [**http://localhost:9200/**](http://localhost:9200/) (_from Host Machine_)
 37 | 
 38 | _Kibana Endpoint URL_ : [**http://localhost:5601/**](http://localhost:5601/) (_from Host Machine_)
 39 | 
 40 | _Logstash Syslog Ports_ : **localhost:5514 (Both TCP and UDP)** (_from Host Machine_)
 41 | 
 42 | _Cluster Nodes :_
 43 | 
 44 | 
 45 | | VM Name| Node Name|Default IP| VM Port <=> Host Port|Description|
 46 | | -------|----------|----------|----------------------|-----------|
 47 | |vm1|thor|10.1.1.11|9200<=>9201<br/>9300<=>9301|1<sup>st</sup> Elasticsearch Node|
 48 | |vm2|zeus|10.1.1.12|9200<=>9202<br/>9300<=>9302|2<sup>nd</sup> Elasticsearch Node|
 49 | |vm3|isis|10.1.1.13|9200<=>9203<br/>9300<=>9303|3<sup>rd</sup> Elasticsearch Node|
 50 | |vm4|baal|10.1.1.14|9200<=>9204<br/>9300<=>9304|4<sup>th</sup> Elasticsearch Node<br/>(_Not started by default_)|
 51 | |vm5|shifu|10.1.1.15|9200<=>9205<br/>9300<=>9305|5<sup>th</sup> Elasticsearch Node<br/>(_Not started by default_)|
 52 | |vm250|kibana|10.1.1.250|**9200<=>9200<br/>9300<=>9300<br/>5601<=>5601**|Kibana + ES Client Node|
 53 | |vm251|logstash|10.1.1.251|**5514<=>5514<br/>(TCP & UDP)**|Logstash Node|
 54 | 
 55 | 
 56 | **WARNING**:  You'll need enough RAM to run VMs in your cluster. Each new VM launched within your cluster will have 1024M of RAM allocated.
 57 | 
 58 | 
 59 | **Elasticsearch Plugins**
 60 | 
 61 | 
 62 | | Plugin              | Version     | URL To Access |
 63 | | ------------------- | ----------- | ------------- |
 64 | | mapper-attachments  | latest      | N.A. |
 65 | | analysis-icu        | latest      | N.A. |
 66 | | lang-javascript     | latest      | N.A. |
 67 | |[elasticsearch-head](https://github.com/mobz/elasticsearch-head)| latest| [http://localhost:9200/\_plugin/head/](http://localhost:9200/_plugin/head/) |
 68 | |[elasticsearch-kopf](https://github.com/lmenezes/elasticsearch-kopf)| 2.1.2| [http://localhost:9200/\_plugin/kopf](http://localhost:9200/_plugin/kopf) |
 69 | |[elasticsearch-paramedic](https://github.com/karmi/elasticsearch-paramedic)|latest | [http://localhost:9200/\_plugin/paramedic/](http://localhost:9200/_plugin/paramedic/) |
 70 | |[elasticsearch-HQ](https://github.com/royrusso/elasticsearch-HQ) | latest| [http://localhost:9200/\_plugin/HQ/](http://localhost:9200/_plugin/HQ/) |
 71 | |[Sense](https://www.elastic.co/guide/en/sense/current/index.html)|latest|[http://localhost:5601/app/sense](http://localhost:5601/app/sense)|
 72 | 
 73 | <br/>
 74 | 
 75 | ---
 76 | 
 77 | 
 78 | 
 79 | 1. Pre Requisite & Set up
 80 | --
 81 | 
 82 | **Must have on your host machine**
 83 | 
 84 | * VirtualBox (last version) OR VMWare desktop|fusion OR parallels
 85 | * Vagrant (>=1.7)
 86 | * Respective vagrant plugins for vmware or parallels
 87 | * cUrl (or another REST client to talk to ES)
 88 | 
 89 | **Clone this repository**
 90 | 
 91 | `git clone https://github.com/bhaskarvk/vagrant-elk-cluster.git`
 92 | 
 93 | **Download Installation Files**
 94 | 
 95 | This needs to be done just once (this should all get done for you).
 96 | 
 97 | *	Download JDK 8u101 64bit RPM from [Oracle](http://www.oracle.com/technetwork/java/javase/downloads/) 
 98 | *	Download elasticsearch from [elastic](https://www.elastic.co/downloads/elasticsearch)
 99 | *	Download kibana from [elastic](https://www.elastic.co/downloads/kibana)
100 | *	Download logstash from [elastic](https://www.elastic.co/downloads/logstash)
101 | *	Place all the above files at the root of this repo.
102 | 
103 | If you need to upgrade any of the above, download respective version and change the version number in `lib/upgrade-es.sh` OR  `lib/upgrade-kibana.sh` Or  `lib/upgrade-logstash.sh` accordingly and re-run provisioning.
104 | 
105 | 
106 | 2. How to run a new ELK Stack cluster
107 | --
108 | 
109 | **Run the cluster**
110 | 
111 | Simply go in the cloned directory (vagrant-elk-cluster by default).
112 | Execute this command :
113 | 
114 | ```bash
115 | vagrant up --no-parallel --provider <virtualbox|parallels|vmware_fusion|vmware_workstation>
116 | ```
117 | 
118 |  I recommend starting in `no-parallel` mode as it is the safest, but you can also try with removing this argument.
119 | 
120 | By default 3 ElasticSearch Nodes are started: vm1, vm2, and vm3. One kibana (vm250) and one logstash (vm251) node are also started.
121 | You can start a maximum of 5 Elasticsearch nodes. If you want you can increase this limit by changing the code but it is pointless to have a bigger cluster for dev|qa purposes.
122 | 
123 | You can change the cluster size with the `CLUSTER_COUNT` variable (min 1 and max 5):
124 | 
125 | ```bash
126 | CLUSTER_COUNT=5 vagrant up 
127 | ```
128 | 
129 | You can change the cluster name with the `CLUSTER_NAME` variable:
130 | 
131 | ```bash
132 | CLUSTER_NAME='es-qa-cluster' vagrant up
133 | ```
134 | 
135 | You can change the cluster RAM used for each node with the `CLUSTER_RAM` variable:
136 | 
137 | ```bash
138 | CLUSTER_RAM=512 vagrant up
139 | ```
140 | 
141 | You can change the cluster CPU used for each node with the `CLUSTER_CPU` variable:
142 | 
143 | ```bash
144 | CLUSTER_CPU=2 vagrant up
145 | ```
146 | 
147 | You can change the cluster network IP address with the `CLUSTER_IP_PATTERN` variable:
148 | 
149 | ```bash
150 | CLUSTER_IP_PATTERN='172.16.15.%d' vagrant up
151 | ```
152 | 
153 | **NOTE** : Providing the `CLUSTER_NAME`, `CLUSTER_COUNT`, `CLUSTER_RAM`, `CLUSTER_CPU`, `CLUSTER_IP_PATTERN` variables is only required when you first start the cluster.
154 | Vagrant will save/cache these values under the `.vagrant` directory, so you can run other commands without repeating yourself.
155 | 
156 | Of course you can use all these variables at the same time :
157 | 
158 | ```bash
159 | $ CLUSTER_NAME='es-qa-cluster' CLUSTER_IP_PATTERN='172.16.25.%d' CLUSTER_COUNT=5 \
160 | CLUSTER_RAM=512 CLUSTER_CPU=2 vagrant up
161 | ```
162 | 
163 | **Sample output**
164 | 
165 | ```
166 | $ vagrant status
167 | ----------------------------------------------------------
168 |           Your ES cluster configurations
169 | ----------------------------------------------------------
170 | Cluster Name: dev-es-cluster
171 | Cluster size: 3
172 | Cluster network IP: 10.1.1.0
173 | Cluster RAM (for each node): 1024
174 | Cluster CPU (for each node): 1
175 | ----------------------------------------------------------
176 | ----------------------------------------------------------
177 | Current machine states:
178 | 
179 | vm1                       stopped (parallels)
180 | vm2                       stopped (parallels)
181 | vm3                       stopped (parallels)
182 | kibana                    stopped (parallels)
183 | logstash                  stopped (parallels)
184 | 
185 | This environment represents multiple VMs. The VMs are all listed
186 | above with their current state. For more information about a specific
187 | VM, run `vagrant status NAME`....
188 | ```
189 | 
190 | The names of the Elasticsearch VMs will follow the following pattern: `vm[0-9]+`.
191 | The trailing number represents the index of the VM, starting at 1. Kibana and Logstash instances are simply named kibana and logstash.
192 | 
193 | ElasticSearch, Kibana, & Logstash instances are started during provisioning of respective VMs.
194 | The command is launched into a new screen as root user inside the vagrant.
195 | 
196 | 
197 | **Stop the cluster**
198 | 
199 | ```bash
200 | vagrant halt
201 | ```
202 | 
203 | This will stop the whole cluster. If you want to only stop one VM, you can use:
204 | 
205 | ```bash
206 | vagrant halt vm2
207 | ```
208 | 
209 | This will stop the `vm2` instance.
210 | 
211 | **Destroy the cluster**
212 | 
213 | ```bash
214 | vagrant destroy
215 | rm -rf .vagrant conf/*.yml conf/*.conf logs/* data/* 
216 | ```
217 | 
218 | This will stop the whole cluster. If you want to only stop one VM, you can use:
219 | 
220 | ```bash
221 | vagrant destroy vm2
222 | ```
223 | 
224 | :warning: If you destroy a VM, I suggest you to destroy all the cluster to be sure to have the same ES version in all of your nodes.
225 | 
226 | **Managing ElasticSearch instances**
227 | 
228 | Each VM has its own ElasticSearch instance running in a `screen` session named `elastic`.
229 | Once connected to the VM, you can manage this instance with the following commands:
230 | 
231 | * `sudo node-start`: starts the ES instance
232 | * `sudo node-stop`: stops the ES instance
233 | * `sudo node-restart`: restarts the ES instance
234 | * `sudo node-status`: displays ES instance's status
235 | * `sudo node-attach`: bring you to the screen session hosting the ES instance. Use `^Ad` to detach.
236 | 
237 | You should be brought to the screen session hosting ElasticSearch and see its log.
238 | 
239 | For Kibana use `sudo kibana-<start|stop|restart|status|attach>`, and similarly for Logstash use `sudo logstash-<start|stop|restart|status|attach>`
240 | 
241 | 
242 | **Default Directories**
243 | 
244 | By default the `data`, `logs` and `config` directories live outside of the VMs on the host, this way you can destroy and rebuild VMs as much as you like without losing your data. You can also upgrade Elasticsearch and not lose data.
245 | 
246 | 3. Access the Cluster
247 | --
248 | 
249 | The 9200 and 9300 ports of the host machine have been setup to forward to respective ports of ES Client Node running on Kibana.
250 | To access ES Rest API from Host machine you can use [**http://localhost:9200/**](http://localhost:9200/) which will route the API Access via the Client Node running on the Kibana Node.
251 | 
252 | TO access ES Rest Endpoint on a data node, use 9200 + \<node number> on the host machine. so for vm1 it would be  9201 so [http://localhost:9201/](http://localhost:9201/) and 9202 for vm2, and so forth. But you will hardly need to access these endpoints from host machine.
253 | 
254 | To access Kibana from host machine use [**http://localhost:5601/**](http://localhost:5601/). Logstash node has been setup to receive syslog messages on port 5514 (TCP & UDP) and the host machine will forward anything on its port 5514 (TCP & UDP) to these ports.
255 | 
256 | 4. Configure your cluster
257 | --
258 | 
259 | If you need or want to change the default working configuration of your cluster,
260 | you can do it adding/editing elasticsearch-<VM-Name>.yml files in `conf` directory.
261 | Each node configuration is shared with VM thanks to this "conf" directory.
262 | 
263 | By default, this configuration files are **auto-generated** by Vagrant when running the cluster for the first time.
264 | In this case, default values listed at the top of this page are used.
265 | 
266 | Similarly for logstash you need to change conf/logstash-<VM Name>.conf file again in `conf` directory.
267 | 
268 | 
269 | 5. Working with your cluster
270 | --
271 | 
272 | Here are a few sample calls to get you started.
273 | 
274 | Send some sample syslog messages to Logstash to be indexed in Elasticsearch.
275 | 
276 | ```bash
277 | # From the host machine
278 | # I'm assuming the host is some sort of UNIX box
279 | # If windows, then you are on your own :)
280 | echo "<133>$0[$$]: Test syslog message from Netcat" | nc -4 localhost 5514
281 | echo "<133>$0[$$]: Second Test syslog message from Netcat" | nc -4 localhost 5514
282 | ```
283 | 
284 | Next go to Kibana Dashboard at [http://localhost:5601/](http://localhost:5601/). Kibana will auto discover the logstash index and ask you some basic question, just select the default, and then go to the discover tab. You should see the two sample messages.
285 | 
286 | ---
287 | 
288 | **TODO**
289 | 
290 | See issues.
291 | 
292 | 


--------------------------------------------------------------------------------