├── .gitignore
├── README.md
└── Vagrantfile


/.gitignore:
--------------------------------------------------------------------------------
1 | ### Vagrant ###
2 | .vagrant/
3 | vagrant_ansible_inventory*


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # A working virtual Hadoop cluster
 2 | 
 3 | With these files you can setup and provision a locally running, virtual Hadoop cluster in real distributed fashion for trying out Hadoop and related technologies. It runs the latest Cloudera Hadoop distribution: **CDH5**. It also allows you to practise the use of [Cloudera Manager](http://www.cloudera.com/content/cloudera/en/products-and-services/cloudera-enterprise/cloudera-manager.html) for installing the Hadoop stack. If you're looking for a fully automated install, without user intervention, look elsewhere. I specifically made this with the goal of creating an environment ideally suited for Cloudera Manager to do its job. This gives you the freedom to actually install the services you want, and change the configuration how you see fit.
 4 | 
 5 | This README describes how to get the cluster with Cloudera Manager up and running. For more detailed instructions on how to install the whole Hadoop stack on that, you can use [this guide](http://dandydev.net/blog/installing-virtual-hadoop-cluster).
 6 | 
 7 | ## Specs
 8 | 
 9 | The cluster conists of 4 nodes:
10 | 
11 | * Master node with 4GB of RAM (Running the NameNode, Hue, ResourceManager etc. after installing the Hadoop services)
12 | * 3 slaves with 2GB of RAM each (Running DataNodes)
13 | 
14 | As you can see, you'll need at least 10GB of free RAM to run this. If you have less, you can try to remove one machine from the Vagrantfile. This will lead to worse performance though!
15 | 
16 | ## Usage
17 | 
18 | Depending on the hardware of your computer, installation will probably take between 15 and 25 minutes.
19 | 
20 | First install [VirtualBox](https://www.virtualbox.org/) and [Vagrant](http://www.vagrantup.com/).
21 | 
22 | Install the Vagrant [Hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager)
23 | 
24 | ```bash
25 | $ vagrant plugin install vagrant-hostmanager
26 | ```
27 | 
28 | Clone this repository.
29 | 
30 | ```bash
31 | $ git clone https://github.com/DandyDev/virtual-hadoop-cluster.git
32 | ```
33 | 
34 | Provision the bare cluster. It will ask you to enter your password, so it can modify your `/etc/hosts` file for easy access in your browser. It uses the Vagrant Hostmanager plugin to do this.
35 | 
36 | ```bash
37 | $ cd virtual-hadoop-cluster
38 | $ vagrant up
39 | ```
40 | 
41 | Go to the [Cloudera Manager web console](http://vm-cluster-node1:7180) and follow the installation instructions. For more detailed instructions on how to do that, you can use [this guide](http://dandydev.net/blog/installing-virtual-hadoop-cluster).
42 | 
43 | **Done!** Have fun with your Hadoop cluster.
44 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
  1 | $master_script = <<SCRIPT
  2 | #!/bin/bash
  3 | 
  4 | apt-get install curl -y
  5 | REPOCM=${REPOCM:-cm5}
  6 | CM_REPO_HOST=${CM_REPO_HOST:-archive.cloudera.com}
  7 | CM_MAJOR_VERSION=$(echo $REPOCM | sed -e 's/cm\\([0-9]\\).*/\\1/')
  8 | CM_VERSION=$(echo $REPOCM | sed -e 's/cm\\([0-9][0-9]*\\)/\\1/')
  9 | OS_CODENAME=$(lsb_release -sc)
 10 | OS_DISTID=$(lsb_release -si | tr '[A-Z]' '[a-z]')
 11 | if [ $CM_MAJOR_VERSION -ge 4 ]; then
 12 |   cat > /etc/apt/sources.list.d/cloudera-$REPOCM.list <<EOF
 13 | deb [arch=amd64] http://$CM_REPO_HOST/cm$CM_MAJOR_VERSION/$OS_DISTID/$OS_CODENAME/amd64/cm $OS_CODENAME-$REPOCM contrib
 14 | deb-src http://$CM_REPO_HOST/cm$CM_MAJOR_VERSION/$OS_DISTID/$OS_CODENAME/amd64/cm $OS_CODENAME-$REPOCM contrib
 15 | EOF
 16 | curl -s http://$CM_REPO_HOST/cm$CM_MAJOR_VERSION/$OS_DISTID/$OS_CODENAME/amd64/cm/archive.key > key
 17 | apt-key add key
 18 | rm key
 19 | fi
 20 | apt-get update
 21 | export DEBIAN_FRONTEND=noninteractive
 22 | apt-get -q -y --force-yes install oracle-j2sdk1.7 cloudera-manager-server-db cloudera-manager-server cloudera-manager-daemons
 23 | service cloudera-scm-server-db initdb
 24 | service cloudera-scm-server-db start
 25 | service cloudera-scm-server start
 26 | SCRIPT
 27 | 
 28 | $hosts_script = <<SCRIPT
 29 | cat > /etc/hosts <<EOF
 30 | 127.0.0.1       localhost
 31 | 
 32 | # The following lines are desirable for IPv6 capable hosts
 33 | ::1     ip6-localhost ip6-loopback
 34 | fe00::0 ip6-localnet
 35 | ff00::0 ip6-mcastprefix
 36 | ff02::1 ip6-allnodes
 37 | ff02::2 ip6-allrouters
 38 | 
 39 | EOF
 40 | SCRIPT
 41 | 
 42 | Vagrant.configure("2") do |config|
 43 | 
 44 |   # Define base image
 45 |   config.vm.box = "precise64"
 46 |   config.vm.box_url = "http://files.vagrantup.com/precise64.box"
 47 | 
 48 |   # Manage /etc/hosts on host and VMs
 49 |   config.hostmanager.enabled = false
 50 |   config.hostmanager.manage_host = true
 51 |   config.hostmanager.include_offline = true
 52 |   config.hostmanager.ignore_private_ip = false
 53 | 
 54 |   config.vm.define :master do |master|
 55 |     master.vm.provider :virtualbox do |v|
 56 |       v.name = "vm-cluster-node1"
 57 |       v.customize ["modifyvm", :id, "--memory", "4096"]
 58 |     end
 59 |     master.vm.network :private_network, ip: "10.211.55.100"
 60 |     master.vm.hostname = "vm-cluster-node1"
 61 |     master.vm.provision :shell, :inline => $hosts_script
 62 |     master.vm.provision :hostmanager
 63 |     master.vm.provision :shell, :inline => $master_script
 64 |   end
 65 | 
 66 |   config.vm.define :slave1 do |slave1|
 67 |     slave1.vm.box = "precise64"
 68 |     slave1.vm.provider :virtualbox do |v|
 69 |       v.name = "vm-cluster-node2"
 70 |       v.customize ["modifyvm", :id, "--memory", "2048"]
 71 |     end
 72 |     slave1.vm.network :private_network, ip: "10.211.55.101"
 73 |     slave1.vm.hostname = "vm-cluster-node2"
 74 |     slave1.vm.provision :shell, :inline => $hosts_script
 75 |     slave1.vm.provision :hostmanager
 76 |   end
 77 | 
 78 |   config.vm.define :slave2 do |slave2|
 79 |     slave2.vm.box = "precise64"
 80 |     slave2.vm.provider :virtualbox do |v|
 81 |       v.name = "vm-cluster-node3"
 82 |       v.customize ["modifyvm", :id, "--memory", "2048"]
 83 |     end
 84 |     slave2.vm.network :private_network, ip: "10.211.55.102"
 85 |     slave2.vm.hostname = "vm-cluster-node3"
 86 |     slave2.vm.provision :shell, :inline => $hosts_script
 87 |     slave2.vm.provision :hostmanager
 88 |   end
 89 | 
 90 |   config.vm.define :slave3 do |slave3|
 91 |     slave3.vm.box = "precise64"
 92 |     slave3.vm.provider :virtualbox do |v|
 93 |       v.name = "vm-cluster-node4"
 94 |       v.customize ["modifyvm", :id, "--memory", "2048"]
 95 |     end
 96 |     slave3.vm.network :private_network, ip: "10.211.55.103"
 97 |     slave3.vm.hostname = "vm-cluster-node4"
 98 |     slave3.vm.provision :shell, :inline => $hosts_script
 99 |     slave3.vm.provision :hostmanager
100 |   end
101 | 
102 | end


--------------------------------------------------------------------------------