├── .env ├── LICENSE ├── Makefile ├── README.md ├── Vagrantfile ├── ansible.cfg ├── bin ├── cluster-ha-up.sh ├── cluster-up.sh ├── get-binaries.sh ├── ssh-gen.sh └── vm-plugins.sh ├── data └── banklist.csv ├── imgs ├── Cassandra-3-NodeCluster.png ├── Flink_Screenshot.png ├── Hadoop-DataNods.png ├── HadoopYarn.png ├── SparkExecutors.png ├── SparkJobs.png ├── SparkStandaloneConsole.png └── VideoDemoScreenshot.png ├── inventory.yml ├── invm.yml ├── roles ├── cassandra │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── cassandra-env.sh.j2 │ │ ├── cassandra.service.j2 │ │ ├── cassandra.yaml.j2 │ │ ├── jmxremote.access.j2 │ │ └── jmxremote.password.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── elasticsearch │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── flink │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── flink-conf.yaml.j2 │ │ └── slaves.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── flume │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── customer.avsc │ │ ├── sample_conf │ │ └── sink-flume.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── hadoop │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── files │ │ ├── id_rsa │ │ └── id_rsa.pub │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── core-site.xml.j2 │ │ ├── hadoop-dfs.service.j2 │ │ ├── hadoop-job-history.service.j2 │ │ ├── hadoop-yarn.service.j2 │ │ ├── hadoop_rsa.pub │ │ ├── hdfs-site.xml.j2 │ │ ├── mapred-site.xml.j2 │ │ ├── slaves │ │ ├── ssh.config.j2 │ │ └── yarn-site.xml.j2 │ └── vars │ │ └── main.yml ├── hbase │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── backup-masters.j2 │ │ ├── hbase-site.xml.j2 │ │ └── regionservers.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── hive │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ └── hive-site.xml.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── kafka │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── kafka.systemd.j2 │ │ └── server.properties.j2 │ └── vars │ │ └── main.yml ├── kibana │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── logstash │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── Procfile.j2 │ │ └── log4j-kakfa-processor.conf.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── mysql │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── pig │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── shared │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── bashrc.j2 │ │ └── hosts.j2 │ └── vars │ │ └── main.yml ├── spark │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── hive-site.xml.j2 │ │ ├── metrics.properties.j2 │ │ ├── slaves.j2 │ │ ├── spark-defaults.conf.j2 │ │ └── spark-env.sh.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── storm │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── Procfile │ │ └── storm.yaml.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── wildfly │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ └── standalone.xml.j2 │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml ├── zeppelin │ ├── .travis.yml │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── tests │ │ ├── inventory │ │ └── test.yml │ └── vars │ │ └── main.yml └── zookeeper │ ├── README.md │ ├── defaults │ └── main.yml │ ├── handlers │ └── main.yml │ ├── meta │ └── main.yml │ ├── tasks │ └── main.yml │ ├── templates │ ├── myid.j2 │ ├── zoo.cfg.j2 │ └── zookeeper-systemd.j2 │ └── vars │ └── main.yml ├── run-playbook.sh ├── site-ha.yml └── site.yml /.env: -------------------------------------------------------------------------------- 1 | MD_HADOOP_MODE=standard 2 | ANSIBLE_SITE_YML=site.yml 3 | HADOOP_MASTER_IP_NN1=192.168.33.40 4 | HADOOP_MASTER_IP_NN2=192.168.33.39 5 | HADOOP_MASTER_IP_NN3=192.168.33.38 6 | HADOOP_MAX_DATANODES=2 7 | PEM_KEY=~/.vagrant.d/insecure_private_key 8 | ID_RSA=/home/vagrant/.ssh/id_rsa 9 | CPU=1 10 | MEMORY=2048 11 | LINUX_IMAGE=bento/centos-7.1 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | vm-up: 2 | vagrant up 3 | run-playbook: 4 | ansible-playbook site.yml 5 | download-artifacts: 6 | ./bin/get-binaries 7 | create-certs: 8 | ./bin/ssh-gen.sh 9 | install-vm-plugins: 10 | ./bin/vm-plugins.sh 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Hadoop Ansible 4 | 5 | This Ansible playbook contains roles for the following technologies: 6 | 7 | * Apache Hadoop v2.7.1 8 | * Apache Spark v1.6.1 9 | * Apache Storm v1.0.1 10 | * Apache Zookeeper v3.5.0 11 | * Apache Kafka v2.11-0.9.0.1 12 | * Apache Pig v0.15.0 13 | * Apache Hive v2.0.0 14 | * Apache Cassandra v3.7 15 | * ElasticSearch v2.3.3 16 | * Logstash v2.3.3 17 | * Kibana v4.5.1 18 | * JBoss Wildfly v10.0 19 | * Apache Zeppelin v0.6.0 20 | * Apache Flink v1.0.3 21 | 22 | Note: This ansible module is not ready yet and is still in development. 23 | 24 | ## Ansible Variables 25 | 26 | If you would like to select which roles you would like to use in this playbook, I've exposed variables for each module `install_$role` so you can include the ones you want and not include others. See below example of inventory.yml variables: 27 | 28 | 29 | ```bash 30 | 31 | [all:vars] 32 | # Required variables for this installation 33 | HOME=/home/vagrant 34 | zk_urls=hadoopmaster:2181,hadoopdata1:2181,hadoopdata2:2181 35 | hadoop_gid=hadoop 36 | hadoop_uid=hadoop 37 | hadoopmaster=192.168.33.40 38 | cassandra_seeds=hadoopmaster,hadoopdata1,hadoopdata2 39 | 40 | # For selecting which modules to install. To enable set each variable below to true. To disable set to false 41 | install_zookeeper=true 42 | install_hadoop=true 43 | install_kafka=true 44 | install_storm=true 45 | install_spark=true 46 | install_cassandra=true 47 | install_elasticsearch=true 48 | install_kibana=true 49 | install_logstash=true 50 | install_pig=true 51 | install_mysql=true 52 | install_hive=true 53 | install_wildfly=true 54 | install_zeppelin=true 55 | install_hbase=true 56 | install_flink=true 57 | install_flume=true 58 | ``` 59 | 60 | ## Prerequisite: 61 | 62 | Hadoop will require creation of public key private key pair. Create your public id_rsa.pub, id_rsa and place it in this folder: 63 | 64 | `/roles/hadoop/files/` 65 | 66 | 67 | ## Vagrant 68 | ```bash 69 | vagrant plugin install vagrant-env 70 | vagrant up 71 | ``` 72 | 73 | ## Screenshots: 74 | 75 | * Apache Cassandra 3 Node Cluster 76 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/Cassandra-3-NodeCluster.png) 77 | 78 | * Apache Hadoop 3 Node Cluster 79 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/Hadoop-DataNods.png) 80 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/HadoopYarn.png) 81 | 82 | * Apache Spark 3 Node Cluster 83 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/SparkJobs.png) 84 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/SparkExecutors.png) 85 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/SparkStandaloneConsole.png) 86 | 87 | * Apache Flink 3 Node Cluster 88 | ![alt tag](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/Flink_Screenshot.png) 89 | 90 | ## Video Demo: 91 | 92 | Click on image below to see video demonstration: 93 | 94 | [![ScreenShot](https://raw.githubusercontent.com/BZCareer/hadoop-ansible/master/imgs/VideoDemoScreenshot.png)](https://vimeo.com/170316038) 95 | 96 | 97 | 98 | ## License 99 | Licensed under the Apache License, Version 2.0. 100 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | # Must install 'vagrant plugin install vagrant-env' for this to work 6 | config.env.enable 7 | config.ssh.insert_key = false 8 | 9 | # Defining Nodes if you would like to demo more then two nodes increase the number below 10 | (1..2).each do |i| 11 | config.vm.define "hadoopdatanode#{i}" do |node| 12 | node.vm.box = ENV['LINUX_IMAGE'] #"centos/7" 13 | node.vm.hostname = "hadoopdata#{i}" 14 | node.vm.network :private_network, ip: "192.168.33.4#{i}" 15 | node.vm.provider "virtualbox" do |v| 16 | v.cpus = ENV['CPU'] 17 | v.memory = ENV['MEMORY'] 18 | end 19 | node.vm.provision :file do |file| 20 | file.source = ENV['PEM_KEY'] 21 | file.destination = ENV['ID_RSA'] 22 | end 23 | end 24 | end 25 | 26 | # Only creates ha mode if environment variable is set 27 | if ENV['MD_HADOOP_MODE'] == 'cluster' 28 | # Defining Master Node 29 | config.vm.define "hadoopmaster3" do |hadoopmaster| 30 | hadoopmaster.vm.hostname = "hadoopmaster3" 31 | hadoopmaster.vm.box = ENV['LINUX_IMAGE'] #"centos/7" 32 | #hadoopmaster.vm.synced_folder ".", "/home/vagrant", mount_options: ["dmode=775,fmode=664"] 33 | hadoopmaster.vm.network "private_network", ip: "192.168.33.38" 34 | hadoopmaster.vm.provider "virtualbox" do |v| 35 | v.name = "hadoopmaster3" 36 | v.cpus = ENV['CPU'] 37 | v.memory = ENV['MEMORY'] 38 | end 39 | # copy private key so hosts can ssh using key authentication (the script below sets permissions to 600) 40 | hadoopmaster.vm.provision :file do |file| 41 | file.source = ENV['PEM_KEY'] 42 | file.destination = ENV['ID_RSA'] 43 | end 44 | end 45 | 46 | # Defining Master Node 47 | config.vm.define "hadoopmaster2" do |hadoopmaster| 48 | hadoopmaster.vm.hostname = "hadoopmaster2" 49 | hadoopmaster.vm.box = ENV['LINUX_IMAGE'] #"centos/7" 50 | #hadoopmaster.vm.synced_folder ".", "/home/vagrant", mount_options: ["dmode=775,fmode=664"] 51 | hadoopmaster.vm.network "private_network", ip: "192.168.33.39" 52 | hadoopmaster.vm.provider "virtualbox" do |v| 53 | v.name = "hadoopmaster2" 54 | v.cpus = ENV['CPU'] 55 | v.memory = ENV['MEMORY'] 56 | end 57 | # copy private key so hosts can ssh using key authentication (the script below sets permissions to 600) 58 | hadoopmaster.vm.provision :file do |file| 59 | file.source = ENV['PEM_KEY'] 60 | file.destination = ENV['ID_RSA'] 61 | end 62 | end 63 | 64 | end 65 | 66 | # Defining Base Master Node 67 | config.vm.define "hadoopmaster" do |hadoopmaster| 68 | hadoopmaster.vm.hostname = "hadoopmaster" 69 | hadoopmaster.vm.box = ENV['LINUX_IMAGE'] #"centos/7" 70 | #hadoopmaster.vm.synced_folder ".", "/home/vagrant", mount_options: ["dmode=775,fmode=664"] 71 | hadoopmaster.vm.network "private_network", ip: "192.168.33.40" 72 | hadoopmaster.vm.provider "virtualbox" do |v| 73 | v.name = "hadoopmaster" 74 | v.cpus = ENV['CPU'] 75 | v.memory = ENV['MEMORY'] 76 | end 77 | # copy private key so hosts can ssh using key authentication (the script below sets permissions to 600) 78 | hadoopmaster.vm.provision :file do |file| 79 | file.source = ENV['PEM_KEY'] 80 | file.destination = ENV['ID_RSA'] 81 | end 82 | 83 | hadoopmaster.vm.provision "ansible" do |ansible| 84 | ansible.playbook = ENV['ANSIBLE_SITE_YML'] 85 | ansible.verbose = true 86 | ansible.limit = "all" # or only "nodes" group, etc. 87 | ansible.inventory_path = "inventory.yml" 88 | end 89 | end 90 | 91 | end 92 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | timeout=20 3 | forks=20 4 | private_key_file = ~/.ssh/id_rsa 5 | remote_user = vagrant 6 | host_key_checking = False 7 | 8 | 9 | [ssh_connection] 10 | ssh_args=-o ControlMaster=auto -o ControlPersist=1800s -o ForwardAgent=yes 11 | #pipelining=True% 12 | -------------------------------------------------------------------------------- /bin/cluster-ha-up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Author: Zak Hassan 3 | # Please install vagrant plugin install vagrant-env. More info can be found here: https://github.com/gosuri/vagrant-env 4 | export MD_HADOOP_MODE='cluster' 5 | export ANSIBLE_SITE_YML = 'site-ha.xml' 6 | vagrant up 7 | -------------------------------------------------------------------------------- /bin/cluster-up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Author: Zak Hassan 3 | # Please install vagrant plugin install vagrant-env. More info can be found here: https://github.com/gosuri/vagrant-env 4 | export MD_HADOOP_MODE='local' 5 | export ANSIBLE_SITE_YML = 'site.xml' 6 | vagrant up 7 | -------------------------------------------------------------------------------- /bin/get-binaries.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Author: Zak Hassan 3 | # Convenience script for downloading artifacts before running ansible script. You will need to change 'get_url' to 'copy' 4 | curl -o ../roles/pig/files/pig-0.15.0.tar.gz http://apache.mirror.gtcomm.net/pig/pig-0.15.0/pig-0.15.0.tar.gz 5 | curl -o ../roles/hive/files/apache-hive-2.0.0-bin.tar.gz http://apache.mirror.gtcomm.net/hive/hive-2.0.0/apache-hive-2.0.0-bin.tar.gz 6 | curl -o ../roles/spark/files/spark-1.6.1-bin-hadoop2.6.tgz http://d3kbcqa49mib13.cloudfront.net/spark-1.6.1-bin-hadoop2.6.tgz 7 | curl -o ../roles/hadoop/files/hadoop-2.7.1.tar.gz http://mirror.csclub.uwaterloo.ca/apache/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz 8 | curl -o ../roles/kafka/files/kafka_2.11-0.9.0.1.tgz http://apachemirror.ovidiudan.com/kafka/0.9.0.1/kafka_2.11-0.9.0.1.tgz 9 | curl -o ../roles/storm/files/apache-storm-1.0.1.tar.gz http://apache.mirror.gtcomm.net/storm/apache-storm-1.0.1/apache-storm-1.0.1.tar.gz 10 | curl -o ../roles/zookeeper/files/zookeeper-3.5.0-alpha.tar.gz http://apache.mirror.gtcomm.net/zookeeper/zookeeper-3.5.0-alpha/zookeeper-3.5.0-alpha.tar.gz 11 | curl -o ../roles/elasticsearch/files/elasticsearch-2.3.3.tar.gz https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.3/elasticsearch-2.3.3.tar.gz 12 | -------------------------------------------------------------------------------- /bin/ssh-gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Author: Zak Hassan 3 | 4 | ssh-keygen 5 | -------------------------------------------------------------------------------- /bin/vm-plugins.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Author: Zak Hassan 3 | vagrant plugin install vagrant-env 4 | -------------------------------------------------------------------------------- /imgs/Cassandra-3-NodeCluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/Cassandra-3-NodeCluster.png -------------------------------------------------------------------------------- /imgs/Flink_Screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/Flink_Screenshot.png -------------------------------------------------------------------------------- /imgs/Hadoop-DataNods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/Hadoop-DataNods.png -------------------------------------------------------------------------------- /imgs/HadoopYarn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/HadoopYarn.png -------------------------------------------------------------------------------- /imgs/SparkExecutors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/SparkExecutors.png -------------------------------------------------------------------------------- /imgs/SparkJobs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/SparkJobs.png -------------------------------------------------------------------------------- /imgs/SparkStandaloneConsole.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/SparkStandaloneConsole.png -------------------------------------------------------------------------------- /imgs/VideoDemoScreenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BZCareer/hadoop-ansible/a08e3b54bfaddf7245d19f5562087e5267d75056/imgs/VideoDemoScreenshot.png -------------------------------------------------------------------------------- /inventory.yml: -------------------------------------------------------------------------------- 1 | [hadoop-master-node-third] 2 | 192.168.33.38 ansible_connection=ssh ansible_ssh_user=vagrant api_hostname=hadoopmaster3 myid=3 ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 3 | 4 | [hadoop-master-node-second] 5 | 192.168.33.39 ansible_connection=ssh ansible_ssh_user=vagrant api_hostname=hadoopmaster2 myid=2 ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 6 | 7 | 8 | 9 | [hadoop-master-node] 10 | 192.168.33.40 server1=0.0.0.0 server2=192.168.33.41 server3=192.168.33.42 ansible_connection=ssh ansible_ssh_user=vagrant api_hostname=hadoopmaster myid=1 ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 11 | 12 | [hadoop-data1-node] 13 | 192.168.33.41 server1=192.168.33.40 server2=0.0.0.0 server3=192.168.33.42 ansible_connection=ssh ansible_ssh_user=vagrant myid=2 api_hostname=hadoopdata1 ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 14 | 15 | [hadoop-data2-node] 16 | 192.168.33.42 server1=192.168.33.40 server2=192.168.33.41 server3=0.0.0.0 ansible_connection=ssh ansible_ssh_user=vagrant myid=3 api_hostname=hadoopdata2 ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 17 | 18 | 19 | 20 | [hadoop-standard-nodes:children] 21 | hadoop-master-node 22 | hadoop-data1-node 23 | hadoop-data2-node 24 | 25 | 26 | [hadoop-master-namenodes:children] 27 | hadoop-master-node 28 | hadoop-master-node-second 29 | hadoop-master-node-third 30 | 31 | [hadoop-nodes:children] 32 | hadoop-data1-node 33 | hadoop-data2-node 34 | 35 | 36 | [all:vars] 37 | # Required variables for this installation 38 | HOME=/home/vagrant 39 | zk_urls=hadoopmaster:2181,hadoopdata1:2181,hadoopdata2:2181 40 | hadoop_gid=hadoop 41 | hadoop_uid=hadoop 42 | hadoopmaster=192.168.33.40 43 | cassandra_seeds=hadoopmaster,hadoopdata1,hadoopdata2 44 | 45 | # For selecting which modules to install. To enable set each variable below to true. To disable set to false 46 | install_zookeeper=true 47 | install_hadoop=true 48 | install_kafka=true 49 | install_storm=true 50 | install_spark=true 51 | install_cassandra=true 52 | install_elasticsearch=true 53 | install_kibana=true 54 | install_logstash=true 55 | install_pig=true 56 | install_mysql=true 57 | install_hive=true 58 | install_wildfly=true 59 | install_zeppelin=true 60 | install_hbase=true 61 | install_flink=true 62 | -------------------------------------------------------------------------------- /invm.yml: -------------------------------------------------------------------------------- 1 | [hadoop-master-node-third] 2 | 192.168.33.38 ansible_connection=ssh ansible_ssh_user=vagrant api_hostname=hadoopmaster3 myid=3 ansible_ssh_private_key_file=~/.ssh/id_rsa 3 | 4 | [hadoop-master-node-second] 5 | 192.168.33.39 ansible_connection=ssh ansible_ssh_user=vagrant api_hostname=hadoopmaster2 myid=2 ansible_ssh_private_key_file=~/.ssh/id_rsa 6 | 7 | 8 | 9 | [hadoop-master-node] 10 | 192.168.33.40 server1=0.0.0.0 server2=192.168.33.41 server3=192.168.33.42 ansible_connection=ssh ansible_ssh_user=vagrant api_hostname=hadoopmaster myid=1 ansible_ssh_private_key_file=~/.ssh/id_rsa 11 | 12 | [hadoop-data1-node] 13 | 192.168.33.41 server1=192.168.33.40 server2=0.0.0.0 server3=192.168.33.42 ansible_connection=ssh ansible_ssh_user=vagrant myid=2 api_hostname=hadoopdata1 ansible_ssh_private_key_file=~/.ssh/id_rsa 14 | 15 | [hadoop-data2-node] 16 | 192.168.33.42 server1=192.168.33.40 server2=192.168.33.41 server3=0.0.0.0 ansible_connection=ssh ansible_ssh_user=vagrant myid=3 api_hostname=hadoopdata2 ansible_ssh_private_key_file=~/.ssh/id_rsa 17 | 18 | 19 | 20 | [hadoop-standard-nodes:children] 21 | hadoop-master-node 22 | hadoop-data1-node 23 | hadoop-data2-node 24 | 25 | 26 | [hadoop-master-namenodes:children] 27 | hadoop-master-node 28 | hadoop-master-node-second 29 | hadoop-master-node-third 30 | 31 | [hadoop-nodes:children] 32 | hadoop-data1-node 33 | hadoop-data2-node 34 | 35 | 36 | [all:vars] 37 | # Required variables for this installation 38 | HOME=/home/vagrant 39 | zk_urls=hadoopmaster:2181,hadoopdata1:2181,hadoopdata2:2181 40 | hadoop_gid=hadoop 41 | hadoop_uid=hadoop 42 | hadoopmaster=192.168.33.40 43 | cassandra_seeds=hadoopmaster,hadoopdata1,hadoopdata2 44 | 45 | # For selecting which modules to install. To enable set each variable below to true. To disable set to false 46 | 47 | #install_zookeeper=true 48 | #install_hadoop=true 49 | #install_kafka=true 50 | #install_storm=true 51 | install_spark=true 52 | #install_cassandra=true 53 | #install_elasticsearch=true 54 | #install_kibana=true 55 | #install_logstash=true 56 | #install_pig=true 57 | #install_mysql=true 58 | #install_hive=true 59 | #install_wildfly=true 60 | #install_zeppelin=true 61 | #install_hbase=true 62 | #install_flink=true 63 | #install_flume=true 64 | skip_shared=true 65 | -------------------------------------------------------------------------------- /roles/cassandra/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/cassandra/README.md: -------------------------------------------------------------------------------- 1 | Cassandra 2 | ========= 3 | 4 | This role was designed to provide a way to provision a cluster of cassandra. 5 | Requirements 6 | ------------ 7 | 8 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 9 | 10 | Role Variables 11 | -------------- 12 | 13 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 14 | 15 | Dependencies 16 | ------------ 17 | 18 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 19 | 20 | Example Playbook 21 | ---------------- 22 | 23 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 24 | 25 | - hosts: servers 26 | roles: 27 | - { role: username.rolename, x: 42 } 28 | 29 | License 30 | ------- 31 | 32 | BSD 33 | 34 | Author Information 35 | ------------------ 36 | 37 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 38 | -------------------------------------------------------------------------------- /roles/cassandra/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for cassandra 3 | -------------------------------------------------------------------------------- /roles/cassandra/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for cassandra 3 | - name: reload systemd 4 | sudo: yes 5 | command: systemctl {{ item }} 6 | with_items: 7 | - stop cassandra 8 | - daemon-reload 9 | - start cassandra 10 | -------------------------------------------------------------------------------- /roles/cassandra/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/cassandra/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for cassandra 3 | # http://apache.mirror.vexxhost.com/cassandra/3.7/apache-cassandra-3.7-bin.tar.gz 4 | # pgrep -u `whoami` -f cassandra | xargs kill -9 5 | # 6 | # For JMX Remote Monitoring 7 | # 8 | # service:jmx:rmi://0.0.0.0/jndi/rmi://0.0.0.0:7199/jmxrmi 9 | # creds = {"monitorRole", "mrpasswd"}; 10 | # java -jar jmxterm-1.0-alpha-4-uber.jar -l service:jmx:rmi://0.0.0.0/jndi/rmi://0.0.0.0:7199/jmxrmi -u monitorRole -p mrpasswd 11 | # 12 | # If you plan on using nodetool make sure your user/pass is the following: 13 | # nodetool -u controlRole -pw crpasswd info 14 | # Note: To change this just update the jmx.remote.access.j2 under ../template folder 15 | # 16 | 17 | 18 | - name: stat /tmp/apache-cassandra-3.10-bin.tar.gz 19 | stat: path=/tmp/apache-cassandra-3.10-bin.tar.gz 20 | register: cassandra_binary_stat 21 | 22 | - name: Downloading Apache Cassandra 23 | get_url: url=http://apache.parentingamerica.com/cassandra/3.10/apache-cassandra-3.10-bin.tar.gz dest=/tmp/apache-cassandra-3.10-bin.tar.gz 24 | when: cassandra_binary_stat.stat.exists == False 25 | 26 | - name: rename file 27 | command: chown hadoop:hadoop /tmp/apache-cassandra-3.10-bin.tar.gz 28 | 29 | 30 | - name: uncompress pig tarball and place in /usr/local/ 31 | command: tar zxf /tmp/apache-cassandra-3.10-bin.tar.gz -C /usr/local/ 32 | 33 | - name: rename file 34 | command: creates="/usr/local/cassandra" mv /usr/local/apache-cassandra-3.10 /usr/local/cassandra 35 | 36 | - name: change ownerships file 37 | command: chown -R hadoop:hadoop /usr/local/cassandra 38 | 39 | - name: setup cassandra.yaml file 40 | template: src=cassandra.yaml.j2 dest=/usr/local/cassandra/conf/cassandra.yaml 41 | 42 | - name: setup cassandra-env.sh file 43 | template: src=cassandra-env.sh.j2 dest=/usr/local/cassandra/conf/cassandra-env.sh 44 | 45 | - name: setup jmxremote.access file 46 | template: src=jmxremote.access.j2 dest=/usr/local/cassandra/conf/jmxremote.access 47 | 48 | - name: setup jmxremote.password file 49 | template: src=jmxremote.password.j2 dest=/usr/local/cassandra/conf/jmxremote.password 50 | 51 | 52 | - name: Downloading Jolokia Agent 53 | get_url: url=http://search.maven.org/remotecontent?filepath=org/jolokia/jolokia-jvm/1.3.3/jolokia-jvm-1.3.3-agent.jar dest=/usr/local/cassandra/lib/jolokia-jvm-1.3.3-agent.jar 54 | 55 | - name: setup systemd unit file 56 | template: src=cassandra.service.j2 dest=/lib/systemd/system/cassandra.service 57 | # notify: reload systemd 58 | 59 | # - name: enable cassandra 60 | # sudo: yes 61 | # service: name=cassandra enabled=yes state=started 62 | -------------------------------------------------------------------------------- /roles/cassandra/templates/cassandra.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Cassandra Node Start Up 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=hadoop 8 | Environment=CASSANDRA_HOME=/usr/local/cassandra JAVA_HOME=/usr/lib/jvm/java-1.8.0 9 | ExecStart=cassandra -f 10 | ExecStop=pgrep -u `whoami` -f cassandra | xargs kill -9 11 | Restart=on-abort 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /roles/cassandra/templates/jmxremote.access.j2: -------------------------------------------------------------------------------- 1 | monitorRole readonly 2 | controlRole readwrite 3 | -------------------------------------------------------------------------------- /roles/cassandra/templates/jmxremote.password.j2: -------------------------------------------------------------------------------- 1 | monitorRole mrpasswd 2 | controlRole crpasswd 3 | -------------------------------------------------------------------------------- /roles/cassandra/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/cassandra/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - cassandra -------------------------------------------------------------------------------- /roles/cassandra/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for cassandra 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/elasticsearch/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/elasticsearch/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for elasticsearch 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for elasticsearch 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for elasticsearch 3 | #https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.3/elasticsearch-2.3.3.tar.gz 4 | 5 | - name: stat /tmp/elasticsearch-2.3.3.tar.gz 6 | stat: path=/tmp/elasticsearch-2.3.3.tar.gz 7 | register: elastic_binary_stat 8 | 9 | - name: Downloading Elastic Search For Hadoop 10 | get_url: url=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.3/elasticsearch-2.3.3.tar.gz dest=/tmp/elasticsearch-2.3.3.tar.gz 11 | when: elastic_binary_stat.stat.exists == False 12 | 13 | - name: uncompress file tarball and place in /usr/local/ 14 | command: creates="/usr/local/elasticsearch-2.3.3" tar zxf /tmp/elasticsearch-2.3.3.tar.gz -C /usr/local/ 15 | 16 | - name: rename file 17 | command: creates="/usr/local/elasticsearch" mv /usr/local/elasticsearch-2.3.3 /usr/local/elasticsearch 18 | 19 | - name: change ownerships file 20 | command: chown -R hadoop:hadoop /usr/local/elasticsearch 21 | -------------------------------------------------------------------------------- /roles/elasticsearch/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/elasticsearch/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - elasticsearch -------------------------------------------------------------------------------- /roles/elasticsearch/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for elasticsearch 3 | -------------------------------------------------------------------------------- /roles/flink/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/flink/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/flink/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for flink 3 | -------------------------------------------------------------------------------- /roles/flink/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for flink 3 | -------------------------------------------------------------------------------- /roles/flink/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: OpenBSD 46 | # versions: 47 | # - all 48 | # - 5.6 49 | # - 5.7 50 | # - 5.8 51 | # - 5.9 52 | # - 6.0 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: MacOSX 73 | # versions: 74 | # - all 75 | # - 10.10 76 | # - 10.11 77 | # - 10.12 78 | # - 10.7 79 | # - 10.8 80 | # - 10.9 81 | #- name: IOS 82 | # versions: 83 | # - all 84 | # - any 85 | #- name: Solaris 86 | # versions: 87 | # - all 88 | # - 10 89 | # - 11.0 90 | # - 11.1 91 | # - 11.2 92 | # - 11.3 93 | #- name: SmartOS 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: eos 98 | # versions: 99 | # - all 100 | # - Any 101 | #- name: Windows 102 | # versions: 103 | # - all 104 | # - 2012R2 105 | #- name: Amazon 106 | # versions: 107 | # - all 108 | # - 2013.03 109 | # - 2013.09 110 | #- name: GenericBSD 111 | # versions: 112 | # - all 113 | # - any 114 | #- name: Junos 115 | # versions: 116 | # - all 117 | # - any 118 | #- name: FreeBSD 119 | # versions: 120 | # - all 121 | # - 10.0 122 | # - 10.1 123 | # - 10.2 124 | # - 10.3 125 | # - 8.0 126 | # - 8.1 127 | # - 8.2 128 | # - 8.3 129 | # - 8.4 130 | # - 9.0 131 | # - 9.1 132 | # - 9.1 133 | # - 9.2 134 | # - 9.3 135 | #- name: Ubuntu 136 | # versions: 137 | # - all 138 | # - lucid 139 | # - maverick 140 | # - natty 141 | # - oneiric 142 | # - precise 143 | # - quantal 144 | # - raring 145 | # - saucy 146 | # - trusty 147 | # - utopic 148 | # - vivid 149 | # - wily 150 | # - xenial 151 | #- name: SLES 152 | # versions: 153 | # - all 154 | # - 10SP3 155 | # - 10SP4 156 | # - 11 157 | # - 11SP1 158 | # - 11SP2 159 | # - 11SP3 160 | # - 11SP4 161 | # - 12 162 | # - 12SP1 163 | #- name: GenericLinux 164 | # versions: 165 | # - all 166 | # - any 167 | #- name: NXOS 168 | # versions: 169 | # - all 170 | # - any 171 | #- name: Debian 172 | # versions: 173 | # - all 174 | # - etch 175 | # - jessie 176 | # - lenny 177 | # - sid 178 | # - squeeze 179 | # - stretch 180 | # - wheezy 181 | 182 | galaxy_tags: [] 183 | # List tags for your role here, one per line. A tag is 184 | # a keyword that describes and categorizes the role. 185 | # Users find roles by searching for tags. Be sure to 186 | # remove the '[]' above if you add tags to this list. 187 | # 188 | # NOTE: A tag is limited to a single word comprised of 189 | # alphanumeric characters. Maximum 20 tags per role. 190 | 191 | dependencies: [] 192 | # List your role dependencies here, one per line. 193 | # Be sure to remove the '[]' above if you add dependencies 194 | # to this list. -------------------------------------------------------------------------------- /roles/flink/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for flink $FLINK_HOME/bin/start-cluster.sh 3 | # http://apache.mirror.gtcomm.net/flink/flink-1.0.3/flink-1.0.3-bin-hadoop2-scala_2.10.tgz 4 | 5 | 6 | - name: stat /tmp/flink-1.0.3-bin-hadoop2-scala_2.10.tgz 7 | stat: path=/tmp/flink-1.0.3-bin-hadoop2-scala_2.10.tgz 8 | register: flink_binary_stat 9 | 10 | - name: Downloading from mirror 11 | get_url: url=http://apache.mirror.gtcomm.net/flink/flink-1.0.3/flink-1.0.3-bin-hadoop2-scala_2.10.tgz dest=/tmp/flink-1.0.3-bin-hadoop2-scala_2.10.tgz 12 | when: flink_binary_stat.stat.exists == False 13 | 14 | - name: rename file 15 | command: chown hadoop:hadoop /tmp/flink-1.0.3-bin-hadoop2-scala_2.10.tgz 16 | 17 | - name: uncompress file tarball and place in /usr/local/ 18 | command: tar zxf /tmp/flink-1.0.3-bin-hadoop2-scala_2.10.tgz -C /usr/local/ 19 | 20 | - name: rename file 21 | command: creates="/usr/local/flink" mv /usr/local/flink-1.0.3/ /usr/local/flink 22 | 23 | - name: set permissions for installation placed in /usr/local/ 24 | command: chown -R hadoop:hadoop /usr/local/flink 25 | 26 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HBASE_HOME=" line="export HBASE_HOME=/usr/local/hbase" 27 | - lineinfile: dest=/home/hadoop/.bashrc regexp="FLINK_HOME=" line="export FLINK_HOME=/usr/local/flink" 28 | 29 | 30 | - name: setup configuration files 31 | template: src={{ item.name }}.j2 32 | dest=/usr/local/flink/conf/{{ item.name }} 33 | with_items: 34 | - { name: flink-conf.yaml } 35 | - { name: slaves } 36 | -------------------------------------------------------------------------------- /roles/flink/templates/flink-conf.yaml.j2: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | 20 | #============================================================================== 21 | # Common 22 | #============================================================================== 23 | 24 | # The host on which the JobManager runs. Only used in non-high-availability mode. 25 | # The JobManager process will use this hostname to bind the listening servers to. 26 | # The TaskManagers will try to connect to the JobManager on that host. 27 | 28 | jobmanager.rpc.address: hadoopmaster 29 | 30 | 31 | # The port where the JobManager's main actor system listens for messages. 32 | 33 | jobmanager.rpc.port: 6123 34 | 35 | 36 | # The heap size for the JobManager JVM 37 | 38 | jobmanager.heap.mb: 256 39 | 40 | 41 | # The heap size for the TaskManager JVM 42 | 43 | taskmanager.heap.mb: 512 44 | 45 | 46 | # The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline. 47 | 48 | taskmanager.numberOfTaskSlots: 1 49 | 50 | # Specify whether TaskManager memory should be allocated when starting up (true) or when 51 | # memory is required in the memory manager (false) 52 | 53 | taskmanager.memory.preallocate: false 54 | 55 | # The parallelism used for programs that did not specify and other parallelism. 56 | 57 | parallelism.default: 1 58 | 59 | 60 | #============================================================================== 61 | # Web Frontend 62 | #============================================================================== 63 | 64 | # The port under which the web-based runtime monitor listens. 65 | # A value of -1 deactivates the web server. 66 | 67 | jobmanager.web.port: 8081 68 | 69 | # Flag to specify whether job submission is enabled from the web-based 70 | # runtime monitor. Uncomment to disable. 71 | 72 | #jobmanager.web.submit.enable: false 73 | 74 | 75 | #============================================================================== 76 | # Streaming state checkpointing 77 | #============================================================================== 78 | 79 | # The backend that will be used to store operator state checkpoints if 80 | # checkpointing is enabled. 81 | # 82 | # Supported backends: jobmanager, filesystem, 83 | # 84 | #state.backend: filesystem 85 | 86 | 87 | # Directory for storing checkpoints in a Flink-supported filesystem 88 | # Note: State backend must be accessible from the JobManager and all TaskManagers. 89 | # Use "hdfs://" for HDFS setups, "file://" for UNIX/POSIX-compliant file systems, 90 | # (or any local file system under Windows), or "S3://" for S3 file system. 91 | # 92 | # state.backend.fs.checkpointdir: hdfs://namenode-host:port/flink-checkpoints 93 | 94 | 95 | #============================================================================== 96 | # Advanced 97 | #============================================================================== 98 | 99 | # The number of buffers for the network stack. 100 | # 101 | # taskmanager.network.numberOfBuffers: 2048 102 | 103 | 104 | # Directories for temporary files. 105 | # 106 | # Add a delimited list for multiple directories, using the system directory 107 | # delimiter (colon ':' on unix) or a comma, e.g.: 108 | # /data1/tmp:/data2/tmp:/data3/tmp 109 | # 110 | # Note: Each directory entry is read from and written to by a different I/O 111 | # thread. You can include the same directory multiple times in order to create 112 | # multiple I/O threads against that directory. This is for example relevant for 113 | # high-throughput RAIDs. 114 | # 115 | # If not specified, the system-specific Java temporary directory (java.io.tmpdir 116 | # property) is taken. 117 | # 118 | # taskmanager.tmp.dirs: /tmp 119 | 120 | 121 | # Path to the Hadoop configuration directory. 122 | # 123 | # This configuration is used when writing into HDFS. Unless specified otherwise, 124 | # HDFS file creation will use HDFS default settings with respect to block-size, 125 | # replication factor, etc. 126 | # 127 | # You can also directly specify the paths to hdfs-default.xml and hdfs-site.xml 128 | # via keys 'fs.hdfs.hdfsdefault' and 'fs.hdfs.hdfssite'. 129 | # 130 | # fs.hdfs.hadoopconf: /path/to/hadoop/conf/ 131 | 132 | 133 | #============================================================================== 134 | # Master High Availability (required configuration) 135 | #============================================================================== 136 | 137 | # The list of ZooKepper quorum peers that coordinate the high-availability 138 | # setup. This must be a list of the form: 139 | # "host1:clientPort,host2[:clientPort],..." (default clientPort: 2181) 140 | # 141 | # recovery.mode: zookeeper 142 | # 143 | # recovery.zookeeper.quorum: localhost:2181,... 144 | # 145 | # Note: You need to set the state backend to 'filesystem' and the checkpoint 146 | # directory (see above) before configuring the storageDir. 147 | # 148 | # recovery.zookeeper.storageDir: hdfs:///recovery 149 | -------------------------------------------------------------------------------- /roles/flink/templates/slaves.j2: -------------------------------------------------------------------------------- 1 | hadoopdata1 2 | hadoopdata2 3 | -------------------------------------------------------------------------------- /roles/flink/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/flink/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - flink -------------------------------------------------------------------------------- /roles/flink/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for flink 3 | -------------------------------------------------------------------------------- /roles/flume/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/flume/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/flume/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for flume 3 | -------------------------------------------------------------------------------- /roles/flume/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for flume 3 | -------------------------------------------------------------------------------- /roles/flume/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: OpenBSD 46 | # versions: 47 | # - all 48 | # - 5.6 49 | # - 5.7 50 | # - 5.8 51 | # - 5.9 52 | # - 6.0 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: MacOSX 73 | # versions: 74 | # - all 75 | # - 10.10 76 | # - 10.11 77 | # - 10.12 78 | # - 10.7 79 | # - 10.8 80 | # - 10.9 81 | #- name: IOS 82 | # versions: 83 | # - all 84 | # - any 85 | #- name: Solaris 86 | # versions: 87 | # - all 88 | # - 10 89 | # - 11.0 90 | # - 11.1 91 | # - 11.2 92 | # - 11.3 93 | #- name: SmartOS 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: eos 98 | # versions: 99 | # - all 100 | # - Any 101 | #- name: Windows 102 | # versions: 103 | # - all 104 | # - 2012R2 105 | #- name: Amazon 106 | # versions: 107 | # - all 108 | # - 2013.03 109 | # - 2013.09 110 | #- name: GenericBSD 111 | # versions: 112 | # - all 113 | # - any 114 | #- name: Junos 115 | # versions: 116 | # - all 117 | # - any 118 | #- name: FreeBSD 119 | # versions: 120 | # - all 121 | # - 10.0 122 | # - 10.1 123 | # - 10.2 124 | # - 10.3 125 | # - 8.0 126 | # - 8.1 127 | # - 8.2 128 | # - 8.3 129 | # - 8.4 130 | # - 9.0 131 | # - 9.1 132 | # - 9.1 133 | # - 9.2 134 | # - 9.3 135 | #- name: Ubuntu 136 | # versions: 137 | # - all 138 | # - lucid 139 | # - maverick 140 | # - natty 141 | # - oneiric 142 | # - precise 143 | # - quantal 144 | # - raring 145 | # - saucy 146 | # - trusty 147 | # - utopic 148 | # - vivid 149 | # - wily 150 | # - xenial 151 | #- name: SLES 152 | # versions: 153 | # - all 154 | # - 10SP3 155 | # - 10SP4 156 | # - 11 157 | # - 11SP1 158 | # - 11SP2 159 | # - 11SP3 160 | # - 11SP4 161 | # - 12 162 | # - 12SP1 163 | #- name: GenericLinux 164 | # versions: 165 | # - all 166 | # - any 167 | #- name: NXOS 168 | # versions: 169 | # - all 170 | # - any 171 | #- name: Debian 172 | # versions: 173 | # - all 174 | # - etch 175 | # - jessie 176 | # - lenny 177 | # - sid 178 | # - squeeze 179 | # - stretch 180 | # - wheezy 181 | 182 | galaxy_tags: [] 183 | # List tags for your role here, one per line. A tag is 184 | # a keyword that describes and categorizes the role. 185 | # Users find roles by searching for tags. Be sure to 186 | # remove the '[]' above if you add tags to this list. 187 | # 188 | # NOTE: A tag is limited to a single word comprised of 189 | # alphanumeric characters. Maximum 20 tags per role. 190 | 191 | dependencies: [] 192 | # List your role dependencies here, one per line. 193 | # Be sure to remove the '[]' above if you add dependencies 194 | # to this list. -------------------------------------------------------------------------------- /roles/flume/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for flume http://apachemirror.ovidiudan.com/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz 3 | 4 | - name: stat /tmp/apache-flume-1.6.0-bin.tar.gz 5 | stat: path=/tmp/apache-flume-1.6.0-bin.tar.gz 6 | register: flume_binary_stat 7 | 8 | - name: Downloading Apache Pig 9 | get_url: url=http://apachemirror.ovidiudan.com/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz dest=/tmp/apache-flume-1.6.0-bin.tar.gz 10 | when: flume_binary_stat.stat.exists == False 11 | 12 | - name: rename file 13 | command: chown hadoop:hadoop /tmp/apache-flume-1.6.0-bin.tar.gz 14 | 15 | - name: uncompress pig tarball and place in /usr/local/ 16 | command: tar zxf /tmp/apache-flume-1.6.0-bin.tar.gz -C /usr/local/ 17 | 18 | - name: rename file 19 | command: creates="/usr/local/flume" mv /usr/local/apache-flume-1.6.0-bin/ /usr/local/flume 20 | 21 | - name: uncompress tarball and place in /usr/local/ 22 | command: chown -R hadoop:hadoop /usr/local/flume 23 | 24 | - lineinfile: dest=/home/hadoop/.bashrc regexp="FLUME_HOME=" line="export FLUME_HOME=/usr/local/flume" 25 | -------------------------------------------------------------------------------- /roles/flume/templates/customer.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "record", 3 | "name" : "GeneratedCustomerBean", 4 | "namespace" : "com.redhat.data.analytics.generated", 5 | "doc" : "Schema for com.redhat.data.analytics.model.CustomerBean", 6 | "fields" : [ { 7 | "name" : "id", 8 | "type" : "int" 9 | }, { 10 | "name" : "name", 11 | "type" : [ "null", "string" ] 12 | } ] 13 | } 14 | -------------------------------------------------------------------------------- /roles/flume/templates/sample_conf: -------------------------------------------------------------------------------- 1 | agent.sources = avro-listener 2 | agent.sources.avro-listener.type=avro 3 | agent.sources.avro-listener.port=41415 4 | agent.sources.avro-listener.channel=my-event-channel 5 | -------------------------------------------------------------------------------- /roles/flume/templates/sink-flume.j2: -------------------------------------------------------------------------------- 1 | agent.sinks.hdfs.type=hdfs 2 | # Very important, *DO NOT* use CompressedStream. Avro itself will do the compression 3 | agent.sinks.hdfs.hdfs.fileType=DataStream 4 | # *MUST* be set to .avro for Hive to work 5 | agent.sinks.hdfs.hdfs.fileSuffix=.avro 6 | # Of course choose your own path 7 | agent.sinks.hdfs.hdfs.path=hdfs://hadoopmaster/customer/datain/key=%{some_partition} 8 | agent.sinks.hdfs.hdfs.writeFormat=Text 9 | # The magic happens here: 10 | agent.sinks.hdfs.serializer=avro_event 11 | agent.sinks.hdfs.serializer.compressionCodec=snappy 12 | -------------------------------------------------------------------------------- /roles/flume/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/flume/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - flume -------------------------------------------------------------------------------- /roles/flume/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for flume 3 | -------------------------------------------------------------------------------- /roles/hadoop/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/hadoop/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for hadoop 3 | -------------------------------------------------------------------------------- /roles/hadoop/files/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEApDElSg8JTkY4ZVmaq5IZEwNPjif7N2f/zkrVFxHVj0Ods7Eb 3 | cOgoavQt249ulSvDy7N/nS2oHaM5VH+PfkdcAQyvVlJn+vPN4OZiIfO7TTYRml8d 4 | z6L2lKA7S2d7u0kiYwOKNlykWNHGFCaGEETr+hFsgvvfn9FmjdD20X4ikWo2/0oG 5 | fxpiGzlEC5Odls5MF2D9cMZ6AXCduzFC61lVYsYcMqwB8oAx7QJA4h5LkVHYBmrs 6 | vu/lz7F6R0FFO3qN3gZ+lZMRe8SbKrp/GiibUbuuw77Xz+XVdRSgkOC8Nsr9iYd+ 7 | 7lq/BNvoNmmTf2VWndxj3IVWjDQPnTcdoSXvBQIDAQABAoIBAQCi3d0PSTsB2cW3 8 | QtqZtaIJqg8QA7i9jPvBtZkJ+TbNlVSpGdUOejsdXHjggbY/YdomBTdgwhkwzOfh 9 | WWcM69vnbp2+Ep/x3kUqAVz2ML5qKGftqMv5Wr0dUJOAiHXDoRfq/2fp5JthPzQs 10 | aMoy+SraMs5gX1fDQANqSQ2LYIMJmauXPYffrnkuGhrtKbrQ58XCN+Qg7pw2TCbh 11 | HBTWYt5GzqcX/Jf54YZD+3rdUaXsxX458ty5D6mbZzZfo0/jICpby8GDsLI6Ch0s 12 | vzaO43eEkyCcDweEAy1yfGhPTG1Z5Tfr6Zel0y3Z/JzInd/m4de6V7dE346KxlBC 13 | rMnJB6YBAoGBAM2lCTApUFU1jSa1Rk6yEPxc2Dp/lAXcNPvdhRe8VzHiqvYfsln5 14 | 9OyrtbABb0jyxi8tQIWFQ1hCbDGLbjD3yhTpK+kwk/xQTyB1lmHDKJ3Q8yS3xqIh 15 | 8za1Aut6oew32JSim/2nd3FIx704hQabYuhGnJfh9C4PnwgUxSdMTPUtAoGBAMxl 16 | nynEqF6Ri1Co4vueWYtLZeSt9QhR73eZPA4AWjR5cHglnGetdush6hbsWpE2OROF 17 | vT9rpz/yJINd56884mJqZumDCteBiq86mv86y8txflvdiMxRmXVlHEAJHwG/PG/D 18 | 8drQI46UqGXyQeTkwjtR1S55H1pZFviAdG7317g5AoGBALG7y78mVdnvWLl1vvhR 19 | CkEy9WdJ0GNoORreECO9khb5cw0ZxAnj8P2EQpmGA+qmeQdZMwSzcssbTWtcuLX1 20 | 02BTTrJSW/ScbGb/eiOwirArYZoAg9qpQQ2tUIK5EK2absO37t5CGQd+AbPW35Ig 21 | RlG+haT3kxUkRyyy1trqJTFZAoGAI4xRACyps7jQZ/jXcTd7D1TPpdPzV70YTs6h 22 | uTWqepHcU6kG6Kksk3xUgCQfJD4v8wtBfLQ2ZFKysv4z+zA1lzQ2LcqSqiRngE5L 23 | ArSxGPLmLQf8cLyIWfh5z1FGyUD/O9jySUIFzRhhqUlW2H5IoZR75Rc/srDn49RD 24 | a8FDDVECgYEAxShc0KAz/CIhvFojlrQmjYcsB+u/8FgE9x3J2PKDu21jXQ55kjQk 25 | MfOP1z/SsCwKqq48v3SBda6PwuBrN17utTCT82XhjroOroXINrzG3UqBIuYV2FNq 26 | rx5pPwzj6fb9siIrDHlPMRYavK0WzdkmgK5iSqhs+vcefR830wuTFA4= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /roles/hadoop/files/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCkMSVKDwlORjhlWZqrkhkTA0+OJ/s3Z//OStUXEdWPQ52zsRtw6Chq9C3bj26VK8PLs3+dLagdozlUf49+R1wBDK9WUmf6883g5mIh87tNNhGaXx3PovaUoDtLZ3u7SSJjA4o2XKRY0cYUJoYQROv6EWyC+9+f0WaN0PbRfiKRajb/SgZ/GmIbOUQLk52WzkwXYP1wxnoBcJ27MULrWVVixhwyrAHygDHtAkDiHkuRUdgGauy+7+XPsXpHQUU7eo3eBn6VkxF7xJsqun8aKJtRu67DvtfP5dV1FKCQ4Lw2yv2Jh37uWr8E2+g2aZN/ZVad3GPchVaMNA+dNx2hJe8F zhassan@zhassan.yyz.redhat.com 2 | -------------------------------------------------------------------------------- /roles/hadoop/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for hadoop 3 | - name: reload systemd 4 | sudo: yes 5 | command: systemctl daemon-reload 6 | -------------------------------------------------------------------------------- /roles/hadoop/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: your name 4 | description: 5 | company: your company (optional) 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | # Some suggested licenses: 10 | # - BSD (default) 11 | # - MIT 12 | # - GPLv2 13 | # - GPLv3 14 | # - Apache 15 | # - CC-BY 16 | license: license (GPLv2, CC-BY, etc) 17 | min_ansible_version: 1.2 18 | # 19 | # Below are all platforms currently available. Just uncomment 20 | # the ones that apply to your role. If you don't see your 21 | # platform on this list, let us know and we'll get it added! 22 | # 23 | #platforms: 24 | #- name: EL 25 | # versions: 26 | # - all 27 | # - 5 28 | # - 6 29 | # - 7 30 | #- name: GenericUNIX 31 | # versions: 32 | # - all 33 | # - any 34 | #- name: Solaris 35 | # versions: 36 | # - all 37 | # - 10 38 | # - 11.0 39 | # - 11.1 40 | # - 11.2 41 | # - 11.3 42 | #- name: Fedora 43 | # versions: 44 | # - all 45 | # - 16 46 | # - 17 47 | # - 18 48 | # - 19 49 | # - 20 50 | # - 21 51 | # - 22 52 | # - 23 53 | #- name: Windows 54 | # versions: 55 | # - all 56 | # - 2012R2 57 | #- name: SmartOS 58 | # versions: 59 | # - all 60 | # - any 61 | #- name: opensuse 62 | # versions: 63 | # - all 64 | # - 12.1 65 | # - 12.2 66 | # - 12.3 67 | # - 13.1 68 | # - 13.2 69 | #- name: Amazon 70 | # versions: 71 | # - all 72 | # - 2013.03 73 | # - 2013.09 74 | #- name: GenericBSD 75 | # versions: 76 | # - all 77 | # - any 78 | #- name: FreeBSD 79 | # versions: 80 | # - all 81 | # - 10.0 82 | # - 10.1 83 | # - 10.2 84 | # - 8.0 85 | # - 8.1 86 | # - 8.2 87 | # - 8.3 88 | # - 8.4 89 | # - 9.0 90 | # - 9.1 91 | # - 9.1 92 | # - 9.2 93 | # - 9.3 94 | #- name: Ubuntu 95 | # versions: 96 | # - all 97 | # - lucid 98 | # - maverick 99 | # - natty 100 | # - oneiric 101 | # - precise 102 | # - quantal 103 | # - raring 104 | # - saucy 105 | # - trusty 106 | # - utopic 107 | # - vivid 108 | # - wily 109 | #- name: SLES 110 | # versions: 111 | # - all 112 | # - 10SP3 113 | # - 10SP4 114 | # - 11 115 | # - 11SP1 116 | # - 11SP2 117 | # - 11SP3 118 | #- name: GenericLinux 119 | # versions: 120 | # - all 121 | # - any 122 | #- name: Debian 123 | # versions: 124 | # - all 125 | # - etch 126 | # - jessie 127 | # - lenny 128 | # - squeeze 129 | # - wheezy 130 | # 131 | # Below are all categories currently available. Just as with 132 | # the platforms above, uncomment those that apply to your role. 133 | # 134 | #categories: 135 | #- cloud 136 | #- cloud:ec2 137 | #- cloud:gce 138 | #- cloud:rax 139 | #- clustering 140 | #- database 141 | #- database:nosql 142 | #- database:sql 143 | #- development 144 | #- monitoring 145 | #- networking 146 | #- packaging 147 | #- system 148 | #- web 149 | dependencies: [] 150 | # List your role dependencies here, one per line. 151 | # Be sure to remove the '[]' above if you add dependencies 152 | # to this list. 153 | 154 | -------------------------------------------------------------------------------- /roles/hadoop/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ######################################################## 3 | # tasks setting up hadoop (only supports single node). I'm working on making it work across cluster. 4 | # Author: Zak Hassan 5 | # Usage: ansible-playbook site.yml or make install 6 | # 7 | # 8 | # Description: Playbook created to demonstrate provisioning cloud environment in like less than 1 min. :) 9 | ##################################################### 10 | 11 | - group: name={{ hadoop_gid }} state=present 12 | 13 | - user: name={{ hadoop_uid }} comment="hadoop" group={{ hadoop_gid }} shell=/bin/bash 14 | 15 | 16 | - name: stat /tmp/hadoop-2.7.1.tar.gz 17 | stat: path=/tmp/hadoop-2.7.1.tar.gz 18 | register: hadoop_binary_stat 19 | 20 | - name: Downloading Apache Hadoop 21 | get_url: url=http://mirror.csclub.uwaterloo.ca/apache/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz dest=/tmp/hadoop-2.7.1.tar.gz 22 | when: hadoop_binary_stat.stat.exists == False 23 | 24 | 25 | 26 | # So hadoop can ssh into alternate servers using hadoop acount 27 | # disabling ssh known key check not for production usage. 28 | - file: path=/home/hadoop/.ssh state=directory mode=0755 29 | 30 | 31 | - file: path=/home/hadoop/hadoop-data/hdfs/namenode state=directory mode=0755 32 | - file: path=/home/hadoop/hadoop-data/hdfs/datanode state=directory mode=0755 33 | # TODO: Need to change permissions in this folder so hadoop can create files in these folders for nodes 34 | - command: sudo chown -Rv hadoop:hadoop /home/hadoop/hadoop-data/ 35 | 36 | - name: setup hosts file 37 | template: src=ssh.config.j2 dest=/home/hadoop/.ssh/config 38 | 39 | - name: change ownership file 40 | command: chown hadoop:hadoop /tmp/hadoop-2.7.1.tar.gz 41 | 42 | - authorized_key: user=hadoop key="{{ lookup('file', 'id_rsa.pub') }}" 43 | 44 | - name: copy hdfs_rsa key 45 | copy: src=id_rsa dest=/home/hadoop/.ssh group={{ hadoop_gid }} 46 | 47 | - name: Uncompress Apache Hadoop tarball and place in /usr/local/ 48 | command: tar zxf /tmp/hadoop-2.7.1.tar.gz -C /usr/local/ 49 | 50 | - name: stat {{ hadoop_install_dir }} 51 | stat: path={{ hadoop_install_dir }} 52 | register: hd_dir_stat 53 | 54 | - name: rename file 55 | command: creates="{{ hadoop_install_dir }}" mv {{ hadoop_prefix_dir }}/{{ hadoop_version }}/ {{ hadoop_install_dir }} 56 | 57 | - name: change permissions to {{ hadoop_uid }} user with {{ hadoop_gid }} group for {{ hadoop_version }} 58 | command: chown -R {{ hadoop_uid }}:{{ hadoop_gid }} {{ hadoop_install_dir }} 59 | # TODO: Need to use with_items here ... 60 | - name: setup core-site.xml file 61 | template: src={{ item.name }}.j2 62 | dest=/usr/local/hadoop/etc/hadoop/{{ item.name }} 63 | with_items: 64 | - { name: core-site.xml } 65 | - { name: hdfs-site.xml } 66 | - { name: mapred-site.xml } 67 | - { name: yarn-site.xml } 68 | tags: 69 | - hadoop_conf 70 | 71 | # TODO: Need to create http://docs.ansible.com/ansible/authorized_key_module.html 72 | # So hadoop can ssh into alternate servers using hadoop acount 73 | # TODO: Add when condition to these lineline file changes so if Hive isn't installed don't need to set its env var. 74 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HADOOP_HOME=" line="export HADOOP_HOME=/usr/local/hadoop" 75 | - lineinfile: dest=/home/hadoop/.bashrc regexp="JAVA_HOME=" line="export JAVA_HOME=/usr/lib/jvm/java-1.8.0" 76 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HIVE_HOME=" line="export HIVE_HOME=/usr/local/hive" 77 | - lineinfile: dest=/home/hadoop/.bashrc regexp="PIG_HOME=" line="export PIG_HOME=/usr/local/pig" 78 | - lineinfile: dest=/home/hadoop/.bashrc regexp="STORM_HOME=" line="export STORM_HOME=/usr/local/storm" 79 | - lineinfile: dest=/home/hadoop/.bashrc regexp="SPARK_HOME=" line="export SPARK_HOME=/usr/local/spark" 80 | - lineinfile: dest=/home/hadoop/.bashrc regexp="CASSANDRA_HOME=" line="export CASSANDRA_HOME=/usr/local/cassandra" 81 | - lineinfile: dest=/home/hadoop/.bashrc regexp="JBOSS_HOME=" line="export JBOSS_HOME=/usr/local/wildfly" 82 | 83 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HADOOP_CONF_DIR=" line="export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop" 84 | 85 | - lineinfile: dest=/home/hadoop/.bashrc regexp="PATH=" line="export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PIG_HOME/bin:$STORM_HOME/bin:$SPARK_HOME/bin:$CASSANDRA_HOME/bin:$JBOSS_HOME/bin" 86 | 87 | 88 | # - name: need to init env variables 89 | # command: source /home/vagrant/.bashrc 90 | # when: api_hostname == "hadoopmaster" 91 | # 92 | # 93 | # # only should be done once. You can skip this task by skip-tag in normal ansible provisioning 94 | # - name: format namenode only done once. 95 | # command: /usr/local/hadoop/bin/hdfs namenode -format 96 | # when: api_hostname == "hadoopmaster" 97 | # tags: 98 | # - hdfs_namenode_format 99 | 100 | # Create systemd files 101 | 102 | - name: copying slaves into conf location 103 | template: src=slaves dest=/usr/local/hadoop/etc/hadoop/slaves owner={{ hadoop_uid }} group={{ hadoop_gid }} 104 | when: api_hostname == "hadoopmaster" 105 | - name: setup systemd unit file 106 | template: src={{ item.name }}.j2 dest=/lib/systemd/system/{{ item.name }} 107 | with_items: 108 | - { name: hadoop-dfs.service } 109 | - { name: hadoop-yarn.service } 110 | - { name: hadoop-job-history.service } 111 | notify: reload systemd 112 | 113 | # export HADOOP_PREFIX=/usr/local/hadoop 114 | # export PATH=/usr/local/hadoop/bin:/usr/local/hadoop/sbin:$PATH 115 | # TODO: hdfs namenode -format & $HADOOP_PREFIX/sbin/start-dfs.sh & $HADOOP_PREFIX/sbin/start-yarn.sh 116 | -------------------------------------------------------------------------------- /roles/hadoop/templates/core-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | hadoop.tmp.dir 22 | /home/{{ hadoop_uid }}/tmp 23 | 24 | 25 | fs.default.name 26 | hdfs://{{ hadoopmaster }}:54310 27 | 28 | 29 | -------------------------------------------------------------------------------- /roles/hadoop/templates/hadoop-dfs.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Hadoop NameNode And DataNode Service 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=hadoop 8 | Environment=HADOOP_PREFIX={{ hadoop_install_dir }} JAVA_HOME=/usr/lib/jvm/java-1.8.0 9 | ExecStart={{ hadoop_install_dir }}/sbin/start-dfs.sh 10 | ExecStop={{ hadoop_install_dir }}/sbin/stop-dfs.sh 11 | Restart=on-abort 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /roles/hadoop/templates/hadoop-job-history.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Hadoop Job History Service 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=hadoop 8 | Environment=HADOOP_PREFIX={{ hadoop_install_dir }} JAVA_HOME=/usr/lib/jvm/java-1.8.0 9 | ExecStart={{ hadoop_install_dir }}/sbin/mr-jobhistory-daemon.sh start historyserver 10 | ExecStart={{ hadoop_install_dir }}/sbin/mr-jobhistory-daemon.sh stop historyserver 11 | Restart=on-abort 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /roles/hadoop/templates/hadoop-yarn.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Hadoop Yarn Resource Manager Service 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=hadoop 8 | Environment=HADOOP_PREFIX={{ hadoop_install_dir }} JAVA_HOME=/usr/lib/jvm/java-1.8.0 9 | ExecStart={{ hadoop_install_dir }}/sbin/start-yarn.sh 10 | ExecStart={{ hadoop_install_dir }}/sbin/stop-yarn.sh 11 | Restart=on-abort 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /roles/hadoop/templates/hadoop_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWeJfgWx7hDeZUJOeaIVzcbmYxzMcWfxhgC2975tvGL5BV6unzLz8ZVak6ju++AvnM5mcQp6Ydv73uWyaoQaFZigAzfuenruQkwc7D5YYuba+FgZdQ8VHon29oQA3iaZWG7xTspagrfq3fcqaz2ZIjzqN+E/MtcW08PwfibN2QRWchBCuZ1Q8AmrW7gClzMcgd/uj3TstabspGaaZMCs8aC9JWzZlMMegXKYHvVQs6xH2AmifpKpLoMTdO8jP4jczmGebPzvaXmvVylgwo6bRJ3tyYAmGwx8PHj2EVVQ0XX9ipgixLyAa2c7+/crPpGmKFRrYibCCT6x65px7nWnn3 -------------------------------------------------------------------------------- /roles/hadoop/templates/hdfs-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | dfs.replication 21 | 2 22 | 23 | 24 | dfs.namenode.name.dir 25 | /home/{{ hadoop_uid }}/hadoop-data/hdfs/namenode 26 | 27 | 28 | dfs.datanode.data.dir 29 | /home/{{ hadoop_uid }}/hadoop-data/hdfs/datanode 30 | 31 | 32 | -------------------------------------------------------------------------------- /roles/hadoop/templates/mapred-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | mapred.job.tracker 21 | {{ hadoopmaster }}:54311 22 | 23 | 24 | mapreduce.framework.name 25 | yarn 26 | 27 | 28 | -------------------------------------------------------------------------------- /roles/hadoop/templates/slaves: -------------------------------------------------------------------------------- 1 | {% for node in servers %} 2 | {{ node['name'] }} 3 | {% endfor %} 4 | -------------------------------------------------------------------------------- /roles/hadoop/templates/ssh.config.j2: -------------------------------------------------------------------------------- 1 | Host hadoopmaster 2 | ForwardAgent yes 3 | ControlMaster auto 4 | ControlPersist 30m 5 | ControlPath /tmp/%r@%h:%p 6 | 7 | Host hadoopdata1 8 | ForwardAgent yes 9 | ControlMaster auto 10 | ControlPersist 30m 11 | ControlPath /tmp/%r@%h:%p 12 | 13 | Host hadoopdata2 14 | ForwardAgent yes 15 | ControlMaster auto 16 | ControlPersist 30m 17 | ControlPath /tmp/%r@%h:%p 18 | -------------------------------------------------------------------------------- /roles/hadoop/templates/yarn-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | yarn.nodemanager.aux-services 19 | mapreduce_shuffle 20 | 21 | 22 | yarn.resourcemanager.scheduler.address 23 | {{ hadoopmaster }}:8030 24 | 25 | 26 | yarn.resourcemanager.address 27 | {{ hadoopmaster }}:8032 28 | 29 | 30 | yarn.resourcemanager.webapp.address 31 | {{ hadoopmaster }}:8088 32 | 33 | 34 | yarn.resourcemanager.resource-tracker.address 35 | {{ hadoopmaster }}:8031 36 | 37 | 38 | yarn.resourcemanager.admin.address 39 | {{ hadoopmaster }}:8033 40 | 41 | 42 | -------------------------------------------------------------------------------- /roles/hadoop/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for hadoop 3 | hadoop_binary: hadoop-2.7.1.tar.gz 4 | hadoop_install_dir: /usr/local/hadoop 5 | hadoop_prefix_dir: /usr/local 6 | hadoop_version: hadoop-2.7.1 7 | hadoop_uid: hadoop 8 | hadoop_gid: hadoop 9 | hadoop_apache_mirror: http://apache.sunsite.ualberta.ca/hadoop/common 10 | servers: 11 | - {name: hadoopdata1, ip: 192.168.33.41 } 12 | - {name: hadoopdata2, ip: 192.168.33.42 } 13 | # - {name: hadoopmaster, ip: 192.168.33.40 } 14 | -------------------------------------------------------------------------------- /roles/hbase/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/hbase/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/hbase/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for hbase 3 | -------------------------------------------------------------------------------- /roles/hbase/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for hbase 3 | -------------------------------------------------------------------------------- /roles/hbase/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: OpenBSD 46 | # versions: 47 | # - all 48 | # - 5.6 49 | # - 5.7 50 | # - 5.8 51 | # - 5.9 52 | # - 6.0 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: MacOSX 73 | # versions: 74 | # - all 75 | # - 10.10 76 | # - 10.11 77 | # - 10.12 78 | # - 10.7 79 | # - 10.8 80 | # - 10.9 81 | #- name: IOS 82 | # versions: 83 | # - all 84 | # - any 85 | #- name: Solaris 86 | # versions: 87 | # - all 88 | # - 10 89 | # - 11.0 90 | # - 11.1 91 | # - 11.2 92 | # - 11.3 93 | #- name: SmartOS 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: eos 98 | # versions: 99 | # - all 100 | # - Any 101 | #- name: Windows 102 | # versions: 103 | # - all 104 | # - 2012R2 105 | #- name: Amazon 106 | # versions: 107 | # - all 108 | # - 2013.03 109 | # - 2013.09 110 | #- name: GenericBSD 111 | # versions: 112 | # - all 113 | # - any 114 | #- name: Junos 115 | # versions: 116 | # - all 117 | # - any 118 | #- name: FreeBSD 119 | # versions: 120 | # - all 121 | # - 10.0 122 | # - 10.1 123 | # - 10.2 124 | # - 10.3 125 | # - 8.0 126 | # - 8.1 127 | # - 8.2 128 | # - 8.3 129 | # - 8.4 130 | # - 9.0 131 | # - 9.1 132 | # - 9.1 133 | # - 9.2 134 | # - 9.3 135 | #- name: Ubuntu 136 | # versions: 137 | # - all 138 | # - lucid 139 | # - maverick 140 | # - natty 141 | # - oneiric 142 | # - precise 143 | # - quantal 144 | # - raring 145 | # - saucy 146 | # - trusty 147 | # - utopic 148 | # - vivid 149 | # - wily 150 | # - xenial 151 | #- name: SLES 152 | # versions: 153 | # - all 154 | # - 10SP3 155 | # - 10SP4 156 | # - 11 157 | # - 11SP1 158 | # - 11SP2 159 | # - 11SP3 160 | # - 11SP4 161 | # - 12 162 | # - 12SP1 163 | #- name: GenericLinux 164 | # versions: 165 | # - all 166 | # - any 167 | #- name: NXOS 168 | # versions: 169 | # - all 170 | # - any 171 | #- name: Debian 172 | # versions: 173 | # - all 174 | # - etch 175 | # - jessie 176 | # - lenny 177 | # - sid 178 | # - squeeze 179 | # - stretch 180 | # - wheezy 181 | 182 | galaxy_tags: [] 183 | # List tags for your role here, one per line. A tag is 184 | # a keyword that describes and categorizes the role. 185 | # Users find roles by searching for tags. Be sure to 186 | # remove the '[]' above if you add tags to this list. 187 | # 188 | # NOTE: A tag is limited to a single word comprised of 189 | # alphanumeric characters. Maximum 20 tags per role. 190 | 191 | dependencies: [] 192 | # List your role dependencies here, one per line. 193 | # Be sure to remove the '[]' above if you add dependencies 194 | # to this list. -------------------------------------------------------------------------------- /roles/hbase/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for hbase # $HBASE_HOME/bin/start-hbase.sh && $HBASE_HOME/bin/hbase.sh shell 3 | 4 | 5 | - name: stat /tmp/hbase-1.2.3-bin.tar.gz 6 | stat: path=/tmp/hbase-1.2.3-bin.tar.gz 7 | register: hbase_binary_stat 8 | 9 | - name: Downloading Apache Hbase 10 | get_url: url=http://mirror.its.dal.ca/apache/hbase/stable/hbase-1.2.3-bin.tar.gz dest=/tmp/hbase-1.2.3-bin.tar.gz 11 | when: hbase_binary_stat.stat.exists == False 12 | 13 | - name: rename file 14 | command: chown hadoop:hadoop /tmp/hbase-1.2.3-bin.tar.gz 15 | 16 | - name: uncompress file tarball and place in /usr/local/ 17 | command: tar zxf /tmp/hbase-1.2.3-bin.tar.gz -C /usr/local/ 18 | 19 | - name: rename file 20 | command: creates="/usr/local/hbase" mv /usr/local/hbase-1.2.3/ /usr/local/hbase 21 | 22 | - name: set permissions for installation placed in /usr/local/ 23 | command: chown -R hadoop:hadoop /usr/local/hbase 24 | 25 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HBASE_HOME=" line="export HBASE_HOME=/usr/local/hbase" 26 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HBASE_REGIONSERVERS=" line="export HBASE_REGIONSERVERS=/usr/local/hbase/conf/regionservers" 27 | - lineinfile: dest=/home/hadoop/.bashrc regexp="HBASE_MANAGES_ZK=" line="export HBASE_MANAGES_ZK=true" 28 | 29 | # copy template files to $HBASE_HOME/conf/* 30 | 31 | - name: setup core-site.xml file 32 | template: src={{ item.name }}.j2 33 | dest=/usr/local/hbase/conf/{{ item.name }} 34 | with_items: 35 | - { name: hbase-site.xml } 36 | - { name: regionservers } 37 | - { name: backup-masters } 38 | tags: 39 | - hbase_conf 40 | 41 | - name: set permissions for installation placed in /usr/local/ 42 | command: cp /usr/local/hadoop/etc/hadoop/{{ item.name }} /usr/local/hbase/conf/ 43 | with_items: 44 | - { name: hdfs-site.xml } 45 | -------------------------------------------------------------------------------- /roles/hbase/templates/backup-masters.j2: -------------------------------------------------------------------------------- 1 | hadoopdata1 2 | hadoopdata2 3 | -------------------------------------------------------------------------------- /roles/hbase/templates/hbase-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | hbase.rootdir 6 | hdfs://hadoopmaster:8020/hbase 7 | 8 | 9 | hbase.cluster.distributed 10 | true 11 | 12 | 13 | dfs.replication 14 | 1 15 | 16 | 17 | hbase.zookeeper.quorum 18 | hadoopmaster,hadoopdata1,hadoopdata2 19 | 20 | 21 | -------------------------------------------------------------------------------- /roles/hbase/templates/regionservers.j2: -------------------------------------------------------------------------------- 1 | hadoopmaster 2 | hadoopdata1 3 | hadoopdata2 4 | -------------------------------------------------------------------------------- /roles/hbase/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/hbase/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - hbase -------------------------------------------------------------------------------- /roles/hbase/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for hbase 3 | -------------------------------------------------------------------------------- /roles/hive/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/hive/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | Hive ansible role is still in development. A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/hive/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for hive 3 | -------------------------------------------------------------------------------- /roles/hive/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for hive 3 | -------------------------------------------------------------------------------- /roles/hive/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: Apache v2 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. 174 | -------------------------------------------------------------------------------- /roles/hive/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for hive 3 | 4 | # http://mirror.its.dal.ca/apache/hive/hive-2.0.0/apache-hive-2.0.0-bin.tar.gz 5 | 6 | - name: stat /tmp/apache-hive-2.0.1-bin.tar.gz 7 | stat: path=/tmp/apache-hive-2.0.1-bin.tar.gz 8 | register: hive_binary_stat 9 | 10 | - name: Downloading Apache Hive 11 | get_url: url=http://muug.ca/mirror/apache-dist/hive/hive-2.0.1/apache-hive-2.0.1-bin.tar.gz dest=/tmp/apache-hive-2.0.1-bin.tar.gz 12 | when: hive_binary_stat.stat.exists == False 13 | 14 | - name: rename file 15 | command: chown hadoop:hadoop /tmp/apache-hive-2.0.1-bin.tar.gz 16 | 17 | # 18 | # - name: copy hive to tmp folder 19 | # copy: src=apache-hive-2.0.0-bin.tar.gz dest=/tmp/ 20 | # when: hivebinary_stat.stat.exists == False 21 | # tags: 22 | # - provision_copy_to_remote 23 | 24 | - name: unzip tarball and place in /usr/local/ 25 | command: tar zxf /tmp/apache-hive-2.0.1-bin.tar.gz -C /usr/local/ 26 | 27 | - name: stat /usr/local/hive 28 | stat: path=/usr/local/hive 29 | register: hive_dir_stat 30 | 31 | - name: rename file 32 | command: creates="/usr/local/hive" mv /usr/local/apache-hive-2.0.1-bin/ /usr/local/hive 33 | when: hive_dir_stat.stat.exists == False 34 | 35 | # TODO: This step is going to be manual. 36 | # - name: create mysql metastore 37 | # command: /usr/local/hive/bin/schematool -dbType mysql -initSchema -userName hiveuser -passWord hivepassword 38 | # become: yes 39 | # become_user: hadoop 40 | 41 | - name: copy mysql jdbc driver 42 | command: cp /usr/share/java/mysql-connector-java.jar /usr/local/hive/lib/mysql-connector-java.jar 43 | 44 | # TODO: Add 'hive.support.concurrency=true, ' 45 | # TODO: Add 'hive.zookeeper.quorum=hadoopmaster,hadoopdata1, hadoopdata2' 46 | - name: setup core-site.xml file 47 | template: src=hive-site.xml.j2 dest=/usr/local/hive/conf/hive-site.xml 48 | -------------------------------------------------------------------------------- /roles/hive/templates/hive-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | javax.jdo.option.ConnectionURL 4 | jdbc:mysql://localhost/metastore?createDatabaseIfNotExist=true 5 | metadata is stored in a MySQL server 6 | 7 | 8 | javax.jdo.option.ConnectionDriverName 9 | com.mysql.jdbc.Driver 10 | MySQL JDBC driver class 11 | 12 | 13 | javax.jdo.option.ConnectionUserName 14 | hiveuser 15 | user name for connecting to mysql server 16 | 17 | 18 | javax.jdo.option.ConnectionPassword 19 | hivepassword 20 | password for connecting to mysql server 21 | 22 | 23 | -------------------------------------------------------------------------------- /roles/hive/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/hive/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - hive -------------------------------------------------------------------------------- /roles/hive/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for hive 3 | -------------------------------------------------------------------------------- /roles/kafka/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/kafka/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for kafka 3 | -------------------------------------------------------------------------------- /roles/kafka/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #handlers file for kafka 3 | - name: reload systemd 4 | sudo: yes 5 | command: systemctl daemon-reload 6 | - name: restart kafka 7 | sudo: yes 8 | command: systemctl restart 9 | -------------------------------------------------------------------------------- /roles/kafka/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: your name 4 | description: 5 | company: your company (optional) 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | # Some suggested licenses: 10 | # - BSD (default) 11 | # - MIT 12 | # - GPLv2 13 | # - GPLv3 14 | # - Apache 15 | # - CC-BY 16 | license: license (GPLv2, CC-BY, etc) 17 | min_ansible_version: 1.2 18 | # 19 | # Below are all platforms currently available. Just uncomment 20 | # the ones that apply to your role. If you don't see your 21 | # platform on this list, let us know and we'll get it added! 22 | # 23 | #platforms: 24 | #- name: EL 25 | # versions: 26 | # - all 27 | # - 5 28 | # - 6 29 | # - 7 30 | #- name: GenericUNIX 31 | # versions: 32 | # - all 33 | # - any 34 | #- name: Solaris 35 | # versions: 36 | # - all 37 | # - 10 38 | # - 11.0 39 | # - 11.1 40 | # - 11.2 41 | # - 11.3 42 | #- name: Fedora 43 | # versions: 44 | # - all 45 | # - 16 46 | # - 17 47 | # - 18 48 | # - 19 49 | # - 20 50 | # - 21 51 | # - 22 52 | # - 23 53 | #- name: Windows 54 | # versions: 55 | # - all 56 | # - 2012R2 57 | #- name: SmartOS 58 | # versions: 59 | # - all 60 | # - any 61 | #- name: opensuse 62 | # versions: 63 | # - all 64 | # - 12.1 65 | # - 12.2 66 | # - 12.3 67 | # - 13.1 68 | # - 13.2 69 | #- name: Amazon 70 | # versions: 71 | # - all 72 | # - 2013.03 73 | # - 2013.09 74 | #- name: GenericBSD 75 | # versions: 76 | # - all 77 | # - any 78 | #- name: FreeBSD 79 | # versions: 80 | # - all 81 | # - 10.0 82 | # - 10.1 83 | # - 10.2 84 | # - 8.0 85 | # - 8.1 86 | # - 8.2 87 | # - 8.3 88 | # - 8.4 89 | # - 9.0 90 | # - 9.1 91 | # - 9.1 92 | # - 9.2 93 | # - 9.3 94 | #- name: Ubuntu 95 | # versions: 96 | # - all 97 | # - lucid 98 | # - maverick 99 | # - natty 100 | # - oneiric 101 | # - precise 102 | # - quantal 103 | # - raring 104 | # - saucy 105 | # - trusty 106 | # - utopic 107 | # - vivid 108 | # - wily 109 | #- name: SLES 110 | # versions: 111 | # - all 112 | # - 10SP3 113 | # - 10SP4 114 | # - 11 115 | # - 11SP1 116 | # - 11SP2 117 | # - 11SP3 118 | #- name: GenericLinux 119 | # versions: 120 | # - all 121 | # - any 122 | #- name: Debian 123 | # versions: 124 | # - all 125 | # - etch 126 | # - jessie 127 | # - lenny 128 | # - squeeze 129 | # - wheezy 130 | # 131 | # Below are all categories currently available. Just as with 132 | # the platforms above, uncomment those that apply to your role. 133 | # 134 | #categories: 135 | #- cloud 136 | #- cloud:ec2 137 | #- cloud:gce 138 | #- cloud:rax 139 | #- clustering 140 | #- database 141 | #- database:nosql 142 | #- database:sql 143 | #- development 144 | #- monitoring 145 | #- networking 146 | #- packaging 147 | #- system 148 | #- web 149 | dependencies: [] 150 | # List your role dependencies here, one per line. 151 | # Be sure to remove the '[]' above if you add dependencies 152 | # to this list. 153 | 154 | -------------------------------------------------------------------------------- /roles/kafka/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ################################################################################### 3 | # Author: Zak Hassan 4 | # Description: Playbook Apache Kafka on openstack like less than 1 min. :) 5 | # 6 | ################################################################################### 7 | # Creating kafka user 8 | - group: name=kafka state=present 9 | - user: name=kafka comment="kafka" group=kafka shell=/bin/bash 10 | 11 | 12 | - name: stat /tmp/kafka_2.11-0.10.2.0 13 | stat: path=/tmp/kafka_2.11-0.10.2.0 14 | register: kfbinary_stat 15 | 16 | - name: Downloading Apache Kafka 17 | get_url: url=http://apache.forsale.plus/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz dest=/tmp/kafka_2.11-0.10.2.0.tgz 18 | when: kfbinary_stat.stat.exists == False 19 | 20 | - name: unzip tarball and place in /usr/local/ 21 | command: tar zxvf /tmp/kafka_2.11-0.10.2.0.tgz -C /usr/local/ 22 | 23 | - name: rename file 24 | command: creates="{{ kafka_install_dir }}" mv /usr/local/kafka_2.11-0.10.2.0/ /usr/local/kafka 25 | 26 | - name: change permissions to kafka user for kafka_2.11-0.10.2.0 27 | command: chown -R kafka:kafka /usr/local/kafka 28 | 29 | - name: setup kafka property file 30 | template: src=server.properties.j2 dest=/usr/local/kafka/config/server.properties 31 | 32 | - name: setup systemd unit file 33 | template: src=kafka.systemd.j2 dest=/lib/systemd/system/kafka.service 34 | register: install_config 35 | notify: reload systemd 36 | 37 | - name: enable kafka 38 | sudo: yes 39 | service: name=kafka enabled=yes state=started 40 | -------------------------------------------------------------------------------- /roles/kafka/templates/kafka.systemd.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kafka Messaging Broker Service 3 | After=network.target 4 | 5 | [Service] 6 | User=kafka 7 | Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0 8 | ExecStart={{ kafka_install_dir }}/bin/kafka-server-start.sh {{ kafka_install_dir }}/config/server.properties 9 | ExecStop={{ kafka_install_dir }}/bin/kafka-server-stop.sh 10 | Restart=on-abort 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /roles/kafka/templates/server.properties.j2: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # see kafka.server.KafkaConfig for additional details and defaults 16 | 17 | ############################# Server Basics ############################# 18 | 19 | # The id of the broker. This must be set to a unique integer for each broker. 20 | broker.id={{ myid }} 21 | 22 | ############################# Socket Server Settings ############################# 23 | 24 | # The port the socket server listens on 25 | port=9092 26 | 27 | # Hostname the broker will bind to. If not set, the server will bind to all interfaces 28 | #host.name=localhost 29 | 30 | # Hostname the broker will advertise to producers and consumers. If not set, it uses the 31 | # value for "host.name" if configured. Otherwise, it will use the value returned from 32 | # java.net.InetAddress.getCanonicalHostName(). 33 | #advertised.host.name= 34 | 35 | # The port to publish to ZooKeeper for clients to use. If this is not set, 36 | # it will publish the same port that the broker binds to. 37 | #advertised.port= 38 | 39 | # The number of threads handling network requests 40 | num.network.threads=3 41 | 42 | # The number of threads doing disk I/O 43 | num.io.threads=8 44 | 45 | # The send buffer (SO_SNDBUF) used by the socket server 46 | socket.send.buffer.bytes=102400 47 | 48 | # The receive buffer (SO_RCVBUF) used by the socket server 49 | socket.receive.buffer.bytes=102400 50 | 51 | # The maximum size of a request that the socket server will accept (protection against OOM) 52 | socket.request.max.bytes=104857600 53 | 54 | 55 | ############################# Log Basics ############################# 56 | 57 | # A comma seperated list of directories under which to store log files 58 | log.dirs={{ kafka_data_dir }} 59 | 60 | # The default number of log partitions per topic. More partitions allow greater 61 | # parallelism for consumption, but this will also result in more files across 62 | # the brokers. 63 | num.partitions=1 64 | 65 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. 66 | # This value is recommended to be increased for installations with data dirs located in RAID array. 67 | num.recovery.threads.per.data.dir=1 68 | 69 | ############################# Log Flush Policy ############################# 70 | 71 | # Messages are immediately written to the filesystem but by default we only fsync() to sync 72 | # the OS cache lazily. The following configurations control the flush of data to disk. 73 | # There are a few important trade-offs here: 74 | # 1. Durability: Unflushed data may be lost if you are not using replication. 75 | # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. 76 | # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. 77 | # The settings below allow one to configure the flush policy to flush data after a period of time or 78 | # every N messages (or both). This can be done globally and overridden on a per-topic basis. 79 | 80 | # The number of messages to accept before forcing a flush of data to disk 81 | #log.flush.interval.messages=10000 82 | 83 | # The maximum amount of time a message can sit in a log before we force a flush 84 | #log.flush.interval.ms=1000 85 | 86 | ############################# Log Retention Policy ############################# 87 | 88 | # The following configurations control the disposal of log segments. The policy can 89 | # be set to delete segments after a period of time, or after a given size has accumulated. 90 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens 91 | # from the end of the log. 92 | 93 | # The minimum age of a log file to be eligible for deletion 94 | log.retention.hours=168 95 | 96 | # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining 97 | # segments don't drop below log.retention.bytes. 98 | #log.retention.bytes=1073741824 99 | 100 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 101 | log.segment.bytes=1073741824 102 | 103 | # The interval at which log segments are checked to see if they can be deleted according 104 | # to the retention policies 105 | log.retention.check.interval.ms=300000 106 | 107 | # By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. 108 | # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. 109 | log.cleaner.enable=false 110 | 111 | ############################# Zookeeper ############################# 112 | 113 | # Zookeeper connection string (see zookeeper docs for details). 114 | # This is a comma separated host:port pairs, each corresponding to a zk 115 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 116 | # You can also append an optional chroot string to the urls to specify the 117 | # root directory for all kafka znodes. 118 | zookeeper.connect={{ zk_urls }} 119 | 120 | # Timeout in ms for connecting to zookeeper 121 | zookeeper.connection.timeout.ms=6000 122 | -------------------------------------------------------------------------------- /roles/kafka/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for kafka 3 | kafka_binary: kafka_2.9.1-0.8.2.2.tgz 4 | kafka_install_dir: /usr/local/kafka 5 | kafka_version: kafka_2.9.1-0.8.2.2 6 | kafka_data_dir: /tmp/kafka-logs 7 | -------------------------------------------------------------------------------- /roles/kibana/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/kibana/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/kibana/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for kibana 3 | -------------------------------------------------------------------------------- /roles/kibana/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for kibana 3 | -------------------------------------------------------------------------------- /roles/kibana/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/kibana/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for kibana 3 | # https://download.elastic.co/kibana/kibana/kibana-4.5.1-linux-x64.tar.gz 4 | # open localhost:5601 5 | 6 | 7 | - name: stat /tmp/kibana-4.5.1-linux-x64.tar.gz 8 | stat: path=/tmp/kibana-4.5.1-linux-x64.tar.gz 9 | register: kibana_binary_stat 10 | 11 | - name: Downloading Elastic Search For Hadoop 12 | get_url: url=https://download.elastic.co/kibana/kibana/kibana-4.5.1-linux-x64.tar.gz dest=/tmp/kibana-4.5.1-linux-x64.tar.gz 13 | when: kibana_binary_stat.stat.exists == False 14 | 15 | - name: uncompress pig tarball and place in /usr/local/ 16 | command: tar zxf /tmp/kibana-4.5.1-linux-x64.tar.gz -C /usr/local/ 17 | 18 | - name: rename file 19 | command: creates="/usr/local/kibana" mv /usr/local/kibana-4.5.1-linux-x64 /usr/local/kibana 20 | 21 | - name: change ownerships file 22 | command: chown -R hadoop:hadoop /usr/local/kibana 23 | -------------------------------------------------------------------------------- /roles/kibana/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/kibana/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - kibana -------------------------------------------------------------------------------- /roles/kibana/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for kibana 3 | -------------------------------------------------------------------------------- /roles/logstash/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/logstash/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/logstash/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for logstash 3 | -------------------------------------------------------------------------------- /roles/logstash/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for logstash 3 | -------------------------------------------------------------------------------- /roles/logstash/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/logstash/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for logstash 3 | # https://download.elastic.co/logstash/logstash/logstash-2.3.3.tar.gz 4 | 5 | - name: stat /tmp/logstash-2.3.3.tar.gz 6 | stat: path=/tmp/logstash-2.3.3.tar.gz 7 | register: logstash_binary_stat 8 | 9 | - name: Downloading file tarball 10 | get_url: url=https://download.elastic.co/logstash/logstash/logstash-2.3.3.tar.gz dest=/tmp/logstash-2.3.3.tar.gz 11 | when: logstash_binary_stat.stat.exists == False 12 | 13 | - name: uncompress file tarball and place in /usr/local/ 14 | command: tar zxf /tmp/logstash-2.3.3.tar.gz -C /usr/local/ 15 | 16 | - name: rename file 17 | command: creates="/usr/local/logstash" mv /usr/local/logstash-2.3.3 /usr/local/logstash 18 | 19 | - name: change ownerships file 20 | command: chown -R hadoop:hadoop /usr/local/logstash 21 | 22 | - name: setup Procfile file 23 | template: src=Procfile.j2 dest=/usr/local/logstash/Procfile 24 | 25 | - name: setup log4j-kakfa-processor.conf file 26 | template: src=log4j-kakfa-processor.conf.j2 dest=/usr/local/logstash/log4j-kakfa-processor.conf 27 | -------------------------------------------------------------------------------- /roles/logstash/templates/Procfile.j2: -------------------------------------------------------------------------------- 1 | elasticsearch: /usr/local/elasticsearch/bin/elasticsearch 2 | logstash: /usr/local/logstash/bin/logstash -f /usr/local/logstash/log4j-kakfa-processor.conf 3 | -------------------------------------------------------------------------------- /roles/logstash/templates/log4j-kakfa-processor.conf.j2: -------------------------------------------------------------------------------- 1 | input { 2 | log4j { 3 | mode => "server" 4 | host => "0.0.0.0" 5 | port => 4712 6 | type => "log4j" 7 | } 8 | } 9 | 10 | output { 11 | 12 | kafka { 13 | topic_id => "logtest" 14 | bootstrap_servers => "hadoopmaster:9092,hadoopdata1:9092,hadoopdata2:9092" 15 | } 16 | elasticsearch { 17 | hosts => ["127.0.0.1"] 18 | index => "logstash-%{+YYYY.MM.dd}" 19 | } 20 | stdout { codec => rubydebug } 21 | } 22 | -------------------------------------------------------------------------------- /roles/logstash/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/logstash/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - logstash -------------------------------------------------------------------------------- /roles/logstash/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for logstash 3 | -------------------------------------------------------------------------------- /roles/mysql/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/mysql/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/mysql/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for mysql 3 | -------------------------------------------------------------------------------- /roles/mysql/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for mysql 3 | -------------------------------------------------------------------------------- /roles/mysql/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/mysql/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for mysql 3 | # Prereq: Install mysql for the metastore: http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm 4 | # yum install mysql-server this will take time from the installation. 5 | # yum install mysql-connector-java 6 | # ln -s /usr/share/java/mysql-connector-java.jar /usr/local/hive/lib/mysql-connector-java.jar 7 | # Need to configure hive to 8 | # 9 | #mysql> CREATE USER 'hiveuser'@'%' IDENTIFIED BY 'hivepassword'; 10 | #mysql> GRANT all on *.* to 'hiveuser'@localhost identified by 'hivepassword'; 11 | #mysql> flush privileges; 12 | # 13 | # schematool -dbType mysql -initSchema -userName hiveuser -passWord hivepassword 14 | # 15 | # 16 | 17 | - name: mysql rpms 18 | yum: name=http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm state=present 19 | 20 | - name: install mysql-rpms 21 | command: yum install {{ item }} -y 22 | with_items: 23 | - gcc 24 | - mysql-server 25 | - mysql-connector-java 26 | - mysql-devel 27 | - python-devel 28 | 29 | - name: epel-release rpms 30 | yum: name=epel-release state=present 31 | 32 | - name: python-pip rpms 33 | yum: name=python-pip state=present 34 | 35 | # Make sure mysql service is running 36 | - name: enable mysql 37 | sudo: yes 38 | service: name=mysql enabled=yes state=started 39 | 40 | 41 | # You will need mysql-python to create users 42 | - name: Install the Python MySQLB module 43 | pip: name=MySQL-python 44 | # ansible localhost -m mysql_user -a "name=hiveuser password=hivepassword priv=*.*:ALL state=present" -s 45 | # create mysql user 46 | - mysql_user: name=hiveuser password=hivepassword priv=*.*:ALL state=present 47 | - mysql_user: name=hiveuser password=hivepassword priv=*.*:ALL state=present 48 | -------------------------------------------------------------------------------- /roles/mysql/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/mysql/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - mysql -------------------------------------------------------------------------------- /roles/mysql/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for mysql 3 | -------------------------------------------------------------------------------- /roles/pig/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/pig/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/pig/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for pig 3 | -------------------------------------------------------------------------------- /roles/pig/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for pig 3 | -------------------------------------------------------------------------------- /roles/pig/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/pig/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for pig 3 | 4 | - name: stat /tmp/pig-0.15.0.tar.gz 5 | stat: path=/tmp/pig-0.15.0.tar.gz 6 | register: pigbinary_stat 7 | 8 | - name: Downloading Apache Pig 9 | get_url: url=http://apache.forsale.plus/pig/pig-0.15.0/pig-0.15.0.tar.gz dest=/tmp/pig-0.15.0.tar.gz 10 | when: pigbinary_stat.stat.exists == False 11 | 12 | - name: rename file 13 | command: chown hadoop:hadoop /tmp/pig-0.15.0.tar.gz 14 | 15 | 16 | - name: uncompress pig tarball and place in /usr/local/ 17 | command: tar zxf /tmp/pig-0.15.0.tar.gz -C /usr/local/ 18 | 19 | - name: rename file 20 | command: creates="/usr/local/pig" mv /usr/local/pig-0.15.0/ /usr/local/pig 21 | 22 | - name: uncompress pig tarball and place in /usr/local/ 23 | command: chown -R hadoop:hadoop /usr/local/pig 24 | -------------------------------------------------------------------------------- /roles/pig/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/pig/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - pig -------------------------------------------------------------------------------- /roles/pig/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for pig 3 | -------------------------------------------------------------------------------- /roles/shared/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | These are environment specific configurations. Like bashrc, hosts, etc. A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/shared/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for env 3 | -------------------------------------------------------------------------------- /roles/shared/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for env 3 | -------------------------------------------------------------------------------- /roles/shared/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: your name 4 | description: 5 | company: your company (optional) 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | # Some suggested licenses: 10 | # - BSD (default) 11 | # - MIT 12 | # - GPLv2 13 | # - GPLv3 14 | # - Apache 15 | # - CC-BY 16 | license: license (GPLv2, CC-BY, etc) 17 | min_ansible_version: 1.2 18 | # 19 | # Below are all platforms currently available. Just uncomment 20 | # the ones that apply to your role. If you don't see your 21 | # platform on this list, let us know and we'll get it added! 22 | # 23 | #platforms: 24 | #- name: EL 25 | # versions: 26 | # - all 27 | # - 5 28 | # - 6 29 | # - 7 30 | #- name: GenericUNIX 31 | # versions: 32 | # - all 33 | # - any 34 | #- name: Solaris 35 | # versions: 36 | # - all 37 | # - 10 38 | # - 11.0 39 | # - 11.1 40 | # - 11.2 41 | # - 11.3 42 | #- name: Fedora 43 | # versions: 44 | # - all 45 | # - 16 46 | # - 17 47 | # - 18 48 | # - 19 49 | # - 20 50 | # - 21 51 | # - 22 52 | # - 23 53 | #- name: Windows 54 | # versions: 55 | # - all 56 | # - 2012R2 57 | #- name: SmartOS 58 | # versions: 59 | # - all 60 | # - any 61 | #- name: opensuse 62 | # versions: 63 | # - all 64 | # - 12.1 65 | # - 12.2 66 | # - 12.3 67 | # - 13.1 68 | # - 13.2 69 | #- name: Amazon 70 | # versions: 71 | # - all 72 | # - 2013.03 73 | # - 2013.09 74 | #- name: GenericBSD 75 | # versions: 76 | # - all 77 | # - any 78 | #- name: FreeBSD 79 | # versions: 80 | # - all 81 | # - 10.0 82 | # - 10.1 83 | # - 10.2 84 | # - 8.0 85 | # - 8.1 86 | # - 8.2 87 | # - 8.3 88 | # - 8.4 89 | # - 9.0 90 | # - 9.1 91 | # - 9.1 92 | # - 9.2 93 | # - 9.3 94 | #- name: Ubuntu 95 | # versions: 96 | # - all 97 | # - lucid 98 | # - maverick 99 | # - natty 100 | # - oneiric 101 | # - precise 102 | # - quantal 103 | # - raring 104 | # - saucy 105 | # - trusty 106 | # - utopic 107 | # - vivid 108 | # - wily 109 | #- name: SLES 110 | # versions: 111 | # - all 112 | # - 10SP3 113 | # - 10SP4 114 | # - 11 115 | # - 11SP1 116 | # - 11SP2 117 | # - 11SP3 118 | #- name: GenericLinux 119 | # versions: 120 | # - all 121 | # - any 122 | #- name: Debian 123 | # versions: 124 | # - all 125 | # - etch 126 | # - jessie 127 | # - lenny 128 | # - squeeze 129 | # - wheezy 130 | # 131 | # Below are all categories currently available. Just as with 132 | # the platforms above, uncomment those that apply to your role. 133 | # 134 | #categories: 135 | #- cloud 136 | #- cloud:ec2 137 | #- cloud:gce 138 | #- cloud:rax 139 | #- clustering 140 | #- database 141 | #- database:nosql 142 | #- database:sql 143 | #- development 144 | #- monitoring 145 | #- networking 146 | #- packaging 147 | #- system 148 | #- web 149 | dependencies: [] 150 | # List your role dependencies here, one per line. 151 | # Be sure to remove the '[]' above if you add dependencies 152 | # to this list. 153 | 154 | -------------------------------------------------------------------------------- /roles/shared/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for env 3 | # TODO: Additional Things to install 'mysql-server' for hive metastore. 4 | # TODO: CREATE PLAYBOOK ROLE FOR GANGLIA 5 | # yum install http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.rpm 6 | 7 | 8 | # - name: install epel 9 | # yum: name=epel-release state=latest 10 | 11 | - name: install java8 12 | yum: name=java-1.8.0-openjdk-devel.x86_64 state=latest 13 | 14 | - name: install java8 15 | yum: name=unzip state=latest 16 | 17 | # - name: install zeromq 18 | # yum: name=zeromq state=latest 19 | 20 | - name: setup hosts file 21 | template: src=hosts.j2 dest=/etc/hosts 22 | register: installing_hosts_config 23 | 24 | - name: setup bashrc file 25 | template: src=bashrc.j2 dest={{ HOME }}/.bashrc 26 | register: installing_hosts_config 27 | -------------------------------------------------------------------------------- /roles/shared/templates/bashrc.j2: -------------------------------------------------------------------------------- 1 | # .bashrc 2 | 3 | # Source global definitions 4 | if [ -f /etc/bashrc ]; then 5 | . /etc/bashrc 6 | fi 7 | 8 | JAVA_HOME="/usr/lib/jvm/java-1.8.0" 9 | KAFKA_HOME="/usr/local/kafka" 10 | ZOOKEEPER_HOME="/usr/local/zookeeper" 11 | HIVE_HOME="/usr/local/hive" 12 | PIG_HOME="/usr/local/pig" 13 | export PATH=$JAVA_HOME'/bin':ZOOKEEPER_HOME'/bin':$PATH 14 | 15 | function zk-start-foreground(){ 16 | $ZOOKEEPER_HOME/bin/zkServer.sh start-foreground $ZOOKEEPER_HOME/conf/zoo.cfg 17 | } 18 | 19 | function zk-start(){ 20 | $ZOOKEEPER_HOME/bin/zkServer.sh start $ZOOKEEPER_HOME/conf/zoo.cfg 21 | } 22 | -------------------------------------------------------------------------------- /roles/shared/templates/hosts.j2: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost.localdomain localhost 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | 192.168.33.40 hadoopmaster 4 | 192.168.33.41 hadoopdata1 5 | 192.168.33.42 hadoopdata2 6 | -------------------------------------------------------------------------------- /roles/shared/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for env 3 | -------------------------------------------------------------------------------- /roles/spark/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/spark/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | This role is still in development and not ready. I will tweet about this once its ready. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | Apache 2.0 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/spark/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for spark 3 | -------------------------------------------------------------------------------- /roles/spark/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for spark 3 | -------------------------------------------------------------------------------- /roles/spark/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/spark/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for spark 3 | # After installed go to http://hadoopmaster:4040/jobs/ to see the jobs executed 4 | # When Starting spark use the following format 5 | # spark-shell --master spark://hadoopmaster:7077 --jars /usr/local/hive/lib/mysql-connector-java.jar 6 | 7 | - name: stat /tmp/spark-2.1.0-bin-hadoop2.7.tgz 8 | stat: path=/tmp/spark-2.1.0-bin-hadoop2.7.tgz 9 | register: spark_binary_stat 10 | 11 | # trying out spark 2.0.0 12 | # http://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-preview-bin-hadoop2.7.tgz 13 | 14 | - name: Downloading Apache Spark 15 | get_url: url=http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz dest=/tmp/spark-2.1.0-bin-hadoop2.7.tgz 16 | when: spark_binary_stat.stat.exists == False 17 | 18 | - name: rename file 19 | command: chown hadoop:hadoop /tmp/spark-2.1.0-bin-hadoop2.7.tgz 20 | 21 | - name: uncompress spark tarball and place in /usr/local/ 22 | command: tar -zxf /tmp/spark-2.1.0-bin-hadoop2.7.tgz -C /usr/local 23 | 24 | - name: rename file 25 | command: creates="/usr/local/spark" mv /usr/local/spark-2.1.0-bin-hadoop2.7/ /usr/local/spark 26 | 27 | - name: Change owner to hadoop for all files under SPARK_HOME 28 | command: chown -R hadoop:hadoop /usr/local/spark 29 | 30 | - name: copying spark slaves conf 31 | template: src=slaves.j2 dest=/usr/local/spark/conf/slaves owner={{ hadoop_uid }} group={{ hadoop_gid }} 32 | when: api_hostname == "hadoopmaster" 33 | 34 | - name: setup spark property files 35 | template: src={{ item.name }}.j2 dest=/usr/local/spark/conf/{{ item.name }} owner={{ hadoop_uid }} group={{ hadoop_gid }} 36 | with_items: 37 | - { name: spark-defaults.conf } 38 | - { name: spark-env.sh } 39 | - { name: metrics.properties } 40 | -------------------------------------------------------------------------------- /roles/spark/templates/hive-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | javax.jdo.option.ConnectionURL 4 | jdbc:mysql://localhost/metastore?createDatabaseIfNotExist=true 5 | metadata is stored in a MySQL server 6 | 7 | 8 | javax.jdo.option.ConnectionDriverName 9 | com.mysql.jdbc.Driver 10 | MySQL JDBC driver class 11 | 12 | 13 | javax.jdo.option.ConnectionUserName 14 | hiveuser 15 | user name for connecting to mysql server 16 | 17 | 18 | javax.jdo.option.ConnectionPassword 19 | hivepassword 20 | password for connecting to mysql server 21 | 22 | 23 | -------------------------------------------------------------------------------- /roles/spark/templates/slaves.j2: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # A Spark Worker will be started on each of the machines listed below. 19 | hadoopdata1 20 | hadoopdata2 21 | -------------------------------------------------------------------------------- /roles/spark/templates/spark-defaults.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | spark.master spark://hadoopmaster:7077 18 | spark.serializer org.apache.spark.serializer.KryoSerializer 19 | 20 | # Default system properties included when running spark-submit. 21 | # This is useful for setting default environmental settings. 22 | 23 | # Example: 24 | # spark.master spark://master:7077 25 | # spark.eventLog.enabled true 26 | # spark.eventLog.dir hdfs://namenode:8021/directory 27 | # spark.serializer org.apache.spark.serializer.KryoSerializer 28 | # spark.driver.memory 5g 29 | # spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" 30 | -------------------------------------------------------------------------------- /roles/spark/templates/spark-env.sh.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one or more 5 | # contributor license agreements. See the NOTICE file distributed with 6 | # this work for additional information regarding copyright ownership. 7 | # The ASF licenses this file to You under the Apache License, Version 2.0 8 | # (the "License"); you may not use this file except in compliance with 9 | # the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # This file is sourced when running various Spark programs. 21 | # Copy it as spark-env.sh and edit that to configure Spark for your site. 22 | 23 | 24 | SPARK_JAVA_OPTS=-Dspark.driver.port=53411 25 | HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop 26 | SPARK_MASTER_IP={{ hadoopmaster }} 27 | 28 | # Options read when launching programs locally with 29 | # ./bin/run-example or ./bin/spark-submit 30 | # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files 31 | # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node 32 | # - SPARK_PUBLIC_DNS, to set the public dns name of the driver program 33 | # - SPARK_CLASSPATH, default classpath entries to append 34 | 35 | # Options read by executors and drivers running inside the cluster 36 | # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node 37 | # - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program 38 | # - SPARK_CLASSPATH, default classpath entries to append 39 | # - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data 40 | # - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos 41 | 42 | # Options read in YARN client mode 43 | # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files 44 | # - SPARK_EXECUTOR_INSTANCES, Number of executors to start (Default: 2) 45 | # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). 46 | # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) 47 | # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) 48 | # - SPARK_YARN_APP_NAME, The name of your application (Default: Spark) 49 | # - SPARK_YARN_QUEUE, The hadoop queue to use for allocation requests (Default: ‘default’) 50 | # - SPARK_YARN_DIST_FILES, Comma separated list of files to be distributed with the job. 51 | # - SPARK_YARN_DIST_ARCHIVES, Comma separated list of archives to be distributed with the job. 52 | 53 | # Options for the daemons used in the standalone deploy mode 54 | # - SPARK_MASTER_IP, to bind the master to a different IP address or hostname 55 | # - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master 56 | # - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y") 57 | # - SPARK_WORKER_CORES, to set the number of cores to use on this machine 58 | # - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g) 59 | # - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker 60 | # - SPARK_WORKER_INSTANCES, to set the number of worker processes per node 61 | # - SPARK_WORKER_DIR, to set the working directory of worker processes 62 | # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") 63 | # - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g). 64 | # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") 65 | # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") 66 | # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") 67 | # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers 68 | 69 | # Generic options for the daemons used in the standalone deploy mode 70 | # - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf) 71 | # - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs) 72 | # - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) 73 | # - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) 74 | # - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) 75 | -------------------------------------------------------------------------------- /roles/spark/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/spark/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - spark -------------------------------------------------------------------------------- /roles/spark/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for spark 3 | -------------------------------------------------------------------------------- /roles/storm/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/storm/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/storm/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for storm 3 | -------------------------------------------------------------------------------- /roles/storm/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for storm 3 | -------------------------------------------------------------------------------- /roles/storm/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/storm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for storm 3 | # - name: install zeromq 4 | # yum: name={{ item }} state=latest 5 | # with_items: 6 | # - epel-release 7 | # - zeromq 8 | - name: stat /tmp/apache-storm-1.0.1.tar.gz 9 | stat: path=/tmp/apache-storm-1.0.1.tar.gz 10 | register: storm_dir_stat 11 | 12 | - name: Downloading Apache Storm 13 | get_url: url=http://apache.mirror.gtcomm.net/storm/apache-storm-1.0.1/apache-storm-1.0.1.tar.gz dest=/tmp/apache-storm-1.0.1.tar.gz 14 | when: storm_dir_stat.stat.exists == False 15 | 16 | - name: rename file 17 | command: chown hadoop:hadoop /tmp/apache-storm-1.0.1.tar.gz 18 | 19 | - name: Uncompress Apache Storm tarball and place in /usr/local/ 20 | command: tar zxf /tmp/apache-storm-1.0.1.tar.gz -C /usr/local/ 21 | 22 | - name: rename file 23 | command: mv /usr/local/apache-storm-1.0.1/ /usr/local/storm creates=/usr/local/storm 24 | 25 | - name: change permissions so hadoop user can modify files in storm dir 26 | command: chown -R hadoop:hadoop /usr/local/storm 27 | 28 | - name: change permissions on storm configuration file 29 | file: path=/usr/local/storm/conf/storm.yaml owner=hadoop group=hadoop mode=0777 30 | 31 | - name: setup storm.yaml file 32 | template: src=storm.yaml.j2 dest=/usr/local/storm/conf/storm.yaml 33 | -------------------------------------------------------------------------------- /roles/storm/templates/Procfile: -------------------------------------------------------------------------------- 1 | storm-nimbus: storm nimbus 2 | storm-supervisor: storm supervisor 3 | storm-ui: storm ui 4 | -------------------------------------------------------------------------------- /roles/storm/templates/storm.yaml.j2: -------------------------------------------------------------------------------- 1 | storm.zookeeper.servers: 2 | - "hadoopmaster" 3 | - "hadoopdata1" 4 | - "hadoopdata2" 5 | 6 | ui.port: 8181 7 | nimbus.host: "{{ server1 }}" 8 | storm.local.dir: /usr/local/storm/ 9 | -------------------------------------------------------------------------------- /roles/storm/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/storm/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - storm -------------------------------------------------------------------------------- /roles/storm/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for storm 3 | -------------------------------------------------------------------------------- /roles/wildfly/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/wildfly/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/wildfly/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for wildfly 3 | -------------------------------------------------------------------------------- /roles/wildfly/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for wildfly 3 | -------------------------------------------------------------------------------- /roles/wildfly/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: IOS 73 | # versions: 74 | # - all 75 | # - any 76 | #- name: SmartOS 77 | # versions: 78 | # - all 79 | # - any 80 | #- name: eos 81 | # versions: 82 | # - all 83 | # - Any 84 | #- name: Windows 85 | # versions: 86 | # - all 87 | # - 2012R2 88 | #- name: Amazon 89 | # versions: 90 | # - all 91 | # - 2013.03 92 | # - 2013.09 93 | #- name: GenericBSD 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: Junos 98 | # versions: 99 | # - all 100 | # - any 101 | #- name: FreeBSD 102 | # versions: 103 | # - all 104 | # - 10.0 105 | # - 10.1 106 | # - 10.2 107 | # - 8.0 108 | # - 8.1 109 | # - 8.2 110 | # - 8.3 111 | # - 8.4 112 | # - 9.0 113 | # - 9.1 114 | # - 9.1 115 | # - 9.2 116 | # - 9.3 117 | #- name: Ubuntu 118 | # versions: 119 | # - all 120 | # - lucid 121 | # - maverick 122 | # - natty 123 | # - oneiric 124 | # - precise 125 | # - quantal 126 | # - raring 127 | # - saucy 128 | # - trusty 129 | # - utopic 130 | # - vivid 131 | # - wily 132 | # - xenial 133 | #- name: SLES 134 | # versions: 135 | # - all 136 | # - 10SP3 137 | # - 10SP4 138 | # - 11 139 | # - 11SP1 140 | # - 11SP2 141 | # - 11SP3 142 | #- name: GenericLinux 143 | # versions: 144 | # - all 145 | # - any 146 | #- name: NXOS 147 | # versions: 148 | # - all 149 | # - any 150 | #- name: Debian 151 | # versions: 152 | # - all 153 | # - etch 154 | # - jessie 155 | # - lenny 156 | # - sid 157 | # - squeeze 158 | # - stretch 159 | # - wheezy 160 | 161 | galaxy_tags: [] 162 | # List tags for your role here, one per line. A tag is 163 | # a keyword that describes and categorizes the role. 164 | # Users find roles by searching for tags. Be sure to 165 | # remove the '[]' above if you add tags to this list. 166 | # 167 | # NOTE: A tag is limited to a single word comprised of 168 | # alphanumeric characters. Maximum 20 tags per role. 169 | 170 | dependencies: [] 171 | # List your role dependencies here, one per line. 172 | # Be sure to remove the '[]' above if you add dependencies 173 | # to this list. -------------------------------------------------------------------------------- /roles/wildfly/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for wildfly 3 | # http://download.jboss.org/wildfly/10.0.0.Final/wildfly-10.0.0.Final.tar.gz 4 | # $JBOSS_HOME/bin/standalone.sh -Djboss.socket.binding.port-offset=10000 5 | - name: stat /tmp/wildfly-10.0.0.Final.tar.gz 6 | stat: path=/tmp/wildfly-10.0.0.Final.tar.gz 7 | register: wildfly_binary_stat 8 | 9 | - name: Downloading wildfly 10 | get_url: url=http://download.jboss.org/wildfly/10.0.0.Final/wildfly-10.0.0.Final.tar.gz dest=/tmp/wildfly-10.0.0.Final.tar.gz 11 | when: wildfly_binary_stat.stat.exists == False 12 | 13 | - name: uncompress file tarball and place in /usr/local/ 14 | command: creates="/usr/local/wildfly-10.0.0.Final" tar zxf /tmp/wildfly-10.0.0.Final.tar.gz -C /usr/local/ 15 | 16 | - name: rename file 17 | command: creates="/usr/local/wildfly" mv /usr/local/wildfly-10.0.0.Final /usr/local/wildfly 18 | 19 | - name: change ownerships file 20 | command: chown -R hadoop:hadoop /usr/local/wildfly 21 | 22 | - name: setup standalone.xml file 23 | template: src=standalone.xml.j2 dest=/usr/local/wildfly/standalone/configuration/standalone.xml 24 | -------------------------------------------------------------------------------- /roles/wildfly/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/wildfly/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - wildfly -------------------------------------------------------------------------------- /roles/wildfly/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for wildfly 3 | -------------------------------------------------------------------------------- /roles/zeppelin/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /roles/zeppelin/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/zeppelin/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for zeppelin 3 | -------------------------------------------------------------------------------- /roles/zeppelin/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for zeppelin 3 | -------------------------------------------------------------------------------- /roles/zeppelin/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: OpenBSD 46 | # versions: 47 | # - all 48 | # - 5.6 49 | # - 5.7 50 | # - 5.8 51 | # - 5.9 52 | # - 6.0 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | # - 23 64 | #- name: opensuse 65 | # versions: 66 | # - all 67 | # - 12.1 68 | # - 12.2 69 | # - 12.3 70 | # - 13.1 71 | # - 13.2 72 | #- name: MacOSX 73 | # versions: 74 | # - all 75 | # - 10.10 76 | # - 10.11 77 | # - 10.12 78 | # - 10.7 79 | # - 10.8 80 | # - 10.9 81 | #- name: IOS 82 | # versions: 83 | # - all 84 | # - any 85 | #- name: Solaris 86 | # versions: 87 | # - all 88 | # - 10 89 | # - 11.0 90 | # - 11.1 91 | # - 11.2 92 | # - 11.3 93 | #- name: SmartOS 94 | # versions: 95 | # - all 96 | # - any 97 | #- name: eos 98 | # versions: 99 | # - all 100 | # - Any 101 | #- name: Windows 102 | # versions: 103 | # - all 104 | # - 2012R2 105 | #- name: Amazon 106 | # versions: 107 | # - all 108 | # - 2013.03 109 | # - 2013.09 110 | #- name: GenericBSD 111 | # versions: 112 | # - all 113 | # - any 114 | #- name: Junos 115 | # versions: 116 | # - all 117 | # - any 118 | #- name: FreeBSD 119 | # versions: 120 | # - all 121 | # - 10.0 122 | # - 10.1 123 | # - 10.2 124 | # - 10.3 125 | # - 8.0 126 | # - 8.1 127 | # - 8.2 128 | # - 8.3 129 | # - 8.4 130 | # - 9.0 131 | # - 9.1 132 | # - 9.1 133 | # - 9.2 134 | # - 9.3 135 | #- name: Ubuntu 136 | # versions: 137 | # - all 138 | # - lucid 139 | # - maverick 140 | # - natty 141 | # - oneiric 142 | # - precise 143 | # - quantal 144 | # - raring 145 | # - saucy 146 | # - trusty 147 | # - utopic 148 | # - vivid 149 | # - wily 150 | # - xenial 151 | #- name: SLES 152 | # versions: 153 | # - all 154 | # - 10SP3 155 | # - 10SP4 156 | # - 11 157 | # - 11SP1 158 | # - 11SP2 159 | # - 11SP3 160 | # - 11SP4 161 | # - 12 162 | # - 12SP1 163 | #- name: GenericLinux 164 | # versions: 165 | # - all 166 | # - any 167 | #- name: NXOS 168 | # versions: 169 | # - all 170 | # - any 171 | #- name: Debian 172 | # versions: 173 | # - all 174 | # - etch 175 | # - jessie 176 | # - lenny 177 | # - sid 178 | # - squeeze 179 | # - stretch 180 | # - wheezy 181 | 182 | galaxy_tags: [] 183 | # List tags for your role here, one per line. A tag is 184 | # a keyword that describes and categorizes the role. 185 | # Users find roles by searching for tags. Be sure to 186 | # remove the '[]' above if you add tags to this list. 187 | # 188 | # NOTE: A tag is limited to a single word comprised of 189 | # alphanumeric characters. Maximum 20 tags per role. 190 | 191 | dependencies: [] 192 | # List your role dependencies here, one per line. 193 | # Be sure to remove the '[]' above if you add dependencies 194 | # to this list. -------------------------------------------------------------------------------- /roles/zeppelin/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for zeppelin 3 | 4 | 5 | - name: stat /tmp/zeppelin-0.6.0-bin-netinst.tgz 6 | stat: path=/tmp/zeppelin-0.6.0-bin-netinst.tgz 7 | register: zepbinary_stat 8 | 9 | - name: Downloading Apache Pig 10 | get_url: url=http://mirror.csclub.uwaterloo.ca/apache/zeppelin/zeppelin-0.6.0/zeppelin-0.6.0-bin-netinst.tgz dest=/tmp/zeppelin-0.6.0-bin-netinst.tgz 11 | when: zepbinary_stat.stat.exists == False 12 | 13 | - name: rename file 14 | command: chown hadoop:hadoop /tmp/zeppelin-0.6.0-bin-netinst.tgz 15 | 16 | 17 | - name: uncompress file tarball and place in /usr/local/ 18 | command: tar zxf /tmp/zeppelin-0.6.0-bin-netinst.tgz -C /usr/local/ 19 | 20 | - name: rename file 21 | command: creates="/usr/local/zeppelin" mv /usr/local/zeppelin-0.6.0-bin-netinst/ /usr/local/zeppelin 22 | 23 | - name: uncompress zeppelin tarball and place in /usr/local/ 24 | command: chown -R hadoop:hadoop /usr/local/zeppelin 25 | -------------------------------------------------------------------------------- /roles/zeppelin/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /roles/zeppelin/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - zeppelin -------------------------------------------------------------------------------- /roles/zeppelin/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for zeppelin 3 | -------------------------------------------------------------------------------- /roles/zookeeper/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /roles/zookeeper/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for zookeeper 3 | -------------------------------------------------------------------------------- /roles/zookeeper/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for zookeeper 3 | - name: reload systemd 4 | sudo: yes 5 | command: systemctl {{ item }} 6 | with_items: 7 | - stop zookeeper 8 | - daemon-reload 9 | - name: start zookeeper 10 | sudo: yes 11 | command: systemctl start zookeeper 12 | -------------------------------------------------------------------------------- /roles/zookeeper/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Zak Hassan 4 | description: Demo of provisioning 3 node zookeeper cluster using zookeeper 3.5.0 5 | company: your company (optional) 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: https://github.com/BZCareer/ansible-playbook/issues 9 | # - Apache 10 | # license: license (GPLv2, CC-BY, etc) 11 | # min_ansible_version: 1.2 12 | # 13 | # Below are all platforms currently available. Just uncomment 14 | # the ones that apply to your role. If you don't see your 15 | # platform on this list, let us know and we'll get it added! 16 | # 17 | # platforms: 18 | # - name: EL 19 | # versions: 20 | # - all 21 | # - 5 22 | # - 6 23 | # - 7 24 | #- name: GenericUNIX 25 | # versions: 26 | # - all 27 | # - any 28 | #- name: Solaris 29 | # versions: 30 | # - all 31 | # - 10 32 | # - 11.0 33 | # - 11.1 34 | # - 11.2 35 | # - 11.3 36 | #- name: Fedora 37 | # versions: 38 | # - all 39 | # - 16 40 | # - 17 41 | # - 18 42 | # - 19 43 | # - 20 44 | # - 21 45 | # - 22 46 | # - 23 47 | #- name: Windows 48 | # versions: 49 | # - all 50 | # - 2012R2 51 | #- name: SmartOS 52 | # versions: 53 | # - all 54 | # - any 55 | #- name: opensuse 56 | # versions: 57 | # - all 58 | # - 12.1 59 | # - 12.2 60 | # - 12.3 61 | # - 13.1 62 | # - 13.2 63 | #- name: Amazon 64 | # versions: 65 | # - all 66 | # - 2013.03 67 | # - 2013.09 68 | #- name: GenericBSD 69 | # versions: 70 | # - all 71 | # - any 72 | #- name: FreeBSD 73 | # versions: 74 | # - all 75 | # - 10.0 76 | # - 10.1 77 | # - 10.2 78 | # - 8.0 79 | # - 8.1 80 | # - 8.2 81 | # - 8.3 82 | # - 8.4 83 | # - 9.0 84 | # - 9.1 85 | # - 9.1 86 | # - 9.2 87 | # - 9.3 88 | #- name: Ubuntu 89 | # versions: 90 | # - all 91 | # - lucid 92 | # - maverick 93 | # - natty 94 | # - oneiric 95 | # - precise 96 | # - quantal 97 | # - raring 98 | # - saucy 99 | # - trusty 100 | # - utopic 101 | # - vivid 102 | # - wily 103 | #- name: SLES 104 | # versions: 105 | # - all 106 | # - 10SP3 107 | # - 10SP4 108 | # - 11 109 | # - 11SP1 110 | # - 11SP2 111 | # - 11SP3 112 | #- name: GenericLinux 113 | # versions: 114 | # - all 115 | # - any 116 | #- name: Debian 117 | # versions: 118 | # - all 119 | # - etch 120 | # - jessie 121 | # - lenny 122 | # - squeeze 123 | # - wheezy 124 | # 125 | # Below are all categories currently available. Just as with 126 | # the platforms above, uncomment those that apply to your role. 127 | # 128 | #categories: 129 | #- cloud 130 | #- cloud:ec2 131 | #- cloud:gce 132 | #- cloud:rax 133 | #- clustering 134 | #- database 135 | #- database:nosql 136 | #- database:sql 137 | #- development 138 | #- monitoring 139 | #- networking 140 | #- packaging 141 | #- system 142 | #- web 143 | dependencies: [] 144 | # List your role dependencies here, one per line. 145 | # Be sure to remove the '[]' above if you add dependencies 146 | # to this list. 147 | -------------------------------------------------------------------------------- /roles/zookeeper/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ################################################################################### 3 | # 4 | # Author: Zak Hassan 5 | # Description: Playbook Apache Zookeeper on Openstack like less than 1 min. :) 6 | # 7 | ################################################################################### 8 | 9 | # Creating zookeeper user 10 | - group: name=zookeeper state=present 11 | - user: name=zookeeper comment="zookeeper" group=zookeeper shell=/bin/bash 12 | 13 | - name: stat /tmp/zookeeper-3.5.0-alpha.tar.gz 14 | stat: path=/tmp/zookeeper-3.5.0-alpha.tar.gz 15 | register: zkbinary_stat 16 | # Warning: If the mirror is down the just udpate the url below. And rerun. 17 | - name: Downloading Apache Zookeeper 18 | get_url: url=http://apache.mirror.gtcomm.net/zookeeper/zookeeper-3.5.0-alpha/zookeeper-3.5.0-alpha.tar.gz dest=/tmp/zookeeper-3.5.0-alpha.tar.gz 19 | when: zkbinary_stat.stat.exists == False 20 | 21 | - name: change owners of file 22 | command: chown hadoop:hadoop /tmp/zookeeper-3.5.0-alpha.tar.gz 23 | 24 | - name: unzip tarball and place in /usr/local/ 25 | command: tar zxf /tmp/zookeeper-3.5.0-alpha.tar.gz -C /usr/local/ 26 | 27 | - name: stat /usr/local/zookeeper 28 | stat: path=/usr/local/zookeeper 29 | register: zk_dir_stat 30 | 31 | - name: rename file 32 | command: creates="/usr/local/zookeeper" mv /usr/local/zookeeper-3.5.0-alpha/ /usr/local/zookeeper 33 | when: zk_dir_stat.stat.exists == False 34 | 35 | - name: create logs directory 36 | file: path=/usr/local/zookeeper/logs state=directory mode=0777 group=hadoop owner=hadoop 37 | 38 | - name: setup configuration file 39 | template: src=zoo.cfg.j2 dest=/usr/local/zookeeper/conf/zoo.cfg 40 | 41 | - name: setup zk myid file 42 | template: src=myid.j2 dest=/usr/local/zookeeper/logs/myid 43 | 44 | 45 | - name: set permissions for installation placed in /usr/local/ 46 | command: chown -R hadoop:hadoop /usr/local/zookeeper 47 | 48 | - name: setup systemd unit file 49 | template: src=zookeeper-systemd.j2 dest=/lib/systemd/system/zookeeper.service 50 | notify: reload systemd 51 | 52 | - name: enable zookeper 53 | sudo: yes 54 | service: name=zookeeper enabled=yes state=started 55 | -------------------------------------------------------------------------------- /roles/zookeeper/templates/myid.j2: -------------------------------------------------------------------------------- 1 | {{ myid }} 2 | -------------------------------------------------------------------------------- /roles/zookeeper/templates/zoo.cfg.j2: -------------------------------------------------------------------------------- 1 | # Created by ansible 2 | tickTime={{ tickTime }} 3 | initLimit={{ initLimit }} 4 | syncLimit={{ syncLimit }} 5 | dataDir={{ dataDir }} 6 | clientPort={{ clientPort }} 7 | server.1={{ server1 }}:2888:3888 8 | server.2={{ server2 }}:2888:3888 9 | server.3={{ server3 }}:2888:3888 10 | -------------------------------------------------------------------------------- /roles/zookeeper/templates/zookeeper-systemd.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Zookeeper Node Start Up 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=zookeeper 8 | Environment=ZOO_LOG_DIR={{ dataDir }} JAVA_HOME=/usr/lib/jvm/java-1.8.0 9 | ExecStart={{ zk_dir }}/bin/zkServer.sh start {{ zk_dir }}/conf/zoo.cfg 10 | ExecStop={{ zk_dir }}/bin/zkServer.sh stop 11 | ExecReload={{ zk_dir }}/bin/zkServer.sh restart 12 | Restart=on-abort 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /roles/zookeeper/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for zookeeper 3 | service_name: zookeeper 4 | zk_dir: /usr/local/zookeeper 5 | dataDir: /usr/local/zookeeper/logs 6 | syncLimit: 3 7 | initLimit: 5 8 | tickTime: 2000 9 | clientPort: 2181 10 | -------------------------------------------------------------------------------- /run-playbook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Author: Zak Hassan 4 | export ANSIBLE_HOST_KEY_CHECKING=False 5 | export ANSIBLE_CONFIG=/vagrant/ansible.cfg 6 | ansible-playbook -i invm.yml site.yml $AN_ARGS 7 | -------------------------------------------------------------------------------- /site-ha.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Author: Zak Hassan 3 | # Usage: ansible-playbook site.yml or make install 4 | # 5 | # Description: Playbook created to demonstrate provisioning cloud environment in like less than 1 min. :) 6 | 7 | # Three servers will be name nodes 1 active, 1 standby and 1 backup 8 | # Will have a layer for messaging for integration with third party apps. 9 | 10 | - name: Configure Hadoop Namenodes with High Availability 11 | sudo: True 12 | hosts: hadoop-master-namenodes # This will be replaced with the name nodes 13 | - { role: shared, when: "ansible_os_family == 'RedHat'" } # Base data 14 | - { role: zookeeper, when: "ansible_os_family == 'RedHat'" } 15 | - { role: kafka, when: "ansible_os_family == 'RedHat'" } 16 | - { role: hadoop, when: "ansible_os_family == 'RedHat'" } 17 | - { role: hive, when: "ansible_os_family == 'RedHat'" } 18 | - { role: pig, when: "ansible_os_family == 'RedHat'" } 19 | - { role: spark, when: "ansible_os_family == 'RedHat'" } 20 | 21 | # We will have 2 datanodes 22 | - name: Configure Hadoop Data Nodes 23 | sudo: True 24 | hosts: hadoop-datanodes 25 | roles: 26 | - { role: shared, when: "ansible_os_family == 'RedHat'" } 27 | - { role: hadoop, when: "ansible_os_family == 'RedHat'" } 28 | - { role: spark, when: "ansible_os_family == 'RedHat'" } 29 | -------------------------------------------------------------------------------- /site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Author: Zak Hassan 3 | # Usage: ansible-playbook site.yml or make install 4 | # 5 | # 6 | # Description: Playbook created to demonstrate provisioning cloud environment in like less than 1 min. :) 7 | # 8 | 9 | 10 | - name: Configure Apache Zookeeper, Kafka, Storm, Spark, Hadoop On Cluster 11 | sudo: True 12 | hosts: 13 | - hadoop-master-node 14 | - hadoop-data1-node 15 | - hadoop-data2-node 16 | roles: 17 | - { role: shared, when: ansible_os_family == "RedHat" and skip_shared is not defined } 18 | - { role: hadoop, when: ansible_os_family == "RedHat" and install_hadoop is defined and install_hadoop == "true" } 19 | - { role: zookeeper, when: ansible_os_family == "RedHat" and install_zookeeper is defined and install_zookeeper == "true" } 20 | - { role: kafka, when: ansible_os_family == "RedHat" and install_kafka is defined and install_kafka == "true" } 21 | - { role: storm , when: ansible_os_family == "RedHat" and install_storm is defined and install_storm == "true" } 22 | - { role: spark, when: ansible_os_family == "RedHat" and install_spark is defined and install_spark == "true" } 23 | - { role: cassandra, when: ansible_os_family == "RedHat" and install_cassandra is defined and install_cassandra == "true" } 24 | # - { role: hbase, when: ansible_os_family == "RedHat" and install_hbase is defined and install_hbase == "true" } 25 | # - { role: flink, when: ansible_os_family == "RedHat" and install_flink is defined and install_flink == "true" } 26 | # - { role: flume, when: ansible_os_family == "RedHat" and install_flume is defined and install_flume == "true" } 27 | 28 | 29 | - name: Installing Big Data Utilities Hive, Spark, Zeppelin, Pig, Logstash, ElasticSearch, etc. 30 | sudo: True 31 | hosts: hadoop-master-node 32 | roles: 33 | - { role: elasticsearch, when: ansible_os_family == "RedHat" and install_elasticsearch is defined and install_elasticsearch == "true" } 34 | - { role: kibana, when: ansible_os_family == "RedHat" and install_kibana is defined and install_kibana == "true" } 35 | - { role: logstash, when: ansible_os_family == "RedHat" and install_logstash is defined and install_logstash == "true" } 36 | - { role: pig, when: ansible_os_family == "RedHat" and install_pig is defined and install_pig == "true" } 37 | - { role: mysql, when: ansible_os_family == "RedHat" and install_mysql is defined and install_mysql == "true" } 38 | - { role: hive, when: ansible_os_family == "RedHat" and install_hive is defined and install_hive == "true" } 39 | - { role: wildfly, when: ansible_os_family == "RedHat" and install_wildfly is defined and install_wildfly == "true" } 40 | # - { role: zeppelin, when: ansible_os_family == "RedHat" and install_zeppelin is defined and install_zeppelin == "true" } 41 | --------------------------------------------------------------------------------