├── roles ├── smokeping │ ├── defaults │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── templates │ │ ├── Probes │ │ ├── General │ │ ├── Targets │ │ └── Alerts │ └── tasks │ │ └── main.yml ├── cdh_hadoop_config │ ├── files │ │ └── dfs.hosts.exclude │ ├── meta │ │ └── main.yml │ ├── templates │ │ ├── fair-scheduler.xml │ │ ├── slaves │ │ ├── hadoop-env.sh │ │ ├── core-site.xml │ │ ├── org-xerial-snappy.properties │ │ ├── configuration.xsl │ │ ├── mapred-site.xml │ │ ├── hadoop-metrics2.properties │ │ ├── yarn-site.xml │ │ └── hdfs-site.xml │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── presto_common │ ├── templates │ │ ├── jmx.properties │ │ ├── tpch.properties │ │ ├── node.properties │ │ ├── hive.properties │ │ ├── jvm.config │ │ └── upstart.conf │ └── tasks │ │ └── main.yml ├── ganglia_web │ ├── meta │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── rsyslog_fluentd │ ├── templates │ │ └── 99-fluentd.conf │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── td_agent │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── files │ │ └── install_debian_libssl0.9.8.sh │ ├── templates │ │ └── td-agent.conf │ └── tasks │ │ └── main.yml ├── cdh_hadoop_common │ ├── meta │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── presto_worker │ ├── meta │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── config.properties │ └── tasks │ │ └── main.yml ├── cdh_hbase_config │ ├── meta │ │ └── main.yml │ ├── defaults │ │ └── main.yml │ ├── templates │ │ ├── regionservers │ │ ├── hbase-site.xml │ │ └── hbase-env.sh │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── presto_coordinator │ ├── meta │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── config.properties │ └── tasks │ │ └── main.yml ├── common │ ├── templates │ │ ├── client.conf │ │ ├── hosts │ │ ├── rsyslog.conf │ │ ├── ntp.conf │ │ ├── limits.conf │ │ └── sysctl.conf │ ├── handlers │ │ └── main.yml │ ├── defaults │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── 2_aggregated_links │ ├── meta │ │ └── main.yml │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── interfaces ├── cdh_hive_mysql_metastore │ ├── meta │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── cdh_hive_postgresql_metastore │ ├── meta │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── cdh_common │ ├── meta │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── ganglia_monitor │ ├── templates │ │ ├── device-metrics.cron.d │ │ ├── device-metrics.php │ │ └── gmond.conf │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── postfix_mandrill │ ├── templates │ │ ├── sasl_passwd │ │ ├── disk-usage-alert.sh │ │ └── 50unattended-upgrades │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── cdh_hadoop_mapreduce │ └── tasks │ │ └── main.yml ├── kibana │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── elasticsearch_curator │ ├── defaults │ │ └── main.yml │ ├── templates │ │ └── elasticsearch-curator.cron.d │ └── tasks │ │ └── main.yml ├── apache2 │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── ganglia_metad │ ├── handlers │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── gmetad.conf ├── cdh_hadoop_datanode │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── rsyslog.conf │ └── tasks │ │ └── main.yml ├── cdh_hadoop_namenode │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── rsyslog.conf │ └── tasks │ │ └── main.yml ├── cdh_hive_config │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── hive-site.xml ├── elasticsearch │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── cdh_hadoop_journalnode │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── rsyslog.conf │ └── tasks │ │ └── main.yml ├── cdh_hadoop_mapreduce_historyserver │ └── tasks │ │ └── main.yml ├── cdh_hive_metastore │ └── tasks │ │ └── main.yml ├── cdh_hbase_common │ └── tasks │ │ └── main.yml ├── cdh_zookeeper_server │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── zoo.cfg │ └── tasks │ │ └── main.yml ├── cdh_hadoop_yarn_proxyserver │ └── tasks │ │ └── main.yml ├── cdh_hive_common │ └── tasks │ │ └── main.yml ├── cdh_hadoop_zkfc │ └── tasks │ │ └── main.yml ├── cdh_hadoop_yarn_resourcemanager │ └── tasks │ │ └── main.yml ├── cdh_hbase_master │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── rsyslog.conf │ └── tasks │ │ └── main.yml ├── cdh_hbase_regionserver │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── rsyslog.conf │ └── tasks │ │ └── main.yml ├── aggregated_links_common │ └── tasks │ │ └── main.yml ├── cdh_hadoop_yarn_nodemanager │ └── tasks │ │ └── main.yml ├── oracle_jdk │ ├── files │ │ └── install_debian_webupd8team_repo.sh │ └── tasks │ │ └── main.yml ├── mysql_server │ └── tasks │ │ └── main.yml ├── postgresql_server │ └── tasks │ │ └── main.yml └── kernel │ └── tasks │ └── main.yml ├── bootstrap ├── ansible_rsa.pub ├── hosts ├── bootstrap.sh ├── ansible.cfg └── bootstrap.yml ├── images ├── kibana.png ├── ganglia.png ├── hmaster01.png ├── smokeping.png └── zookeeper.png ├── ansible.cfg ├── site.sh ├── travis.ssh.pub ├── group_vars └── all ├── screenshot.js ├── hosts ├── travis.ssh ├── do_cluster.yml ├── .travis.yml ├── site.yml ├── README.md └── LICENSE /roles/smokeping/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/files/dfs.hosts.exclude: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bootstrap/ansible_rsa.pub: -------------------------------------------------------------------------------- 1 | paste your public SSH RSA key here -------------------------------------------------------------------------------- /roles/presto_common/templates/jmx.properties: -------------------------------------------------------------------------------- 1 | connector.name=jmx -------------------------------------------------------------------------------- /roles/ganglia_web/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - apache2 -------------------------------------------------------------------------------- /roles/presto_common/templates/tpch.properties: -------------------------------------------------------------------------------- 1 | connector.name=tpch -------------------------------------------------------------------------------- /roles/smokeping/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - apache2 -------------------------------------------------------------------------------- /roles/rsyslog_fluentd/templates/99-fluentd.conf: -------------------------------------------------------------------------------- 1 | *.* @127.0.0.1:5140 2 | -------------------------------------------------------------------------------- /roles/td_agent/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | elasticsearch_port: 9200 -------------------------------------------------------------------------------- /roles/cdh_hadoop_common/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - cdh_common -------------------------------------------------------------------------------- /roles/presto_worker/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - presto_common -------------------------------------------------------------------------------- /roles/cdh_hbase_config/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - cdh_hbase_common -------------------------------------------------------------------------------- /roles/presto_coordinator/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - presto_common -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - cdh_hadoop_common -------------------------------------------------------------------------------- /roles/common/templates/client.conf: -------------------------------------------------------------------------------- 1 | [sysinfo] 2 | exclude_sysinfo_plugins = LandscapeLink -------------------------------------------------------------------------------- /roles/2_aggregated_links/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - aggregated_links_common -------------------------------------------------------------------------------- /roles/cdh_hive_mysql_metastore/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - cdh_hive_common -------------------------------------------------------------------------------- /images/kibana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/analytically/hadoop-ansible/HEAD/images/kibana.png -------------------------------------------------------------------------------- /roles/cdh_hive_postgresql_metastore/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - cdh_hive_common -------------------------------------------------------------------------------- /images/ganglia.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/analytically/hadoop-ansible/HEAD/images/ganglia.png -------------------------------------------------------------------------------- /images/hmaster01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/analytically/hadoop-ansible/HEAD/images/hmaster01.png -------------------------------------------------------------------------------- /images/smokeping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/analytically/hadoop-ansible/HEAD/images/smokeping.png -------------------------------------------------------------------------------- /images/zookeeper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/analytically/hadoop-ansible/HEAD/images/zookeeper.png -------------------------------------------------------------------------------- /bootstrap/hosts: -------------------------------------------------------------------------------- 1 | monitor01 2 | hmaster01 3 | hmaster02 4 | hslave01 5 | hslave02 6 | hslave03 7 | hslave04 -------------------------------------------------------------------------------- /roles/td_agent/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: restart td-agent 4 | command: /etc/init.d/td-agent restart -------------------------------------------------------------------------------- /roles/cdh_common/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - { role: oracle_jdk, when: jdk_installed is not defined } -------------------------------------------------------------------------------- /roles/smokeping/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: restart smokeping 4 | service: name=smokeping state=restarted -------------------------------------------------------------------------------- /roles/ganglia_monitor/templates/device-metrics.cron.d: -------------------------------------------------------------------------------- 1 | 5-55/10 * * * * root php /usr/local/sbin/device-metrics.php 1 1 2 | -------------------------------------------------------------------------------- /roles/postfix_mandrill/templates/sasl_passwd: -------------------------------------------------------------------------------- 1 | [smtp.mandrillapp.com] {{ mandrill_username }}:{{ mandrill_api_key }} 2 | -------------------------------------------------------------------------------- /bootstrap/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export ANSIBLE_HOST_KEY_CHECKING=False 3 | ansible-playbook -i hosts -k -K -u ansibler bootstrap.yml -------------------------------------------------------------------------------- /roles/cdh_common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_common/handlers/main.yml 3 | 4 | - name: ldconfig 5 | command: ldconfig -------------------------------------------------------------------------------- /roles/cdh_hadoop_mapreduce/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: install hadoop-mapreduce via apt 4 | apt: name=hadoop-mapreduce 5 | tags: hadoop -------------------------------------------------------------------------------- /roles/cdh_hbase_config/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_config/defaults/main.yml 3 | 4 | # The HBase heap size 5 | hbase_heapsize: 4096 -------------------------------------------------------------------------------- /roles/kibana/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/kibana/handlers/main.yml 3 | 4 | - name: restart apache2 5 | service: name=apache2 state=restarted -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/fair-scheduler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /roles/elasticsearch_curator/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # https://github.com/elasticsearch/curator arguments 4 | elasticsearch_curator_args: -C space -g 40 -------------------------------------------------------------------------------- /roles/apache2/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/apache2/handlers/main.yml 3 | 4 | - name: reload apache config 5 | service: name=apache2 state=reloaded -------------------------------------------------------------------------------- /roles/presto_common/templates/node.properties: -------------------------------------------------------------------------------- 1 | node.environment=production 2 | node.id={{ site_name|lower }}-{{ ansible_hostname }} 3 | node.data-dir=/data/presto -------------------------------------------------------------------------------- /bootstrap/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | timeout=20 3 | forks=20 4 | 5 | [ssh_connection] 6 | ssh_args=-o ControlMaster=auto -o ControlPersist=1800s 7 | pipelining=True -------------------------------------------------------------------------------- /roles/presto_common/templates/hive.properties: -------------------------------------------------------------------------------- 1 | connector.name=hive-cdh4 2 | hive.metastore.uri=thrift://{{ hostvars[groups["hive_metastore"][0]]["ansible_fqdn"] }}:9083 -------------------------------------------------------------------------------- /roles/presto_worker/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/presto_worker/handlers/main.yml 3 | 4 | - name: restart presto 5 | service: name=presto state=restarted -------------------------------------------------------------------------------- /roles/ganglia_metad/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/ganglia_metad/handlers/main.yml 3 | 4 | - name: restart gmetad 5 | service: name=gmetad state=restarted 6 | -------------------------------------------------------------------------------- /roles/ganglia_web/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/ganglia_monitor/handlers/main.yml 3 | 4 | - name: restart apache2 5 | service: name=apache2 state=restarted 6 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | timeout=20 3 | forks=20 4 | 5 | [ssh_connection] 6 | ssh_args=-o ControlMaster=auto -o ControlPersist=1800s -o ForwardAgent=yes 7 | pipelining=True -------------------------------------------------------------------------------- /roles/presto_coordinator/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/presto_coordinator/handlers/main.yml 3 | 4 | - name: restart presto 5 | service: name=presto state=restarted -------------------------------------------------------------------------------- /roles/cdh_hadoop_datanode/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_datanode/handlers/main.yml 3 | 4 | - name: restart rsyslog 5 | service: name=rsyslog state=restarted -------------------------------------------------------------------------------- /roles/cdh_hadoop_namenode/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_namenode/handlers/main.yml 3 | 4 | - name: restart rsyslog 5 | service: name=rsyslog state=restarted -------------------------------------------------------------------------------- /roles/cdh_hive_config/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hive_config/defaults/main.yml 3 | 4 | # Choose postgresql or mysql as hive metastore 5 | hive_metastore: postgresql -------------------------------------------------------------------------------- /roles/elasticsearch/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/elasticsearch/handlers/main.yml 3 | 4 | - name: restart elasticsearch 5 | service: name=elasticsearch state=restarted -------------------------------------------------------------------------------- /roles/cdh_hadoop_journalnode/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_journalnode/handlers/main.yml 3 | 4 | - name: restart rsyslog 5 | service: name=rsyslog state=restarted -------------------------------------------------------------------------------- /roles/cdh_hadoop_mapreduce_historyserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: install hadoop-mapreduce-historyserver via apt 4 | apt: name=hadoop-mapreduce-historyserver 5 | tags: hadoop -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/slaves: -------------------------------------------------------------------------------- 1 | {% for host in groups['datanodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }} 2 | {% endfor %} 3 | -------------------------------------------------------------------------------- /roles/cdh_hive_metastore/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hive_metastore/tasks/main.yml 3 | 4 | - name: install hive-metastore via apt 5 | apt: name=hive-metastore 6 | tags: hive -------------------------------------------------------------------------------- /roles/2_aggregated_links/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # see https://www.kernel.org/doc/Documentation/networking/bonding.txt - balance-alb doesn't require any switch setup 4 | bond_mode: balance-alb -------------------------------------------------------------------------------- /roles/elasticsearch_curator/templates/elasticsearch-curator.cron.d: -------------------------------------------------------------------------------- 1 | 0 2 * * * * root curator --host {{ hostvars[groups["elasticsearch"][0]]["ansible_fqdn"] }} {{ elasticsearch_curator_args }} 2 | -------------------------------------------------------------------------------- /roles/cdh_hbase_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_common/tasks/main.yml 3 | 4 | - name: install base HBase package via apt 5 | apt: name=hbase update_cache=yes 6 | tags: hbase -------------------------------------------------------------------------------- /roles/cdh_hbase_config/templates/regionservers: -------------------------------------------------------------------------------- 1 | {% for host in groups['regionservers'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }} 2 | {% endfor %} 3 | -------------------------------------------------------------------------------- /roles/cdh_hive_config/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hive_config/handlers/main.yml 3 | 4 | - name: restart hive-metastore 5 | service: name=hive-metastore state=restarted 6 | ignore_errors: yes -------------------------------------------------------------------------------- /roles/cdh_zookeeper_server/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_zookeeper_server/handlers/main.yml 3 | 4 | - name: restart zookeeper 5 | service: name=zookeeper-server state=restarted 6 | ignore_errors: yes -------------------------------------------------------------------------------- /roles/cdh_hadoop_yarn_proxyserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_yarn_proxyserver/tasks/main.yml 3 | 4 | - name: install hadoop-yarn-proxyserver via apt 5 | apt: name=hadoop-yarn-proxyserver 6 | tags: hadoop -------------------------------------------------------------------------------- /roles/cdh_hive_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hive_common/tasks/main.yml 3 | 4 | - name: install base Hive package via apt 5 | apt: name=hive update_cache=yes 6 | tags: 7 | - postgres 8 | - hive 9 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_zkfc/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_zkfc/tasks/main.yml 3 | 4 | - name: install hadoop-hdfs-zkfc (Hadoop HDFS Zookeeper Failover Controller) via apt 5 | apt: name=hadoop-hdfs-zkfc 6 | tags: hadoop -------------------------------------------------------------------------------- /roles/cdh_hadoop_yarn_resourcemanager/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_yarn_resourcemanager/tasks/main.yml 3 | 4 | - name: install hadoop-yarn-resourcemanager via apt 5 | apt: name=hadoop-yarn-resourcemanager 6 | tags: hadoop -------------------------------------------------------------------------------- /roles/smokeping/templates/Probes: -------------------------------------------------------------------------------- 1 | *** Probes *** 2 | 3 | + FPing 4 | binary = /usr/bin/fping 5 | 6 | + Curl 7 | binary = /usr/bin/curl 8 | step = 60 9 | 10 | # a default for this target-specific variable 11 | urlformat = http://%host%/ -------------------------------------------------------------------------------- /roles/cdh_hbase_master/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_master/handlers/main.yml 3 | 4 | - name: restart rsyslog 5 | service: name=rsyslog state=restarted 6 | 7 | - name: start hbase-master 8 | service: name=hbase-master state=restarted -------------------------------------------------------------------------------- /roles/cdh_hbase_master/templates/rsyslog.conf: -------------------------------------------------------------------------------- 1 | $InputFileName /var/log/hbase/hbase-hbase-master-{{ ansible_fqdn }}.log 2 | $InputFileTag hbase-master: 3 | $InputFileStateFile state-hbase-master-error 4 | $InputFileSeverity info 5 | $InputRunFileMonitor 6 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_datanode/templates/rsyslog.conf: -------------------------------------------------------------------------------- 1 | $InputFileName /var/log/hadoop-hdfs/hadoop-hdfs-datanode-{{ ansible_fqdn }}.log 2 | $InputFileTag hadoop-hdfs-datanode: 3 | $InputFileStateFile state-hadoop-hdfs-datanode-error 4 | $InputFileSeverity info 5 | $InputRunFileMonitor 6 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_namenode/templates/rsyslog.conf: -------------------------------------------------------------------------------- 1 | $InputFileName /var/log/hadoop-hdfs/hadoop-hdfs-namenode-{{ ansible_fqdn }}.log 2 | $InputFileTag hadoop-hdfs-namenode: 3 | $InputFileStateFile state-hadoop-hdfs-namenode-error 4 | $InputFileSeverity info 5 | $InputRunFileMonitor 6 | -------------------------------------------------------------------------------- /roles/cdh_hbase_regionserver/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_regionserver/handlers/main.yml 3 | 4 | - name: restart rsyslog 5 | service: name=rsyslog state=restarted 6 | 7 | - name: start hbase-regionserver 8 | service: name=hbase-regionserver state=restarted -------------------------------------------------------------------------------- /roles/cdh_hbase_regionserver/templates/rsyslog.conf: -------------------------------------------------------------------------------- 1 | $InputFileName /var/log/hbase/hbase-hbase-regionserver-{{ ansible_fqdn }}.log 2 | $InputFileTag hbase-regionserver: 3 | $InputFileStateFile state-hbase-regionserver-error 4 | $InputFileSeverity info 5 | $InputRunFileMonitor 6 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_journalnode/templates/rsyslog.conf: -------------------------------------------------------------------------------- 1 | $InputFileName /var/log/hadoop-hdfs/hadoop-hdfs-journalnode-{{ ansible_fqdn }}.log 2 | $InputFileTag hadoop-hdfs-journalnode: 3 | $InputFileStateFile state-hadoop-hdfs-journalnode-error 4 | $InputFileSeverity info 5 | $InputRunFileMonitor 6 | -------------------------------------------------------------------------------- /roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/common/handlers/main.yml 3 | 4 | - name: restart ntp 5 | service: name=ntp state=restarted 6 | 7 | - name: restart procps 8 | service: name=procps state=restarted 9 | 10 | - name: restart rsyslog 11 | service: name=rsyslog state=restarted -------------------------------------------------------------------------------- /roles/common/templates/hosts: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | 127.0.0.1 localhost 4 | 5 | {% for host in groups['all'] %} 6 | {{ hostvars[host].ipv4_address|default(hostvars[host].ansible_default_ipv4.address) }} {{ hostvars[host]["ansible_fqdn"] }} {{ hostvars[host]["ansible_hostname"] }} 7 | {% endfor %} 8 | -------------------------------------------------------------------------------- /roles/ganglia_monitor/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/ganglia_monitor/handlers/main.yml 3 | 4 | # for some reason gmond hangs when restarting it on Ubuntu 12.04+, use kill here 5 | - name: restart ganglia-monitor 6 | shell: killall -v gmond && /usr/sbin/gmond --pid-file=/var/run/ganglia-monitor.pid -------------------------------------------------------------------------------- /roles/td_agent/files/install_debian_libssl0.9.8.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | dpkg --list libssl > /dev/null 4 | if [ $? != 0 ]; then 5 | cd /tmp 6 | wget http://ftp.us.debian.org/debian/pool/main/o/openssl/libssl0.9.8_0.9.8o-4squeeze14_amd64.deb 7 | dpkg -i libssl0.9.8_0.9.8o-4squeeze14_amd64.deb 8 | fi -------------------------------------------------------------------------------- /roles/rsyslog_fluentd/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/rsylog_fluentd/handlers/main.yml 3 | 4 | - name: restart rsyslog 5 | service: name=rsyslog state=restarted 6 | 7 | - name: log installation message 8 | command: logger -t ansible "Configured rsyslog integration with Fluentd on {{ ansible_fqdn }}" 9 | -------------------------------------------------------------------------------- /roles/common/templates/rsyslog.conf: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | $ModLoad imfile 4 | 5 | # don't escape newlines 6 | $EscapeControlCharactersOnReceive off 7 | $SystemLogRateLimitInterval 0 8 | $RepeatedMsgReduction off 9 | $MaxMessageSize 32768 10 | 11 | $InputFilePollInterval {{ rsyslog_polling_interval_secs }} -------------------------------------------------------------------------------- /roles/presto_worker/templates/config.properties: -------------------------------------------------------------------------------- 1 | coordinator=false 2 | datasources=jmx,hive,tpch 3 | http-server.http.port=8081 4 | presto-metastore.db.type=h2 5 | presto-metastore.db.filename=var/db/MetaStore 6 | task.max-memory=1GB 7 | discovery.uri=http://{{ hostvars[groups["presto_coordinators"][0]]["ansible_fqdn"] }}:8081 -------------------------------------------------------------------------------- /roles/cdh_hbase_config/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_config/handlers/main.yml 3 | 4 | - name: restart hbase-master 5 | service: name=hbase-master state=restarted 6 | ignore_errors: yes 7 | 8 | - name: restart hbase-regionserver 9 | service: name=hbase-regionserver state=restarted 10 | ignore_errors: yes -------------------------------------------------------------------------------- /roles/postfix_mandrill/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/postfix_mandrill/handlers/main.yml 3 | 4 | - name: restart postfix 5 | service: name=postfix state=restarted 6 | 7 | - name: send test email 8 | shell: echo "This is a test message during an Ansible configuration of {{ ansible_hostname }} at $(date -R)" | sendmail {{ notify_email }} -------------------------------------------------------------------------------- /roles/presto_coordinator/templates/config.properties: -------------------------------------------------------------------------------- 1 | coordinator=true 2 | datasources=jmx 3 | http-server.http.port=8081 4 | presto-metastore.db.type=h2 5 | presto-metastore.db.filename=var/db/MetaStore 6 | task.max-memory=1GB 7 | discovery-server.enabled=true 8 | discovery.uri=http://{{ hostvars[groups["presto_coordinators"][0]]["ansible_fqdn"] }}:8081 -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_config/defaults/main.yml 3 | 4 | # The default block size for new files, in bytes - here 256 MB 5 | dfs_blocksize: 268435456 6 | 7 | # Upper bound on the number of files that it will serve at any one time 8 | max_xcievers: 4096 9 | 10 | # The Hadoop DataNode heap size 11 | heapsize: 12278 -------------------------------------------------------------------------------- /roles/kibana/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/kibana/tasks/main.yml 3 | 4 | - name: download and extract Kibana to /var/www/kibana 5 | shell: creates=/var/www/kibana chdir=/tmp curl -O http://download.elasticsearch.org/kibana/kibana/kibana-latest.zip && unzip kibana-latest.zip && mv kibana-latest /var/www/kibana 6 | tags: kibana 7 | notify: 8 | - restart apache2 -------------------------------------------------------------------------------- /roles/presto_worker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/presto_worker/tasks/main.yml 3 | 4 | - name: configure presto worker 5 | template: src={{ item }} dest=/usr/lib/presto/etc/{{ item }} owner=root group=root mode=0644 6 | with_items: 7 | - config.properties 8 | notify: 9 | - restart presto 10 | tags: 11 | - configuration 12 | - presto -------------------------------------------------------------------------------- /roles/presto_coordinator/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/presto_coordinator/tasks/main.yml 3 | 4 | - name: configure presto coordinator 5 | template: src={{ item }} dest=/usr/lib/presto/etc/{{ item }} owner=root group=root mode=0644 6 | with_items: 7 | - config.properties 8 | notify: 9 | - restart presto 10 | tags: 11 | - configuration 12 | - presto -------------------------------------------------------------------------------- /roles/aggregated_links_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/aggregated_links_common/tasks/main.yml 3 | # see https://help.ubuntu.com/community/UbuntuBonding 4 | 5 | - name: install ifenslave-2.6 6 | apt: pkg=ifenslave-2.6 7 | tags: bonding 8 | 9 | - name: configure bonding kernel module 10 | lineinfile: dest=/etc/modules regexp="bonding" line=bonding 11 | tags: bonding -------------------------------------------------------------------------------- /roles/presto_common/templates/jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xmx8G 3 | -XX:+UseConcMarkSweepGC 4 | -XX:+ExplicitGCInvokesConcurrent 5 | -XX:+CMSClassUnloadingEnabled 6 | -XX:+AggressiveOpts 7 | -XX:+HeapDumpOnOutOfMemoryError 8 | -XX:OnOutOfMemoryError=kill -9 %p 9 | -XX:PermSize=150M 10 | -XX:MaxPermSize=150M 11 | -XX:ReservedCodeCacheSize=150M 12 | -Xbootclasspath/p:/usr/lib/presto/lib/floatingdecimal-0.1.jar -------------------------------------------------------------------------------- /roles/rsyslog_fluentd/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/rsylog_fluentd/tasks/main.yml 3 | 4 | - name: configure fluentd in /etc/rsyslog.d/ 5 | template: src={{ item }} dest=/etc/rsyslog.d/{{ item }} owner=root group=root mode=0644 6 | with_items: 7 | - 99-fluentd.conf 8 | tags: 9 | - rsyslog 10 | - fluentd 11 | notify: 12 | - restart rsyslog 13 | - log installation message -------------------------------------------------------------------------------- /site.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | IFS=$',' 4 | 5 | export ANSIBLE_ERROR_ON_UNDEFINED_VARS=True 6 | export ANSIBLE_HOST_KEY_CHECKING=False 7 | 8 | if [ $# -gt 0 ] 9 | then 10 | echo 'Running ansible-playbook -i hosts site.yml --tags' "$*" 11 | ansible-playbook -i hosts site.yml --tags "$*" 12 | else 13 | echo 'Running ansible-playbook -i hosts site.yml' 14 | ansible-playbook -i hosts site.yml 15 | fi -------------------------------------------------------------------------------- /travis.ssh.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD1J3LsiGCDBPqwdnDyDHif7DnusJvTCydtg6cT0414J+8+jssgrxSzdM8da7V+CjFssD3X8KUM2SrSOoVEKq7ZJ1KGnGapKKmiW2oZ8Ndy5y3Xox7mP1kLYoFEunjy4zOe6kftiI/rKytEBjFCbkqtVa/giF9j5u/GeKcZAsEMHa2UqFe6w6n+MNkqCZzaPBBXlI+9gXdWXVv3d74rdHh0i24zVr+MjelFXetOLUGHOiyr29DQ5wd/V5GHCyZ/7nBNS0GszVDX8XWbwHjdFZ1Oaqswu+okZCSN2SrTXIF2YMWK7tupmSNagUVa9xPZrs8GEC64AZeXNQh/UJO+TzRn noreply@travis-ci.org 2 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | export HADOOP_HEAPSIZE={{ heapsize }} 4 | 5 | export HADOOP_OPTS="-Xbootclasspath/p:/usr/lib/jvm/floatingdecimal-0.1.jar -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -XX:MaxPermSize=300M -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+ExplicitGCInvokesConcurrent -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/var/log/hadoop-hdfs/hadoop-hdfs-gc.log" -------------------------------------------------------------------------------- /roles/cdh_hadoop_namenode/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_namenode/tasks/main.yml 3 | 4 | - name: install hadoop-hdfs-namenode via apt 5 | apt: name=hadoop-hdfs-namenode 6 | tags: hadoop 7 | 8 | - name: configure rsyslog for hadoop-hdfs-namenode 9 | template: src=rsyslog.conf dest=/etc/rsyslog.d/60-hadoop-hdfs-namenode.conf owner=root group=root mode=0644 10 | tags: rsyslog 11 | notify: 12 | - restart rsyslog -------------------------------------------------------------------------------- /roles/cdh_hbase_master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase-master/tasks/main.yml 3 | 4 | - name: configre rsyslog for hbase-master 5 | template: src=rsyslog.conf dest=/etc/rsyslog.d/60-hbase-master.conf owner=root group=root mode=0644 6 | tags: rsyslog 7 | notify: 8 | - restart rsyslog 9 | 10 | - name: install hbase-master via apt 11 | apt: name=hbase-master 12 | tags: hbase 13 | notify: 14 | - start hbase-master -------------------------------------------------------------------------------- /roles/apache2/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/apache2/tasks/main.yml 3 | 4 | - name: install apache2 via apt 5 | apt: pkg={{ item }} 6 | with_items: 7 | - apache2 8 | tags: apache 9 | 10 | - name: configure apache2 so it doesn't complain 'can't determine fqdn' 11 | lineinfile: dest=/etc/apache2/apache2.conf regexp="{{ ansible_fqdn }}" line="ServerName {{ ansible_fqdn }}" 12 | notify: 13 | - reload apache config 14 | tags: apache 15 | -------------------------------------------------------------------------------- /roles/ganglia_metad/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/ganglia_metad/tasks/main.yml 3 | 4 | - name: install Ganglia meta deamon and rrdtool via apt 5 | apt: name={{ item }} 6 | with_items: 7 | - gmetad 8 | - rrdtool 9 | tags: ganglia 10 | 11 | - name: configure gmetad.conf in /etc/ganglia/gmetad.conf 12 | template: src=gmetad.conf dest=/etc/ganglia/gmetad.conf owner=root group=root mode=0644 13 | tags: ganglia 14 | notify: 15 | - restart gmetad -------------------------------------------------------------------------------- /group_vars/all: -------------------------------------------------------------------------------- 1 | site_name: acme 2 | 3 | # IPv4 maximum transmission unit, 9216 is the maximum for Intel/Cisco hardware 4 | mtu: 9216 5 | 6 | # To enable Postfix SMTP through Mandrill (@mandrillapp), set the following variables: 7 | 8 | # notify_email: notifications@example.com 9 | # postfix_domain: example.com 10 | # mandrill_username: your_username 11 | # mandrill_api_key: your_api_key 12 | 13 | # Upgrade kernel to 3.13, much improved epoll performance 14 | upgrade_kernel: no -------------------------------------------------------------------------------- /roles/cdh_hbase_regionserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_regionserver/tasks/main.yml 3 | 4 | - name: configure rsyslog for hbase-regionserver 5 | template: src=rsyslog.conf dest=/etc/rsyslog.d/60-hbase-regionserver.conf owner=root group=root mode=0644 6 | tags: rsyslog 7 | notify: 8 | - restart rsyslog 9 | 10 | - name: install hbase-regionserver via apt 11 | apt: name=hbase-regionserver 12 | tags: hbase 13 | notify: 14 | - start hbase-regionserver -------------------------------------------------------------------------------- /roles/cdh_hadoop_yarn_nodemanager/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_yarn_nodemanager/tasks/main.yml 3 | 4 | - name: create the /data/yarn/local directory 5 | file: path=/data/yarn/local owner=yarn group=yarn state=directory 6 | tags: hadoop 7 | 8 | - name: create the /data/yarn/logs directory 9 | file: path=/data/yarn/logs owner=yarn group=yarn state=directory 10 | tags: hadoop 11 | 12 | - name: install hadoop-yarn-nodemanager via apt 13 | apt: name=hadoop-yarn-nodemanager 14 | tags: hadoop -------------------------------------------------------------------------------- /roles/presto_common/templates/upstart.conf: -------------------------------------------------------------------------------- 1 | description "Facebook Presto" 2 | 3 | start on started hive-metastore 4 | stop on stopped hive-metastore 5 | 6 | respawn 7 | respawn limit 5 10 8 | 9 | expect fork 10 | 11 | chdir /usr/lib/presto/bin 12 | 13 | script 14 | CMD="./launcher start" 15 | logger -is -t "$UPSTART_JOB" "[`date -u +%Y-%m-%dT%T.%3NZ`] executing: $CMD" 16 | 17 | exec su -c "$CMD" 18 | end script 19 | 20 | pre-stop script 21 | logger -is -t "$UPSTART_JOB" "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Stopping" 22 | end script -------------------------------------------------------------------------------- /roles/cdh_hadoop_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_common/tasks/main.yml 3 | 4 | - name: install base Hadoop packages via apt 5 | apt: name={{ item }} update_cache=yes 6 | with_items: 7 | - hadoop 8 | - hadoop-hdfs 9 | - hadoop-lzo-cdh4 10 | - hadoop-yarn 11 | tags: hadoop 12 | 13 | - name: create the /data directory 14 | file: path=/data owner=hdfs group=hdfs state=directory 15 | tags: hadoop 16 | 17 | - name: create the /data/dfs directory 18 | file: path=/data/dfs owner=hdfs group=hdfs state=directory 19 | tags: hadoop -------------------------------------------------------------------------------- /roles/cdh_hadoop_datanode/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_datanode/tasks/main.yml 3 | 4 | - name: create the /data/dfs/dn directory 5 | file: path=/data/dfs/dn owner=hdfs group=hdfs state=directory 6 | tags: hadoop 7 | 8 | - name: install hadoop-hdfs-datanode via apt 9 | apt: name=hadoop-hdfs-datanode 10 | tags: hadoop 11 | 12 | - name: configure rsyslog for hadoop-hdfs-datanode 13 | template: src=rsyslog.conf dest=/etc/rsyslog.d/60-hadoop-hdfs-datanode.conf owner=root group=root mode=0644 14 | tags: rsyslog 15 | notify: 16 | - restart rsyslog -------------------------------------------------------------------------------- /roles/oracle_jdk/files/install_debian_webupd8team_repo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # see http://docs.ansible.com/script_module.html 4 | 5 | grep -Fqx "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" /etc/apt/sources.list 6 | 7 | if [ $? != 0 ]; then 8 | echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list 9 | echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list 10 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 11 | apt-get update 12 | fi -------------------------------------------------------------------------------- /roles/common/templates/ntp.conf: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | driftfile /var/lib/ntp/ntp.drift 4 | 5 | server {{ ntp_server1 }} 6 | server {{ ntp_server2 }} 7 | server {{ ntp_server3 }} 8 | server {{ ntp_server4 }} 9 | server {{ ntp_server5 }} 10 | server {{ ntp_server6 }} 11 | server {{ ntp_server7 }} 12 | 13 | # By default, exchange time with everybody, but don't allow configuration. 14 | restrict -4 default kod notrap nomodify nopeer noquery 15 | restrict -6 default kod notrap nomodify nopeer noquery 16 | 17 | # Local users may interrogate the ntp server more closely. 18 | restrict 127.0.0.1 19 | restrict ::1 20 | -------------------------------------------------------------------------------- /roles/elasticsearch_curator/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/elasticsearch_curator/tasks/main.yml 3 | 4 | - name: install python-pip via apt 5 | apt: pkg=python-pip 6 | tags: elasticsearch_curator 7 | 8 | - name: install elasticsearch-curator via pip 9 | command: pip install --upgrade https://github.com/elasticsearch/curator/tarball/master 10 | tags: elasticsearch_curator 11 | 12 | - name: install the elasticsearch-curator crontab at /etc/cron.d/elasticsearch-curator 13 | template: src=elasticsearch-curator.cron.d dest=/etc/cron.d/elasticsearch-curator owner=root group=root mode=644 14 | tags: elasticsearch_curator -------------------------------------------------------------------------------- /screenshot.js: -------------------------------------------------------------------------------- 1 | var args = require('system').args; 2 | 3 | if (args.length === 3) { 4 | var url = args[1]; 5 | var filename = args[2]; 6 | 7 | var page = require('webpage').create(); 8 | page.viewportSize = { width: 1280, height: 800 }; 9 | page.open(url, function () { 10 | page.onConsoleMessage = function (msg) { 11 | console.log(msg); 12 | }; 13 | 14 | window.setTimeout(function () { 15 | page.render(filename); 16 | phantom.exit(); 17 | }, 200); 18 | }); 19 | } 20 | else { 21 | console.log('usage: phantomjs screenshot.js url filename'); 22 | } -------------------------------------------------------------------------------- /roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Using the value 1 in order to avoid swapping and OOMs, since servers should have plenty of memory. 4 | kernel_swappiness: 1 5 | 6 | # Open file limits. 7 | limits_nofile_hard: 65536 8 | limits_nofile_soft: 65536 9 | 10 | # Configure which NTP servers are used to synchronize time. 11 | ntp_server1: ntp0.ox.ac.uk 12 | ntp_server2: ntp1.ox.ac.uk 13 | ntp_server3: ntp2.ox.ac.uk 14 | ntp_server4: ntp3.ox.ac.uk 15 | ntp_server5: 0.uk.pool.ntp.org 16 | ntp_server6: 1.uk.pool.ntp.org 17 | ntp_server7: ntp.ubuntu.com # fallback 18 | 19 | # Configure rsyslog polling. 20 | rsyslog_polling_interval_secs: 10 -------------------------------------------------------------------------------- /roles/mysql_server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/mysql_server/tasks/main.yml 3 | 4 | - name: predefine MySQL root passwd 5 | shell: executable=/bin/bash debconf-set-selections <<< "mysql-server-5.5 mysql-server/root_password password hive_{{ site_name }}" 6 | tags: mysql 7 | 8 | - name: predefine MySQL root passwd confirm 9 | shell: executable=/bin/bash debconf-set-selections <<< "mysql-server-5.5 mysql-server/root_password_again password hive_{{ site_name }}" 10 | tags: mysql 11 | 12 | - name: install MySQL via apt 13 | apt: name={{ item }} 14 | with_items: 15 | - mysql-server-5.5 16 | - libmysql-java 17 | - python-mysqldb 18 | tags: mysql -------------------------------------------------------------------------------- /roles/postgresql_server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/postgresql_server/tasks/main.yml 3 | 4 | - name: add PostgreSQL repository 5 | apt_repository: repo='deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main' 6 | tags: postgres 7 | 8 | - name: add PostgreSQL key 9 | apt_key: url=http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc 10 | tags: postgres 11 | 12 | - name: install PostgreSQL via apt 13 | apt: name={{ item }} force=yes 14 | with_items: 15 | - postgresql-9.1 16 | - python-psycopg2 17 | - libpostgresql-jdbc-java 18 | tags: postgres 19 | 20 | - name: install PostgreSQL contrib via apt 21 | apt: name=postgresql-contrib-9.1 force=yes 22 | tags: postgres -------------------------------------------------------------------------------- /roles/2_aggregated_links/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/2_aggregated_links/tasks/main.yml 3 | 4 | - name: configure /etc/network/interfaces with a bonding configuration 5 | template: backup=yes src=interfaces dest=/etc/network/interfaces 6 | tags: bonding 7 | 8 | - name: reboot after bonding the interfaces 9 | shell: sleep 2s && /sbin/reboot & 10 | tags: bonding 11 | 12 | - name: wait for the server to go down (reboot) 13 | sudo: false 14 | local_action: wait_for host={{ inventory_hostname }} port=22 state=stopped 15 | tags: bonding 16 | 17 | - name: wait for the server to come up 18 | sudo: false 19 | local_action: wait_for host={{ inventory_hostname }} port=22 delay=30 20 | tags: bonding -------------------------------------------------------------------------------- /roles/cdh_hadoop_journalnode/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_journalnode/tasks/main.yml 3 | # see http://touk.pl/blog/en/2012/10/30/hadoop-ha-setup/ and 4 | # http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/HDFSHighAvailabilityWithQJM.html 5 | 6 | - name: create the /data/dfs/jn directory 7 | file: path=/data/dfs/jn owner=hdfs group=hdfs state=directory 8 | tags: hadoop 9 | 10 | - name: install hadoop-hdfs-journalnode via apt 11 | apt: name=hadoop-hdfs-journalnode 12 | tags: hadoop 13 | 14 | - name: configure rsyslog for hadoop-hdfs-journalnode 15 | template: src=rsyslog.conf dest=/etc/rsyslog.d/60-hadoop-hdfs-journalnode.conf owner=root group=root mode=0644 16 | tags: rsyslog 17 | notify: 18 | - restart rsyslog -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/elasticsearch/tasks/main.yml 3 | 4 | - name: install Elasticsearch 5 | shell: creates=/usr/share/elasticsearch/bin/elasticsearch chdir=/tmp wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.0.deb && dpkg -i elasticsearch-1.1.0.deb 6 | tags: elasticsearch 7 | 8 | - name: make sure Elasticsearch is enabled and started 9 | service: name=elasticsearch enabled=yes state=started 10 | tags: elasticsearch 11 | 12 | - name: install Elasticsearch marvel plugin 13 | command: creates=/usr/share/elasticsearch/plugins/marvel chdir=/usr/share/elasticsearch/bin ./plugin -install elasticsearch/marvel/latest 14 | tags: elasticsearch 15 | notify: 16 | - restart elasticsearch -------------------------------------------------------------------------------- /roles/smokeping/templates/General: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | *** General *** 4 | 5 | owner = {{ site_name|capitalize }} 6 | {% if notify_email is defined %} 7 | contact = {{ notify_email }} 8 | {% else %} 9 | contact = root@localhost 10 | {% endif %} 11 | 12 | # NOTE: do not put the Image Cache below cgi-bin 13 | # since all files under cgi-bin will be executed ... this is not 14 | # good for images. 15 | cgiurl = http://{{ hostvars[groups["monitors"][0]]["ansible_fqdn"] }}/cgi-bin/smokeping.cgi 16 | 17 | # specify this to get syslog logging 18 | syslogfacility = local0 19 | 20 | # each probe is now run in its own process 21 | # disable this to revert to the old behaviour 22 | # concurrentprobes = no 23 | 24 | @include /etc/smokeping/config.d/pathnames 25 | -------------------------------------------------------------------------------- /roles/cdh_hive_mysql_metastore/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hive_mysql_metastore/tasks/main.yml 3 | 4 | - name: create MySQL db 'metastore' 5 | mysql_db: name=metastore state=present 6 | login_user=root login_password=hive_{{ site_name }} 7 | tags: 8 | - mysql 9 | - hive 10 | 11 | - name: init Hive metastore schema 12 | shell: mysql -uroot -phive_{{ site_name }} metastore < /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.10.0.mysql.sql 13 | tags: 14 | - mysql 15 | - hive 16 | 17 | - name: add and grant MySQL user 'hiveuser' 18 | mysql_user: name=hiveuser host=localhost password=hive_{{ site_name }} priv=metastore.*:ALL state=present 19 | login_user=root login_password=hive_{{ site_name }} 20 | tags: 21 | - mysql 22 | - hive -------------------------------------------------------------------------------- /roles/td_agent/templates/td-agent.conf: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | #### 4 | ## Output descriptions: 5 | ## 6 | 7 | 8 | type syslog 9 | port 5140 10 | tag syslog 11 | 12 | 13 | ## built-in TCP input 14 | ## @see http://docs.fluentd.org/articles/in_forward 15 | 16 | type forward 17 | 18 | 19 | 20 | 21 | type elasticsearch 22 | logstash_format true 23 | {% if groups["elasticsearch"]|length == 1 %} 24 | host {{ hostvars[groups["elasticsearch"][0]].ipv4_address|default(hostvars[groups["elasticsearch"][0]].ansible_default_ipv4.address) }} 25 | port {{ elasticsearch_port }} 26 | {% else %} 27 | hosts {% for host in groups['elasticsearch'] %}{{ hostvars[host].ipv4_address|default(hostvars[host].ansible_default_ipv4.address) }}:{{ elasticsearch_port }}{% if not loop.last %},{% endif %}{% endfor %} 28 | {% endif %} 29 | 30 | 31 | -------------------------------------------------------------------------------- /roles/ganglia_monitor/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/ganglia_monitor/tasks/main.yml 3 | 4 | - name: install Ganglia Monitor and required packages via apt 5 | apt: name={{ item }} 6 | with_items: 7 | - sysstat 8 | - php5-cli 9 | - ganglia-monitor 10 | - hddtemp 11 | tags: ganglia 12 | 13 | - name: configure gmond.conf in /etc/ganglia/gmond.conf 14 | template: src=gmond.conf dest=/etc/ganglia/gmond.conf owner=root group=root mode=0644 15 | tags: ganglia 16 | notify: 17 | - restart ganglia-monitor 18 | 19 | - name: install the device-metrics script at /usr/local/sbin 20 | template: src=device-metrics.php dest=/usr/local/sbin/device-metrics.php owner=root group=root mode=0644 21 | tags: ganglia 22 | 23 | - name: install the device-metrics crontab at /etc/cron.d/device-metrics 24 | template: src=device-metrics.cron.d dest=/etc/cron.d/device-metrics owner=root group=root mode=644 25 | tags: ganglia -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | fs.defaultFS 10 | hdfs://{{ site_name|lower }} 11 | true 12 | 13 | 14 | 15 | io.file.buffer.size 16 | 131072 17 | true 18 | 19 | 20 | 21 | io.compression.codecs 22 | org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec 23 | true 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /roles/cdh_hbase_config/templates/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | hbase.rootdir 9 | hdfs://{{ site_name|lower }}/hbase 10 | 11 | 12 | 13 | hbase.cluster.distributed 14 | true 15 | 16 | 17 | 18 | hbase.zookeeper.quorum 19 | {% for host in groups['zookeepers'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}{% if not loop.last %},{% endif %}{% endfor %} 20 | 21 | 22 | 23 | 24 | zookeeper.session.timeout 25 | 180000 26 | 27 | 28 | -------------------------------------------------------------------------------- /roles/cdh_hive_postgresql_metastore/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/postgresql_metastore/tasks/main.yml 3 | 4 | - name: create PostgreSQL db 'metastore' 5 | sudo: true 6 | sudo_user: postgres 7 | postgresql_db: name=metastore 8 | tags: 9 | - postgres 10 | - hive 11 | 12 | - name: add PostgreSQL user 'hiveuser' 13 | sudo: true 14 | sudo_user: postgres 15 | postgresql_user: db=metastore name=hiveuser password=hive_{{ site_name }} priv=CONNECT 16 | tags: 17 | - postgres 18 | - hive 19 | 20 | - name: init Hive metastore schema 21 | sudo: true 22 | sudo_user: postgres 23 | shell: psql --dbname=metastore --file=/usr/lib/hive/scripts/metastore/upgrade/postgres/hive-schema-0.10.0.postgres.sql 24 | tags: 25 | - postgres 26 | - hive 27 | 28 | - name: grant PostgreSQL user 'hiveuser' 29 | sudo: true 30 | sudo_user: postgres 31 | shell: psql --dbname=metastore --command="GRANT ALL ON ALL TABLES IN SCHEMA public TO hiveuser;" 32 | tags: 33 | - postgres 34 | - hive 35 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_config/handlers/main.yml 3 | 4 | - name: restart hadoop-hdfs-namenode 5 | service: name=hadoop-hdfs-namenode state=restarted 6 | ignore_errors: yes 7 | 8 | - name: restart hadoop-hdfs-datanode 9 | service: name=hadoop-hdfs-datanode state=restarted 10 | ignore_errors: yes 11 | 12 | - name: restart hadoop-hdfs-journalnode 13 | service: name=hadoop-hdfs-journalnode state=restarted 14 | ignore_errors: yes 15 | 16 | - name: restart hadoop-mapreduce-historyserver 17 | service: name=hadoop-mapreduce-historyserver state=restarted 18 | ignore_errors: yes 19 | 20 | - name: restart hadoop-yarn-nodemanager 21 | service: name=hadoop-yarn-nodemanager state=restarted 22 | ignore_errors: yes 23 | 24 | - name: restart hadoop-yarn-resourcemanager 25 | service: name=hadoop-yarn-resourcemanager state=restarted 26 | ignore_errors: yes 27 | 28 | - name: refresh datanodes 29 | sudo_user: hdfs 30 | command: hdfs dfsadmin -refreshNodes 31 | ignore_errors: yes -------------------------------------------------------------------------------- /roles/smokeping/templates/Targets: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | *** Targets *** 4 | 5 | # default probe 6 | probe = FPing 7 | 8 | menu = Top 9 | title = Network Latency Grapher 10 | remark = SmokePing for {{ site_name|capitalize }}. 11 | 12 | + {{ site_name|capitalize }} 13 | menu = {{ site_name|capitalize }} 14 | title = {{ site_name|capitalize }} 15 | 16 | {% for group in groups|sort if group != 'all' and not 'aggreg' in group and group != 'ungrouped' %} 17 | ++ {{ group }} 18 | menu = {{ group|replace("_", " ")|title }} 19 | title = {{ group|replace("_", " ")|title }} 20 | 21 | {% for host in groups[group] %} 22 | +++ {{ hostvars[host]['ansible_hostname'] }} 23 | menu = {{ hostvars[host]['ansible_hostname'] }} 24 | title = {{ group|replace("_", " ")|title }} {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host].ipv4_address|default(hostvars[host].ansible_default_ipv4.address) }} 25 | host = {{ hostvars[host]['ansible_fqdn'] }} 26 | alerts = bigloss,someloss,startloss,rttdetect,hostdown 27 | {% endfor %} 28 | 29 | {% endfor %} -------------------------------------------------------------------------------- /roles/smokeping/templates/Alerts: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | *** Alerts *** 4 | {% if notify_email is defined %} 5 | to = {{ notify_email }} 6 | {% else %} 7 | to = root@localhost 8 | {% endif %} 9 | 10 | from = smokealert@{{ site_name|lower }}.com 11 | 12 | +bigloss 13 | type = loss 14 | # in percent 15 | pattern = ==0%,==0%,==0%,==0%,>0%,>0%,>0% 16 | comment = suddenly there is packet loss 17 | 18 | +someloss 19 | type = loss 20 | # in percent 21 | pattern = >0%,*12*,>0%,*12*,>0% 22 | comment = loss 3 times in a row 23 | 24 | +startloss 25 | type = loss 26 | # in percent 27 | pattern = ==S,>0%,>0%,>0% 28 | comment = loss at startup 29 | 30 | +rttdetect 31 | type = rtt 32 | # in milliseconds 33 | pattern = <10,<10,<10,<10,<10,<100,>100,>100,>100 34 | comment = routing messed up again? 35 | 36 | +hostdown 37 | type = loss 38 | # in percent 39 | pattern = ==0%,==0%,==0%, ==U 40 | comment = no reply 41 | 42 | +lossdetect 43 | type = loss 44 | # in percent 45 | pattern = ==0%,==0%,==0%,==0%,>20%,>20%,>20% 46 | comment = suddenly there is packet loss -------------------------------------------------------------------------------- /roles/cdh_hbase_config/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hbase_config/tasks/main.yml 3 | 4 | - name: copy /etc/hbase/conf.empty to /etc/hbase/conf.{{ site_name|lower }} 5 | command: creates=/etc/hbase/conf.{{ site_name|lower }} cp -R -p /etc/hbase/conf.dist /etc/hbase/conf.{{ site_name|lower }} 6 | tags: hbase 7 | 8 | - name: configure HBase in /etc/hbase/conf.{{ site_name|lower }} 9 | template: src={{ item }} dest=/etc/hbase/conf.{{ site_name|lower }}/{{ item }} owner=root group=root mode=0644 10 | with_items: 11 | - hbase-env.sh 12 | - hbase-site.xml 13 | - regionservers 14 | notify: 15 | - restart hbase-master 16 | - restart hbase-regionserver 17 | tags: 18 | - hbase 19 | - configuration 20 | 21 | - name: run 'update-alternatives' to install HBase configuration 22 | command: update-alternatives --install /etc/hbase/conf hbase-conf /etc/hbase/conf.{{ site_name|lower }} 50 23 | tags: hbase 24 | 25 | - name: run 'update-alternatives' to set HBase configuration 26 | command: update-alternatives --set hbase-conf /etc/hbase/conf.{{ site_name|lower }} 27 | tags: hbase -------------------------------------------------------------------------------- /roles/smokeping/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/smokeping/tasks/main.yml 3 | 4 | - name: install smokeping via apt 5 | apt: pkg=smokeping 6 | tags: smokeping 7 | 8 | - name: configure smokeping in /etc/smokeping/config.d 9 | template: src={{ item }} dest=/etc/smokeping/config.d/{{ item }} owner=root group=root mode=644 10 | with_items: 11 | - Alerts 12 | - General 13 | - Targets 14 | tags: smokeping 15 | notify: 16 | - restart smokeping 17 | 18 | - name: link to smokeping config in /etc/apache2/conf-available 19 | command: creates=/etc/apache2/conf-available/smokeping.conf chdir=/etc/apache2/conf-available ln -s ../../smokeping/apache2.conf smokeping.conf 20 | when: ansible_distribution_version not in ["12.04", "12.10", "13.04"] 21 | tags: smokeping 22 | 23 | - name: enable the smokeping apache2 configuration 24 | command: a2enconf smokeping 25 | when: ansible_distribution_version not in ["12.04", "12.10", "13.04"] 26 | tags: smokeping 27 | 28 | - name: enable the cgid apache2 module 29 | command: a2enmod cgid 30 | tags: smokeping 31 | notify: 32 | - reload apache config -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/org-xerial-snappy.properties: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one or more contributor license 2 | # agreements. See the NOTICE file distributed with this work for 3 | # additional information regarding copyright ownership. Cloudera, 4 | # Inc. licenses this file to you under the Apache License, Version 5 | # 2.0 (the "License"); you may not use this file except in compliance 6 | # with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # Copyright (c) 2011 Cloudera, inc. 17 | 18 | # Disables loading Snappy-Java native library bundled in the 19 | # snappy-java-*.jar file forcing to load the Snappy-Java native 20 | # library from the java.library.path. 21 | org.xerial.snappy.disable.bundled.libs=true 22 | 23 | -------------------------------------------------------------------------------- /roles/ganglia_web/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/ganglia_web/tasks/main.yml 3 | 4 | - name: install Ganglia web dependencies via apt 5 | apt: name={{ item }} 6 | with_items: 7 | - apache2 8 | - php5 9 | - rrdtool 10 | tags: ganglia 11 | 12 | - name: install ganglia-web at /var/www/ganglia 13 | shell: creates=/var/www/ganglia chdir=/var/www curl http://heanet.dl.sourceforge.net/project/ganglia/ganglia-web/3.5.12/ganglia-web-3.5.12.tar.gz | tar xz && mv ganglia-web-3.5.12 ganglia 14 | tags: ganglia 15 | 16 | - name: make sure ganglia-web makefile has the correct path 17 | command: creates=/var/lib/ganglia-web/dwoo chdir=/var/www/ganglia sed -i "s=/var/www/html/ganglia=/var/www/ganglia=" Makefile 18 | tags: ganglia 19 | 20 | - name: make sure ganglia-web makefile has the correct user 21 | command: creates=/var/lib/ganglia-web/dwoo chdir=/var/www/ganglia sed -i "s/APACHE_USER = apache/APACHE_USER = www-data/" Makefile 22 | tags: ganglia 23 | 24 | - name: make install ganglia-web 25 | command: creates=/var/lib/ganglia-web/dwoo chdir=/var/www/ganglia make install 26 | notify: 27 | - restart apache2 28 | tags: ganglia 29 | -------------------------------------------------------------------------------- /roles/kernel/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/kernel/tasks/main.yml 3 | 4 | - name: download the latest kernel debian packages 5 | get_url: url={{ item }} dest=/tmp 6 | with_items: 7 | - http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.13.7-trusty/linux-headers-3.13.7-031307-generic_3.13.7-031307.201403240156_amd64.deb 8 | - http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.13.7-trusty/linux-image-3.13.7-031307-generic_3.13.7-031307.201403240156_amd64.deb 9 | - http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.13.7-trusty/linux-headers-3.13.7-031307_3.13.7-031307.201403240156_all.deb 10 | tags: kernel 11 | 12 | - name: install the latest kernel 13 | shell: chdir=/tmp dpkg -i linux-*.deb 14 | tags: kernel 15 | 16 | - name: reboot after kernel upgrade 17 | shell: sleep 2s && /sbin/reboot & 18 | tags: kernel 19 | 20 | - name: wait for the server to go down (reboot) 21 | sudo: false 22 | local_action: wait_for host={{ inventory_hostname }} port=22 state=stopped 23 | tags: kernel 24 | 25 | - name: wait for the server to come up 26 | sudo: false 27 | local_action: wait_for host={{ inventory_hostname }} port=22 delay=30 28 | tags: kernel -------------------------------------------------------------------------------- /roles/cdh_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_common/tasks/main.yml 3 | 4 | - name: install Google Snappy 5 | shell: creates=/usr/local/lib/libsnappy.so.1.2.0 chdir=/tmp curl https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz | tar xz && cd snappy-1.1.1 && CFLAGS="-funroll-loops -O2 -DNDEBUG" CXXFLAGS="-funroll-loops -O2 -DNDEBUG" ./configure && make && checkinstall -pkgname=snappy-1.1.1 --backup=no --deldoc=yes --fstrans=no --default 6 | notify: 7 | - ldconfig 8 | tags: hadoop 9 | 10 | - name: add Cloudera CDH4 repository 11 | apt_repository: repo='deb [arch=amd64] http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh precise-cdh4 contrib' 12 | tags: 13 | - hadoop 14 | - hbase 15 | - hive 16 | 17 | - name: add Cloudera CDH4 GPL extras repository 18 | apt_repository: repo='deb [arch=amd64] http://archive.cloudera.com/gplextras/ubuntu/precise/amd64/gplextras precise-gplextras4 contrib' 19 | tags: 20 | - hadoop 21 | - hbase 22 | - hive 23 | 24 | - name: add Cloudera CDH4 key 25 | apt_key: url=http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh/archive.key 26 | tags: 27 | - hadoop 28 | - hbase 29 | - hive -------------------------------------------------------------------------------- /roles/postfix_mandrill/templates/disk-usage-alert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set -x 3 | 4 | # {{ ansible_managed }} 5 | 6 | # Shell script to monitor or watch the disk space 7 | # It will send an email to root when the (free available) percentage of space is >= 80%. 8 | # ------------------------------------------------------------------------- 9 | 10 | USE_ALERT_PCT=80 11 | 12 | # Exclude list of unwanted monitoring, if several partions then use "|" to separate the partitions. 13 | # An example: EXCLUDE_LIST="/dev/sda1|/dev/sda5" 14 | EXCLUDE_LIST="" 15 | 16 | function do_mail() { 17 | while read output; 18 | do 19 | usep=$(echo $output | awk '{ print $1}' | cut -d'%' -f1) 20 | partition=$(echo $output | awk '{print $2}') 21 | if [ $usep -ge $USE_ALERT_PCT ] ; then 22 | echo "Running out of disk space: $partition is $usep% used on server $(hostname)." | \ 23 | mail -s "WARNING: $(hostname) running out of disk space: $usep% used" root 24 | fi 25 | done 26 | } 27 | 28 | if [ "$EXCLUDE_LIST" != "" ] ; then 29 | df -H | grep -vE "^Filesystem|tmpfs|cdrom|${EXCLUDE_LIST}" | awk '{print $5 " " $6}' | do_mail 30 | else 31 | df -H | grep -vE "^Filesystem|tmpfs|cdrom" | awk '{print $5 " " $6}' | do_mail 32 | fi -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/configuration.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 21 | 24 | 27 | 28 | 29 |
namevaluedescription
17 | 18 | 19 | 20 | 22 | 23 | 25 | 26 |
30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /roles/oracle_jdk/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/oracle_jdk/tasks/main.yml 3 | 4 | - name: add webupd8team java repository for Ubuntu 5 | apt_repository: repo='ppa:webupd8team/java' 6 | when: ansible_distribution == "Ubuntu" 7 | tags: java 8 | 9 | - name: add webupd8team java repository for Debian "wheezy" 10 | script: install_debian_webupd8team_repo.sh 11 | when: 12 | - ansible_distribution == "Debian" 13 | - ansible_distribution_release == "wheezy" 14 | tags: java 15 | 16 | - name: autoaccept license for java 17 | shell: executable=/bin/bash debconf-set-selections <<< "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" 18 | tags: java 19 | 20 | - name: install or update Oracle Java 7 JDK via apt 21 | apt: pkg={{ item }} update_cache=yes state=latest install_recommends=yes 22 | with_items: 23 | - oracle-java7-installer 24 | - oracle-java7-set-default 25 | tags: java 26 | 27 | - name: download floatingdecimal-0.1.jar to /usr/lib/jvm (fixes monitor contention when parsing doubles due to a static synchronized method) 28 | command: creates=/usr/lib/jvm/floatingdecimal-0.1.jar chdir=/usr/lib/jvm curl -O http://repo1.maven.org/maven2/io/airlift/floatingdecimal/0.1/floatingdecimal-0.1.jar 29 | tags: java 30 | 31 | - name: set fact jdk_installed 32 | set_fact: jdk_installed=true 33 | tags: java -------------------------------------------------------------------------------- /bootstrap/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Ansible Bootstrap 3 | # ================= 4 | 5 | # Found on the ansible mailing list and adopted 6 | 7 | # Creates the 'ansibler' user and adds him to the 'sudo' group. Then concatenates ansible_rsa.pub to 8 | # the ~/.ssh/authorized_keys on the target instance with group 'sudo'. Then make sure that sudoers is set to allow 9 | # NOPASSWD from sudo, and run everything else with ansible with 'user: ansibler' and sudo:true. This way, all ansible 10 | # activity is logged, and we can specifically revoke the key (and terminate ansible's control) if either 11 | # (a) security is compromised or (b) we hand over control of an instance to a client. 12 | 13 | - hosts: all 14 | sudo: true 15 | vars: 16 | - authorized_rsa_key: "{{ lookup('file', 'ansible_rsa.pub') }}" 17 | tasks: 18 | - name: hostname should match inventory name 19 | hostname: name={{ inventory_hostname }} 20 | 21 | - name: create user 'ansibler' 22 | user: name=ansibler groups=sudo shell=/bin/bash 23 | 24 | - name: add 'ansibler' RSA SSH key 25 | authorized_key: user=ansibler key="{{ authorized_rsa_key }}" 26 | 27 | - name: change sudoers to contains NOPASSWD for sudo group 28 | shell: "creates=/etc/sudoers.bak chdir=/etc cp sudoers sudoers.bak && sed -ri -e 's/(%sudo\\s+ALL=\\(ALL:ALL\\))\\s+ALL/\\1 NOPASSWD: ALL/' /etc/sudoers" -------------------------------------------------------------------------------- /hosts: -------------------------------------------------------------------------------- 1 | # Hadoop Nodes 2 | # ============ 3 | 4 | [zookeepers] # minimal 3, must be odd number of hosts 5 | hmaster01 zookeeper_id=1 6 | hmaster02 zookeeper_id=2 7 | hslave01 zookeeper_id=3 8 | 9 | [namenodes] 10 | hmaster01 ganglia_accept=true 11 | hmaster02 ganglia_accept=true 12 | 13 | [journalnodes] # minimal 3, must be odd number of hosts 14 | hmaster01 15 | hmaster02 16 | hslave01 17 | 18 | [resourcemanager] 19 | hmaster01 20 | 21 | [datanodes] 22 | hslave[01:04] 23 | 24 | [nodemanagers:children] 25 | datanodes 26 | 27 | [historyserver] 28 | hmaster01 29 | 30 | # HBase Nodes 31 | # =========== 32 | 33 | [hbase_masters:children] 34 | namenodes 35 | 36 | [regionservers:children] 37 | datanodes 38 | 39 | [elasticsearch] 40 | monitor01 41 | 42 | [monitors] 43 | monitor01 44 | 45 | # Hive Nodes 46 | # ========== 47 | 48 | [hive_metastore] 49 | hmaster02 50 | 51 | # Presto Nodes 52 | # ============ 53 | 54 | [presto_coordinators] 55 | hmaster02 56 | 57 | [presto_workers] 58 | hslave03 59 | hslave04 60 | 61 | # Groups 62 | # ====== 63 | 64 | [masters:children] 65 | namenodes 66 | resourcemanager 67 | hbase_masters 68 | 69 | [slaves:children] 70 | zookeepers 71 | datanodes 72 | nodemanagers 73 | regionservers 74 | 75 | [2_link_aggregation_balance_alb:children] 76 | zookeepers 77 | namenodes 78 | journalnodes 79 | datanodes 80 | 81 | [2_links_aggregated:children] 82 | 2_link_aggregation_balance_alb -------------------------------------------------------------------------------- /roles/cdh_hive_config/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hive_config/tasks/main.yml 3 | 4 | - name: copy /etc/hive/conf.empty to /etc/hive/conf.{{ site_name|lower }} 5 | command: creates=/etc/hive/conf.{{ site_name|lower }} cp -R -p /etc/hive/conf.dist /etc/hive/conf.{{ site_name|lower }} 6 | tags: hive 7 | 8 | - name: configure hive in /etc/hive/conf.{{ site_name|lower }} 9 | template: src={{ item }} dest=/etc/hive/conf.{{ site_name|lower }}/{{ item }} owner=root group=root mode=0644 10 | with_items: 11 | - hive-site.xml 12 | notify: 13 | - restart hive-metastore 14 | tags: 15 | - hive 16 | - configuration 17 | 18 | - name: run 'update-alternatives' to install hive configuration 19 | command: update-alternatives --install /etc/hive/conf hive-conf /etc/hive/conf.{{ site_name|lower }} 50 20 | tags: hive 21 | 22 | - name: run 'update-alternatives' to set hive configuration 23 | command: update-alternatives --set hive-conf /etc/hive/conf.{{ site_name|lower }} 24 | tags: hive 25 | 26 | - name: soft link postgresql-jdbc4.jar 27 | command: creates=/usr/lib/hive/lib/postgresql-jdbc4.jar ln -s /usr/share/java/postgresql-jdbc4.jar /usr/lib/hive/lib/postgresql-jdbc4.jar 28 | when: hive_metastore == "postgresql" 29 | tags: hive 30 | 31 | - name: symbolically link libmysql-java.jar 32 | command: creates=/usr/lib/hive/lib/libmysql-java.jar 33 | ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/libmysql-java.jar 34 | when: hive_metastore == "mysql" 35 | tags: hive -------------------------------------------------------------------------------- /roles/cdh_zookeeper_server/templates/zoo.cfg: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | # see http://zookeeper.apache.org/doc/current/zookeeperAdmin.html 3 | 4 | # The number of milliseconds of each tick 5 | tickTime=5000 6 | 7 | # The number of ticks that the initial 8 | # synchronization phase can take 9 | initLimit=10 10 | 11 | # The number of ticks that can pass between 12 | # sending a request and getting an acknowledgement 13 | syncLimit=5 14 | 15 | # the directory where the snapshot is stored. 16 | dataDir=/var/lib/zookeeper 17 | 18 | # the port at which the clients will connect 19 | clientPort=2181 20 | 21 | # Be sure to read the maintenance section of the 22 | # administrator guide before turning on autopurge. 23 | # 24 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 25 | # 26 | # The number of snapshots to retain in dataDir 27 | #autopurge.snapRetainCount=3 28 | # Purge task interval in hours 29 | # Set to "0" to disable auto purge feature 30 | 31 | # Specify all zookeeper servers 32 | # The fist port is used by followers to connect to the leader 33 | # The second one is used for leader election 34 | 35 | {% for host in groups['zookeepers'] %} 36 | server.{{ hostvars[host].zookeeper_id }}={{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:2888:3888 37 | {% endfor %} 38 | 39 | # Allow larger than default maximum client connections. 40 | maxClientCnxns=200 41 | 42 | # ZooKeeper and HBase need same session timeout values. 43 | maxSessionTimeout=180000 44 | -------------------------------------------------------------------------------- /roles/cdh_zookeeper_server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_zookeeper_server/tasks/main.yml 3 | 4 | - name: install zookeeper-server via apt 5 | apt: name={{ item }} force=yes update-cache=yes 6 | with_items: 7 | - zookeeper 8 | - zookeeper-server 9 | tags: zookeeper 10 | 11 | - name: copy /etc/zookeeper/conf.dist to /etc/zookeeper/conf.{{ site_name|lower }} 12 | command: creates=/etc/zookeeper/conf.{{ site_name|lower }} cp -R -p /etc/zookeeper/conf.dist /etc/zookeeper/conf.{{ site_name|lower }} 13 | tags: zookeeper 14 | 15 | - name: run 'update-alternatives' to install ZooKeeper configuration 16 | command: update-alternatives --install /etc/zookeeper/conf zookeeper-conf /etc/zookeeper/conf.{{ site_name|lower }} 50 17 | tags: zookeeper 18 | 19 | - name: run 'update-alternatives' to set ZooKeeper configuration 20 | command: update-alternatives --set zookeeper-conf /etc/zookeeper/conf.{{ site_name|lower }} 21 | tags: zookeeper 22 | 23 | - name: create /var/lib/zookeeper directory 24 | file: path=/var/lib/zookeeper owner=zookeeper group=zookeeper state=directory 25 | tags: zookeeper 26 | 27 | - name: configure ZooKeeper in /etc/zookeeper/conf.{{ site_name|lower }} 28 | template: src=zoo.cfg dest=/etc/zookeeper/conf.{{ site_name|lower }}/zoo.cfg owner=zookeeper group=zookeeper mode=0644 29 | notify: 30 | - restart zookeeper 31 | tags: zookeeper 32 | 33 | - name: initialize ZooKeeper 34 | command: creates=/var/lib/zookeeper/myid service zookeeper-server init --myid={{ zookeeper_id }} 35 | tags: zookeeper 36 | -------------------------------------------------------------------------------- /roles/2_aggregated_links/templates/interfaces: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # This file describes the network interfaces available on your system 4 | # and how to activate them. For more information, see interfaces(5). 5 | 6 | # The loopback network interface 7 | auto lo 8 | iface lo inet loopback 9 | 10 | {% for interface in ansible_interfaces|sort if not interface.startswith('lo') %} 11 | auto {{ interface }} 12 | iface {{ interface }} inet manual 13 | hardware-dma-ring-rx 512 14 | hardware-dma-ring-tx 512 15 | {% if loop.index > 1 %} 16 | pre-up sleep 2 17 | {% endif %} 18 | bond-master bond0 19 | {% endfor %} 20 | 21 | # The primary (bonded) network interface 22 | auto bond0 23 | iface bond0 inet dhcp 24 | post-up /sbin/ifconfig bond0 mtu {{ mtu }} 25 | bond-mode {{ bond_mode }} 26 | bond-slaves none 27 | 28 | # Indicates that the devices are polled every 100ms to check for connection changes, such as 29 | # a link being down or a link duplex having changed 30 | bond-miimon 100 31 | 32 | {% if bond_mode == 'lacp' %} 33 | # Sets the rate for transmitting LACPDU packets, 0 is once every 30 seconds, 1 is every 1 second, which allows our 34 | # network devices to automatically configure a single logical connection at the switch quickly 35 | bond-lacp-rate 1 36 | {% endif %} 37 | 38 | # Instructs the driver to spread the load over interfaces based on the source and destination 39 | # IP address instead of MAC address 40 | bond-xmit-hash-policy layer3+4 41 | 42 | # Google Public DNS - see https://developers.google.com/speed/public-dns/ 43 | dns-nameservers 8.8.8.8 8.8.4.4 44 | -------------------------------------------------------------------------------- /travis.ssh: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEA9Sdy7IhggwT6sHZw8gx4n+w57rCb0wsnbYOnE9ONeCfvPo7L 3 | IK8Us3TPHWu1fgoxbLA91/ClDNkq0jqFRCqu2SdShpxmqSipoltqGfDXcuct16Me 4 | 5j9ZC2KBRLp48uMznupH7YiP6ysrRAYxQm5KrVWv4IhfY+bvxninGQLBDB2tlKhX 5 | usOp/jDZKgmc2jwQV5SPvYF3Vl1b93e+K3R4dItuM1a/jI3pRV3rTi1Bhzosq9vQ 6 | 0OcHf1eRhwsmf+5wTUtBrM1Q1/F1m8B43RWdTmqrMLvqJGQkjdkq01yBdmDFiu7b 7 | qZkjWoFFWvcT2a7PBhAuuAGXlzUIf1CTvk80ZwIDAQABAoIBAGTEVE65N8Ji95A0 8 | /vPfAG3rsVxVJ5e1y/0N0k0p9cpfz80M8J18maux5bgYH7+w/OPKBYGPHqiLUzia 9 | aVC9oqULrrY571C0sBVWMeR7xHMsZwFtwTOu8jDlynCvnStpDDzT28DdExr84YnS 10 | acj+gJ+Cywolw9/Setg/m5+CLEuH0W9ic11SCnFsSxfIe9AAH7h/213AOhU/NvTz 11 | 96lbpPlnSriNLFsEeJZ341pH2N8rlzQaya3R2PmUMNE0w3vkAE75BBwl3eTKsSPM 12 | imKDrCGuHcTYxjrj9iWp+d3RHDc++G3jDmmEEmjE3zg3oWqd21Z3GNY4WXL9wH/J 13 | tFZQYRECgYEA/I2f7PNOiodMOzQWXJZJSU+yzja8K3mm3aBSNls3gEJ57AyJOiif 14 | D10yO8mCotquSCCyVSKtjaC+kgg3Pc7rLcFI3SWsIpZ3NEDgba8qH+8P6X2rFJD+ 15 | l+XjxThl3ffnLcCLWWxbwZBHh8OKWYiUZ7+6bps54sP6Qk1ICP7bWuUCgYEA+H/5 16 | FzWRhWIJKW41gwzPj5Ukmn2YKD+v27fhpu8ENm6or49SN3FEjkVMaqy4lCdYAEvQ 17 | J3Td+H86/NI7lVw6qY2wN4JXFUF9sjYoThd0GmRobXjX5jWxOVDP5rz52bdtL0i7 18 | AyfVb4E1gJHnm+dlMeHS6mO9UBirFs6NBFBfAVsCgYBai0vy36gg8jDaaOo7d4X+ 19 | ypwIYodD7vFsM7ZF1p4nVa3weu4X88hS8c64sJ4tVMl3AoHtE+m1BRRNtrRXbH/U 20 | f59n6cG1BMzn5vqPWGV53XOO+ocDcJhWTQ1D/9xFoEM5m/ZZsQeUde2vNuO7BcRv 21 | DcYBMZWnPyvR6XMSBo0OIQKBgA8/ItRWuA/7/N3q4iOhRJblqrJD5HMLvUl41cJ9 22 | Rb/lODpVS9EVBQTJoa+l20lTxKV4AVR6aigV/lOGvpsiZoMCq34sZBVsyD6H7h8k 23 | dz4rTPDyiiSYxL/kgm0AyT+fbnaJOtgtuH236sVxrkk1gFL4LQ2ugu4IJW8nyyUp 24 | nTqjAoGACiljS67rQR5+uTWgy1/g4dzZOLD6C+d1QMvgiFffoBIciU+ZapyzFPth 25 | p0Y82CoTMpaUvPvGaP3a32kA6qO9mJOwnagwCQkkMho/fKnx7Y9FlvXRiKzKPoHF 26 | /nM5h3MubDafGPejfrAsaTFa1H5UaeFIAhLLGEihA61JSTNaftQ= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/cdh_hadoop_config/tasks/main.yml 3 | 4 | - name: copy /etc/hadoop/conf.empty to /etc/hadoop/conf.{{ site_name|lower }} 5 | command: creates=/etc/hadoop/conf.{{ site_name|lower }} cp -R -p /etc/hadoop/conf.empty /etc/hadoop/conf.{{ site_name|lower }} 6 | tags: 7 | - hadoop 8 | - configuration 9 | 10 | - name: configure hadoop in /etc/hadoop/conf.{{ site_name|lower }} 11 | template: src={{ item }} dest=/etc/hadoop/conf.{{ site_name|lower }}/{{ item }} owner=root group=root mode=0644 12 | with_items: 13 | - core-site.xml 14 | - hadoop-env.sh 15 | - hadoop-metrics2.properties 16 | - hdfs-site.xml 17 | - org-xerial-snappy.properties 18 | - slaves 19 | - mapred-site.xml 20 | - yarn-site.xml 21 | - fair-scheduler.xml 22 | notify: 23 | - restart hadoop-hdfs-namenode 24 | - restart hadoop-hdfs-journalnode 25 | - restart hadoop-hdfs-datanode 26 | - restart hadoop-mapreduce-historyserver 27 | - restart hadoop-yarn-nodemanager 28 | - restart hadoop-yarn-resourcemanager 29 | tags: 30 | - hadoop 31 | - configuration 32 | 33 | - name: update excluded datanodes 34 | copy: src=dfs.hosts.exclude dest=/etc/hadoop/conf/dfs.hosts.exclude owner=root group=root mode=644 35 | notify: 36 | - refresh datanodes 37 | tags: 38 | - hadoop 39 | - configuration 40 | 41 | - name: run 'update-alternatives' to install hadoop configuration 42 | command: update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.{{ site_name|lower }} 50 43 | tags: hadoop 44 | 45 | - name: run 'update-alternatives' to set hadoop configuration 46 | command: update-alternatives --set hadoop-conf /etc/hadoop/conf.{{ site_name|lower }} 47 | tags: hadoop 48 | -------------------------------------------------------------------------------- /roles/presto_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/presto_common/tasks/main.yml 3 | 4 | - name: create the /data/presto directory 5 | file: path=/data/presto state=directory 6 | tags: presto 7 | 8 | - name: create the /usr/lib/presto/etc directory 9 | file: path=/usr/lib/presto/etc state=directory 10 | tags: presto 11 | 12 | - name: create the /usr/lib/presto/etc/catalog directory 13 | file: path=/usr/lib/presto/etc/catalog state=directory 14 | tags: presto 15 | 16 | - name: install Presto at /usr/lib/presto 17 | shell: creates=/usr/lib/presto/bin/launcher chdir=/tmp curl http://central.maven.org/maven2/com/facebook/presto/presto-server/0.65/presto-server-0.65.tar.gz | tar xz && mv presto-server-0.65/* /usr/lib/presto 18 | tags: presto 19 | 20 | - name: configure presto in /usr/lib/presto/etc 21 | template: src={{ item }} dest=/usr/lib/presto/etc/{{ item }} owner=root group=root mode=0644 22 | with_items: 23 | - node.properties 24 | - jvm.config 25 | tags: 26 | - configuration 27 | - presto 28 | 29 | - name: configure presto hive catalog in /usr/lib/presto/etc/catalog 30 | template: src={{ item }} dest=/usr/lib/presto/etc/catalog/{{ item }} owner=root group=root mode=0644 31 | with_items: 32 | - jmx.properties 33 | - hive.properties 34 | - tpch.properties 35 | tags: 36 | - configuration 37 | - presto 38 | 39 | - name: install presto command line client 40 | shell: chdir=/usr/bin curl -o presto http://central.maven.org/maven2/com/facebook/presto/presto-cli/0.65/presto-cli-0.65-executable.jar && chmod +x presto 41 | tags: presto 42 | 43 | - name: install upstart config for presto 44 | template: src=upstart.conf dest=/etc/init/presto.conf owner=root group=root mode=0644 45 | tags: 46 | - configuration 47 | - presto -------------------------------------------------------------------------------- /do_cluster.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: localhost 4 | gather_facts: False 5 | tasks: 6 | - name: create DigitalOcean SSH key 7 | digital_ocean: > 8 | command=ssh 9 | name=travisci 10 | ssh_pub_key='ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD1J3LsiGCDBPqwdnDyDHif7DnusJvTCydtg6cT0414J+8+jssgrxSzdM8da7V+CjFssD3X8KUM2SrSOoVEKq7ZJ1KGnGapKKmiW2oZ8Ndy5y3Xox7mP1kLYoFEunjy4zOe6kftiI/rKytEBjFCbkqtVa/giF9j5u/GeKcZAsEMHa2UqFe6w6n+MNkqCZzaPBBXlI+9gXdWXVv3d74rdHh0i24zVr+MjelFXetOLUGHOiyr29DQ5wd/V5GHCyZ/7nBNS0GszVDX8XWbwHjdFZ1Oaqswu+okZCSN2SrTXIF2YMWK7tupmSNagUVa9xPZrs8GEC64AZeXNQh/UJO+TzRn noreply@travis-ci.org' 11 | client_id={{ client_id }} 12 | api_key={{ api_key_password }} 13 | register: traviscikey 14 | 15 | - name: create DigitalOcean droplets to deploy cluster on (1 monitor + 2 master + 4 slave droplets) 16 | digital_ocean: > 17 | name={{ item }} 18 | ssh_key_ids="{{ traviscikey.ssh_key.id }}" 19 | client_id={{ client_id }} 20 | api_key={{ api_key_password }} 21 | size_id=62 22 | region_id=4 23 | image_id=3101918 24 | register: hosts 25 | with_items: 26 | - monitor01 27 | - hmaster01 28 | - hmaster02 29 | - hslave01 30 | - hslave02 31 | - hslave03 32 | - hslave04 33 | 34 | - name: add delete line to /etc/rc6.d/K10do_destroy.sh 35 | sudo: true 36 | lineinfile: create=yes dest=/etc/rc6.d/K10do_destroy.sh line="wget -qO- https://api.digitalocean.com/droplets/{{ item.droplet.id }}/destroy/\?client_id\={{ client_id }}\&api_key\={{ api_key_password }}" owner=root group=root mode=755 37 | with_items: hosts.results 38 | 39 | - name: add the newly created hosts to /etc/hosts 40 | sudo: true 41 | lineinfile: dest=/etc/hosts line="{{ item.droplet.ip_address }} {{ item.droplet.name }}" 42 | with_items: hosts.results -------------------------------------------------------------------------------- /roles/td_agent/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/td_agent/tasks/main.yml 3 | 4 | - name: add Treasure Data repository 5 | apt_repository: repo='deb http://packages.treasure-data.com/precise/ precise contrib' 6 | tags: tdagent 7 | 8 | - name: install libssl0.9.8 for Debian "wheezy" 9 | script: install_debian_libssl0.9.8.sh 10 | when: 11 | - ansible_distribution == "Debian" 12 | - ansible_distribution_release == "wheezy" 13 | tags: tdagent 14 | 15 | - name: install td-agent and libcurl via apt 16 | apt: pkg={{ item }} update_cache=yes force=yes 17 | with_items: 18 | - td-agent 19 | - libcurl3 20 | - libcurl3-gnutls 21 | - libcurl4-openssl-dev 22 | tags: tdagent 23 | 24 | - name: remove older fluent Elasticsearch plugin versions v0.1.* 25 | shell: rm -Rf /usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.1.* 26 | ignore_errors: yes 27 | tags: tdagent 28 | 29 | - name: remove older fluent Elasticsearch plugin versions v0.2.0 30 | command: removes=/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.2.0 rm -Rf /usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.2.0 31 | tags: tdagent 32 | 33 | - name: install the fluent Elasticsearch plugin 34 | command: creates=/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.3.0/Gemfile /usr/lib/fluent/ruby/bin/fluent-gem install fluent-plugin-elasticsearch 35 | tags: tdagent 36 | 37 | - name: install the fluent tail-multiline plugin 38 | command: creates=/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-tail-multiline-0.1.5/Gemfile /usr/lib/fluent/ruby/bin/fluent-gem install fluent-plugin-tail-multiline 39 | tags: tdagent 40 | 41 | - name: configure td-agent in /etc/td-agent/td-agent.conf 42 | template: src=td-agent.conf dest=/etc/td-agent/td-agent.conf owner=root group=root mode=0644 43 | tags: 44 | - tdagent 45 | - configuration 46 | notify: 47 | - restart td-agent -------------------------------------------------------------------------------- /roles/common/templates/limits.conf: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | # 3 | # /etc/security/limits.conf 4 | # 5 | #Each line describes a limit for a user in the form: 6 | # 7 | # 8 | # 9 | #Where: 10 | # can be: 11 | # - an user name 12 | # - a group name, with @group syntax 13 | # - the wildcard *, for default entry 14 | # - the wildcard %, can be also used with %group syntax, 15 | # for maxlogin limit 16 | # - NOTE: group and wildcard limits are not applied to root. 17 | # To apply a limit to the root user, must be 18 | # the literal username root. 19 | # 20 | # can have the two values: 21 | # - "soft" for enforcing the soft limits 22 | # - "hard" for enforcing hard limits 23 | # 24 | # can be one of the following: 25 | # - core - limits the core file size (KB) 26 | # - data - max data size (KB) 27 | # - fsize - maximum filesize (KB) 28 | # - memlock - max locked-in-memory address space (KB) 29 | # - nofile - max number of open files 30 | # - rss - max resident set size (KB) 31 | # - stack - max stack size (KB) 32 | # - cpu - max CPU time (MIN) 33 | # - nproc - max number of processes 34 | # - as - address space limit (KB) 35 | # - maxlogins - max number of logins for this user 36 | # - maxsyslogins - max number of logins on the system 37 | # - priority - the priority to run user process with 38 | # - locks - max number of file locks the user can hold 39 | # - sigpending - max number of pending signals 40 | # - msgqueue - max memory used by POSIX message queues (bytes) 41 | # - nice - max nice priority allowed to raise to values: [-20, 19] 42 | # - rtprio - max realtime priority 43 | # - chroot - change root to directory (Debian-specific) 44 | # 45 | # 46 | # 47 | 48 | * soft nofile {{ limits_nofile_soft }} 49 | * hard nofile {{ limits_nofile_hard }} 50 | 51 | * soft memlock unlimited 52 | * hard memlock unlimited 53 | 54 | # End of file 55 | -------------------------------------------------------------------------------- /roles/postfix_mandrill/templates/50unattended-upgrades: -------------------------------------------------------------------------------- 1 | // Automatically upgrade packages from these (origin:archive) pairs 2 | Unattended-Upgrade::Allowed-Origins { 3 | "${distro_id}:${distro_codename}-security"; 4 | // "${distro_id}:${distro_codename}-updates"; 5 | // "${distro_id}:${distro_codename}-proposed"; 6 | // "${distro_id}:${distro_codename}-backports"; 7 | }; 8 | 9 | // List of packages to not update 10 | Unattended-Upgrade::Package-Blacklist { 11 | // "vim"; 12 | // "libc6"; 13 | // "libc6-dev"; 14 | // "libc6-i686"; 15 | }; 16 | 17 | // This option allows you to control if on a unclean dpkg exit 18 | // unattended-upgrades will automatically run 19 | // dpkg --force-confold --configure -a 20 | // The default is true, to ensure updates keep getting installed 21 | //Unattended-Upgrade::AutoFixInterruptedDpkg "false"; 22 | 23 | // Split the upgrade into the smallest possible chunks so that 24 | // they can be interrupted with SIGUSR1. This makes the upgrade 25 | // a bit slower but it has the benefit that shutdown while a upgrade 26 | // is running is possible (with a small delay) 27 | //Unattended-Upgrade::MinimalSteps "true"; 28 | 29 | // Install all unattended-upgrades when the machine is shuting down 30 | // instead of doing it in the background while the machine is running 31 | // This will (obviously) make shutdown slower 32 | //Unattended-Upgrade::InstallOnShutdown "true"; 33 | 34 | // Send email to this address for problems or packages upgrades 35 | // If empty or unset then no email is sent, make sure that you 36 | // have a working mail setup on your system. A package that provides 37 | // 'mailx' must be installed. E.g. "user@example.com" 38 | Unattended-Upgrade::Mail "{{ notify_email }}"; 39 | 40 | // Set this value to "true" to get emails only on errors. Default 41 | // is to always send a mail if Unattended-Upgrade::Mail is set 42 | //Unattended-Upgrade::MailOnlyOnError "true"; 43 | 44 | // Do automatic removal of new unused dependencies after the upgrade 45 | // (equivalent to apt-get autoremove) 46 | Unattended-Upgrade::Remove-Unused-Dependencies "true"; 47 | 48 | // Automatically reboot *WITHOUT CONFIRMATION* if a 49 | // the file /var/run/reboot-required is found after the upgrade 50 | Unattended-Upgrade::Automatic-Reboot "true"; 51 | 52 | 53 | // Use apt bandwidth limit feature, this example limits the download 54 | // speed to 70kb/sec 55 | //Acquire::http::Dl-Limit "70"; 56 | -------------------------------------------------------------------------------- /roles/cdh_hive_config/templates/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | javax.jdo.option.ConnectionURL 25 | {% if hostvars[groups["hive_metastore"][0]]["ansible_fqdn"] == ansible_fqdn %} 26 | jdbc:{{ hive_metastore }}://localhost/metastore 27 | {% else %} 28 | jdbc:{{ hive_metastore }}://{{ hostvars[groups["hive_metastore"][0]]["ansible_fqdn"] }}/metastore 29 | {% endif %} 30 | 31 | 32 | 33 | javax.jdo.option.ConnectionDriverName 34 | {% if hive_metastore == "postgresql" %} 35 | org.postgresql.Driver 36 | {% endif %} 37 | {% if hive_metastore == "mysql" %} 38 | com.mysql.jdbc.Driver 39 | {% endif %} 40 | 41 | 42 | 43 | javax.jdo.option.ConnectionUserName 44 | hiveuser 45 | 46 | 47 | 48 | javax.jdo.option.ConnectionPassword 49 | hive_{{ site_name|lower }} 50 | 51 | 52 | 53 | datanucleus.autoCreateSchema 54 | false 55 | 56 | 57 | 58 | hive.metastore.uris 59 | thrift://{{ hostvars[groups["hive_metastore"][0]]["ansible_fqdn"] }}:9083 60 | IP address (or fully-qualified domain name) and port of the metastore host 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /roles/common/templates/sysctl.conf: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # Auto-reboot linux 10 seconds after a kernel panic 4 | kernel.panic = 10 5 | kernel.panic_on_oops = 10 6 | kernel.unknown_nmi_panic = 10 7 | kernel.panic_on_unrecovered_nmi = 10 8 | kernel.panic_on_io_nmi = 10 9 | 10 | # Controls whether core dumps will append the PID to the core filename, useful for debugging multi-threaded applications 11 | kernel.core_uses_pid = 1 12 | 13 | # Allow for more PIDs 14 | kernel.pid_max = 4194303 15 | 16 | # Turn off address space randomization - the servers are behind a firewall 17 | kernel.randomize_va_space = 0 18 | 19 | # ------ VM ------ 20 | 21 | # See http://en.wikipedia.org/wiki/Swappiness 22 | vm.swappiness = {{ kernel_swappiness }} 23 | 24 | # ------ VM ------ 25 | 26 | fs.file-max = 204708 27 | fs.epoll.max_user_instances = 4096 28 | 29 | # ------ NETWORK SECURITY ------ 30 | 31 | # Protect ICMP attacks 32 | net.ipv4.icmp_echo_ignore_broadcasts = 1 33 | 34 | # Turn on protection for bad icmp error messages 35 | net.ipv4.icmp_ignore_bogus_error_responses = 1 36 | 37 | # Turn on syncookies for SYN flood attack protection 38 | net.ipv4.tcp_syncookies = 1 39 | net.ipv4.tcp_max_syn_backlog = 4096 40 | net.ipv4.tcp_synack_retries = 3 41 | 42 | # Log suspicious packets, such as spoofed, source-routed, and redirect 43 | net.ipv4.conf.all.log_martians = 1 44 | net.ipv4.conf.default.log_martians = 1 45 | 46 | # Disables these ipv4 features, not very legitimate uses 47 | net.ipv4.conf.all.accept_source_route = 0 48 | net.ipv4.conf.default.accept_source_route = 0 49 | 50 | # ------ NETWORK PERFORMANCE ------ 51 | 52 | # don't cache ssthresh from previous connection 53 | net.ipv4.tcp_no_metrics_save = 1 54 | 55 | # Allow reusing sockets in TIME_WAIT state for new connections 56 | net.ipv4.tcp_tw_reuse = 1 57 | 58 | # Socket max connections waiting to get accepted; the listen() backlog. 59 | # Default is 128. 60 | net.core.somaxconn = 4096 61 | 62 | # Enable receiver autotuning. Receiver autotuning is 'new'; sender autotuning has been around a long time. 63 | # Default is disabled. 64 | net.ipv4.tcp_moderate_rcvbuf = 1 65 | 66 | # Reduce TCP retries. 67 | # Default is 15. 68 | net.ipv4.tcp_retries2 = 3 69 | 70 | # Tune TCP keepalive. 71 | net.ipv4.tcp_keepalive_time = 600 72 | net.ipv4.tcp_keepalive_probes = 3 73 | net.ipv4.tcp_keepalive_intvl = 30 74 | 75 | # Decrease fin timeout. After telling the client we are closing, how long to wait for a FIN, ACK? 76 | # Default is 60. 77 | net.ipv4.tcp_fin_timeout = 10 78 | 79 | # Enable TCP FAST_OPEN for client and server. Still rarely used by applications. See https://lwn.net/Articles/508865/. Default from kernel 3.13. 80 | net.ipv4.tcp_fastopen = 3 81 | -------------------------------------------------------------------------------- /roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/common/tasks/main.yml 3 | 4 | - name: mount / with inode_readahead_blks=128,noatime,commit=30,errors=remount-ro 5 | shell: creates=/etc/fstab.bak chdir=/etc cp fstab fstab.bak && sed -i "s/ errors=remount-ro/ inode_readahead_blks=128,noatime,commit=30,errors=remount-ro/" fstab 6 | 7 | - name: write sysctl config file to /etc/sysctl.d/60-hadoop.conf 8 | template: src=sysctl.conf dest=/etc/sysctl.d/60-hadoop.conf owner=root group=root mode=0644 9 | notify: 10 | - restart procps 11 | 12 | - name: configure /etc/security/limits.conf with high max file descriptors (soft/hard) 13 | template: src=limits.conf dest=/etc/security/limits.conf mode=0644 14 | 15 | - name: configure /etc/pam.d/common-session and /etc/pam.d/common-session-noninteractive with pam_limits.so 16 | lineinfile: name={{ item }} regexp='pam_limits.so' line='session required pam_limits.so' 17 | with_items: 18 | - /etc/pam.d/common-session 19 | - /etc/pam.d/common-session-noninteractive 20 | 21 | - name: install commonly used packages via apt 22 | apt: pkg={{ item }} update_cache=yes cache_valid_time=3600 23 | with_items: 24 | - checkinstall 25 | - curl 26 | - dstat 27 | - git 28 | - htop 29 | - liblzo2-dev 30 | - net-tools 31 | - ntp 32 | - numactl 33 | - python-pycurl 34 | - software-properties-common 35 | - zip 36 | tags: ntp 37 | 38 | - name: install intel-microcode 39 | apt: pkg=intel-microcode 40 | when: (ansible_virtualization_role is not defined or ansible_virtualization_role == "host") and "Intel" in ansible_processor[0] 41 | 42 | - name: install amd64-microcode 43 | apt: pkg=amd64-microcode 44 | when: (ansible_virtualization_role is not defined or ansible_virtualization_role == "host") and "AMD" in ansible_processor[0] 45 | 46 | - name: remove various useless files 47 | command: removes={{ item }} rm {{ item }} 48 | with_items: 49 | - /etc/update-motd.d/10-help-text 50 | - /etc/update-motd.d/91-release-upgrade 51 | 52 | - name: create the hosts file for all machines 53 | template: backup=yes src=hosts dest=/etc/hosts 54 | tags: configuration 55 | 56 | - name: configure landscape-sysinfo to hide link 57 | template: src=client.conf dest=/etc/landscape/client.conf owner=root group=root mode=0644 58 | 59 | - name: configure rsyslog with the imfile module 60 | template: src=rsyslog.conf dest=/etc/rsyslog.d/30-imfile.conf owner=root group=root mode=0644 61 | tags: rsyslog 62 | notify: 63 | - restart rsyslog 64 | 65 | - name: configure NTP in /etc/ntp.conf 66 | template: src=ntp.conf dest=/etc/ntp.conf 67 | notify: 68 | - restart ntp 69 | tags: ntp 70 | 71 | - name: make sure NTP is enabled and started 72 | service: name=ntp enabled=yes state=started 73 | tags: ntp -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | mapreduce.framework.name 9 | yarn 10 | 11 | 12 | 13 | yarn.app.mapreduce.am.staging-dir 14 | /user 15 | 16 | 17 | 18 | mapreduce.jobhistory.address 19 | {{ hostvars[groups['historyserver'][0]]['ansible_fqdn'] }}:10020 20 | 21 | 22 | 23 | mapreduce.jobhistory.webapp.address 24 | {{ hostvars[groups['historyserver'][0]]['ansible_fqdn'] }}:19888 25 | 26 | 27 | 28 | yarn.web-proxy.address 29 | {{ hostvars[groups['resourcemanager'][0]]['ansible_fqdn'] }}:8088 30 | 31 | 32 | 33 | mapred.map.tasks 34 | 16 35 | 36 | 37 | 38 | mapred.reduce.tasks 39 | 6 40 | 41 | 42 | 43 | mapreduce.task.io.sort.factor 44 | 100 45 | 46 | 47 | 48 | io.sort.mb 49 | 512 50 | Higher memory-limit while sorting data. 51 | 52 | 53 | 54 | mapreduce.reduce.shuffle.parallelcopies 55 | 30 56 | 57 | 58 | 59 | 60 | mapreduce.map.output.compress 61 | true 62 | 63 | 64 | mapreduce.map.output.compress.codec 65 | org.apache.hadoop.io.compress.SnappyCodec 66 | 67 | 68 | 69 | 70 | mapreduce.output.fileoutputformat.compress 71 | true 72 | 73 | 74 | mapred.output.fileoutputformat.compress.codec 75 | org.apache.hadoop.io.compress.SnappyCodec 76 | 77 | 78 | mapreduce.output.fileoutputformat.compress.type 79 | BLOCK 80 | 81 | 82 | 83 | mapreduce.map.java.opts 84 | -Xmx1024m 85 | Higher Java heap for mapper to work. 86 | 87 | 88 | -------------------------------------------------------------------------------- /roles/ganglia_monitor/templates/device-metrics.php: -------------------------------------------------------------------------------- 1 | 0 && count($cols) > 1) { 18 | $data[] = $cols; 19 | } 20 | } 21 | 22 | $titles = array(); 23 | $titles['rrqm_s'] = 'Merged Reads'; 24 | $titles['wrqm_s'] = 'Merged Writes'; 25 | $titles['r_s'] = 'Completed Reads'; 26 | $titles['w_s'] = 'Completed Writes'; 27 | $titles['rkB_s'] = 'Data Read'; 28 | $titles['wkB_s'] = 'Data Written'; 29 | $titles['avgrq-sz'] = 'Average Req Size'; 30 | $titles['avgqu-sz'] = 'Average Req Queue Length'; 31 | $titles['await'] = 'Await'; 32 | $titles['r_await'] = 'Average Read Await'; 33 | $titles['w_await'] = 'Average Write Await'; 34 | $titles['util'] = 'CPU Time'; 35 | 36 | $units = array(); 37 | $units['rrqm_s'] = 'queued reqs/sec'; 38 | $units['wrqm_s'] = 'queued reqs/sec'; 39 | $units['r_s'] = 'reqs/sec'; 40 | $units['w_s'] = 'reqs/sec'; 41 | $units['rkB_s'] = 'kB/sec'; 42 | $units['wkB_s'] = 'kB/sec'; 43 | $units['avgrq-sz'] = 'sectors'; 44 | $units['avgqu-sz'] = 'sectors'; 45 | $units['await'] = 'ms'; 46 | $units['r_await'] = 'ms'; 47 | $units['w_await'] = 'ms'; 48 | $units['util'] = '%'; 49 | 50 | for ($col = 0; $col < count($header); $col++) { 51 | $header[$col] = str_replace("/", "_", $header[$col]); 52 | $header[$col] = str_replace("%", "", $header[$col]); 53 | } 54 | 55 | for ($row = 0; $row < count($data); $row++) { 56 | $namePrefix = "dev_" . $data[$row][0] . "-"; 57 | $titlePrefix = "dev/" . $data[$row][0] . " "; 58 | 59 | for ($col = 1; $col < count($header); $col++) { 60 | if ($header[$col] == 'svctm') 61 | continue; 62 | 63 | if ($header[$col] == '') 64 | continue; 65 | 66 | if (!isset($data[$row][$col])) 67 | continue; 68 | 69 | $name = $namePrefix . $header[$col]; 70 | $title = $titlePrefix . $titles[$header[$col]]; 71 | $value = $data[$row][$col]; 72 | $unit = $units[$header[$col]]; 73 | 74 | exec('gmetric -c /etc/ganglia/gmond.conf --name="' . $name . '" --title="' . $title . '" --value="' . $value . '" --type="float" --units="' . $unit . '" -g disk'); 75 | } 76 | 77 | exec('/usr/sbin/hddtemp -q -n /dev/' . $data[$row][0], $temperature); 78 | exec('gmetric -c /etc/ganglia/gmond.conf --name="' . $namePrefix . 'temp" --title="' . $titlePrefix . 'Temperature" --value="' . $temperature[0] . '" --type="int8" --units="degrees C" -g disk'); 79 | } 80 | ?> -------------------------------------------------------------------------------- /roles/postfix_mandrill/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: roles/postfix_mandrill/tasks/main.yml 3 | 4 | - name: debconf-set-selections mailname for postfix 5 | shell: executable=/bin/bash debconf-set-selections <<< "postfix postfix/mailname string {{ ansible_hostname }}.{{ postfix_domain }}" 6 | tags: postfix 7 | 8 | - name: debconf-set-selections mailer type for postfix 9 | shell: executable=/bin/bash debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'" 10 | tags: postfix 11 | 12 | - name: install postfix via apt 13 | apt: pkg={{ item }} 14 | with_items: 15 | - heirloom-mailx 16 | - postfix 17 | - mailutils 18 | - libsasl2-2 19 | - libsasl2-modules 20 | - ca-certificates 21 | tags: postfix 22 | 23 | - name: setup Mandrill credentials for postfix 24 | template: src=sasl_passwd dest=/etc/postfix/sasl_passwd owner=root group=root mode=600 25 | tags: postfix 26 | 27 | - name: run postmap to configure postfix lookup tables 28 | command: postmap /etc/postfix/sasl_passwd 29 | tags: postfix 30 | 31 | - name: install Mandrill (Thawte Premium Server) SSL certificates 32 | shell: creates=/etc/postfix/cacert.pem cat /etc/ssl/certs/Thawte_Premium_Server_CA.pem | tee /etc/postfix/cacert.pem 33 | tags: postfix 34 | 35 | - name: configure postfix in /etc/postfix/main.cf 36 | lineinfile: dest=/etc/postfix/main.cf regexp='{{ item }}' line='{{ item }}' 37 | with_items: 38 | - 'smtp_sasl_auth_enable = yes' 39 | - 'smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd' 40 | - 'smtp_sasl_security_options = noanonymous' 41 | - 'smtp_use_tls = yes' 42 | - "mydomain = {{ postfix_domain }}" 43 | - 'inet_interfaces = 127.0.0.1' # Make sure Postfix listens to localhost only 44 | tags: postfix 45 | 46 | - name: configure postfix's relayhost in /etc/postfix/main.cf 47 | lineinfile: dest=/etc/postfix/main.cf regexp='mandrillapp' line='relayhost = [smtp.mandrillapp.com]' 48 | notify: 49 | - restart postfix 50 | - send test email 51 | tags: postfix 52 | 53 | - name: forward root user email to {{ notify_email }} if not already forwarding 54 | shell: creates=~/.forward chdir=~ echo "{{ notify_email }}" > .forward 55 | tags: postfix 56 | 57 | - name: forward ansible user email to {{ notify_email }} if not already forwarding 58 | sudo: false 59 | shell: creates=~/.forward chdir=~ echo "{{ notify_email }}" > .forward 60 | tags: postfix 61 | 62 | - name: copy disk-usage-alert script to /etc/cron.hourly (emails when disk full) 63 | template: src=disk-usage-alert.sh dest=/etc/cron.hourly/disk-usage-alert owner=root group=root mode=755 64 | tags: postfix 65 | 66 | - name: configure unattended upgrades in /etc/apt/apt.conf.d/50unattended-upgrades (emails upon upgrade failure) 67 | template: src=50unattended-upgrades dest=/etc/apt/apt.conf.d/50unattended-upgrades 68 | tags: postfix -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: '2.6' 3 | addons: 4 | firefox: "25.0.1" 5 | cache: 6 | directories: 7 | - $HOME/.pip-cache/ 8 | install: 9 | - pip install ansible --download-cache $HOME/.pip-cache 10 | - pip install dopy --download-cache $HOME/.pip-cache 11 | before_script: 12 | - ansible-playbook -i hosts --extra-vars "api_key_password=$DO_API_KEY client_id=$DO_CLIENT_ID" do_cluster.yml 13 | - "export DISPLAY=:99.0" 14 | - "sh -e /etc/init.d/xvfb start" 15 | - "wget http://download.slimerjs.org/v0.9/0.9.1/slimerjs-0.9.1.zip" 16 | - "unzip slimerjs-0.9.1.zip && mv slimerjs-0.9.1 ./slimerjs" 17 | - "gem install --version 0.8.9 faraday" 18 | - "gem install travis-artifacts" 19 | - eval `ssh-agent` 20 | - "chmod 400 travis.ssh && ssh-add travis.ssh" 21 | - sleep 300 # DigitalOcean can be damn slow creating/booting droplets 22 | script: 23 | - ansible-playbook -i hosts --extra-vars "authorized_rsa_key='$(cat travis.ssh.pub)'" -u root bootstrap/bootstrap.yml 24 | - ansible-playbook -i hosts site.yml 25 | after_success: 26 | - "slimerjs screenshot.js http://hmaster01:50070 active-namenode.png" 27 | - "slimerjs screenshot.js http://hmaster02:50070 standby-namenode.png" 28 | - "slimerjs screenshot.js http://monitor01/ganglia ganglia.png" 29 | - "slimerjs screenshot.js http://monitor01:9200 elasticsearch.png" 30 | - "slimerjs screenshot.js http://monitor01/smokeping/smokeping.cgi?target=_charts smokeping.png" 31 | - "slimerjs screenshot.js http://hmaster02:8081 presto.png" 32 | - "travis-artifacts upload --path active-namenode.png" 33 | - "travis-artifacts upload --path standby-namenode.png" 34 | - "travis-artifacts upload --path ganglia.png" 35 | - "travis-artifacts upload --path elasticsearch.png" 36 | - "travis-artifacts upload --path smokeping.png" 37 | - "travis-artifacts upload --path presto.png" 38 | - sudo /etc/rc6.d/K10do_destroy.sh 39 | after_failure: 40 | - sudo /etc/rc6.d/K10do_destroy.sh 41 | env: 42 | global: 43 | - ANSIBLE_HOST_KEY_CHECKING=False ANSIBLE_SSH_ARGS="-o ForwardAgent=yes" 44 | - SLIMERJSLAUNCHER=$(which firefox) DISPLAY=:99.0 PATH=$TRAVIS_BUILD_DIR/slimerjs:$PATH 45 | - secure: "YBnTdEJ6U6i5no2eRlVC5dsqvSmYqULG+zWyTk2KqWGGMCza3RFi0wUueK3NuN+tl4Ijzf26Kma7wqdjkort0UrfxiH2xd4DWv3LvWz5vpKFP4GaSQGcd5zokHAJJ5hs0JgKY5jwGfoNHcnL16oUJEtptfLqTjGKRIHS+Ye1LmQ=" 46 | - secure: "DmsCmxwGBWVnYV6CDuCiJsATk85eFUr/Zs7BeYWjlKvAQz4Np9GTDECe3rE1KAA42PZrbvyAdWD31Np9QF0zj8Aq6poCvZ0c0Mib2bRi3eOP8tPL6EUggAl125UkoAZzxtgRWvILZfeRGhBzYsRX9C7/UtA8F8Ij0hXknkgdY48=" 47 | - ARTIFACTS_AWS_REGION=us-east-1 ARTIFACTS_S3_BUCKET=hadoop-ansible 48 | - secure: "QVbUIhd/KU8HmXf0hnU4DRKjT5zzAZNKEhW6zAr7hLHvgCYFQEyc3idtmbjMPxEtIqpudR/cu+W2LA0Sl6FWIPVKweTJLU5/ocT/oDEBR4R/J2jZD8yEun6fiNPhbJLfAGX65sZLJYuPows4NVXAxt37jHR2z3tMHLVzALVSuOE=" 49 | - secure: "Ipe61XLKD1rAo/3NDUsrlw40Ab6rXwQibDLALBGJikGQvbMiQrXRdEniGdlHyV9cnSoZ5+Akqn4SNZ0GmgnWIbhkUwzhFn7/ylL8O+KlGd8mAk8PKJyFTuVwSNBFluhnSH49xDobZp9OSMiColteNALxhXYh53de4ard1W27i88=" 50 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/hadoop-metrics2.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # syntax: [prefix].[source|sink].[instance].[options] 19 | # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details 20 | 21 | # {{ ansible_managed }} 22 | 23 | # default sampling period, in seconds 24 | *.period=10 25 | 26 | *.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 27 | 28 | # default sampling period, in seconds 29 | *.sink.ganglia.period=10 30 | 31 | # default for supportsparse is false 32 | *.sink.ganglia.supportsparse=true 33 | 34 | *.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both 35 | *.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 36 | 37 | # journalnodes 38 | journalnode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 39 | 40 | # namenodes 41 | namenode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 42 | 43 | # datanodes 44 | datanode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 45 | 46 | # jobtrackers 47 | jobtracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 48 | 49 | # tasktrackers 50 | tasktracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 51 | 52 | # maptasks 53 | maptask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 54 | 55 | # reducetasks 56 | reducetask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8649{% if not loop.last %},{% endif %}{% endfor %} 57 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | yarn.nodemanager.resource.cpu-vcores 10 | {{ ansible_processor_count * ansible_processor_cores * ansible_processor_threads_per_core }} 11 | 12 | 13 | 14 | 15 | yarn.scheduler.maximum-allocation-mb 16 | {{ ansible_memtotal_mb - 1024 }} 17 | 18 | 19 | yarn.nodemanager.resource.memory-mb 20 | {{ ansible_memtotal_mb - 1024 }} 21 | 22 | 23 | 24 | yarn.resourcemanager.resource-tracker.address 25 | {{ hostvars[groups['resourcemanager'][0]]['ansible_fqdn'] }}:8031 26 | 27 | 28 | yarn.resourcemanager.address 29 | {{ hostvars[groups['resourcemanager'][0]]['ansible_fqdn'] }}:8032 30 | 31 | 32 | yarn.resourcemanager.scheduler.address 33 | {{ hostvars[groups['resourcemanager'][0]]['ansible_fqdn'] }}:8030 34 | 35 | 36 | yarn.resourcemanager.admin.address 37 | {{ hostvars[groups['resourcemanager'][0]]['ansible_fqdn'] }}:8033 38 | 39 | 40 | yarn.resourcemanager.webapp.address 41 | {{ hostvars[groups['resourcemanager'][0]]['ansible_fqdn'] }}:8088 42 | 43 | 44 | Classpath for typical applications. 45 | yarn.application.classpath 46 | 47 | $HADOOP_CONF_DIR, 48 | $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*, 49 | $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*, 50 | $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*, 51 | $YARN_HOME/*,$YARN_HOME/lib/* 52 | 53 | 54 | 55 | yarn.nodemanager.aux-services 56 | mapreduce.shuffle 57 | 58 | 59 | yarn.nodemanager.aux-services.mapreduce.shuffle.class 60 | org.apache.hadoop.mapred.ShuffleHandler 61 | 62 | 63 | yarn.nodemanager.local-dirs 64 | /data/yarn/local 65 | 66 | 67 | yarn.nodemanager.log-dirs 68 | /data/yarn/logs 69 | 70 | 71 | Where to aggregate logs 72 | yarn.nodemanager.remote-app-log-dir 73 | /var/log/hadoop-yarn/apps 74 | 75 | 76 | 78 | 79 | yarn.resourcemanager.scheduler.class 80 | org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler 81 | 82 | 83 | -------------------------------------------------------------------------------- /roles/cdh_hadoop_config/templates/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | dfs.hosts.exclude 9 | /etc/hadoop/conf/dfs.hosts.exclude 10 | 11 | 12 | 13 | 14 | dfs.nameservices 15 | {{ site_name|lower }} 16 | 17 | 18 | 19 | 20 | dfs.ha.namenodes.{{ site_name|lower }} 21 | {% for host in groups['namenodes'] %}nn{{ loop.index }}{% if not loop.last %},{% endif %}{% endfor %} 22 | 23 | 24 | {% for host in groups['namenodes'] %} 25 | 26 | dfs.namenode.rpc-address.{{ site_name|lower }}.nn{{ loop.index }} 27 | {{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8020 28 | 29 | {% endfor %} 30 | 31 | {% for host in groups['namenodes'] %} 32 | 33 | dfs.namenode.http-address.{{ site_name|lower }}.nn{{ loop.index }} 34 | {{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:50070 35 | 36 | {% endfor %} 37 | 38 | 39 | 40 | dfs.namenode.shared.edits.dir 41 | qjournal://{% for host in groups['journalnodes'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}:8485{% if not loop.last %};{% endif %}{% endfor %}/{{ site_name|lower }} 42 | 43 | 44 | 45 | 46 | dfs.client.failover.proxy.provider.{{ site_name|lower }} 47 | org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider 48 | 49 | 50 | 51 | 52 | dfs.ha.fencing.methods 53 | shell(/bin/true) 54 | 55 | 56 | 57 | 58 | dfs.ha.automatic-failover.enabled 59 | true 60 | 61 | 62 | ha.zookeeper.quorum 63 | {% for host in groups['zookeepers'] %}{{ hostvars[host].ipv4_address|default(hostvars[host]['ansible_default_ipv4']['address']) }}{% if not loop.last %},{% endif %}{% endfor %} 64 | 65 | 66 | 67 | 68 | dfs.replication 69 | 3 70 | true 71 | 72 | 73 | 74 | dfs.blocksize 75 | {{ dfs_blocksize }} 76 | true 77 | 78 | 79 | 80 | dfs.journalnode.edits.dir 81 | /data/dfs/jn 82 | 83 | 84 | 85 | dfs.name.dir 86 | file:///data/dfs/nn 87 | 88 | 89 | 90 | dfs.data.dir 91 | file:///data/dfs/dn 92 | 93 | 94 | 95 | dfs.datanode.max.xcievers 96 | {{ max_xcievers }} 97 | true 98 | 99 | 100 | 101 | dfs.namenode.handler.count 102 | 40 103 | true 104 | 105 | 106 | 107 | dfs.datanode.handler.count 108 | 8 109 | true 110 | 111 | 112 | 113 | dfs.namenode.avoid.read.stale.datanode 114 | true 115 | 116 | 117 | 118 | dfs.namenode.avoid.write.stale.datanode 119 | true 120 | 121 | 122 | 123 | dfs.namenode.stale.datanode.interval 124 | 30000 125 | 126 | 127 | 128 | dfs.client.read.shortcircuit 129 | true 130 | 131 | 132 | 133 | dfs.domain.socket.path 134 | /var/run/hadoop-hdfs/dn._PORT 135 | 136 | 137 | 138 | dfs.client.file-block-storage-locations.timeout 139 | 3000 140 | 141 | 142 | -------------------------------------------------------------------------------- /roles/cdh_hbase_config/templates/hbase-env.sh: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | # 3 | #/** 4 | # * Copyright 2007 The Apache Software Foundation 5 | # * 6 | # * Licensed to the Apache Software Foundation (ASF) under one 7 | # * or more contributor license agreements. See the NOTICE file 8 | # * distributed with this work for additional information 9 | # * regarding copyright ownership. The ASF licenses this file 10 | # * to you under the Apache License, Version 2.0 (the 11 | # * "License"); you may not use this file except in compliance 12 | # * with the License. You may obtain a copy of the License at 13 | # * 14 | # * http://www.apache.org/licenses/LICENSE-2.0 15 | # * 16 | # * Unless required by applicable law or agreed to in writing, software 17 | # * distributed under the License is distributed on an "AS IS" BASIS, 18 | # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | # * See the License for the specific language governing permissions and 20 | # * limitations under the License. 21 | # */ 22 | 23 | # Set environment variables here. 24 | 25 | # This script sets variables multiple times over the course of starting an hbase process, 26 | # so try to keep things idempotent unless you want to take an even deeper look 27 | # into the startup scripts (bin/hbase, etc.) 28 | 29 | # The java implementation to use. Java 1.6 required. 30 | # export JAVA_HOME=/usr/java/jdk1.6.0/ 31 | 32 | # Extra Java CLASSPATH elements. Optional. 33 | # export HBASE_CLASSPATH= 34 | 35 | # The maximum amount of heap to use, in MB. Default is 4096. 36 | export HBASE_HEAPSIZE={{ hbase_heapsize }} 37 | 38 | # Extra Java runtime options. 39 | # Below are what we set by default. May only work with SUN JVM. 40 | # For more on why as well as other possible settings, 41 | # see http://wiki.apache.org/hadoop/PerformanceTuning 42 | export HBASE_OPTS="-Xbootclasspath/p:/usr/lib/jvm/floatingdecimal-0.1.jar -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -XX:MaxPermSize=300M -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+ExplicitGCInvokesConcurrent" 43 | 44 | # Uncomment below to enable java garbage collection logging for the server-side processes 45 | # this enables basic gc logging for the server processes to the .out file 46 | # export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" 47 | 48 | # this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above 49 | export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS" 50 | 51 | # Uncomment below to enable java garbage collection logging for the client processes in the .out file. 52 | export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" 53 | 54 | # Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS). 55 | # This applies to both the server and client GC options above 56 | export HBASE_USE_GC_LOGFILE=true 57 | 58 | 59 | # Uncomment below if you intend to use the EXPERIMENTAL off heap cache. 60 | # export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" 61 | # Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. 62 | 63 | 64 | # Uncomment and adjust to enable JMX exporting 65 | # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. 66 | # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html 67 | # 68 | # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" 69 | # export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" 70 | # export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" 71 | # export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" 72 | # export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" 73 | # export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" 74 | 75 | # File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. 76 | # export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers 77 | 78 | # Uncomment and adjust to keep all the Region Server pages mapped to be memory resident 79 | #HBASE_REGIONSERVER_MLOCK=true 80 | #HBASE_REGIONSERVER_UID="hbase" 81 | 82 | # File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. 83 | # export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters 84 | 85 | # Extra ssh options. Empty by default. 86 | # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" 87 | 88 | # Where log files are stored. $HBASE_HOME/logs by default. 89 | # export HBASE_LOG_DIR=${HBASE_HOME}/logs 90 | 91 | # Enable remote JDWP debugging of major HBase processes. Meant for Core Developers 92 | # export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" 93 | # export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" 94 | # export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" 95 | # export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" 96 | 97 | # A string representing this instance of hbase. $USER by default. 98 | # export HBASE_IDENT_STRING=$USER 99 | 100 | # The scheduling priority for daemon processes. See 'man nice'. 101 | # export HBASE_NICENESS=10 102 | 103 | # The directory where pid files are stored. /tmp by default. 104 | # export HBASE_PID_DIR=/var/hadoop/pids 105 | 106 | # Seconds to sleep between slave commands. Unset by default. This 107 | # can be useful in large clusters, where, e.g., slave rsyncs can 108 | # otherwise arrive faster than the master can service them. 109 | # export HBASE_SLAVE_SLEEP=0.1 110 | 111 | # Tell HBase whether it should manage it's own instance of Zookeeper or not. 112 | # export HBASE_MANAGES_ZK=true 113 | # The default log rolling policy is RFA, where the log file is rolled as per the size defined for the 114 | # RFA appender. Please refer to the log4j.properties file to see more details on this appender. 115 | # In case one needs to do log rolling on a date change, one should set the environment property 116 | # HBASE_ROOT_LOGGER to ",DRFA". 117 | # For example: 118 | # HBASE_ROOT_LOGGER=INFO,DRFA 119 | # The reason for changing default to RFA is to avoid the boundary case of filling out disk space as 120 | # DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. 121 | -------------------------------------------------------------------------------- /roles/ganglia_metad/templates/gmetad.conf: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | # 3 | #------------------------------------------------------------------------------- 4 | # Setting the debug_level to 1 will keep daemon in the forground and 5 | # show only error messages. Setting this value higher than 1 will make 6 | # gmetad output debugging information and stay in the foreground. 7 | # default: 0 8 | # debug_level 10 9 | # 10 | #------------------------------------------------------------------------------- 11 | # What to monitor. The most important section of this file. 12 | # 13 | # The data_source tag specifies either a cluster or a grid to 14 | # monitor. If we detect the source is a cluster, we will maintain a complete 15 | # set of RRD databases for it, which can be used to create historical 16 | # graphs of the metrics. If the source is a grid (it comes from another gmetad), 17 | # we will only maintain summary RRDs for it. 18 | # 19 | # Format: 20 | # data_source "my cluster" [polling interval] address1:port addreses2:port ... 21 | # 22 | # The keyword 'data_source' must immediately be followed by a unique 23 | # string which identifies the source, then an optional polling interval in 24 | # seconds. The source will be polled at this interval on average. 25 | # If the polling interval is omitted, 15sec is asssumed. 26 | # 27 | # If you choose to set the polling interval to something other than the default, 28 | # note that the web frontend determines a host as down if its TN value is less 29 | # than 4 * TMAX (20sec by default). Therefore, if you set the polling interval 30 | # to something around or greater than 80sec, this will cause the frontend to 31 | # incorrectly display hosts as down even though they are not. 32 | # 33 | # A list of machines which service the data source follows, in the 34 | # format ip:port, or name:port. If a port is not specified then 8649 35 | # (the default gmond port) is assumed. 36 | # default: There is no default value 37 | # 38 | 39 | # data_source "my cluster" 10 localhost my.machine.edu:8649 1.2.3.5:8655 40 | # data_source "my grid" 50 1.3.4.7:8655 grid.org:8651 grid-backup.org:8651 41 | # data_source "another source" 1.3.4.7:8655 1.3.4.8 42 | 43 | data_source "Hadoop" {{ hostvars[groups["masters"][0]].ipv4_address|default(hostvars[groups["masters"][0]].ansible_default_ipv4.address) }} {{ hostvars[groups["masters"][1]].ipv4_address|default(hostvars[groups["masters"][1]].ansible_default_ipv4.address) }} 44 | 45 | # 46 | # Round-Robin Archives 47 | # You can specify custom Round-Robin archives here (defaults are listed below) 48 | # 49 | # Old Default RRA: Keep 1 hour of metrics at 15 second resolution. 1 day at 6 minute 50 | # RRAs "RRA:AVERAGE:0.5:1:244" "RRA:AVERAGE:0.5:24:244" "RRA:AVERAGE:0.5:168:244" "RRA:AVERAGE:0.5:672:244" \ 51 | # "RRA:AVERAGE:0.5:5760:374" 52 | # New Default RRA 53 | # Keep 5856 data points at 15 second resolution assuming 15 second (default) polling. That's 1 day 54 | # Two weeks of data points at 1 minute resolution (average) 55 | #RRAs "RRA:AVERAGE:0.5:1:5856" "RRA:AVERAGE:0.5:4:20160" "RRA:AVERAGE:0.5:40:52704" 56 | 57 | # 58 | #------------------------------------------------------------------------------- 59 | # Scalability mode. If on, we summarize over downstream grids, and respect 60 | # authority tags. If off, we take on 2.5.0-era behavior: we do not wrap our output 61 | # in tags, we ignore all tags we see, and always assume 62 | # we are the "authority" on data source feeds. This approach does not scale to 63 | # large groups of clusters, but is provided for backwards compatibility. 64 | # default: on 65 | # scalable off 66 | # 67 | #------------------------------------------------------------------------------- 68 | # The name of this Grid. All the data sources above will be wrapped in a GRID 69 | # tag with this name. 70 | # default: unspecified 71 | 72 | gridname "{{ site_name|capitalize }}" 73 | 74 | # 75 | #------------------------------------------------------------------------------- 76 | # The authority URL for this grid. Used by other gmetads to locate graphs 77 | # for our data sources. Generally points to a ganglia/ 78 | # website on this machine. 79 | # default: "http://hostname/ganglia/", 80 | # where hostname is the name of this machine, as defined by gethostname(). 81 | # authority "http://mycluster.org/newprefix/" 82 | # 83 | #------------------------------------------------------------------------------- 84 | # List of machines this gmetad will share XML with. Localhost 85 | # is always trusted. 86 | # default: There is no default value 87 | # trusted_hosts 127.0.0.1 169.229.50.165 my.gmetad.org 88 | # 89 | #------------------------------------------------------------------------------- 90 | # If you want any host which connects to the gmetad XML to receive 91 | # data, then set this value to "on" 92 | # default: off 93 | # all_trusted on 94 | # 95 | #------------------------------------------------------------------------------- 96 | # If you don't want gmetad to setuid then set this to off 97 | # default: on 98 | # setuid off 99 | # 100 | #------------------------------------------------------------------------------- 101 | # User gmetad will setuid to (defaults to "nobody") 102 | # default: "nobody" 103 | # setuid_username "nobody" 104 | # 105 | #------------------------------------------------------------------------------- 106 | # Umask to apply to created rrd files and grid directory structure 107 | # default: 0 (files are public) 108 | # umask 022 109 | # 110 | #------------------------------------------------------------------------------- 111 | # The port gmetad will answer requests for XML 112 | # default: 8651 113 | # xml_port 8651 114 | # 115 | #------------------------------------------------------------------------------- 116 | # The port gmetad will answer queries for XML. This facility allows 117 | # simple subtree and summation views of the XML tree. 118 | # default: 8652 119 | # interactive_port 8652 120 | # 121 | #------------------------------------------------------------------------------- 122 | # The number of threads answering XML requests 123 | # default: 4 124 | # server_threads 10 125 | # 126 | #------------------------------------------------------------------------------- 127 | # Where gmetad stores its round-robin databases 128 | # default: "/var/lib/ganglia/rrds" 129 | # rrd_rootdir "/some/other/place" 130 | # 131 | #------------------------------------------------------------------------------- 132 | # In earlier versions of gmetad, hostnames were handled in a case 133 | # sensitive manner 134 | # If your hostname directories have been renamed to lower case, 135 | # set this option to 0 to disable backward compatibility. 136 | # From version 3.2, backwards compatibility will be disabled by default. 137 | # default: 1 (for gmetad < 3.2) 138 | # default: 0 (for gmetad >= 3.2) 139 | case_sensitive_hostnames 0 140 | 141 | #------------------------------------------------------------------------------- 142 | # It is now possible to export all the metrics collected by gmetad directly to 143 | # graphite by setting the following attributes. 144 | # 145 | # The hostname or IP address of the Graphite server 146 | # default: unspecified 147 | # carbon_server "my.graphite.box" 148 | # 149 | # The port on which Graphite is listening 150 | # default: 2003 151 | # carbon_port 2003 152 | # 153 | # A prefix to prepend to the metric names exported by gmetad. Graphite uses dot- 154 | # separated paths to organize and refer to metrics. 155 | # default: unspecified 156 | # graphite_prefix "datacenter1.gmetad" 157 | # 158 | # Number of milliseconds gmetad will wait for a response from the graphite server 159 | # default: 500 160 | # carbon_timeout 500 161 | # 162 | -------------------------------------------------------------------------------- /site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Site Configuration 3 | # ================== 4 | 5 | - hosts: all 6 | user: ansibler 7 | tasks: 8 | - name: determine interface 9 | set_fact: ipv4_address="{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" 10 | # eg. to use a eth1: ipv4_address="{{ hostvars[inventory_hostname].ansible_eth1.ipv4.address }}" 11 | 12 | - hosts: all 13 | user: ansibler 14 | sudo: true 15 | roles: 16 | - { role: kernel, when: upgrade_kernel and not ansible_kernel.startswith('3.13') } 17 | - common 18 | - { role: postfix_mandrill, when: notify_email is defined and postfix_domain is defined and mandrill_username is defined and mandrill_api_key is defined } 19 | - ganglia_monitor 20 | - { role: oracle_jdk, when: jdk_installed is not defined } 21 | 22 | - hosts: 2_links_aggregated 23 | user: ansibler 24 | sudo: true 25 | roles: 26 | - { role: 2_aggregated_links, when: bond_mode is defined and mtu is defined and ansible_virtualization_role == "host" and ansible_interfaces|length == 3 and "bond0" not in ansible_interfaces } 27 | 28 | - hosts: elasticsearch 29 | user: ansibler 30 | sudo: true 31 | roles: 32 | - elasticsearch 33 | - elasticsearch_curator 34 | 35 | - hosts: monitors[0] 36 | user: ansibler 37 | sudo: true 38 | roles: 39 | - ganglia_metad 40 | - ganglia_web 41 | - kibana 42 | - smokeping 43 | 44 | - hosts: all 45 | user: ansibler 46 | sudo: true 47 | roles: 48 | - td_agent 49 | - rsyslog_fluentd 50 | 51 | # Hadoop 52 | 53 | - hosts: zookeepers:journalnodes:resourcemanager:nodemanagers:historyserver:hbase_masters:regionservers 54 | user: ansibler 55 | sudo: true 56 | roles: 57 | - cdh_common 58 | - cdh_hadoop_config 59 | 60 | - hosts: zookeepers 61 | user: ansibler 62 | sudo: true 63 | roles: 64 | - cdh_zookeeper_server 65 | 66 | - hosts: journalnodes 67 | user: ansibler 68 | sudo: true 69 | roles: 70 | - cdh_hadoop_journalnode 71 | 72 | - hosts: namenodes[0] 73 | user: ansibler 74 | sudo: true 75 | sudo_user: hdfs 76 | tasks: 77 | - name: create the /data/dfs/nn directory 78 | file: path=/data/dfs/nn owner=hdfs group=hdfs state=directory 79 | tags: 80 | - hadoop 81 | - hbase 82 | - hive 83 | - name: format the namenode - WILL NOT FORMAT IF /data/dfs/nn/current/VERSION EXISTS TO AVOID DATA LOSS 84 | command: creates=/data/dfs/nn/current/VERSION hdfs namenode -format -force 85 | tags: 86 | - hadoop 87 | - hbase 88 | - hive 89 | 90 | - hosts: namenodes[0] 91 | user: ansibler 92 | sudo: true 93 | roles: 94 | - cdh_hadoop_namenode 95 | - cdh_hadoop_zkfc 96 | 97 | - hosts: namenodes[1] 98 | user: ansibler 99 | sudo: true 100 | sudo_user: hdfs 101 | tasks: 102 | - name: wait for the first namenode to come online 103 | wait_for: host={{ hostvars[groups['namenodes'][0]].ipv4_address|default(hostvars[groups['namenodes'][0]].ansible_default_ipv4.address) }} port=50070 104 | tags: 105 | - hadoop 106 | - hbase 107 | - hive 108 | - name: create the /data/dfs/nn directory 109 | file: path=/data/dfs/nn owner=hdfs group=hdfs state=directory 110 | tags: 111 | - hadoop 112 | - hbase 113 | - hive 114 | - name: bootstrap the standby namenode 115 | shell: creates=/data/dfs/nn/.bootstrapped hdfs namenode -bootstrapStandby && touch /data/dfs/nn/.bootstrapped 116 | tags: 117 | - hadoop 118 | - hbase 119 | - hive 120 | 121 | - hosts: namenodes[1] 122 | user: ansibler 123 | sudo: true 124 | roles: 125 | - cdh_hadoop_namenode 126 | - cdh_hadoop_zkfc 127 | 128 | - hosts: namenodes[0] 129 | user: ansibler 130 | sudo: true 131 | tasks: 132 | - name: format hadoop-hdfs-zkfc 133 | shell: creates=/data/dfs/.zkfsformatted hdfs zkfc -formatZK -force && touch /data/dfs/.zkfsformatted 134 | tags: 135 | - hadoop 136 | - hbase 137 | - hive 138 | - name: start zkfc 139 | service: name=hadoop-hdfs-zkfc state=restarted 140 | tags: 141 | - hadoop 142 | - hbase 143 | - hive 144 | - name: restart namenode 145 | service: name=hadoop-hdfs-namenode state=restarted 146 | tags: 147 | - hadoop 148 | - hbase 149 | - hive 150 | 151 | - hosts: datanodes 152 | user: ansibler 153 | sudo: true 154 | roles: 155 | - cdh_hadoop_datanode 156 | 157 | - hosts: nodemanagers 158 | user: ansibler 159 | sudo: true 160 | roles: 161 | - cdh_hadoop_mapreduce 162 | 163 | - hosts: hbase_masters:regionservers 164 | user: ansibler 165 | sudo: true 166 | roles: 167 | - cdh_hbase_config 168 | 169 | - hosts: namenodes[0] 170 | user: ansibler 171 | sudo: true 172 | sudo_user: hdfs 173 | tasks: 174 | - name: create /tmp directory on the cluster 175 | shell: creates=/data/dfs/.tmpdircreated hadoop fs -mkdir /tmp && touch /data/dfs/.tmpdircreated 176 | tags: 177 | - hadoop 178 | - hadoop_fs 179 | - name: make sure the /tmp directory has the correct permissions 180 | shell: creates=/data/dfs/.tmpdirchowned sleep 5 && hadoop fs -chmod -R 1777 /tmp && touch /data/dfs/.tmpdirchowned 181 | tags: 182 | - hadoop 183 | - hadoop_fs 184 | - name: create a /user/history directory on the cluster 185 | shell: creates=/data/dfs/.historydircreated hadoop fs -mkdir /user/history && touch /data/dfs/.historydircreated 186 | tags: 187 | - hadoop 188 | - hadoop_fs 189 | - name: make sure the /user/history directory has the correct permissions 190 | shell: creates=/data/dfs/.historydirchmodded sleep 5 && hadoop fs -chmod -R 1777 /user/history && touch /data/dfs/.historydirchmodded 191 | tags: 192 | - hadoop 193 | - hadoop_fs 194 | - name: make sure the /user/history directory has the correct owner 195 | shell: creates=/data/dfs/.historydirchowned sleep 5 && hadoop fs -chown yarn:mapred /user/history && touch /data/dfs/.historydirchowned 196 | tags: 197 | - hadoop 198 | - hadoop_fs 199 | - name: create the /hbase directory on the cluster 200 | shell: creates=/data/dfs/.hbasedircreated hadoop fs -mkdir /hbase && touch /data/dfs/.hbasedircreated 201 | tags: 202 | - hbase 203 | - hadoop_fs 204 | - name: make sure the /hbase directory has the correct owner 205 | shell: creates=/data/dfs/.hbasedirchowned sleep 5 && hadoop fs -chown hbase:hbase /hbase && touch /data/dfs/.hbasedirchowned 206 | tags: 207 | - hbase 208 | - hadoop_fs 209 | 210 | - hosts: resourcemanager 211 | user: ansibler 212 | sudo: true 213 | roles: 214 | - cdh_hadoop_yarn_resourcemanager 215 | 216 | - hosts: nodemanagers 217 | user: ansibler 218 | sudo: true 219 | roles: 220 | - cdh_hadoop_yarn_nodemanager 221 | 222 | - hosts: historyserver 223 | user: ansibler 224 | sudo: true 225 | roles: 226 | - cdh_hadoop_mapreduce_historyserver 227 | - cdh_hadoop_yarn_proxyserver 228 | 229 | - hosts: hbase_masters 230 | user: ansibler 231 | sudo: true 232 | roles: 233 | - cdh_hbase_master 234 | 235 | - hosts: regionservers 236 | user: ansibler 237 | sudo: true 238 | roles: 239 | - cdh_hbase_regionserver 240 | 241 | - hosts: hive_metastore 242 | user: ansibler 243 | sudo: true 244 | roles: 245 | - { role: postgresql_server, when: hive_metastore == "postgresql" } 246 | - { role: cdh_hive_postgresql_metastore, when: hive_metastore == "postgresql" } 247 | - { role: mysql_server, when: hive_metastore == "mysql" } 248 | - { role: cdh_hive_mysql_metastore, when: hive_metastore == "mysql" } 249 | - cdh_hive_config 250 | 251 | - hosts: hive_metastore 252 | user: ansibler 253 | sudo: true 254 | roles: 255 | - cdh_hive_metastore 256 | 257 | - hosts: namenodes[0] 258 | user: ansibler 259 | sudo: true 260 | sudo_user: hdfs 261 | tasks: 262 | - name: create a /user/hive/warehouse directory on the cluster 263 | shell: creates=/data/dfs/.hivewarehousedircreated hadoop fs -mkdir /user/hive/warehouse && touch /data/dfs/.hivewarehousedircreated 264 | tags: 265 | - hive 266 | - hadoop_fs 267 | - name: make sure the /user/hive/warehouse directory has the correct permissions 268 | shell: creates=/data/dfs/.hivewarehousedirchmodded sleep 5 && hadoop fs -chmod -R 1777 /user/hive/warehouse && touch /data/dfs/.hivewarehousedirchmodded 269 | tags: 270 | - hive 271 | - hadoop_fs 272 | 273 | - hosts: presto_coordinators 274 | user: ansibler 275 | sudo: true 276 | roles: 277 | - presto_coordinator 278 | 279 | - hosts: presto_workers 280 | user: ansibler 281 | sudo: true 282 | roles: 283 | - presto_worker 284 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Hadoop Ansible Playbook [![Build Status](https://travis-ci.org/analytically/hadoop-ansible.svg?branch=master)](https://travis-ci.org/analytically/hadoop-ansible) 2 | ======================= 3 | 4 | [Ansible](http://www.ansibleworks.com/) playbook that installs a CDH 4.6.0 [Hadoop](http://hadoop.apache.org/) 5 | cluster (running on Java 7, supported from [CDH 4.4](http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH4-Release-Notes/Whats_New_in_4-4.html)), 6 | with [HBase](http://hbase.apache.org/), Hive, [Presto](http://prestodb.io/) for analytics, and [Ganglia](http://ganglia.sourceforge.net/), 7 | [Smokeping](http://oss.oetiker.ch/smokeping/), [Fluentd](http://fluentd.org/), [Elasticsearch](http://www.elasticsearch.org/) 8 | and [Kibana](http://www.elasticsearch.org/overview/kibana/) for monitoring and centralized log indexing. 9 | 10 | Follow [@analytically](http://twitter.com/analytically). Browse the CI [build screenshots](http://hadoop-ansible.s3-website-us-east-1.amazonaws.com/#artifacts/). 11 | 12 | ### Requirements 13 | 14 | - [Ansible](http://www.ansibleworks.com/) 1.5 or later (`pip install ansible`) 15 | - 6 + 1 Ubuntu 12.04 LTS/13.04/13.10 or Debian "wheezy" hosts - see [ubuntu-netboot-tftp](https://github.com/analytically/ubuntu-netboot-tftp) if you need automated server installation 16 | - [Mandrill](http://mandrill.com/) username and API key for sending email notifications 17 | - `ansibler` user in sudo group without sudo password prompt (see Bootstrapping section below) 18 | 19 | ### Cloudera ([CDH4](http://www.cloudera.com/content/support/en/documentation/cdh4-documentation/cdh4-documentation-v4-latest.html)) Hadoop Roles 20 | 21 | If you're assembling your own Hadoop playbook, these roles are available for you to reuse: 22 | 23 | - [`cdh_common`](roles/cdh_common/) - sets up Cloudera's Ubuntu repository and key 24 | - [`cdh_hadoop_common`](roles/cdh_hadoop_common/) - common packages shared by all Hadoop nodes 25 | - [`cdh_hadoop_config`](roles/cdh_hadoop_config/) - common configuration shared by all Hadoop nodes 26 | - [`cdh_hadoop_datanode`](roles/cdh_hadoop_datanode/) - installs Hadoop DataNode 27 | - [`cdh_hadoop_journalnode`](roles/cdh_hadoop_journalnode/) - installs Hadoop JournalNode 28 | - [`cdh_hadoop_mapreduce`](roles/cdh_hadoop_mapreduce/) - installs Hadoop MapReduce 29 | - [`cdh_hadoop_mapreduce_historyserver`](roles/cdh_hadoop_mapreduce_historyserver/) - installs Hadoop MapReduce history server 30 | - [`cdh_hadoop_namenode`](roles/cdh_hadoop_namenode/) - installs Hadoop NameNode 31 | - [`cdh_hadoop_yarn_nodemanager`](roles/cdh_hadoop_yarn_nodemanager/) - installs Hadoop YARN node manager 32 | - [`cdh_hadoop_yarn_proxyserver`](roles/cdh_hadoop_yarn_proxyserver/) - installs Hadoop YARN proxy server 33 | - [`cdh_hadoop_yarn_resourcemanager`](roles/cdh_hadoop_yarn_resourcemanager/) - installs Hadoop YARN resource manager 34 | - [`cdh_hadoop_zkfc`](roles/cdh_hadoop_zkfc/) - installs Hadoop Zookeeper Failover Controller 35 | - [`cdh_hbase_common`](roles/cdh_hbase_common/) - common packages shared by all HBase nodes 36 | - [`cdh_hbase_config`](roles/cdh_hbase_common/) - common configuration shared by all HBase nodes 37 | - [`cdh_hbase_master`](roles/cdh_hbase_master/) - installs HBase-Master 38 | - [`cdh_hbase_regionserver`](roles/cdh_hbase_regionserver/) - installs HBase RegionServer 39 | - [`cdh_hive_common`](roles/cdh_hive_common/) - common packages shared by all Hive nodes 40 | - [`cdh_hive_config`](roles/cdh_hive_config/) - common configuration shared by all Hive nodes 41 | - [`cdh_hive_metastore`](roles/cdh_hive_metastore/) - installs Hive metastore (with PostgreSQL database) 42 | - [`cdh_zookeeper_server`](roles/cdh_zookeeper_server/) - installs ZooKeeper Server 43 | 44 | ### [Facebook Presto](http://prestodb.io/) Roles 45 | 46 | - [`presto_common`](roles/presto_common/) - downloads Presto to /usr/local/presto and prepares the node configuration 47 | - [`presto_coordinator`](roles/presto_coordinator/) - installs Presto coordinator config 48 | - [`presto_worker`](roles/presto_worker/) - installs Presto worker config 49 | 50 | ### Configuration 51 | 52 | Set the following variables using `--extra-vars` or editing [`group_vars/all`](group_vars/all): 53 | 54 | Required: 55 | 56 | - `site_name` - used as Hadoop nameservices and various directory names. Alphanumeric only. 57 | 58 | Optional: 59 | 60 | - Network interface: if you'd like to use a different IP address per host (eg. internal interface), change `site.yml` and 61 | change `set_fact: ipv4_address=...` to determine the correct IP address to use per host. If this fact is not set, 62 | `ansible_default_ipv4.address` will be used. 63 | - Email notification: `notify_email`, `postfix_domain`, `mandrill_username`, `mandrill_api_key` 64 | - [`roles/common`](roles/common/defaults/main.yml): `kernel_swappiness`(0), `nofile` limits, ntp servers and `rsyslog_polling_interval_secs`(10) 65 | - [`roles/2_aggregated_links`](roles/2_aggregated_links/defaults/main.yml): `bond_mode` (balance-alb) and `mtu` (9216) 66 | - [`roles/cdh_hadoop_config`](roles/cdh_hadoop_config/defaults/main.yml): `dfs_blocksize` (268435456), `max_xcievers` (4096), `heapsize` (12278) 67 | 68 | #### Adding hosts 69 | 70 | Edit the [hosts](hosts) file and list hosts per group (see [Inventory](http://www.ansibleworks.com/docs/intro_inventory.html) for more examples): 71 | 72 | ``` 73 | [datanodes] 74 | hslave010 75 | hslave[090:252] 76 | hadoop-slave-[a:f].example.com 77 | ``` 78 | 79 | Make sure that the `zookeepers` and `journalnodes` groups contain at least 3 hosts and have an odd number of hosts. 80 | 81 | #### Ganglia nodes 82 | 83 | Since we're using unicast mode for Ganglia (which significantly reduces chatter), you may have to wait 60 seconds 84 | after node startup before it is seen/shows up in the web interface. 85 | 86 | ### Installation 87 | 88 | To run Ansible: 89 | 90 | ```sh 91 | ./site.sh 92 | ``` 93 | 94 | To e.g. just install ZooKeeper, add the `zookeeper` tag as argument (available tags: apache, bonding, configuration, 95 | elasticsearch, elasticsearch_curator, fluentd, ganglia, hadoop, hbase, hive, java, kibana, ntp, postfix, postgres, presto, 96 | rsyslog, tdagent, zookeeper): 97 | 98 | ```sh 99 | ./site.sh zookeeper 100 | ``` 101 | 102 | #### What else is installed? 103 | 104 | - To improve performance, [sysctl tuning](roles/common/templates/sysctl.conf) 105 | - [link aggregation](roles/2_aggregated_links/templates/interfaces) configures [Link Aggregation](https://help.ubuntu.com/community/UbuntuBonding) if 2 interfaces are available 106 | - [htop](http://htop.sourceforge.net/), curl, checkinstall, heirloom-mailx, intel-microcode/amd64-microcode, net-tools, zip 107 | - [NTP](http://www.ntp.org/) configured with the [Oxford University NTP service](http://www.oucs.ox.ac.uk/network/ntp/) by default 108 | - [Postfix](http://www.postfix.org/) with [Mandrill](http://mandrill.com/) configuration 109 | - unattended upgrades [email to inform success/failure](roles/postfix_mandrill/templates/50unattended-upgrades) 110 | - php5-cli, sysstat, hddtemp to report [device metrics](roles/ganglia_monitor/templates/device-metrics.php) 111 | (reads/writes/temp) to Ganglia [every 10 minutes](roles/ganglia_monitor/templates/device-metrics.cron.d) 112 | - LZO (Lempel–Ziv–Oberhumer) and [Google Snappy 1.1.1](https://code.google.com/p/snappy/) compression 113 | - a [fork of openjdk's FloatingDecimal](https://github.com/airlift/floatingdecimal) to fix monitor contention when parsing doubles due to a static synchronized method 114 | - [Elasticsearch Curator](https://github.com/elasticsearch/curator), defaults to maximum 30 GB of data in Elasticsearch, via cron daily at 2:00AM 115 | - [Elasticsearch Marvel](http://www.elasticsearch.org/overview/marvel/), monitor your Elasticsearch cluster's heartbeat 116 | - [SmokePing](http://oss.oetiker.ch/smokeping/) to keep track of network latency 117 | 118 | #### URL's 119 | 120 | After the installation, go here: 121 | 122 | - Ganglia at [monitor01/ganglia](http://monitor01/ganglia/) 123 | - Kibana at [monitor01/kibana/index.html#/dashboard/file/logstash.json](http://monitor01/kibana/index.html#/dashboard/file/logstash.json) 124 | - Smokeping at [monitor01/smokeping/smokeping.cgi](http://monitor01/smokeping/smokeping.cgi) 125 | - hmaster01 at [hmaster01:50070](http://hmaster01:50070) - active namenode 126 | - hmaster02 at [hmaster02:50070](http://hmaster02:50070) - standby namenode 127 | - Presto at [hmaster02:8081](http://hmaster02:8081) - Presto coordinator 128 | 129 | ### Performance testing 130 | 131 | Instructions on how to test the performance of your CDH4 cluster. 132 | 133 | - SSH into one of the machines. 134 | - Change to the `hdfs` user: `sudo su - hdfs` 135 | - Set HADOOP_MAPRED_HOME: `export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce` 136 | - `cd /usr/lib/hadoop-mapreduce` 137 | 138 | ##### TeraGen and TeraSort 139 | 140 | - `hadoop jar hadoop-mapreduce-examples.jar teragen -Dmapred.map.tasks=1000 10000000000 /tera/in` to run TeraGen 141 | - `hadoop jar hadoop-mapreduce-examples.jar terasort /tera/in /tera/out` to run TeraSort 142 | 143 | ##### DFSIO 144 | 145 | - `hadoop jar hadoop-mapreduce-client-jobclient-2.0.0-cdh4.6.0-tests.jar TestDFSIO -write` 146 | 147 | ### Bootstrapping 148 | 149 | Paste your public SSH RSA key in `bootstrap/ansible_rsa.pub` and run `bootstrap.sh` to bootstrap the nodes 150 | specified in `bootstrap/hosts`. See [`bootstrap/bootstrap.yml`](bootstrap/bootstrap.yml) for more information. 151 | 152 | ### What about Pig, Flume, etc? 153 | 154 | You can manually install additional components after running this playbook. Follow the 155 | official [CDH4 Installation Guide](http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH4-Installation-Guide/CDH4-Installation-Guide.html). 156 | 157 | ### Screenshots 158 | 159 | ![zookeeper](images/zookeeper.png) 160 | 161 | ![hmaster01](images/hmaster01.png) 162 | 163 | ![ganglia](images/ganglia.png) 164 | 165 | ![kibana](images/kibana.png) 166 | 167 | ![smokeping](images/smokeping.png) 168 | 169 | ### License 170 | 171 | Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). 172 | 173 | Copyright 2013-2014 [Mathias Bogaert](mailto:mathias.bogaert@gmail.com). -------------------------------------------------------------------------------- /roles/ganglia_monitor/templates/gmond.conf: -------------------------------------------------------------------------------- 1 | /* {{ ansible_managed }} */ 2 | 3 | globals { 4 | daemonize = yes 5 | setuid = yes 6 | user = ganglia 7 | debug_level = 0 8 | max_udp_msg_len = 1472 9 | mute = no 10 | deaf = no 11 | host_dmax = 86400 /* secs - expires (removes from web interface) hosts in 1 day */ 12 | cleanup_threshold = 300 /* secs */ 13 | gexec = no 14 | send_metadata_interval = 60 /* secs */ 15 | } 16 | 17 | /* If a cluster attribute is specified, then all gmond hosts are wrapped inside 18 | * of a tag. If you do not specify a cluster tag, then all will 19 | * NOT be wrapped inside of a tag. */ 20 | cluster { 21 | name = "{{ ganglia_cluster|default('Hadoop') }}" 22 | owner = "{{ site_name|capitalize }}" 23 | } 24 | 25 | /* The host section describes attributes of the host, like the location */ 26 | host { 27 | location = "Unknown" 28 | } 29 | 30 | /* Feel free to specify as many udp_send_channels as you like. Gmond 31 | used to only support having a single channel */ 32 | 33 | /* All nodes must have a udp_send_channel section. This tells gmond where to send the data it has collected about the 34 | local node (even if that is to itself). You can configure this to broadcast the information or send it to a particular 35 | host and port. If you specify a particular host, you probably want all nodes to send data to the same place. You can 36 | also have each node send the same information to more than one place for redundancy. */ 37 | 38 | udp_send_channel { 39 | host = {{ hostvars[groups["masters"][0]].ipv4_address|default(hostvars[groups["masters"][0]].ansible_default_ipv4.address) }} 40 | port = {{ ganglia_udp_send|default('8649') }} 41 | } 42 | 43 | udp_send_channel { 44 | host = {{ hostvars[groups["masters"][1]].ipv4_address|default(hostvars[groups["masters"][1]].ansible_default_ipv4.address) }} 45 | port = {{ ganglia_udp_accept|default('8649') }} 46 | } 47 | 48 | {% if ganglia_accept is defined %} 49 | /* At least one node must have a udp_recv_channel section. Data received by this section forms a snapshot of the state 50 | of all nodes. You can configure this to receive the data via broadcast or receive it on a particular IP interface and 51 | port. More than one node could be receiving the same data. You can use the 'deaf' keyword in the 'globals' section 52 | to disable this section, even if it is defined. */ 53 | /* You can specify as many udp_recv_channels as you like as well. */ 54 | udp_recv_channel { 55 | port = {{ ganglia_udp_recv|default('8649') }} 56 | } 57 | {% endif %} 58 | 59 | {% if ganglia_accept is defined %} 60 | /* For Ganglia to really be useful, at least one node which has udp_recv_channel defined must have a tcp_accept_channel 61 | section also. This section describes a particular IP interface and port where a query can be sent. Gmond will return 62 | an XML string of the summary information it has collected. This interface is the one gmetad will talk to. */ 63 | /* You can specify as many tcp_accept_channels as you like to share an xml description of the state of the cluster */ 64 | tcp_accept_channel { 65 | port = {{ ganglia_tcp_accept|default('8649') }} 66 | } 67 | {% endif %} 68 | 69 | /* Each metrics module that is referenced by gmond must be specified and 70 | loaded. If the module has been statically linked with gmond, it does not 71 | require a load path. However all dynamically loadable modules must include 72 | a load path. */ 73 | modules { 74 | module { 75 | name = "core_metrics" 76 | } 77 | module { 78 | name = "cpu_module" 79 | path = "/usr/lib/ganglia/modcpu.so" 80 | } 81 | module { 82 | name = "disk_module" 83 | path = "/usr/lib/ganglia/moddisk.so" 84 | } 85 | module { 86 | name = "load_module" 87 | path = "/usr/lib/ganglia/modload.so" 88 | } 89 | module { 90 | name = "mem_module" 91 | path = "/usr/lib/ganglia/modmem.so" 92 | } 93 | module { 94 | name = "net_module" 95 | path = "/usr/lib/ganglia/modnet.so" 96 | } 97 | module { 98 | name = "proc_module" 99 | path = "/usr/lib/ganglia/modproc.so" 100 | } 101 | module { 102 | name = "sys_module" 103 | path = "/usr/lib/ganglia/modsys.so" 104 | } 105 | } 106 | 107 | include ('/etc/ganglia/conf.d/*.conf') 108 | 109 | /* The old internal 2.5.x metric array has been replaced by the following 110 | collection_group directives. What follows is the default behavior for 111 | collecting and sending metrics that is as close to 2.5.x behavior as 112 | possible. */ 113 | 114 | /* This collection group will cause a heartbeat (or beacon) to be sent every 115 | 20 seconds. In the heartbeat is the GMOND_STARTED data which expresses 116 | the age of the running gmond. */ 117 | collection_group { 118 | collect_once = yes 119 | time_threshold = 20 120 | metric { 121 | name = "heartbeat" 122 | } 123 | } 124 | 125 | /* This collection group will send general info about this host every 1200 secs. 126 | This information doesn't change between reboots and is only collected once. */ 127 | collection_group { 128 | collect_once = yes 129 | time_threshold = 1200 130 | metric { 131 | name = "cpu_num" 132 | title = "CPU Count" 133 | } 134 | metric { 135 | name = "cpu_speed" 136 | title = "CPU Speed" 137 | } 138 | metric { 139 | name = "mem_total" 140 | title = "Memory Total" 141 | } 142 | /* Should this be here? Swap can be added/removed between reboots. */ 143 | metric { 144 | name = "swap_total" 145 | title = "Swap Space Total" 146 | } 147 | metric { 148 | name = "boottime" 149 | title = "Last Boot Time" 150 | } 151 | metric { 152 | name = "machine_type" 153 | title = "Machine Type" 154 | } 155 | metric { 156 | name = "os_name" 157 | title = "Operating System" 158 | } 159 | metric { 160 | name = "os_release" 161 | title = "Operating System Release" 162 | } 163 | metric { 164 | name = "location" 165 | title = "Location" 166 | } 167 | } 168 | 169 | /* This collection group will send the status of gexecd for this host every 300 secs */ 170 | /* Unlike 2.5.x the default behavior is to report gexecd OFF. */ 171 | collection_group { 172 | collect_once = yes 173 | time_threshold = 300 174 | metric { 175 | name = "gexec" 176 | title = "Gexec Status" 177 | } 178 | } 179 | 180 | /* This collection group will collect the CPU status info every 20 secs. 181 | The time threshold is set to 90 seconds. In honesty, this time_threshold could be 182 | set significantly higher to reduce unneccessary network chatter. */ 183 | collection_group { 184 | collect_every = 20 185 | time_threshold = 90 186 | /* CPU status */ 187 | metric { 188 | name = "cpu_user" 189 | value_threshold = "1.0" 190 | title = "CPU User" 191 | } 192 | metric { 193 | name = "cpu_system" 194 | value_threshold = "1.0" 195 | title = "CPU System" 196 | } 197 | metric { 198 | name = "cpu_idle" 199 | value_threshold = "5.0" 200 | title = "CPU Idle" 201 | } 202 | metric { 203 | name = "cpu_nice" 204 | value_threshold = "1.0" 205 | title = "CPU Nice" 206 | } 207 | metric { 208 | name = "cpu_aidle" 209 | value_threshold = "5.0" 210 | title = "CPU aidle" 211 | } 212 | metric { 213 | name = "cpu_wio" 214 | value_threshold = "1.0" 215 | title = "CPU wio" 216 | } 217 | metric { 218 | name = "cpu_intr" 219 | value_threshold = "1.0" 220 | title = "CPU intr" 221 | } 222 | metric { 223 | name = "cpu_sintr" 224 | value_threshold = "1.0" 225 | title = "CPU sintr" 226 | } 227 | } 228 | 229 | collection_group { 230 | collect_every = 20 231 | time_threshold = 90 232 | /* Load Averages */ 233 | metric { 234 | name = "load_one" 235 | value_threshold = "1.0" 236 | title = "One Minute Load Average" 237 | } 238 | metric { 239 | name = "load_five" 240 | value_threshold = "1.0" 241 | title = "Five Minute Load Average" 242 | } 243 | metric { 244 | name = "load_fifteen" 245 | value_threshold = "1.0" 246 | title = "Fifteen Minute Load Average" 247 | } 248 | } 249 | 250 | /* This group collects the number of running and total processes */ 251 | collection_group { 252 | collect_every = 80 253 | time_threshold = 950 254 | metric { 255 | name = "proc_run" 256 | value_threshold = "1.0" 257 | title = "Total Running Processes" 258 | } 259 | metric { 260 | name = "proc_total" 261 | value_threshold = "1.0" 262 | title = "Total Processes" 263 | } 264 | } 265 | 266 | /* This collection group grabs the volatile memory metrics every 40 secs and 267 | sends them at least every 180 secs. This time_threshold can be increased 268 | significantly to reduce unneeded network traffic. */ 269 | collection_group { 270 | collect_every = 40 271 | time_threshold = 180 272 | metric { 273 | name = "mem_free" 274 | value_threshold = "1024.0" 275 | title = "Free Memory" 276 | } 277 | metric { 278 | name = "mem_shared" 279 | value_threshold = "1024.0" 280 | title = "Shared Memory" 281 | } 282 | metric { 283 | name = "mem_buffers" 284 | value_threshold = "1024.0" 285 | title = "Memory Buffers" 286 | } 287 | metric { 288 | name = "mem_cached" 289 | value_threshold = "1024.0" 290 | title = "Cached Memory" 291 | } 292 | metric { 293 | name = "swap_free" 294 | value_threshold = "1024.0" 295 | title = "Free Swap Space" 296 | } 297 | } 298 | 299 | collection_group { 300 | collect_every = 40 301 | time_threshold = 300 302 | metric { 303 | name = "bytes_out" 304 | value_threshold = 4096 305 | title = "Bytes Sent" 306 | } 307 | metric { 308 | name = "bytes_in" 309 | value_threshold = 4096 310 | title = "Bytes Received" 311 | } 312 | metric { 313 | name = "pkts_in" 314 | value_threshold = 256 315 | title = "Packets Received" 316 | } 317 | metric { 318 | name = "pkts_out" 319 | value_threshold = 256 320 | title = "Packets Sent" 321 | } 322 | } 323 | 324 | collection_group { 325 | collect_every = 1800 326 | time_threshold = 3600 327 | metric { 328 | name = "disk_total" 329 | value_threshold = 1.0 330 | title = "Total Disk Space" 331 | } 332 | } 333 | 334 | collection_group { 335 | collect_every = 40 336 | time_threshold = 180 337 | metric { 338 | name = "disk_free" 339 | value_threshold = 1.0 340 | title = "Disk Space Available" 341 | } 342 | metric { 343 | name = "part_max_used" 344 | value_threshold = 1.0 345 | title = "Maximum Disk Space Used" 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------