├── .github └── stale.yml ├── .gitignore ├── INSTALL-CDH.md ├── INSTALL-ENV.md ├── INSTALL-HDP.md ├── README.md ├── add_keys.sh ├── ansible.cfg ├── bootstrap_kafka_rax.sh ├── bootstrap_kafka_static.sh ├── bootstrap_rax.sh ├── bootstrap_static.sh ├── cloudera_rax.sh ├── cloudera_rm_mngr.rax.sh ├── cloudera_static.sh ├── confluent_community_rax.sh ├── confluent_community_static.sh ├── get_facts.sh ├── get_site_facts.sh ├── hortonworks_rax.sh ├── hortonworks_static.sh ├── inventory ├── localhost ├── rax.py └── static ├── kube_rax.sh ├── playbooks ├── add_keys.yml ├── bootstrap.yml ├── bootstrap_kafka.yml ├── cloudera.yml ├── confluent-community.yml ├── create_groups.yml ├── create_kafka_groups.yml ├── group_vars │ ├── all │ ├── cloudera │ ├── confluent-community │ ├── edge-nodes │ ├── hortonworks │ ├── kafka-nodes │ ├── kafka-nodes-templates │ ├── kafka-server-nodes │ ├── kafka-zookeeper-nodes │ ├── kube-all.yml │ ├── kube-masters │ ├── kube-nodes │ ├── master-nodes │ ├── master-nodes-templates │ ├── postgres_vars.yml │ ├── slave-nodes │ └── slave-nodes-templates ├── hortonworks.yml ├── kube.yml ├── library │ ├── cloudera │ │ ├── cdh.py │ │ └── templates │ │ │ ├── analytic-database.yaml.j2 │ │ │ ├── basic.yaml.j2 │ │ │ ├── cluster-ha.yaml.j2 │ │ │ ├── cluster.yaml.j2 │ │ │ ├── data-engineering.yaml.j2 │ │ │ ├── enterprise-datahub.yaml.j2 │ │ │ └── operational-database.yaml.j2 │ └── site_facts │ │ ├── sitefacts.py │ │ └── updateconfigs.py ├── provision_cbs.yml ├── provision_rax.yml ├── remove_cloudera.yml ├── roles │ ├── ambari-agent │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── unbound-dns.yml │ │ ├── templates │ │ │ ├── redhat-unbound.j2 │ │ │ └── resolv.conf.j2 │ │ └── vars │ │ │ ├── debian.yml │ │ │ ├── redhat-amazon.yml │ │ │ └── redhat.yml │ ├── ambari-server │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── arcadia.yml │ │ │ ├── custom.yml │ │ │ ├── main.yml │ │ │ ├── multi-nodes.yml │ │ │ ├── prerequisites.yml │ │ │ ├── single-node.yml │ │ │ └── unbound-dns.yml │ │ └── templates │ │ │ ├── alert_targets.j2 │ │ │ ├── blueprint-custom.j2 │ │ │ ├── blueprint-multi-node-1-master.j2 │ │ │ ├── blueprint-multi-node-2-masters.j2 │ │ │ ├── blueprint-multi-node-3-masters.j2 │ │ │ ├── blueprint-single-node.j2 │ │ │ ├── blueprint-site-facts-2-masters.j2 │ │ │ ├── blueprint-site-facts-3-masters.j2 │ │ │ ├── cluster-template-custom.j2 │ │ │ ├── cluster-template-multi-nodes.j2 │ │ │ ├── cluster-template-single-node.j2 │ │ │ ├── hdprepo.j2 │ │ │ ├── redhat-unbound-master.j2 │ │ │ ├── redhat-unbound-zonefile.j2 │ │ │ ├── resolv.conf.j2 │ │ │ ├── spark-params.j2 │ │ │ ├── tachyon-master.j2 │ │ │ └── tachyon-params.j2 │ ├── cloudera-agent │ │ ├── files │ │ │ ├── cloudera-manager.repo │ │ │ ├── java_home.sh │ │ │ └── jce_policy-8.zip │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── jce_config.yml │ │ │ ├── jce_install.yml │ │ │ ├── jdk_install.yml │ │ │ ├── main.yml │ │ │ ├── mysql.yml │ │ │ └── unbound-dns.yml │ │ ├── templates │ │ │ ├── config.ini │ │ │ ├── redhat-unbound.j2 │ │ │ └── resolv.conf.j2 │ │ └── vars │ │ │ ├── debian.yml │ │ │ └── redhat.yml │ ├── cloudera-mngr │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ ├── prerequisites.yml │ │ │ └── unbound-dns.yml │ │ ├── templates │ │ │ ├── db.mgmt.properties │ │ │ ├── db.properties │ │ │ ├── redhat-unbound-master.j2 │ │ │ ├── redhat-unbound-zonefile.j2 │ │ │ └── resolv.conf.j2 │ │ └── vars │ │ │ ├── debian.yml │ │ │ ├── defaults.yml │ │ │ └── redhat.yml │ ├── cloudera-mysql │ │ ├── files │ │ │ ├── amon.sql │ │ │ ├── dbs.sql │ │ │ ├── hive-schema-1.1.0.mysql.sql │ │ │ ├── hive-txn-schema-0.13.0.mysql.sql │ │ │ ├── metastore.sql │ │ │ ├── nav.sql │ │ │ ├── navms.sql │ │ │ ├── oozie.sql │ │ │ ├── rman.sql │ │ │ ├── scm.sql │ │ │ ├── scm_prepare_database.sh │ │ │ ├── sentry.sql │ │ │ └── sqoop.sql │ │ ├── scripts │ │ │ └── hive-schema-mysql.sh │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── my.cnf-mariadb │ │ │ └── my.cnf-mysqld │ │ └── vars │ │ │ ├── mysql_vars.yml │ │ │ ├── redhat-6.yml │ │ │ └── redhat-7.yml │ ├── cloudera-postgres │ │ ├── files │ │ │ ├── amon.sql │ │ │ ├── dbs.sql │ │ │ ├── hive-schema-1.1.0.mysql.sql │ │ │ ├── hive-schema-3.1.2000.postgres.sql │ │ │ ├── hive-txn-schema-0.13.0.mysql.sql │ │ │ ├── metastore.sql │ │ │ ├── nav.sql │ │ │ ├── navms.sql │ │ │ ├── oozie.sql │ │ │ ├── postgresql-9.6.service │ │ │ ├── postgresql.conf │ │ │ ├── rman.sql │ │ │ ├── scm.sql │ │ │ ├── sentry.sql │ │ │ └── sqoop.sql │ │ ├── tasks │ │ │ ├── hive_postgres_schema.yml │ │ │ ├── main.yml │ │ │ ├── pg_dbs.yml │ │ │ ├── pg_hba.yml │ │ │ ├── pg_user.yml │ │ │ └── postgres_repo_install.yml │ │ └── vars │ │ │ ├── redhat-6.yml │ │ │ └── redhat-7.yml │ ├── cni │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── calico-etcd.yml.j2 │ │ │ ├── calico-etcd.yml.j2-bak │ │ │ ├── calico-rbac.yml.j2 │ │ │ ├── calico.yml.j2 │ │ │ ├── calico.yml.j2-bak │ │ │ ├── flannel-rbac.yml.j2 │ │ │ └── flannel.yml.j2 │ ├── common │ │ ├── files │ │ │ ├── isdct-3.0.2.400-17.x86_64.rpm │ │ │ └── tuned.conf │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── bonding.yml │ │ │ ├── cachedisks.yml │ │ │ ├── firewall.yml │ │ │ ├── main.yml │ │ │ ├── partitioning.yml │ │ │ └── slavetest.yml │ │ ├── templates │ │ │ ├── debian-ifcfg-bond.j2 │ │ │ ├── debian-ifcfg-eth.j2 │ │ │ ├── hosts.j2 │ │ │ ├── kafka_hosts.j2 │ │ │ ├── mdadm.conf.j2 │ │ │ ├── redhat-ifcfg-bond.j2 │ │ │ ├── redhat-ifcfg-eth.j2 │ │ │ ├── redhat-unbound-master.j2 │ │ │ ├── redhat-unbound-zonefile.j2 │ │ │ ├── redhat-unbound.j2 │ │ │ ├── resolv.conf.j2 │ │ │ └── sparkstack-repo.j2 │ │ └── vars │ │ │ ├── debian.yml │ │ │ ├── redhat-6.yml │ │ │ ├── redhat-7.yml │ │ │ └── redhat-amazon.yml │ ├── confluent-community-broker │ │ ├── files │ │ │ ├── confluent-kafka.service │ │ │ ├── prometheus.service │ │ │ └── prometheus.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── prometheus-install.yml │ │ └── templates │ │ │ ├── consumer.properties │ │ │ ├── kafka-rest.properties │ │ │ ├── ksql-server.properties │ │ │ ├── producer.properties │ │ │ ├── schema-registry.properties │ │ │ ├── server.properties │ │ │ └── zookeeper.properties │ ├── confluent-community-common │ │ ├── files │ │ │ ├── confluent.repo │ │ │ ├── java_home.sh │ │ │ └── tuned.conf │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── bonding.yml │ │ │ ├── firewall.yml │ │ │ ├── main.yml │ │ │ └── partitioning.yml │ │ ├── templates │ │ │ ├── debian-ifcfg-bond.j2 │ │ │ ├── debian-ifcfg-eth.j2 │ │ │ ├── hosts.j2 │ │ │ ├── mdadm.conf.j2 │ │ │ ├── redhat-ifcfg-bond.j2 │ │ │ ├── redhat-ifcfg-eth.j2 │ │ │ ├── redhat-unbound-master.j2 │ │ │ ├── redhat-unbound-zonefile.j2 │ │ │ ├── redhat-unbound.j2 │ │ │ └── resolv.conf.j2 │ │ └── vars │ │ │ └── redhat-7.yml │ ├── confluent-community-mngr │ │ ├── files │ │ │ └── prometheus.service │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── prometheus-install.yml │ │ └── templates │ │ │ ├── kafka.alerts.yml │ │ │ └── prometheus.yml │ ├── confluent-community-zookeeper │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── myid.j2 │ │ │ └── zookeeper.properties │ ├── docker │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── pkg.yml │ │ └── templates │ │ │ ├── docker.j2 │ │ │ └── docker.service.j2 │ ├── healthcheck │ │ ├── tasks │ │ │ └── main.yml │ │ └── vars │ │ │ └── main.yml │ ├── helm │ │ ├── files │ │ │ └── rbac-config.yml │ │ └── tasks │ │ │ └── main.yml │ ├── kube-commons │ │ ├── os-checker │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ └── main.yml │ │ └── pre-install │ │ │ ├── meta │ │ │ └── main.yml │ │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── pkg.yml │ │ │ └── templates │ │ │ └── 20-extra-args.conf.j2 │ ├── master-nodes │ │ └── tasks │ │ │ └── main.yml │ ├── metallb │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ └── metallb-layer-2-config.yml.j2 │ │ └── vars │ │ │ └── main.yml │ └── templates │ │ ├── calico-etcd.yml.j2 │ │ ├── calico-etcd.yml.j2-bak │ │ ├── calico-rbac.yml.j2 │ │ ├── calico.yml.j2 │ │ ├── calico.yml.j2-bak │ │ ├── flannel-rbac.yml.j2 │ │ └── flannel.yml.j2 ├── site_facts.yml └── spark-stack.yml └── provision_rax.sh /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 30 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - security 8 | # Label to use when marking an issue as stale 9 | staleLabel: stale 10 | # Comment to post when marking an issue as stale. Set to `false` to disable 11 | markComment: > 12 | This issue has been automatically marked as stale because it has not had 13 | recent activity. It will be closed if no further activity occurs. Thank you 14 | for your contributions. 15 | # Comment to post when closing a stale issue. Set to `false` to disable 16 | closeComment: false 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](http://104.130.29.12/buildStatus/icon?job=ansible-hadoop)](http://104.130.29.12/job/ansible-hadoop/) 2 | 3 | ansible-hadoop 4 | --------- 5 | These Ansible playbooks will build a Hadoop cluster. 6 | 7 | You can pre-build a Rackspace cloud environment or run the playbooks against an existing environment. 8 | 9 | --- 10 | 11 | ## [Installation] (id:installation) 12 | 13 | See [INSTALL-ENV.md](../master/INSTALL-ENV.md) for installation and build instructions. 14 | 15 | 16 | ## [Requirements] (id:requirements) 17 | 18 | - Ansible == 2.1.3.0 (2.2 is not supported at the moment) 19 | 20 | - Expects RHEL/CentOS 6/7 or Ubuntu 14 hosts. 21 | 22 | - Building the Rackspace Cloud environment requires the `pyrax` Python module: [pyrax link](https://github.com/rackspace/pyrax). 23 | 24 | 25 | ## [Features] (id:features) 26 | 27 | - It installs Hortonworks Data Platform using [Ambari Blueprints](https://cwiki.apache.org/confluence/display/AMBARI/Blueprints). 28 | 29 | - It supports static inventory if the environment is pre-built (`inventory/static` file). 30 | 31 | - The data drives can be customized and can be put on top of Cloud Block Storage (partitioning is automatic). 32 | 33 | - If there are 2 or 3 masternodes, it will also enable HA NameNode. 34 | 35 | - Memory settings are scaled with the hardware configuration of the nodes. 36 | 37 | 38 | ## [Inventory] (id:inventory) 39 | 40 | - The cloud environment requires the standard `pyrax` credentials file that looks like this: 41 | ```` 42 | [rackspace_cloud] 43 | username = my_username 44 | api_key = 01234567890abcdef 45 | ```` 46 | 47 | This file will be referenced in `playbooks/group_vars/all` (the `rax_credentials_file` variable). 48 | 49 | By default, the file is expected to be: `~/.raxpub`. 50 | 51 | - When provisioning HDP on existing infrastructure edit `inventory/static` and add the nodes. 52 | 53 | 54 | ## [Configuration files] (id:configuration) 55 | 56 | To customize, change the variables under `playbooks/group_vars` folder: 57 | 58 | 1. **`playbooks/group_vars/all`**: contains global cluster and cloud settings 59 | 1. **`playbooks/group_vars/master-nodes`**: master-nodes configuration 60 | 1. **`playbooks/group_vars/slave-nodes`**: slave-nodes configuration 61 | 1. **`playbooks/group_vars/edge-nodes`**: edge-nodes configuration 62 | 63 | For a one-node cluster, set `cloud_nodes_count` in master-nodes to 1 and `cloud_nodes_count` in slave-nodes to 0. 64 | 65 | 66 | ## [Scripts] (id:scripts) 67 | 68 | ###`provision_rax.sh` 69 | 70 | To provision a cloud environment, run the `provision_rax.sh` script after you've customized the variables under `playbooks/group_vars`: 71 | ```` 72 | bash provision_rax.sh 73 | ```` 74 | 75 | Continue with the HDP deployment steps here : 76 | [HDP Install](../master/INSTALL-HDP.md) 77 | -------------------------------------------------------------------------------- /add_keys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | 6 | ansible-playbook -vvv -i inventory/rax.py playbooks/add_keys.yml 7 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | timeout = 60 4 | ansible_keep_remote_files = True 5 | library = playbooks/library/cloudera:playbooks/library/site_facts 6 | #callback_plugins = playbooks/library/human_log/ 7 | -------------------------------------------------------------------------------- /bootstrap_kafka_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -i inventory/rax.py playbooks/bootstrap_kafka.yml 9 | -------------------------------------------------------------------------------- /bootstrap_kafka_static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -i inventory/static playbooks/bootstrap_kafka.yml 9 | -------------------------------------------------------------------------------- /bootstrap_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -i inventory/rax.py playbooks/bootstrap.yml 9 | -------------------------------------------------------------------------------- /bootstrap_static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 4 | 5 | export $VARS 6 | ansible-playbook -f 20 -i inventory/static playbooks/bootstrap.yml 7 | -------------------------------------------------------------------------------- /cloudera_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | 6 | # ansible-playbook -vvv -i inventory/rax.py playbooks/cloudera.yml --tags cluster_deploy 7 | ansible-playbook -vvv -i inventory/rax.py playbooks/cloudera.yml 8 | -------------------------------------------------------------------------------- /cloudera_rm_mngr.rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | 6 | ansible-playbook -vvv -i inventory/rax.py playbooks/remove_cloudera.yml 7 | -------------------------------------------------------------------------------- /cloudera_static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ansible-playbook -f 20 -i inventory/static playbooks/cloudera.yml 4 | -------------------------------------------------------------------------------- /confluent_community_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -vvv -i inventory/rax.py playbooks/confluent-community.yml 9 | -------------------------------------------------------------------------------- /confluent_community_static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -vvv -i inventory/static playbooks/confluent-community.yml 9 | -------------------------------------------------------------------------------- /get_facts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | 6 | ansible-playbook -vvv -i inventory/rax.py playbooks/get_facts.yml 7 | -------------------------------------------------------------------------------- /get_site_facts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | 6 | ansible-playbook -vvv -i inventory/rax.py playbooks/site_facts.yml --extra-vars="debug=true, compare=true" 7 | -------------------------------------------------------------------------------- /hortonworks_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -i inventory/rax.py playbooks/hortonworks.yml 9 | -------------------------------------------------------------------------------- /hortonworks_static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 4 | 5 | export $VARS 6 | ansible-playbook -f 20 -i inventory/static playbooks/hortonworks.yml 7 | -------------------------------------------------------------------------------- /inventory/localhost: -------------------------------------------------------------------------------- 1 | [localhost] 2 | localhost ansible_connection=local -------------------------------------------------------------------------------- /inventory/static: -------------------------------------------------------------------------------- 1 | [master-nodes] 2 | master01 ansible_host=192.168.0.2 bond_ip=172.16.0.2 ansible_user=rack ansible_ssh_pass=changeme 3 | #master02 ansible_host=192.168.0.2 bond_ip=172.16.0.2 ansible_user=root ansible_ssh_pass=changeme 4 | 5 | [slave-nodes] 6 | slave01 ansible_host=192.168.0.3 bond_ip=172.16.0.3 ansible_user=rack ansible_ssh_pass=changeme 7 | slave02 ansible_host=192.168.0.4 bond_ip=172.16.0.4 ansible_user=rack ansible_ssh_pass=changeme 8 | 9 | [edge-nodes] 10 | #edge01 ansible_host=192.168.0.5 bond_ip=172.16.0.5 ansible_user=rack ansible_ssh_pass=changeme 11 | -------------------------------------------------------------------------------- /kube_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export RAX_CREDS_FILE=$(grep rax_credentials_file playbooks/group_vars/all|cut -d"'" -f2) 4 | export RAX_REGION=$(grep rax_region playbooks/group_vars/all|cut -d"'" -f2) 5 | VARS="${VARS} ANSIBLE_SCP_IF_SSH=y ANSIBLE_HOST_KEY_CHECKING=False" 6 | 7 | export $VARS 8 | ansible-playbook -f 20 -vvv -i inventory/rax.py playbooks/kube.yml 9 | -------------------------------------------------------------------------------- /playbooks/add_keys.yml: -------------------------------------------------------------------------------- 1 | - include: create_groups.yml 2 | 3 | - name: "add keys to the cluster" 4 | hosts: hadoop-cluster 5 | any_errors_fatal: true 6 | become: yes 7 | pre_tasks: 8 | - name: "Show hadoop-cluster info" 9 | debug: var="{{ hostvars[inventory_hostname] }}" 10 | when: debug 11 | tasks: 12 | - name: "add key" 13 | authorized_key: user=root key="{{ lookup('file', 'keys/some-key') }}" 14 | -------------------------------------------------------------------------------- /playbooks/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_groups.yml 3 | 4 | - name: Apply the common role to all nodes 5 | hosts: hadoop-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | pre_tasks: 9 | - name: Show hadoop-cluster info 10 | debug: var=hostvars[inventory_hostname] 11 | when: debug 12 | roles: 13 | - common 14 | -------------------------------------------------------------------------------- /playbooks/bootstrap_kafka.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_kafka_groups.yml 3 | 4 | - name: Apply the common role to all nodes 5 | hosts: kafka-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | pre_tasks: 9 | - name: Show hadoop-cluster info 10 | debug: var=hostvars[inventory_hostname] 11 | when: debug 12 | roles: 13 | - common 14 | -------------------------------------------------------------------------------- /playbooks/cloudera.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_groups.yml 3 | tags: 4 | - always 5 | 6 | - name: "Apply the ambari-agent role to all nodes" 7 | hosts: hadoop-cluster 8 | any_errors_fatal: true 9 | become: yes 10 | pre_tasks: 11 | - name: "Show hadoop-cluster info" 12 | debug: var="{{ hostvars[inventory_hostname] }}" 13 | when: debug 14 | roles: 15 | - cloudera-agent 16 | 17 | - name: "setup postgres on cloudera master nodes" 18 | hosts: master-nodes 19 | any_errors_fatal: true 20 | ## become: yes 21 | pre_tasks: 22 | - name: "Show cluster info" 23 | debug: var="{{ hostvars[inventory_hostname] }}" 24 | when: debug 25 | roles: 26 | - cloudera-postgres 27 | 28 | - name: "Apply the cloudera-mngr role to cm_node group" 29 | hosts: cm_node 30 | become: yes 31 | pre_tasks: 32 | - name: "Show cluster info" 33 | debug: var="{{ hostvars[inventory_hostname] }}" 34 | when: debug 35 | roles: 36 | - cloudera-mngr 37 | 38 | - name: Build a Cloudera cluster 39 | gather_facts: True 40 | hosts: cm_node 41 | become: yes 42 | pre_tasks: 43 | - name: Install python-pip (yum) 44 | yum: name='python-pip' state=installed 45 | when: ansible_os_family == "RedHat" 46 | tags: 47 | - cluster_deploy 48 | 49 | - name: Install python-pip (apt) 50 | apt: name='python-pip' state=installed 51 | when: ansible_os_family == "Debian" 52 | tags: 53 | - cluster_deploy 54 | 55 | - name: Install cm_api 56 | pip: name='cm_api' state=latest 57 | tags: 58 | - cluster_deploy 59 | 60 | - name: Install pyyaml 61 | pip: name='pyyaml' state=latest 62 | tags: 63 | - cluster_deploy 64 | 65 | - include_vars: group_vars/postgres_vars.yml 66 | tags: 67 | - cluster_deploy 68 | 69 | - include_vars: group_vars/cloudera 70 | tags: 71 | - cluster_deploy 72 | 73 | - debug: var=vars 74 | 75 | - name: Copy cluster template 76 | template: src=library/cloudera/{{ cluster_template_file }} dest=/opt/cluster.yaml 77 | tags: 78 | - cluster_deploy 79 | 80 | - include_vars: group_vars/postgres_vars.yml 81 | tags: 82 | - cluster_deploy 83 | 84 | - include_vars: group_vars/cloudera 85 | 86 | 87 | tasks: 88 | - name: Cloudera cluster create request 89 | action: 90 | module: cdh.py 91 | trial: true 92 | register: my_cdh 93 | tags: 94 | - cluster_deploy 95 | -------------------------------------------------------------------------------- /playbooks/confluent-community.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_kafka_groups.yml 3 | 4 | - name: Apply the common role to all nodes 5 | hosts: kafka-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | pre_tasks: 9 | - name: Show kafka-cluster info 10 | debug: var=hostvars[inventory_hostname] 11 | when: debug 12 | 13 | - name: include confluent community vars 14 | include_vars: group_vars/confluent-community 15 | 16 | roles: 17 | - confluent-community-common 18 | 19 | - name: Apply the zookeeper role to all zk nodes 20 | hosts: kafka-zookeeper-cluster 21 | any_errors_fatal: true 22 | become: yes 23 | pre_tasks: 24 | - name: Show kafka-zookeeper-cluster info 25 | debug: var=hostvars[inventory_hostname] 26 | when: debug 27 | 28 | - name: include confluent-community vars 29 | include_vars: group_vars/confluent-community 30 | 31 | roles: 32 | - confluent-community-zookeeper 33 | 34 | - name: Apply the broker role to all broker nodes 35 | hosts: kafka-broker-cluster 36 | any_errors_fatal: true 37 | become: yes 38 | pre_tasks: 39 | - name: Show hadoop-broker-cluster info 40 | debug: var=hostvars[inventory_hostname] 41 | when: debug 42 | 43 | - name: include confluent community vars 44 | include_vars: group_vars/confluent-community 45 | roles: 46 | - confluent-community-broker 47 | 48 | #- name: "generate site facts" 49 | # hosts: localhost 50 | # any_errors_fatal: true 51 | # become: no 52 | # dnmemory: "{{ hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] / 1024 }}" 53 | # mnmemory: "{{ hostvars[groups['master-nodes'][0]]['ansible_memtotal_mb'] / 1024 }}" 54 | # cores: "{{ hostvars[groups['slave-nodes'][0]]['ansible_processor_count'] }}" 55 | # tasks: 56 | # - name: "gather site facts" 57 | # action: 58 | # module: confluentsitefacts.py 59 | # dnmemory="{{ dnmemory }}" 60 | # mnmemory="{{ mnmemory }}" 61 | # cores="{{ cores }}" 62 | # manager_server="localhost" 63 | # ambari_pass="admin" 64 | # cluster_name="{{ cluster_name }}" 65 | # compare="false" 66 | # current_facts="false" 67 | # 68 | - name: Apply the confluent-community manager role to manager node group 69 | hosts: kafka-manager 70 | become: yes 71 | pre_tasks: 72 | - name: include confluent community vars 73 | include_vars: group_vars/confluent-community 74 | roles: 75 | - confluent-community-mngr 76 | -------------------------------------------------------------------------------- /playbooks/create_groups.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add nodes to required groups 3 | hosts: localhost 4 | connection: local 5 | gather_facts: False 6 | tasks: 7 | - name: Add all cluster nodes to the hadoop-cluster group 8 | check_mode: no 9 | add_host: 10 | name: "{{ hostvars[item].inventory_hostname }}" 11 | ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}" 12 | ansible_user: "{{ hostvars[item].ansible_user|default('root') }}" 13 | ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}" 14 | ansible_become_user: root 15 | ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}" 16 | groups: hadoop-cluster 17 | with_flattened: 18 | - "{{ groups['master-nodes']|default([]) }}" 19 | - "{{ groups['slave-nodes']|default([]) }}" 20 | - "{{ groups['edge-nodes']|default([]) }}" 21 | register: hadoop-cluster 22 | when: "'hadoop-cluster' not in groups or groups['hadoop-cluster']|length < 1" 23 | 24 | - name: debugging some vars 25 | debug: 26 | var: groups['hadoop-cluster'] 27 | 28 | - name: "include cdh vars" 29 | include_vars: group_vars/cloudera 30 | when: distro == "cdh" 31 | 32 | - name: "include hdp vars" 33 | include_vars: group_vars/hortonworks 34 | when: distro == "hdp" 35 | 36 | - name: Add the last masternode to ambari-node variable group 37 | check_mode: no 38 | add_host: 39 | name: "{{ hostvars[item].inventory_hostname }}" 40 | ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}" 41 | ansible_user: "{{ hostvars[item].ansible_user|default('root') }}" 42 | ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}" 43 | ansible_become_user: root 44 | ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}" 45 | groups: ambari-node 46 | with_items: "{{ groups['master-nodes']|sort|last }}" 47 | register: ambari-node 48 | when: 49 | - distro == "hdp" 50 | - "'ambari-node' not in groups or groups['ambari-node']|length < 1" 51 | 52 | - name: Add the last masternode to cm_node variable group 53 | check_mode: no 54 | add_host: 55 | name: "{{ hostvars[item].inventory_hostname }}" 56 | ansible_host: "{{ hostvars[item].ansible_host|default(hostvars[item].ansible_ssh_host) }}" 57 | ansible_user: "{{ hostvars[item].ansible_user|default('root') }}" 58 | ansible_ssh_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}" 59 | ansible_become_user: root 60 | ansible_become_pass: "{{ hostvars[item].ansible_ssh_pass|default('') }}" 61 | groups: cm_node 62 | with_items: "{{ groups['master-nodes']|sort|last }}" 63 | register: cm_node 64 | when: 65 | - distro == "cdh" 66 | - "'cm_node' not in groups or groups['cm_node']|length < 1" 67 | 68 | 69 | -------------------------------------------------------------------------------- /playbooks/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | #valid hdp/cdh/cc 3 | distro: 'cdh' 4 | use_dns: false 5 | dnsfwd: ['173.203.4.8', '173.203.4.9'] 6 | dnsres: ['173.203.4.8', '173.203.4.9'] 7 | 8 | cloud_config: 9 | rax_credentials_file: '~/.raxpub' 10 | rax_region: 'DFW' 11 | domain: 'localnet' 12 | allowed_external_ips: ['127.0.0.1'] 13 | ssh: 14 | keyname: 'hadoop-ssh-key' 15 | keyfile: '~/.ssh/id_rsa.pub' 16 | 17 | # If deployment is in Microsoft Azure 18 | # This assumes that all the vm deployments are done thru Azure tools outside 19 | # of this ansible setup and the bootstrap and Hortonworks setup are done using 20 | # ansible-hadoop. 21 | # NOTE: This skips a few steps in bootstrap that are not need by Azure VM's 22 | azure: false 23 | 24 | # set to true to show host variables 25 | debug: false 26 | -------------------------------------------------------------------------------- /playbooks/group_vars/cloudera: -------------------------------------------------------------------------------- 1 | --- 2 | cluster_name: 'hadoop-poc' 3 | cloudera_version: '7' 4 | full_version: '7.0.3' 5 | custom_repo: false 6 | 7 | adminnode: 'cm_node' 8 | 9 | #templates 10 | #analytic-database.yaml.j2 basic.yaml.j2 cluster-ha.yaml.j2 data-engineering.yaml.j2 enterprise-datahub.yaml.j2 operational-database.yaml.j2 11 | 12 | scm_repo_url: "http://clouder_url/cloudera-repos/cm7/" 13 | scm_repo_gpgkey: "http://clouder_url/cloudera-repos/cm7/RPM-GPG-KEY-cloudera" 14 | 15 | scm_repo_user: "" 16 | scm_repo_pass: "" 17 | 18 | parcel_repo_url: "http://clouder_url/cloudera-repos/p/cdh7/7.0.3.0/parcels/" 19 | cdh_parcel: "CDH-7.0.3-1.cdh7.0.3.p0.1635019-el7.parcel" 20 | cdh_parcel_sha: "CDH-7.0.3-1.cdh7.0.3.p0.1635019-el7.parcel.sha" 21 | 22 | pdgd_repo_rpm_url: "https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/" 23 | pdgd_repo_rpm: "pgdg-redhat-repo-latest.noarch.rpm" 24 | 25 | admin_password: 'admin' 26 | services_pass: '' 27 | tough_pass: '' 28 | alerts_contact: 'root@localhost.localdomain' 29 | data_disks_filesystem: xfs 30 | configure_firewall: true 31 | cluster_template_file: 'templates/operational-database.yaml.j2' 32 | 33 | hdfs: 34 | dfs_replication: 3 35 | failed_volumes_tolerated: 1 36 | 37 | # set to true to show host variables 38 | debug: true 39 | -------------------------------------------------------------------------------- /playbooks/group_vars/confluent-community: -------------------------------------------------------------------------------- 1 | --- 2 | cluster_name: 'KafkaPoc' 3 | managernode: 'manager-node' 4 | confluent_community_version: '3.1' 5 | admin_password: 'admin' 6 | services_password: 'AsdQwe123' 7 | alerts_contact: 'root@localhost.localdomain' 8 | wait: true 9 | wait_timeout: 1800 # 30 minutes 10 | 11 | data_disks_filesystem: xfs 12 | configure_firewall: false 13 | custom_blueprint: false 14 | custom_repo: false 15 | custom_repo_url: '' 16 | 17 | zookeeper_starting_id: 1 18 | 19 | broker_starting_myid: 1 20 | 21 | prometheus_url: "https://github.com/prometheus/prometheus/releases/download/v2.16.0/" 22 | prometheus_file: "prometheus-2.16.0.linux-amd64.tar.gz" 23 | prometheus_jmx_url: "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar" 24 | prometheus_kafka_yaml: "https://github.com/prometheus/jmx_exporter/raw/master/example_configs/kafka-2_0_0.yml" 25 | prometheus_dir: "/opt/prometheus" 26 | 27 | alertmanager: "127.0.0.1" 28 | 29 | tmp_dir: "/tmp" 30 | 31 | jolokia_enabled: true 32 | jmxexporter_enabled: true 33 | 34 | # set to true to show host variables 35 | debug: false 36 | -------------------------------------------------------------------------------- /playbooks/group_vars/edge-nodes: -------------------------------------------------------------------------------- 1 | --- 2 | ######################################### 3 | ## example for Rackspace cloud servers ## 4 | ## general1-2 flavor and CentOS 7 ## 5 | ## root filesystem used for /hadoop ## 6 | ## using the default public network ## 7 | ######################################### 8 | 9 | cloud_nodes_count: 0 10 | cloud_image: 'CentOS 7 (PVHVM)' 11 | # cloud_image: 'CentOS 6 (PVHVM)' 12 | cloud_flavor: 'general1-2' 13 | 14 | 15 | ############################################# 16 | ## example for Rackspace cloud servers ## 17 | ## performance2-15 flavor and Ubuntu 14 ## 18 | ## ephemeral disk used for /hadoop ## 19 | ## using ServiceNet as the cluster network ## 20 | ############################################# 21 | 22 | # cluster_interface: 'eth1' 23 | # cloud_nodes_count: 1 24 | # cloud_image: 'Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)' 25 | # cloud_flavor: 'performance2-15' 26 | # hadoop_disk: xvde 27 | 28 | 29 | ###################################### 30 | ## example for Rackspace OnMetal v2 ## 31 | ###################################### 32 | 33 | # cloud_nodes_count: 2 34 | # cloud_image: 'OnMetal - CentOS 7' 35 | ## cloud_image: 'OnMetal - Ubuntu 14.04 LTS (Trusty Tahr)' 36 | # cloud_flavor: 'onmetal-general2-small' 37 | 38 | 39 | ################################## 40 | ## example for static inventory ## 41 | ################################## 42 | 43 | # cluster_interface: 'bond1' 44 | # bond_interfaces: ['eth4', 'eth6'] 45 | # bond_netmask: '255.255.255.0' 46 | # hadoop_disk: sdb 47 | -------------------------------------------------------------------------------- /playbooks/group_vars/hortonworks: -------------------------------------------------------------------------------- 1 | --- 2 | cluster_name: 'hadoop-poc' 3 | adminnode: 'ambari-node' 4 | hdp_version: '2.5' 5 | ambari_version: '2.4.2.0' 6 | admin_password: 'admin' 7 | services_password: 'AsdQwe123' 8 | alerts_contact: 'root@localhost.localdomain' 9 | wait: true 10 | wait_timeout: 1800 # 30 minutes 11 | 12 | install_spark: false # <= HDP 2.6 13 | install_zeppelin: false # >= HDP 2.5 14 | install_flume: true 15 | install_hbase: true 16 | install_storm: true 17 | install_kafka: true 18 | install_falcon: false 19 | tachyon_service: false 20 | 21 | data_disks_filesystem: xfs 22 | configure_firewall: false 23 | custom_blueprint: false 24 | custom_blueprint_template: blueprint-custom.j2 25 | custom_cluster_template: cluster-template-custom.j2 26 | custom_repo: false 27 | custom_repo_url: 'http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/2.5.0.0-7/centos7/' 28 | custom_repo_target: 'api/v1/stacks/HDP/versions/2.5/operating_systems/redhat7/repositories/HDP-2.5' 29 | 30 | #requires HDP2.3 RHEL/CentOS 6 31 | #available in 2 and 3 masternode blueprints 32 | arcadia: false 33 | arcadia_config: 34 | archive_file: ARCADIA-ENTERPRISE-3.1.0.0_1464326436-1.tar.gz 35 | archive_src: 'http://get.arcadiadata.com' 36 | archive_dest: '/var/lib/ambari-server/resources/stacks/HDP/2.4/services/' 37 | hdfs-site: 'dfs.datanode.hdfs-blocks-metadata.enabled' 38 | 39 | hdfs: 40 | dfs_replication: 3 41 | failed_volumes_tolerated: 1 42 | ha_namenode: true 43 | 44 | # set to true to show host variables 45 | debug: false 46 | -------------------------------------------------------------------------------- /playbooks/group_vars/kafka-nodes: -------------------------------------------------------------------------------- 1 | ############################################################### 2 | # use template file for example references # 3 | # Default Rackspace kafka server node 4 | ############################################################### 5 | cluster_interface: 'eth0' 6 | cloud_nodes_count: 0 7 | cloud_image: 'CentOS 7 (PVHVM)' 8 | # cloud_image: 'CentOS 6 (PVHVM)' 9 | cloud_flavor: 'performance2-15' 10 | build_kafka_cbs: true 11 | cbs_disks_size: 200 12 | cbs_disks_type: 'SATA' 13 | hadoop_disk: xvde 14 | datanode_disks: ['xvdf', 'xvdg'] 15 | -------------------------------------------------------------------------------- /playbooks/group_vars/kafka-server-nodes: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################### 3 | # use template file for example references # 4 | # ~/ansible-hadoop/playbooks/group_vars/master-nodes-templates# 5 | ############################################################### 6 | cluster_interface: 'eth0' 7 | cloud_nodes_count: 3 8 | cloud_image: 'CentOS 7 (PVHVM)' 9 | # cloud_image: 'CentOS 6 (PVHVM)' 10 | cloud_flavor: 'performance1-8' 11 | build_kafka_cbs: true 12 | cbs_disks_size: 200 13 | cbs_disks_type: 'SATA' 14 | kafka_disk: xvdf 15 | kafka-streams: xvdg 16 | #datanode_disks: ['xvdf', 'xvdg'] 17 | -------------------------------------------------------------------------------- /playbooks/group_vars/kafka-zookeeper-nodes: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################### 3 | # use template file for example references # 4 | # ~/ansible-hadoop/playbooks/group_vars/master-nodes-templates# 5 | ############################################################### 6 | cluster_interface: 'eth0' 7 | cloud_nodes_count: 3 8 | cloud_image: 'CentOS 7 (PVHVM)' 9 | # cloud_image: 'CentOS 6 (PVHVM)' 10 | cloud_flavor: 'performance1-4' 11 | build_kafka_cbs: true 12 | cbs_disks_size: 100 13 | cbs_disks_type: 'SATA' 14 | zk_disk: xvdf 15 | #datanode_disks: ['xvdf', 'xvdg'] 16 | -------------------------------------------------------------------------------- /playbooks/group_vars/kube-all.yml: -------------------------------------------------------------------------------- 1 | # Ansible 2 | # ansible_user: root 3 | 4 | # Kubernetes 5 | kube_version: v1.15.0 6 | token: b0f7b8.8d1767876297d85c 7 | 8 | # 1.8.x feature: --feature-gates SelfHosting=true 9 | init_opts: "" 10 | 11 | # Any other additional opts you want to add.. 12 | kubeadm_opts: "" 13 | # For example: 14 | # kubeadm_opts: '--apiserver-cert-extra-sans "k8s.domain.com,kubernetes.domain.com"' 15 | 16 | service_cidr: "10.96.0.0/12" 17 | pod_network_cidr: "10.244.0.0/16" 18 | 19 | #calico_etcd_service: "10.96.232.136" 20 | 21 | # Network implementation('flannel', 'calico') 22 | network: flannel 23 | 24 | # Change this to an appropriate interface, preferably a private network. 25 | # For example, on DigitalOcean, you would use eth1 as that is the default private network interface. 26 | network_interface: "eth0" 27 | 28 | enable_dashboard: yes 29 | 30 | # A list of insecure registries you might need to define 31 | # insecure_registries: [] 32 | insecure_registries: ['gcr.io'] 33 | 34 | systemd_dir: /lib/systemd/system 35 | system_env_dir: /etc/sysconfig 36 | network_dir: /etc/kubernetes/network 37 | kubeadmin_config: /etc/kubernetes/admin.conf 38 | kube_addon_dir: /etc/kubernetes/addon 39 | 40 | # Additional feature to install 41 | additional_features: 42 | helm: false 43 | metallb: false 44 | healthcheck: false 45 | 46 | # temporary directory used by additional features 47 | tmp_dir: /tmp/kubeadm-ansible-files 48 | 49 | -------------------------------------------------------------------------------- /playbooks/group_vars/kube-masters: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################### 3 | # use template file for example references # 4 | # ~/ansible-hadoop/playbooks/group_vars/master-nodes-templates# 5 | ############################################################### 6 | cluster_interface: 'eth0' 7 | cloud_nodes_count: 0 8 | cloud_image: 'CentOS 7 (PVHVM)' 9 | cloud_flavor: 'performance2-15' 10 | build_datanode_cbs: true 11 | cbs_disks_size: 200 12 | cbs_disks_type: 'SATA' 13 | hadoop_disk: xvde 14 | datanode_disks: ['xvdf', 'xvdg'] 15 | 16 | -------------------------------------------------------------------------------- /playbooks/group_vars/kube-nodes: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################### 3 | # use template file for example references # 4 | # ~/ansible-hadoop/playbooks/group_vars/master-nodes-templates# 5 | ############################################################### 6 | cluster_interface: 'eth0' 7 | cloud_nodes_count: 0 8 | cloud_image: 'CentOS 7 (PVHVM)' 9 | cloud_flavor: 'performance2-15' 10 | build_datanode_cbs: true 11 | cbs_disks_size: 200 12 | cbs_disks_type: 'SATA' 13 | hadoop_disk: xvde 14 | datanode_disks: ['xvdf', 'xvdg'] 15 | 16 | -------------------------------------------------------------------------------- /playbooks/group_vars/master-nodes: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################### 3 | # use template file for example references # 4 | # ~/ansible-hadoop/playbooks/group_vars/master-nodes-templates# 5 | ############################################################### 6 | -------------------------------------------------------------------------------- /playbooks/group_vars/master-nodes-templates: -------------------------------------------------------------------------------- 1 | -- 2 | ######################################### 3 | ## example for Rackspace cloud servers ## 4 | ## general1-8 flavor and CentOS 7 ## 5 | ## root filesystem used for /hadoop ## 6 | ## using the default public network ## 7 | ######################################### 8 | 9 | #cluster_interface: 'eth0' 10 | #cloud_nodes_count: 3 11 | #cloud_image: 'CentOS 7 (PVHVM)' 12 | # cloud_image: 'CentOS 6 (PVHVM)' 13 | #cloud_flavor: 'performance2-15' 14 | #build_datanode_cbs: true 15 | #cbs_disks_size: 200 16 | #cbs_disks_type: 'SATA' 17 | #hadoop_disk: xvde 18 | #datanode_disks: ['xvdf', 'xvdg'] 19 | 20 | ################################## 21 | ## example for static inventory ## 22 | ################################## 23 | 24 | # cluster_interface: 'bond1' 25 | # bond_interfaces: ['eth4', 'eth6'] 26 | # bond_netmask: '255.255.255.0' 27 | # hadoop_disk: sdb 28 | 29 | ############################################## 30 | ## example for Rackspace cloud servers ## 31 | ## performance2-15 flavor and CentOS 6 or 7 ## 32 | ## root filesystem used for /hadoop ## 33 | ## Namenode and Masterservices extra mounts ## 34 | ## using the default public network ## 35 | ############################################## 36 | #cluster_interface: 'eth0' 37 | #cloud_nodes_count: 3 38 | #cloud_image: 'CentOS 7 (PVHVM)' 39 | #cloud_image: 'CentOS 6 (PVHVM)' 40 | #cloud_flavor: 'performance2-15' 41 | #build_datanode_cbs: true 42 | #cbs_disks_size: 200 43 | #cbs_disks_type: 'SATA' 44 | #hadoop_disk: xvde 45 | #namenode_disk: xvdf 46 | #masterservices_disk: xvdg 47 | #datanode_disks: ['xvdf', 'xvdg'] 48 | 49 | ############################################## 50 | ## example for Rackspace OnMetal servers ## 51 | ## performance2-15 flavor and CentOS 6 or 7 ## 52 | ## root filesystem used for /hadoop ## 53 | ## Namenode and Masterservices on SSD ## 54 | ## using the default public network ## 55 | ############################################## 56 | #cluster_interface: 'bond0.101' 57 | #cloud_nodes_count: 3 58 | #cloud_image: 'OnMetal - CentOS 7' 59 | #cloud_flavor: 'onmetal-io1' 60 | #build_datanode_cbs: true 61 | #cbs_disks_size: 200 62 | #cbs_disks_type: 'SATA' 63 | #hadoop_disk: sdb 64 | #namenode_disk: sdb 65 | #masterservices_disk: sdc 66 | #datanode_disks: ['sdd', 'sde'] 67 | 68 | ############################################# 69 | ## example for Rackspace cloud servers ## 70 | ## performance2-15 flavor and Ubuntu 14 ## 71 | ## ephemeral disk used for /hadoop ## 72 | ## using ServiceNet as the cluster network ## 73 | ############################################# 74 | 75 | # cluster_interface: 'eth1' 76 | # cloud_nodes_count: 2 77 | # cloud_image: 'Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)' 78 | # cloud_flavor: 'performance2-15' 79 | # hadoop_disk: xvde 80 | 81 | 82 | ###################################### 83 | ## example for Rackspace OnMetal v2 ## 84 | ###################################### 85 | 86 | #cluster_interface: bond0 87 | #cloud_nodes_count: 3 88 | #cloud_image: 'OnMetal - CentOS 7' 89 | ## cloud_image: 'OnMetal - Ubuntu 14.04 LTS (Trusty Tahr)' 90 | # cloud_flavor: 'onmetal-general2-small' 91 | #cloud_flavor: 'onmetal-io1' 92 | #hadoop_disk: sdb 93 | #datanode_disks: sdc 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /playbooks/group_vars/postgres_vars.yml: -------------------------------------------------------------------------------- 1 | amon_pass: "{{ services_pass }}" 2 | rman_pass: "{{ services_pass }}" 3 | sentry_pass: "{{ services_pass }}" 4 | nav_pass: "{{ services_pass }}" 5 | navms_pass: "{{ services_pass }}" 6 | hive_pass: "{{ services_pass }}" 7 | sqoop_pass: "{{ services_pass }}" 8 | oozie_pass: "{{ services_pass }}" 9 | scm_pass: "{{ services_pass }}" 10 | db_names: 11 | - amon 12 | - metastore 13 | - navms 14 | - nav 15 | - rman 16 | - sentry 17 | - oozie 18 | - sqoop 19 | - scm 20 | -------------------------------------------------------------------------------- /playbooks/group_vars/slave-nodes: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################## 3 | # use template file for example references # 4 | # ~/ansible-hadoop/playbooks/group_vars/slave-nodes-templates# 5 | ############################################################## 6 | -------------------------------------------------------------------------------- /playbooks/hortonworks.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_groups.yml 3 | 4 | - name: Apply the ambari-agent role to all nodes 5 | hosts: hadoop-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | pre_tasks: 9 | - name: Show hadoop-cluster info 10 | debug: var=hostvars[inventory_hostname] 11 | when: debug 12 | 13 | - name: include hortonworks vars 14 | include_vars: group_vars/hortonworks 15 | 16 | roles: 17 | - ambari-agent 18 | 19 | - name: "generate site facts" 20 | hosts: localhost 21 | any_errors_fatal: true 22 | become: no 23 | vars: 24 | dnmemory: "{{ hostvars[groups['slave-nodes'][0]]['ansible_memtotal_mb'] / 1024 }}" 25 | mnmemory: "{{ hostvars[groups['master-nodes'][0]]['ansible_memtotal_mb'] / 1024 }}" 26 | cores: "{{ hostvars[groups['slave-nodes'][0]]['ansible_processor_count'] }}" 27 | tasks: 28 | - name: "gather site facts" 29 | action: 30 | module: sitefacts.py 31 | dnmemory="{{ dnmemory }}" 32 | mnmemory="{{ mnmemory }}" 33 | cores="{{ cores }}" 34 | ambari_server="localhost" 35 | ambari_pass="admin" 36 | cluster_name="{{ cluster_name }}" 37 | compare="false" 38 | current_facts="false" 39 | 40 | - name: Apply the ambari-server role to ambari-node group 41 | hosts: ambari-node 42 | become: yes 43 | pre_tasks: 44 | - name: include hortonworks vars 45 | include_vars: group_vars/hortonworks 46 | 47 | roles: 48 | - ambari-server 49 | post_tasks: 50 | - name: Cleanup the temporary files 51 | file: path={{ item }} state=absent 52 | with_items: 53 | - /tmp/cluster_blueprint 54 | - /tmp/cluster_template 55 | - /tmp/alert_targets 56 | - /tmp/hdprepo 57 | tags: 58 | - ambari-server-only 59 | -------------------------------------------------------------------------------- /playbooks/kube.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_groups.yml 3 | 4 | - name: Apply the to all nodes 5 | hosts: kube-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | pre_tasks: 9 | - name: Show hadoop-cluster info 10 | debug: var=hostvars[inventory_hostname] 11 | when: debug 12 | 13 | - name: include kube vars 14 | include_vars: group_vars/kube-all.yml 15 | 16 | - name: include kube vars 17 | include_vars: group_vars/kube-cluster.yml 18 | roles: 19 | - { role: docker, tags: docker } 20 | 21 | - name: kubernetes role 22 | hosts: kube-masters 23 | gather_facts: yes 24 | become: yes 25 | pre_tasks: 26 | - name: Show hadoop-cluster info 27 | debug: var=hostvars[inventory_hostname] 28 | when: debug 29 | 30 | - name: include kube vars 31 | include_vars: group_vars/kube-all.yml 32 | 33 | - name: include kube vars 34 | include_vars: group_vars/kube-cluster.yml 35 | roles: 36 | - { role: kubernetes/master, tags: master } 37 | - { role: cni, tags: cni } 38 | 39 | - name: kubernetes nodes 40 | hosts: kube-nodes 41 | gather_facts: yes 42 | become: yes 43 | pre_tasks: 44 | - name: Show hadoop-cluster info 45 | debug: var=hostvars[inventory_hostname] 46 | when: debug 47 | 48 | - name: include kube vars 49 | include_vars: group_vars/kube-all.yml 50 | 51 | - name: include kube vars 52 | include_vars: group_vars/kube-cluster.yml 53 | roles: 54 | - { role: kubernetes/node, tags: node } 55 | 56 | 57 | # 58 | # 59 | #- name: Apply the kluster 60 | # hosts: kube-cluster 61 | # become: yes 62 | # pre_tasks: 63 | # - name: include kube vars 64 | # include_vars: group_vars/kube 65 | # tasks: 66 | # - name: run kubeadm ansible on cluser 67 | # include: /root/ansible-hadoop/playbooks/roles/kubeadm-ansible/site.yaml 68 | # 69 | #- hosts: kube-cluster 70 | # gather_facts: yes 71 | # become: yes 72 | # roles: 73 | # - { role: docker, tags: docker } 74 | # 75 | - name: extra kube roles 76 | hosts: kube-masters 77 | gather_facts: yes 78 | become: yes 79 | pre_tasks: 80 | - name: Show hadoop-cluster info 81 | debug: var=hostvars[inventory_hostname] 82 | when: debug 83 | 84 | - name: include kube vars 85 | include_vars: group_vars/kube-all.yml 86 | 87 | - name: include kube vars 88 | include_vars: group_vars/kube-cluster.yml 89 | tasks: 90 | - name: "Helm role" 91 | include_role: 92 | name: helm 93 | when: "additional_features.helm" 94 | run_once: yes 95 | tags: helm 96 | 97 | - name: "MetalLB role" 98 | include_role: 99 | name: metallb 100 | when: "additional_features.metallb" 101 | run_once: yes 102 | tags: metallb 103 | 104 | - name: "Healthcheck role" 105 | include_role: 106 | name: healthcheck 107 | when: "additional_features.healthcheck" 108 | run_once: yes 109 | tags: healthcheck 110 | 111 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reload systemd 3 | command: systemctl daemon-reload 4 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 5 | 6 | - name: Restart ambari-agent 7 | service: name=ambari-agent state=restarted 8 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load OS specific variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml" 7 | - "{{ ansible_os_family|lower }}.yml" 8 | - defaults.yml 9 | paths: 10 | - ../vars 11 | 12 | - name: Download the Ambari repo 13 | get_url: url={{ ambari_repo }} dest={{ ambari_repo_file }} 14 | 15 | - name: Add apt key 16 | apt_key: keyserver=keyserver.ubuntu.com id={{ ambari_repo_key }} 17 | when: ansible_os_family == "Debian" 18 | 19 | - name: Ensure required packages are installed (yum) 20 | yum: 21 | name: "{{ item }}" 22 | update_cache: yes 23 | state: installed 24 | with_items: 25 | - ambari-agent 26 | notify: Reload systemd 27 | when: ansible_os_family == "RedHat" 28 | 29 | - name: Ensure required packages are installed (apt) 30 | apt: 31 | name: "{{ item }}" 32 | update_cache: yes 33 | state: installed 34 | with_items: 35 | - ambari-agent 36 | when: ansible_os_family == "Debian" 37 | 38 | - include: unbound-dns.yml 39 | when: use_dns 40 | 41 | - name: Configure the Ambari agent 42 | lineinfile: dest=/etc/ambari-agent/conf/ambari-agent.ini 43 | regexp='^hostname\s*=' 44 | line='hostname={{ hostvars[groups['ambari-node'][0]]['ansible_nodename'] }}' 45 | state=present 46 | notify: Restart ambari-agent 47 | 48 | - meta: flush_handlers 49 | 50 | - name: Make sure ambari-agent is running 51 | service: name=ambari-agent state=started enabled=yes 52 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/tasks/unbound-dns.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Unbound 3 | yum: 4 | name: "unbound" 5 | state: present 6 | update_cache: yes 7 | ignore_errors: true 8 | when: ansible_os_family == "RedHat" 9 | 10 | - name: Start the unbound service 11 | service: name=unbound state=restarted enabled=yes 12 | 13 | - name: Rearrange Ubuntu unbound configuration to match default RHEL 14 | shell: sed -i 's/unbound\.conf\.d/conf\.d/g' /etc/unbound/unbound.conf; 15 | mv /etc/unbound/unbound.conf.d /etc/unbound/conf.d; 16 | mkdir /etc/unbound/local.d; 17 | grep -q -F '/etc/unbound/local.d/*.conf' /etc/unbound/unbound.conf || echo "include{{ ":" }} \"/etc/unbound/local.d/*.conf\"" >> /etc/unbound/unbound.conf; 18 | sed -i 's/RESOLVCONF_FORWARDERS=true/RESOLVCONF_FORWARDERS=false/g' /etc/default/unbound 19 | when: ansible_os_family == "Debian" 20 | 21 | - name: Configure unbound service 22 | template: src=redhat-unbound.j2 dest=/etc/unbound/conf.d/mbd.conf 23 | notify: Restart unbound 24 | 25 | - name: Place resolv template in place 26 | template: src=resolv.conf.j2 dest={{ resolv_conf }} 27 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/templates/redhat-unbound.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | access-control: 127.0.0.0/8 allow 4 | access-control: 10.0.0.0/8 allow 5 | domain-insecure: "{{ ansible_domain }}" 6 | local-zone: "10.in-addr.arpa." nodefault 7 | 8 | stub-zone: 9 | name: "10.in-addr.arpa." 10 | stub-addr: {{ hostvars[groups['ambari-node'][0]][['ansible_', hostvars[groups['ambari-node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 11 | 12 | forward-zone: 13 | name: "." 14 | forward-addr: {{ hostvars[groups['ambari-node'][0]][['ansible_', hostvars[groups['ambari-node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 15 | 16 | remote-control: 17 | control-interface: 127.0.0.1 18 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | domain {{ ansible_domain }} 2 | nameserver 127.0.0.1 3 | nameserver {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 4 | nameserver 173.203.4.8 5 | nameserver 173.203.4.9 6 | search {{ ansible_domain }} 7 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/vars/debian.yml: -------------------------------------------------------------------------------- 1 | ambari_repo: "http://public-repo-1.hortonworks.com/ambari/{{ ansible_distribution|lower }}{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}/ambari.list" 2 | ambari_repo_file: "/etc/apt/sources.list.d/ambari.list" 3 | ambari_repo_key: "B9733A7A07513CAD" 4 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | ambari_repo: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/{{ ambari_version }}/ambari.repo" 2 | ambari_repo_file: "/etc/yum.repos.d/ambari.repo" 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | resolv_conf: "/etc/resolv.conf" 2 | 3 | ambari_repo: "http://public-repo-1.hortonworks.com/ambari/centos{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}/ambari.repo" 4 | ambari_repo_file: "/etc/yum.repos.d/ambari.repo" 5 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reload systemd 3 | command: systemctl daemon-reload 4 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 5 | 6 | - name: Restart ambari-server 7 | service: name=ambari-server state=restarted 8 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/tasks/arcadia.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: copy arcadia enterprise to target server 3 | copy: src={{ arcadia_config.archive_file }} dest=/tmp/ 4 | 5 | - name: Extract archive file to ambari service destination 6 | command: tar -zxf /tmp/{{ arcadia_config.archive_file }} -C {{ arcadia_config.archive_dest }} 7 | 8 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/tasks/custom.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Upload custom blueprint 3 | template: src={{ custom_blueprint_template }} dest=/tmp/cluster_blueprint mode=0644 4 | 5 | - name: Upload custom cluster creation template 6 | template: src={{ custom_cluster_template }} dest=/tmp/cluster_template mode=0644 7 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/tasks/multi-nodes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Upload 1-masternode blueprint 3 | template: src=blueprint-multi-node-1-master.j2 dest=/tmp/cluster_blueprint mode=0644 4 | when: groups['master-nodes']|length == 1 5 | 6 | - name: Upload 2-masternodes blueprint 7 | template: src=blueprint-multi-node-2-masters.j2 dest=/tmp/cluster_blueprint mode=0644 8 | when: groups['master-nodes']|length == 2 9 | 10 | - name: Upload 3-masternodes blueprint 11 | template: src=blueprint-multi-node-3-masters.j2 dest=/tmp/cluster_blueprint mode=0644 12 | when: groups['master-nodes']|length > 2 13 | 14 | - name: Upload cluster creation template 15 | template: src=cluster-template-multi-nodes.j2 dest=/tmp/cluster_template mode=0644 16 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/tasks/prerequisites.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure required packages are installed (yum) 3 | yum: 4 | name: "{{ item }}" 5 | update_cache: yes 6 | state: installed 7 | with_items: 8 | - mysql-connector-java 9 | - ambari-server 10 | notify: Reload systemd 11 | when: ansible_os_family == "RedHat" 12 | 13 | - name: Ensure required packages are installed (apt) 14 | apt: 15 | name: "{{ item }}" 16 | update_cache: yes 17 | state: installed 18 | with_items: 19 | - ambari-server 20 | when: ansible_os_family == "Debian" 21 | 22 | - include: unbound-dns.yml 23 | when: use_dns 24 | 25 | - meta: flush_handlers 26 | 27 | - include: arcadia.yml 28 | when: arcadia and ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" 29 | 30 | - name: Run Ambari Server setup for mysql connector 31 | shell: /usr/sbin/ambari-server setup -s --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar 32 | 33 | - name: Run Ambari Server setup for default config 34 | shell: /usr/sbin/ambari-server setup -s 35 | 36 | - name: Make sure ambari-server is running 37 | command: service ambari-server restart 38 | 39 | - name: Waiting for ambari-server to start listening on port 8080 40 | wait_for: host={{ ansible_nodename }} port=8080 41 | 42 | - name: Change Ambari admin user password 43 | uri: url=http://{{ ansible_nodename }}:8080/api/v1/users/admin 44 | method=PUT 45 | force_basic_auth=yes 46 | user=admin 47 | password=admin 48 | HEADER_X-Requested-By="ambari" 49 | body=' {"Users":{ "password":"{{ admin_password }}", "old_password":"admin"}}' 50 | body_format=raw 51 | status_code=200,201,202,403 52 | when: admin_password != 'admin' 53 | 54 | - name: Waiting for ambari-agents to register 55 | uri: url=http://{{ ansible_nodename }}:8080/api/v1/hosts/{{ hostvars[item]['ansible_nodename'] | lower }} 56 | method=GET 57 | force_basic_auth=yes 58 | user=admin 59 | password={{ admin_password }} 60 | HEADER_X-Requested-By="ambari" 61 | status_code=200,201,202,404 62 | with_items: "{{ groups['hadoop-cluster'] }}" 63 | register: result 64 | until: "result.status != 404" 65 | retries: 200 66 | delay: 5 67 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/tasks/single-node.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Upload single-node blueprint 3 | template: src=blueprint-single-node.j2 dest=/tmp/cluster_blueprint mode=0644 4 | 5 | - name: Upload single-node cluster creation template 6 | template: src=cluster-template-single-node.j2 dest=/tmp/cluster_template mode=0644 7 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/tasks/unbound-dns.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Unbound 3 | yum: 4 | name: "unbound" 5 | state: present 6 | update_cache: yes 7 | ignore_errors: true 8 | when: ansible_os_family == "RedHat" 9 | 10 | - name: Start the unbound service 11 | service: name=unbound state=restarted enabled=yes 12 | 13 | - name: Rearrange Ubuntu unbound configuration to match default RHEL 14 | shell: sed -i 's/unbound\.conf\.d/conf\.d/g' /etc/unbound/unbound.conf; 15 | mv /etc/unbound/unbound.conf.d /etc/unbound/conf.d; 16 | mkdir /etc/unbound/local.d; 17 | grep -q -F '/etc/unbound/local.d/*.conf' /etc/unbound/unbound.conf || echo "include{{ ":" }} \"/etc/unbound/local.d/*.conf\"" >> /etc/unbound/unbound.conf; 18 | sed -i 's/RESOLVCONF_FORWARDERS=true/RESOLVCONF_FORWARDERS=false/g' /etc/default/unbound 19 | when: ansible_os_family == "Debian" 20 | 21 | - name: Configure unbound service on the master node 22 | template: src=redhat-unbound-master.j2 dest=/etc/unbound/conf.d/mbd.conf 23 | notify: Restart unbound 24 | 25 | - name: Configure unbound cluster zone on the master node 26 | template: src=redhat-unbound-zonefile.j2 dest=/etc/unbound/local.d/{{ ansible_domain }}_zone.conf force=yes 27 | notify: Restart unbound 28 | 29 | - name: Place resolv template in place 30 | template: src=resolv.conf.j2 dest={{ resolv_conf }} 31 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/alert_targets.j2: -------------------------------------------------------------------------------- 1 | { 2 | "AlertTarget": { 3 | "name" : "Initial Notification", 4 | "description" : "Notification created during cluster installing", 5 | "notification_type" : "EMAIL", 6 | "global" : true, 7 | "properties" : { 8 | "ambari.dispatch.recipients" : ["{{ alerts_contact }}"], 9 | "mail.smtp.host" : "localhost", 10 | "mail.smtp.port" : "25", 11 | "mail.smtp.auth" : "false", 12 | "mail.smtp.starttls.enable" : "false", 13 | "mail.smtp.from" : "ambari@{{ hostvars[groups['ambari-node'][0]]['ansible_nodename'] }}" 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/cluster-template-custom.j2: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint" : "cluster_blueprint", 3 | "default_password" : "{{ services_password }}", 4 | "host_groups" :[ 5 | {% if groups['slave-nodes']|length > 0 -%} 6 | { 7 | "name" : "slavenode_simple", 8 | "hosts" : [{% for node in groups['slave-nodes']|sort %} 9 | 10 | { 11 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 12 | }{% if not loop.last %},{% endif %} 13 | {% endfor %} 14 | 15 | ] 16 | }, 17 | {% endif -%} 18 | {% if groups['master-nodes']|length > 1 -%} 19 | { 20 | "name" : "masternode_2", 21 | "hosts" : [ 22 | {% for node in groups['master-nodes']|sort %}{% if loop.index == 2 %} 23 | { 24 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 25 | } 26 | {% endif %}{% endfor %} 27 | ] 28 | }, 29 | {% endif -%} 30 | {% if groups['master-nodes']|length > 2 -%} 31 | { 32 | "name" : "masternode_3", 33 | "hosts" : [ 34 | { 35 | "fqdn" : "{{ hostvars[groups['ambari-node'][0]]['ansible_nodename'] | lower }}" 36 | } 37 | ] 38 | }, 39 | {% endif -%} 40 | {% if 'edge-nodes' in groups and groups['edge-nodes']|length > 0 -%} 41 | { 42 | "name" : "edgenode", 43 | "hosts" : [{% for node in groups['edge-nodes']|sort %} 44 | 45 | { 46 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 47 | }{% if not loop.last %},{% endif %} 48 | {% endfor %} 49 | 50 | ] 51 | }, 52 | {% endif -%} 53 | { 54 | "name" : "masternode_1", 55 | "hosts" : [ 56 | { 57 | "fqdn" : "{{ hostvars[groups['master-nodes']|sort|first]['ansible_nodename'] | lower }}" 58 | } 59 | ] 60 | } 61 | ] 62 | } 63 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/cluster-template-multi-nodes.j2: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint" : "cluster_blueprint", 3 | "default_password" : "{{ services_password }}", 4 | "host_groups" :[ 5 | {% if groups['slave-nodes']|length > 0 and not (groups['slave-nodes']|length == 1 and groups['master-nodes']|length == 2) -%} 6 | { 7 | "name" : "slavenode_simple", 8 | "hosts" : [{% for node in groups['slave-nodes']|sort %}{% if groups['master-nodes']|length == 2 %}{% if not loop.first %} 9 | 10 | { 11 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 12 | }{% if not loop.last %},{% endif %} 13 | 14 | {% endif %} 15 | {% else %} 16 | 17 | { 18 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 19 | }{% if not loop.last %},{% endif %} 20 | {% endif %} 21 | {% endfor %} 22 | 23 | ] 24 | }, 25 | {% endif -%} 26 | {% if groups['master-nodes']|length > 1 -%} 27 | { 28 | "name" : "masternode_2", 29 | "hosts" : [ 30 | {% for node in groups['master-nodes']|sort %}{% if loop.index == 2 %} 31 | { 32 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 33 | } 34 | {% endif %}{% endfor %} 35 | ] 36 | }, 37 | {% endif -%} 38 | {% if groups['master-nodes']|length > 2 -%} 39 | { 40 | "name" : "masternode_3", 41 | "hosts" : [ 42 | { 43 | "fqdn" : "{{ hostvars[groups['ambari-node'][0]]['ansible_nodename'] | lower }}" 44 | } 45 | ] 46 | }, 47 | {% endif -%} 48 | {% if 'edge-nodes' in groups and groups['edge-nodes']|length > 0 -%} 49 | { 50 | "name" : "edgenode", 51 | "hosts" : [{% for node in groups['edge-nodes']|sort %} 52 | 53 | { 54 | "fqdn" : "{{ hostvars[node]['ansible_nodename'] | lower }}" 55 | }{% if not loop.last %},{% endif %} 56 | {% endfor %} 57 | 58 | ] 59 | }, 60 | {% endif -%} 61 | {% if groups['master-nodes']|length == 2 -%} 62 | { 63 | "name" : "slavenode_zookeeper", 64 | "hosts" : [ 65 | { 66 | "fqdn" : "{{ hostvars[groups['slave-nodes']|sort|first]['ansible_nodename'] | lower }}" 67 | } 68 | ] 69 | }, 70 | {% endif -%} 71 | { 72 | "name" : "masternode_1", 73 | "hosts" : [ 74 | { 75 | "fqdn" : "{{ hostvars[groups['master-nodes']|sort|first]['ansible_nodename'] | lower }}" 76 | } 77 | ] 78 | } 79 | ] 80 | } 81 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/cluster-template-single-node.j2: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint" : "cluster_blueprint", 3 | "default_password" : "{{ services_password }}", 4 | "host_groups" :[ 5 | { 6 | "name" : "single_node", 7 | "hosts" : [ 8 | { 9 | "fqdn" : "{{ hostvars[groups['ambari-node'][0]]['ansible_nodename'] | lower }}" 10 | } 11 | ] 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/hdprepo.j2: -------------------------------------------------------------------------------- 1 | {"Repositories" : 2 | { "base_url" : "{{ custom_repo_url }}" } 3 | } 4 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/redhat-unbound-master.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | interface: {{ hostvars[groups['ambari-node'][0]][['ansible_', hostvars[groups['ambari-node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 4 | access-control: 127.0.0.0/8 allow 5 | access-control: 10.0.0.0/8 allow 6 | {% for node in groups['hadoop-cluster'] %} 7 | access-control: {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}/32 allow 8 | {% endfor %} 9 | domain-insecure: "{{ ansible_domain }}" 10 | local-zone: "10.in-addr.arpa." nodefault 11 | 12 | stub-zone: 13 | name: "10.in-addr.arpa." 14 | stub-addr: {{ hostvars[groups['ambari-node'][0]][['ansible_', hostvars[groups['ambari-node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 15 | 16 | forward-zone: 17 | name: "." 18 | forward-addr: {{ dnsfwd[0] }} 19 | forward-addr: {{ dnsfwd[1] }} 20 | 21 | remote-control: 22 | control-interface: 127.0.0.1 23 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/redhat-unbound-zonefile.j2: -------------------------------------------------------------------------------- 1 | # cluster zone 2 | 3 | local-zone: "{{ ansible_domain }}." transparent 4 | 5 | {% for node in groups['hadoop-cluster'] %} 6 | local-data: "{{ hostvars[node]['ansible_fqdn'] }}. IN A {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}" 7 | local-data-ptr: "{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_fqdn'] }}" 8 | 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | nameserver 127.0.0.1 2 | nameserver {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 3 | nameserver {{ dnsres[0] }} 4 | nameserver {{ dnsres[1] }} 5 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/spark-params.j2: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | from resource_management.libraries.script.script import Script 4 | 5 | 6 | # config object that holds the configurations declared under the configurations folder 7 | config = Script.get_config() 8 | 9 | spark_master = config['clusterHostInfo']['sparks_master_hosts'][0] 10 | tachyon_master_hosts = config['clusterHostInfo'].get('tachyon_master_hosts', []) 11 | if len(tachyon_master_hosts) > 0: 12 | tachyon_master = config['clusterHostInfo']['tachyon_master_hosts'][0] 13 | else: 14 | tachyon_master = "localhost" 15 | hdfs_root = 'hdfs://{{ hostvars[groups['master-nodes'][1]]['ansible_hostname'] }}' 16 | 17 | node_hostname = socket.getfqdn() 18 | 19 | # spark-env.sh configs 20 | spark_local_dirs = config['configurations']['spark-env']['spark_local_dirs'] 21 | 22 | # Content of spark-env.sh 23 | spark_env_content = config['configurations']['spark-env']['content'] 24 | 25 | # Content of spark-defaults.conf 26 | spark_defaults_content = config['configurations']['spark-defaults']['content'] 27 | spark_executor_memory = config['configurations']['spark-defaults']['spark_executor_memory'] 28 | spark_driver_memory = config['configurations']['spark-defaults']['spark_driver_memory'] 29 | spark_driver_maxResultSize = config['configurations']['spark-defaults']['spark_driver_maxResultSize'] 30 | 31 | 32 | spark_conf_dir = "/etc/spark/conf" 33 | spark_user = 'spark' 34 | spark_events = '/apps/spark/events' 35 | hdfs_user = hdfs_group = 'hdfs' 36 | users_group = 'users' 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/tachyon-params.j2: -------------------------------------------------------------------------------- 1 | from resource_management.libraries.script.script import Script 2 | 3 | 4 | # config object that holds the configurations declared under the configurations folder 5 | config = Script.get_config() 6 | 7 | # tachyon master address 8 | tachyon_master_address = config['clusterHostInfo']['tachyon_master_hosts'][0] 9 | 10 | # tachyon underfs address 11 | tachyon_underfs_address = 'hdfs://{{ hostvars[groups['master-nodes'][1]]['ansible_hostname'] }}' 12 | tachyon_underfs_root = config['configurations']['tachyon-env']['tachyon_underfs_root'] 13 | 14 | # tachyon worker memory alotment 15 | tachyon_worker_memory = config['configurations']['tachyon-env']['tachyon_worker_memory'] 16 | tachyon_ram_folder = config['configurations']['tachyon-env']['tachyon_ram_folder'] 17 | 18 | # Content of tachyon-env.sh 19 | tachyon_env_content = config['configurations']['tachyon-env']['content'] 20 | 21 | tachyon_conf_dir = "/etc/tachyon/conf" 22 | hdfs_user = hdfs_group = 'hdfs' 23 | tachyon_user = 'tachyon' 24 | users_group = 'users' 25 | 26 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/files/cloudera-manager.repo: -------------------------------------------------------------------------------- 1 | [cloudera-manager] 2 | # Packages for Cloudera Manager, Version 5, on RedHat or CentOS 7 x86_64 3 | name=Cloudera Manager 4 | baseurl=https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5/ 5 | gpgkey =https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera 6 | gpgcheck = 1 7 | 8 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/files/java_home.sh: -------------------------------------------------------------------------------- 1 | #set java home for all users 2 | #modify with oracle jdk 1.8 3 | export JAVA_HOME=/usr/java/jdk1.8.0_241-amd64 4 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/files/jce_policy-8.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/objectrocket/ansible-hadoop/d0d9dec5330d20ecce85e8ea29ebc38f27cde347/playbooks/roles/cloudera-agent/files/jce_policy-8.zip -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart unbound 3 | service: name=unbound state=restarted enabled=yes 4 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/tasks/jce_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - lineinfile: 4 | path: "{{ java_home }}/jre/lib/security/java.security" 5 | regexp: '#?crypto.policy=' 6 | line: crypto.policy=unlimited 7 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/tasks/jce_install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install unzip package 4 | package: 5 | name: 6 | - unzip 7 | state: installed 8 | 9 | - name: Copy JCE policy zip to temp directory 10 | get_url: 11 | url: "{{ java_jce_url }}/{{ java_jce_zip }}" 12 | dest: "{{ tmp_dir }}/jce.zip" 13 | 14 | - name: Extract JCE policy zip 15 | unarchive: 16 | src: "{{ tmp_dir }}/jce.zip" 17 | dest: "{{ tmp_dir }}" 18 | copy: no 19 | 20 | - name: Copy JCE policy jars into correct location 21 | copy: 22 | src: "{{ item }}" 23 | dest: "{{ java_home }}/jre/lib/security/" 24 | backup: yes 25 | with_fileglob: 26 | - "{{ tmp_dir }}/{{ unarchived_directory }}/*.jar" 27 | 28 | - name: Cleanup tmp files 29 | file: 30 | path: "{{ tmp_dir }}/{{ item }}" 31 | state: absent 32 | with_items: 33 | - jce.zip 34 | - "{{ unarchived_directory }}" 35 | ignore_errors: True 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/tasks/jdk_install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Copy jdk to temp directory 4 | get_url: 5 | url: "{{ java_jdk_url }}{{ java_jdk_rpm }}" 6 | dest: "{{ tmp_dir }}/" 7 | 8 | - name: install JDK 9 | yum: 10 | name: "{{ tmp_dir }}/{{ java_jdk_rpm }}" 11 | update_cache: yes 12 | state: installed 13 | when: ansible_os_family == "RedHat" 14 | 15 | - name: Cleanup tmp files 16 | file: 17 | path: "{{ tmp_dir }}/{{ java_jdk_rpm }}" 18 | state: absent 19 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load OS specific variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}.yml" 7 | - defaults.yml 8 | paths: 9 | - ../vars 10 | 11 | - include_vars: ../vars/{{ ansible_os_family|lower }}.yml 12 | - include_vars: ../../../group_vars/cloudera 13 | 14 | - name: Add Cloudera Manager yum repository 15 | yum_repository: 16 | name: cloudera-manager 17 | description: Cloudera Manager 18 | baseurl: "{{ scm_repo_url }}" 19 | gpgkey: "{{ scm_repo_gpgkey }}" 20 | gpgcheck: yes 21 | enabled: yes 22 | when: 23 | - ansible_os_family|lower == "redhat" 24 | 25 | - include: jdk_install.yml 26 | 27 | - include: jce_install.yml 28 | 29 | - include: jce_config.yml 30 | 31 | - name: setup java home in profile.d 32 | copy: src=java_home.sh dest=/etc/profile.d/ 33 | 34 | - name: Add apt key 35 | apt_key: keyserver=keyserver.ubuntu.com id={{ cloudera_repo_key }} 36 | when: ansible_os_family == "Debian" 37 | 38 | - name: Ensure required packages are installed (yum) 39 | yum: 40 | name: "{{ item }}" 41 | update_cache: yes 42 | state: installed 43 | with_items: 44 | - cloudera-manager-agent 45 | - cloudera-manager-daemons 46 | when: ansible_os_family == "RedHat" 47 | 48 | - name: Ensure required packages are installed (apt) 49 | apt: 50 | name: "{{ item }}" 51 | update_cache: yes 52 | state: installed 53 | with_items: "{{ package_info.pkgs }}" 54 | when: ansible_os_family == "Debian" 55 | 56 | - name: Configure Cloudera Manager Agent 'server_host' 57 | lineinfile: 58 | dest: /etc/cloudera-scm-agent/config.ini 59 | regexp: "^server_host" 60 | line: "server_host={{ hostvars[groups['cm_node'][0]]['ansible_nodename'] }}" 61 | 62 | - name: Restart Cloudera Manager Agents 63 | service: 64 | name: cloudera-scm-agent 65 | state: restarted 66 | enabled: yes 67 | 68 | - include: unbound-dns.yml 69 | when: use_dns 70 | 71 | - name: Enable the Cloudera agent 72 | service: name=cloudera-scm-agent state=restarted enabled=yes 73 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/tasks/mysql.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: mysql_vars.yml 3 | 4 | - name: mysql config template 5 | template: src=my.cnf dest=/etc/ 6 | 7 | - name: start mysql 8 | service: name=mariadb state=started enabled=yes 9 | 10 | - name: copy db dump to server 11 | copy: src={{ item }}.sql dest=/tmp 12 | with_items: "{{ db_names }}" 13 | 14 | - name: create DB references 15 | mysql_db: name={{ item }} state=present 16 | with_items: "{{ db_names }}" 17 | 18 | - name: import DBs 19 | mysql_db: name={{ item }} state=import target=/tmp/{{ item }}.sql 20 | with_items: "{{ db_names }}" 21 | 22 | - name: setup Activity Monitor DB user 23 | mysql_user: name=amon password={{ amon_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=amon.*:ALL state=present 24 | with_items: "{{ hadoop-cluster }}" 25 | 26 | #- name: Set firewalld rules between cluster nodes 27 | # command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" accept' 28 | # firewalld: source={{ hostvars[item][['ansible_', hostvars[item]['cluster_interface']]|join]['ipv4']['address'] }} state=enabled 29 | # with_items: "{{ hadoop-cluster }}" 30 | 31 | 32 | - name: setup Report Manager DB user 33 | mysql_user: name=rman password={{ rman_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=rman.*:ALL state=present 34 | with_items: "{{ hadoop-cluster }}" 35 | 36 | - name: setup Sentry DB user 37 | mysql_user: name=sentry password={{ sentry_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=sentry.*:ALL state=present 38 | with_items: "{{ hadoop-cluster }}" 39 | 40 | - name: setup Cloudera Navigator DB user 41 | mysql_user: name=nav password={{ nav_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=nav.*:ALL state=present 42 | with_items: "{{ hadoop-cluster }}" 43 | 44 | - name: setup Cloudera Navigator Metadata DB user 45 | mysql_user: name=navms password={{ navms_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=navms.*:ALL state=present 46 | with_items: "{{ hadoop-cluster }}" 47 | 48 | - name: setup Hive DB user 49 | mysql_user: name=metastore password={{ hive_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=hive.*:ALL state=present 50 | with_items: "{{ hadoop-cluster }}" 51 | 52 | - name: setup SCM DB user 53 | mysql_user: name=scm password={{ scm_pass }} host="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" priv=scm.*:ALL state=present 54 | with_items: "{{ hadoop-cluster }}" 55 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/tasks/unbound-dns.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Unbound 3 | yum: 4 | name: "unbound" 5 | state: present 6 | update_cache: yes 7 | ignore_errors: true 8 | when: ansible_os_family == "RedHat" 9 | 10 | - name: Start the unbound service 11 | service: name=unbound state=restarted enabled=yes 12 | 13 | - name: Rearrange Ubuntu unbound configuration to match default RHEL 14 | shell: sed -i 's/unbound\.conf\.d/conf\.d/g' /etc/unbound/unbound.conf; 15 | mv /etc/unbound/unbound.conf.d /etc/unbound/conf.d; 16 | mkdir /etc/unbound/local.d; 17 | grep -q -F '/etc/unbound/local.d/*.conf' /etc/unbound/unbound.conf || echo "include{{ ":" }} \"/etc/unbound/local.d/*.conf\"" >> /etc/unbound/unbound.conf; 18 | sed -i 's/RESOLVCONF_FORWARDERS=true/RESOLVCONF_FORWARDERS=false/g' /etc/default/unbound 19 | when: ansible_os_family == "Debian" 20 | 21 | - name: Configure unbound service 22 | template: src=redhat-unbound.j2 dest=/etc/unbound/conf.d/mbd.conf 23 | notify: Restart unbound 24 | 25 | - name: Place resolv template in place 26 | template: src=resolv.conf.j2 dest={{ resolv_conf }} 27 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/templates/redhat-unbound.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | access-control: 127.0.0.0/8 allow 4 | access-control: 10.0.0.0/8 allow 5 | domain-insecure: "{{ ansible_domain }}" 6 | local-zone: "10.in-addr.arpa." nodefault 7 | 8 | stub-zone: 9 | name: "10.in-addr.arpa." 10 | stub-addr: {{ hostvars[groups['cm_node'][0]][['ansible_', hostvars[groups['cm_node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 11 | 12 | forward-zone: 13 | name: "." 14 | forward-addr: {{ hostvars[groups['cm_node'][0]][['ansible_', hostvars[groups['cm_node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 15 | 16 | remote-control: 17 | control-interface: 127.0.0.1 18 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | domain {{ ansible_domain }} 2 | nameserver 127.0.0.1 3 | nameserver {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 4 | nameserver 173.203.4.8 5 | nameserver 173.203.4.9 6 | search {{ ansible_domain }} 7 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/vars/debian.yml: -------------------------------------------------------------------------------- 1 | package_info: 2 | pkg_mgr: apt 3 | args: 4 | pkg: "ambari-agent" 5 | update_cache: yes 6 | state: latest 7 | ambari_repo: "http://public-repo-1.hortonworks.com/ambari/{{ ansible_distribution|lower }}{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}/ambari.list" 8 | ambari_repo_file: "/etc/apt/sources.list.d/ambari.list" 9 | ambari_repo_key: "B9733A7A07513CAD" 10 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-agent/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | package_info: 2 | pkg_mgr: yum 3 | args: 4 | name: "{{ item }}" 5 | state: latest 6 | update_cache: yes 7 | pkgs: 8 | - cloudera-manager-daemons 9 | - cloudera-manager-agent 10 | 11 | resolv_conf: "/etc/resolv.conf" 12 | 13 | cloudera_repo_file: "/etc/yum.repos.d/cloudera.repo" 14 | 15 | java_jce_url: "http://jdk_url/jdk-files/" 16 | java_jce_zip: "jce_policy-8.zip" 17 | unarchived_directory: "UnlimitedJCEPolicyJDK8" 18 | java_jdk_url: "http://jdk_url/jdk-files/" 19 | java_jdk_rpm: "jdk-8u241-linux-x64.rpm" 20 | 21 | tmp_dir: "/tmp/" 22 | 23 | java_jce_remote_src: "yes" 24 | 25 | java_jdk_remote_src: "yes" 26 | 27 | java_home: "/usr/java/jdk1.8.0_241-amd64/" 28 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart unbound 3 | service: name=unbound state=restarted enabled=yes 4 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: group_vars/cloudera 3 | 4 | - include_vars: group_vars/postgres_vars.yml 5 | 6 | - include: prerequisites.yml 7 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/tasks/prerequisites.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load OS specific variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}.yml" 7 | - defaults.yml 8 | paths: 9 | - ../vars 10 | 11 | - include_vars: group_vars/cloudera 12 | 13 | - include_vars: group_vars/postgres_vars.yml 14 | 15 | - name: Ensure required packages are installed (yum) 16 | yum: 17 | name: "{{ item }}" 18 | update_cache: yes 19 | state: installed 20 | with_items: "{{ pkgs }}" 21 | when: ansible_os_family == "RedHat" 22 | 23 | - include: unbound-dns.yml 24 | when: use_dns 25 | 26 | #- name: setup SCM DB 27 | # command: /opt/cloudera/cm/schema/scm_prepare_database.sh postgresql -h {{ groups['cm_node'][0] }} scm scm {{ scm_pass }} 28 | 29 | #- name: download cdh parcel 30 | # get_url: 31 | # url: "{{ parcel_repo_url }}{{ cdh_parcel }}" 32 | # dest: /opt/cloudera/parcel-repo/ 33 | # owner: cloudera-scm 34 | # group: cloudera-scm 35 | # 36 | #- name: download cdh parcel sha 37 | # get_url: 38 | # url: "{{ parcel_repo_url }}{{ cdh_parcel_sha }}" 39 | # dest: /opt/cloudera/parcel-repo/ 40 | # owner: cloudera-scm 41 | # group: cloudera-scm 42 | # 43 | - name: Upload db properties template 44 | template: src=db.properties dest=/etc/cloudera-scm-server/ 45 | 46 | - name: Upload db properties template 47 | template: src=db.mgmt.properties dest=/etc/cloudera-scm-server/ 48 | 49 | - name: Enable the cloudera manager server service 50 | service: name=cloudera-scm-server state=started enabled=yes 51 | 52 | - name: Waiting for mngr-server to start listening on port 7180 53 | wait_for: host={{ ansible_nodename }} port=7180 54 | 55 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/tasks/unbound-dns.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Unbound 3 | yum: 4 | name: "unbound" 5 | state: present 6 | update_cache: yes 7 | ignore_errors: true 8 | when: ansible_os_family == "RedHat" 9 | 10 | - name: Start the unbound service 11 | service: name=unbound state=restarted enabled=yes 12 | 13 | - name: Rearrange Ubuntu unbound configuration to match default RHEL 14 | shell: sed -i 's/unbound\.conf\.d/conf\.d/g' /etc/unbound/unbound.conf; 15 | mv /etc/unbound/unbound.conf.d /etc/unbound/conf.d; 16 | mkdir /etc/unbound/local.d; 17 | grep -q -F '/etc/unbound/local.d/*.conf' /etc/unbound/unbound.conf || echo "include{{ ":" }} \"/etc/unbound/local.d/*.conf\"" >> /etc/unbound/unbound.conf; 18 | sed -i 's/RESOLVCONF_FORWARDERS=true/RESOLVCONF_FORWARDERS=false/g' /etc/default/unbound 19 | when: ansible_os_family == "Debian" 20 | 21 | - name: Configure unbound service on the master node 22 | template: src=redhat-unbound-master.j2 dest=/etc/unbound/conf.d/mbd.conf 23 | notify: Restart unbound 24 | 25 | - name: Configure unbound cluster zone on the master node 26 | template: src=redhat-unbound-zonefile.j2 dest=/etc/unbound/local.d/{{ ansible_domain }}_zone.conf force=yes 27 | notify: Restart unbound 28 | 29 | - name: Place resolv template in place 30 | template: src=resolv.conf.j2 dest={{ resolv_conf }} 31 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/templates/db.mgmt.properties: -------------------------------------------------------------------------------- 1 | # 2 | # These are database credentials for databases 3 | # created by "cloudera-scm-server-db" for 4 | # Cloudera Manager Management Services, 5 | # to be used during the installation wizard if 6 | # the embedded database route is taken. 7 | # 8 | # The source of truth for these settings 9 | # is the Cloudera Manager databases and 10 | # changes made here will not be reflected 11 | # there automatically. 12 | # 13 | com.cloudera.cmf.ACTIVITYMONITOR.db.type=postgres 14 | com.cloudera.cmf.ACTIVITYMONITOR.db.host={{ ansible_nodename }}:5432 15 | com.cloudera.cmf.ACTIVITYMONITOR.db.name=amon 16 | com.cloudera.cmf.ACTIVITYMONITOR.db.user=amon 17 | com.cloudera.cmf.ACTIVITYMONITOR.db.password={{ amon_pass }} 18 | com.cloudera.cmf.REPORTSMANAGER.db.type=postgres 19 | com.cloudera.cmf.REPORTSMANAGER.db.host={{ ansible_nodename }}:5432 20 | com.cloudera.cmf.REPORTSMANAGER.db.name=rman 21 | com.cloudera.cmf.REPORTSMANAGER.db.user=rman 22 | com.cloudera.cmf.REPORTSMANAGER.db.password={{ rman_pass }} 23 | com.cloudera.cmf.NAVIGATOR.db.type=postgres 24 | com.cloudera.cmf.NAVIGATOR.db.host={{ ansible_nodename }}:5432 25 | com.cloudera.cmf.NAVIGATOR.db.name=nav 26 | com.cloudera.cmf.NAVIGATOR.db.user=nav 27 | com.cloudera.cmf.NAVIGATOR.db.password={{ nav_pass }} 28 | com.cloudera.cmf.NAVIGATORMETASERVER.db.type=postgres 29 | com.cloudera.cmf.NAVIGATORMETASERVER.db.host={{ ansible_nodename }}:5432 30 | com.cloudera.cmf.NAVIGATORMETASERVER.db.name=navms 31 | com.cloudera.cmf.NAVIGATORMETASERVER.db.user=navms 32 | com.cloudera.cmf.NAVIGATORMETASERVER.db.password={{ navms_pass }} 33 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/templates/db.properties: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Cloudera, Inc. All rights reserved. 2 | # 3 | # This file describes the database connection. 4 | # 5 | 6 | # The database type 7 | # Currently 'mysql', 'postgresql' and 'oracle' are valid databases. 8 | com.cloudera.cmf.db.type=postgresql 9 | 10 | # The database host 11 | # If a non standard port is needed, use 'hostname:port' 12 | com.cloudera.cmf.db.host={{ ansible_nodename }}:5432 13 | 14 | # The database name 15 | com.cloudera.cmf.db.name=scm 16 | 17 | # The database user 18 | com.cloudera.cmf.db.user=scm 19 | 20 | # The database user's password 21 | com.cloudera.cmf.db.password={{ scm_pass }} 22 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/templates/redhat-unbound-master.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | interface: {{ hostvars[groups['cm_node'][0]][['ansible_', hostvars[groups['cm_node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 4 | access-control: 127.0.0.0/8 allow 5 | access-control: 10.0.0.0/8 allow 6 | {% for node in groups['hadoop-cluster'] %} 7 | access-control: {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}/32 allow 8 | {% endfor %} 9 | domain-insecure: "{{ ansible_domain }}" 10 | local-zone: "10.in-addr.arpa." nodefault 11 | 12 | stub-zone: 13 | name: "10.in-addr.arpa." 14 | stub-addr: {{ hostvars[groups['cm_node'][0]][['ansible_', hostvars[groups['cm_node'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 15 | 16 | forward-zone: 17 | name: "." 18 | forward-addr: {{ dnsfwd[0] }} 19 | forward-addr: {{ dnsfwd[1] }} 20 | 21 | remote-control: 22 | control-interface: 127.0.0.1 23 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/templates/redhat-unbound-zonefile.j2: -------------------------------------------------------------------------------- 1 | # cluster zone 2 | 3 | local-zone: "{{ ansible_domain }}." transparent 4 | 5 | {% for node in groups['hadoop-cluster'] %} 6 | local-data: "{{ hostvars[node]['ansible_fqdn'] }}. IN A {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}" 7 | local-data-ptr: "{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_fqdn'] }}" 8 | 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | nameserver 127.0.0.1 2 | nameserver {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 3 | nameserver {{ dnsres[0] }} 4 | nameserver {{ dnsres[1] }} 5 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/vars/debian.yml: -------------------------------------------------------------------------------- 1 | package_info: 2 | pkg_mgr: apt 3 | args: 4 | name: "{{ item }}" 5 | update_cache: yes 6 | cache_valid_time: 600 7 | state: latest 8 | pkgs: 9 | - cloudera-manager-server 10 | - cloudera-manager-daemons 11 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/vars/defaults.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ansible_ssh_user: 'root' 3 | cluster_name: 'hadoop-poc' 4 | cloudera_version: '5' 5 | custom_repo: false 6 | install_spark: true 7 | install_flume: false 8 | install_hbase: false 9 | install_storm: false 10 | install_kafka: false 11 | install_falcon: false 12 | tachyon_service: false 13 | custom_blueprint: false 14 | admin_password: 'admin' 15 | services_password: 'AsdQwe123' 16 | alerts_contact: 'root@localhost.localdomain' 17 | 18 | 19 | spark_stack_config: 20 | tachyon_worker_memory: '1GB' 21 | spark_exector_memory: '1g' 22 | spark_driver_memory: '1g' 23 | spark_driver_maxResultSize: '512M' 24 | zeppelin_spark_executor_memory: '1g' 25 | zeppelin_spark_cores_max: '1' 26 | default_user: 'dgrier' 27 | 28 | hdfs: 29 | dfs_replication: 3 30 | failed_volumes_tolerated: 1 31 | ha_namenode: true 32 | 33 | cloud_config: 34 | rax_credentials_file: '~/.raxpub' 35 | rax_region: 'DFW' 36 | domain: 'localnet' 37 | allowed_external_ips: ['127.0.0.1'] 38 | ssh: 39 | keyname: 'hadoop-ssh-key' 40 | keyfile: '~/.ssh/id_rsa.pub' 41 | 42 | # set to true to show host variables 43 | debug: false 44 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mngr/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | pkgs: 2 | - cloudera-manager-server 3 | - cloudera-manager-daemons 4 | 5 | resolv_conf: "/etc/resolv.conf" 6 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/amon.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: amon 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `amon` 20 | -- 21 | 22 | ALTER DATABASE `amon` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `amon`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/dbs.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: amon 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `amon` 20 | -- 21 | 22 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `amon` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `amon`; 25 | 26 | -- 27 | -- Current Database: `rman` 28 | -- 29 | 30 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `rman` /*!40100 DEFAULT CHARACTER SET utf8 */; 31 | 32 | USE `rman`; 33 | 34 | -- 35 | -- Current Database: `sentry` 36 | -- 37 | 38 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `sentry` /*!40100 DEFAULT CHARACTER SET utf8 */; 39 | 40 | USE `sentry`; 41 | 42 | -- 43 | -- Current Database: `nav` 44 | -- 45 | 46 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `nav` /*!40100 DEFAULT CHARACTER SET utf8 */; 47 | 48 | USE `nav`; 49 | 50 | -- 51 | -- Current Database: `navms` 52 | -- 53 | 54 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `navms` /*!40100 DEFAULT CHARACTER SET utf8 */; 55 | 56 | USE `navms`; 57 | 58 | 59 | -- 60 | -- Current Database: `metastore` 61 | -- 62 | 63 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `metastore` /*!40100 DEFAULT CHARACTER SET utf8 */; 64 | 65 | USE `metastore`; 66 | 67 | 68 | -- 69 | -- Current Database: `scm` 70 | -- 71 | 72 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `scm` /*!40100 DEFAULT CHARACTER SET utf8 */; 73 | 74 | USE `scm`; 75 | 76 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 77 | 78 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 79 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 80 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 81 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 82 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 83 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 84 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 85 | 86 | -- Dump completed on 2016-02-22 20:56:52 87 | 88 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/hive-txn-schema-0.13.0.mysql.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one or more 2 | -- contributor license agreements. See the NOTICE file distributed with 3 | -- this work for additional information regarding copyright ownership. 4 | -- The ASF licenses this file to You under the Apache License, Version 2.0 5 | -- (the "License"); you may not use this file except in compliance with 6 | -- the License. You may obtain a copy of the License at 7 | -- 8 | -- http://www.apache.org/licenses/LICENSE-2.0 9 | -- 10 | -- Unless required by applicable law or agreed to in writing, software 11 | -- distributed under the License is distributed on an "AS IS" BASIS, 12 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | -- See the License for the specific language governing permissions and 14 | -- limitations under the License. 15 | 16 | -- 17 | -- Tables for transaction management 18 | -- 19 | 20 | CREATE TABLE TXNS ( 21 | TXN_ID bigint PRIMARY KEY, 22 | TXN_STATE char(1) NOT NULL, 23 | TXN_STARTED bigint NOT NULL, 24 | TXN_LAST_HEARTBEAT bigint NOT NULL, 25 | TXN_USER varchar(128) NOT NULL, 26 | TXN_HOST varchar(128) NOT NULL 27 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 28 | 29 | CREATE TABLE TXN_COMPONENTS ( 30 | TC_TXNID bigint, 31 | TC_DATABASE varchar(128) NOT NULL, 32 | TC_TABLE varchar(128), 33 | TC_PARTITION varchar(767), 34 | FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 36 | 37 | CREATE TABLE COMPLETED_TXN_COMPONENTS ( 38 | CTC_TXNID bigint, 39 | CTC_DATABASE varchar(128) NOT NULL, 40 | CTC_TABLE varchar(128), 41 | CTC_PARTITION varchar(767) 42 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 43 | 44 | CREATE TABLE NEXT_TXN_ID ( 45 | NTXN_NEXT bigint NOT NULL 46 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 47 | INSERT INTO NEXT_TXN_ID VALUES(1); 48 | 49 | CREATE TABLE HIVE_LOCKS ( 50 | HL_LOCK_EXT_ID bigint NOT NULL, 51 | HL_LOCK_INT_ID bigint NOT NULL, 52 | HL_TXNID bigint, 53 | HL_DB varchar(128) NOT NULL, 54 | HL_TABLE varchar(128), 55 | HL_PARTITION varchar(767), 56 | HL_LOCK_STATE char(1) not null, 57 | HL_LOCK_TYPE char(1) not null, 58 | HL_LAST_HEARTBEAT bigint NOT NULL, 59 | HL_ACQUIRED_AT bigint, 60 | HL_USER varchar(128) NOT NULL, 61 | HL_HOST varchar(128) NOT NULL, 62 | PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), 63 | KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) 64 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 65 | 66 | CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); 67 | 68 | CREATE TABLE NEXT_LOCK_ID ( 69 | NL_NEXT bigint NOT NULL 70 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 71 | INSERT INTO NEXT_LOCK_ID VALUES(1); 72 | 73 | CREATE TABLE COMPACTION_QUEUE ( 74 | CQ_ID bigint PRIMARY KEY, 75 | CQ_DATABASE varchar(128) NOT NULL, 76 | CQ_TABLE varchar(128) NOT NULL, 77 | CQ_PARTITION varchar(767), 78 | CQ_STATE char(1) NOT NULL, 79 | CQ_TYPE char(1) NOT NULL, 80 | CQ_WORKER_ID varchar(128), 81 | CQ_START bigint, 82 | CQ_RUN_AS varchar(128) 83 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 84 | 85 | CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( 86 | NCQ_NEXT bigint NOT NULL 87 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 88 | INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); 89 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/metastore.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: metastore 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `metastore` 20 | -- 21 | 22 | ALTER DATABASE `metastore` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `metastore`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/nav.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: nav 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `nav` 20 | -- 21 | 22 | ALTER DATABASE `nav` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `nav`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/navms.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: navms 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `navms` 20 | -- 21 | 22 | ALTER DATABASE `navms` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `navms`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/oozie.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: metastore 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `oozie` 20 | -- 21 | 22 | ALTER DATABASE `oozie` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `oozie`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/rman.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: rman 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `rman` 20 | -- 21 | 22 | ALTER DATABASE `rman` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `rman`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/scm.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: scm 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `scm` 20 | -- 21 | 22 | ALTER DATABASE `scm` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `scm`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 37 | 38 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 39 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 40 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 41 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 42 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 43 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 44 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 45 | 46 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/sentry.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: sentry 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `sentry` 20 | -- 21 | 22 | ALTER DATABASE `sentry` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `sentry`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/files/sqoop.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: sqoop 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `sqoop` 20 | -- 21 | 22 | ALTER DATABASE `sqoop` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `sqoop`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/scripts/hive-schema-mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PATH=/opt/cloudera/; 4 | 5 | cd $PATH 6 | 7 | 8 | /usr/bin/mysql -h$1 -u$2 -p$3 $4 < hive-schema-1.1.0.mysql.sql; 9 | 10 | 11 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/templates/my.cnf-mariadb: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | transaction-isolation = READ-COMMITTED 3 | # Disabling symbolic-links is recommended to prevent assorted security risks; 4 | # to do so, uncomment this line: 5 | # symbolic-links = 0 6 | 7 | key_buffer = 16M 8 | key_buffer_size = 32M 9 | max_allowed_packet = 32M 10 | thread_stack = 256K 11 | thread_cache_size = 64 12 | query_cache_limit = 8M 13 | query_cache_size = 64M 14 | query_cache_type = 1 15 | 16 | max_connections = 550 17 | 18 | #log_bin should be on a disk with enough free space. Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your system and chown the specified folder to the mysql user. 19 | #log_bin=/var/lib/mysql/mysql_binary_log 20 | #expire_logs_days = 10 21 | #max_binlog_size = 100M 22 | 23 | # For MySQL version 5.1.8 or later. Comment out binlog_format for older versions. 24 | binlog_format = mixed 25 | 26 | read_buffer_size = 2M 27 | read_rnd_buffer_size = 16M 28 | sort_buffer_size = 8M 29 | join_buffer_size = 8M 30 | 31 | # InnoDB settings 32 | innodb_file_per_table = 1 33 | innodb_flush_log_at_trx_commit = 2 34 | innodb_log_buffer_size = 64M 35 | innodb_buffer_pool_size = 4G 36 | innodb_thread_concurrency = 8 37 | innodb_flush_method = O_DIRECT 38 | innodb_log_file_size = 512M 39 | 40 | [mysqld_safe] 41 | log-error=/var/log/mariadb/mariadb.log 42 | pid-file=/var/run/mariadb/mariadb.pid 43 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/vars/mysql_vars.yml: -------------------------------------------------------------------------------- 1 | services_pass: 'temp' 2 | mysql_root_pass: 'temp' 3 | amon_pass: "{{ services_pass }}" 4 | rman_pass: "{{ services_pass }}" 5 | sentry_pass: "{{ services_pass }}" 6 | nav_pass: "{{ services_pass }}" 7 | navms_pass: "{{ services_pass }}" 8 | hive_pass: "{{ services_pass }}" 9 | sqoop_pass: "{{ services_pass }}" 10 | oozie_pass: "{{ services_pass }}" 11 | scm_pass: "{{ services_pass }}" 12 | db_names: 13 | - amon 14 | - metastore 15 | - navms 16 | - nav 17 | - rman 18 | - sentry 19 | - oozie 20 | - sqoop 21 | - scm 22 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/vars/redhat-6.yml: -------------------------------------------------------------------------------- 1 | pkgs: 2 | - mysql-server 3 | - mysql-connector-java 4 | 5 | mycnf_template: my.cnf-mysqld 6 | 7 | mysql_service: mysqld 8 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-mysql/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | pkgs: 2 | - mariadb-server 3 | - mysql-connector-java 4 | 5 | mycnf_template: my.cnf-mariadb 6 | 7 | mysql_service: mariadb 8 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/amon.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: amon 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `amon` 20 | -- 21 | 22 | ALTER DATABASE `amon` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `amon`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/dbs.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: amon 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `amon` 20 | -- 21 | 22 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `amon` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `amon`; 25 | 26 | -- 27 | -- Current Database: `rman` 28 | -- 29 | 30 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `rman` /*!40100 DEFAULT CHARACTER SET utf8 */; 31 | 32 | USE `rman`; 33 | 34 | -- 35 | -- Current Database: `sentry` 36 | -- 37 | 38 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `sentry` /*!40100 DEFAULT CHARACTER SET utf8 */; 39 | 40 | USE `sentry`; 41 | 42 | -- 43 | -- Current Database: `nav` 44 | -- 45 | 46 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `nav` /*!40100 DEFAULT CHARACTER SET utf8 */; 47 | 48 | USE `nav`; 49 | 50 | -- 51 | -- Current Database: `navms` 52 | -- 53 | 54 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `navms` /*!40100 DEFAULT CHARACTER SET utf8 */; 55 | 56 | USE `navms`; 57 | 58 | 59 | -- 60 | -- Current Database: `metastore` 61 | -- 62 | 63 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `metastore` /*!40100 DEFAULT CHARACTER SET utf8 */; 64 | 65 | USE `metastore`; 66 | 67 | 68 | -- 69 | -- Current Database: `scm` 70 | -- 71 | 72 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `scm` /*!40100 DEFAULT CHARACTER SET utf8 */; 73 | 74 | USE `scm`; 75 | 76 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 77 | 78 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 79 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 80 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 81 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 82 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 83 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 84 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 85 | 86 | -- Dump completed on 2016-02-22 20:56:52 87 | 88 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/hive-txn-schema-0.13.0.mysql.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one or more 2 | -- contributor license agreements. See the NOTICE file distributed with 3 | -- this work for additional information regarding copyright ownership. 4 | -- The ASF licenses this file to You under the Apache License, Version 2.0 5 | -- (the "License"); you may not use this file except in compliance with 6 | -- the License. You may obtain a copy of the License at 7 | -- 8 | -- http://www.apache.org/licenses/LICENSE-2.0 9 | -- 10 | -- Unless required by applicable law or agreed to in writing, software 11 | -- distributed under the License is distributed on an "AS IS" BASIS, 12 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | -- See the License for the specific language governing permissions and 14 | -- limitations under the License. 15 | 16 | -- 17 | -- Tables for transaction management 18 | -- 19 | 20 | CREATE TABLE TXNS ( 21 | TXN_ID bigint PRIMARY KEY, 22 | TXN_STATE char(1) NOT NULL, 23 | TXN_STARTED bigint NOT NULL, 24 | TXN_LAST_HEARTBEAT bigint NOT NULL, 25 | TXN_USER varchar(128) NOT NULL, 26 | TXN_HOST varchar(128) NOT NULL 27 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 28 | 29 | CREATE TABLE TXN_COMPONENTS ( 30 | TC_TXNID bigint, 31 | TC_DATABASE varchar(128) NOT NULL, 32 | TC_TABLE varchar(128), 33 | TC_PARTITION varchar(767), 34 | FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 36 | 37 | CREATE TABLE COMPLETED_TXN_COMPONENTS ( 38 | CTC_TXNID bigint, 39 | CTC_DATABASE varchar(128) NOT NULL, 40 | CTC_TABLE varchar(128), 41 | CTC_PARTITION varchar(767) 42 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 43 | 44 | CREATE TABLE NEXT_TXN_ID ( 45 | NTXN_NEXT bigint NOT NULL 46 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 47 | INSERT INTO NEXT_TXN_ID VALUES(1); 48 | 49 | CREATE TABLE HIVE_LOCKS ( 50 | HL_LOCK_EXT_ID bigint NOT NULL, 51 | HL_LOCK_INT_ID bigint NOT NULL, 52 | HL_TXNID bigint, 53 | HL_DB varchar(128) NOT NULL, 54 | HL_TABLE varchar(128), 55 | HL_PARTITION varchar(767), 56 | HL_LOCK_STATE char(1) not null, 57 | HL_LOCK_TYPE char(1) not null, 58 | HL_LAST_HEARTBEAT bigint NOT NULL, 59 | HL_ACQUIRED_AT bigint, 60 | HL_USER varchar(128) NOT NULL, 61 | HL_HOST varchar(128) NOT NULL, 62 | PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), 63 | KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) 64 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 65 | 66 | CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); 67 | 68 | CREATE TABLE NEXT_LOCK_ID ( 69 | NL_NEXT bigint NOT NULL 70 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 71 | INSERT INTO NEXT_LOCK_ID VALUES(1); 72 | 73 | CREATE TABLE COMPACTION_QUEUE ( 74 | CQ_ID bigint PRIMARY KEY, 75 | CQ_DATABASE varchar(128) NOT NULL, 76 | CQ_TABLE varchar(128) NOT NULL, 77 | CQ_PARTITION varchar(767), 78 | CQ_STATE char(1) NOT NULL, 79 | CQ_TYPE char(1) NOT NULL, 80 | CQ_WORKER_ID varchar(128), 81 | CQ_START bigint, 82 | CQ_RUN_AS varchar(128) 83 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 84 | 85 | CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( 86 | NCQ_NEXT bigint NOT NULL 87 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 88 | INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); 89 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/metastore.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: metastore 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `metastore` 20 | -- 21 | 22 | ALTER DATABASE `metastore` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `metastore`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/nav.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: nav 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `nav` 20 | -- 21 | 22 | ALTER DATABASE `nav` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `nav`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/navms.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: navms 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `navms` 20 | -- 21 | 22 | ALTER DATABASE `navms` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `navms`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/oozie.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: metastore 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `oozie` 20 | -- 21 | 22 | ALTER DATABASE `oozie` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `oozie`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/postgresql-9.6.service: -------------------------------------------------------------------------------- 1 | # It's not recommended to modify this file in-place, because it will be 2 | # overwritten during package upgrades. If you want to customize, the 3 | # best way is to create a file "/etc/systemd/system/postgresql-9.6.service", 4 | # containing 5 | # .include /lib/systemd/system/postgresql-9.6.service 6 | # ...make your changes here... 7 | # For more info about custom unit files, see 8 | # http://fedoraproject.org/wiki/Systemd#How_do_I_customize_a_unit_file.2F_add_a_custom_unit_file.3F 9 | 10 | # Note: changing PGDATA will typically require adjusting SELinux 11 | # configuration as well. 12 | 13 | # Note: do not use a PGDATA pathname containing spaces, or you will 14 | # break postgresql-setup. 15 | [Unit] 16 | Description=PostgreSQL 9.6 database server 17 | Documentation=https://www.postgresql.org/docs/9.6/static/ 18 | After=syslog.target 19 | After=network.target 20 | 21 | [Service] 22 | Type=notify 23 | 24 | User=postgres 25 | Group=postgres 26 | 27 | # Note: avoid inserting whitespace in these Environment= lines, or you may 28 | # break postgresql-setup. 29 | 30 | # Location of database directory 31 | Environment=PGDATA=/var/lib/sql/postgres/ 32 | 33 | # Where to send early-startup messages from the server (before the logging 34 | # options of postgresql.conf take effect) 35 | # This is normally controlled by the global default set by systemd 36 | # StandardOutput=syslog 37 | 38 | # Disable OOM kill on the postmaster 39 | OOMScoreAdjust=-1000 40 | Environment=PG_OOM_ADJUST_FILE=/proc/self/oom_score_adj 41 | 42 | ExecStartPre=/usr/pgsql-9.6/bin/postgresql96-check-db-dir ${PGDATA} 43 | ExecStart=/usr/pgsql-9.6/bin/postmaster -D ${PGDATA} 44 | ExecReload=/bin/kill -HUP $MAINPID 45 | KillMode=mixed 46 | KillSignal=SIGINT 47 | 48 | 49 | # Do not set any timeout value, so that systemd will not kill postmaster 50 | # during crash recovery. 51 | TimeoutSec=0 52 | 53 | [Install] 54 | WantedBy=multi-user.target 55 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/rman.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: rman 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `rman` 20 | -- 21 | 22 | ALTER DATABASE `rman` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `rman`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/scm.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: scm 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `scm` 20 | -- 21 | 22 | ALTER DATABASE `scm` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `scm`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 37 | 38 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 39 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 40 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 41 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 42 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 43 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 44 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 45 | 46 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/sentry.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: sentry 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `sentry` 20 | -- 21 | 22 | ALTER DATABASE `sentry` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `sentry`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/files/sqoop.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.14 Distrib 5.5.44-MariaDB, for Linux (x86_64) 2 | -- 3 | -- Host: localhost Database: sqoop 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.44-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `sqoop` 20 | -- 21 | 22 | ALTER DATABASE `sqoop` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `sqoop`; 25 | 26 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 27 | 28 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 29 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 30 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 31 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 32 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 33 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 34 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 35 | 36 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/tasks/hive_postgres_schema.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: setup metastore schema 4 | copy: 5 | src: "hive-schema-3.1.2000.postgres.sql" 6 | dest: "{{ tmp_dir }}/" 7 | 8 | 9 | - name: setup Hive Metastore schema 10 | shell: psql -U hive -h localhost metastore < {{ tmp_dir }}/hive-schema-3.1.2000.postgres.sql 11 | environment: PGPASSWD={{ services_pass }} 12 | 13 | 14 | - name: Cleanup tmp files 15 | file: 16 | path: "{{ tmp_dir }}/hive-schema-3.1.2000.postgres.sql" 17 | state: absent 18 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/tasks/pg_dbs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load OS specific variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version }}.yml" 7 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml" 8 | - "{{ ansible_os_family|lower }}.yml" 9 | - defaults.yml 10 | paths: 11 | - ../vars 12 | 13 | - include_vars: group_vars/cloudera 14 | 15 | - include_vars: group_vars/postgres_vars.yml 16 | 17 | - name: setup dbs 18 | block: 19 | - name: Create a new database 20 | postgresql_db: 21 | name: scm 22 | owner: scm 23 | 24 | - name: Create a new database 25 | postgresql_db: 26 | name: amon 27 | owner: amon 28 | 29 | - name: Create a new database 30 | postgresql_db: 31 | name: rman 32 | owner: rman 33 | 34 | - name: Create a new database 35 | postgresql_db: 36 | name: hue 37 | owner: hue 38 | 39 | - name: Create a new database 40 | postgresql_db: 41 | name: metastore 42 | owner: hive 43 | 44 | - name: Create a new database 45 | postgresql_db: 46 | name: oozie 47 | owner: oozie 48 | 49 | - name: Create a new database 50 | postgresql_db: 51 | name: das 52 | owner: das 53 | 54 | - name: Create a new database 55 | postgresql_db: 56 | name: ranger 57 | owner: rangeradmin 58 | 59 | - name: Create a new database 60 | postgresql_db: 61 | name: nav 62 | owner: nav 63 | 64 | - name: Create a new database 65 | postgresql_db: 66 | name: navms 67 | owner: navms 68 | 69 | become: true 70 | become_method: sudo 71 | become_user: postgres 72 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/tasks/pg_user.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load OS specific variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version }}.yml" 7 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml" 8 | - "{{ ansible_os_family|lower }}.yml" 9 | - defaults.yml 10 | paths: 11 | - ../vars 12 | 13 | - include_vars: group_vars/cloudera 14 | 15 | - include_vars: group_vars/postgres_vars.yml 16 | 17 | - name: setup postgres db owner users 18 | postgresql_user: 19 | name: "{{ item }}" 20 | password: "{{ services_pass }}" 21 | with_items: 22 | - scm 23 | - amon 24 | - rman 25 | - nav 26 | - navms 27 | - hue 28 | - hive 29 | - oozie 30 | - das 31 | - rangeradmin 32 | become: true 33 | become_method: sudo 34 | become_user: postgres 35 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/tasks/postgres_repo_install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: download postgres repo 4 | block: 5 | - name: download postgres repo rpm 6 | get_url: 7 | url: "{{ pdgd_repo_rpm_url }}{{ pdgd_repo_rpm }}" 8 | dest: "{{ tmp_dir }}/" 9 | 10 | - name: install postgres repo 11 | yum: 12 | name: "{{ tmp_dir }}/{{ pdgd_repo_rpm }}" 13 | update_cache: yes 14 | state: installed 15 | when: ansible_os_family == "RedHat" 16 | 17 | - name: Cleanup tmp files 18 | file: 19 | path: "{{ tmp_dir }}/{{ pdgd_repo_rpm }}" 20 | state: absent 21 | become: true 22 | become_user: root 23 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/vars/redhat-6.yml: -------------------------------------------------------------------------------- 1 | pkgs: 2 | - mysql-server 3 | - mysql-connector-java 4 | 5 | mycnf_template: my.cnf-mysqld 6 | 7 | mysql_service: mysqld 8 | -------------------------------------------------------------------------------- /playbooks/roles/cloudera-postgres/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | pkgs: 2 | - postgresql96-server 3 | - python-pip 4 | - postgresql-jdbc 5 | -------------------------------------------------------------------------------- /playbooks/roles/cni/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | calico_cni_opts: "interface={{ network_interface }}" 4 | flannel_cni_opts: "--iface={{ network_interface }}" 5 | 6 | -------------------------------------------------------------------------------- /playbooks/roles/cni/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Create Kubernetes addon directory 4 | file: 5 | path: "{{ network_dir }}" 6 | state: directory 7 | 8 | - name: "Copy {{ network }} YAML files" 9 | template: 10 | src: "{{ item }}" 11 | dest: "{{ network_dir }}/{{ item | basename | regex_replace('\\.j2','') }}" 12 | with_fileglob: 13 | - ../templates/{{ network }}*.j2 14 | 15 | - name: "Check {{ network }} daemonset is working" 16 | shell: kubectl --kubeconfig={{ kubeadmin_config }} get ds --all-namespaces | grep {{ network }} 17 | delegate_to: "{{ groups['kube-masters'][0] }}" 18 | run_once: true 19 | register: check_net 20 | ignore_errors: true 21 | changed_when: false 22 | 23 | - name: "Create {{ network }} network daemonset" 24 | when: check_net is failed 25 | command: kubectl apply --kubeconfig={{ kubeadmin_config }} -f {{ network_dir }}/ 26 | delegate_to: "{{ groups['kube-masters'][0] }}" 27 | run_once: true 28 | -------------------------------------------------------------------------------- /playbooks/roles/cni/templates/calico-etcd.yml.j2-bak: -------------------------------------------------------------------------------- 1 | # This manifest installs the Calico etcd on the kubeadm master. This uses a DaemonSet 2 | # to force it to run on the master even when the master isn't schedulable, and uses 3 | # nodeSelector to ensure it only runs on the master. 4 | apiVersion: extensions/v1beta1 5 | kind: DaemonSet 6 | metadata: 7 | name: calico-etcd 8 | namespace: kube-system 9 | labels: 10 | k8s-app: calico-etcd 11 | spec: 12 | template: 13 | metadata: 14 | labels: 15 | k8s-app: calico-etcd 16 | annotations: 17 | # Mark this pod as a critical add-on; when enabled, the critical add-on scheduler 18 | # reserves resources for critical add-on pods so that they can be rescheduled after 19 | # a failure. This annotation works in tandem with the toleration below. 20 | scheduler.alpha.kubernetes.io/critical-pod: '' 21 | spec: 22 | tolerations: 23 | # This taint is set by all kubelets running `--cloud-provider=external` 24 | # so we should tolerate it to schedule the Calico pods 25 | - key: node.cloudprovider.kubernetes.io/uninitialized 26 | value: "true" 27 | effect: NoSchedule 28 | # Allow this pod to run on the master. 29 | - key: node-role.kubernetes.io/master 30 | effect: NoSchedule 31 | # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode. 32 | # This, along with the annotation above marks this pod as a critical add-on. 33 | - key: CriticalAddonsOnly 34 | operator: Exists 35 | # Only run this pod on the master. 36 | nodeSelector: 37 | node-role.kubernetes.io/master: "" 38 | hostNetwork: true 39 | containers: 40 | - name: calico-etcd 41 | image: quay.io/coreos/etcd:v3.3.9 42 | env: 43 | - name: CALICO_ETCD_IP 44 | valueFrom: 45 | fieldRef: 46 | fieldPath: status.podIP 47 | command: 48 | - /usr/local/bin/etcd 49 | args: 50 | - --name=calico 51 | - --data-dir=/var/etcd/calico-data 52 | - --advertise-client-urls=http://$(CALICO_ETCD_IP):6666 53 | - --listen-client-urls=http://0.0.0.0:6666 54 | - --listen-peer-urls=http://0.0.0.0:6667 55 | - --auto-compaction-retention=1 56 | volumeMounts: 57 | - name: var-etcd 58 | mountPath: /var/etcd 59 | volumes: 60 | - name: var-etcd 61 | hostPath: 62 | path: /var/etcd 63 | 64 | --- 65 | 66 | # This manifest installs the Service which gets traffic to the Calico 67 | # etcd. 68 | apiVersion: v1 69 | kind: Service 70 | metadata: 71 | labels: 72 | k8s-app: calico-etcd 73 | name: calico-etcd 74 | namespace: kube-system 75 | spec: 76 | # Select the calico-etcd pod running on the master. 77 | selector: 78 | k8s-app: calico-etcd 79 | # This ClusterIP needs to be known in advance, since we cannot rely 80 | # on DNS to get access to etcd. 81 | clusterIP: {{ calico_etcd_service }} 82 | ports: 83 | - port: 6666 84 | -------------------------------------------------------------------------------- /playbooks/roles/cni/templates/calico-rbac.yml.j2: -------------------------------------------------------------------------------- 1 | # Calico Version v3.3.0 2 | # https://docs.projectcalico.org/v3.3/releases#v3.3.0 3 | 4 | --- 5 | 6 | kind: ClusterRole 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | metadata: 9 | name: calico-kube-controllers 10 | rules: 11 | - apiGroups: 12 | - "" 13 | - extensions 14 | resources: 15 | - pods 16 | - namespaces 17 | - networkpolicies 18 | - nodes 19 | - serviceaccounts 20 | verbs: 21 | - watch 22 | - list 23 | - apiGroups: 24 | - networking.k8s.io 25 | resources: 26 | - networkpolicies 27 | verbs: 28 | - watch 29 | - list 30 | --- 31 | kind: ClusterRoleBinding 32 | apiVersion: rbac.authorization.k8s.io/v1beta1 33 | metadata: 34 | name: calico-kube-controllers 35 | roleRef: 36 | apiGroup: rbac.authorization.k8s.io 37 | kind: ClusterRole 38 | name: calico-kube-controllers 39 | subjects: 40 | - kind: ServiceAccount 41 | name: calico-kube-controllers 42 | namespace: kube-system 43 | 44 | --- 45 | 46 | kind: ClusterRole 47 | apiVersion: rbac.authorization.k8s.io/v1beta1 48 | metadata: 49 | name: calico-node 50 | rules: 51 | - apiGroups: [""] 52 | resources: 53 | - pods 54 | - nodes 55 | - namespaces 56 | verbs: 57 | - get 58 | 59 | --- 60 | 61 | apiVersion: rbac.authorization.k8s.io/v1beta1 62 | kind: ClusterRoleBinding 63 | metadata: 64 | name: calico-node 65 | roleRef: 66 | apiGroup: rbac.authorization.k8s.io 67 | kind: ClusterRole 68 | name: calico-node 69 | subjects: 70 | - kind: ServiceAccount 71 | name: calico-node 72 | namespace: kube-system 73 | -------------------------------------------------------------------------------- /playbooks/roles/cni/templates/flannel-rbac.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ClusterRole 3 | apiVersion: rbac.authorization.k8s.io/v1beta1 4 | metadata: 5 | name: flannel 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | verbs: 12 | - get 13 | - apiGroups: 14 | - "" 15 | resources: 16 | - nodes 17 | verbs: 18 | - list 19 | - watch 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - nodes/status 24 | verbs: 25 | - patch 26 | --- 27 | kind: ClusterRoleBinding 28 | apiVersion: rbac.authorization.k8s.io/v1beta1 29 | metadata: 30 | name: flannel 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: flannel 35 | subjects: 36 | - kind: ServiceAccount 37 | name: flannel 38 | namespace: kube-system 39 | --- 40 | apiVersion: v1 41 | kind: ServiceAccount 42 | metadata: 43 | name: flannel 44 | namespace: kube-system 45 | --- 46 | -------------------------------------------------------------------------------- /playbooks/roles/common/files/isdct-3.0.2.400-17.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/objectrocket/ansible-hadoop/d0d9dec5330d20ecce85e8ea29ebc38f27cde347/playbooks/roles/common/files/isdct-3.0.2.400-17.x86_64.rpm -------------------------------------------------------------------------------- /playbooks/roles/common/files/tuned.conf: -------------------------------------------------------------------------------- 1 | # 2 | # tuned configuration 3 | # 4 | 5 | [main] 6 | include=throughput-performance 7 | 8 | [vm] 9 | transparent_hugepages=never 10 | 11 | [sysctl] 12 | vm.swappiness = 1 13 | -------------------------------------------------------------------------------- /playbooks/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart iptables 3 | service: name=iptables state=restarted enabled=yes 4 | 5 | - name: Reload firewalld 6 | command: firewall-cmd --reload 7 | 8 | - name: Run update-grub 9 | shell: "{{ update_grub }}" 10 | when: ansible_distribution_major_version|int > 6 11 | 12 | - name: Run resolvconf 13 | shell: resolvconf -u 14 | when: ansible_os_family == "Debian" 15 | 16 | - name: Restart network interfaces 17 | shell: ifdown {{ item }}; ifup {{ item }}; ifup {{ cluster_interface }} 18 | with_flattened: 19 | - cluster_interface 20 | - bond_interfaces 21 | 22 | - name: Run depmod 23 | command: depmod 24 | -------------------------------------------------------------------------------- /playbooks/roles/common/tasks/bonding.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Write bonding.conf file 3 | lineinfile: dest=/etc/modprobe.d/bonding.conf line='alias {{ cluster_interface }} bonding' state=present create=yes mode=0664 owner=root group=root 4 | notify: Run depmod 5 | 6 | - meta: flush_handlers 7 | 8 | - name: Make sure the interfaces.d files are included 9 | lineinfile: line="source /etc/network/interfaces.d/ifcfg-*" 10 | dest=/etc/network/interfaces 11 | state=present 12 | insertafter=EOF 13 | when: ansible_os_family == "Debian" 14 | 15 | - name: Write ifcfg files for bonded interfaces 16 | template: src={{ ansible_os_family|lower }}-ifcfg-eth.j2 dest={{ networking_path }}/ifcfg-{{ item }} 17 | with_items: "{{ bond_interfaces }}" 18 | notify: Restart network interfaces 19 | 20 | - name: Write ifcfg file for bond interface 21 | template: src={{ ansible_os_family|lower }}-ifcfg-bond.j2 dest={{ networking_path }}/ifcfg-{{ cluster_interface }} 22 | notify: Restart network interfaces 23 | 24 | - meta: flush_handlers 25 | 26 | - name: Test slave interfaces 27 | include: slavetest.yml 28 | with_items: "{{ bond_interfaces }}" 29 | loop_control: 30 | loop_var: slave_interface 31 | 32 | - name: Ping between all hosts on bond interface to verify network connectivity 33 | command: /bin/ping -q -c 5 -i 0.2 -W 1 {{ hostvars[item]['bond_ip'] }} 34 | when: bond_ip is defined and hostvars[item]['bond_ip'] is defined 35 | with_items: "{{ groups['all'] }}" 36 | changed_when: 0 37 | -------------------------------------------------------------------------------- /playbooks/roles/common/tasks/firewall.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set iptables between cluster nodes 3 | lineinfile: dest=/etc/sysconfig/iptables 4 | insertbefore="^-A INPUT" 5 | line="-A INPUT -s {{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}/32 -j ACCEPT" 6 | state=present 7 | with_items: "{{ play_hosts }}" 8 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" 9 | notify: Restart iptables 10 | 11 | - name: Set iptables to allow cluster access from external IPs 12 | lineinfile: dest=/etc/sysconfig/iptables 13 | insertbefore="^-A INPUT" 14 | line="-A INPUT -s {{ item }}/32 -j ACCEPT" 15 | state=present 16 | with_items: "{{ cloud_config.allowed_external_ips }}" 17 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" 18 | notify: Restart iptables 19 | 20 | - name: Set firewalld rules between cluster nodes 21 | command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" accept' 22 | # firewalld: source={{ hostvars[item][['ansible_', hostvars[item]['cluster_interface']]|join]['ipv4']['address'] }} state=enabled 23 | with_items: "{{ play_hosts }}" 24 | notify: Reload firewalld 25 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 26 | 27 | - name: Set firewalld to allow cluster access from external IPs 28 | command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ item }}" accept' 29 | # firewalld: source={{ item }} state=enabled 30 | with_items: "{{ cloud_config.allowed_external_ips }}" 31 | notify: Reload firewalld 32 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 33 | 34 | - name: Set UFW rules between cluster nodes 35 | ufw: rule=allow src={{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }} 36 | with_items: "{{ play_hosts }}" 37 | when: ansible_distribution == "Ubuntu" 38 | 39 | - name: Set UFW to allow cluster access from external IPs 40 | ufw: rule=allow src={{ item }} 41 | with_items: "{{ cloud_config.allowed_external_ips }}" 42 | when: ansible_distribution == "Ubuntu" 43 | -------------------------------------------------------------------------------- /playbooks/roles/common/tasks/partitioning.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get disk alignment for disk {{ item }} 3 | shell: if [[ -e /sys/block/{{ item }}/queue/optimal_io_size && -e /sys/block/{{ item }}/alignment_offset && -e /sys/block/{{ item }}/queue/physical_block_size ]]; then echo $[$(( ($(cat /sys/block/{{ item }}/queue/optimal_io_size) + $(cat /sys/block/{{ item }}/alignment_offset)) / $(cat /sys/block/{{ item }}/queue/physical_block_size) )) | 2048]; else echo 2048; fi 4 | when: ansible_devices[item] is defined 5 | register: disk_offset 6 | 7 | - name: Partition disk {{ item }} 8 | shell: parted -s -a optimal /dev/{{ item }} mklabel gpt mkpart primary {{ data_disks_filesystem }} {{ disk_offset.stdout|default("2048") }}s 100% && sleep 5 && partprobe /dev/{{ item }}; sleep 5 9 | when: ansible_devices[item] is defined and ansible_devices[item]['partitions'] == {} 10 | 11 | - name: Create the filesystem on disk {{ item }} 12 | filesystem: dev="/dev/{{ item }}1" fstype="{{ data_disks_filesystem }}" 13 | when: ansible_devices[item] is defined 14 | 15 | - name: Disable periodic fsck on {{ item }} 16 | shell: tune2fs -c0 -i0 /dev/{{ item }}1 17 | when: ansible_devices[item] is defined and (data_disks_filesystem == "ext4" or data_disks_filesystem == "ext3") 18 | -------------------------------------------------------------------------------- /playbooks/roles/common/tasks/slavetest.yml: -------------------------------------------------------------------------------- 1 | - name: "Ensure all slaves are enabled" 2 | shell: ifup {{ item }} 3 | with_items: "{{ bond_interfaces }}" 4 | 5 | - name: "Disable all slaves other than {{ slave_interface }}" 6 | shell: ifdown {{ item }} 7 | with_items: "{{ bond_interfaces }}" 8 | when: item != slave_interface 9 | 10 | - name: Pause to allow bond to come online 11 | pause: seconds=5 12 | 13 | - name: "Ping all hosts using {{ slave_interface }}" 14 | shell: ping -q -c 5 -i 0.2 -W 1 {{ hostvars[item]['bond_ip'] }} 15 | when: bond_ip is defined and hostvars[item]['bond_ip'] is defined 16 | with_items: "{{ groups['all'] }}" 17 | changed_when: 0 18 | register: result 19 | retries: 1 20 | delay: 1 21 | until: result.rc == 0 22 | ignore_errors: yes 23 | 24 | - name: Re-enable slaves 25 | shell: ifup {{ item }} 26 | with_items: "{{ bond_interfaces }}" 27 | when: item != slave_interface 28 | 29 | - name: Pause if ping test failed 30 | pause: prompt="Ping test failed using {{ slave_interface }}. Would you like to proceed anyway? Press Enter to continue or ctrl-c A to abort" 31 | when: result|failed 32 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/debian-ifcfg-bond.j2: -------------------------------------------------------------------------------- 1 | auto {{ cluster_interface }} 2 | iface {{ cluster_interface }} inet static 3 | address {{ bond_ip }} 4 | netmask {{ bond_netmask }} 5 | bond-mode active-backup 6 | bond-miimon 100 7 | bond-slaves none 8 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/debian-ifcfg-eth.j2: -------------------------------------------------------------------------------- 1 | auto {{ item }} 2 | iface {{ item }} inet manual 3 | bond-master {{ cluster_interface }} 4 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/hosts.j2: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | {% for node in groups['hadoop-cluster'] %} 4 | {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_nodename'] }} {{ hostvars[node]['ansible_hostname'] }} 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/kafka_hosts.j2: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | {% for node in groups['kafka-cluster'] %} 4 | {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_nodename'] }} {{ hostvars[node]['ansible_hostname'] }} 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/mdadm.conf.j2: -------------------------------------------------------------------------------- 1 | {{ mdadm_scan.stdout }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/redhat-ifcfg-bond.j2: -------------------------------------------------------------------------------- 1 | DEVICE={{ cluster_interface }} 2 | IPADDR={{ bond_ip }} 3 | NETMASK={{ bond_netmask }} 4 | NETWORK= 5 | BROADCAST= 6 | ONBOOT=yes 7 | BOOTPROTO=none 8 | USERCTL=no 9 | BONDING_OPTS="mode=active-backup miimon=100" 10 | NM_CONTROLLED=no 11 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/redhat-ifcfg-eth.j2: -------------------------------------------------------------------------------- 1 | DEVICE={{ item }} 2 | ONBOOT=yes 3 | BOOTPROTO=none 4 | USERCTL=no 5 | MASTER={{ cluster_interface }} 6 | SLAVE=yes 7 | NM_CONTROLLED=no 8 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/redhat-unbound-master.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | interface: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 4 | access-control: 127.0.0.0/8 allow 5 | access-control: 10.0.0.0/8 allow 6 | domain-insecure: "{{ ansible_domain }}" 7 | local-zone: "10.in-addr.arpa." nodefault 8 | 9 | stub-zone: 10 | name: "10.in-addr.arpa." 11 | stub-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 12 | 13 | forward-zone: 14 | name: "." 15 | forward-addr: 8.8.8.8 16 | forward-addr: 8.8.4.4 17 | 18 | remote-control: 19 | control-interface: 127.0.0.1 20 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/redhat-unbound-zonefile.j2: -------------------------------------------------------------------------------- 1 | # Local zone 2 | 3 | local-zone: "{{ ansible_domain }}." static 4 | 5 | {% for node in groups['hadoop-cluster'] %} 6 | local-data: "{{ hostvars[node]['ansible_fqdn'] }}. IN A {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}" 7 | local-data-ptr: "{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_fqdn'] }}" 8 | 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/redhat-unbound.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | access-control: 127.0.0.0/8 allow 4 | access-control: 10.0.0.0/8 allow 5 | domain-insecure: "{{ ansible_domain }}" 6 | local-zone: "10.in-addr.arpa." nodefault 7 | 8 | stub-zone: 9 | name: "10.in-addr.arpa." 10 | stub-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 11 | 12 | forward-zone: 13 | name: "." 14 | forward-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 15 | 16 | remote-control: 17 | control-interface: 127.0.0.1 18 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | domain {{ ansible_domain }} 2 | nameserver 173.203.4.8 3 | nameserver 173.203.4.9 4 | search {{ ansible_domain }} 5 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/sparkstack-repo.j2: -------------------------------------------------------------------------------- 1 | /dev/block/202:1: UUID="39b71bf0-828f-4fd2-acb0-555b9501de41" TYPE="ext4" 2 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/debian.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - python-simplejson 3 | - python-httplib2 4 | - man-db 5 | - vim 6 | - sysstat 7 | - lsof 8 | - telnet 9 | - netcat 10 | - python-selinux 11 | - ntp 12 | - parted 13 | - mc 14 | - perl 15 | - iotop 16 | - links 17 | - unzip 18 | - curl 19 | - wget 20 | - ifenslave 21 | - xfsprogs 22 | 23 | update_grub: "/usr/sbin/update-grub" 24 | 25 | ntp_service: ntp 26 | ntp_sync: "/usr/sbin/ntpd -gq" 27 | 28 | resolv_conf: "/etc/resolvconf/resolv.conf.d/tail" 29 | 30 | networking_path: "/etc/network/interfaces.d" 31 | networking_eth: "debian-ifcfg-eth.j2" 32 | networking_bond: "debian-ifcfg-bond.j2" 33 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/redhat-6.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - python-simplejson 3 | - python-httplib2 4 | - man 5 | - man-pages 6 | - vim-enhanced 7 | - sysstat 8 | - lsof 9 | - jwhois 10 | - telnet 11 | - nc 12 | - libselinux-python 13 | - ntp 14 | - parted 15 | - mc 16 | - perl 17 | - openssh-clients 18 | - util-linux-ng 19 | - iotop 20 | - links 21 | - unzip 22 | - curl 23 | - wget 24 | - xfsprogs 25 | 26 | ntp_service: ntpd 27 | ntp_sync: "/usr/sbin/ntpd -gq" 28 | 29 | epel_rpm_url: "http://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm" 30 | 31 | epel_yum: "epel-release" 32 | 33 | resolv_conf: "/etc/resolv.conf" 34 | 35 | networking_path: "/etc/sysconfig/network-scripts" 36 | networking_eth: "redhat-ifcfg-eth.j2" 37 | networking_bond: "redhat-ifcfg-bond.j2" 38 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - python-simplejson 3 | - python-httplib2 4 | - man 5 | - man-pages 6 | - vim-enhanced 7 | - sysstat 8 | - lsof 9 | - jwhois 10 | - telnet 11 | - nc 12 | - libselinux-python 13 | - ntpdate 14 | - chrony 15 | - parted 16 | - mc 17 | - perl 18 | - openssh-clients 19 | - util-linux-ng 20 | - iotop 21 | - links 22 | - unzip 23 | - curl 24 | - wget 25 | - xfsprogs 26 | 27 | update_grub: 'cd && grub2-mkconfig -o "$(readlink -n /etc/grub2.cfg)"' 28 | 29 | ntp_service: chronyd 30 | ntp_sync: "systemctl restart ntpdate" 31 | 32 | epel_rpm_url: "http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" 33 | 34 | epel_yum: "epel-release" 35 | 36 | resolv_conf: "/etc/resolv.conf" 37 | 38 | networking_path: "/etc/sysconfig/network-scripts" 39 | networking_eth: "redhat-ifcfg-eth.j2" 40 | networking_bond: "redhat-ifcfg-bond.j2" 41 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - python-simplejson 3 | - python26-httplib2 4 | - python27-httplib2 5 | - man 6 | - man-pages 7 | - vim-enhanced 8 | - sysstat 9 | - lsof 10 | - jwhois 11 | - telnet 12 | - nc 13 | - libselinux-python 14 | - ntp 15 | - parted 16 | - mc 17 | - perl 18 | - openssh-clients 19 | - util-linux-ng 20 | - iotop 21 | - links 22 | - unzip 23 | - curl 24 | - wget 25 | - xfsprogs 26 | 27 | ntp_service: ntpd 28 | ntp_sync: "/usr/sbin/ntpd -gq" 29 | 30 | epel_rpm_url: "http://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm" 31 | 32 | epel_yum: "epel-release" 33 | 34 | resolv_conf: "/etc/resolv.conf" 35 | 36 | networking_path: "/etc/sysconfig/network-scripts" 37 | networking_eth: "redhat-ifcfg-eth.j2" 38 | networking_bond: "redhat-ifcfg-bond.j2" 39 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/files/confluent-kafka.service: -------------------------------------------------------------------------------- 1 | [Service] 2 | Type=simple 3 | User=cp-kafka 4 | Group=confluent 5 | ExecStart=/usr/bin/kafka-server-start /etc/kafka/server.properties 6 | LimitNOFILE=1000000 7 | TimeoutStopSec=180 8 | Restart=no 9 | Environment="KAFKA_OPTS=-javaagent:/opt/prometheus/jmx_prometheus_javaagent-0.12.0.jar=7071:/opt/prometheus/kafka-2_0_0.yml" "KAFKA_HEAP_OPTS=-Xms6g -Xmx6g -XX:MetaspaceSize=96m -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80" 10 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/files/prometheus.service: -------------------------------------------------------------------------------- 1 | # /etc/systemd/system/prometheus.service 2 | [Unit] 3 | Description=Prometheus Server 4 | Documentation=https://prometheus.io/docs/introduction/overview/ 5 | After=network-online.target 6 | 7 | [Service] 8 | User=prometheus 9 | Restart=on-failure 10 | ExecStart=/opt/prometheus/prometheus-2.16.0.linux-amd64/prometheus \ 11 | --config.file=/opt/prometheus/prometheus.yml \ 12 | --storage.tsdb.path=/opt/prometheus/prometheus-2.16.0.linux-amd64/data 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/files/prometheus.yml: -------------------------------------------------------------------------------- 1 | # my global config 2 | global: 3 | scrape_interval: 10s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 4 | evaluation_interval: 10s # Evaluate rules every 15 seconds. The default is every 1 minute. 5 | # scrape_timeout is set to the global default (10s). 6 | 7 | # Alertmanager configuration 8 | alerting: 9 | alertmanagers: 10 | - static_configs: 11 | - targets: 12 | # - alertmanager:9093 13 | 14 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 15 | rule_files: 16 | # - "first_rules.yml" 17 | # - "second_rules.yml" 18 | 19 | # A scrape configuration containing exactly one endpoint to scrape: 20 | # Here it's Prometheus itself. 21 | scrape_configs: 22 | # The job name is added as a label `job=` to any timeseries scraped from this config. 23 | - job_name: 'kafka' 24 | 25 | # metrics_path defaults to '/metrics' 26 | # scheme defaults to 'http'. 27 | 28 | static_configs: 29 | - targets: ['localhost:7071'] 30 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: group_vars/kafka-server-nodes 3 | 4 | - name: Set Broker Id 5 | set_fact: broker_id={{item.0 + 1}} 6 | with_indexed_items: "{{ groups['kafka-server-nodes'] }}" 7 | when: item.1 == "{{inventory_hostname}}" 8 | 9 | - name: setup kafka dir 10 | file: 11 | path: /var/lib/kafka 12 | owner: cp-kafka 13 | group: confluent 14 | mode: '750' 15 | state: directory 16 | 17 | - name: setup kafka streams dir 18 | file: 19 | path: /var/lib/kafka-streams 20 | owner: cp-ksql 21 | group: confluent 22 | mode: '750' 23 | state: directory 24 | 25 | - name: Template kafka server properties 26 | template: 27 | src: server.properties 28 | dest: /etc/kafka/ 29 | 30 | - name: Template kafka consumer properties 31 | template: 32 | src: consumer.properties 33 | dest: /etc/kafka/ 34 | 35 | - name: Template kafka-rest.properties 36 | template: 37 | src: kafka-rest.properties 38 | dest: /etc/kafka/ 39 | 40 | - name: Template ksql-server.properties 41 | template: 42 | src: ksql-server.properties 43 | dest: /etc/kafka/ 44 | 45 | - name: Template producer.properties 46 | template: 47 | src: producer.properties 48 | dest: /etc/kafka/ 49 | 50 | - name: Template zookeeper.properties 51 | template: 52 | src: zookeeper.properties 53 | dest: /etc/kafka/ 54 | 55 | - name: Template schema-registry.properties 56 | template: 57 | src: schema-registry.properties 58 | dest: /etc/schema-registry/ 59 | 60 | - name: start kafka server 61 | systemd: 62 | name: confluent-kafka 63 | state: started 64 | 65 | - name: start kafka rest 66 | systemd: 67 | name: confluent-kafka-rest 68 | state: started 69 | 70 | - name: start ksql server 71 | systemd: 72 | name: confluent-ksqldb 73 | state: started 74 | 75 | - name: start schema registry 76 | systemd: 77 | name: confluent-schema-registry 78 | state: started 79 | 80 | - name: start connect service 81 | systemd: 82 | name: confluent-kafka-connect 83 | state: started 84 | 85 | - name: setup prometheus 86 | include: prometheus-install.yml 87 | 88 | - debug: var=ansible_facts 89 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/tasks/prometheus-install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: add prometheus system user 4 | user: 5 | name: prometheus 6 | system: yes 7 | 8 | - name: setup prometheus dir 9 | file: 10 | path: "{{ prometheus_dir }}" 11 | state: directory 12 | 13 | - name: download prometheus 14 | get_url: 15 | url: "{{ prometheus_url }}/{{ prometheus_file }}" 16 | dest: "{{ tmp_dir }}/" 17 | 18 | - name: download prometheus jmx exporter 19 | get_url: 20 | url: "{{ prometheus_jmx_url }}" 21 | dest: "{{ prometheus_dir }}/" 22 | 23 | - name: download kafka prometheus yaml 24 | get_url: 25 | url: "{{ prometheus_kafka_yaml }}" 26 | dest: "{{ prometheus_dir }}/" 27 | 28 | - name: chown prometheus dirs 29 | file: 30 | dest: "{{ prometheus_dir }}" 31 | owner: prometheus 32 | group: prometheus 33 | recurse: yes 34 | 35 | - name: update kafka service 36 | copy: 37 | src: "confluent-kafka.service" 38 | dest: "/usr/lib/systemd/system/" 39 | 40 | - name: reload systemd 41 | systemd: 42 | daemon_reload: yes 43 | 44 | - name: restart kafka service 45 | systemd: 46 | name: confluent-kafka 47 | state: started 48 | 49 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/consumer.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # see org.apache.kafka.clients.consumer.ConsumerConfig for more details 16 | 17 | # list of brokers used for bootstrapping knowledge about the rest of the cluster 18 | # format: host1:port1,host2:port2 ... 19 | bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:9092{% if not loop.last %},{% endif %}{% endfor %} 20 | 21 | # consumer group id 22 | group.id=test-consumer-group 23 | 24 | # What to do when there is no initial offset in Kafka or if the current 25 | # offset does not exist any more on the server: latest, earliest, none 26 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/kafka-rest.properties: -------------------------------------------------------------------------------- 1 | # Maintained by Ansible 2 | #bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092 3 | bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:9092{% if not loop.last %},{% endif %}{% endfor %} 4 | 5 | listeners=http://0.0.0.0:8082 6 | client.ssl.endpoint.identification.algorithm= 7 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/ksql-server.properties: -------------------------------------------------------------------------------- 1 | # Maintained by Ansible 2 | 3 | #bootstrap.servers=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092 4 | bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:9092{% if not loop.last %},{% endif %}{% endfor %} 5 | 6 | #ksql.schema.registry.url=http://104.130.220.122:8081/ 7 | ksql.schema.registry.url=http://{{ hostvars[groups['kafka-server-nodes'][0]][['ansible_', hostvars[groups['kafka-server-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }}:8081/ 8 | 9 | application.id=ksql-server 10 | 11 | listeners=http://0.0.0.0:8088 12 | 13 | ksql.streams.state.dir=/var/lib/kafka-streams 14 | 15 | ssl.endpoint.identification.algorithm= 16 | 17 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/producer.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # see org.apache.kafka.clients.producer.ProducerConfig for more details 16 | 17 | ############################# Producer Basics ############################# 18 | 19 | # list of brokers used for bootstrapping knowledge about the rest of the cluster 20 | # format: host1:port1,host2:port2 ... 21 | bootstrap.servers={% for node in groups['kafka-server-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:9092{% if not loop.last %},{% endif %}{% endfor %} 22 | 23 | # specify the compression codec for all data generated: none, gzip, snappy, lz4, zstd 24 | compression.type=none 25 | 26 | # name of the partitioner class for partitioning events; default partition spreads data randomly 27 | #partitioner.class= 28 | 29 | # the maximum amount of time the client will wait for the response of a request 30 | #request.timeout.ms= 31 | 32 | # how long `KafkaProducer.send` and `KafkaProducer.partitionsFor` will block for 33 | #max.block.ms= 34 | 35 | # the producer will wait for up to the given delay to allow other records to be sent so that the sends can be batched together 36 | #linger.ms= 37 | 38 | # the maximum size of a request in bytes 39 | #max.request.size= 40 | 41 | # the default batch size in bytes when batching multiple records sent to a partition 42 | #batch.size= 43 | 44 | # the total bytes of memory the producer can use to buffer records waiting to be sent to the server 45 | #buffer.memory= 46 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/schema-registry.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 Confluent Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # The address the socket server listens on. 18 | # FORMAT: 19 | # listeners = listener_name://host_name:port 20 | # EXAMPLE: 21 | # listeners = PLAINTEXT://your.host.name:9092 22 | listeners=http://0.0.0.0:8081 23 | 24 | # Zookeeper connection string for the Zookeeper cluster used by your Kafka cluster 25 | # (see zookeeper docs for details). 26 | # This is a comma separated host:port pairs, each corresponding to a zk 27 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 28 | kafkastore.connection.url={% for node in groups['kafka-zk-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:2181{% if not loop.last %},{% endif %}{% endfor %} 29 | 30 | # Alternatively, Schema Registry can now operate without Zookeeper, handling all coordination via 31 | # Kafka brokers. Use this setting to specify the bootstrap servers for your Kafka cluster and it 32 | # will be used both for selecting the master schema registry instance and for storing the data for 33 | # registered schemas. 34 | # (Note that you cannot mix the two modes; use this mode only on new deployments or by shutting down 35 | # all instances, switching to the new configuration, and then starting the schema registry 36 | # instances again.) 37 | #kafkastore.bootstrap.servers=PLAINTEXT://localhost:9092 38 | 39 | # The name of the topic to store schemas in 40 | kafkastore.topic=_schemas 41 | 42 | # If true, API requests that fail will include extra debugging information, including stack traces 43 | debug=false 44 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/server.properties: -------------------------------------------------------------------------------- 1 | # Maintained by Ansible 2 | listeners=PLAINTEXT://:9092 3 | #listeners=104.130.220.120:9092,104.130.220.119:9092,104.130.220.122:9092 4 | 5 | #{% for node in groups['kafka-server-nodes'] %} 6 | #listeners={{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:2888:3888 7 | #{% endfor %} 8 | 9 | #listeners={% for node in groups['kafka-server-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:9092{% if not loop.last %},{% endif %}{% endfor %} 10 | 11 | #zookeeper.connect=104.130.220.120:2181,104.130.220.119:2181,104.130.220.122:2181 12 | zookeeper.connect={% for node in groups['kafka-zk-nodes'] %}{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:2181{% if not loop.last %},{% endif %}{% endfor %} 13 | 14 | log.dirs=/var/lib/kafka/data 15 | change to proper mount 16 | broker.id={{ broker_id }} 17 | 18 | log.segment.bytes=1073741824 19 | socket.receive.buffer.bytes=102400 20 | socket.send.buffer.bytes=102400 21 | confluent.metrics.reporter.topic.replicas=3 22 | num.network.threads=8 23 | ssl.endpoint.identification.algorithm= 24 | num.io.threads=16 25 | confluent.metrics.reporter.ssl.endpoint.identification.algorithm= 26 | transaction.state.log.min.isr=2 27 | zookeeper.connection.timeout.ms=6000 28 | offsets.topic.replication.factor=3 29 | socket.request.max.bytes=104857600 30 | log.retention.check.interval.ms=300000 31 | group.initial.rebalance.delay.ms=0 32 | #metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter 33 | num.recovery.threads.per.data.dir=2 34 | transaction.state.log.replication.factor=3 35 | #confluent.metrics.reporter.bootstrap.servers=104.130.220.116:9092 36 | log.retention.hours=168 37 | num.partitions=1 38 | 39 | # Confluent Support 40 | #confluent.support.metrics.enable=true 41 | #confluent.support.customer.id=anonymous 42 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-broker/templates/zookeeper.properties: -------------------------------------------------------------------------------- 1 | tickTime=2000 2 | dataDir=/var/lib/zookeeper 3 | clientPort=2181 4 | initLimit=5 5 | syncLimit=2 6 | #server.1=10.0.0.10:2888:3888 7 | #server.2=10.0.0.11:2888:3888 8 | #server.3=10.0.0.12:2888:3888 9 | {% for node in groups['kafka-zk-nodes'] %} 10 | 11 | server.{{ hostvars[node].zk_id }}={{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:2888:3888 12 | 13 | {% endfor %} 14 | autopurge.snapRetainCount=3 15 | autopurge.purgeInterval=24 16 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/files/confluent.repo: -------------------------------------------------------------------------------- 1 | [Confluent.dist] 2 | name=Confluent repository (dist) 3 | baseurl=https://packages.confluent.io/rpm/5.5/7 4 | gpgcheck=1 5 | gpgkey=https://packages.confluent.io/rpm/5.5/archive.key 6 | enabled=1 7 | 8 | [Confluent] 9 | name=Confluent repository 10 | baseurl=https://packages.confluent.io/rpm/5.5 11 | gpgcheck=1 12 | gpgkey=https://packages.confluent.io/rpm/5.5/archive.key 13 | enabled=1 14 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/files/java_home.sh: -------------------------------------------------------------------------------- 1 | #set java home for all users 2 | 3 | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.222.b10-1.el7_7.x86_64 4 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/files/tuned.conf: -------------------------------------------------------------------------------- 1 | # 2 | # tuned configuration 3 | # 4 | 5 | [main] 6 | include=throughput-performance 7 | 8 | [vm] 9 | transparent_hugepages=never 10 | 11 | [sysctl] 12 | vm.swappiness = 1 13 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: yum-clean-metadata 3 | command: yum clean metadata 4 | args: 5 | warn: no 6 | 7 | - name: Restart iptables 8 | service: name=iptables state=restarted enabled=yes 9 | 10 | - name: Reload firewalld 11 | command: firewall-cmd --reload 12 | 13 | - name: Run update-grub 14 | shell: "{{ update_grub }}" 15 | when: ansible_distribution_major_version|int > 6 16 | 17 | - name: Run resolvconf 18 | shell: resolvconf -u 19 | when: ansible_os_family == "Debian" 20 | 21 | - name: Restart network interfaces 22 | shell: ifdown {{ item }}; ifup {{ item }}; ifup {{ cluster_interface }} 23 | with_flattened: 24 | - cluster_interface 25 | - bond_interfaces 26 | 27 | - name: Run depmod 28 | command: depmod 29 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/tasks/bonding.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Write bonding.conf file 3 | lineinfile: dest=/etc/modprobe.d/bonding.conf line='alias {{ cluster_interface }} bonding' state=present create=yes mode=0664 owner=root group=root 4 | notify: Run depmod 5 | 6 | - meta: flush_handlers 7 | 8 | - name: Make sure the interfaces.d files are included 9 | lineinfile: line="source /etc/network/interfaces.d/ifcfg-*" 10 | dest=/etc/network/interfaces 11 | state=present 12 | insertafter=EOF 13 | when: ansible_os_family == "Debian" 14 | 15 | - name: Write ifcfg files for bonded interfaces 16 | template: src={{ ansible_os_family|lower }}-ifcfg-eth.j2 dest={{ networking_path }}/ifcfg-{{ item }} 17 | with_items: "{{ bond_interfaces }}" 18 | notify: Restart network interfaces 19 | 20 | - name: Write ifcfg file for bond interface 21 | template: src={{ ansible_os_family|lower }}-ifcfg-bond.j2 dest={{ networking_path }}/ifcfg-{{ cluster_interface }} 22 | notify: Restart network interfaces 23 | 24 | - meta: flush_handlers 25 | 26 | - name: Test slave interfaces 27 | include: slavetest.yml 28 | with_items: "{{ bond_interfaces }}" 29 | loop_control: 30 | loop_var: slave_interface 31 | 32 | - name: Ping between all hosts on bond interface to verify network connectivity 33 | command: /bin/ping -q -c 5 -i 0.2 -W 1 {{ hostvars[item]['bond_ip'] }} 34 | when: bond_ip is defined and hostvars[item]['bond_ip'] is defined 35 | with_items: "{{ groups['all'] }}" 36 | changed_when: 0 37 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/tasks/firewall.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set iptables between cluster nodes 3 | lineinfile: dest=/etc/sysconfig/iptables 4 | insertbefore="^-A INPUT" 5 | line="-A INPUT -s {{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}/32 -j ACCEPT" 6 | state=present 7 | with_items: "{{ play_hosts }}" 8 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" 9 | notify: Restart iptables 10 | 11 | - name: Set iptables to allow cluster access from external IPs 12 | lineinfile: dest=/etc/sysconfig/iptables 13 | insertbefore="^-A INPUT" 14 | line="-A INPUT -s {{ item }}/32 -j ACCEPT" 15 | state=present 16 | with_items: "{{ cloud_config.allowed_external_ips }}" 17 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "6" 18 | notify: Restart iptables 19 | 20 | - name: Set firewalld rules between cluster nodes 21 | command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }}" accept' 22 | # firewalld: source={{ hostvars[item][['ansible_', hostvars[item]['cluster_interface']]|join]['ipv4']['address'] }} state=enabled 23 | with_items: "{{ play_hosts }}" 24 | notify: Reload firewalld 25 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 26 | 27 | - name: Set firewalld to allow cluster access from external IPs 28 | command: firewall-cmd --permanent --zone=public --add-rich-rule='rule family="ipv4" source address="{{ item }}" accept' 29 | # firewalld: source={{ item }} state=enabled 30 | with_items: "{{ cloud_config.allowed_external_ips }}" 31 | notify: Reload firewalld 32 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 33 | 34 | - name: Set UFW rules between cluster nodes 35 | ufw: rule=allow src={{ hostvars[item]['ansible_'~hostvars[item].cluster_interface|default(hostvars[item].ansible_default_ipv4.alias)]['ipv4']['address'] }} 36 | with_items: "{{ play_hosts }}" 37 | when: ansible_distribution == "Ubuntu" 38 | 39 | - name: Set UFW to allow cluster access from external IPs 40 | ufw: rule=allow src={{ item }} 41 | with_items: "{{ cloud_config.allowed_external_ips }}" 42 | when: ansible_distribution == "Ubuntu" 43 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/tasks/partitioning.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get disk alignment for disk {{ item }} 3 | shell: if [[ -e /sys/block/{{ item }}/queue/optimal_io_size && -e /sys/block/{{ item }}/alignment_offset && -e /sys/block/{{ item }}/queue/physical_block_size ]]; then echo $[$(( ($(cat /sys/block/{{ item }}/queue/optimal_io_size) + $(cat /sys/block/{{ item }}/alignment_offset)) / $(cat /sys/block/{{ item }}/queue/physical_block_size) )) | 2048]; else echo 2048; fi 4 | when: ansible_devices[item] is defined 5 | register: disk_offset 6 | 7 | - name: Partition disk {{ item }} 8 | shell: parted -s -a optimal /dev/{{ item }} mklabel gpt mkpart primary {{ data_disks_filesystem }} {{ disk_offset.stdout|default("2048") }}s 100% && sleep 5 && partprobe /dev/{{ item }}; sleep 5 9 | when: ansible_devices[item] is defined and ansible_devices[item]['partitions'] == {} 10 | 11 | - name: Create the filesystem on disk {{ item }} 12 | filesystem: dev="/dev/{{ item }}1" fstype="{{ data_disks_filesystem }}" 13 | when: ansible_devices[item] is defined 14 | 15 | - name: Disable periodic fsck on {{ item }} 16 | shell: tune2fs -c0 -i0 /dev/{{ item }}1 17 | when: ansible_devices[item] is defined and (data_disks_filesystem == "ext4" or data_disks_filesystem == "ext3") 18 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/debian-ifcfg-bond.j2: -------------------------------------------------------------------------------- 1 | auto {{ cluster_interface }} 2 | iface {{ cluster_interface }} inet static 3 | address {{ bond_ip }} 4 | netmask {{ bond_netmask }} 5 | bond-mode active-backup 6 | bond-miimon 100 7 | bond-slaves none 8 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/debian-ifcfg-eth.j2: -------------------------------------------------------------------------------- 1 | auto {{ item }} 2 | iface {{ item }} inet manual 3 | bond-master {{ cluster_interface }} 4 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/hosts.j2: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | {% for node in groups['kafka-cluster'] %} 4 | {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_nodename'] }} {{ hostvars[node]['ansible_hostname'] }} 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/mdadm.conf.j2: -------------------------------------------------------------------------------- 1 | {{ mdadm_scan.stdout }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/redhat-ifcfg-bond.j2: -------------------------------------------------------------------------------- 1 | DEVICE={{ cluster_interface }} 2 | IPADDR={{ bond_ip }} 3 | NETMASK={{ bond_netmask }} 4 | NETWORK= 5 | BROADCAST= 6 | ONBOOT=yes 7 | BOOTPROTO=none 8 | USERCTL=no 9 | BONDING_OPTS="mode=active-backup miimon=100" 10 | NM_CONTROLLED=no 11 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/redhat-ifcfg-eth.j2: -------------------------------------------------------------------------------- 1 | DEVICE={{ item }} 2 | ONBOOT=yes 3 | BOOTPROTO=none 4 | USERCTL=no 5 | MASTER={{ cluster_interface }} 6 | SLAVE=yes 7 | NM_CONTROLLED=no 8 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/redhat-unbound-master.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | interface: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 4 | access-control: 127.0.0.0/8 allow 5 | access-control: 10.0.0.0/8 allow 6 | domain-insecure: "{{ ansible_domain }}" 7 | local-zone: "10.in-addr.arpa." nodefault 8 | 9 | stub-zone: 10 | name: "10.in-addr.arpa." 11 | stub-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 12 | 13 | forward-zone: 14 | name: "." 15 | forward-addr: 8.8.8.8 16 | forward-addr: 8.8.4.4 17 | 18 | remote-control: 19 | control-interface: 127.0.0.1 20 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/redhat-unbound-zonefile.j2: -------------------------------------------------------------------------------- 1 | # Local zone 2 | 3 | local-zone: "{{ ansible_domain }}." static 4 | 5 | {% for node in groups['hadoop-cluster'] %} 6 | local-data: "{{ hostvars[node]['ansible_fqdn'] }}. IN A {{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}" 7 | local-data-ptr: "{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }} {{ hostvars[node]['ansible_fqdn'] }}" 8 | 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/redhat-unbound.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 127.0.0.1 3 | access-control: 127.0.0.0/8 allow 4 | access-control: 10.0.0.0/8 allow 5 | domain-insecure: "{{ ansible_domain }}" 6 | local-zone: "10.in-addr.arpa." nodefault 7 | 8 | stub-zone: 9 | name: "10.in-addr.arpa." 10 | stub-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 11 | 12 | forward-zone: 13 | name: "." 14 | forward-addr: {{ hostvars[groups['master-nodes'][0]][['ansible_', hostvars[groups['master-nodes'][0]]['cluster_interface']]|join]['ipv4']['address'] }} 15 | 16 | remote-control: 17 | control-interface: 127.0.0.1 18 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | domain {{ ansible_domain }} 2 | nameserver 173.203.4.8 3 | nameserver 173.203.4.9 4 | search {{ ansible_domain }} 5 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-common/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - curl 3 | - wget 4 | - xfsprogs 5 | - confluent-community-2.12 6 | - java-11-openjdk-devel 7 | 8 | 9 | epel_rpm_url: "http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" 10 | 11 | epel_yum: "epel-release" 12 | 13 | resolv_conf: "/etc/resolv.conf" 14 | 15 | networking_path: "/etc/sysconfig/network-scripts" 16 | networking_eth: "redhat-ifcfg-eth.j2" 17 | networking_bond: "redhat-ifcfg-bond.j2" 18 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-mngr/files/prometheus.service: -------------------------------------------------------------------------------- 1 | # /etc/systemd/system/prometheus.service 2 | [Unit] 3 | Description=Prometheus Server 4 | Documentation=https://prometheus.io/docs/introduction/overview/ 5 | After=network-online.target 6 | 7 | [Service] 8 | User=prometheus 9 | Restart=on-failure 10 | ExecStart=/opt/prometheus/prometheus-2.16.0.linux-amd64/prometheus \ 11 | --config.file=/opt/prometheus/prometheus.yml \ 12 | --storage.tsdb.path=/opt/prometheus/prometheus-2.16.0.linux-amd64/data 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-mngr/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: group_vars/kafka-server-nodes 3 | 4 | - name: setup prometheus 5 | include: prometheus-install.yml 6 | 7 | - debug: var=ansible_facts 8 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-mngr/tasks/prometheus-install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: setup prometheus dir 4 | file: 5 | path: "{{ prometheus_dir }}" 6 | state: directory 7 | 8 | - name: download prometheus 9 | get_url: 10 | url: "{{ prometheus_url }}/{{ prometheus_file }}" 11 | dest: "{{ tmp_dir }}/" 12 | 13 | - name: Extract prometheus.tgz into prometheus dir 14 | unarchive: 15 | src: "{{ tmp_dir }}/{{ prometheus_file }}" 16 | dest: "{{ prometheus_dir }}/" 17 | remote_src: yes 18 | 19 | - name: install prometheus config 20 | template: 21 | src: "prometheus.yml" 22 | dest: "{{ prometheus_dir }}/" 23 | 24 | - name: install alerts config 25 | template: 26 | src: "kafka.alerts.yml" 27 | dest: "{{ prometheus_dir }}/" 28 | 29 | - name: Cleanup tmp files 30 | file: 31 | path: "{{ tmp_dir }}/{{ prometheus_file }}" 32 | state: absent 33 | 34 | - name: setup prometheus user 35 | user: 36 | name: prometheus 37 | shell: /sbin/nologin 38 | home: "{{ prometheus_dir }}" 39 | ignore_errors: yes 40 | 41 | - name: chown prometheus dirs 42 | file: 43 | dest: "{{ prometheus_dir }}" 44 | owner: prometheus 45 | group: prometheus 46 | recurse: yes 47 | 48 | - name: add prometheus service 49 | copy: 50 | src: prometheus.service 51 | dest: /usr/lib/systemd/system/prometheus.service 52 | 53 | - name: reload systemd 54 | systemd: 55 | daemon_reload: yes 56 | 57 | - name: start prometheus service 58 | systemd: 59 | name: prometheus 60 | state: started 61 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-mngr/templates/kafka.alerts.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: kafka 3 | rules: 4 | - alert: BytesInPerSec50kThreshold 5 | expr: kafka_server_brokertopicmetrics_oneminuterate{name="BytesInPerSec"} > 5000 6 | for: 10s 7 | labels: 8 | severity: warning 9 | annotations: 10 | summary: Kafka High Bytes In CRIT 5k (10sec) 11 | - alert: BytesInPerSec100kThreshold 12 | expr: kafka_server_brokertopicmetrics_oneminuterate{name="BytesInPerSec"} > 10000 13 | for: 10s 14 | labels: 15 | severity: critical 16 | annotations: 17 | summary: Kafka High Bytes In CRIT 10k (10sec) 18 | - alert: BytesInPerSec150kThreshold 19 | expr: kafka_server_brokertopicmetrics_oneminuterate{name="BytesInPerSec"} > 15000 20 | for: 10s 21 | labels: 22 | severity: critical 23 | annotations: 24 | summary: Kafka High Bytes In CRIT 15k (10sec) 25 | - alert: CPU-90%+ 26 | expr: java_lang_operatingsystem_systemcpuload > 90 27 | for: 1s 28 | labels: 29 | severity: critical 30 | annotations: 31 | summary: CPU Load 90%+ 32 | - alert: EndpointDown 33 | expr: up == 0 34 | for: 10s 35 | labels: 36 | severity: "critical" 37 | annotations: 38 | summary: "Endpoint down" 39 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-mngr/templates/prometheus.yml: -------------------------------------------------------------------------------- 1 | # my global config 2 | global: 3 | scrape_interval: 10s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 4 | evaluation_interval: 10s # Evaluate rules every 15 seconds. The default is every 1 minute. 5 | # scrape_timeout is set to the global default (10s). 6 | 7 | # Alertmanager configuration 8 | alerting: 9 | alertmanagers: 10 | - static_configs: 11 | - targets: 12 | - {{ alertmanager }}:9093 13 | 14 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 15 | rule_files: 16 | # - "first_rules.yml" 17 | # - "second_rules.yml" 18 | - "kafka.alerts.yml" 19 | 20 | # A scrape configuration containing exactly one endpoint to scrape: 21 | # Here it's Prometheus itself. 22 | scrape_configs: 23 | # The job name is added as a label `job=` to any timeseries scraped from this config. 24 | - job_name: 'kafka' 25 | 26 | # metrics_path defaults to '/metrics' 27 | # scheme defaults to 'http'. 28 | 29 | static_configs: 30 | {% for node in groups['kafka-broker-cluster'] %} 31 | - targets: ['{{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:7071'] 32 | {% endfor %} 33 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-zookeeper/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: group_vars/kafka-zookeeper-nodes 3 | 4 | - name: Set Zookeeper Id 5 | set_fact: zk_id={{item.0 + 1}} 6 | with_indexed_items: "{{ groups['kafka-zk-nodes'] }}" 7 | when: item.1 == "{{inventory_hostname}}" 8 | 9 | - name: setup zookeeper dir 10 | file: 11 | path: /var/lib/zookeeper 12 | owner: cp-kafka 13 | group: confluent 14 | mode: '750' 15 | state: directory 16 | 17 | - name: Template zk id 18 | template: 19 | src: myid.j2 20 | dest: /var/lib/zookeeper/myid 21 | 22 | - name: Template zookeeper.properties 23 | template: 24 | src: zookeeper.properties 25 | dest: /etc/kafka/ 26 | 27 | - name: start zookeeper 28 | systemd: 29 | name: confluent-zookeeper 30 | state: started 31 | 32 | - debug: var=ansible_facts 33 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-zookeeper/templates/myid.j2: -------------------------------------------------------------------------------- 1 | {{ zk_id }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/confluent-community-zookeeper/templates/zookeeper.properties: -------------------------------------------------------------------------------- 1 | tickTime=2000 2 | dataDir=/var/lib/zookeeper 3 | clientPort=2181 4 | initLimit=5 5 | syncLimit=2 6 | #server.1=10.0.0.10:2888:3888 7 | #server.2=10.0.0.11:2888:3888 8 | #server.3=10.0.0.12:2888:3888 9 | {% for node in groups['kafka-zk-nodes'] %} 10 | 11 | server.{{ hostvars[node].zk_id }}={{ hostvars[node]['ansible_'~hostvars[node].cluster_interface|default(hostvars[node].ansible_default_ipv4.alias)]['ipv4']['address'] }}:2888:3888 12 | 13 | {% endfor %} 14 | autopurge.snapRetainCount=3 15 | autopurge.purgeInterval=24 16 | -------------------------------------------------------------------------------- /playbooks/roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | docker_version: 17.03 -------------------------------------------------------------------------------- /playbooks/roles/docker/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - { role: kube-commons/os-checker } 4 | -------------------------------------------------------------------------------- /playbooks/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Docker container engine 3 | include_tasks: pkg.yml 4 | 5 | - name: Copy Docker engine service file 6 | register: change_docker 7 | template: 8 | src: "docker.service.j2" 9 | dest: "{{ systemd_dir }}/docker.service" 10 | owner: root 11 | group: root 12 | mode: 0755 13 | 14 | - name: Copy Docker environment config file 15 | template: src=docker.j2 dest={{ system_env_dir }}/docker 16 | 17 | - name: Add any insecure registries to Docker config 18 | when: insecure_registries is defined and insecure_registries | length > 0 19 | lineinfile: dest={{ system_env_dir }}/docker regexp=^INSECURE_REGISTRY= line=INSECURE_REGISTRY="{% for reg in insecure_registries %}--insecure-registry={{ reg }} {% endfor %}" 20 | 21 | - name: Add registry to Docker config 22 | when: add_registry is defined and add_registry > 0 23 | lineinfile: dest={{ system_env_dir }}/docker regexp=^ADD_REGISTRY= line=ADD_REGISTRY="{% for reg in add_registry %}--add-registry={{ reg }} {%endfor %}" 24 | 25 | - name: Enable and check Docker service 26 | systemd: 27 | name: docker 28 | daemon_reload: yes 29 | state: started 30 | enabled: yes 31 | register: started_docker 32 | -------------------------------------------------------------------------------- /playbooks/roles/docker/tasks/pkg.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install apt-transport-https 3 | when: ansible_os_family == "Debian" 4 | apt: 5 | name: "apt-transport-https" 6 | state: present 7 | update_cache: yes 8 | 9 | - name: Add Docker APT GPG key 10 | when: ansible_os_family == "Debian" 11 | apt_key: 12 | url: https://download.docker.com/linux/ubuntu/gpg 13 | 14 | - name: Add Docker APT repository 15 | when: ansible_os_family == "Debian" 16 | apt_repository: 17 | repo: deb https://download.docker.com/linux/ubuntu xenial stable 18 | state: present 19 | filename: 'docker' 20 | 21 | - name: Add Docker yum repository 22 | when: ansible_os_family == "RedHat" 23 | yum_repository: 24 | name: Docker 25 | description: Docker Repository 26 | file: docker 27 | baseurl: https://yum.dockerproject.org/repo/main/centos/7/ 28 | enabled: yes 29 | gpgcheck: yes 30 | gpgkey: https://yum.dockerproject.org/gpg 31 | 32 | - name: Install docker engine (RHEL/CentOS) 33 | when: ansible_os_family == "RedHat" 34 | yum: 35 | name: "docker-engine-{{ docker_version }}.*" 36 | state: present 37 | 38 | - name: Install docker engine (Debian/Ubuntu) 39 | when: ansible_os_family == "Debian" 40 | apt: 41 | update_cache: yes 42 | name: "docker-ce={{ docker_version }}*" 43 | state: present 44 | 45 | - name: Hold docker version 46 | when: ansible_os_family == "Debian" 47 | dpkg_selections: 48 | name: docker-ce 49 | selection: hold 50 | -------------------------------------------------------------------------------- /playbooks/roles/docker/templates/docker.j2: -------------------------------------------------------------------------------- 1 | INSECURE_REGISTRY="" 2 | DOCKER_OPTS="" 3 | {% if ansible_os_family == "RedHat" -%} 4 | DOCKER_STORAGE_OPTIONS="--storage-driver=overlay" 5 | {% endif -%} 6 | -------------------------------------------------------------------------------- /playbooks/roles/docker/templates/docker.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Docker Engine 3 | After=network.target 4 | 5 | [Service] 6 | Type=notify 7 | EnvironmentFile=-{{ system_env_dir }}/docker 8 | ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT 9 | ExecStart=/usr/bin/dockerd {% if ansible_os_family == 'Debian' -%} -H fd:// {% endif -%} \ 10 | $OPTIONS \ 11 | $DOCKER_STORAGE_OPTIONS \ 12 | $DOCKER_OPTS \ 13 | $DOCKER_NETWORK_OPTIONS \ 14 | $ADD_REGISTRY \ 15 | $BLOCK_REGISTRY \ 16 | $INSECURE_REGISTRY 17 | 18 | ExecReload=/bin/kill -s HUP $MAINPID 19 | Restart=on-failure 20 | LimitNOFILE=1048576 21 | LimitNPROC=infinity 22 | LimitCORE=infinity 23 | TimeoutStartSec=0 24 | Delegate=yes 25 | KillMode=process 26 | 27 | [Install] 28 | WantedBy=multi-user.target 29 | -------------------------------------------------------------------------------- /playbooks/roles/healthcheck/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Create tmp directory" 3 | file: 4 | path: "{{ tmp_dir }}" 5 | state: directory 6 | mode: 0755 7 | tags: healthcheck 8 | 9 | - name: "Create checkout directory" 10 | file: 11 | path: "{{ tmp_dir }}/healthcheck" 12 | state: directory 13 | mode: 0755 14 | tags: healthcheck 15 | 16 | - name: "Clone git repo" 17 | git: 18 | repo: "{{ healthcheck_git_url }}" 19 | dest: "{{ tmp_dir }}/healthcheck" 20 | tags: healthcheck 21 | 22 | - name: "Install Healthcheck" 23 | shell: "kubectl apply -f {{ tmp_dir }}/healthcheck/kubernetes/" 24 | tags: healthcheck 25 | 26 | - name: "Clean-up" 27 | file: 28 | path: "{{ tmp_dir }}" 29 | state: absent 30 | ignore_errors: yes 31 | tags: healthcheck 32 | -------------------------------------------------------------------------------- /playbooks/roles/healthcheck/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | healthcheck_git_url: https://github.com/emrekenci/k8s-healthcheck.git 3 | -------------------------------------------------------------------------------- /playbooks/roles/helm/files/rbac-config.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: tiller 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: ClusterRoleBinding 9 | metadata: 10 | name: tiller 11 | roleRef: 12 | apiGroup: rbac.authorization.k8s.io 13 | kind: ClusterRole 14 | name: cluster-admin 15 | subjects: 16 | - kind: ServiceAccount 17 | name: tiller 18 | namespace: kube-system 19 | -------------------------------------------------------------------------------- /playbooks/roles/helm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Create tmp directory" 3 | file: 4 | path: "{{ tmp_dir }}" 5 | state: directory 6 | mode: 0755 7 | tags: helm 8 | 9 | - name: "Check if Helm is installed" 10 | shell: command -v helm >/dev/null 2>&1 11 | register: helm_exists 12 | ignore_errors: yes 13 | tags: helm 14 | 15 | - name: "Install Helm" 16 | block: 17 | - name: "Get Helm installer" 18 | get_url: 19 | url: https://raw.githubusercontent.com/helm/helm/master/scripts/get 20 | dest: "{{ tmp_dir }}/get_helm.sh" 21 | mode: 0755 22 | 23 | - name: "Run the installer" 24 | shell: "{{ tmp_dir }}/get_helm.sh" 25 | 26 | when: helm_exists.rc > 0 27 | tags: helm 28 | 29 | - name: "Copy yaml file" 30 | copy: 31 | src: "rbac-config.yml" 32 | dest: "{{ tmp_dir }}/rbac-config.yml" 33 | mode: 0644 34 | tags: helm 35 | 36 | - name: "RBAC configuration" 37 | shell: "kubectl apply -f {{ tmp_dir }}/rbac-config.yml" 38 | tags: helm 39 | 40 | - name: "Init Helm" 41 | shell: "helm init --service-account tiller" 42 | tags: helm 43 | 44 | - name: "Update Helm repo" 45 | shell: "helm repo update" 46 | tags: helm 47 | 48 | - name: "Clean-up" 49 | file: 50 | path: "{{ tmp_dir }}" 51 | state: absent 52 | ignore_errors: yes 53 | tags: helm 54 | -------------------------------------------------------------------------------- /playbooks/roles/kube-commons/os-checker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | systemd_dir: /lib/systemd/system 4 | system_env_dir: /etc/sysconfig 5 | network_dir: /etc/kubernetes/network 6 | kubeadmin_config: /etc/kubernetes/admin.conf 7 | kube_addon_dir: /etc/kubernetes/addon 8 | -------------------------------------------------------------------------------- /playbooks/roles/kube-commons/os-checker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Fact os vars 3 | 4 | - name: Get os_version from /etc/os-release 5 | when: ansible_os_family is not defined 6 | raw: "grep '^VERSION_ID=' /etc/os-release | sed s'/VERSION_ID=//'" 7 | register: os_version 8 | changed_when: False 9 | 10 | - name: Get distro name from /etc/os-release 11 | when: ansible_os_family is not defined 12 | raw: "grep '^NAME=' /etc/os-release | sed s'/NAME=//'" 13 | register: distro 14 | changed_when: False 15 | 16 | - name: Set fact ansible_os_family var to Debian 17 | when: 18 | - ansible_os_family is not defined 19 | - "'Debian' in distro.stdout" 20 | set_fact: 21 | ansible_os_family: Debian 22 | 23 | - name: Set fact ansible_os_family var to Debian 24 | when: 25 | - ansible_os_family is not defined 26 | - "'Ubuntu' in distro.stdout" 27 | set_fact: 28 | ansible_os_family: Debian 29 | 30 | - name: Set fact ansible_os_family var to RedHat 31 | when: 32 | - ansible_os_family is not defined 33 | - "'CentOS' in distro.stdout" 34 | set_fact: 35 | ansible_os_family: RedHat 36 | 37 | - name: Override config file directory for Debian 38 | when: ansible_os_family == "Debian" 39 | set_fact: 40 | system_env_dir: "/etc/default" 41 | -------------------------------------------------------------------------------- /playbooks/roles/kube-commons/pre-install/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - { role: kube-commons/os-checker } 4 | -------------------------------------------------------------------------------- /playbooks/roles/kube-commons/pre-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install Kubernetes packages 4 | include_tasks: pkg.yml 5 | 6 | - name: Disable system swap 7 | shell: "swapoff -a" 8 | 9 | - name: Remove current swaps from fstab 10 | lineinfile: 11 | dest: /etc/fstab 12 | regexp: '(?i)^([^#][\S]+\s+(none|swap)\s+swap.*)' 13 | line: '# \1' 14 | backrefs: yes 15 | state: present 16 | 17 | - name: ensure br_netfiler 18 | modprobe: 19 | name: br_netfilter 20 | state: present 21 | 22 | - name: Disable swappiness and pass bridged IPv4 traffic to iptable's chains 23 | sysctl: 24 | name: "{{ item.name }}" 25 | value: "{{ item.value }}" 26 | state: present 27 | with_items: 28 | - { name: 'vm.swappiness', value: '0' } 29 | - { name: 'net.bridge.bridge-nf-call-iptables', value: '1' } 30 | 31 | - name: Create service drop-in directory 32 | file: 33 | path: /etc/systemd/system/kubelet.service.d/ 34 | state: directory 35 | owner: "{{ ansible_user | default(ansible_user_id) }}" 36 | group: "{{ ansible_user | default(ansible_user_id) }}" 37 | mode: 0755 38 | 39 | - name: Copy kubeadm conf to drop-in directory 40 | template: src=20-extra-args.conf.j2 dest=/etc/systemd/system/kubelet.service.d/20-extra-args.conf 41 | 42 | - name: Reload kubelet daemon 43 | systemd: 44 | name: kubelet 45 | daemon_reload: yes 46 | enabled: yes 47 | -------------------------------------------------------------------------------- /playbooks/roles/kube-commons/pre-install/tasks/pkg.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Add Kubernetes APT GPG key 4 | when: ansible_os_family == "Debian" 5 | apt_key: 6 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 7 | state: present 8 | 9 | - name: Add Kubernetes APT repository 10 | when: ansible_os_family == "Debian" 11 | apt_repository: 12 | repo: deb http://apt.kubernetes.io/ kubernetes-xenial main 13 | state: present 14 | filename: 'kubernetes' 15 | 16 | - name: Add Kubernetes yum repository 17 | when: ansible_os_family == "RedHat" 18 | yum_repository: 19 | name: Kubernetes 20 | description: Kubernetes Repository 21 | file: kubernetes 22 | baseurl: http://yum.kubernetes.io/repos/kubernetes-el7-x86_64 23 | enabled: yes 24 | gpgcheck: no 25 | 26 | - name: Install kubernetes packages (RHEL/CentOS) 27 | when: ansible_os_family == "RedHat" 28 | yum: 29 | name: "{{ item }}-{{ kube_version | replace('v', '') }}" 30 | update_cache: yes 31 | state: installed 32 | with_items: "{{ pkgs }}" 33 | 34 | - name: Install kubernetes packages (Debian/Ubuntu) 35 | when: ansible_os_family == "Debian" 36 | apt: 37 | name: "{{ item }}-{{ kube_version | replace('v', '') }}" 38 | update_cache: yes 39 | state: installed 40 | with_items: "{{ pkgs }}" 41 | -------------------------------------------------------------------------------- /playbooks/roles/kube-commons/pre-install/templates/20-extra-args.conf.j2: -------------------------------------------------------------------------------- 1 | [Service] 2 | Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false" 3 | -------------------------------------------------------------------------------- /playbooks/roles/master-nodes/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: group_vars/cloudera 3 | 4 | #- include: mysql.yml 5 | -------------------------------------------------------------------------------- /playbooks/roles/metallb/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Create tmp directory" 3 | file: 4 | path: "{{ tmp_dir }}" 5 | state: directory 6 | mode: 0755 7 | tags: metallb 8 | 9 | - name: "Install MetalLB" 10 | shell: "kubectl apply -f {{ metallb_yaml_url }}" 11 | tags: metallb 12 | 13 | - name: "Create configmap file" 14 | template: 15 | src: metallb-layer-2-config.yml.j2 16 | dest: "{{ tmp_dir }}/metallb-layer-2-config.yml" 17 | tags: metallb 18 | 19 | - name: "Create MetalLB configmap in kubernetes" 20 | shell: "kubectl apply -f {{ tmp_dir }}/metallb-layer-2-config.yml" 21 | tags: metallb 22 | 23 | - name: "Clean-up" 24 | file: 25 | path: "{{ tmp_dir }}" 26 | state: absent 27 | ignore_errors: yes 28 | tags: metallb 29 | -------------------------------------------------------------------------------- /playbooks/roles/metallb/templates/metallb-layer-2-config.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: metallb-system 5 | name: config 6 | data: 7 | config: | 8 | address-pools: 9 | - name: metallb-ip-space 10 | protocol: layer2 11 | addresses: 12 | - {{ metallb_address_space }} 13 | -------------------------------------------------------------------------------- /playbooks/roles/metallb/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | metallb_version: v0.7.3 3 | metallb_yaml_url: "https://raw.githubusercontent.com/google/metallb/{{ metallb_version }}/manifests/metallb.yaml" 4 | metallb_address_space: 192.168.205.200-192.168.205.210 5 | -------------------------------------------------------------------------------- /playbooks/roles/templates/calico-etcd.yml.j2-bak: -------------------------------------------------------------------------------- 1 | # This manifest installs the Calico etcd on the kubeadm master. This uses a DaemonSet 2 | # to force it to run on the master even when the master isn't schedulable, and uses 3 | # nodeSelector to ensure it only runs on the master. 4 | apiVersion: extensions/v1beta1 5 | kind: DaemonSet 6 | metadata: 7 | name: calico-etcd 8 | namespace: kube-system 9 | labels: 10 | k8s-app: calico-etcd 11 | spec: 12 | template: 13 | metadata: 14 | labels: 15 | k8s-app: calico-etcd 16 | annotations: 17 | # Mark this pod as a critical add-on; when enabled, the critical add-on scheduler 18 | # reserves resources for critical add-on pods so that they can be rescheduled after 19 | # a failure. This annotation works in tandem with the toleration below. 20 | scheduler.alpha.kubernetes.io/critical-pod: '' 21 | spec: 22 | tolerations: 23 | # This taint is set by all kubelets running `--cloud-provider=external` 24 | # so we should tolerate it to schedule the Calico pods 25 | - key: node.cloudprovider.kubernetes.io/uninitialized 26 | value: "true" 27 | effect: NoSchedule 28 | # Allow this pod to run on the master. 29 | - key: node-role.kubernetes.io/master 30 | effect: NoSchedule 31 | # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode. 32 | # This, along with the annotation above marks this pod as a critical add-on. 33 | - key: CriticalAddonsOnly 34 | operator: Exists 35 | # Only run this pod on the master. 36 | nodeSelector: 37 | node-role.kubernetes.io/master: "" 38 | hostNetwork: true 39 | containers: 40 | - name: calico-etcd 41 | image: quay.io/coreos/etcd:v3.3.9 42 | env: 43 | - name: CALICO_ETCD_IP 44 | valueFrom: 45 | fieldRef: 46 | fieldPath: status.podIP 47 | command: 48 | - /usr/local/bin/etcd 49 | args: 50 | - --name=calico 51 | - --data-dir=/var/etcd/calico-data 52 | - --advertise-client-urls=http://$(CALICO_ETCD_IP):6666 53 | - --listen-client-urls=http://0.0.0.0:6666 54 | - --listen-peer-urls=http://0.0.0.0:6667 55 | - --auto-compaction-retention=1 56 | volumeMounts: 57 | - name: var-etcd 58 | mountPath: /var/etcd 59 | volumes: 60 | - name: var-etcd 61 | hostPath: 62 | path: /var/etcd 63 | 64 | --- 65 | 66 | # This manifest installs the Service which gets traffic to the Calico 67 | # etcd. 68 | apiVersion: v1 69 | kind: Service 70 | metadata: 71 | labels: 72 | k8s-app: calico-etcd 73 | name: calico-etcd 74 | namespace: kube-system 75 | spec: 76 | # Select the calico-etcd pod running on the master. 77 | selector: 78 | k8s-app: calico-etcd 79 | # This ClusterIP needs to be known in advance, since we cannot rely 80 | # on DNS to get access to etcd. 81 | clusterIP: {{ calico_etcd_service }} 82 | ports: 83 | - port: 6666 84 | -------------------------------------------------------------------------------- /playbooks/roles/templates/calico-rbac.yml.j2: -------------------------------------------------------------------------------- 1 | # Calico Version v3.3.0 2 | # https://docs.projectcalico.org/v3.3/releases#v3.3.0 3 | 4 | --- 5 | 6 | kind: ClusterRole 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | metadata: 9 | name: calico-kube-controllers 10 | rules: 11 | - apiGroups: 12 | - "" 13 | - extensions 14 | resources: 15 | - pods 16 | - namespaces 17 | - networkpolicies 18 | - nodes 19 | - serviceaccounts 20 | verbs: 21 | - watch 22 | - list 23 | - apiGroups: 24 | - networking.k8s.io 25 | resources: 26 | - networkpolicies 27 | verbs: 28 | - watch 29 | - list 30 | --- 31 | kind: ClusterRoleBinding 32 | apiVersion: rbac.authorization.k8s.io/v1beta1 33 | metadata: 34 | name: calico-kube-controllers 35 | roleRef: 36 | apiGroup: rbac.authorization.k8s.io 37 | kind: ClusterRole 38 | name: calico-kube-controllers 39 | subjects: 40 | - kind: ServiceAccount 41 | name: calico-kube-controllers 42 | namespace: kube-system 43 | 44 | --- 45 | 46 | kind: ClusterRole 47 | apiVersion: rbac.authorization.k8s.io/v1beta1 48 | metadata: 49 | name: calico-node 50 | rules: 51 | - apiGroups: [""] 52 | resources: 53 | - pods 54 | - nodes 55 | - namespaces 56 | verbs: 57 | - get 58 | 59 | --- 60 | 61 | apiVersion: rbac.authorization.k8s.io/v1beta1 62 | kind: ClusterRoleBinding 63 | metadata: 64 | name: calico-node 65 | roleRef: 66 | apiGroup: rbac.authorization.k8s.io 67 | kind: ClusterRole 68 | name: calico-node 69 | subjects: 70 | - kind: ServiceAccount 71 | name: calico-node 72 | namespace: kube-system 73 | -------------------------------------------------------------------------------- /playbooks/roles/templates/flannel-rbac.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ClusterRole 3 | apiVersion: rbac.authorization.k8s.io/v1beta1 4 | metadata: 5 | name: flannel 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | verbs: 12 | - get 13 | - apiGroups: 14 | - "" 15 | resources: 16 | - nodes 17 | verbs: 18 | - list 19 | - watch 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - nodes/status 24 | verbs: 25 | - patch 26 | --- 27 | kind: ClusterRoleBinding 28 | apiVersion: rbac.authorization.k8s.io/v1beta1 29 | metadata: 30 | name: flannel 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: flannel 35 | subjects: 36 | - kind: ServiceAccount 37 | name: flannel 38 | namespace: kube-system 39 | --- 40 | apiVersion: v1 41 | kind: ServiceAccount 42 | metadata: 43 | name: flannel 44 | namespace: kube-system 45 | --- 46 | -------------------------------------------------------------------------------- /playbooks/spark-stack.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: create_groups.yml 3 | 4 | - name: Apply the ambari-agent role to all nodes 5 | hosts: hadoop-cluster 6 | become: yes 7 | pre_tasks: 8 | - name: Show hadoop-cluster info 9 | debug: var=hostvars[inventory_hostname] 10 | when: debug 11 | roles: 12 | - ambari-agent 13 | 14 | - name: Apply the ambari-server role to ambari-node group 15 | hosts: ambari-node 16 | become: yes 17 | roles: 18 | - ambari-server 19 | -------------------------------------------------------------------------------- /provision_rax.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ansible-playbook -i inventory/localhost --extra-vars="ansible_python_interpreter=$(which python)" playbooks/provision_rax.yml 4 | --------------------------------------------------------------------------------