├── inventory ├── empty ├── openstack │ └── group_vars │ │ └── all ├── static ├── azure │ ├── azure_rm.ini │ └── group_vars │ │ └── all ├── gce │ ├── group_vars │ │ └── all │ └── gce.ini └── aws │ └── group_vars │ └── all ├── playbooks ├── roles │ ├── common │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── templates │ │ │ └── tuned.conf.j2 │ │ ├── tasks │ │ │ ├── tuned.yml │ │ │ └── java.yml │ │ └── vars │ │ │ ├── suse-11.yml │ │ │ ├── debian-ubuntu-14.yml │ │ │ ├── redhat-6.yml │ │ │ ├── redhat-amazon.yml │ │ │ ├── suse-12.yml │ │ │ ├── redhat-7.yml │ │ │ ├── debian-ubuntu-16.yml │ │ │ └── debian-ubuntu-18.yml │ ├── database │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── vars │ │ │ ├── suse-11.yml │ │ │ ├── debian-ubuntu-14.yml │ │ │ ├── debian-ubuntu-16.yml │ │ │ ├── debian-ubuntu-18.yml │ │ │ ├── suse-12.yml │ │ │ ├── redhat-6.yml │ │ │ ├── redhat-amazon.yml │ │ │ └── redhat-7.yml │ │ └── tasks │ │ │ ├── mysql_play_db_access.yml │ │ │ ├── mysql_play.yml │ │ │ ├── postgres_play_db_access.yml │ │ │ └── postgres_play.yml │ ├── ambari-config │ │ ├── templates │ │ │ ├── vdf-HDF-3.1-latest.xml.j2 │ │ │ ├── vdf-HDF-3.2-latest.xml.j2 │ │ │ ├── vdf-HDF-3.3-latest.xml.j2 │ │ │ ├── vdf-HDF-3.4-latest.xml.j2 │ │ │ ├── vdf-HDP-2.4-latest.xml.j2 │ │ │ ├── vdf-HDP-2.5-latest.xml.j2 │ │ │ ├── vdf-HDP-2.6-latest.xml.j2 │ │ │ ├── vdf-HDP-3.0-latest.xml.j2 │ │ │ ├── vdf-HDP-3.1-latest.xml.j2 │ │ │ ├── vdf-HDF-part-release.j2 │ │ │ ├── vdf-HDP-part-release.j2 │ │ │ ├── vdf-HDF-3-latest.xml.j2 │ │ │ ├── vdf-HDP-2-latest.xml.j2 │ │ │ ├── vdf-HDP-3-latest.xml.j2 │ │ │ ├── vdf-HDF-part-repository-info.j2 │ │ │ ├── vdf-HDF-2.0-latest.xml.j2 │ │ │ ├── vdf-HDF-2.1-latest.xml.j2 │ │ │ ├── vdf-HDF-3.0-latest.xml.j2 │ │ │ ├── vdf-HDF-3.1.0.0.xml.j2 │ │ │ ├── vdf-HDF-3.1.2.0.xml.j2 │ │ │ ├── vdf-HDF-3.2.0.0.xml.j2 │ │ │ ├── vdf-HDF-3.3.1.0.xml.j2 │ │ │ ├── vdf-HDF-3.4.0.0.xml.j2 │ │ │ ├── vdf-HDP-part-repository-info.j2 │ │ │ ├── repo_version_template.json.j2 │ │ │ ├── vdf-HDP-3.0.0.0.xml.j2 │ │ │ ├── vdf-HDP-3.1.0.0.xml.j2 │ │ │ ├── vdf-HDP-2.5.0.0.xml.j2 │ │ │ ├── vdf-HDP-2.5.3.0.xml.j2 │ │ │ ├── vdf-HDP-2.5.5.0.xml.j2 │ │ │ ├── vdf-HDP-2.5.6.0.xml.j2 │ │ │ ├── vdf-HDP-2.6.0.3.xml.j2 │ │ │ ├── vdf-HDP-2.6.1.0.xml.j2 │ │ │ ├── vdf-HDP-2.6.2.0.xml.j2 │ │ │ ├── vdf-HDP-2.6.2.14.xml.j2 │ │ │ ├── vdf-HDP-2.6.5.0.xml.j2 │ │ │ ├── vdf-HDP-2.6.3.0.xml.j2 │ │ │ ├── vdf-HDP-2.6.4.0.xml.j2 │ │ │ └── vdf-HDP-2.4.3.0.xml.j2 │ │ ├── vars │ │ │ ├── redhat-amazon.yml │ │ │ ├── suse-11.yml │ │ │ ├── suse-12.yml │ │ │ ├── redhat.yml │ │ │ └── debian.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── krb5-client │ │ ├── vars │ │ │ ├── debian.yml │ │ │ ├── suse.yml │ │ │ └── redhat.yml │ │ ├── templates │ │ │ └── krb5.conf.j2 │ │ └── tasks │ │ │ └── main.yml │ ├── ambari-server │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ └── pgpass.j2 │ │ ├── vars │ │ │ ├── suse-11.yml │ │ │ ├── suse-12.yml │ │ │ ├── debian-ubuntu-14.yml │ │ │ ├── debian-ubuntu-16.yml │ │ │ ├── debian-ubuntu-18.yml │ │ │ ├── redhat-7.yml │ │ │ ├── redhat-6.yml │ │ │ └── redhat-amazon.yml │ │ └── handlers │ │ │ └── main.yml │ ├── ambari-repo │ │ ├── vars │ │ │ ├── redhat-amazon.yml │ │ │ ├── redhat.yml │ │ │ ├── suse-11.yml │ │ │ ├── suse-12.yml │ │ │ └── debian-ubuntu.yml │ │ ├── files │ │ │ └── ambari_pubkey.asc │ │ └── tasks │ │ │ └── main.yml │ ├── ambari-agent │ │ ├── meta │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── ambari-blueprint │ │ ├── vars │ │ │ ├── suse.yml │ │ │ ├── debian-ubuntu.yml │ │ │ └── redhat.yml │ │ ├── files │ │ │ ├── simple_nifi.json │ │ │ ├── blueprint_hdfs_only.json │ │ │ └── single_node.json │ │ ├── templates │ │ │ └── cluster_template.j2 │ │ └── defaults │ │ │ └── main.yml │ ├── mit-kdc │ │ ├── handlers │ │ │ └── main.yml │ │ ├── templates │ │ │ └── kdc.conf.j2 │ │ ├── vars │ │ │ ├── debian-ubuntu.yml │ │ │ ├── redhat.yml │ │ │ └── suse.yml │ │ └── tasks │ │ │ └── main.yml │ ├── post-install │ │ └── tasks │ │ │ └── main.yml │ └── README.md ├── post_install.yml ├── clouds │ ├── build_openstack.yml │ ├── add_nodes_azure.yml │ ├── add_nodes_gce.yml │ ├── add_nodes_openstack.yml │ ├── add_nodes_static.yml │ ├── add_nodes_aws.yml │ ├── build_openstack_nodes.yml │ ├── build_gce.yml │ ├── build_azure.yml │ ├── build_aws.yml │ └── build_gce_nodes.yml ├── install_cluster.yml ├── install_ambari.yml ├── configure_ambari.yml ├── apply_blueprint.yml ├── group_vars │ ├── example-hdf-singlenode │ ├── example-hdf-cluster │ ├── example-hdf-cluster-with-streamline │ ├── example-hdf-cluster-with-ranger │ ├── example-hdp3-singlenode-with-nifi │ ├── example-hdp-singlenode-with-nifi │ ├── example-hdp-singlenode │ ├── example-hdp3-singlenode │ ├── example-hdp3-singlenode-no-monitoring │ ├── example-hdp-singlenode-with-accumulo │ ├── example-hdp3-singlenode-with-druid │ ├── example-hdp-singlenode-with-storm-kafka │ ├── example-hdp-singlenode-with-ranger-atlas │ ├── example-hdp3-singlenode-with-ranger-atlas │ ├── example-hdp-singlenode-with-druid-solr-kafka-ranger │ ├── example-hdp3-singlenode-all-services │ ├── example-hdp-singlenode-all-services │ ├── example-hdp-ha-2-masters-with-solr │ ├── example-hdp-ha-3-masters │ ├── example-hdp3-ha-3-masters-with-accumulo │ ├── example-hdp-ha-3-masters-with-ranger-atlas │ ├── example-hdp-ha-3-masters-with-nifi │ ├── example-hdp-ha-3-masters-with-storm-kafka │ ├── example-hdp3-ha-3-masters-with-druid-atlas-knox-log │ └── example-hdp-ha-3-masters-with-nifi-kafka-druid └── prepare_nodes.yml ├── ansible.cfg ├── post_install.sh ├── install_ambari.sh ├── prepare_nodes.sh ├── apply_blueprint.sh ├── configure_ambari.sh ├── install_cluster.sh ├── molecule └── default │ ├── tests │ └── test_default.py │ ├── INSTALL.rst │ ├── Dockerfile.j2 │ ├── playbook.yml │ └── molecule.yml ├── set_inventory.sh ├── .travis.yml └── set_cloud.sh /inventory/empty: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /playbooks/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | swappiness: 1 3 | -------------------------------------------------------------------------------- /playbooks/roles/database/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | max_connections: 200 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.1-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDF-3.1.2.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.2-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDF-3.2.0.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.3-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDF-3.3.1.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.4-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDF-3.4.0.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.4-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDP-2.4.3.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.5-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDP-2.5.6.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDP-2.6.5.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-3.0-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDP-3.0.0.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-3.1-latest.xml.j2: -------------------------------------------------------------------------------- 1 | vdf-HDP-3.1.0.0.xml.j2 -------------------------------------------------------------------------------- /playbooks/roles/krb5-client/vars/debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kerberos_client_package: krb5-user 3 | -------------------------------------------------------------------------------- /playbooks/roles/krb5-client/vars/suse.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kerberos_client_package: krb5-client 3 | -------------------------------------------------------------------------------- /playbooks/roles/krb5-client/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kerberos_client_package: krb5-workstation 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_port: 5432 3 | mysql_port: 3306 4 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | repo_os_family: "amazonlinux2" 3 | baseurl_os_family: "amazonlinux2" 4 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory=inventory/empty 3 | forks=20 4 | host_key_checking = False 5 | timeout = 60 6 | ansible_keep_remote_files = True 7 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: ambari-repo 4 | when: ambari_repo_enabled | default(True) | bool 5 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ambari_repo_url: "{{ repo_base_url }}/ambari/amazonlinux2/2.x/updates/{{ ambari_version }}" 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: ambari-repo 4 | when: ambari_repo_enabled | default(True) | bool 5 | 6 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/vars/suse-11.yml: -------------------------------------------------------------------------------- 1 | --- 2 | repo_os_family: "suse11sp3" 3 | baseurl_os_family: "suse{{ ansible_distribution_major_version }}" 4 | 5 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/vars/suse-12.yml: -------------------------------------------------------------------------------- 1 | --- 2 | repo_os_family: "sles12" 3 | baseurl_os_family: "suse{{ ansible_distribution_major_version }}" 4 | 5 | 6 | -------------------------------------------------------------------------------- /playbooks/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Generate the Grub config file 3 | shell: "{{ update_grub_command }}" 4 | args: 5 | chdir: / 6 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ambari_repo_url: "{{ repo_base_url }}/ambari/centos{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}" 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/vars/suse-11.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ambari_repo_url: "{{ repo_base_url }}/ambari/suse{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}" 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/vars/suse-12.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ambari_repo_url: "{{ repo_base_url }}/ambari/sles{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}" 3 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | repo_os_family: "centos{{ ansible_distribution_major_version }}" 3 | baseurl_os_family: "redhat{{ ansible_distribution_major_version }}" 4 | 5 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/vars/debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | repo_os_family: "{{ ansible_distribution|lower }}{{ ansible_distribution_major_version }}" 3 | baseurl_os_family: "{{ ansible_distribution|lower }}{{ ansible_distribution_major_version }}" 4 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/templates/pgpass.j2: -------------------------------------------------------------------------------- 1 | {{ database_options.external_hostname|default(ansible_fqdn,true) }}:{{ postgres_port }}:{{ database_options.ambari_db_name }}:{{ database_options.ambari_db_username }}:{{ database_options.ambari_db_password }} -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/vars/suse.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 3 | 4 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 5 | 6 | mariadb_jdbc_location: /usr/share/java/mysql-connector-java.jar 7 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/vars/debian-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_jdbc_location: /usr/share/java/postgresql.jar 3 | 4 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 5 | 6 | mariadb_jdbc_location: /usr/share/java/mysql-connector-java.jar 7 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 3 | 4 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 5 | 6 | mariadb_jdbc_location: /usr/share/java/mysql-connector-java.jar 7 | -------------------------------------------------------------------------------- /playbooks/post_install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: "set_variables.yml" 3 | 4 | - name: Apply the post-install role to the hadoop-cluster group 5 | hosts: hadoop-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | roles: 9 | - post-install 10 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/vars/debian-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ambari_repo_url: "{{ repo_base_url }}/ambari/{{ ansible_distribution|lower }}{{ ansible_distribution_major_version }}/2.x/updates/{{ ambari_version }}" 3 | ambari_repo_key: "B9733A7A07513CAD" 4 | ambari_repo_keyfile: 'ambari_pubkey.asc' 5 | -------------------------------------------------------------------------------- /playbooks/roles/mit-kdc/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart krb5 3 | service: 4 | name: "{{ kdc_service_name }}" 5 | state: restarted 6 | notify: Restart kadmin 7 | 8 | - name: Restart kadmin 9 | service: 10 | name: "{{ kadmin_service_name }}" 11 | state: restarted 12 | -------------------------------------------------------------------------------- /playbooks/clouds/build_openstack.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Build the Cloud Environment 3 | hosts: localhost 4 | connection: local 5 | gather_facts: False 6 | tasks: 7 | - name: Build Openstack Nodes 8 | include_tasks: build_openstack_nodes.yml 9 | when: item.count > 0 10 | with_items: "{{ nodes }}" 11 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/suse-11.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_packages: 3 | - postgresql94 4 | - postgresql-jdbc 5 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 6 | 7 | mysql_packages: 8 | - python-mysql 9 | - mysql-connector-java 10 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 11 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/tuned.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # tuned configuration 3 | # 4 | 5 | [main] 6 | include=throughput-performance 7 | 8 | [sysctl] 9 | vm.swappiness={{ swappiness }} 10 | net.ipv6.conf.all.disable_ipv6 = 1 11 | net.ipv6.conf.default.disable_ipv6 = 1 12 | net.ipv6.conf.lo.disable_ipv6 = 1 13 | 14 | [vm] 15 | transparent_hugepages=never 16 | -------------------------------------------------------------------------------- /playbooks/roles/mit-kdc/templates/kdc.conf.j2: -------------------------------------------------------------------------------- 1 | [realms] 2 | {{ security_options.realm | upper }} = { 3 | database_name = {{ kdc_database_file }} 4 | acl_file = {{ kdc_acl_file }} 5 | supported_enctypes = aes256-cts-hmac-sha1-96:normal aes128-cts-hmac-sha1-96:normal des3-cbc-sha1:normal des-cbc-md5:normal arcfour-hmac-md5:normal 6 | } 7 | -------------------------------------------------------------------------------- /post_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "${BASH_SOURCE[0]}")/set_cloud.sh 4 | source $(dirname "${BASH_SOURCE[0]}")/set_inventory.sh 5 | 6 | ansible-playbook "playbooks/post_install.yml" \ 7 | --inventory="inventory/${inventory_to_use}" \ 8 | --extra-vars="cloud_name=${cloud_to_use}" \ 9 | "$@" 10 | -------------------------------------------------------------------------------- /install_ambari.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "${BASH_SOURCE[0]}")/set_cloud.sh 4 | source $(dirname "${BASH_SOURCE[0]}")/set_inventory.sh 5 | 6 | ansible-playbook "playbooks/install_ambari.yml" \ 7 | --inventory="inventory/${inventory_to_use}" \ 8 | --extra-vars="cloud_name=${cloud_to_use}" \ 9 | "$@" 10 | -------------------------------------------------------------------------------- /prepare_nodes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "${BASH_SOURCE[0]}")/set_cloud.sh 4 | source $(dirname "${BASH_SOURCE[0]}")/set_inventory.sh 5 | 6 | ansible-playbook "playbooks/prepare_nodes.yml" \ 7 | --inventory="inventory/${inventory_to_use}" \ 8 | --extra-vars="cloud_name=${cloud_to_use}" \ 9 | "$@" 10 | -------------------------------------------------------------------------------- /apply_blueprint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "${BASH_SOURCE[0]}")/set_cloud.sh 4 | source $(dirname "${BASH_SOURCE[0]}")/set_inventory.sh 5 | 6 | ansible-playbook "playbooks/apply_blueprint.yml" \ 7 | --inventory="inventory/${inventory_to_use}" \ 8 | --extra-vars="cloud_name=${cloud_to_use}" \ 9 | "$@" 10 | -------------------------------------------------------------------------------- /configure_ambari.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "${BASH_SOURCE[0]}")/set_cloud.sh 4 | source $(dirname "${BASH_SOURCE[0]}")/set_inventory.sh 5 | 6 | ansible-playbook "playbooks/configure_ambari.yml" \ 7 | --inventory="inventory/${inventory_to_use}" \ 8 | --extra-vars="cloud_name=${cloud_to_use}" \ 9 | "$@" 10 | -------------------------------------------------------------------------------- /install_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "${BASH_SOURCE[0]}")/set_cloud.sh 4 | source $(dirname "${BASH_SOURCE[0]}")/set_inventory.sh 5 | 6 | ansible-playbook "playbooks/install_cluster.yml" \ 7 | --inventory="inventory/${inventory_to_use}" \ 8 | --extra-vars="cloud_name=${cloud_to_use}" \ 9 | "$@" 10 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/suse-12.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib64/jvm/java-1.8.0-openjdk 3 | 4 | postgres_packages: 5 | - postgresql96 6 | - postgresql-jdbc 7 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 8 | 9 | mysql_packages: 10 | - python-mysql 11 | - mysql-connector-java 12 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 13 | -------------------------------------------------------------------------------- /playbooks/roles/post-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Set correct permissions to the http secret key 4 | file: 5 | path: /etc/security/http_secret 6 | owner: hdfs 7 | group: hadoop 8 | mode: 0440 9 | when: security != "none" and security_options.http_authentication|default(false) and install_hdp is defined and install_hdp 10 | -------------------------------------------------------------------------------- /playbooks/roles/database/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart postgres 3 | service: 4 | name: "{{ postgres_service_name }}" 5 | state: restarted 6 | 7 | - name: Restart mysql 8 | service: 9 | name: "{{ mysql_service_name }}" 10 | state: restarted 11 | 12 | - name: Restart mariadb 13 | service: 14 | name: "{{ mariadb_service_name }}" 15 | state: restarted 16 | -------------------------------------------------------------------------------- /playbooks/install_cluster.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: "prepare_nodes.yml" 3 | tags: prepare_nodes 4 | 5 | - import_playbook: "install_ambari.yml" 6 | tags: ambari 7 | 8 | - import_playbook: "configure_ambari.yml" 9 | tags: ambari,blueprint 10 | 11 | - import_playbook: "apply_blueprint.yml" 12 | tags: blueprint 13 | 14 | - import_playbook: "post_install.yml" 15 | tags: post_install 16 | -------------------------------------------------------------------------------- /molecule/default/tests/test_default.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE'] 7 | ).get_hosts('all') 8 | 9 | 10 | def test_hosts_file(host): 11 | f = host.file('/etc/hosts') 12 | 13 | assert f.exists 14 | assert f.user == 'root' 15 | assert f.group == 'root' 16 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reload systemd 3 | systemd: 4 | daemon_reload: yes 5 | when: (ansible_os_family|lower == "redhat" and ansible_distribution_major_version|int == 7) or (ansible_os_family|lower == "suse" and ansible_distribution_major_version|int == 12) 6 | 7 | - name: Restart ambari-agent 8 | service: 9 | name: ambari-agent 10 | state: restarted 11 | -------------------------------------------------------------------------------- /playbooks/roles/mit-kdc/vars/debian-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kdc_packages: 3 | - krb5-kdc 4 | - krb5-admin-server 5 | - krb5-user 6 | kdc_service_name: krb5-kdc 7 | kadmin_service_name: krb5-admin-server 8 | 9 | kdc_init_command: kdb5_util create -s -W 10 | kadmin_command: kadmin.local 11 | 12 | kdc_config_file: /etc/krb5kdc/kdc.conf 13 | kdc_acl_file: /etc/krb5kdc/kadm5.acl 14 | kdc_database_file: /var/lib/krb5kdc/principal 15 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/debian-ubuntu-14.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib/jvm/java-7-openjdk-amd64 3 | 4 | postgres_packages: 5 | - postgresql-client 6 | - libpq-dev 7 | - python-psycopg2 8 | - libpostgresql-jdbc-java 9 | postgres_jdbc_location: /usr/share/java/postgresql.jar 10 | 11 | mysql_packages: 12 | - python-mysqldb 13 | - libmysql-java 14 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 15 | -------------------------------------------------------------------------------- /playbooks/roles/mit-kdc/vars/redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kdc_packages: 3 | - krb5-server 4 | - krb5-libs 5 | - krb5-workstation 6 | kdc_service_name: krb5kdc 7 | kadmin_service_name: kadmin 8 | 9 | kdc_init_command: kdb5_util create -s -W 10 | kadmin_command: kadmin.local 11 | 12 | kdc_config_file: /var/kerberos/krb5kdc/kdc.conf 13 | kdc_acl_file: /var/kerberos/krb5kdc/kadm5.acl 14 | kdc_database_file: /var/kerberos/krb5kdc/principal 15 | -------------------------------------------------------------------------------- /playbooks/install_ambari.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: "set_variables.yml" 3 | 4 | - name: Apply the ambari-agent role to the hadoop-cluster group 5 | hosts: hadoop-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | roles: 9 | - ambari-agent 10 | 11 | - name: Apply the ambari-server role to the ambari-server group 12 | hosts: ambari-server 13 | any_errors_fatal: true 14 | become: yes 15 | roles: 16 | - ambari-server 17 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-part-release.j2: -------------------------------------------------------------------------------- 1 | STANDARD 2 | HDF-{{ hdf_minor_version }} 3 | {{ hdf_version }} 4 | {{ hdf_build_number_auto|default(hdf_build_number) }} 5 | {{ hdf_major_version }}\.[0-2]\.\d+\.\d+ 6 | http://example.com 7 | HDF-{{ hdf_version }} 8 | 9 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-part-release.j2: -------------------------------------------------------------------------------- 1 | STANDARD 2 | HDP-{{ hdp_minor_version }} 3 | {{ hdp_version }} 4 | {{ hdp_build_number_auto|default(hdp_build_number) }} 5 | {{ hdp_major_version }}\.\d+\.\d+\.\d+ 6 | http://example.com 7 | HDP-{{ hdp_version }} 8 | 9 | -------------------------------------------------------------------------------- /playbooks/roles/mit-kdc/vars/suse.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kdc_packages: 3 | - krb5-server 4 | - krb5-client 5 | kdc_service_name: krb5kdc 6 | kadmin_service_name: kadmind 7 | 8 | kdc_init_command: /usr/lib/mit/sbin/kdb5_util create -s 9 | kadmin_command: /usr/lib/mit/sbin/kadmin.local 10 | 11 | kdc_config_file: /var/lib/kerberos/krb5kdc/kdc.conf 12 | kdc_acl_file: /var/lib/kerberos/krb5kdc/kadm5.acl 13 | kdc_database_file: /var/lib/kerberos/krb5kdc/principal 14 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart ambari-agent 3 | service: 4 | name: ambari-agent 5 | state: restarted 6 | 7 | - name: Restart ambari-server 8 | service: 9 | name: ambari-server 10 | state: restarted 11 | notify: "Wait for Ambari Server to start listening on port 8080" 12 | 13 | - name: Wait for Ambari Server to start listening on port 8080 14 | wait_for: 15 | port: 8080 16 | host: 0.0.0.0 17 | delay: 1 18 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/debian-ubuntu-16.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib/jvm/java-8-openjdk-amd64 3 | 4 | postgres_packages: 5 | - postgresql-client 6 | - libpq-dev 7 | - python-psycopg2 8 | - python3-psycopg2 9 | - libpostgresql-jdbc-java 10 | postgres_jdbc_location: /usr/share/java/postgresql.jar 11 | 12 | mysql_packages: 13 | - python-mysqldb 14 | - python3-mysqldb 15 | - libmysql-java 16 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 17 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/debian-ubuntu-18.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib/jvm/java-8-openjdk-amd64 3 | 4 | postgres_packages: 5 | - postgresql-client 6 | - libpq-dev 7 | - python-psycopg2 8 | - python3-psycopg2 9 | - libpostgresql-jdbc-java 10 | postgres_jdbc_location: /usr/share/java/postgresql.jar 11 | 12 | mysql_packages: 13 | - python-mysqldb 14 | - python3-mysqldb 15 | - libmysql-java 16 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 17 | -------------------------------------------------------------------------------- /playbooks/configure_ambari.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: "set_variables.yml" 3 | 4 | - name: Gather facts from all nodes 5 | hosts: hadoop-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | 9 | - import_playbook: "check_dynamic_blueprint.yml" 10 | when: blueprint_file == 'blueprint_dynamic.j2' 11 | 12 | - name: Apply the ambari-config role to the ambari-server group 13 | hosts: ambari-server 14 | any_errors_fatal: true 15 | become: yes 16 | roles: 17 | - ambari-config 18 | -------------------------------------------------------------------------------- /playbooks/apply_blueprint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: "set_variables.yml" 3 | 4 | - name: Gather facts from all nodes 5 | hosts: hadoop-cluster 6 | any_errors_fatal: true 7 | become: yes 8 | 9 | - import_playbook: "check_dynamic_blueprint.yml" 10 | when: blueprint_file == 'blueprint_dynamic.j2' 11 | 12 | - name: Apply the ambari-blueprint role to the ambari-server group 13 | hosts: ambari-server 14 | any_errors_fatal: true 15 | become: yes 16 | roles: 17 | - ambari-blueprint 18 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib/jvm/java-1.8.0-openjdk 3 | 4 | postgres_packages: 5 | - postgresql 6 | - python-psycopg2 7 | - postgresql-jdbc 8 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 9 | 10 | mysql_packages: 11 | - MySQL-python 12 | - mysql-connector-java 13 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 14 | 15 | mariadb_packages: 16 | - MySQL-python 17 | - mysql-connector-java 18 | mariadb_jdbc_location: /usr/share/java/mysql-connector-java.jar 19 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3-latest.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | {% include 'vdf-HDF-part-repository-info.j2' %} 12 | 13 | 14 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2-latest.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | {% include 'vdf-HDP-part-repository-info.j2' %} 12 | 13 | 14 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-3-latest.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | {% include 'vdf-HDP-part-repository-info.j2' %} 12 | 13 | 14 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/redhat-6.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib/jvm/java-1.8.0-openjdk.x86_64 3 | 4 | postgres_packages: 5 | - postgresql 6 | - python-psycopg2 7 | - postgresql-jdbc 8 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 9 | 10 | mysql_packages: 11 | - MySQL-python 12 | - mysql-connector-java 13 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 14 | 15 | mariadb_packages: 16 | - MySQL-python 17 | - mysql-connector-java 18 | mariadb_jdbc_location: /usr/share/java/mysql-connector-java.jar 19 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openjdk_path: /usr/lib/jvm/java-1.8.0-openjdk 3 | 4 | postgres_packages: 5 | - postgresql 6 | - python-psycopg2 7 | - postgresql-jdbc 8 | postgres_jdbc_location: /usr/share/java/postgresql-jdbc.jar 9 | 10 | mysql_packages: 11 | - MySQL-python 12 | - mysql-connector-java 13 | mysql_jdbc_location: /usr/share/java/mysql-connector-java.jar 14 | 15 | mariadb_packages: 16 | - MySQL-python 17 | - mysql-connector-java 18 | mariadb_jdbc_location: /usr/share/java/mysql-connector-java.jar 19 | -------------------------------------------------------------------------------- /set_inventory.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -z "$INVENTORY_TO_USE" ]; then 4 | echo -e "\e[94m[INFO]\e[0m - INVENTORY_TO_USE environment variable not set, defaulting to 'inventory/${cloud_to_use}'" 5 | echo -e "See 'https://github.com/hortonworks/ansible-hortonworks/blob/master/INSTALL_${cloud_to_use}.md' for more details" 6 | inventory_to_use="${cloud_to_use}" 7 | else 8 | echo -e "\e[94m[INFO]\e[0m - INVENTORY_TO_USE environment variable set to '${INVENTORY_TO_USE}'" 9 | inventory_to_use="${INVENTORY_TO_USE}" 10 | fi 11 | 12 | 13 | -------------------------------------------------------------------------------- /playbooks/roles/common/tasks/tuned.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Gather the rpm package facts 3 | package_facts: 4 | manager: auto 5 | 6 | - block: 7 | - name: Create tuned directory 8 | file: 9 | path: /etc/tuned/hortonworks/ 10 | state: directory 11 | 12 | - name: Upload the tuned profile 13 | template: 14 | src: tuned.conf.j2 15 | dest: /etc/tuned/hortonworks/tuned.conf 16 | mode: 0755 17 | 18 | - name: Activate the tuned profile 19 | shell: tuned-adm profile hortonworks 20 | when: ansible_facts.packages['tuned'] is defined -------------------------------------------------------------------------------- /inventory/openstack/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | cloud_config: 3 | name_suffix: '' 4 | zone: PS 5 | admin_username: centos 6 | ssh: 7 | keyname: field 8 | privatekey: ~/.ssh/field.pem 9 | 10 | nodes: 11 | - host_group: "hdp-master" 12 | count: 1 13 | image: CentOS 7.4 14 | flavor: m3.xlarge 15 | public_ip: false 16 | 17 | - host_group: "hdp-slave" 18 | count: 2 19 | image: CentOS 7.4 20 | flavor: m3.large 21 | public_ip: false 22 | 23 | - host_group: "hdp-edge" 24 | count: 0 25 | image: CentOS 7.4 26 | flavor: m3.medium 27 | public_ip: false 28 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/suse-11.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_packages: 3 | - postgresql94-server 4 | postgres_service_name: postgresql 5 | postgres_initdb_command: service postgresql start 6 | postgres_config_file: /var/lib/pgsql/data/postgresql.conf 7 | postgres_hba_file: /var/lib/pgsql/data/pg_hba.conf 8 | 9 | mysql_version: '5.6' 10 | mysql_repo_url: 'http://repo.mysql.com/yum/mysql-{{ mysql_version }}-community/sles/11/$basearch/' 11 | mysql_packages: 12 | - mysql-community-client 13 | - mysql-community-server 14 | - python-mysql 15 | mysql_service_name: mysql 16 | mysql_config_file: /etc/my.cnf 17 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-part-repository-info.j2: -------------------------------------------------------------------------------- 1 | 2 | {{ hdf_version|replace('-','_') }}_* 3 | 4 | {{ hdf_main_repo_url }} 5 | HDF-{{ hdf_minor_version }} 6 | HDF 7 | true 8 | 9 | 10 | {{ utils_repo_url }} 11 | HDP-UTILS-{{ utils_version }} 12 | HDP-UTILS 13 | false 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /playbooks/roles/krb5-client/templates/krb5.conf.j2: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = {{ security_options.realm | upper }} 3 | default_ccache_name = /tmp/krb5cc_%{uid} 4 | [realms] 5 | {{ security_options.realm | upper }} = { 6 | kdc = {{ security_options.external_hostname|default(hostvars[groups['ambari-server']|first]['ansible_fqdn'],true) }} 7 | admin_server = {{ security_options.external_hostname|default(hostvars[groups['ambari-server']|first]['ansible_fqdn'],true) }} 8 | } 9 | [logging] 10 | default = FILE:/var/log/krb5kdc.log 11 | admin_server = FILE:/var/log/kadmind.log 12 | kdc = FILE:/var/log/krb5kdc.log 13 | -------------------------------------------------------------------------------- /molecule/default/INSTALL.rst: -------------------------------------------------------------------------------- 1 | ******* 2 | Docker driver installation guide 3 | ******* 4 | 5 | Requirements 6 | ============ 7 | 8 | * Docker Engine 9 | 10 | Install 11 | ======= 12 | 13 | Please refer to the `Virtual environment`_ documentation for installation best 14 | practices. If not using a virtual environment, please consider passing the 15 | widely recommended `'--user' flag`_ when invoking ``pip``. 16 | 17 | .. _Virtual environment: https://virtualenv.pypa.io/en/latest/ 18 | .. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site 19 | 20 | .. code-block:: bash 21 | 22 | $ pip install 'molecule[docker]' 23 | -------------------------------------------------------------------------------- /playbooks/roles/database/tasks/mysql_play_db_access.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Create the {{ db_username }} database user and IP based access from {{ outer_loop }} ({{ database }}) 5 | mysql_user: 6 | name: "{{ db_username }}" 7 | host: "{{ hostvars[outer_loop]['ansible_'~local_loop | regex_replace('-','_')]['ipv4']['address'] }}" 8 | priv: "{{ db_name }}.*:ALL" 9 | password: "{{ db_password }}" 10 | state: present 11 | loop_control: 12 | loop_var: local_loop 13 | with_items: "{{ hostvars[outer_loop]['ansible_interfaces'] }}" 14 | when: local_loop != 'lo' and 'ipv4' in hostvars[outer_loop]['ansible_'~local_loop | regex_replace('-','_')] 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | sudo: required 3 | 4 | language: python 5 | 6 | services: 7 | - docker 8 | 9 | install: 10 | # molecule version >v2.22 required for compatibility with ansible 2.8.x (provided in the used docker image) 11 | - pip install molecule==2.22 12 | # Use ansible 2.8 due to issue in 2.9.0 in the blueprint role task setting the 'cluster_template' fact 13 | # Related issue (most probably): https://github.com/ansible/ansible/issues/64169 14 | - pip install ansible==2.8.2 15 | - pip install docker 16 | script: 17 | - molecule test 18 | # TODO test 2nd scenario (multi-node !?) 19 | #- molecule test --scenario-name xxx 20 | 21 | notifications: 22 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ 23 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-server/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reload systemd 3 | systemd: 4 | daemon_reload: yes 5 | when: (ansible_os_family|lower == "redhat" and ansible_distribution_major_version|int == 7) or (ansible_os_family|lower == "suse" and ansible_distribution_major_version|int == 12) 6 | 7 | - name: Restart ambari-agent 8 | service: 9 | name: ambari-agent 10 | state: restarted 11 | 12 | - name: Restart ambari-server 13 | service: 14 | name: ambari-server 15 | state: restarted 16 | notify: "Wait for Ambari Server to start listening on port 8080" 17 | 18 | - name: Wait for Ambari Server to start listening on port 8080 19 | wait_for: 20 | port: 8080 21 | host: 0.0.0.0 22 | delay: 1 23 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/suse-11.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - php53-curl # php-curl requred by Ambari in SLES 6 | - unzip # unzip required by Ambari 7 | - tar # tar required by Ambari 8 | - wget # wget required by Ambari 9 | - openssl # openssl required by Ambari 10 | - ntp # ntp required by Hadoop 11 | - libopenssl-devel # openssl-devel recommended for Ranger KMS 12 | - libtirpc-devel # libtirpc required by HDP 2.6+ 13 | 14 | ntp_service_name: ntp 15 | 16 | firewall_service_name: SuSEfirewall2_setup 17 | -------------------------------------------------------------------------------- /playbooks/roles/database/tasks/mysql_play.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create the {{ db_name }} database ({{ database }}) 3 | mysql_db: 4 | name: "{{ db_name }}" 5 | encoding: "{{ db_encoding | default(omit) }}" 6 | state: present 7 | 8 | - name: Create the {{ db_username }} database user and host based access ({{ database }}) 9 | mysql_user: 10 | name: "{{ db_username }}" 11 | host: "{{ hostvars[item]['ansible_fqdn'] }}" 12 | priv: "{{ db_name }}.*:ALL" 13 | password: "{{ db_password }}" 14 | state: present 15 | with_items: "{{ db_client_hosts }}" 16 | 17 | - name: Configure IP based access for the {{ db_username }} user ({{ database }}) 18 | include_tasks: mysql_play_db_access.yml 19 | with_items: "{{ db_client_hosts }}" 20 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/debian-ubuntu-14.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh-client # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - unzip # unzip required by Ambari 6 | - tar # tar required by Ambari 7 | - wget # wget required by Ambari 8 | - openssl # openssl required by Ambari 9 | - ntp # ntp required by Hadoop 10 | - libssl-dev # openssl-devel recommended for Ranger KMS 11 | - libtirpc-dev # libtirpc required by HDP 2.6+ 12 | 13 | openjdk_package: openjdk-7-jdk 14 | 15 | ntp_service_name: ntp 16 | 17 | firewall_service_name: ufw 18 | 19 | update_grub_command: "/usr/sbin/update-grub" 20 | -------------------------------------------------------------------------------- /playbooks/clouds/add_nodes_azure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Add nodes to hadoop-cluster and {{ outer_loop.host_group }} groups 5 | add_host: 6 | name: "{{ hostvars[local_loop].name }}" 7 | ansible_host: "{{ (outer_loop.public_ip|default(false)== true) | ternary(hostvars[local_loop].public_ip,hostvars[local_loop].private_ip) }}" 8 | ansible_user: "{{ cloud_config.admin_username|default('root') }}" 9 | ansible_ssh_private_key_file: "{{ cloud_config.ssh.privatekey }}" 10 | ansible_become_user: root 11 | groups: hadoop-cluster, {{ outer_loop.host_group }} 12 | loop_control: 13 | loop_var: local_loop 14 | when: outer_loop.count > 0 15 | with_items: "{{ groups['Group_'~cluster_name~'-'~outer_loop.host_group] }}" 16 | -------------------------------------------------------------------------------- /playbooks/clouds/add_nodes_gce.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Add nodes to hadoop-cluster and {{ outer_loop.host_group }} groups 5 | add_host: 6 | name: "{{ hostvars[local_loop].gce_name }}" 7 | ansible_host: "{{ (outer_loop.public_ip|default(false)== true) | ternary(hostvars[local_loop].gce_public_ip,hostvars[local_loop].gce_private_ip) }}" 8 | ansible_user: "{{ cloud_config.admin_username|default('root') }}" 9 | ansible_ssh_private_key_file: "{{ cloud_config.ssh.privatekey }}" 10 | ansible_become_user: root 11 | groups: hadoop-cluster, {{ outer_loop.host_group }} 12 | loop_control: 13 | loop_var: local_loop 14 | when: outer_loop.count > 0 15 | with_items: "{{ groups[cluster_name~'-'~outer_loop.host_group] }}" 16 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/redhat-6.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh-clients # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - unzip # unzip required by Ambari 6 | - tar # tar required by Ambari 7 | - wget # wget required by Ambari 8 | - openssl # openssl required by Ambari 9 | - ntp # ntp required by Hadoop 10 | - openssl-devel # openssl-devel recommended for Ranger KMS 11 | - libtirpc-devel # libtirpc required by HDP 2.6+ 12 | - libselinux-python # libselinux-python required by Ansible 13 | 14 | openjdk_package: java-1.8.0-openjdk-devel 15 | 16 | ntp_service_name: ntpd 17 | 18 | firewall_service_name: iptables 19 | -------------------------------------------------------------------------------- /playbooks/clouds/add_nodes_openstack.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Add nodes to hadoop-cluster and {{ outer_loop.host_group }} groups 5 | add_host: 6 | name: "{{ hostvars[local_loop].openstack.name }}" 7 | ansible_host: "{{ (outer_loop.public_ip|default(false)== true) | ternary(hostvars[local_loop].openstack.public_v4,hostvars[local_loop].openstack.private_v4) }}" 8 | ansible_user: "{{ cloud_config.admin_username|default('root') }}" 9 | ansible_ssh_private_key_file: "{{ cloud_config.ssh.privatekey }}" 10 | ansible_become_user: root 11 | groups: hadoop-cluster, {{ outer_loop.host_group }} 12 | loop_control: 13 | loop_var: local_loop 14 | when: outer_loop.count > 0 15 | with_items: "{{ groups['meta-Group_'~cluster_name~'-'~outer_loop.host_group] }}" 16 | -------------------------------------------------------------------------------- /playbooks/clouds/add_nodes_static.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add nodes to hadoop-cluster group 3 | add_host: 4 | name: "{{ hostvars[local_loop]['inventory_hostname'] }}" 5 | ansible_host: "{{ hostvars[local_loop]['ansible_host']|default(hostvars[local_loop]['inventory_hostname']) }}" 6 | ansible_user: "{{ hostvars[local_loop]['ansible_user']|default('root') }}" 7 | ansible_ssh_private_key_file: "{{ hostvars[local_loop]['ansible_ssh_private_key_file']|default(omit) }}" 8 | ansible_ssh_pass: "{{ hostvars[local_loop]['ansible_ssh_pass']|default(omit) }}" 9 | ansible_become_user: root 10 | groups: hadoop-cluster 11 | loop_control: 12 | loop_var: local_loop 13 | with_items: "{{ groups['all'] }}" 14 | when: groups['hadoop-cluster'] is not defined or groups['hadoop-cluster']|length == 0 15 | 16 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-2.0-latest.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {% include 'vdf-HDF-part-repository-info.j2' %} 16 | 17 | 18 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh-clients # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - unzip # unzip required by Ambari 6 | - tar # tar required by Ambari 7 | - wget # wget required by Ambari 8 | - openssl # openssl required by Ambari 9 | - chrony # ntp required by Hadoop 10 | - openssl-devel # openssl-devel recommended for Ranger KMS 11 | - libtirpc-devel # libtirpc required by HDP 2.6+ 12 | - libselinux-python # libselinux-python required by Ansible 13 | 14 | openjdk_package: java-1.8.0-openjdk-devel 15 | 16 | ntp_service_name: chronyd 17 | 18 | update_grub_command: 'grub2-mkconfig -o "$(readlink -n /etc/grub2.cfg)"' 19 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/debian-ubuntu-14.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_version: 9.4 3 | postgres_repo_url: http://apt.postgresql.org/pub/repos/apt/ 4 | postgres_repo_key: ACCC4CF8 5 | postgres_repo_keyfile: postgres_pubkey.asc 6 | postgres_packages: 7 | - postgresql-{{ postgres_version }} 8 | postgres_service_name: postgresql 9 | postgres_initdb_command: /bin/true 10 | postgres_config_file: "/etc/postgresql/{{ postgres_version }}/main/postgresql.conf" 11 | postgres_hba_file: "/etc/postgresql/{{ postgres_version }}/main/pg_hba.conf" 12 | 13 | mysql_version: 5.6 14 | mysql_repo_url: http://repo.mysql.com/apt/ubuntu/ 15 | mysql_repo_key: 5072E1F5 16 | mysql_repo_keyfile: mysql_pubkey.asc 17 | mysql_packages: 18 | - mysql-community-server 19 | - python-mysqldb 20 | mysql_service_name: mysql 21 | mysql_config_file: /etc/mysql/mysql.conf.d/mysqld.cnf 22 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/suse-12.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - php5-curl # php-curl requred by Ambari in SLES 6 | - unzip # unzip required by Ambari 7 | - tar # tar required by Ambari 8 | - wget # wget required by Ambari 9 | - openssl # openssl required by Ambari 10 | - ntp # ntp required by Hadoop 11 | - libopenssl-devel # openssl-devel recommended for Ranger KMS 12 | - libtirpc-devel # libtirpc required by HDP 2.6+ 13 | 14 | openjdk_package: java-1_8_0-openjdk-devel 15 | 16 | ntp_service_name: ntpd 17 | 18 | firewall_service_name: SuSEfirewall2 19 | 20 | update_grub_command: 'grub2-mkconfig -o /boot/grub2/grub.cfg' 21 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/debian-ubuntu-16.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_version: 9.6 3 | postgres_repo_url: http://apt.postgresql.org/pub/repos/apt/ 4 | postgres_repo_key: ACCC4CF8 5 | postgres_repo_keyfile: postgres_pubkey.asc 6 | postgres_packages: 7 | - postgresql-{{ postgres_version }} 8 | postgres_service_name: postgresql 9 | postgres_initdb_command: /bin/true 10 | postgres_config_file: "/etc/postgresql/{{ postgres_version }}/main/postgresql.conf" 11 | postgres_hba_file: "/etc/postgresql/{{ postgres_version }}/main/pg_hba.conf" 12 | 13 | mysql_version: 5.7 14 | mysql_repo_url: http://repo.mysql.com/apt/ubuntu/ 15 | mysql_repo_key: 5072E1F5 16 | mysql_repo_keyfile: mysql_pubkey.asc 17 | mysql_packages: 18 | - mysql-community-server 19 | - python-mysqldb 20 | - python3-mysqldb 21 | mysql_service_name: mysql 22 | mysql_config_file: /etc/mysql/mysql.conf.d/mysqld.cnf 23 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/debian-ubuntu-18.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_version: 9.6 3 | postgres_repo_url: http://apt.postgresql.org/pub/repos/apt/ 4 | postgres_repo_key: ACCC4CF8 5 | postgres_repo_keyfile: postgres_pubkey.asc 6 | postgres_packages: 7 | - postgresql-{{ postgres_version }} 8 | postgres_service_name: postgresql 9 | postgres_initdb_command: /bin/true 10 | postgres_config_file: "/etc/postgresql/{{ postgres_version }}/main/postgresql.conf" 11 | postgres_hba_file: "/etc/postgresql/{{ postgres_version }}/main/pg_hba.conf" 12 | 13 | mysql_version: 5.7 14 | mysql_repo_url: http://repo.mysql.com/apt/ubuntu/ 15 | mysql_repo_key: 5072E1F5 16 | mysql_repo_keyfile: mysql_pubkey.asc 17 | mysql_packages: 18 | - mysql-community-server 19 | - python-mysqldb 20 | - python3-mysqldb 21 | mysql_service_name: mysql 22 | mysql_config_file: /etc/mysql/mysql.conf.d/mysqld.cnf 23 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/suse-12.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_packages: 3 | - postgresql96-server 4 | postgres_service_name: postgresql 5 | postgres_initdb_command: systemctl start postgresql 6 | postgres_config_file: /var/lib/pgsql/data/postgresql.conf 7 | postgres_hba_file: /var/lib/pgsql/data/pg_hba.conf 8 | 9 | mysql_version: '5.6' 10 | mysql_repo_url: 'http://repo.mysql.com/yum/mysql-{{ mysql_version }}-community/sles/12/$basearch/' 11 | mysql_packages: 12 | - mysql-community-client 13 | - mysql-community-server 14 | - python-mysql 15 | mysql_service_name: mysql 16 | mysql_config_file: /etc/my.cnf 17 | 18 | mariadb_version: '10.2' 19 | mariadb_repo_url: 'http://yum.mariadb.org/{{ mariadb_version }}/sles12-amd64' 20 | mariadb_packages: 21 | - MariaDB-server 22 | - python-mysql 23 | mariadb_service_name: mariadb 24 | mariadb_config_file: /etc/my.cnf.d/server.cnf 25 | -------------------------------------------------------------------------------- /inventory/static: -------------------------------------------------------------------------------- 1 | ###################################################### 2 | ## 3 | ## The static inventory puts the hosts in different groups as described here: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#hosts-and-groups 4 | ## Variables would be defined directly for each host, including the ansible_user, the login user with sudoers permissions. 5 | ## For authentication, either the password or the SSH Private Key can be used. 6 | ## 7 | ###################################################### 8 | 9 | [hdp-master] 10 | master01 ansible_host=192.168.0.2 ansible_user=root ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 11 | 12 | [hdp-slave] 13 | slave01 ansible_host=192.168.0.3 ansible_user=root ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 14 | #slave02 ansible_host=192.168.0.4 ansible_user=root ansible_ssh_pass=changeme 15 | -------------------------------------------------------------------------------- /playbooks/roles/database/tasks/postgres_play_db_access.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Configure access for the {{ db_username }} user from {{ outer_loop }} ({{ database }}) 5 | lineinfile: 6 | path: "{{ postgres_hba_file }}" 7 | regexp: "^host {{ db_name }} {{ db_username }} {{ hostvars[outer_loop]['ansible_'~local_loop | regex_replace('-','_')]['ipv4']['address'] }}/32 .*" 8 | line: "host {{ db_name }} {{ db_username }} {{ hostvars[outer_loop]['ansible_'~local_loop | regex_replace('-','_')]['ipv4']['address'] }}/32 md5" 9 | insertbefore: EOF 10 | state: present 11 | loop_control: 12 | loop_var: local_loop 13 | notify: Restart postgres 14 | with_items: "{{ hostvars[outer_loop]['ansible_interfaces'] }}" 15 | when: local_loop != 'lo' and 'ipv4' in hostvars[outer_loop]['ansible_'~local_loop | regex_replace('-','_')] 16 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-2.1-latest.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% include 'vdf-HDF-part-repository-info.j2' %} 17 | 18 | 19 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.0-latest.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% include 'vdf-HDF-part-repository-info.j2' %} 17 | 18 | 19 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh-clients # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - unzip # unzip required by Ambari 6 | - tar # tar required by Ambari 7 | - wget # wget required by Ambari 8 | - openssl # openssl required by Ambari 9 | - chrony # ntp required by Hadoop 10 | - openssl-devel # openssl-devel recommended for Ranger KMS 11 | - libtirpc-devel # libtirpc required by HDP 2.6+ 12 | - libselinux-python # libselinux-python required by Ansible 13 | 14 | openjdk_package: java-1.8.0-openjdk-devel 15 | 16 | ntp_service_name: chronyd 17 | 18 | firewall_service_name: firewalld 19 | 20 | update_grub_command: 'grub2-mkconfig -o "$(readlink -n /etc/grub2.cfg)"' 21 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/debian-ubuntu-16.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh-client # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - unzip # unzip required by Ambari 6 | - tar # tar required by Ambari 7 | - wget # wget required by Ambari 8 | - openssl # openssl required by Ambari 9 | - ntp # ntp required by Hadoop 10 | - python3-apt # python-apt required by Ansible 11 | - libssl-dev # openssl-devel recommended for Ranger KMS 12 | - python2.7 # python 2.7 required by Ambari 13 | - libtirpc-dev # libtirpc required by HDP 2.6+ 14 | 15 | openjdk_package: openjdk-8-jdk 16 | 17 | ntp_service_name: ntp 18 | 19 | firewall_service_name: ufw 20 | 21 | update_grub_command: "/usr/sbin/update-grub" 22 | -------------------------------------------------------------------------------- /playbooks/roles/common/vars/debian-ubuntu-18.yml: -------------------------------------------------------------------------------- 1 | --- 2 | packages: 3 | - openssh-client # scp required by Ambari 4 | - curl # curl required by Ambari 5 | - unzip # unzip required by Ambari 6 | - tar # tar required by Ambari 7 | - wget # wget required by Ambari 8 | - openssl # openssl required by Ambari 9 | - ntp # ntp required by Hadoop 10 | - python3-apt # python-apt required by Ansible 11 | - libssl-dev # openssl-devel recommended for Ranger KMS 12 | - python2.7 # python 2.7 required by Ambari 13 | - libtirpc-dev # libtirpc required by HDP 2.6+ 14 | 15 | openjdk_package: openjdk-8-jdk 16 | 17 | ntp_service_name: ntp 18 | 19 | firewall_service_name: ufw 20 | 21 | update_grub_command: "/usr/sbin/update-grub" 22 | -------------------------------------------------------------------------------- /inventory/azure/azure_rm.ini: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration file for azure_rm.py 3 | # 4 | [azure] 5 | # Control which resource groups are included. By default all resources groups are included. 6 | # Set resource_groups to a comma separated list of resource groups names. 7 | #resource_groups= 8 | 9 | # Control which tags are included. Set tags to a comma separated list of keys or key:value pairs 10 | #tags= 11 | 12 | # Control which locations are included. Set locations to a comma separated list (e.g. eastus,eastus2,westus) 13 | #locations= 14 | 15 | # Include powerstate. If you don't need powerstate information, turning it off improves runtime performance. 16 | include_powerstate=yes 17 | 18 | # Control grouping with the following boolean flags. Valid values: yes, no, true, false, True, False, 0, 1. 19 | group_by_resource_group=yes 20 | group_by_location=yes 21 | group_by_security_group=yes 22 | group_by_tag=yes 23 | -------------------------------------------------------------------------------- /playbooks/clouds/add_nodes_aws.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Add nodes to hadoop-cluster and {{ outer_loop.host_group }} groups 5 | add_host: 6 | name: "{{ (outer_loop.public_ip|default(false)== true) | ternary(hostvars[local_loop].ec2_dns_name,hostvars[local_loop].ec2_private_dns_name) }}" 7 | ansible_host: "{{ (outer_loop.public_ip|default(false)== true) | ternary(hostvars[local_loop].ec2_ip_address,hostvars[local_loop].ec2_private_ip_address) }}" 8 | ansible_user: "{{ cloud_config.admin_username|default('root') }}" 9 | ansible_ssh_private_key_file: "{{ cloud_config.ssh.privatekey }}" 10 | ansible_become_user: root 11 | groups: hadoop-cluster, {{ outer_loop.host_group }} 12 | loop_control: 13 | loop_var: local_loop 14 | when: outer_loop.count > 0 15 | with_items: "{{ groups[('tag_Group_'~cluster_name~'-'~outer_loop.host_group)|replace('-','_')] }}" 16 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ### These are defaults originaly in each OS type vars 4 | ### by moving these here it allows for over-riding 5 | 6 | gpl_repo_url: "{{ repo_base_url }}/HDP-GPL/{{ repo_os_family }}/{{ hdp_major_version }}.x/updates/{{ hdp_version }}" 7 | 8 | hdf_main_repo_url: "{{ repo_base_url }}/HDF/{{ repo_os_family }}/{{ hdf_major_version }}.x/updates/{{ hdf_version }}" 9 | 10 | hdp_main_repo_url: "{{ repo_base_url }}/HDP/{{ repo_os_family }}/{{ hdp_major_version }}.x/updates/{{ hdp_version }}" 11 | hdpsearch_main_repo_url: "{{ repo_base_url }}/HDP-SOLR-{{ hdpsearch_version }}-{{ hdpsearch_build_number }}/repos/{{ repo_os_family }}" 12 | 13 | main_os_target: "operating_systems/{{ baseurl_os_family }}" 14 | 15 | utils_os_target: "operating_systems/{{ baseurl_os_family }}/repositories/HDP-UTILS-{{ utils_version }}" 16 | utils_repo_url: "{{ repo_base_url }}/HDP-UTILS-{{ utils_version }}/repos/{{ repo_os_family }}" 17 | -------------------------------------------------------------------------------- /playbooks/roles/krb5-client/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version }}.yml" 7 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}-{{ ansible_distribution_version }}.yml" 8 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml" 9 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version }}.yml" 10 | - "{{ ansible_os_family|lower }}.yml" 11 | - defaults.yml 12 | paths: 13 | - ../krb5-client/vars 14 | 15 | - name: Install the Kerberos client package 16 | package: 17 | name: "{{ item }}" 18 | update_cache: yes 19 | state: present 20 | with_items: "{{ kerberos_client_package }}" 21 | 22 | - name: Upload the krb5.conf file 23 | template: 24 | src: krb5.conf.j2 25 | dest: /etc/krb5.conf 26 | mode: 0644 27 | force: yes 28 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/files/simple_nifi.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations" : [ 3 | { 4 | "nifi-ambari-config" : { 5 | "nifi.node.ssl.port": "9091", 6 | "nifi.node.port": "9090", 7 | "nifi.security.encrypt.configuration.password": "{{ default_password }}", 8 | "nifi.sensitive.props.key": "{{ default_password }}" 9 | } 10 | }, 11 | { 12 | "nifi-env" : { 13 | "nifi_group" : "nifi", 14 | "nifi_user" : "nifi" 15 | } 16 | } 17 | ], 18 | "host_groups" : [ 19 | { 20 | "name" : "hdf-master", 21 | "configurations" : [ ], 22 | "components" : [ 23 | { "name" : "ZOOKEEPER_CLIENT" }, 24 | { "name" : "ZOOKEEPER_SERVER" }, 25 | { "name" : "AMBARI_SERVER" }, 26 | { "name" : "NIFI_MASTER" }, 27 | { "name" : "METRICS_COLLECTOR" }, 28 | { "name" : "METRICS_GRAFANA" }, 29 | { "name" : "METRICS_MONITOR" } 30 | ] 31 | } 32 | ], 33 | "Blueprints" : { 34 | "stack_name" : "HDF", 35 | "stack_version" : "3.1" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdf-singlenode: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDF instance with Ambari and NiFi. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdf-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - KAFKA_BROKER 16 | - NIFI_MASTER 17 | - NIFI_REGISTRY_MASTER 18 | - AMBARI_SERVER 19 | - INFRA_SOLR 20 | - METRICS_COLLECTOR 21 | - METRICS_GRAFANA 22 | - METRICS_MONITOR 23 | -------------------------------------------------------------------------------- /playbooks/prepare_nodes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: "set_variables.yml" 3 | tags: always 4 | 5 | - name: Apply the common role to the hadoop-cluster group 6 | hosts: hadoop-cluster 7 | any_errors_fatal: true 8 | become: yes 9 | roles: 10 | - common 11 | tags: common 12 | 13 | - name: Apply the database role to the ambari-server group 14 | hosts: ambari-server 15 | any_errors_fatal: true 16 | become: yes 17 | roles: 18 | - role: database 19 | when: database != "embedded" and not database_options.external_hostname|default('') 20 | tags: database 21 | 22 | - name: Apply the krb5-client role to the hadoop-cluster group 23 | hosts: hadoop-cluster 24 | any_errors_fatal: true 25 | become: yes 26 | roles: 27 | - role: krb5-client 28 | when: security == "mit-kdc" 29 | tags: mit-kdc 30 | 31 | - name: Apply the mit-kdc role to the ambari-server group 32 | hosts: ambari-server 33 | any_errors_fatal: true 34 | become: yes 35 | roles: 36 | - role: mit-kdc 37 | when: security == "mit-kdc" and not security_options.external_hostname|default('') 38 | tags: mit-kdc 39 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.1.0.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% include 'vdf-HDF-part-repository-info.j2' %} 21 | 22 | 23 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.1.2.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% include 'vdf-HDF-part-repository-info.j2' %} 21 | 22 | 23 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.2.0.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | {% include 'vdf-HDF-part-repository-info.j2' %} 22 | 23 | 24 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.3.1.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | {% include 'vdf-HDF-part-repository-info.j2' %} 22 | 23 | 24 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDF-3.4.0.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDF-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | {% include 'vdf-HDF-part-repository-info.j2' %} 22 | 23 | 24 | -------------------------------------------------------------------------------- /molecule/default/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | # Molecule managed 2 | 3 | {% if item.registry is defined %} 4 | FROM {{ item.registry.url }}/{{ item.image }} 5 | {% else %} 6 | FROM {{ item.image }} 7 | {% endif %} 8 | 9 | {% if item.env is defined %} 10 | {% for var, value in item.env.items() %} 11 | {% if value %} 12 | ENV {{ var }} {{ value }} 13 | {% endif %} 14 | {% endfor %} 15 | {% endif %} 16 | 17 | RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \ 18 | elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ 19 | elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ 20 | elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ 21 | elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ 22 | elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi 23 | -------------------------------------------------------------------------------- /playbooks/clouds/build_openstack_nodes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Create {{ outer_loop.host_group }} node(s) 5 | os_server: 6 | name: "{{ cluster_name }}-{{ outer_loop.host_group }}{{ (outer_loop.count > 1) | ternary(local_loop,'') }}{{ cloud_config.name_suffix|default('') }}" 7 | image: "{{ outer_loop.image }}" 8 | flavor: "{{ outer_loop.flavor }}" 9 | key_name: "{{ cloud_config.ssh.keyname }}" 10 | availability_zone: "{{ cloud_config.zone }}" 11 | state: present 12 | wait: true 13 | timeout: 900 14 | validate_certs: false 15 | meta: 16 | Cluster: "{{ cluster_name }}" 17 | Role: "{{ outer_loop.host_group }}" 18 | Group: "{{ cluster_name }}-{{ outer_loop.host_group }}" 19 | loop_control: 20 | loop_var: local_loop 21 | with_sequence: count="{{ outer_loop.count }}" format=-%02x 22 | async: 1000 23 | poll: 0 24 | register: current_nodes_async 25 | 26 | - name: Wait for {{ outer_loop.host_group }} node(s) to be built 27 | async_status: 28 | jid: "{{ local_loop.ansible_job_id }}" 29 | loop_control: 30 | loop_var: local_loop 31 | with_items: "{{ current_nodes_async.results }}" 32 | register: current_nodes 33 | until: current_nodes.finished 34 | retries: 120 35 | -------------------------------------------------------------------------------- /set_cloud.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -z "$CLOUD_TO_USE" ]; then 4 | echo "CLOUD_TO_USE environment variable must be set to one of the following: aws, azure, gce, openstack, static" 5 | exit 1 6 | fi 7 | 8 | cloud_to_use=$(echo "$CLOUD_TO_USE" | tr '[:upper:]' '[:lower:]') 9 | case $cloud_to_use in 10 | aws|amazon) 11 | cloud_to_use=aws 12 | message="Cloud to be used is AWS.\nMake sure you've set the AWS authentication variables." 13 | ;; 14 | azure|microsoft) 15 | cloud_to_use=azure 16 | message="Cloud to be used is Microsoft Azure.\nMake sure $HOME/.azure/credentials exists and it's correct." 17 | ;; 18 | gce|google|gcp) 19 | cloud_to_use=gce 20 | message="Cloud to be used is Google Compute Engine.\nMake sure you've set the GCE authentication variables and JSON credentials file." 21 | ;; 22 | openstack) 23 | message="Cloud to be used is OpenStack.\nMake sure you've sourced the OpenStack RC file." 24 | ;; 25 | static) 26 | message="The static inventory will be used." 27 | ;; 28 | terraform) 29 | message="The terraform inventory will be used." 30 | ;; 31 | *) 32 | message="CLOUD_TO_USE environment variable was set to \"$CLOUD_TO_USE\" but must be set to one of the following: aws, azure, gce, openstack, static, terraform" 33 | echo -e $message 34 | exit 1 35 | ;; 36 | esac 37 | 38 | echo -e "${message}" 39 | 40 | -------------------------------------------------------------------------------- /inventory/gce/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | cloud_config: 3 | name_suffix: '' 4 | region: 'europe-west1' 5 | zone: 'europe-west1-b' 6 | vpc_name: "{{ cluster_name }}-vpc" 7 | subnet_name: "{{ cluster_name }}-subnet" 8 | subnet_cidr: '10.0.10.0/24' 9 | admin_username: google-user 10 | ssh: 11 | privatekey: '~/.ssh/id_rsa' 12 | 13 | firewall_rules: 14 | - name: "{{ cluster_name }}-allow-internal" # default firewall rule - do not change 15 | allowed: 'tcp:0-65535;udp:0-65535' 16 | src_range: "{{ cloud_config.subnet_cidr }}" 17 | - name: "{{ cluster_name }}-ssh-access" 18 | allowed: 'tcp:22' 19 | src_range: '0.0.0.0/0' 20 | - name: "{{ cluster_name }}-ambari-access" 21 | allowed: 'tcp:8080,8443' 22 | src_range: '0.0.0.0/0' 23 | 24 | nodes: 25 | - host_group: "hdp-master" 26 | count: 1 27 | image: centos-7 28 | type: n1-highmem-4 29 | public_ip: true 30 | root_disk: 31 | type: pd-ssd 32 | size: 15 33 | 34 | - host_group: "hdp-slave" 35 | count: 2 36 | image: centos-7 37 | type: n1-standard-2 38 | public_ip: true 39 | root_disk: 40 | type: pd-ssd 41 | size: 15 42 | 43 | - host_group: "hdp-edge" 44 | count: 0 45 | image: centos-7 46 | type: n1-standard-1 47 | public_ip: true 48 | root_disk: 49 | type: pd-standard 50 | size: 20 51 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdf-cluster: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDF cluster with 1 management node and 1 or more worker nodes running NiFi, Kafka and Storm Supervisors. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdf-management" 12 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 13 | services: 14 | - AMBARI_SERVER 15 | - INFRA_SOLR 16 | - NIFI_REGISTRY_MASTER 17 | - NIMBUS 18 | - DRPC_SERVER 19 | - STORM_UI_SERVER 20 | - METRICS_COLLECTOR 21 | - METRICS_GRAFANA 22 | - METRICS_MONITOR 23 | - host_group: "hdf-worker" 24 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 25 | services: 26 | - ZOOKEEPER_SERVER 27 | - NIFI_MASTER 28 | - KAFKA_BROKER 29 | - SUPERVISOR 30 | - METRICS_MONITOR 31 | -------------------------------------------------------------------------------- /playbooks/clouds/build_gce.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Build the Cloud Environment 3 | hosts: localhost 4 | connection: local 5 | gather_facts: False 6 | tasks: 7 | - name: Create the {{ cloud_config.vpc_name }} VPC network 8 | gce_net: 9 | name: "{{ cloud_config.vpc_name }}" 10 | mode: custom 11 | subnet_name: "{{ cloud_config.subnet_name }}" 12 | subnet_region: "{{ cloud_config.region }}" 13 | ipv4_range: "{{ cloud_config.subnet_cidr }}" 14 | 15 | - name: Create firewall rules 16 | gce_net: 17 | name: "{{ cloud_config.vpc_name }}" 18 | fwname: "{{ item.name }}" 19 | allowed: "{{ item.allowed }}" 20 | src_range: "{{ item.src_range }}" 21 | target_tags: "{{ item.target_groups|default(omit) }}" 22 | with_items: "{{ firewall_rules }}" 23 | async: 1000 24 | poll: 0 25 | register: current_firewall_rules_async 26 | 27 | - name: Wait for firewall rules to be applied 28 | async_status: 29 | jid: "{{ local_loop.ansible_job_id }}" 30 | loop_control: 31 | loop_var: local_loop 32 | with_items: "{{ current_firewall_rules_async.results }}" 33 | register: current_firewall_rules 34 | until: current_firewall_rules.finished 35 | retries: 120 36 | 37 | - name: Build GCE Nodes 38 | include_tasks: build_gce_nodes.yml 39 | when: item.count > 0 40 | with_items: "{{ nodes }}" 41 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/redhat-6.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_version: 9.4 3 | postgres_repo_name: "pgdg{{ postgres_version | replace('.','') }}" 4 | postgres_repo_url: "https://download.postgresql.org/pub/repos/yum/{{ postgres_version }}/redhat/rhel-$releasever-$basearch" 5 | postgres_repo_key: "https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG-{{ postgres_version | replace('.','') }}" 6 | postgres_packages: 7 | - postgresql{{ postgres_version | replace('.','') }}-server 8 | postgres_service_name: "postgresql-{{ postgres_version }}" 9 | postgres_initdb_command: "service {{ postgres_service_name }} initdb" 10 | postgres_data_directory: "/var/lib/pgsql/{{ postgres_version }}/data" 11 | postgres_config_file: "{{ postgres_data_directory }}/postgresql.conf" 12 | postgres_hba_file: "{{ postgres_data_directory }}/pg_hba.conf" 13 | 14 | mysql_version: 5.6 15 | mysql_repo_url: "http://repo.mysql.com/yum/mysql-{{ mysql_version }}-community/el/6/$basearch/" 16 | mysql_repo_key: http://repo.mysql.com/RPM-GPG-KEY-mysql 17 | mysql_packages: 18 | - mysql-community-server 19 | - MySQL-python 20 | mysql_service_name: mysqld 21 | mysql_config_file: /etc/my.cnf 22 | 23 | mariadb_version: 10.2 24 | mariadb_repo_url: "http://yum.mariadb.org/{{ mariadb_version }}/centos6-amd64" 25 | mariadb_repo_key: http://yum.mariadb.org/RPM-GPG-KEY-MariaDB 26 | mariadb_packages: 27 | - MariaDB-server 28 | - MySQL-python 29 | mariadb_service_name: mysql 30 | mariadb_config_file: /etc/my.cnf.d/server.cnf 31 | -------------------------------------------------------------------------------- /playbooks/roles/database/tasks/postgres_play.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Generate the {{ db_username }} MD5 password 3 | set_fact: 4 | db_password_md5: "md5{{ (db_password~db_username) | hash('md5') }}" 5 | 6 | - name: Create the {{ db_username }} database user ({{ database }}) 7 | shell: su - postgres -c "psql -c \"CREATE ROLE {{ db_username }} ENCRYPTED PASSWORD '{{ db_password_md5 }}' NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT LOGIN;\"" 8 | args: 9 | warn: false 10 | register: createuser_results 11 | ignore_errors: true 12 | 13 | - name: Fail if the user could not be created ({{ database }}) 14 | fail: 15 | msg: The user could not be created. Error is {{ createuser_results.stderr }}! 16 | when: createuser_results is failed and not createuser_results.stderr is search("role .* already exists") 17 | 18 | - name: Create the {{ db_name }} database ({{ database }}) 19 | shell: su - postgres -c "createdb -O {{ db_username }} {{ db_name }}" 20 | args: 21 | warn: false 22 | register: createdb_results 23 | ignore_errors: true 24 | 25 | - name: Fail if the database could not be created ({{ database }}) 26 | fail: 27 | msg: The database could not be created. Error is {{ createdb_results.stderr }}! 28 | when: createdb_results is failed and not createdb_results.stderr is search("database .* already exists") 29 | 30 | - name: Configure access for the {{ db_username }} user ({{ database }}) 31 | include_tasks: postgres_play_db_access.yml 32 | with_items: "{{ db_client_hosts }}" 33 | -------------------------------------------------------------------------------- /inventory/azure/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | cloud_config: 3 | name_suffix: '' # Set this to a domain name (like '.example.com') when using the Azure internal DNS otherwise the cluster won't use FQDN nodes 4 | location: 'West Europe' 5 | admin_username: azure-user 6 | ssh: 7 | publickey: '~/.ssh/id_rsa.pub' 8 | privatekey: '~/.ssh/id_rsa' 9 | resource_group: Testing 10 | storage_account: 11 | name: teststorage000111111 12 | type: Standard_LRS 13 | network: 14 | name: "{{ cluster_name }}-network" 15 | address: "10.10.0.0/16" 16 | subnet: 17 | name: "{{ cluster_name }}-subnet" 18 | address: "10.10.0.0/24" 19 | security_groups: 20 | - name: ssh_access 21 | port: 22 22 | 23 | nodes: 24 | - host_group: "hdp-master" 25 | count: 1 26 | image: 27 | offer: CentOS 28 | publisher: OpenLogic 29 | sku: '7.4' 30 | version: latest 31 | flavor: Standard_D3_v2 32 | public_ip: true 33 | security_group: ssh_access 34 | 35 | - host_group: "hdp-slave" 36 | count: 2 37 | image: 38 | offer: CentOS 39 | publisher: OpenLogic 40 | sku: '7.4' 41 | version: latest 42 | flavor: Standard_A5 43 | public_ip: true 44 | security_group: ssh_access 45 | 46 | - host_group: "hdp-edge" 47 | count: 0 48 | image: 49 | offer: CentOS 50 | publisher: OpenLogic 51 | sku: '7.4' 52 | version: latest 53 | flavor: Standard_A2 54 | public_ip: true 55 | security_group: ssh_access 56 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdf-cluster-with-streamline: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDF cluster with 1 management node (including Storm Master services, SAM and Schema Registry) and 1 or more worker nodes running NiFi, Kafka and Storm Supervisors. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdf-management" 12 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 13 | services: 14 | - AMBARI_SERVER 15 | - INFRA_SOLR 16 | - NIFI_REGISTRY_MASTER 17 | - NIMBUS 18 | - DRPC_SERVER 19 | - STORM_UI_SERVER 20 | - REGISTRY_SERVER 21 | - STREAMLINE_SERVER 22 | - METRICS_COLLECTOR 23 | - METRICS_GRAFANA 24 | - METRICS_MONITOR 25 | - host_group: "hdf-worker" 26 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 27 | services: 28 | - ZOOKEEPER_SERVER 29 | - NIFI_MASTER 30 | - KAFKA_BROKER 31 | - SUPERVISOR 32 | - METRICS_MONITOR 33 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdf-cluster-with-ranger: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDF cluster with 1 management node and 1 or more worker nodes running NiFi, Kafka and Storm Supervisors. 5 | # It will also install Ranger (both Admin and UserSync) on the management node. 6 | # 7 | ########### 8 | 9 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 10 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 11 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 12 | - host_group: "hdf-management" 13 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 14 | services: 15 | - AMBARI_SERVER 16 | - INFRA_SOLR 17 | - NIFI_REGISTRY_MASTER 18 | - NIMBUS 19 | - DRPC_SERVER 20 | - STORM_UI_SERVER 21 | - RANGER_ADMIN 22 | - RANGER_USERSYNC 23 | - METRICS_COLLECTOR 24 | - METRICS_GRAFANA 25 | - METRICS_MONITOR 26 | - host_group: "hdf-worker" 27 | clients: ['ZOOKEEPER_CLIENT', 'INFRA_SOLR_CLIENT'] 28 | services: 29 | - ZOOKEEPER_SERVER 30 | - NIFI_MASTER 31 | - KAFKA_BROKER 32 | - SUPERVISOR 33 | - METRICS_MONITOR 34 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/redhat-amazon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_version: 9.6 3 | postgres_repo_name: "pgdg{{ postgres_version | replace('.','') }}" 4 | postgres_repo_url: "https://download.postgresql.org/pub/repos/yum/{{ postgres_version }}/redhat/rhel-7-x86_64" 5 | postgres_repo_key: "https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG-{{ postgres_version | replace('.','') }}" 6 | postgres_packages: 7 | - postgresql{{ postgres_version | replace('.','') }}-server 8 | postgres_service_name: "postgresql-{{ postgres_version }}" 9 | postgres_initdb_command: "/usr/pgsql-{{ postgres_version }}/bin/postgresql{{ (postgres_version is version_compare('10', '>=')) | ternary('-','') }}{{ postgres_version | replace('.','') }}-setup initdb" 10 | postgres_data_directory: "/var/lib/pgsql/{{ postgres_version }}/data" 11 | postgres_config_file: "{{ postgres_data_directory }}/postgresql.conf" 12 | postgres_hba_file: "{{ postgres_data_directory }}/pg_hba.conf" 13 | 14 | mysql_version: 5.6 15 | mysql_repo_url: "http://repo.mysql.com/yum/mysql-{{ mysql_version }}-community/el/7/x86_64/" 16 | mysql_repo_key: http://repo.mysql.com/RPM-GPG-KEY-mysql 17 | mysql_packages: 18 | - mysql-community-server 19 | - MySQL-python 20 | mysql_service_name: mysql 21 | mysql_config_file: /etc/my.cnf 22 | 23 | mariadb_version: 10.2 24 | mariadb_repo_url: "http://yum.mariadb.org/{{ mariadb_version }}/centos7-amd64" 25 | mariadb_repo_key: http://yum.mariadb.org/RPM-GPG-KEY-MariaDB 26 | mariadb_packages: 27 | - MariaDB-server 28 | - MySQL-python 29 | mariadb_service_name: mariadb 30 | mariadb_config_file: /etc/my.cnf.d/server.cnf 31 | -------------------------------------------------------------------------------- /playbooks/roles/database/vars/redhat-7.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_version: 9.6 3 | postgres_repo_name: "pgdg{{ postgres_version | replace('.','') }}" 4 | postgres_repo_url: "https://download.postgresql.org/pub/repos/yum/{{ postgres_version }}/redhat/rhel-$releasever-$basearch" 5 | postgres_repo_key: "https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG-{{ postgres_version | replace('.','') }}" 6 | postgres_packages: 7 | - postgresql{{ postgres_version | replace('.','') }}-server 8 | postgres_service_name: "postgresql-{{ postgres_version }}" 9 | postgres_initdb_command: "/usr/pgsql-{{ postgres_version }}/bin/postgresql{{ (postgres_version is version_compare('10', '>=')) | ternary('-','') }}{{ postgres_version | replace('.','') }}-setup initdb" 10 | postgres_data_directory: "/var/lib/pgsql/{{ postgres_version }}/data" 11 | postgres_config_file: "{{ postgres_data_directory }}/postgresql.conf" 12 | postgres_hba_file: "{{ postgres_data_directory }}/pg_hba.conf" 13 | 14 | mysql_version: 5.6 15 | mysql_repo_url: "http://repo.mysql.com/yum/mysql-{{ mysql_version }}-community/el/7/$basearch/" 16 | mysql_repo_key: http://repo.mysql.com/RPM-GPG-KEY-mysql 17 | mysql_packages: 18 | - mysql-community-server 19 | - MySQL-python 20 | mysql_service_name: mysql 21 | mysql_config_file: /etc/my.cnf 22 | 23 | mariadb_version: 10.2 24 | mariadb_repo_url: "http://yum.mariadb.org/{{ mariadb_version }}/centos7-amd64" 25 | mariadb_repo_key: http://yum.mariadb.org/RPM-GPG-KEY-MariaDB 26 | mariadb_packages: 27 | - MariaDB-server 28 | - MySQL-python 29 | mariadb_service_name: mariadb 30 | mariadb_config_file: /etc/my.cnf.d/server.cnf 31 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-part-repository-info.j2: -------------------------------------------------------------------------------- 1 | 2 | {{ hdp_version|replace('-','_') }}_* 3 | 4 | {{ hdp_main_repo_url }} 5 | HDP-{{ hdp_minor_version }} 6 | HDP 7 | true 8 | 9 | {% if install_hdf|default(false) %} 10 | 11 | {{ hdf_main_repo_url }} 12 | HDF-{{ hdf_minor_version }} 13 | HDF 14 | true 15 | 16 | {% endif %} 17 | {% if install_hdpsearch|default(false) and hdpsearch_main_repo_url is defined %} 18 | 19 | {{ hdpsearch_main_repo_url }} 20 | HDP-SOLR-{{ hdpsearch_version }}-{{ hdpsearch_build_number }} 21 | HDP-SOLR 22 | true 23 | 24 | {% endif %} 25 | {% if hdp_version is version_compare('2.6.4.0', '>=') %} 26 | 27 | {{ gpl_repo_url }} 28 | HDP-{{ hdp_minor_version }}-GPL 29 | HDP-GPL 30 | true 31 | 32 | GPL 33 | 34 | 35 | {% endif %} 36 | 37 | {{ utils_repo_url }} 38 | HDP-UTILS-{{ utils_version }} 39 | HDP-UTILS 40 | false 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/repo_version_template.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "operating_systems":[ 3 | { 4 | "OperatingSystems":{ 5 | "ambari_managed_repositories":false, 6 | "os_type":"{{ baseurl_os_family }}" 7 | }, 8 | "repositories":[ 9 | { 10 | "Repositories":{ 11 | "base_url":"{{ hdp_main_repo_url }}", 12 | "repo_id":"HDP-{{ hdp_minor_version }}", 13 | "repo_name":"HDP" 14 | } 15 | }, 16 | {% if install_hdf|default(false) %} 17 | { 18 | "Repositories":{ 19 | "base_url":"{{ hdf_main_repo_url }}", 20 | "repo_id":"HDF-{{ hdf_minor_version }}", 21 | "repo_name":"HDF" 22 | } 23 | }, 24 | {% endif %} 25 | {% if install_hdpsearch|default(false) and hdpsearch_main_repo_url is defined %} 26 | { 27 | "Repositories":{ 28 | "base_url":"{{ hdpsearch_main_repo_url }}", 29 | "repo_id":"HDP-SOLR-{{ hdpsearch_version }}-{{ hdpsearch_build_number }}", 30 | "repo_name":"HDP-SOLR" 31 | } 32 | }, 33 | {% endif %} 34 | { 35 | "Repositories":{ 36 | "base_url":"{{ utils_repo_url }}", 37 | "repo_id":"HDP-UTILS-{{ utils_version }}", 38 | "repo_name":"HDP-UTILS" 39 | } 40 | }, 41 | { 42 | "Repositories":{ 43 | "base_url":"{{ gpl_repo_url }}", 44 | "repo_id":"HDP-{{ hdp_minor_version }}-GPL", 45 | "repo_name":"HDP-GPL" 46 | } 47 | } 48 | ] 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-singlenode-with-nifi: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common HDP services plus NiFi. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - YARN_REGISTRY_DNS 22 | - TIMELINE_READER 23 | - HISTORYSERVER 24 | - HIVE_SERVER 25 | - HIVE_METASTORE 26 | - SPARK2_JOBHISTORYSERVER 27 | - ZEPPELIN_MASTER 28 | - HST_SERVER 29 | - ACTIVITY_ANALYZER 30 | - ACTIVITY_EXPLORER 31 | - HST_AGENT 32 | - AMBARI_SERVER 33 | - INFRA_SOLR 34 | - METRICS_COLLECTOR 35 | - METRICS_GRAFANA 36 | - METRICS_MONITOR 37 | - NIFI_MASTER 38 | - NIFI_REGISTRY_MASTER 39 | - KAFKA_BROKER 40 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/files/blueprint_hdfs_only.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations" : [ 3 | { 4 | "core-site": { 5 | "hadoop.proxyuser.hcat.groups" : "*", 6 | "hadoop.proxyuser.hcat.hosts" : "*", 7 | "hadoop.proxyuser.hue.groups" : "*", 8 | "hadoop.proxyuser.hue.hosts" : "*" 9 | } 10 | }, 11 | { 12 | "hadoop-env" : { 13 | "dtnode_heapsize" : "1024m", 14 | "namenode_heapsize" : "2048m", 15 | "namenode_opt_maxnewsize" : "384m", 16 | "namenode_opt_newsize" : "384m" 17 | } 18 | }, 19 | { 20 | "hdfs-site" : { 21 | "dfs.datanode.data.dir" : "/hadoop/hdfs/data", 22 | "dfs.datanode.failed.volumes.tolerated" : "0", 23 | "dfs.replication" : "3" 24 | } 25 | } 26 | ], 27 | "host_groups" : [ 28 | { 29 | "name" : "hdp-slave", 30 | "configurations" : [ ], 31 | "components" : [ 32 | { "name" : "ZOOKEEPER_CLIENT" }, 33 | { "name" : "HDFS_CLIENT" }, 34 | { "name" : "DATANODE" }, 35 | { "name" : "METRICS_MONITOR" } 36 | ] 37 | }, 38 | { 39 | "name" : "hdp-master", 40 | "configurations" : [ ], 41 | "components" : [ 42 | { "name" : "ZOOKEEPER_CLIENT" }, 43 | { "name" : "HDFS_CLIENT" }, 44 | { "name" : "ZOOKEEPER_SERVER" }, 45 | { "name" : "SECONDARY_NAMENODE" }, 46 | { "name" : "NAMENODE" }, 47 | { "name" : "AMBARI_SERVER" }, 48 | { "name" : "METRICS_COLLECTOR" }, 49 | { "name" : "METRICS_GRAFANA" }, 50 | { "name" : "METRICS_MONITOR" } 51 | ] 52 | } 53 | ], 54 | "Blueprints" : { 55 | "stack_name" : "HDP", 56 | "stack_version" : "2.6" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode-with-nifi: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common HDP services plus NiFi. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - HISTORYSERVER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - SPARK2_JOBHISTORYSERVER 26 | - ZEPPELIN_MASTER 27 | - KNOX_GATEWAY 28 | - HST_SERVER 29 | - ACTIVITY_ANALYZER 30 | - ACTIVITY_EXPLORER 31 | - HST_AGENT 32 | - AMBARI_SERVER 33 | - INFRA_SOLR 34 | - METRICS_COLLECTOR 35 | - METRICS_GRAFANA 36 | - METRICS_MONITOR 37 | - NIFI_MASTER 38 | - NIFI_REGISTRY_MASTER 39 | - KAFKA_BROKER 40 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-3.0.0.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {% include 'vdf-HDP-part-repository-info.j2' %} 31 | 32 | 33 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-3.1.0.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {% include 'vdf-HDP-part-repository-info.j2' %} 31 | 32 | 33 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - HISTORYSERVER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - OOZIE_SERVER 26 | - HBASE_MASTER 27 | - HBASE_REGIONSERVER 28 | - SPARK2_JOBHISTORYSERVER 29 | - ZEPPELIN_MASTER 30 | - KNOX_GATEWAY 31 | - HST_SERVER 32 | - ACTIVITY_ANALYZER 33 | - ACTIVITY_EXPLORER 34 | - HST_AGENT 35 | - AMBARI_SERVER 36 | - INFRA_SOLR 37 | - METRICS_COLLECTOR 38 | - METRICS_GRAFANA 39 | - METRICS_MONITOR 40 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-singlenode: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - YARN_REGISTRY_DNS 22 | - TIMELINE_READER 23 | - HISTORYSERVER 24 | - HIVE_SERVER 25 | - HIVE_METASTORE 26 | - OOZIE_SERVER 27 | - HBASE_MASTER 28 | - HBASE_REGIONSERVER 29 | - SPARK2_JOBHISTORYSERVER 30 | - ZEPPELIN_MASTER 31 | - KNOX_GATEWAY 32 | - HST_SERVER 33 | - ACTIVITY_ANALYZER 34 | - ACTIVITY_EXPLORER 35 | - HST_AGENT 36 | - AMBARI_SERVER 37 | - INFRA_SOLR 38 | - METRICS_COLLECTOR 39 | - METRICS_GRAFANA 40 | - METRICS_MONITOR 41 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-singlenode-no-monitoring: -------------------------------------------------------------------------------- 1 | ########### 2 | # Blueprint derived from "example-hdp3-singlenode" but without components: HST*,METRICS*, OOZIE, HBASE*, KNOX 3 | # 4 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 5 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 6 | # 7 | ########### 8 | 9 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 10 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 11 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 12 | - host_group: "hdp-singlenode" 13 | #clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 14 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'SPARK2_CLIENT'] 15 | services: 16 | - AMBARI_SERVER 17 | - ZOOKEEPER_SERVER 18 | - NAMENODE 19 | - SECONDARY_NAMENODE 20 | - DATANODE 21 | - RESOURCEMANAGER 22 | - NODEMANAGER 23 | - APP_TIMELINE_SERVER 24 | - YARN_REGISTRY_DNS 25 | - TIMELINE_READER 26 | - HISTORYSERVER 27 | - HIVE_SERVER 28 | - HIVE_METASTORE 29 | # - HBASE_MASTER 30 | # - HBASE_REGIONSERVER 31 | - SPARK2_JOBHISTORYSERVER 32 | - ZEPPELIN_MASTER 33 | # - KNOX_GATEWAY 34 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/files/ambari_pubkey.asc: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP PUBLIC KEY BLOCK----- 2 | Version: GnuPG v1 3 | 4 | mQINBE/kz5QBEACz65O0pyfgvYrPxXNBKtMaP5HWX1X6rIReg3K24Awn2ULxvFNb 5 | 7/zepCopt7IbEwqfMSRI3DbdhA1kNbQzRBgyKXQajH3k8Ows7VPit6m6Scwfyyaa 6 | dCIBaZWF8jcRsFjHUF4kgK4uZ3gx27bns8HDCpXUKkuZ08n0ggDiZ7Jx5Lnnfk6i 7 | 4iHWSXPyS6x0XPVyJYsdXRnONOKN/8KJosMQEzEjPx7/y4S4MycshARkq8g6gK+E 8 | +sHtwfFqJDxYQmh7e77Fr3tLquE86VIVdPjjas2z+sttm+TPlfyoAAGKBhSh6OKX 9 | RRhNXngMJcSMYQ5UIFDzc2rOapTSd+zO7tNJZCD64mbKDSr3Bt9uZ+dtEUEdkp2v 10 | 3byuR3E6mEITxEbImtPoVcXCAECqC7SKecT8BTDgt7ZcdOLsnnH5lTadSkYm+aAq 11 | XUEqVBONzxMEGzTzwPy8lHqKuZ1vFgehHRu1lxGpR30cVZLSEXHdIKWB3amt+BlH 12 | 7aF/lGpjmxwmFESxFnDpXG4DrxuIOjicnAWD0nBqVActPaSCq0WCSjh11lodOr/2 13 | 9lbKCgXlh6Ynb84ZCy5T8Crx+j3h5J3InbUyoFj4gQP/3AHbC3Ig3Oq6udZ8LEHW 14 | jOpA2+eY7FbB9FOvK0jNkmvDJ2f8mVBGaBI4OL+jkKe7Qcn/UwLA8foObwARAQAB 15 | tC1KZW5raW5zIChIRFAgQnVpbGRzKSA8amVua2luQGhvcnRvbndvcmtzLmNvbT6J 16 | AjgEEwECACIFAk/kz5QCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELlz 17 | OnoHUTytmekP/0Mlg0VmV033+JzLShpt5uEcIGF1ZCJd2Mwygu1I4JAu/fezntN9 18 | xjPxNvpUTf4OtbrSJKRBZp4awENOXfNz/LCjysBCOP4MoVHS8Vu2oM8pknuNp5aA 19 | lgkYX+z8YFYEdQ3JHtB6ZAt/b0XNQs3BAH71lYHY2yuUbFlB5IHztaEGBYbWuKI2 20 | haI9/31TWEyz4L2Uf/kelHT4vjadJvftQidoRKVFUY21JPgTgwXAtaFOtUNbx2xT 21 | xklPYb8mBqiibNFn3L/hYvlvm+LMhR3LLS3OI5wh5Rr7jWIPY5YBVp0k9OYOiH7i 22 | AWwA786SXa4oqin9IPQUaflpfVNlCjVzJzKRvuFP99R2v1f44IwTJ9QLyYOC46i6 23 | uChEI2rBCI19pKuOH3L9xXxbeEDYDQ7j7eSrl/BrFRYoB8AH2lXmB7IRWRKU3+Ll 24 | FterDA76O7EDPrBKJ9gH6S8sAAE8RiFfNxj4TuYWvzoX9bMe0TLoAZcltAbRuG87 25 | VPzDLVP1lgkBL/BsIywDG423dSZLFm1KF4ptVMGhM+wbEVPsno1AjkOzwmVzVLjZ 26 | 5iZJNVf/ruxY0iHhfYnyxz8xCqMQVv9BJ0XOuzk2xU2hFXD9rKg6UCuU/S25X0f4 27 | WdfF0yTKCqONNpTRqL+/hPP61Tql7zZEBSaCaQEfBnC9qMJaZprK0ccz 28 | =I2D5 29 | -----END PGP PUBLIC KEY BLOCK----- 30 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode-with-accumulo: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services plus Accumulo. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - HISTORYSERVER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - OOZIE_SERVER 26 | - ACCUMULO_MASTER 27 | - ACCUMULO_MONITOR 28 | - ACCUMULO_TRACER 29 | - ACCUMULO_GC 30 | - ACCUMULO_TSERVER 31 | - SPARK2_JOBHISTORYSERVER 32 | - ZEPPELIN_MASTER 33 | - KNOX_GATEWAY 34 | - HST_SERVER 35 | - ACTIVITY_ANALYZER 36 | - ACTIVITY_EXPLORER 37 | - HST_AGENT 38 | - AMBARI_SERVER 39 | - INFRA_SOLR 40 | - METRICS_COLLECTOR 41 | - METRICS_GRAFANA 42 | - METRICS_MONITOR 43 | -------------------------------------------------------------------------------- /molecule/default/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Notes: 3 | # - deployment with blueprint playbooks/group_vars/example-hdp3-singlenode-with-nifi, hung during services startup with docker mem max at 9GB 4 | # 5 | # Important: Any variable overrides must be read not just for 'all' hosts, but also on 'localhost' (used in the set_variables.yml playbooks) 6 | - name: Pre-reqs 7 | hosts: "localhost:all" 8 | vars: 9 | ## Note: As travis provide only 7.5GB VMs, to get a working deployment (including started services started) you have to use a more minimal blueprint: 10 | blueprint_vars_file: example-hdp3-singlenode-no-monitoring 11 | ## Note: All below blueprints require ~12GB dedicated RAM 12 | #blueprint_vars_file: example-hdp3-singlenode 13 | #blueprint_vars_file: example-hdp3-singlenode-with-nifi 14 | ## Note: To install Ranger, you need to set a non-embedded DB 15 | #blueprint_vars_file: example-hdp3-singlenode-with-ranger-atlas 16 | tasks: 17 | - debug: 18 | var: cloud_name 19 | - name: "load blueprint vars for the selected blueprint cfg file {{ blueprint_vars_file }}" 20 | include_vars: ../../playbooks/group_vars/{{ blueprint_vars_file }} 21 | - debug: 22 | var: blueprint_dynamic 23 | verbosity: 1 24 | 25 | - name: Converge 26 | # Note: To override the repo's "playbook group_vars" it's easier to use a "vars:" block here, as inventory vars have lower precedence! 27 | # Note: Another reason for vars *here* is: the 'set_variables.yml' playbook runs on 'localhost', so only sees 'all' group_vars 28 | import_playbook: ../../playbooks/install_cluster.yml 29 | vars: 30 | is_vm_docker_containers: "yes" 31 | cloud_name: static 32 | java: 'openjdk' 33 | ambari_version: '2.7.3.0' 34 | # Currently postgres startup fails in database 'role' with error "Unable to start service postgresql-9.6" 35 | #database: 'postgres' 36 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-singlenode-with-druid: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services plus Druid and Superset. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - YARN_REGISTRY_DNS 22 | - TIMELINE_READER 23 | - HISTORYSERVER 24 | - HIVE_SERVER 25 | - HIVE_METASTORE 26 | - OOZIE_SERVER 27 | - DRUID_COORDINATOR 28 | - DRUID_OVERLORD 29 | - DRUID_ROUTER 30 | - DRUID_BROKER 31 | - DRUID_HISTORICAL 32 | - DRUID_MIDDLEMANAGER 33 | - SUPERSET 34 | - HBASE_MASTER 35 | - HBASE_REGIONSERVER 36 | - SPARK2_JOBHISTORYSERVER 37 | - ZEPPELIN_MASTER 38 | - HST_SERVER 39 | - ACTIVITY_ANALYZER 40 | - ACTIVITY_EXPLORER 41 | - HST_AGENT 42 | - AMBARI_SERVER 43 | - INFRA_SOLR 44 | - METRICS_COLLECTOR 45 | - METRICS_GRAFANA 46 | - METRICS_MONITOR 47 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode-with-storm-kafka: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # It will also install Storm and Kafka services. 6 | # 7 | ########### 8 | 9 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 10 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 11 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 12 | - host_group: "hdp-singlenode" 13 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 14 | services: 15 | - ZOOKEEPER_SERVER 16 | - NAMENODE 17 | - SECONDARY_NAMENODE 18 | - DATANODE 19 | - RESOURCEMANAGER 20 | - NODEMANAGER 21 | - APP_TIMELINE_SERVER 22 | - HISTORYSERVER 23 | - HIVE_SERVER 24 | - HIVE_METASTORE 25 | - WEBHCAT_SERVER 26 | - OOZIE_SERVER 27 | - HBASE_MASTER 28 | - HBASE_REGIONSERVER 29 | - SPARK2_JOBHISTORYSERVER 30 | - ZEPPELIN_MASTER 31 | - NIMBUS 32 | - DRPC_SERVER 33 | - STORM_UI_SERVER 34 | - SUPERVISOR 35 | - KAFKA_BROKER 36 | - RANGER_ADMIN 37 | - RANGER_USERSYNC 38 | - HST_SERVER 39 | - ACTIVITY_ANALYZER 40 | - ACTIVITY_EXPLORER 41 | - HST_AGENT 42 | - AMBARI_SERVER 43 | - INFRA_SOLR 44 | - METRICS_COLLECTOR 45 | - METRICS_GRAFANA 46 | - METRICS_MONITOR 47 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode-with-ranger-atlas: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # It will also install Ranger (both Admin and UserSync). 6 | # 7 | ########### 8 | 9 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 10 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 11 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 12 | - host_group: "hdp-singlenode" 13 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 14 | services: 15 | - ZOOKEEPER_SERVER 16 | - NAMENODE 17 | - SECONDARY_NAMENODE 18 | - DATANODE 19 | - RESOURCEMANAGER 20 | - NODEMANAGER 21 | - APP_TIMELINE_SERVER 22 | - HISTORYSERVER 23 | - HIVE_SERVER 24 | - HIVE_METASTORE 25 | - WEBHCAT_SERVER 26 | - OOZIE_SERVER 27 | - HBASE_MASTER 28 | - HBASE_REGIONSERVER 29 | - SPARK2_JOBHISTORYSERVER 30 | - ZEPPELIN_MASTER 31 | - RANGER_ADMIN 32 | - RANGER_USERSYNC 33 | - RANGER_KMS_SERVER 34 | - ATLAS_SERVER 35 | - KNOX_GATEWAY 36 | - KAFKA_BROKER 37 | - HST_SERVER 38 | - ACTIVITY_ANALYZER 39 | - ACTIVITY_EXPLORER 40 | - HST_AGENT 41 | - AMBARI_SERVER 42 | - INFRA_SOLR 43 | - METRICS_COLLECTOR 44 | - METRICS_GRAFANA 45 | - METRICS_MONITOR 46 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-singlenode-with-ranger-atlas: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # It will also install Ranger (both Admin and UserSync). 6 | # 7 | ########### 8 | 9 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 10 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 11 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 12 | - host_group: "hdp-singlenode" 13 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 14 | services: 15 | - ZOOKEEPER_SERVER 16 | - NAMENODE 17 | - SECONDARY_NAMENODE 18 | - DATANODE 19 | - RESOURCEMANAGER 20 | - NODEMANAGER 21 | - APP_TIMELINE_SERVER 22 | - YARN_REGISTRY_DNS 23 | - TIMELINE_READER 24 | - HISTORYSERVER 25 | - HIVE_SERVER 26 | - HIVE_METASTORE 27 | - OOZIE_SERVER 28 | - HBASE_MASTER 29 | - HBASE_REGIONSERVER 30 | - SPARK2_JOBHISTORYSERVER 31 | - ZEPPELIN_MASTER 32 | - RANGER_ADMIN 33 | - RANGER_USERSYNC 34 | - RANGER_KMS_SERVER 35 | - ATLAS_SERVER 36 | - KNOX_GATEWAY 37 | - KAFKA_BROKER 38 | - HST_SERVER 39 | - ACTIVITY_ANALYZER 40 | - ACTIVITY_EXPLORER 41 | - HST_AGENT 42 | - AMBARI_SERVER 43 | - INFRA_SOLR 44 | - METRICS_COLLECTOR 45 | - METRICS_GRAFANA 46 | - METRICS_MONITOR 47 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.5.0.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | {% include 'vdf-HDP-part-repository-info.j2' %} 35 | 36 | 37 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.5.3.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | {% include 'vdf-HDP-part-repository-info.j2' %} 35 | 36 | 37 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.5.5.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | {% include 'vdf-HDP-part-repository-info.j2' %} 35 | 36 | 37 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.5.6.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | {% include 'vdf-HDP-part-repository-info.j2' %} 35 | 36 | 37 | -------------------------------------------------------------------------------- /playbooks/clouds/build_azure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Build the Cloud Environment 3 | hosts: localhost 4 | connection: local 5 | gather_facts: False 6 | tasks: 7 | - name: Build {{ cloud_config.resource_group }} Resource Group 8 | azure_rm_resourcegroup: 9 | name: "{{ cloud_config.resource_group }}" 10 | location: "{{ cloud_config.location }}" 11 | state: present 12 | force: yes 13 | ignore_errors: yes 14 | 15 | - name: Create {{ cloud_config.storage_account.name }} Storage Account 16 | azure_rm_storageaccount: 17 | resource_group: "{{ cloud_config.resource_group }}" 18 | name: "{{ cloud_config.storage_account.name }}" 19 | account_type: "{{ cloud_config.storage_account.type }}" 20 | 21 | - name: Create {{ cloud_config.network.name }} Virtual Network 22 | azure_rm_virtualnetwork: 23 | resource_group: "{{ cloud_config.resource_group }}" 24 | name: "{{ cloud_config.network.name }}" 25 | address_prefixes_cidr: "{{ cloud_config.network.address }}" 26 | 27 | - name: Create {{ cloud_config.subnet.name }} Subnet 28 | azure_rm_subnet: 29 | resource_group: "{{ cloud_config.resource_group }}" 30 | virtual_network_name: "{{ cloud_config.network.name }}" 31 | name: "{{ cloud_config.subnet.name }}" 32 | address_prefix_cidr : "{{ cloud_config.subnet.address }}" 33 | 34 | - name: Create security groups 35 | azure_rm_securitygroup: 36 | resource_group: "{{ cloud_config.resource_group }}" 37 | name: "{{ item.name }}" 38 | rules: 39 | - name: "AllowPort_{{ item.port }}" 40 | protocol: Tcp 41 | destination_port_range: "{{ item.port }}" 42 | access: Allow 43 | direction: Inbound 44 | priority: 101 45 | with_items: "{{ cloud_config.security_groups }}" 46 | 47 | - name: Build Azure Nodes 48 | include_tasks: build_azure_nodes.yml 49 | when: item.count > 0 50 | with_items: "{{ nodes }}" 51 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/templates/cluster_template.j2: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint" : "{{ blueprint_name }}", 3 | "config_recommendation_strategy" : "{{ config_recommendation_strategy }}", 4 | {% if ambari_version is version_compare('2.6', '>=') %} 5 | "repository_version_id": "{{ vdf_id }}", 6 | {% endif %} 7 | "default_password" : "{{ default_password }}", 8 | {% if security|lower != "none" %} 9 | "credentials" : [ 10 | { 11 | "alias" : "kdc.admin.credential", 12 | {% if security|lower == "mit-kdc" %} 13 | "principal" : "{{ security_options.admin_principal }}/admin@{{ security_options.realm | upper }}", 14 | {% else %} 15 | "principal" : "{{ security_options.admin_principal }}@{{ security_options.realm | upper }}", 16 | {% endif %} 17 | "key" : "{{ security_options.admin_password }}", 18 | "type" : "TEMPORARY" 19 | } 20 | ], 21 | "security" : { 22 | "type" : "KERBEROS" 23 | }, 24 | {% endif %} 25 | {# 26 | The following attempts to find out the Ansible groups that map on the blueprint's host_groups. 27 | When using a dynamic blueprint this should work fine as it knows the host_groups from the 'blueprint_dynamic' variable. 28 | When using a static blueprint, this might sometimes fail, as it simply does the regex check in the blueprint's content. 29 | #} 30 | "host_groups" :[ 31 | {% for group in groups|sort if group not in ['all', 'ungrouped'] and groups[group]|length > 0 and (blueprint_file == 'blueprint_dynamic.j2' and group in blueprint_dynamic|map(attribute='host_group')|list) or (blueprint_file != 'blueprint_dynamic.j2' and blueprint_static['host_groups'] | string is search(group)) %} 32 | 33 | { 34 | "name" : "{{ group }}", 35 | "hosts" : [ 36 | {% for node in groups[group]|sort -%} 37 | 38 | { "fqdn" : "{{ hostvars[node]['ansible_fqdn'] | lower }}", "rack_info": "{{ hostvars[node]['rack'] | default('/default-rack') }}" }{% if not loop.last %},{% endif %} 39 | 40 | {% endfor %} 41 | 42 | ] 43 | }{% if not loop.last %},{% endif %} 44 | {% endfor %} 45 | 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.0.3.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.1.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.2.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.2.14.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.5.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.3.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.6.4.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include 'vdf-HDP-part-release.j2' %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% include 'vdf-HDP-part-repository-info.j2' %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode-with-druid-solr-kafka-ranger: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari, common services and many extras such as Solr, Kafka, Druid, Superset (similar to a sandbox). 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - HISTORYSERVER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - OOZIE_SERVER 26 | - HBASE_MASTER 27 | - HBASE_REGIONSERVER 28 | - SPARK2_JOBHISTORYSERVER 29 | - ZEPPELIN_MASTER 30 | - KAFKA_BROKER 31 | - SOLR_SERVER 32 | - DRUID_COORDINATOR 33 | - DRUID_OVERLORD 34 | - DRUID_ROUTER 35 | - DRUID_BROKER 36 | - DRUID_HISTORICAL 37 | - DRUID_MIDDLEMANAGER 38 | - SUPERSET 39 | - RANGER_ADMIN 40 | - RANGER_USERSYNC 41 | - RANGER_KMS_SERVER 42 | - HST_SERVER 43 | - ACTIVITY_ANALYZER 44 | - ACTIVITY_EXPLORER 45 | - HST_AGENT 46 | - AMBARI_SERVER 47 | - INFRA_SOLR 48 | - METRICS_COLLECTOR 49 | - METRICS_GRAFANA 50 | - METRICS_MONITOR 51 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-agent/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install the ambari-agent package 3 | package: 4 | name: ambari-agent 5 | update_cache: yes 6 | state: latest 7 | notify: Reload systemd 8 | 9 | - name: Set the Ambari Server in the agent configuration 10 | lineinfile: 11 | path: /etc/ambari-agent/conf/ambari-agent.ini 12 | regexp: "^hostname=" 13 | line: "hostname={{ hostvars[groups['ambari-server']|first]['ansible_fqdn'] }}" 14 | state: present 15 | notify: Restart ambari-agent 16 | 17 | ### The following is done on Python >= 2.7 to prevent issues with the latest Python libraries rejecting lower TLS protocols 18 | - name: Configure the Ambari Agents to use TLS 1.2 19 | ini_file: 20 | path: /etc/ambari-agent/conf/ambari-agent.ini 21 | create: no 22 | section: "security" 23 | option: "force_https_protocol" 24 | value: "PROTOCOL_TLSv1_2" 25 | state: present 26 | notify: Restart ambari-agent 27 | when: (ansible_python_version is version_compare('2.7.9', '>=') and ansible_os_family|lower == "debian") or 28 | (ansible_python_version is version_compare('2.7', '>=') and ((ansible_os_family|lower == "redhat" and ansible_distribution_major_version|int >= 7) or 29 | (ansible_os_family|lower == "suse" and ansible_distribution_major_version|int >= 12) or 30 | (ansible_os_family|lower == "redhat" and ansible_distribution|lower == "amazon"))) 31 | 32 | - name: Update ambari-agent log dir 33 | ini_file: 34 | path: /etc/ambari-agent/conf/ambari-agent.ini 35 | create: no 36 | section: "agent" 37 | option: "logdir" 38 | value: "{{ base_log_dir }}/ambari-agent" 39 | state: present 40 | notify: Restart ambari-agent 41 | 42 | - meta: flush_handlers 43 | 44 | - name: Make sure the ambari-agent service is started 45 | service: 46 | name: ambari-agent 47 | state: started 48 | 49 | - name: Make sure the ambari-agent service is enabled 50 | service: 51 | name: ambari-agent 52 | enabled: yes 53 | ignore_errors: true 54 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-singlenode-all-services: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - YARN_REGISTRY_DNS 22 | - TIMELINE_READER 23 | - HISTORYSERVER 24 | - HIVE_SERVER 25 | - HIVE_METASTORE 26 | - OOZIE_SERVER 27 | - HBASE_MASTER 28 | - HBASE_REGIONSERVER 29 | - SPARK2_JOBHISTORYSERVER 30 | - ZEPPELIN_MASTER 31 | - NIMBUS 32 | - DRPC_SERVER 33 | - STORM_UI_SERVER 34 | - SUPERVISOR 35 | - KAFKA_BROKER 36 | - HST_SERVER 37 | - ACTIVITY_ANALYZER 38 | - ACTIVITY_EXPLORER 39 | - HST_AGENT 40 | - AMBARI_SERVER 41 | - INFRA_SOLR 42 | - LOGSEARCH_SERVER 43 | - LOGSEARCH_LOGFEEDER 44 | - METRICS_COLLECTOR 45 | - METRICS_GRAFANA 46 | - METRICS_MONITOR 47 | - RANGER_ADMIN 48 | - RANGER_USERSYNC 49 | - RANGER_KMS_SERVER 50 | - ATLAS_SERVER 51 | - KNOX_GATEWAY 52 | - DRUID_COORDINATOR 53 | - DRUID_OVERLORD 54 | - DRUID_ROUTER 55 | - DRUID_BROKER 56 | - DRUID_HISTORICAL 57 | - DRUID_MIDDLEMANAGER 58 | - SUPERSET -------------------------------------------------------------------------------- /playbooks/roles/ambari-config/templates/vdf-HDP-2.4.3.0.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | STANDARD 5 | HDP-{{ hdp_minor_version }} 6 | {{ hdp_version }} 7 | {{ hdp_build_number_auto|default(hdp_build_number) }} 8 | 2.4.[0-9]+ 9 | http://example.com 10 | HDP-{{ hdp_version }} 11 | {{ hdp_version|replace('-','_') }}_* 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | {% include 'vdf-HDP-part-repository-info.j2' %} 40 | 41 | 42 | -------------------------------------------------------------------------------- /playbooks/roles/common/tasks/java.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Fail if trying to install OpenJDK on SLES 11 3 | fail: 4 | msg: "OpenJDK is not available or supported on SLES 11." 5 | when: java == "openjdk" and ansible_os_family|lower == "suse" and ansible_distribution_major_version|int == 11 6 | 7 | - name: Install OpenJDK 8 | package: 9 | name: "{{ openjdk_package }}" 10 | update_cache: yes 11 | state: present 12 | when: java == "openjdk" 13 | 14 | - block: 15 | - name: Create the {{ oraclejdk_options.base_folder }} directory 16 | file: 17 | path: "{{ oraclejdk_options.base_folder }}" 18 | state: directory 19 | 20 | - name: Extract the Oracle JDK tarball 21 | unarchive: 22 | src: "{{ oraclejdk_options.tarball_location }}" 23 | dest: "{{ oraclejdk_options.base_folder }}" 24 | owner: root 25 | group: root 26 | mode: 0755 27 | remote_src: "{{ oraclejdk_options.remote_files }}" 28 | list_files: yes 29 | keep_newer: yes 30 | register: tarball_content 31 | 32 | - name: Set the new JDK folder name 33 | set_fact: 34 | jdk_folder: "{{ tarball_content.files[0] | regex_replace('/(.*)$','') }}" 35 | 36 | - name: Create the {{ oraclejdk_options.base_folder }}/latest symlink 37 | file: 38 | src: "{{ oraclejdk_options.base_folder }}/{{ jdk_folder }}" 39 | path: "{{ oraclejdk_options.base_folder }}/latest" 40 | state: link 41 | force: yes 42 | 43 | - name: Extract the JCE archive 44 | unarchive: 45 | src: "{{ oraclejdk_options.jce_location }}" 46 | dest: "{{ oraclejdk_options.base_folder }}/latest/jre/lib/security" 47 | owner: root 48 | group: root 49 | mode: 0755 50 | remote_src: "{{ oraclejdk_options.remote_files }}" 51 | 52 | - name: Copy the JCE jar files to {{ oraclejdk_options.base_folder }}/latest/jre/lib/security 53 | copy: 54 | src: "{{ oraclejdk_options.base_folder }}/latest/jre/lib/security/UnlimitedJCEPolicyJDK8/{{ item }}" 55 | dest: "{{ oraclejdk_options.base_folder }}/latest/jre/lib/security" 56 | force: yes 57 | remote_src: yes 58 | with_items: 59 | - local_policy.jar 60 | - US_export_policy.jar 61 | when: java == "oraclejdk" 62 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-singlenode-all-services: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build a single node HDP instance with Ambari and some of the more common services. 5 | # 6 | ########### 7 | 8 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 9 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 10 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 11 | - host_group: "hdp-singlenode" 12 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'SPARK_CLIENT', 'ATLAS_CLIENT'] 13 | services: 14 | - ZOOKEEPER_SERVER 15 | - NAMENODE 16 | - SECONDARY_NAMENODE 17 | - DATANODE 18 | - RESOURCEMANAGER 19 | - NODEMANAGER 20 | - APP_TIMELINE_SERVER 21 | - HISTORYSERVER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - OOZIE_SERVER 26 | - HBASE_MASTER 27 | - HBASE_REGIONSERVER 28 | - SPARK2_JOBHISTORYSERVER 29 | - SPARK_JOBHISTORYSERVER 30 | - ZEPPELIN_MASTER 31 | - NIMBUS 32 | - DRPC_SERVER 33 | - STORM_UI_SERVER 34 | - SUPERVISOR 35 | - KAFKA_BROKER 36 | - HST_SERVER 37 | - ACTIVITY_ANALYZER 38 | - ACTIVITY_EXPLORER 39 | - HST_AGENT 40 | - AMBARI_SERVER 41 | - INFRA_SOLR 42 | - LOGSEARCH_LOGFEEDER 43 | - LOGSEARCH_SERVER 44 | - METRICS_COLLECTOR 45 | - METRICS_GRAFANA 46 | - METRICS_MONITOR 47 | - RANGER_ADMIN 48 | - RANGER_USERSYNC 49 | - RANGER_KMS_SERVER 50 | - ATLAS_SERVER 51 | - KNOX_GATEWAY 52 | - DRUID_COORDINATOR 53 | - DRUID_OVERLORD 54 | - DRUID_ROUTER 55 | - DRUID_BROKER 56 | - DRUID_HISTORICAL 57 | - DRUID_MIDDLEMANAGER 58 | - SUPERSET 59 | 60 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | postgres_port: "5432" 3 | mysql_port: "3306" 4 | 5 | ################################# 6 | ## dynamic blueprint variables ## 7 | ################################# 8 | registry_dns_bind_port: "53" 9 | 10 | 11 | ######################## 12 | ## path configuration ## 13 | ######################## 14 | # Common base dirs 15 | base_log_dir: "/var/log" 16 | base_tmp_dir: "/tmp" 17 | 18 | # Services base dirs 19 | hadoop_base_dir: "/hadoop" 20 | hdfs_data_base_dir: "/hadoop" 21 | kafka_log_base_dir: "/kafka-logs" 22 | nifi_repositories_base_dir: "/var/lib/nifi" 23 | ams_base_dir: "/var/lib" 24 | smartsense_base_dir: "/var/lib/smartsense" 25 | ranger_audit_hdfs_filespool_base_dir: "{{ base_log_dir }}" 26 | ranger_audit_solr_filespool_base_dir: "{{ base_log_dir }}" 27 | 28 | # HDFS main dirs 29 | hdfs_dfs_namenode_checkpoint_dir: "{{ hadoop_base_dir }}/hdfs/namesecondary" 30 | hdfs_dfs_namenode_name_dir: "{{ hadoop_base_dir }}/hdfs/namenode" 31 | hdfs_dfs_journalnode_edits_dir: "{{ hadoop_base_dir }}/hdfs/journalnode" 32 | hdfs_dfs_datanode_data_dir: "{{ hdfs_data_base_dir }}/hdfs/data" 33 | 34 | # YARN main dirs 35 | yarn_nodemanager_local_dirs: "{{ hadoop_base_dir }}/yarn/local" 36 | yarn_nodemanager_log_dirs: "{{ hadoop_base_dir }}/yarn/log" 37 | yarn_timeline_leveldb_dir: "{{ hadoop_base_dir }}/yarn/timeline" 38 | 39 | # NiFi main dirs 40 | nifi_internal_dir: "/var/lib/nifi" 41 | nifi_database_dir: "{{ nifi_internal_dir }}/database_repository" 42 | nifi_content_repository_dir: "{{ nifi_repositories_base_dir }}/content_repository" 43 | nifi_flowfile_repository_dir: "{{ nifi_repositories_base_dir }}/flowfile_repository" 44 | nifi_provenance_repository_dir: "{{ nifi_repositories_base_dir }}/provenance_repository" 45 | 46 | ## NiFi-Registry main dirs 47 | nifi_registry_internal_dir: "/var/lib/nifi-registry" 48 | nifi_registry_database_dir: "{{ nifi_registry_internal_dir }}/database" 49 | 50 | # Other dirs 51 | zookeeper_data_dir: "{{ hadoop_base_dir }}/zookeeper" 52 | oozie_data_dir: "{{ hadoop_base_dir }}/oozie/data" 53 | storm_local_dir: "{{ hadoop_base_dir }}/storm" 54 | infra_solr_datadir: "{{ hadoop_base_dir }}/ambari-infra-solr/data" 55 | atlas_data_dir: "/var/lib/atlas/data" 56 | heap_dump_location: "{{ base_tmp_dir }}" 57 | nfs_file_dump_dir: "{{ base_tmp_dir }}/.hdfs-nfs" 58 | hive_downloaded_resources_dir: "{{ base_tmp_dir }}/hive/${hive.session.id}_resources" 59 | -------------------------------------------------------------------------------- /playbooks/clouds/build_aws.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Build the Cloud Environment 3 | hosts: localhost 4 | connection: local 5 | gather_facts: False 6 | tasks: 7 | - name: Create the VPC 8 | ec2_vpc_net: 9 | region: "{{ cloud_config.region }}" 10 | cidr_block: "{{ cloud_config.vpc_cidr }}" 11 | name: "{{ cloud_config.vpc_name }}" 12 | state: present 13 | register: cluster_vpc 14 | tags: always 15 | 16 | - name: Create the Subnet 17 | ec2_vpc_subnet: 18 | region: "{{ cloud_config.region }}" 19 | az: "{{ cloud_config.zone }}" 20 | vpc_id: "{{ cluster_vpc.vpc.id }}" 21 | cidr: "{{ cloud_config.subnet_cidr }}" 22 | resource_tags: 23 | Name: "{{ cloud_config.subnet_name }}" 24 | state: present 25 | register: cluster_subnet 26 | 27 | - name: Add an Internet Gateway to the VPC 28 | ec2_vpc_igw: 29 | region: "{{ cloud_config.region }}" 30 | vpc_id: "{{ cluster_vpc.vpc.id }}" 31 | state: present 32 | when: cloud_config.internet_gateway 33 | register: cluster_igw 34 | 35 | - name: Set up the public subnet route table for the Internet Gateway 36 | ec2_vpc_route_table: 37 | region: "{{ cloud_config.region }}" 38 | vpc_id: "{{ cluster_vpc.vpc.id }}" 39 | subnets: "{{ cluster_subnet.subnet.id }}" 40 | routes: 41 | - dest: 0.0.0.0/0 42 | gateway_id: "{{ cluster_igw.gateway_id }}" 43 | state: present 44 | when: cluster_igw.gateway_id is defined 45 | 46 | - name: Create Security Groups 47 | ec2_group: 48 | region: "{{ cloud_config.region }}" 49 | name: "{{ item.name }}" 50 | vpc_id: "{{ cluster_vpc.vpc.id }}" 51 | description: "{{ item.description }}" 52 | purge_rules: false 53 | purge_rules_egress: false 54 | rules: "{{ item.rules }}" 55 | with_items: "{{ cloud_config.security_groups }}" 56 | register: cluster_security_groups 57 | tags: security_groups 58 | 59 | - name: Upload the SSH Key 60 | ec2_key: 61 | region: "{{ cloud_config.region }}" 62 | name: "{{ cloud_config.ssh.keyname }}" 63 | key_material: "{{ lookup('file', cloud_config.ssh.publickey) }}" 64 | state: present 65 | wait: yes 66 | 67 | - name: Build AWS Nodes 68 | include_tasks: build_aws_nodes.yml 69 | when: item.count > 0 70 | with_items: "{{ nodes }}" 71 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-blueprint/files/single_node.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations" : [ 3 | { 4 | "core-site": { 5 | "hadoop.proxyuser.hcat.groups" : "*", 6 | "hadoop.proxyuser.hcat.hosts" : "*", 7 | "hadoop.proxyuser.hue.groups" : "*", 8 | "hadoop.proxyuser.hue.hosts" : "*" 9 | } 10 | }, 11 | { 12 | "hadoop-env" : { 13 | "dtnode_heapsize" : "1024m", 14 | "namenode_heapsize" : "2048m", 15 | "namenode_opt_maxnewsize" : "384m", 16 | "namenode_opt_newsize" : "384m" 17 | } 18 | }, 19 | { 20 | "hdfs-site" : { 21 | "dfs.datanode.data.dir" : "/hadoop/hdfs/data", 22 | "dfs.datanode.failed.volumes.tolerated" : "0", 23 | "dfs.replication" : "3" 24 | } 25 | } 26 | ], 27 | "host_groups" : [ 28 | { 29 | "name" : "hdp-singlenode", 30 | "configurations" : [ ], 31 | "components" : [ 32 | { "name" : "ZOOKEEPER_CLIENT" }, 33 | { "name" : "HDFS_CLIENT" }, 34 | { "name" : "PIG" }, 35 | { "name" : "SQOOP" }, 36 | { "name" : "HIVE_CLIENT" }, 37 | { "name" : "YARN_CLIENT" }, 38 | { "name" : "HCAT" }, 39 | { "name" : "MAPREDUCE2_CLIENT" }, 40 | { "name" : "TEZ_CLIENT" }, 41 | { "name" : "SPARK_CLIENT" }, 42 | { "name" : "SPARK2_CLIENT" }, 43 | { "name" : "OOZIE_CLIENT" }, 44 | { "name" : "HBASE_CLIENT" }, 45 | { "name" : "INFRA_SOLR_CLIENT" }, 46 | { "name" : "SLIDER" }, 47 | { "name" : "ZOOKEEPER_SERVER" }, 48 | { "name" : "SECONDARY_NAMENODE" }, 49 | { "name" : "NAMENODE" }, 50 | { "name" : "DATANODE" }, 51 | { "name" : "RESOURCEMANAGER" }, 52 | { "name" : "NODEMANAGER" }, 53 | { "name" : "APP_TIMELINE_SERVER" }, 54 | { "name" : "HISTORYSERVER" }, 55 | { "name" : "MYSQL_SERVER" }, 56 | { "name" : "HIVE_SERVER" }, 57 | { "name" : "HIVE_METASTORE" }, 58 | { "name" : "WEBHCAT_SERVER" }, 59 | { "name" : "OOZIE_SERVER" }, 60 | { "name" : "HBASE_MASTER" }, 61 | { "name" : "HBASE_REGIONSERVER" }, 62 | { "name" : "SPARK_JOBHISTORYSERVER" }, 63 | { "name" : "SPARK2_JOBHISTORYSERVER" }, 64 | { "name" : "ZEPPELIN_MASTER" }, 65 | { "name" : "KAFKA_BROKER" }, 66 | { "name" : "HST_SERVER" }, 67 | { "name" : "HST_AGENT" }, 68 | { "name" : "AMBARI_SERVER" }, 69 | { "name" : "INFRA_SOLR" }, 70 | { "name" : "METRICS_COLLECTOR" }, 71 | { "name" : "METRICS_GRAFANA" }, 72 | { "name" : "METRICS_MONITOR" } 73 | ] 74 | } 75 | ], 76 | "Blueprints" : { 77 | "stack_name" : "HDP", 78 | "stack_version" : "2.6" 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /inventory/aws/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | cloud_config: 3 | region: 'eu-west-1' 4 | zone: 'eu-west-1a' 5 | vpc_name: "{{ cluster_name }}-vpc" 6 | vpc_cidr: '10.0.0.0/16' 7 | subnet_name: '{{ cluster_name }}-subnet' 8 | subnet_cidr: '10.0.10.0/24' 9 | internet_gateway: true # Only set to false if using local repos (node will be without Internet access) 10 | admin_username: ubuntu # Depends on the image, usually can be set to ubuntu, centos or ec2-user 11 | ssh: 12 | keyname: "{{ cluster_name }}-key" 13 | publickey: '~/.ssh/id_rsa.pub' 14 | privatekey: '~/.ssh/id_rsa' 15 | security_groups: 16 | - name: ambari_access 17 | rules: 18 | - {"proto":"tcp", "from_port":"8080", "to_port":"8080", "cidr_ip":"0.0.0.0/0"} 19 | description: "Ambari access - {{ cluster_name }}" 20 | - name: default_cluster_access 21 | rules: 22 | - {"proto":"tcp", "from_port":"22", "to_port":"22", "cidr_ip":"0.0.0.0/0"} 23 | - {"proto":"all", "group_name":"default_cluster_access"} 24 | description: "SSH and inter-nodes access - {{ cluster_name }}" 25 | 26 | nodes: 27 | - host_group: "hdp-master" 28 | count: 1 29 | image: ami-0773391ae604c49a4 # Ubuntu 16.04 AMI in eu-west-1 only (change this ID if using a different region) 30 | type: r4.xlarge 31 | spot: false 32 | spot_price: "0.1" 33 | public_ip: true 34 | security_groups: default_cluster_access,ambari_access 35 | root_volume: 36 | ebs: true # non-EBS root volumes are not supported at the moment 37 | type: gp2 38 | size: 25 39 | 40 | - host_group: "hdp-slave" 41 | count: 3 42 | image: ami-0773391ae604c49a4 # Ubuntu 16.04 AMI in eu-west-1 only (change this ID if using a different region) 43 | type: m5.xlarge 44 | spot: false 45 | spot_price: "0.1" 46 | public_ip: true 47 | security_groups: default_cluster_access 48 | root_volume: 49 | ebs: true # non-EBS root volumes are not supported at the moment 50 | type: gp2 51 | size: 25 52 | 53 | - host_group: "hdp-edge" 54 | count: 0 55 | image: ami-0773391ae604c49a4 # Ubuntu 16.04 AMI in eu-west-1 only (change this ID if using a different region) 56 | type: m4.large 57 | spot: true 58 | spot_price: "0.05" 59 | public_ip: true 60 | security_groups: default_cluster_access 61 | root_volume: 62 | ebs: true # non-EBS root volumes are not supported at the moment 63 | type: gp2 64 | size: 20 65 | -------------------------------------------------------------------------------- /playbooks/clouds/build_gce_nodes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: outer_loop="{{ item }}" 3 | 4 | - name: Create {{ outer_loop.host_group }} root disk(s) 5 | gce_pd: 6 | zone: "{{ cloud_config.zone }}" 7 | name: "{{ cluster_name }}-{{ outer_loop.host_group }}{{ (outer_loop.count > 1) | ternary(local_loop,'') }}" 8 | image: "{{ outer_loop.image }}" 9 | disk_type: "{{ outer_loop.root_disk.type }}" 10 | size_gb: "{{ outer_loop.root_disk.size }}" 11 | mode: READ_WRITE 12 | state: present 13 | loop_control: 14 | loop_var: local_loop 15 | with_sequence: count="{{ outer_loop.count }}" format=-%02x 16 | async: 1000 17 | poll: 0 18 | register: current_disks_async 19 | 20 | - name: Wait for {{ outer_loop.host_group }} root disk(s) to be created 21 | async_status: 22 | jid: "{{ local_loop.ansible_job_id }}" 23 | loop_control: 24 | loop_var: local_loop 25 | with_items: "{{ current_disks_async.results }}" 26 | register: current_disks 27 | until: current_disks.finished 28 | retries: 120 29 | 30 | - name: Create {{ outer_loop.host_group }} node(s) 31 | gce: 32 | zone: "{{ cloud_config.zone }}" 33 | name: "{{ cluster_name }}-{{ outer_loop.host_group }}{{ (outer_loop.count > 1) | ternary(local_loop,'') }}{{ cloud_config.name_suffix|default('') }}" 34 | image: "{{ outer_loop.image }}" 35 | machine_type: "{{ outer_loop.type }}" 36 | network: "{{ cloud_config.vpc_name }}" 37 | subnetwork: "{{ cloud_config.subnet_name }}" 38 | external_ip: "{{ (outer_loop.public_ip|default(false)|bool == true) | ternary('ephemeral','none') }}" 39 | disks: 40 | - "{{ cluster_name }}-{{ outer_loop.host_group }}{{ (outer_loop.count > 1) | ternary(local_loop,'') }}" 41 | tags: 42 | - "cluster-{{ cluster_name }}" 43 | - "role-{{ outer_loop.host_group }}" 44 | - "group-{{ cluster_name }}-{{ outer_loop.host_group }}" 45 | state: present 46 | loop_control: 47 | loop_var: local_loop 48 | with_sequence: count="{{ outer_loop.count }}" format=-%02x 49 | async: 1000 50 | poll: 0 51 | register: current_nodes_async 52 | 53 | - name: Wait for {{ outer_loop.host_group }} node(s) to be built 54 | async_status: 55 | jid: "{{ local_loop.ansible_job_id }}" 56 | loop_control: 57 | loop_var: local_loop 58 | with_items: "{{ current_nodes_async.results }}" 59 | register: current_nodes 60 | until: current_nodes.finished 61 | retries: 120 62 | 63 | - name: Wait for SSH to start 64 | wait_for: 65 | host: "{{ (outer_loop.public_ip|default(false) == true) | ternary(local_loop.instance_data.0.public_ip,local_loop.instance_data.0.private_ip) }}" 66 | port: 22 67 | state: started 68 | delay: 5 69 | timeout: 300 70 | loop_control: 71 | loop_var: local_loop 72 | with_items: '{{ current_nodes.results }}' 73 | -------------------------------------------------------------------------------- /playbooks/roles/ambari-repo/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version }}.yml" 7 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}-{{ ansible_distribution_version }}.yml" 8 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml" 9 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version }}.yml" 10 | - "{{ ansible_os_family|lower }}.yml" 11 | - defaults.yml 12 | paths: 13 | - ../ambari-repo/vars 14 | 15 | - name: Add the Ambari repository (yum) 16 | yum_repository: 17 | name: "ambari" 18 | description: "ambari Version - ambari-{{ ambari_version }}" 19 | baseurl: "{{ ambari_repo_url }}" 20 | file: 'ambari' 21 | state: present 22 | gpgcheck: yes 23 | gpgkey: "{{ ambari_repo_url }}/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins" 24 | when: ansible_os_family|lower == "redhat" 25 | 26 | - name: Add the Ambari repository (zypper) 27 | zypper_repository: 28 | name: "ambari" 29 | description: "ambari Version - ambari-{{ ambari_version }}" 30 | repo: "{{ ambari_repo_url }}" 31 | state: present 32 | disable_gpg_check : yes 33 | runrefresh: yes 34 | when: ansible_os_family|lower == "suse" 35 | 36 | - name: Add Ambari repository key (apt) 37 | apt_key: 38 | data: "{{ lookup('file', ambari_repo_keyfile) }}" 39 | id: "{{ ambari_repo_key }}" 40 | when: ansible_os_family|lower == "debian" 41 | 42 | - name: Add the Ambari repository (apt) 43 | apt_repository: 44 | filename: ambari 45 | repo: "deb {{ ambari_repo_url }} Ambari main" 46 | state: present 47 | update_cache: yes 48 | when: ansible_os_family|lower == "debian" 49 | 50 | - block: 51 | - name: Temporarily add the HDP-UTILS repository (SLES 12) 52 | zypper_repository: 53 | name: temp-HDP-UTILS 54 | description: "HDP-UTILS-1.1.0.21" 55 | repo: "{{ repo_base_url }}/HDP-UTILS-1.1.0.21/repos/sles12" 56 | state: present 57 | disable_gpg_check : yes 58 | runrefresh: yes 59 | 60 | - name: Make sure libsnappy1 is absent (bug in SLES 12 HDP-UTILS repo) 61 | zypper: 62 | name: libsnappy1 63 | state: absent 64 | 65 | - name: Install snappy from HDP-UTILS repo (bug in SLES 12 HDP-UTILS repo) 66 | zypper: 67 | name: snappy-1.0.5-1,snappy-devel-1.0.5-1 68 | state: present 69 | oldpackage: yes 70 | 71 | - name: Remove the HDP-UTILS repository previously added (SLES 12) 72 | zypper_repository: 73 | name: temp-HDP-UTILS 74 | state: absent 75 | runrefresh: yes 76 | when: ansible_os_family|lower == "suse" and ansible_distribution_major_version|int == 12 77 | -------------------------------------------------------------------------------- /playbooks/roles/mit-kdc/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load variables 3 | include_vars: "{{ item }}" 4 | with_first_found: 5 | - files: 6 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version }}.yml" 7 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}-{{ ansible_distribution_version }}.yml" 8 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution|lower }}.yml" 9 | - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version }}.yml" 10 | - "{{ ansible_os_family|lower }}.yml" 11 | - defaults.yml 12 | paths: 13 | - ../mit-kdc/vars 14 | 15 | - name: Install MIT KDC packages 16 | package: 17 | name: "{{ item }}" 18 | update_cache: yes 19 | state: present 20 | with_items: "{{ kdc_packages | join(',') }}" 21 | 22 | - name: Upload the KDC configuration file 23 | template: 24 | src: kdc.conf.j2 25 | dest: "{{ kdc_config_file }}" 26 | mode: 0600 27 | force: yes 28 | notify: Restart krb5 29 | 30 | - name: Create the KDC database for realm {{ security_options.realm | upper }} 31 | shell: "{{ kdc_init_command }} -r {{ security_options.realm | upper }} -P {{ security_options.kdc_master_key }}" 32 | args: 33 | creates: "{{ kdc_database_file }}" 34 | notify: Restart krb5 35 | 36 | - meta: flush_handlers 37 | 38 | - name: Add the admin principal for realm {{ security_options.realm | upper }} 39 | shell: "{{ kadmin_command }} -q \"add_principal -pw {{ security_options.admin_password }} {{ security_options.admin_principal }}/admin@{{ security_options.realm | upper }}\"" 40 | register: addprinc_result 41 | 42 | - name: Fail if the admin principal could not be created 43 | fail: 44 | msg: The admin principal could not be created. Error is {{ addprinc_result.stderr }} 45 | when: addprinc_result is failed or not (addprinc_result.stdout is search("Principal .* created") or addprinc_result.stderr is search("Principal or policy already exists while creating")) 46 | 47 | - name: Set the ACL for the admin user 48 | lineinfile: 49 | path: "{{ kdc_acl_file }}" 50 | regexp: "^\\*/admin@{{ security_options.realm | upper }}.*" 51 | line: "*/admin@{{ security_options.realm | upper }} *" 52 | insertbefore: EOF 53 | state: present 54 | create: yes 55 | mode: 0600 56 | notify: Restart kadmin 57 | 58 | - meta: flush_handlers 59 | 60 | - name: Make sure the kdc service is started 61 | service: 62 | name: "{{ kdc_service_name }}" 63 | state: started 64 | 65 | - name: Make sure the kdc service is enabled 66 | service: 67 | name: "{{ kdc_service_name }}" 68 | enabled: yes 69 | ignore_errors: true 70 | 71 | - name: Make sure the kadmin service is started 72 | service: 73 | name: "{{ kadmin_service_name }}" 74 | state: started 75 | 76 | - name: Make sure the kadmin service is enabled 77 | service: 78 | name: "{{ kadmin_service_name }}" 79 | enabled: yes 80 | ignore_errors: true 81 | -------------------------------------------------------------------------------- /molecule/default/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Molecule config for a single-node HDP cluster 3 | # 4 | # Notes: 5 | # - Linters are set enabled=false because the roles currently contains many lint warnings 6 | # - The 'destroy' action is commented out to avoid accidentally destroying your molecule cluster 7 | # 8 | dependency: 9 | name: galaxy 10 | driver: 11 | name: docker 12 | lint: 13 | name: yamllint 14 | enabled: false 15 | # Using geerlingguy's image that provides centos7 with ansible AND systemd 16 | platforms: 17 | - name: "${MOLECULE_DISTRO:-centos7}-hdp01" 18 | image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" 19 | pre_build_image: true 20 | privileged: true 21 | volume_mounts: 22 | - "/sys/fs/cgroup:/sys/fs/cgroup:rw" 23 | command: "/usr/sbin/init" 24 | groups: 25 | - hadoop-cluster 26 | - hdp-singlenode 27 | networks: 28 | - name: molecule_hdp 29 | network_mode: bridge 30 | # Host networking did not work on my Macbook (it hang at task waiting for the Ambari Agents registration) 31 | #network_mode: host 32 | published_ports: 33 | # Ambari 34 | - 0.0.0.0:8080:8080/tcp 35 | # YARN 36 | - 0.0.0.0:8088:8088/tcp 37 | # HDFS 38 | - 0.0.0.0:50070:50070/tcp 39 | - 0.0.0.0:50075:50075/tcp 40 | # Spark-History 41 | - 0.0.0.0:18081:18081/tcp 42 | # Knox 43 | - 0.0.0.0:8443:8443/tcp 44 | # Zeppelin 45 | - 0.0.0.0:9995:9995/tcp 46 | # Ranger 47 | - 0.0.0.0:6080:6080/tcp 48 | # NiFi 49 | - 0.0.0.0:9090:9090/tcp 50 | # NiFi-Registry 51 | - 0.0.0.0:18080:18080/tcp 52 | #Not needed: 53 | #exposed_ports: 54 | 55 | provisioner: 56 | name: ansible 57 | enabled: false 58 | config_options: 59 | defaults: 60 | gathering: smart 61 | fact_caching: jsonfile 62 | fact_caching_connection: /tmp/facts_cache 63 | ssh_connection: 64 | pipelining: true 65 | options: 66 | diff: true 67 | v: true 68 | inventory: 69 | group_vars: 70 | # Note: Need to use specific inventory group_vars here, since (due to vars precedence) inventory group_vars/all will be overriden by the playbook group_vars/all ! 71 | # However we still need to use 'all' group_vars so that the 'set_variables.yml' playbook (running on host=localhost) can see them! 72 | all: 73 | cluster_name: 'cluster-name that wont be used since the var is overriden anyway by playbook group_vars' 74 | hadoop-cluster: 75 | cluster_name: 'cluster_name_overriden_by_import_playbook_vars' 76 | # ansible-lint config 77 | lint: 78 | name: ansible-lint 79 | enabled: false 80 | 81 | scenario: 82 | name: default 83 | test_sequence: 84 | - lint 85 | # - destroy 86 | - dependency 87 | - syntax 88 | - create 89 | - prepare 90 | - converge 91 | # - idempotence 92 | # - side_effect 93 | - verify 94 | # - destroy 95 | verifier: 96 | name: testinfra 97 | lint: 98 | name: flake8 99 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-ha-2-masters-with-solr: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 2 master services nodes, 1 "special" worker node with Zookeeper and 1 or more normal worker nodes. 5 | # One of the master services nodes is running Ambari and both are have HA NameNode enabled. 6 | # The worker nodes are running Solr from HDP Search. 7 | # 8 | ########### 9 | 10 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 11 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 12 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 13 | - host_group: "hdp-master1" 14 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 15 | services: 16 | - ZOOKEEPER_SERVER 17 | - NAMENODE 18 | - ZKFC 19 | - JOURNALNODE 20 | - RESOURCEMANAGER 21 | - APP_TIMELINE_SERVER 22 | - HISTORYSERVER 23 | - SPARK2_JOBHISTORYSERVER 24 | - HIVE_SERVER 25 | - HIVE_METASTORE 26 | - WEBHCAT_SERVER 27 | - HBASE_MASTER 28 | - HST_SERVER 29 | - ACTIVITY_ANALYZER 30 | - ACTIVITY_EXPLORER 31 | - HST_AGENT 32 | - METRICS_MONITOR 33 | - host_group: "hdp-master2" 34 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 35 | services: 36 | - AMBARI_SERVER 37 | - INFRA_SOLR 38 | - ZOOKEEPER_SERVER 39 | - NAMENODE 40 | - ZKFC 41 | - JOURNALNODE 42 | - HIVE_SERVER 43 | - HIVE_METASTORE 44 | - OOZIE_SERVER 45 | - ACTIVITY_ANALYZER 46 | - KNOX_GATEWAY 47 | - HST_AGENT 48 | - METRICS_COLLECTOR 49 | - METRICS_GRAFANA 50 | - METRICS_MONITOR 51 | - host_group: "hdp-worker-zk" 52 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 53 | services: 54 | - ZOOKEEPER_SERVER 55 | - JOURNALNODE 56 | - DATANODE 57 | - NODEMANAGER 58 | - HBASE_REGIONSERVER 59 | - ACTIVITY_ANALYZER 60 | - HST_AGENT 61 | - METRICS_MONITOR 62 | - SOLR_SERVER 63 | - host_group: "hdp-worker" 64 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 65 | services: 66 | - DATANODE 67 | - NODEMANAGER 68 | - HBASE_REGIONSERVER 69 | - HST_AGENT 70 | - METRICS_MONITOR 71 | - SOLR_SERVER 72 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-ha-3-masters: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 3 master services nodes and 1 or more worker nodes. 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services. 7 | # 8 | ########### 9 | 10 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 11 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 12 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 13 | - host_group: "hdp-masternode-01" 14 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 15 | services: 16 | - ZOOKEEPER_SERVER 17 | - NAMENODE 18 | - ZKFC 19 | - JOURNALNODE 20 | - RESOURCEMANAGER 21 | - HBASE_MASTER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - ACTIVITY_ANALYZER 26 | - HST_AGENT 27 | - METRICS_MONITOR 28 | - LOGSEARCH_LOGFEEDER 29 | - host_group: "hdp-masternode-02" 30 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 31 | services: 32 | - ZOOKEEPER_SERVER 33 | - NAMENODE 34 | - ZKFC 35 | - JOURNALNODE 36 | - RESOURCEMANAGER 37 | - HBASE_MASTER 38 | - HIVE_SERVER 39 | - HIVE_METASTORE 40 | - ACTIVITY_ANALYZER 41 | - HST_AGENT 42 | - METRICS_MONITOR 43 | - LOGSEARCH_LOGFEEDER 44 | - host_group: "hdp-management" 45 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 46 | services: 47 | - ZOOKEEPER_SERVER 48 | - JOURNALNODE 49 | - OOZIE_SERVER 50 | - AMBARI_SERVER 51 | - INFRA_SOLR 52 | - LOGSEARCH_SERVER 53 | - ZEPPELIN_MASTER 54 | - APP_TIMELINE_SERVER 55 | - HISTORYSERVER 56 | - SPARK2_JOBHISTORYSERVER 57 | - KNOX_GATEWAY 58 | - HST_SERVER 59 | - ACTIVITY_ANALYZER 60 | - ACTIVITY_EXPLORER 61 | - HST_AGENT 62 | - METRICS_COLLECTOR 63 | - METRICS_GRAFANA 64 | - METRICS_MONITOR 65 | - LOGSEARCH_LOGFEEDER 66 | - host_group: "hdp-worker" 67 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 68 | services: 69 | - DATANODE 70 | - NODEMANAGER 71 | - HBASE_REGIONSERVER 72 | - HST_AGENT 73 | - METRICS_MONITOR 74 | - LOGSEARCH_LOGFEEDER 75 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-ha-3-masters-with-accumulo: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with Accumulo, made of 3 master services nodes and 1 or more worker nodes. 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services, including the Accumulo masters. 7 | # 8 | ########### 9 | 10 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 11 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 12 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 13 | - host_group: "hdp-masternode-01" 14 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT'] 15 | services: 16 | - ZOOKEEPER_SERVER 17 | - NAMENODE 18 | - ZKFC 19 | - JOURNALNODE 20 | - RESOURCEMANAGER 21 | - HIVE_SERVER 22 | - HIVE_METASTORE 23 | - ACCUMULO_MASTER 24 | - ACCUMULO_MONITOR 25 | - ACCUMULO_TRACER 26 | - ACCUMULO_GC 27 | - KAFKA_BROKER 28 | - ACTIVITY_ANALYZER 29 | - HST_AGENT 30 | - METRICS_MONITOR 31 | - host_group: "hdp-masternode-02" 32 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT'] 33 | services: 34 | - ZOOKEEPER_SERVER 35 | - NAMENODE 36 | - ZKFC 37 | - JOURNALNODE 38 | - RESOURCEMANAGER 39 | - APP_TIMELINE_SERVER 40 | - YARN_REGISTRY_DNS 41 | - TIMELINE_READER 42 | - HIVE_SERVER 43 | - HIVE_METASTORE 44 | - ACCUMULO_MASTER 45 | - ACCUMULO_MONITOR 46 | - ACCUMULO_TRACER 47 | - ACCUMULO_GC 48 | - KAFKA_BROKER 49 | - HISTORYSERVER 50 | - SPARK2_JOBHISTORYSERVER 51 | - ACTIVITY_ANALYZER 52 | - HST_AGENT 53 | - METRICS_MONITOR 54 | - host_group: "hdp-management" 55 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT'] 56 | services: 57 | - ZOOKEEPER_SERVER 58 | - JOURNALNODE 59 | - OOZIE_SERVER 60 | - KNOX_GATEWAY 61 | - ZEPPELIN_MASTER 62 | - AMBARI_SERVER 63 | - INFRA_SOLR 64 | - HST_SERVER 65 | - ACTIVITY_ANALYZER 66 | - ACTIVITY_EXPLORER 67 | - HST_AGENT 68 | - METRICS_COLLECTOR 69 | - METRICS_GRAFANA 70 | - METRICS_MONITOR 71 | - host_group: "hdp-worker" 72 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'ACCUMULO_CLIENT'] 73 | services: 74 | - DATANODE 75 | - NODEMANAGER 76 | - ACCUMULO_TSERVER 77 | - HST_AGENT 78 | - METRICS_MONITOR 79 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-ha-3-masters-with-ranger-atlas: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 3 master services nodes and 1 or more slave nodes. 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services. 7 | # It will also install Ranger (both Admin and UserSync) on the management node. 8 | # 9 | ########### 10 | 11 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 12 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 13 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 14 | - host_group: "hdp-masternode-01" 15 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 16 | services: 17 | - ZOOKEEPER_SERVER 18 | - NAMENODE 19 | - ZKFC 20 | - JOURNALNODE 21 | - RESOURCEMANAGER 22 | - HBASE_MASTER 23 | - HIVE_SERVER 24 | - HIVE_METASTORE 25 | - WEBHCAT_SERVER 26 | - ACTIVITY_ANALYZER 27 | - HST_AGENT 28 | - METRICS_MONITOR 29 | - host_group: "hdp-masternode-02" 30 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 31 | services: 32 | - ZOOKEEPER_SERVER 33 | - NAMENODE 34 | - ZKFC 35 | - JOURNALNODE 36 | - RESOURCEMANAGER 37 | - HBASE_MASTER 38 | - HIVE_SERVER 39 | - HIVE_METASTORE 40 | - ACTIVITY_ANALYZER 41 | - HST_AGENT 42 | - METRICS_MONITOR 43 | - host_group: "hdp-management" 44 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 45 | services: 46 | - ZOOKEEPER_SERVER 47 | - JOURNALNODE 48 | - OOZIE_SERVER 49 | - AMBARI_SERVER 50 | - INFRA_SOLR 51 | - ZEPPELIN_MASTER 52 | - APP_TIMELINE_SERVER 53 | - HISTORYSERVER 54 | - SPARK2_JOBHISTORYSERVER 55 | - RANGER_ADMIN 56 | - RANGER_USERSYNC 57 | - RANGER_KMS_SERVER 58 | - ATLAS_SERVER 59 | - KNOX_GATEWAY 60 | - KAFKA_BROKER 61 | - HST_SERVER 62 | - ACTIVITY_ANALYZER 63 | - ACTIVITY_EXPLORER 64 | - HST_AGENT 65 | - METRICS_COLLECTOR 66 | - METRICS_GRAFANA 67 | - METRICS_MONITOR 68 | - host_group: "hdp-worker" 69 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 70 | services: 71 | - DATANODE 72 | - NODEMANAGER 73 | - HBASE_REGIONSERVER 74 | - HST_AGENT 75 | - METRICS_MONITOR 76 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-ha-3-masters-with-nifi: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 3 master services nodes and 1 or more hdp worker nodes and 1 or more streaming nodes (kafka, nifi). 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services. 7 | # 8 | ########### 9 | 10 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 11 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 12 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 13 | - host_group: "hdp-masternode-01" 14 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 15 | services: 16 | - ZOOKEEPER_SERVER 17 | - NAMENODE 18 | - ZKFC 19 | - JOURNALNODE 20 | - RESOURCEMANAGER 21 | - HBASE_MASTER 22 | - HIVE_SERVER 23 | - HIVE_METASTORE 24 | - WEBHCAT_SERVER 25 | - ACTIVITY_ANALYZER 26 | - HST_AGENT 27 | - METRICS_MONITOR 28 | - host_group: "hdp-masternode-02" 29 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 30 | services: 31 | - ZOOKEEPER_SERVER 32 | - NAMENODE 33 | - ZKFC 34 | - JOURNALNODE 35 | - RESOURCEMANAGER 36 | - HBASE_MASTER 37 | - HIVE_SERVER 38 | - HIVE_METASTORE 39 | - OOZIE_SERVER 40 | - ACTIVITY_ANALYZER 41 | - HST_AGENT 42 | - METRICS_MONITOR 43 | - host_group: "hdp-management" 44 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 45 | services: 46 | - ZOOKEEPER_SERVER 47 | - JOURNALNODE 48 | - AMBARI_SERVER 49 | - INFRA_SOLR 50 | - ZEPPELIN_MASTER 51 | - APP_TIMELINE_SERVER 52 | - HISTORYSERVER 53 | - SPARK2_JOBHISTORYSERVER 54 | - HST_SERVER 55 | - ACTIVITY_ANALYZER 56 | - ACTIVITY_EXPLORER 57 | - HST_AGENT 58 | - METRICS_COLLECTOR 59 | - METRICS_GRAFANA 60 | - METRICS_MONITOR 61 | - NIFI_REGISTRY_MASTER 62 | - host_group: "hdp-worker" 63 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 64 | services: 65 | - DATANODE 66 | - NODEMANAGER 67 | - HBASE_REGIONSERVER 68 | - HST_AGENT 69 | - METRICS_MONITOR 70 | - host_group: "hdf-worker" 71 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 72 | services: 73 | - KAFKA_BROKER 74 | - NIFI_MASTER 75 | - HST_AGENT 76 | - METRICS_MONITOR 77 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-ha-3-masters-with-storm-kafka: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 3 master services nodes and 1 or more slave nodes. 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services. 7 | # It will also install Storm and Kafka services. 8 | # 9 | ########### 10 | 11 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 12 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 13 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 14 | - host_group: "hdp-masternode-01" 15 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 16 | services: 17 | - ZOOKEEPER_SERVER 18 | - NAMENODE 19 | - ZKFC 20 | - JOURNALNODE 21 | - RESOURCEMANAGER 22 | - HBASE_MASTER 23 | - HIVE_SERVER 24 | - HIVE_METASTORE 25 | - NIMBUS 26 | - DRPC_SERVER 27 | - STORM_UI_SERVER 28 | - ACTIVITY_ANALYZER 29 | - HST_AGENT 30 | - METRICS_MONITOR 31 | - host_group: "hdp-masternode-02" 32 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 33 | services: 34 | - ZOOKEEPER_SERVER 35 | - NAMENODE 36 | - ZKFC 37 | - JOURNALNODE 38 | - RESOURCEMANAGER 39 | - HBASE_MASTER 40 | - HIVE_SERVER 41 | - HIVE_METASTORE 42 | - WEBHCAT_SERVER 43 | - ACTIVITY_ANALYZER 44 | - HST_AGENT 45 | - METRICS_MONITOR 46 | - host_group: "hdp-management" 47 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 48 | services: 49 | - ZOOKEEPER_SERVER 50 | - JOURNALNODE 51 | - OOZIE_SERVER 52 | - AMBARI_SERVER 53 | - INFRA_SOLR 54 | - ZEPPELIN_MASTER 55 | - APP_TIMELINE_SERVER 56 | - HISTORYSERVER 57 | - SPARK2_JOBHISTORYSERVER 58 | - HST_SERVER 59 | - ACTIVITY_ANALYZER 60 | - ACTIVITY_EXPLORER 61 | - HST_AGENT 62 | - METRICS_COLLECTOR 63 | - METRICS_GRAFANA 64 | - METRICS_MONITOR 65 | - host_group: "hdp-worker" 66 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 67 | services: 68 | - DATANODE 69 | - NODEMANAGER 70 | - HBASE_REGIONSERVER 71 | - HST_AGENT 72 | - METRICS_MONITOR 73 | - host_group: "hdp-stream" 74 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 75 | services: 76 | - KAFKA_BROKER 77 | - SUPERVISOR 78 | - HST_AGENT 79 | - METRICS_MONITOR 80 | -------------------------------------------------------------------------------- /inventory/gce/gce.ini: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Google Inc. 2 | # 3 | # This file is part of Ansible 4 | # 5 | # Ansible is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # Ansible is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with Ansible. If not, see . 17 | 18 | # The GCE inventory script has the following dependencies: 19 | # 1. A valid Google Cloud Platform account with Google Compute Engine 20 | # enabled. See https://cloud.google.com 21 | # 2. An OAuth2 Service Account flow should be enabled. This will generate 22 | # a private key file that the inventory script will use for API request 23 | # authorization. See https://developers.google.com/accounts/docs/OAuth2 24 | # 3. Convert the private key from PKCS12 to PEM format 25 | # $ openssl pkcs12 -in pkey.pkcs12 -passin pass:notasecret \ 26 | # > -nodes -nocerts | openssl rsa -out pkey.pem 27 | # 4. The libcloud (>=0.13.3) python libray. See http://libcloud.apache.org 28 | # 29 | # (See ansible/test/gce_tests.py comments for full install instructions) 30 | # 31 | # Author: Eric Johnson 32 | # Contributors: John Roach 33 | 34 | [gce] 35 | # GCE Service Account configuration information can be stored in the 36 | # libcloud 'secrets.py' file. Ideally, the 'secrets.py' file will already 37 | # exist in your PYTHONPATH and be picked up automatically with an import 38 | # statement in the inventory script. However, you can specify an absolute 39 | # path to the secrets.py file with 'libcloud_secrets' parameter. 40 | # This option will be deprecated in a future release. 41 | libcloud_secrets = 42 | 43 | # If you are not going to use a 'secrets.py' file, you can set the necessary 44 | # authorization parameters here. 45 | # You can add multiple gce projects to by using a comma separated list. Make 46 | # sure that the service account used has permissions on said projects. 47 | gce_service_account_email_address = 48 | gce_service_account_pem_file_path = 49 | gce_project_id = 50 | gce_zone = 51 | 52 | # Filter inventory based on state. Leave undefined to return instances regardless of state. 53 | # example: Uncomment to only return inventory in the running or provisioning state 54 | #instance_states = RUNNING,PROVISIONING 55 | 56 | # Filter inventory based on instance tags. Leave undefined to return instances regardless of tags. 57 | # example: Uncomment to only return inventory with the http-server or https-server tag 58 | #instance_tags = http-server,https-server 59 | 60 | 61 | [inventory] 62 | # The 'inventory_ip_type' parameter specifies whether 'ansible_ssh_host' should 63 | # contain the instance internal or external address. Values may be either 64 | # 'internal' or 'external'. If 'external' is specified but no external instance 65 | # address exists, the internal address will be used. 66 | # The INVENTORY_IP_TYPE environment variable will override this value. 67 | inventory_ip_type = 68 | 69 | [cache] 70 | # directory in which cache should be created 71 | cache_path = ~/.ansible/tmp 72 | 73 | # The number of seconds a cache file is considered valid. After this many 74 | # seconds, a new API call will be made, and the cache file will be updated. 75 | # To disable the cache, set this value to 0 76 | cache_max_age = 300 77 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp3-ha-3-masters-with-druid-atlas-knox-log: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 3 master services nodes and 1 or more worker nodes. 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services. 7 | # It will also install Druid services and Superset. 8 | # 9 | ########### 10 | 11 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 12 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 13 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 14 | - host_group: "hdp-masternode-01" 15 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 16 | services: 17 | - ZOOKEEPER_SERVER 18 | - NAMENODE 19 | - ZKFC 20 | - JOURNALNODE 21 | - RESOURCEMANAGER 22 | - HBASE_MASTER 23 | - HIVE_SERVER 24 | - HIVE_METASTORE 25 | - DRUID_BROKER 26 | - OOZIE_SERVER 27 | - HISTORYSERVER 28 | - SPARK2_JOBHISTORYSERVER 29 | - KAFKA_BROKER 30 | - ACTIVITY_ANALYZER 31 | - HST_AGENT 32 | - METRICS_MONITOR 33 | - LOGSEARCH_LOGFEEDER 34 | - host_group: "hdp-masternode-02" 35 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 36 | services: 37 | - ZOOKEEPER_SERVER 38 | - NAMENODE 39 | - ZKFC 40 | - JOURNALNODE 41 | - RESOURCEMANAGER 42 | - APP_TIMELINE_SERVER 43 | - YARN_REGISTRY_DNS 44 | - TIMELINE_READER 45 | - HBASE_MASTER 46 | - HIVE_SERVER 47 | - HIVE_METASTORE 48 | - DRUID_BROKER 49 | - KAFKA_BROKER 50 | - ACTIVITY_ANALYZER 51 | - HST_AGENT 52 | - METRICS_MONITOR 53 | - LOGSEARCH_LOGFEEDER 54 | - host_group: "hdp-management" 55 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 56 | services: 57 | - ZOOKEEPER_SERVER 58 | - JOURNALNODE 59 | - DRUID_COORDINATOR 60 | - DRUID_OVERLORD 61 | - DRUID_ROUTER 62 | - SUPERSET 63 | - ATLAS_SERVER 64 | - KNOX_GATEWAY 65 | - ZEPPELIN_MASTER 66 | - AMBARI_SERVER 67 | - INFRA_SOLR 68 | - LOGSEARCH_SERVER 69 | - HST_SERVER 70 | - ACTIVITY_ANALYZER 71 | - ACTIVITY_EXPLORER 72 | - HST_AGENT 73 | - METRICS_COLLECTOR 74 | - METRICS_GRAFANA 75 | - METRICS_MONITOR 76 | - LOGSEARCH_LOGFEEDER 77 | - host_group: "hdp-worker" 78 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'OOZIE_CLIENT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT', 'ATLAS_CLIENT'] 79 | services: 80 | - DATANODE 81 | - NODEMANAGER 82 | - HBASE_REGIONSERVER 83 | - DRUID_HISTORICAL 84 | - DRUID_MIDDLEMANAGER 85 | - HST_AGENT 86 | - METRICS_MONITOR 87 | - LOGSEARCH_LOGFEEDER 88 | -------------------------------------------------------------------------------- /playbooks/group_vars/example-hdp-ha-3-masters-with-nifi-kafka-druid: -------------------------------------------------------------------------------- 1 | ########### 2 | # 3 | # These variables would be used by the 'blueprint_dynamic.j2' Jinja2 template to generate the blueprint JSON. 4 | # This blueprint will build an HDP cluster with 3 master services nodes and 1 or more hdp worker nodes and 1 or more streaming nodes (kafka, nifi). 5 | # One of the master services nodes is running Ambari and other utility services - here called the management node. 6 | # The other two master services are running the main HA Hadoop services. 7 | # It will also install Druid services and Superset. 8 | # 9 | ########### 10 | 11 | blueprint_name: '{{ cluster_name }}_blueprint' # the name of the blueprint as it will be stored in Ambari 12 | blueprint_file: 'blueprint_dynamic.j2' # the blueprint JSON file - 'blueprint_dynamic.j2' is a Jinja2 template that generates the required JSON 13 | blueprint_dynamic: # properties for the dynamic blueprint - these are only used by the 'blueprint_dynamic.j2' template to generate the JSON 14 | - host_group: "hdp-masternode-01" 15 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 16 | services: 17 | - ZOOKEEPER_SERVER 18 | - NAMENODE 19 | - ZKFC 20 | - JOURNALNODE 21 | - RESOURCEMANAGER 22 | - HBASE_MASTER 23 | - HIVE_SERVER 24 | - HIVE_METASTORE 25 | - WEBHCAT_SERVER 26 | - DRUID_BROKER 27 | - ACTIVITY_ANALYZER 28 | - HST_AGENT 29 | - METRICS_MONITOR 30 | - host_group: "hdp-masternode-02" 31 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 32 | services: 33 | - ZOOKEEPER_SERVER 34 | - NAMENODE 35 | - ZKFC 36 | - JOURNALNODE 37 | - RESOURCEMANAGER 38 | - HBASE_MASTER 39 | - HIVE_SERVER 40 | - HIVE_METASTORE 41 | - DRUID_BROKER 42 | - ACTIVITY_ANALYZER 43 | - HST_AGENT 44 | - METRICS_MONITOR 45 | - host_group: "hdp-management" 46 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 47 | services: 48 | - ZOOKEEPER_SERVER 49 | - JOURNALNODE 50 | - DRUID_COORDINATOR 51 | - DRUID_OVERLORD 52 | - DRUID_ROUTER 53 | - SUPERSET 54 | - AMBARI_SERVER 55 | - INFRA_SOLR 56 | - ZEPPELIN_MASTER 57 | - APP_TIMELINE_SERVER 58 | - HISTORYSERVER 59 | - SPARK2_JOBHISTORYSERVER 60 | - HST_SERVER 61 | - ACTIVITY_ANALYZER 62 | - ACTIVITY_EXPLORER 63 | - HST_AGENT 64 | - METRICS_COLLECTOR 65 | - METRICS_GRAFANA 66 | - METRICS_MONITOR 67 | - host_group: "hdp-worker" 68 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 69 | services: 70 | - DATANODE 71 | - NODEMANAGER 72 | - HBASE_REGIONSERVER 73 | - DRUID_HISTORICAL 74 | - HST_AGENT 75 | - METRICS_MONITOR 76 | - host_group: "hdp-stream" 77 | clients: ['ZOOKEEPER_CLIENT', 'HDFS_CLIENT', 'YARN_CLIENT', 'MAPREDUCE2_CLIENT', 'TEZ_CLIENT', 'SLIDER', 'PIG', 'SQOOP', 'HIVE_CLIENT', 'HCAT', 'INFRA_SOLR_CLIENT', 'SPARK2_CLIENT', 'HBASE_CLIENT'] 78 | services: 79 | - KAFKA_BROKER 80 | - NIFI_MASTER 81 | - DRUID_MIDDLEMANAGER 82 | - HST_AGENT 83 | - METRICS_MONITOR 84 | -------------------------------------------------------------------------------- /playbooks/roles/README.md: -------------------------------------------------------------------------------- 1 | ansible-hortonworks roles 2 | ----------- 3 | 4 | The following roles are being used throughout this project: 5 | 6 | 7 | ## common 8 | This role is applied to all nodes and it's responsible with preparing the OS specific prerequisites: 9 | - installs the required OS packages 10 | - installs Java 11 | - starts NTP 12 | - adds all nodes to /etc/hosts (if requested using the `external_dns` variable) 13 | - sets ulimits 14 | - sets swappiness 15 | - stops the firewall service (if requested using the `disable_firewall` variable) 16 | - disables selinux 17 | - disables THP 18 | - configures tuned (if needed) 19 | - creates the http secret key (if Kerberos is enabled) 20 | 21 | ## database 22 | This role is applied to database nodes (by default this is the `ambari-server` group) and it's responsible with installing the database packages and setting up the databases and users as per the [documentation](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.0.0/administering-ambari/content/amb_using_existing_or_installing_default_databases.html). 23 | 24 | The role will only execute if the `database` variable is not `embedded` and `database_options.external_hostname` is empty, as otherwise it's implied the database prerequisites are being taken care of outside these playbooks. 25 | 26 | It also won't do much when using a static blueprint as most helper variables are not being set with a static blueprint: when using a static blueprint, the database prerequisites need to be done before running these playbooks. 27 | 28 | ## krb5-client 29 | This role is applied to all nodes when the `security` variable is set to `mit-kdc`. 30 | 31 | It installs the Kerberos client packages and sets a basic `/etc/krb5.conf` file. 32 | 33 | ## mit-kdc 34 | This role is applied to Kerberos server nodes (by default this is the `ambari-server` group) and it's responsible with installing the MIT KDC packages and setting up the KDC as per the [documentation](https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.2.2/bk_ambari-security/content/optional_install_a_new_mit_kdc.html). 35 | 36 | The role will only execute if `security` is set to `mit-kdc` and `security_options.external_hostname` is empty, as otherwise it's implied there is already a KDC or AD running and configured. 37 | 38 | ## ambari-repo 39 | This role is executed as a [dependency](ambari-agent/meta/main.yml) of the `ambari-agent` and `ambari-server` roles. 40 | 41 | It sets up the Ambari repositories. 42 | 43 | ## ambari-agent 44 | This role is applied to all nodes and it's responsible with installing, configuring and starting the Ambari Agents. 45 | 46 | ## ambari-server 47 | This role is applied to the Ambari Server node (by default this is the `ambari-server` group) and it's responsible with installing, setting up (with the required Java and database options) and starting the Ambari Server. 48 | 49 | ## ambari-config 50 | This role is applied to the Ambari Server node (by default this is the `ambari-server` group) and it's responsible with configuring the Ambari Server: 51 | - sets up several settings (accepts GPL license, home folder creation) 52 | - installs any required mpacks (like HDF, HDP Search) 53 | - registers the cluster repository information to Ambari 54 | - at the end it checks that all Ambari Agents are registered to the Server 55 | 56 | ## ambari-blueprint 57 | This role is applied to the Ambari Server node (by default this is the `ambari-server` group) and it's responsible with applying a blueprint to the Ambari Server. 58 | 59 | If using the `blueprint_dynamic` option, it generates the blueprint using a [Jinja2 template](ambari-blueprint/templates/blueprint_dynamic.j2). 60 | 61 | If using a static blueprint, it uploads that file as a blueprint. It still uses the [template](https://docs.ansible.com/ansible/latest/modules/template_module.html) module so this static blueprint can include [Jinja2 variables](ambari-blueprint/files/blueprint_hdfs_only.j2#L56) if needed. 62 | 63 | Then it generates the [Cluster Creation Template](https://cwiki.apache.org/confluence/display/AMBARI/Blueprints#Blueprints-ClusterCreationTemplateStructure) which is a [Jinja2 template](ambari-blueprint/templates/cluster_template.j2) irrespective of the blueprint type (this can be changed using the `cluster_template_file` variable). 64 | 65 | Depending on the `wait` variable, it will wait for the cluster to be fully deployed by Ambari. 66 | 67 | At the end of this role, the HDP/HDF/HDP Search cluster should be fully installed. 68 | 69 | ## post-install 70 | This role is applied to all nodes and it's responsible with doing any work after the installation of the cluster. 71 | 72 | Currently it fixes a file ownership that can only be done after the cluster is built as it requires the `hdfs` user to be present. 73 | --------------------------------------------------------------------------------