├── .gitignore ├── helper ├── windowsad │ ├── hosts.txt │ ├── createsmbvolume │ │ ├── template.yml │ │ └── createvolume.sh │ ├── create_dns.bat │ └── createkeytabs.bat ├── jumboframe_disable.yml ├── jumboframe_enable.yml ├── debug.yml ├── create-user-chufe.yml ├── site-run-installer-nativesecurity.yml ├── ssl │ ├── generate_root.sh │ ├── cert_tpl │ └── generate_certs.sh ├── haproxy │ ├── mapr-grafana.cfg │ ├── mapr-hue.cfg │ ├── mapr-httpfs.cfg │ ├── mapr-sparkhistory │ ├── mapr-mcs.cfg │ ├── mapr-hive-metastore.cfg │ ├── mapr-hiveserver2.cfg │ ├── mapr-yarn-rm.cfg │ ├── mapr-kafka-rest.cfg │ ├── mapr-drill.cfg │ └── haproxy-global.cfg ├── site-run-installer.yml ├── set-latest-version.yml ├── fetch-mapr-keys.yml ├── set-mapruser-authkeys-sudo.yml ├── ssh_unsecure.yml ├── create-user-ace.yml ├── notes.md └── deploy.sh ├── molecule ├── mapr-core-docker │ ├── plays │ │ ├── files │ │ ├── mapr-repository.yml │ │ ├── mapr-precheck.yml │ │ ├── common.yml │ │ └── prepare-one-node-with-mep.yml │ ├── playbook.yml │ ├── INSTALL.rst │ ├── Dockerfile.j2 │ └── molecule.yml ├── lint │ ├── tests │ │ ├── test_default.pyc │ │ └── test_default.py │ ├── yamllint.conf │ ├── INSTALL.rst │ ├── molecule.yml │ └── Dockerfile.j2 └── common │ ├── playbook.yml │ ├── INSTALL.rst │ ├── tests │ ├── test_ntp.py │ ├── test_rpcbind.py │ ├── test_prereq.py │ ├── test_openjdk18.py │ ├── test_sysctl.py │ ├── test_mapruser.py │ └── test_misc.py │ ├── molecule.yml │ └── Dockerfile.j2 ├── roles ├── ext-zeppelin │ └── templates │ │ ├── bowerrc │ │ └── zeppelin-env.unsecure.sh ├── ext-rstudioserverpro │ ├── templates │ │ ├── rsession.conf.ubuntu │ │ └── rserver.conf.ubuntu │ └── tasks │ │ └── main.yml ├── common-sysctl │ ├── handlers │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── mapr-pig-install │ └── tasks │ │ └── main.yml ├── mapr-flume-install │ └── tasks │ │ └── main.yml ├── mapr-pig-uninstall │ └── tasks │ │ └── main.yml ├── mapr-flume-uninstall │ └── tasks │ │ └── main.yml ├── mapr-sqoop1-install │ └── tasks │ │ └── main.yml ├── mapr-gateway-install │ └── tasks │ │ └── main.yml ├── mapr-kafka-uninstall │ └── tasks │ │ └── main.yml ├── mapr-sqoop1-uninstall │ └── tasks │ │ └── main.yml ├── mapr-fluentd-install │ └── tasks │ │ └── main.yml ├── mapr-grafana-install │ └── tasks │ │ └── main.yml ├── mapr-hbase-cli-uninstall │ └── tasks │ │ └── main.yml ├── mapr-hue-install │ ├── files │ │ └── cx_Oracle-5.3.tar.gz │ └── templates │ │ ├── create_mysql_tables.sh │ │ └── create_oracle_tables.sh ├── mapr-impala-core-uninstall │ └── tasks │ │ └── main.yml ├── ext-ad-createusers │ └── files │ │ └── ad-change-pwd-0.1.jar ├── mapr-collectd-install │ └── tasks │ │ └── main.yml ├── mapr-dsr │ ├── defaults │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── mapr-kafka-ksql-uninstall │ └── tasks │ │ └── main.yml ├── mapr-mapreduce2-install │ └── tasks │ │ └── main.yml ├── mapr-mapreduce2-uninstall │ └── tasks │ │ └── main.yml ├── mapr-mastgateway-install │ └── tasks │ │ └── main.yml ├── mapr-mcs-install │ └── tasks │ │ └── main.yml ├── mapr-nodemanager-install │ └── tasks │ │ └── main.yml ├── mapr-patch-client-install │ └── tasks │ │ └── main.yml ├── mapr-drill-yarn-uninstall │ └── tasks │ │ └── main.yml ├── mapr-kafka-ksql-install │ └── tasks │ │ └── main.yml ├── mapr-sqoop2-client-install │ └── tasks │ │ └── main.yml ├── mapr-configure-zookeeper-restart │ └── tasks │ │ └── main.yml ├── mapr-sqoop2-client-uninstall │ └── tasks │ │ └── main.yml ├── mapr-elasticsearch-install │ └── tasks │ │ └── main.yml ├── mapr-configure-roles │ └── tasks │ │ └── main.yml ├── mapr-resourcemanager-install │ └── tasks │ │ └── main.yml ├── common-oraclejdk18 │ ├── files │ │ └── UnlimitedJCEPolicyJDK8 │ │ │ ├── local_policy.jar │ │ │ └── US_export_policy.jar │ └── tasks │ │ └── main.yml ├── mapr-unregister-node │ └── tasks │ │ └── main.yml ├── mapr-hive-cli-uninstall │ └── tasks │ │ └── main.yml ├── common-rpcbind │ └── tasks │ │ └── main.yml ├── mapr-kafka-install │ └── tasks │ │ └── main.yml ├── mapr-spark-yarn-uninstall │ └── tasks │ │ └── main.yml ├── mapr-trial-license │ └── tasks │ │ └── main.yml ├── mapr-cldb-install │ └── tasks │ │ └── main.yml ├── ui-setup-init │ └── tasks │ │ └── main.yml ├── mapr-client-configure-unsecure │ └── tasks │ │ └── main.yml ├── mapr-fileserver-install │ └── tasks │ │ └── main.yml ├── common-selinux │ └── tasks │ │ └── main.yml ├── mapr-kafka-rest-install │ └── tasks │ │ └── main.yml ├── ext-sssd-ad-setup │ ├── templates │ │ ├── smb.conf │ │ └── sssd.conf │ └── tasks │ │ └── main.yml ├── mapr-data-access-gateway-install │ └── tasks │ │ └── main.yml ├── mapr-historyserver-install │ └── tasks │ │ └── main.yml ├── mapr-drill-yarn-install │ └── tasks │ │ └── main.yml ├── mapr-zookeeper-install │ └── tasks │ │ └── main.yml ├── mapr-client-configure-fetch-keys │ └── tasks │ │ └── main.yml ├── mapr-kafka-connect-jdbc-install │ └── tasks │ │ └── main.yml ├── ext-kerberos5-setup │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── krb5.conf ├── ext-mysql-uninstall │ └── tasks │ │ └── main.yml ├── ext-nscd-setup │ └── tasks │ │ └── main.yml ├── ext-docker-redhat │ ├── defaults │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── mapr-sqoop2-server-vars │ └── tasks │ │ └── main.yml ├── ext-kerberos-keytabs-ad-generate │ ├── templates │ │ └── ktutil.txt │ └── tasks │ │ └── main.yml ├── mapr-cldb-changes │ └── tasks │ │ └── main.yml ├── common-ntp │ └── tasks │ │ └── main.yml ├── mapr-hive-vars │ └── tasks │ │ └── main.yml ├── mapr-hive-cli-install │ └── templates │ │ └── hive-env.sh ├── mapr-httpfs-uninstall │ └── tasks │ │ └── main.yml ├── mapr-kibana-uninstall │ └── tasks │ │ └── main.yml ├── common-openjdk18 │ └── tasks │ │ └── main.yml ├── mapr-fluentd-uninstall │ └── tasks │ │ └── main.yml ├── mapr-grafana-uninstall │ └── tasks │ │ └── main.yml ├── mapr-collectd-uninstall │ └── tasks │ │ └── main.yml ├── mapr-configure-cleanup │ └── tasks │ │ └── main.yml ├── mapr-oozie-uninstall │ └── tasks │ │ └── main.yml ├── mapr-opentsdb-uninstall │ └── tasks │ │ └── main.yml ├── mapr-hive-webhcat-uninstall │ └── tasks │ │ └── main.yml ├── mapr-configure-warden-restart │ └── tasks │ │ └── main.yml ├── mapr-global-vars │ └── tasks │ │ └── main.yml ├── mapr-hive-server2-uninstall │ └── tasks │ │ └── main.yml ├── mapr-patch-posix-client-basic-uninstall │ └── tasks │ │ └── main.yml ├── mapr-configure-vars-cldb │ └── tasks │ │ └── main.yml ├── mapr-drill-standalone-uninstall │ └── tasks │ │ └── main.yml ├── mapr-zookeeper-changes │ └── tasks │ │ └── main.yml ├── mapr-sqoop2-server-uninstall │ └── tasks │ │ └── main.yml ├── mapr-hive-metastore-uninstall │ └── tasks │ │ └── main.yml ├── mapr-impala-server-uninstall │ └── tasks │ │ └── main.yml ├── mapr-nfs-v3-install │ ├── templates │ │ ├── mapr_fstab │ │ └── exports │ └── tasks │ │ └── main.yml ├── mapr-patch-posix-client-platinum-uninstall │ └── tasks │ │ └── main.yml ├── mapr-elasticsearch-uninstall │ └── tasks │ │ └── main.yml ├── mapr-filemigrate-uninstall │ └── tasks │ │ └── main.yml ├── mapr-impala-catalog-uninstall │ └── tasks │ │ └── main.yml ├── mapr-impala-statestore-uninstall │ └── tasks │ │ └── main.yml ├── mapr-patch-posix-client-basic-install │ └── tasks │ │ └── main.yml ├── mapr-spark-historyserver-uninstall │ └── tasks │ │ └── main.yml ├── mapr-configure-vars-hs │ └── tasks │ │ └── main.yml ├── mapr-historyserver-changes │ └── tasks │ │ └── main.yml ├── mapr-log-monitoring-configure-vars │ └── tasks │ │ └── main.yml ├── mapr-patch-posix-client-platinum-install │ └── tasks │ │ └── main.yml ├── mapr-hbase-cli-install │ └── tasks │ │ └── main.yml ├── mapr-configure-env-override │ └── tasks │ │ └── main.yml ├── mapr-posix-client-basic-uninstall │ └── tasks │ │ └── main.yml ├── mapr-unregister-node-vars │ └── tasks │ │ └── main.yml ├── ext-anaconda-python-spark │ └── tasks │ │ └── main.yml ├── common-mapruser │ └── tasks │ │ └── main.yml ├── mapr-posix-client-platinum-uninstall │ └── tasks │ │ └── main.yml ├── mapr-kibana-customssl │ └── tasks │ │ └── main.yml ├── mapr-gateway-uninstall │ └── tasks │ │ └── main.yml ├── mapr-kafka-rest-uninstall │ └── tasks │ │ └── main.yml ├── mapr-grafana-customssl │ └── tasks │ │ └── main.yml ├── mapr-configure-vars-rm │ └── tasks │ │ └── main.yml ├── mapr-mcs-uninstall │ └── tasks │ │ └── main.yml ├── mapr-mastgateway-uninstall │ └── tasks │ │ └── main.yml ├── mapr-nodemanager-uninstall │ └── tasks │ │ └── main.yml ├── mapr-fileserver-uninstall │ └── tasks │ │ └── main.yml ├── ext-anaconda-python │ ├── templates │ │ └── condarc │ └── tasks │ │ └── main.yml ├── ui-setup-nativesecurity-genkeys │ └── tasks │ │ └── main.yml ├── mapr-kafka-connect-jdbc-uninstall │ └── tasks │ │ └── main.yml ├── mapr-core-uninstall │ └── tasks │ │ └── main.yml ├── mapr-data-access-gateway-uninstall │ └── tasks │ │ └── main.yml ├── mapr-spark-thriftserver-uninstall │ └── tasks │ │ └── main.yml ├── mapr-zookeeper-uninstall │ └── tasks │ │ └── main.yml ├── mapr-resourcemanager-uninstall │ └── tasks │ │ └── main.yml ├── mapr-configure-unsecure │ └── tasks │ │ └── main.yml ├── mapr-spark-yarn-shuffle-distribute │ └── tasks │ │ └── main.yml ├── mapr-cldb-uninstall │ └── tasks │ │ └── main.yml ├── mapr-spark-yarn-shuffle-retrieve │ └── tasks │ │ └── main.yml ├── mapr-filemigrate-install │ └── tasks │ │ └── main.yml ├── mapr-configure-nativesecurity-secure-genkeys │ └── tasks │ │ └── main.yml ├── mapr-spark-thriftserver-install │ └── tasks │ │ └── main.yml ├── mapr-configure-security-fetch-keys │ └── tasks │ │ └── main.yml ├── mapr-historyserver-uninstall │ └── tasks │ │ └── main.yml ├── mapr-hue-uninstall │ └── tasks │ │ └── main.yml ├── common-prereq │ └── tasks │ │ └── main.yml ├── common-misc │ └── tasks │ │ └── main.yml ├── mapr-configure-disks │ └── tasks │ │ └── main.yml ├── mapr-client-configure-nativesecurity-secure │ └── tasks │ │ └── main.yml ├── mapr-client-configure-nativesecurity-fetch │ └── tasks │ │ └── main.yml ├── mapr-nfsloopback-install │ └── tasks │ │ └── main.yml ├── mapr-impala-catalog-install │ └── tasks │ │ └── main.yml ├── mapr-kibana-install │ └── tasks │ │ └── main.yml ├── mapr-log-monitoring-configure │ └── tasks │ │ └── main.yml ├── mapr-hbase-thrift-rest-uninstall │ └── tasks │ │ └── main.yml ├── mapr-impala-server-install │ └── tasks │ │ └── main.yml ├── mapr-impala-statestore-install │ └── tasks │ │ └── main.yml ├── mapr-patch-install │ └── tasks │ │ └── main.yml ├── mapr-nfs-v3-uninstall │ └── tasks │ │ └── main.yml ├── mapr-patch-uninstall │ └── tasks │ │ └── main.yml ├── mapr-configure-vars-zk │ └── tasks │ │ └── main.yml ├── mapr-kibana-install-secure │ └── tasks │ │ └── main.yml ├── mapr-configure-kerberos-genkeys │ └── tasks │ │ └── main.yml ├── mapr-client-install │ └── tasks │ │ └── main.yml ├── ext-rlang │ └── tasks │ │ └── main.yml ├── mapr-posix-client-basic-install │ └── tasks │ │ └── main.yml ├── mapr-posix-client-platinum-install │ └── tasks │ │ └── main.yml ├── ext-mysql-install │ └── tasks │ │ └── main.yml ├── mapr-metric-monitoring-configure │ └── tasks │ │ └── main.yml ├── mapr-core-install │ ├── templates │ │ ├── sample-timelineserver.xml │ │ └── core-site.xml │ └── tasks │ │ └── main.yml ├── mapr-hive-mysql │ └── tasks │ │ └── main.yml ├── mapr-fluentd-install-secure │ └── tasks │ │ └── main.yml ├── mapr-hue-mysql │ └── tasks │ │ └── main.yml ├── mapr-impala-core-install │ └── tasks │ │ └── main.yml ├── mapr-impala-vars │ └── tasks │ │ └── main.yml ├── mapr-core-customssl-validate │ └── tasks │ │ └── main.yml ├── mapr-configure-warden-cldb-restart │ └── tasks │ │ └── main.yml ├── mapr-hive-server2-install │ └── tasks │ │ └── main.yml ├── mapr-oozie-mysql │ └── tasks │ │ └── main.yml ├── mapr-opentsdb-install │ └── tasks │ │ └── main.yml ├── mapr-spark-historyserver-install │ └── tasks │ │ └── main.yml ├── mapr-hive-webhcat-install │ └── tasks │ │ └── main.yml ├── mapr-hive-metastore-install │ └── tasks │ │ └── main.yml ├── mapr-spark-yarn-install │ └── templates │ │ ├── hbase-site.xml │ │ └── hive-site.xml ├── mapr-sqoop2-server-install │ └── tasks │ │ └── main.yml ├── ext-kerberos-verify-keytabs │ └── tasks │ │ └── main.yml ├── ui-setup-nativesecurity-secure │ └── tasks │ │ └── main.yml ├── mapr-log-monitoring-configure-genkeys │ └── tasks │ │ └── main.yml └── mapr-elasticsearch-install-secure │ └── tasks │ └── main.yml ├── myhosts ├── rollingnodes_order └── hosts_client ├── sites ├── ext-kerberos-keytabs-verify.yml ├── files │ └── mysql-connector-java-5.1.40-bin.jar ├── ext-kerberos-keytabs-ad-generate.yml ├── ext-zeppelin.yml ├── mapr-include-vars.yml ├── mapr-repository.yml ├── ext-kerberos-createadusers.yml ├── mapr-ssl-validate.yml ├── ext-kerberos-sssd-setup.yml ├── ext-mysql.yml ├── mapr-precheck.yml ├── ext-kerberos-all.yml ├── mapr-mep-pig.yml ├── ext-anaconda.yml ├── mapr-mep-flume.yml ├── ext-rstudio.yml ├── mapr-mep-httpfs.yml ├── mapr-mep-filemigrate.yml ├── mapr-trial-license.yml ├── mapr-mep-data-access-gateway.yml ├── mapr-unprovision.yml ├── mapr-mep-hue.yml ├── common.yml ├── mapr-mep-oozie.yml ├── mapr-mep-drill.yml ├── mapr-mep-hbase.yml ├── mapr-mep-all.yml ├── mapr-mep-sqoop.yml ├── mapr-mep-kafka.yml ├── mapr-mep-spyglass-metric.yml ├── mapr-mep-impala.yml ├── mapr-mep-hive.yml ├── mapr-mep-spark.yml ├── mapr-dsr.yml ├── mapr-ssl.yml └── mapr-mep-spyglass-log.yml ├── host_templates ├── hosts_rstudio ├── hosts_zeppelin ├── hosts_dsr ├── hosts_anaconda ├── hosts_kerberos └── hosts_client ├── site-cluster.yml ├── rolling-upgrade.sh ├── TESTING.md └── .travis.yml /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | *.retry 3 | .DS_Store -------------------------------------------------------------------------------- /helper/windowsad/hosts.txt: -------------------------------------------------------------------------------- 1 | myhostname1 2 | myhostname2 -------------------------------------------------------------------------------- /molecule/mapr-core-docker/plays/files: -------------------------------------------------------------------------------- 1 | ../../../sites/files -------------------------------------------------------------------------------- /roles/ext-zeppelin/templates/bowerrc: -------------------------------------------------------------------------------- 1 | { "allow_root": true } -------------------------------------------------------------------------------- /myhosts/rollingnodes_order: -------------------------------------------------------------------------------- 1 | ip-10-0-0-148.ps.mapr.com 2 | ip-10-0-0-203.ps.mapr.com 3 | -------------------------------------------------------------------------------- /roles/ext-rstudioserverpro/templates/rsession.conf.ubuntu: -------------------------------------------------------------------------------- 1 | # R Session Configuration File 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /roles/common-sysctl/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reload Sysctl 3 | command: sysctl -p 4 | ignore_errors: true 5 | -------------------------------------------------------------------------------- /molecule/lint/tests/test_default.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mapr-emea/mapr-ansible/HEAD/molecule/lint/tests/test_default.pyc -------------------------------------------------------------------------------- /roles/mapr-pig-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Pig 3 | - name: Install mapr-pig 4 | package: name=mapr-pig state=present 5 | -------------------------------------------------------------------------------- /molecule/common/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: ../../sites/mapr-include-vars.yml 3 | - import_playbook: ../../sites/common.yml 4 | -------------------------------------------------------------------------------- /roles/mapr-flume-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Flume 3 | - name: Install mapr-flume 4 | package: name=mapr-flume state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-pig-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Pig 3 | - name: Uninstall mapr-pig 4 | package: name=mapr-pig state=absent 5 | -------------------------------------------------------------------------------- /roles/mapr-flume-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Flume 3 | - name: Uninstall mapr-flume 4 | package: name=mapr-flume state=absent 5 | -------------------------------------------------------------------------------- /roles/mapr-sqoop1-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Sqoop1 3 | - name: Install mapr-sqoop 4 | package: name=mapr-sqoop state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-gateway-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Gateway 3 | - name: Install mapr-gateway 4 | package: name=mapr-gateway state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-kafka-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR Kafka 3 | - name: Uninstall mapr-kafka 4 | package: name=mapr-kafka state=absent 5 | -------------------------------------------------------------------------------- /roles/mapr-sqoop1-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Sqoop1 3 | - name: Uninstall mapr-sqoop 4 | package: name=mapr-sqoop state=absent 5 | -------------------------------------------------------------------------------- /sites/ext-kerberos-keytabs-verify.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Verify keytabs 3 | hosts: ext-kerberos 4 | roles: 5 | - ../roles/ext-kerberos-verify-keytabs -------------------------------------------------------------------------------- /sites/files/mysql-connector-java-5.1.40-bin.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mapr-emea/mapr-ansible/HEAD/sites/files/mysql-connector-java-5.1.40-bin.jar -------------------------------------------------------------------------------- /roles/mapr-fluentd-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR fluentd 3 | - name: Install mapr-fluentd 4 | package: name=mapr-fluentd state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-grafana-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR grafana 3 | - name: Install mapr-grafana 4 | package: name=mapr-grafana state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-hbase-cli-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall HBase CLI 3 | - name: Uninstall mapr-hbase 4 | package: name=mapr-hbase state=absent 5 | -------------------------------------------------------------------------------- /roles/mapr-hue-install/files/cx_Oracle-5.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mapr-emea/mapr-ansible/HEAD/roles/mapr-hue-install/files/cx_Oracle-5.3.tar.gz -------------------------------------------------------------------------------- /roles/mapr-impala-core-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Impala 3 | - name: Uninstall mapr-impala 4 | package: name=mapr-impala state=absent 5 | -------------------------------------------------------------------------------- /sites/ext-kerberos-keytabs-ad-generate.yml: -------------------------------------------------------------------------------- 1 | - name: Create user 2 | hosts: ext-kerberos 3 | roles: 4 | - ../roles/ext-kerberos-keytabs-ad-generate 5 | -------------------------------------------------------------------------------- /molecule/mapr-core-docker/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - import_playbook: plays/prepare-one-node.yml 4 | - import_playbook: plays/prepare-one-node-with-mep.yml 5 | -------------------------------------------------------------------------------- /roles/ext-ad-createusers/files/ad-change-pwd-0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mapr-emea/mapr-ansible/HEAD/roles/ext-ad-createusers/files/ad-change-pwd-0.1.jar -------------------------------------------------------------------------------- /roles/mapr-collectd-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR collectd 3 | 4 | - name: Install mapr-collectd 5 | package: name=mapr-collectd state=present 6 | -------------------------------------------------------------------------------- /roles/mapr-dsr/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | mapr_user_container_password: mapr123 3 | mapr_dsr_docker_image: maprtech/data-science-refinery:v1.3.2_6.1.0_6.1.0_ubuntu16 4 | -------------------------------------------------------------------------------- /roles/mapr-kafka-ksql-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # Uninstall MapR Kafka KSQL 2 | - name: Uninstall mapr-kafka-ksql 3 | package: name=mapr-kafka-ksql state=absent 4 | -------------------------------------------------------------------------------- /roles/mapr-mapreduce2-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapReduce2 3 | - name: Install mapr-mapreduce2 4 | package: name=mapr-mapreduce2 state=present 5 | 6 | -------------------------------------------------------------------------------- /roles/mapr-mapreduce2-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapReduce2 3 | - name: Uninstall mapr-mapreduce2 4 | package: name=mapr-mapreduce2 state=absent 5 | -------------------------------------------------------------------------------- /roles/mapr-mastgateway-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MAST Gateway 3 | - name: Install mapr-mastgateway 4 | package: name=mapr-mastgateway state=present 5 | -------------------------------------------------------------------------------- /sites/ext-zeppelin.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Zeppelin 3 | hosts: ext-zeppelin 4 | roles: 5 | - ../roles/ext-zeppelin 6 | -------------------------------------------------------------------------------- /sites/mapr-include-vars.yml: -------------------------------------------------------------------------------- 1 | - name: Group vars 2 | hosts: all 3 | tasks: 4 | - name: include global config 5 | include_vars: file="../group_vars/all" 6 | -------------------------------------------------------------------------------- /roles/mapr-mcs-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Control System (MCS) 3 | - name: Install mapr-webserver 4 | package: name=mapr-webserver state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-nodemanager-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install YARN Node Manager 3 | - name: Install mapr-nodemanager 4 | package: name=mapr-nodemanager state=present 5 | -------------------------------------------------------------------------------- /roles/mapr-patch-client-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Patches Client 3 | - name: Install mapr-patch-client 4 | package: name=mapr-patch-client state=present -------------------------------------------------------------------------------- /roles/mapr-drill-yarn-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Apache Drill on YARN 3 | - name: Uninstall mapr-drill-yarn 4 | package: name=mapr-drill-yarn state=absent 5 | -------------------------------------------------------------------------------- /roles/mapr-kafka-ksql-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # Install MapR mapr-kafka-ksql 2 | - name: Install mapr-kafka-ksql 3 | package: name=mapr-kafka-ksql state=present 4 | 5 | 6 | -------------------------------------------------------------------------------- /roles/mapr-sqoop2-client-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Sqoop2 Client 3 | - name: Install mapr-sqoop2-client 4 | package: name=mapr-sqoop2-client state=present 5 | -------------------------------------------------------------------------------- /helper/windowsad/createsmbvolume/template.yml: -------------------------------------------------------------------------------- 1 | volumename: testcarsten 2 | readgroups: 3 | - maprgrp 4 | - domain users 5 | writegroups: 6 | - maprgrp 7 | - domain users 8 | -------------------------------------------------------------------------------- /roles/mapr-configure-zookeeper-restart/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Restart Zookeeper 3 | - name: Restart Zookeeper 4 | service: name=mapr-zookeeper state=restarted enabled=yes 5 | 6 | -------------------------------------------------------------------------------- /roles/mapr-sqoop2-client-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Sqoop2 Client 3 | - name: Uninstall mapr-sqoop2-client 4 | package: name=mapr-sqoop2-client state=absent 5 | -------------------------------------------------------------------------------- /sites/mapr-repository.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Setup MapR repositories 3 | hosts: mapr-core 4 | roles: 5 | - ../roles/mapr-configure-repository -------------------------------------------------------------------------------- /host_templates/hosts_rstudio: -------------------------------------------------------------------------------- 1 | # If you do not want to install a component leave the block empty 2 | 3 | # R Lang 4 | [ext-rlang] 5 | 6 | # RStudioServer Pro 7 | [ext-rstudioserverpro] 8 | -------------------------------------------------------------------------------- /roles/mapr-elasticsearch-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR elasticsearch 3 | 4 | - name: Install mapr-elasticsearch 5 | package: name=mapr-elasticsearch state=present 6 | -------------------------------------------------------------------------------- /sites/ext-kerberos-createadusers.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create AD user 3 | hosts: ext-kerberos 4 | roles: 5 | - ../roles/common-openjdk18 6 | - ../roles/ext-ad-createusers 7 | -------------------------------------------------------------------------------- /roles/mapr-configure-roles/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure unsecure cluster 3 | 4 | - name: Configure.sh -R 5 | shell: /opt/mapr/server/configure.sh -R 6 | - pause: seconds=10 7 | 8 | -------------------------------------------------------------------------------- /roles/mapr-resourcemanager-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install YARN Resource Manager 3 | - name: Install mapr-resourcemanager 4 | package: name=mapr-resourcemanager state=present 5 | -------------------------------------------------------------------------------- /roles/common-oraclejdk18/files/UnlimitedJCEPolicyJDK8/local_policy.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mapr-emea/mapr-ansible/HEAD/roles/common-oraclejdk18/files/UnlimitedJCEPolicyJDK8/local_policy.jar -------------------------------------------------------------------------------- /sites/mapr-ssl-validate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: mapr-include-vars.yml 3 | - name: Validate certificates 4 | hosts: mapr-core 5 | roles: 6 | - ../roles/mapr-core-customssl-validate 7 | 8 | -------------------------------------------------------------------------------- /roles/common-oraclejdk18/files/UnlimitedJCEPolicyJDK8/US_export_policy.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mapr-emea/mapr-ansible/HEAD/roles/common-oraclejdk18/files/UnlimitedJCEPolicyJDK8/US_export_policy.jar -------------------------------------------------------------------------------- /roles/mapr-unregister-node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Unregister node 2 | shell: maprcli node remove -nodes {{ unprovision_string }} 3 | environment: 4 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket -------------------------------------------------------------------------------- /helper/jumboframe_disable.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | tasks: 3 | - lineinfile: state=absent dest=/etc/sysconfig/network-scripts/ifcfg-bond0 line="MTU=9216" 4 | - service: name=network state=restarted enabled=yes 5 | -------------------------------------------------------------------------------- /helper/jumboframe_enable.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | tasks: 3 | - lineinfile: state=present dest=/etc/sysconfig/network-scripts/ifcfg-bond0 line="MTU=9216" 4 | - service: name=network state=restarted enabled=yes 5 | -------------------------------------------------------------------------------- /sites/ext-kerberos-sssd-setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Setup Kerberos Client 3 | hosts: all 4 | roles: 5 | - ../roles/ext-kerberos5-setup 6 | - ../roles/ext-nscd-setup 7 | - ../roles/ext-sssd-ad-setup 8 | -------------------------------------------------------------------------------- /helper/debug.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | tasks: 4 | - shell: "/opt/mapr/initscripts/mapr-mfs status" 5 | register: mfs_status 6 | failed_when: mfs_status.stdout.find('is running') == -1 7 | serial: 1 -------------------------------------------------------------------------------- /roles/mapr-hive-cli-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Hive CLI 3 | - name: Uninstall mapr-hive 4 | package: name=mapr-hive state=absent 5 | - name: Uninstall mapr-tez 6 | package: name=mapr-tez state=absent -------------------------------------------------------------------------------- /sites/ext-mysql.yml: -------------------------------------------------------------------------------- 1 | - name: Install MySQL 2 | hosts: ext-mysql 3 | roles: 4 | - ../roles/ext-mysql-install 5 | - name: Uninstall MySQL 6 | hosts: all:!ext-mysql 7 | roles: 8 | - ../roles/ext-mysql-uninstall -------------------------------------------------------------------------------- /host_templates/hosts_zeppelin: -------------------------------------------------------------------------------- 1 | # If you do not want to install a component leave the block empty 2 | 3 | # Zeppelin 4 | [ext-zeppelin] 5 | 10.0.0.132 ansible_user=centos ansible_become=yes ansible_become_method=sudo 6 | 7 | -------------------------------------------------------------------------------- /molecule/mapr-core-docker/plays/mapr-repository.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # - import_playbook: mapr-include-vars.yml 3 | - name: Setup MapR repositories 4 | hosts: mapr-core 5 | roles: 6 | - ../../roles/mapr-configure-repository 7 | -------------------------------------------------------------------------------- /roles/ext-zeppelin/templates/zeppelin-env.unsecure.sh: -------------------------------------------------------------------------------- 1 | export SPARK_HOME={{ spark_path_result.files[0].path }} 2 | export HADOOP_HOME=/opt/mapr/hadoop/hadoop-{{ hadoop_version }} 3 | export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop -------------------------------------------------------------------------------- /roles/common-rpcbind/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install RPC bind 3 | package: name=rpcbind state=present 4 | 5 | # Other actions 6 | - name: Start the RPC bind service 7 | service: name=rpcbind state=started enabled=yes 8 | -------------------------------------------------------------------------------- /roles/mapr-kafka-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR-Kafka 3 | - name: Install mapr-kafka 4 | package: name=mapr-kafka state=present 5 | - name: Install mapr-librdkafka 6 | package: name=mapr-librdkafka state=present 7 | -------------------------------------------------------------------------------- /roles/mapr-spark-yarn-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Spark on YARN 3 | - name: Uninstall mapr-spark 4 | package: name=mapr-spark state=absent 5 | - name: Remove /opt/mapr/spark 6 | file: path=/opt/mapr/spark state=absent -------------------------------------------------------------------------------- /roles/mapr-trial-license/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Copy trial_license.sh 3 | template: src=trial_license.sh dest=/opt/mapr/bin/trial_license.sh mode=0755 4 | - name: Apply Trial license 5 | shell: /opt/mapr/bin/trial_license.sh 6 | -------------------------------------------------------------------------------- /sites/mapr-precheck.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Precheck 3 | hosts: all 4 | roles: 5 | - ../roles/mapr-precheck 6 | - {role: ../roles/mapr-core-customssl-validate, when: https_custom_certs_enabled } 7 | -------------------------------------------------------------------------------- /helper/windowsad/create_dns.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | ECHO Start of Loop 3 | 4 | FOR /L %%i IN (1,1,254) DO ( 5 | dnscmd /RecordAdd ps.mapr.com ip-10-0-0-%%i A 10.0.0.%%i 6 | dnscmd /RecordAdd 0.0.10.in-addr.arpa %%i PTR ip-10-0-0-%%i.ps.mapr.com 7 | ) -------------------------------------------------------------------------------- /sites/ext-kerberos-all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: ext-kerberos-sssd-setup.yml 3 | - import_playbook: ext-kerberos-createadusers.yml 4 | - import_playbook: ext-kerberos-keytabs-ad-generate.yml 5 | - import_playbook: ext-kerberos-keytabs-verify.yml -------------------------------------------------------------------------------- /helper/create-user-chufe.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | tasks: 3 | # - name: Create staff Group 4 | # group: name=staff gid=20 state=present 5 | - name: Create chufe User 6 | user: name=chufe uid=501 password=mapr123 createhome=yes state=present 7 | -------------------------------------------------------------------------------- /molecule/common/INSTALL.rst: -------------------------------------------------------------------------------- 1 | ******* 2 | Install 3 | ******* 4 | 5 | Requirements 6 | ============ 7 | 8 | * Docker Engine 9 | * docker-py 10 | 11 | Install 12 | ======= 13 | 14 | .. code-block:: bash 15 | 16 | $ sudo pip install docker-py 17 | -------------------------------------------------------------------------------- /roles/mapr-cldb-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install CLDB 3 | - name: Install mapr-cldb 4 | package: name=mapr-cldb state=present 5 | register: cldb_result 6 | - set_fact: cldb_haschanged={{ cldb_haschanged | int + 1 }} 7 | when: cldb_result.changed 8 | -------------------------------------------------------------------------------- /sites/mapr-mep-pig.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Pig 3 | hosts: mapr-pig 4 | roles: 5 | - ../roles/mapr-pig-install 6 | - name: Uninstall Pig 7 | hosts: all:!mapr-pig 8 | roles: 9 | - ../roles/mapr-pig-uninstall -------------------------------------------------------------------------------- /roles/ui-setup-init/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Download installer 4 | get_url: url=http://package.mapr.com/releases/installer/mapr-setup.sh force=yes mode=0755 dest=/tmp/mapr-setup.sh 5 | 6 | - name: Start Setup 7 | shell: y '' | /tmp/mapr-setup.sh 8 | -------------------------------------------------------------------------------- /sites/ext-anaconda.yml: -------------------------------------------------------------------------------- 1 | - name: Install Anaconda Python 2 | hosts: ext-anaconda-python 3 | roles: 4 | - ../roles/ext-anaconda-python 5 | - name: Set Python Environment in Spark 6 | hosts: mapr-spark 7 | roles: 8 | - ../roles/ext-anaconda-python-spark -------------------------------------------------------------------------------- /molecule/mapr-core-docker/INSTALL.rst: -------------------------------------------------------------------------------- 1 | ******* 2 | Install 3 | ******* 4 | 5 | Requirements 6 | ============ 7 | 8 | * Docker Engine 9 | * docker-py 10 | 11 | Install 12 | ======= 13 | 14 | .. code-block:: bash 15 | 16 | $ sudo pip install docker-py 17 | -------------------------------------------------------------------------------- /roles/mapr-client-configure-unsecure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Configure.sh for mapr-client - unsecure 4 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -Z {{ zookeeper_string }} -C {{ cldb_string }} -u {{ mapr_user }} -g {{mapr_group}} -unsecure -c" -------------------------------------------------------------------------------- /sites/mapr-mep-flume.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Flume 3 | hosts: mapr-flume 4 | roles: 5 | - ../roles/mapr-flume-install 6 | - name: Uninstall Flume 7 | hosts: all:!mapr-flume 8 | roles: 9 | - ../roles/mapr-flume-uninstall -------------------------------------------------------------------------------- /helper/site-run-installer-nativesecurity.yml: -------------------------------------------------------------------------------- 1 | - name: apply common configuration to all nodes 2 | hosts: master 3 | roles: 4 | - ../roles/ui-setup-nativesecurity-genkeys 5 | - name: apply security 6 | hosts: all 7 | roles: 8 | - ../roles/ui-setup-nativesecurity-secure -------------------------------------------------------------------------------- /roles/mapr-fileserver-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR FileServer (MFS) 3 | - name: Install mapr-fileserver 4 | package: name=mapr-fileserver state=present 5 | register: fileserver_result 6 | - set_fact: mfs_local_changes=1 7 | when: fileserver_result.changed 8 | -------------------------------------------------------------------------------- /sites/ext-rstudio.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install R Lang 3 | hosts: ext-rlang 4 | roles: 5 | - ../roles/ext-rlang 6 | - name: Install R Studio Server Pro 7 | hosts: ext-rstudioserverpro 8 | roles: 9 | - ../roles/ext-rstudioserverpro -------------------------------------------------------------------------------- /sites/mapr-mep-httpfs.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install HttpFS 3 | hosts: mapr-httpfs 4 | roles: 5 | - ../roles/mapr-httpfs-install 6 | - name: Uninstall HttpFS 7 | hosts: all:!mapr-httpfs 8 | roles: 9 | - ../roles/mapr-httpfs-uninstall -------------------------------------------------------------------------------- /molecule/mapr-core-docker/plays/mapr-precheck.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # - import_playbook: mapr-include-vars.yml 3 | - name: Precheck 4 | hosts: all 5 | roles: 6 | - ../../roles/mapr-precheck 7 | - {role: ../../roles/mapr-core-customssl-validate, when: https_custom_certs_enabled} 8 | -------------------------------------------------------------------------------- /roles/common-selinux/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install required Ansible package to handle SElinux 3 | package: name=libselinux-python state=present 4 | when: ansible_os_family != 'Suse' 5 | - name: Disable SElinux 6 | selinux: state=disabled 7 | when: ansible_os_family != 'Suse' 8 | -------------------------------------------------------------------------------- /site-cluster.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: sites/common.yml 2 | - import_playbook: sites/mapr-precheck.yml 3 | - import_playbook: sites/mapr-repository.yml 4 | - import_playbook: sites/mapr-core.yml 5 | - import_playbook: sites/mapr-mep-all.yml 6 | - import_playbook: sites/mapr-unprovision.yml 7 | 8 | -------------------------------------------------------------------------------- /helper/ssl/generate_root.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CERT_PATH=~/testsslkeys 4 | 5 | openssl genrsa -des3 -out $CERT_PATH/rootCA-private.key 4096 6 | openssl req -x509 -new -nodes -key $CERT_PATH/rootCA-private.key -sha256 -days 1024 -subj "/C=DE/O=MapR/OU=MapR PS/CN=ps.mapr.com" -out $CERT_PATH/rootCA.pem -------------------------------------------------------------------------------- /roles/mapr-hue-install/templates/create_mysql_tables.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/mapr/hue/hue-{{ version_output.stdout }} 4 | source ./build/env/bin/activate 5 | #pip uninstall -y MySQL-python 6 | pip install --upgrade --force-reinstall MySQL-python 7 | hue syncdb --noinput 8 | hue migrate 9 | deactivate -------------------------------------------------------------------------------- /roles/mapr-kafka-rest-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR mapr-kafka-rest 3 | - name: Install mapr-kafka-rest 4 | package: name=mapr-kafka-rest state=present 5 | register: kafka_rest_result 6 | - name: Configure.sh -R 7 | shell: /opt/mapr/server/configure.sh -R 8 | when: kafka_rest_result.changed -------------------------------------------------------------------------------- /sites/mapr-mep-filemigrate.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Filemigrate 3 | hosts: mapr-filemigrate 4 | roles: 5 | - ../roles/mapr-filemigrate-install 6 | - name: Uninstall Filemigrate 7 | hosts: all:!mapr-filemigrate 8 | roles: 9 | - ../roles/mapr-filemigrate-uninstall -------------------------------------------------------------------------------- /roles/ext-sssd-ad-setup/templates/smb.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | security = ads 3 | workgroup = {{ mapr_kerberos_ad_workgroup }} 4 | realm = {{ mapr_kerberos_realm }} 5 | kerberos method = secrets and keytab 6 | passdb backend = tdbsam 7 | unix extensions = yes 8 | client signing = yes 9 | client use spnego = yes 10 | -------------------------------------------------------------------------------- /sites/mapr-trial-license.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: mapr-include-vars.yml 3 | - name: Setup Trial License 4 | hosts: mapr-cldb[0] 5 | roles: 6 | - ../roles/mapr-trial-license 7 | - name: Restarting mapr-warden on all nodes 8 | hosts: mapr-core 9 | roles: 10 | - ../roles/mapr-configure-warden-restart -------------------------------------------------------------------------------- /helper/haproxy/mapr-grafana.cfg: -------------------------------------------------------------------------------- 1 | # Grafana 2 | frontend mapr-grafana 3 | description "Grafana" 4 | bind *:3000 5 | mode tcp 6 | default_backend mapr-grafana 7 | 8 | backend mapr-grafana 9 | description "Grafana" 10 | mode tcp 11 | balance first 12 | server mapr-grafana1 10.0.0.63:3000 check -------------------------------------------------------------------------------- /roles/mapr-data-access-gateway-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR mapr-data-access-gateway 3 | - name: Install mapr-data-access-gateway 4 | package: name=mapr-data-access-gateway 5 | register: gateway_result 6 | - name: Configure.sh -R 7 | shell: /opt/mapr/server/configure.sh -R 8 | when: gateway_result.changed -------------------------------------------------------------------------------- /roles/mapr-historyserver-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install History Server 3 | - name: Install mapr-historyserver 4 | package: name=mapr-historyserver state=present 5 | register: historyserver_result 6 | - set_fact: historyserver_haschanged={{ historyserver_haschanged | int + 1 }} 7 | when: historyserver_result.changed -------------------------------------------------------------------------------- /helper/site-run-installer.yml: -------------------------------------------------------------------------------- 1 | - name: Apply common configuration to all nodes 2 | hosts: all 3 | roles: 4 | - ../roles/common-oracle-jdk 5 | - ../roles/common-mapruser 6 | - ../roles/common-rpcbind 7 | - ../roles/common-ntp 8 | - name: Start setup 9 | hosts: master 10 | roles: 11 | - ../roles/ui-setup-init -------------------------------------------------------------------------------- /roles/mapr-drill-yarn-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Apache Drill on YARN 3 | 4 | - name: Install mapr-drill-yarn 5 | package: name=mapr-drill-yarn state=present 6 | 7 | # Other actions 8 | - name: Set owner mapr-system user 9 | file: path=/opt/mapr/drill owner="{{ mapr_user }}" group="{{ mapr_group }}" state=directory recurse=yes -------------------------------------------------------------------------------- /roles/mapr-zookeeper-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Zookeeper 3 | - name: Install mapr-zookeeper 4 | package: name=mapr-zookeeper state=present 5 | register: zookeeper_result 6 | #- debug: msg={{ zookeeper_result.changed }} 7 | - set_fact: zookeeper_haschanged={{ zookeeper_haschanged | int + 1 }} 8 | when: zookeeper_result.changed -------------------------------------------------------------------------------- /molecule/common/tests/test_ntp.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_ntp_installed(host): 10 | p = host.package('ntp') 11 | 12 | assert p.is_installed 13 | -------------------------------------------------------------------------------- /roles/mapr-client-configure-fetch-keys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # key store is required for Spark UI 2 | - name: Copy keys to local 3 | fetch: src=/opt/mapr/conf/{{ item }} dest="{{ local_tmp_path }}/{{ item }}" flat=yes 4 | with_items: 5 | - ssl_truststore 6 | - mapruserticket 7 | - ssl_keystore 8 | when: inventory_hostname == groups["mapr-cldb"][0] 9 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-hue.cfg: -------------------------------------------------------------------------------- 1 | # Hue 2 | frontend mapr-hue 3 | description "Hue" 4 | bind *:8888 5 | mode tcp 6 | default_backend mapr-hue 7 | 8 | backend mapr-hue 9 | description "Hue" 10 | mode tcp 11 | balance first 12 | server mapr-hue1 10.0.0.232:8888 check 13 | # server mapr-httpfs1 10.0.0.110:14000 check backup -------------------------------------------------------------------------------- /molecule/common/tests/test_rpcbind.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_rpcbind_running(host): 10 | s = host.service('rpcbind') 11 | 12 | assert s.is_running 13 | -------------------------------------------------------------------------------- /roles/mapr-kafka-connect-jdbc-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR mapr-kafka-connect-jdbc 3 | - name: Install mapr-kafka-connect-jdbc 4 | package: name=mapr-kafka-connect-jdbc state=present 5 | register: kafka_connect_jdbc_result 6 | - name: Configure.sh -R 7 | shell: /opt/mapr/server/configure.sh -R 8 | when: kafka_connect_jdbc_result.changed -------------------------------------------------------------------------------- /molecule/lint/yamllint.conf: -------------------------------------------------------------------------------- 1 | extends: default 2 | 3 | rules: 4 | line-length: disable 5 | document-start: disable 6 | new-line-at-end-of-file: disable 7 | truthy: disable 8 | comments: disable 9 | empty-lines: disable 10 | indentation: disable 11 | braces: disable 12 | brackets: disable 13 | colons: disable 14 | new-lines: disable 15 | -------------------------------------------------------------------------------- /roles/mapr-hue-install/templates/create_oracle_tables.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/mapr/hue/hue-{{ version_output.stdout }} 4 | source ./build/env/bin/activate 5 | export LD_LIBRARY_PATH="{{ oracle_version_arch_path.files[0].path }}/lib/libclntsh.so:$LD_LIBRARY_PATH" 6 | pip install /tmp/cx_Oracle-5.3.tar.gz 7 | hue syncdb --noinput 8 | hue migrate 9 | deactivate -------------------------------------------------------------------------------- /molecule/lint/INSTALL.rst: -------------------------------------------------------------------------------- 1 | ******* 2 | Docker driver installation guide 3 | ******* 4 | 5 | Requirements 6 | ============ 7 | 8 | * General molecule dependencies (see https://molecule.readthedocs.io/en/latest/installation.html) 9 | * Docker Engine 10 | * docker-py 11 | * docker 12 | 13 | Install 14 | ======= 15 | 16 | $ sudo pip install docker-py 17 | -------------------------------------------------------------------------------- /roles/ext-kerberos5-setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Kerberos 5 client 3 | vars: 4 | packages_RedHat: ['krb5-workstation', 'krb5-libs', "openldap-clients", "python-ldap"] 5 | package: name={{ item }} state=present 6 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 7 | - name: Configure krb5.conf 8 | template: src=krb5.conf dest=/etc/krb5.conf -------------------------------------------------------------------------------- /roles/ext-mysql-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MySQL 3 | - name: Uninstall MySQL / MariaDB 4 | vars: 5 | packages_Suse: ['mariadb'] 6 | packages_RedHat: ['mariadb-server'] 7 | packages_Debian: ['mysql-server'] 8 | package: name={{ item }} state=absent 9 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-httpfs.cfg: -------------------------------------------------------------------------------- 1 | # HttpFS 2 | frontend mapr-httpfs 3 | description "HttpFS" 4 | bind *:14000 5 | mode tcp 6 | default_backend mapr-httpfs 7 | 8 | backend mapr-httpfs 9 | description "HttpFS" 10 | mode tcp 11 | balance roundrobin 12 | server mapr-httpfs1 10.0.0.63:14000 check 13 | server mapr-httpfs2 10.0.0.110:14000 check -------------------------------------------------------------------------------- /sites/mapr-mep-data-access-gateway.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Data Access Gateway 3 | hosts: mapr-data-access-gateway 4 | roles: 5 | - ../roles/mapr-data-access-gateway-install 6 | - name: Uninstall Data Access Gateway 7 | hosts: all:!mapr-data-access-gateway 8 | roles: 9 | - ../roles/mapr-data-access-gateway-uninstall 10 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-sparkhistory: -------------------------------------------------------------------------------- 1 | # Spark HistoryServer 2 | frontend mapr-sparkhistory 3 | description "Spark HistoryServer" 4 | bind *:18480 5 | mode tcp 6 | default_backend mapr-sparkhistory 7 | 8 | backend mapr-sparkhistory 9 | description "Spark HistoryServer" 10 | mode tcp 11 | balance first 12 | server mapr-sparkhistory1 10.0.0.110:18480 check -------------------------------------------------------------------------------- /helper/set-latest-version.yml: -------------------------------------------------------------------------------- 1 | - name: Update version in MapR config 2 | hosts: mapr-cldb[0] 3 | tasks: 4 | - name: Upgrade version in configs 5 | shell: 'maprcli config save -values {"mapr.targetversion":"`cat /opt/mapr/MapRBuildVersion`"}' 6 | environment: 7 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 8 | when: rolling_upgrade is undefined or not rolling_upgrade -------------------------------------------------------------------------------- /molecule/common/tests/test_prereq.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_pam_directory(host): 10 | f = host.file('/etc/pam.d/system-auth-pc') 11 | 12 | assert f.exists 13 | assert f.is_directory 14 | -------------------------------------------------------------------------------- /roles/ext-rstudioserverpro/templates/rserver.conf.ubuntu: -------------------------------------------------------------------------------- 1 | # Server Configuration File 2 | ## Set R Server Studio Port 3 | www-port=8787 4 | ## Set R Binary Path 5 | rsession-which-r=/usr/lib/R/bin/R 6 | #r-cran-repos=http://mapr6" 7 | server-shared-storage-path=/mapr/{{ cluster_name }}/R-Shared/rprojects 8 | auth-pam-sessions-profile=login 9 | auth-pam-sessions-use-password=1 10 | 11 | 12 | -------------------------------------------------------------------------------- /roles/ext-nscd-setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install NSCD 3 | vars: 4 | packages_RedHat: ['nscd'] 5 | package: name={{ item }} state=present 6 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 7 | - name: Configure nscd.conf 8 | template: src=nscd.conf dest=/etc/nscd.conf 9 | - name: Enable and start the NSCD service 10 | service: name=nscd state=restarted enabled=yes -------------------------------------------------------------------------------- /helper/haproxy/mapr-mcs.cfg: -------------------------------------------------------------------------------- 1 | # MCS 2 | frontend mapr-mcs 3 | description "MapR Control System" 4 | bind *:8443 5 | mode tcp 6 | default_backend mapr-mcs 7 | 8 | backend mapr-mcs 9 | description "MapR Control System" 10 | mode tcp 11 | balance first 12 | option ssl-hello-chk 13 | server mapr-mcs 10.0.0.63:8443 check 14 | # server mapr-mcs2 10.0.0.64:8443 check backup -------------------------------------------------------------------------------- /rolling-upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | ansible-playbook -i myhosts/hosts_4nodes helper/fetch-mapr-keys.yml 6 | 7 | input="myhosts/rollingnodes_order" 8 | while IFS= read -r var 9 | do 10 | ansible-playbook -i myhosts/hosts_3nodes site-upgrade.yml --limit $var -e rolling_upgrade=true 11 | done < "$input" 12 | ansible-playbook -i myhosts/hosts_3nodes helper/set-latest-version.yml 13 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-hive-metastore.cfg: -------------------------------------------------------------------------------- 1 | # Hive MetaStore 2 | frontend mapr-hivemeta 3 | description "Hive MetaStore" 4 | bind *:9083 5 | mode tcp 6 | default_backend mapr-hivemeta 7 | 8 | backend mapr-hivemeta 9 | description "Hive MetaStore" 10 | mode tcp 11 | balance roundrobin 12 | server mapr-hivemeta1 10.0.0.63:9083 check 13 | server mapr-hivemeta2 10.0.0.110:9083 check -------------------------------------------------------------------------------- /helper/haproxy/mapr-hiveserver2.cfg: -------------------------------------------------------------------------------- 1 | # HiveServer2 2 | frontend mapr-hiveserver 3 | description "HiveServer2" 4 | bind *:10000 5 | mode tcp 6 | default_backend mapr-hiveserver 7 | 8 | backend mapr-hiveserver 9 | description "HiveServer2" 10 | mode tcp 11 | balance roundrobin 12 | server mapr-hiveserver1 10.0.0.63:10000 check 13 | server mapr-hiveserver2 10.0.0.110:10000 check -------------------------------------------------------------------------------- /molecule/lint/tests/test_default.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_hosts_file(host): 10 | f = host.file('/etc/hosts') 11 | 12 | assert f.exists 13 | assert f.user == 'root' 14 | assert f.group == 'root' 15 | -------------------------------------------------------------------------------- /roles/ext-docker-redhat/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | docker_yum_repo_url: https://download.docker.com/linux/{{ (ansible_distribution == "Fedora") | ternary("fedora","centos") }}/docker-ce.repo 3 | docker_gpg_key: https://download.docker.com/linux/centos/gpg 4 | container_selinux_url: http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.74-1.el7.noarch.rpm 5 | docker_users: ["{{ mapr_user }}"] 6 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-yarn-rm.cfg: -------------------------------------------------------------------------------- 1 | # YARN Resource Manager 2 | frontend mapr-yarn-rm 3 | description "YARN Resource Manager" 4 | bind *:8090 5 | mode tcp 6 | default_backend mapr-yarn-rm 7 | 8 | backend mapr-yarn-rm 9 | description "YARN Resource Manager" 10 | mode tcp 11 | balance roundrobin 12 | server mapr-yarn-rm1 10.0.0.110:8090 check 13 | server mapr-yarn-rm2 10.0.0.232:8090 check -------------------------------------------------------------------------------- /roles/mapr-sqoop2-server-vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #- name: Get MySQL Host 3 | # set_fact: mysql_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 4 | # with_items: 5 | # - "{{ groups['ext-mysql'] }}" 6 | # register: mysql_hosts 7 | #- name: Make MySQL Host String 8 | # set_fact: sqoop_db_host={{ mysql_hosts.results | map(attribute='ansible_facts.mysql_item') | list | join(',')}} 9 | 10 | -------------------------------------------------------------------------------- /roles/ext-kerberos-keytabs-ad-generate/templates/ktutil.txt: -------------------------------------------------------------------------------- 1 | add_entry -password -p {{ mapr_user_cldb_principal }}/{{ cluster_name }} -k 1 -e RC4-HMAC 2 | {{ mapr_kerberos_keytab_passwd }} 3 | add_entry -password -p {{ mapr_user }}/{{ ansible_fqdn }} -k 1 -e RC4-HMAC 4 | {{ mapr_kerberos_keytab_passwd }} 5 | add_entry -password -p HTTP/{{ ansible_fqdn }} -k 1 -e RC4-HMAC 6 | {{ mapr_kerberos_keytab_passwd }} 7 | wkt /tmp/mapr.keytab 8 | quit 9 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-kafka-rest.cfg: -------------------------------------------------------------------------------- 1 | # Kafka REST 2 | frontend mapr-kafka-rest 3 | description "Kafka REST" 4 | bind *:8082 5 | mode tcp 6 | default_backend mapr-kafka-rest 7 | 8 | backend mapr-kafka-rest 9 | description "Kafka REST" 10 | mode tcp 11 | balance roundrobin 12 | server mapr-kafka-rest2 10.0.0.63:8082 check 13 | # Test https://10.0.0.131:8082/streams/%2Fvar%2Fmapr%2Fmapr.monitoring%2Fstreams%2F0/topics -------------------------------------------------------------------------------- /roles/mapr-cldb-changes/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Retrieve all CLDB changes 2 | set_fact: cldb_changes_item={{ hostvars[item]['cldb_haschanged'] if 'cldb_haschanged' in hostvars[item] else 0 }} 3 | with_items: 4 | - "{{ groups['all'] }}" 5 | register: cldb_changes_hosts 6 | - name: Calculate number of CLDB changes 7 | set_fact: cldb_changes={{ cldb_changes_hosts.results | map(attribute='ansible_facts.cldb_changes_item') | map('int') | sum }} 8 | -------------------------------------------------------------------------------- /roles/common-ntp/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install NTP 3 | package: name=ntp state=present 4 | 5 | # ntp can't start in a container, hence no molecule tests 6 | - name: Start the ntp service 7 | vars: 8 | service_Suse: ntpd 9 | service_RedHat: ntpd 10 | service_Debian: ntp 11 | service: 12 | name: "{{ vars['service_' + ansible_os_family] }}" 13 | state: started 14 | enabled: true 15 | tags: 16 | - skip-docker 17 | -------------------------------------------------------------------------------- /roles/mapr-hive-vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get Thrift server 3 | set_fact: metastore_item=thrift://{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}:9083 4 | with_items: 5 | - "{{ groups['mapr-hive-metastore'] }}" 6 | register: metastore_hosts 7 | - name: Make Hive MetaStore String 8 | set_fact: metastore_string={{ metastore_hosts.results | map(attribute='ansible_facts.metastore_item') | list | join(',')}} 9 | -------------------------------------------------------------------------------- /roles/mapr-hive-cli-install/templates/hive-env.sh: -------------------------------------------------------------------------------- 1 | export METASTORE_PORT=9083 2 | {% if hive_with_tez %} 3 | export TEZ_CONF_DIR={{ tez_path_result.files[0].path }}/conf 4 | export TEZ_JARS={{ tez_path_result.files[0].path }}/*:{{ tez_path_result.files[0].path }}/lib/* 5 | export HADOOP_CLASSPATH=$TEZ_CONF_DIR:$TEZ_JARS:$HADOOP_CLASSPATH 6 | {% endif %} 7 | {% if hive_server_security == 'pam' %} 8 | export HADOOP_OPTS="-Dmapr_sec_enabled=true $HADOOP_OPTS" 9 | {% endif %} -------------------------------------------------------------------------------- /roles/mapr-httpfs-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall HttpFS 3 | - name: Stop HttpFS 4 | shell: "maprcli node services -name httpfs -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-httpfs 11 | package: name=mapr-httpfs state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-kibana-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR Kibana 3 | - name: Stop Kibana 4 | shell: "maprcli node services -name kibana -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-kibana 11 | package: name=mapr-kibana state=absent 12 | -------------------------------------------------------------------------------- /roles/common-openjdk18/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install OpenJDK 1.8.0 3 | vars: 4 | packages_RedHat: ['java-1.8.0-openjdk', 5 | 'java-1.8.0-openjdk-devel', 6 | 'java-1.8.0-openjdk-headless'] 7 | packages_Debian: ['openjdk-8-jdk'] 8 | packages_Suse: ['java-1_8_0-openjdk', 'java-1_8_0-openjdk-devel'] 9 | package: name={{ item }} state=present 10 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 11 | -------------------------------------------------------------------------------- /roles/mapr-fluentd-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR fluentd 3 | - name: Stop Fluentd 4 | shell: "maprcli node services -name fluentd -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-fluentd 11 | package: name=mapr-fluentd state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-grafana-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR grafana 3 | - name: Stop Grafana 4 | shell: "maprcli node services -name grafana -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-grafana 11 | package: name=mapr-grafana state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-collectd-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR collectd 3 | - name: Stop Collectd 4 | shell: "maprcli node services -name collectd -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-collectd 11 | package: name=mapr-collectd state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-configure-cleanup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure secure cluster 3 | - name: Clean up of security files copied locally 4 | file: path="{{ local_tmp_path }}/{{ item }}" state=absent 5 | become: no 6 | with_items: 7 | - maprserverticket 8 | - cldb.key 9 | - ssl_keystore 10 | - ssl_keystore.pem 11 | - ssl_keystore.p12 12 | - ssl_truststore 13 | - ssl_truststore.pem 14 | - ssl_truststore.p12 15 | - dare.master.key 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /roles/mapr-oozie-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Oozie and MySQL 3 | - name: Stop Oozie 4 | shell: "maprcli node services -name oozie -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-oozie 11 | package: name=mapr-oozie state=absent 12 | 13 | 14 | -------------------------------------------------------------------------------- /roles/mapr-opentsdb-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR OpenTSDB 3 | - name: Stop OpenTSDB 4 | shell: "maprcli node services -name opentsdb -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-opentsdb 11 | package: name=mapr-opentsdb state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-hive-webhcat-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Hive WebHCat 3 | - name: Stop WebHCat 4 | shell: "maprcli node services -name hcat -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-hivewebhcat 11 | package: name=mapr-hivewebhcat state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-configure-warden-restart/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Restart Warden 3 | - name: Restart MapR Warden 4 | service: name=mapr-warden state=restarted enabled=yes 5 | - pause: seconds="{{ warden_restart_wait_time }}" 6 | when: ansible_check_mode is undefined or not ansible_check_mode 7 | #- name: Check if MapR Fileserver is running 8 | # shell: "/opt/mapr/initscripts/mapr-mfs status" 9 | # register: mfs_status 10 | # failed_when: mfs_status.stdout.find('is running') == -1 11 | -------------------------------------------------------------------------------- /roles/mapr-global-vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Initialize Zookeeper changed variable 3 | set_fact: zookeeper_haschanged=0 4 | - name: Initialize CLDB changed variable 5 | set_fact: cldb_haschanged=0 6 | - name: Initialize History server changed variable 7 | set_fact: historyserver_haschanged=0 8 | - name: Initialize mapr-core local changes variable 9 | set_fact: core_local_changes=0 10 | - name: Initialize mapr-mfs local changes variable 11 | set_fact: mfs_local_changes=0 -------------------------------------------------------------------------------- /roles/mapr-hive-server2-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Hive Server 2 3 | - name: Stop HiveServer2 4 | shell: "maprcli node services -name hs2 -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-hiveserver2 11 | package: name=mapr-hiveserver2 state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-patch-posix-client-basic-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR Patches Client 3 | - name: Check if Service Exists 4 | stat: path=/etc/init.d/mapr-posix-client-basic 5 | register: service_status 6 | - name: Stop the mapr-posix-client-basic service 7 | service: name=mapr-posix-client-basic state=stopped 8 | when: service_status.stat.exists 9 | - name: Uninstall mapr-patch-posix-client-basic 10 | package: name=mapr-patch-posix-client-basic state=absent 11 | -------------------------------------------------------------------------------- /roles/mapr-configure-vars-cldb/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get CLDB FQDNs 3 | set_fact: cldb_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 4 | with_items: 5 | - "{{ groups['mapr-cldb'] }}" 6 | register: cldb_hosts 7 | - name: Make CLDB String 8 | set_fact: cldb_string={{ cldb_hosts.results | map(attribute='ansible_facts.cldb_item') | list | join(',')}} 9 | 10 | - name: Debug CLDB 11 | debug: msg="{{ cldb_string }}" 12 | 13 | 14 | -------------------------------------------------------------------------------- /roles/mapr-drill-standalone-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Apache Drill Standalone 3 | - name: Stop Drill 4 | shell: "maprcli node services -name drill-bits -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-drill 11 | package: name=mapr-drill state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-zookeeper-changes/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Retrieve all Zookeeper changes 2 | set_fact: zookeeper_changes_item={{ hostvars[item]['zookeeper_haschanged'] if 'zookeeper_haschanged' in hostvars[item] else 0 }} 3 | with_items: 4 | - "{{ groups['all'] }}" 5 | register: zookeeper_changes_hosts 6 | - name: Calculate number of Zookeeper changes 7 | set_fact: zookeeper_changes={{ zookeeper_changes_hosts.results | map(attribute='ansible_facts.zookeeper_changes_item') | map('int') | sum }} 8 | -------------------------------------------------------------------------------- /helper/fetch-mapr-keys.yml: -------------------------------------------------------------------------------- 1 | - hosts: mapr-cldb[0] 2 | tasks: 3 | - name: Check if Service Exists 4 | stat: path=/opt/mapr/conf/cldb.key 5 | register: cldb_key 6 | - name: Copy keys to local 7 | fetch: src=/opt/mapr/conf/{{ item }} dest="{{ local_tmp_path }}/{{ item }}" flat=yes 8 | when: cldb_key.stat.exists 9 | with_items: 10 | - maprserverticket 11 | - cldb.key 12 | - ssl_keystore 13 | - ssl_truststore 14 | - ssl_truststore.pem -------------------------------------------------------------------------------- /roles/mapr-sqoop2-server-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Sqoop2 Server 3 | - name: Stop Sqoop2 Server 4 | shell: "maprcli node services -name sqoop2 -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-sqoop2-server 11 | package: name=mapr-sqoop2-server state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-hive-metastore-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Hive Metastore 3 | - name: Stop Hive Metastore 4 | shell: "maprcli node services -name hivemeta -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-hivemetastore 11 | package: name=mapr-hivemetastore state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-impala-server-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Impala Server 3 | - name: Stop Impala Server 4 | shell: "maprcli node services -name impalaserver -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-impala-server 11 | package: name=mapr-impala-server state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-nfs-v3-install/templates/mapr_fstab: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2009 & onwards. MapR Tech, Inc., All rights reserved 2 | 3 | # MapR fstab: MapR filestem information 4 | # Filesystems in this file are mounted after nfsserver is 5 | # started (/etc/init.d/mapr-nfsserver start) 6 | # on shutdown the listed Filesystems are unmounted 7 | # (/etc/init.d/mapr-nfsserver stop) 8 | # 9 | # format: 10 | # 11 | 12 | localhost:/mapr /mapr nolock,nfsvers=3,nosuid -------------------------------------------------------------------------------- /roles/mapr-patch-posix-client-platinum-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR Patches Client 3 | - name: Check if Service Exists 4 | stat: path=/etc/init.d/mapr-posix-client-platinum 5 | register: service_status 6 | - name: Stop the mapr-posix-client-platinum service 7 | service: name=mapr-posix-client-platinum state=stopped 8 | when: service_status.stat.exists 9 | - name: Uninstall mapr-patch-posix-client-platinum 10 | package: name=mapr-patch-posix-client-platinum state=absent 11 | -------------------------------------------------------------------------------- /molecule/common/tests/test_openjdk18.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_java_installed(host): 10 | list = ['java-1.8.0-openjdk', 11 | 'java-1.8.0-openjdk-devel', 12 | 'java-1.8.0-openjdk-headless'] 13 | for j in list: 14 | p = host.package(j) 15 | 16 | assert p.is_installed 17 | -------------------------------------------------------------------------------- /molecule/common/tests/test_sysctl.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_sysctl_file(host): 10 | f = host.file('/etc/sysctl.conf') 11 | 12 | assert f.exists 13 | assert f.contains("vm.swappiness = 1") 14 | assert f.contains("net.ipv4.tcp_retries2 = 5") 15 | assert f.contains("vm.overcommit_memory = 0") 16 | -------------------------------------------------------------------------------- /roles/mapr-elasticsearch-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR elasticsearch 3 | - name: Stop Elasticsearch 4 | shell: "maprcli node services -name elasticsearch -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-elasticsearch 11 | package: name=mapr-elasticsearch state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-filemigrate-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall File Migrate Service 3 | - name: Stop File Migrate Service 4 | shell: "maprcli node services -name filemigrate -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-filemigrate 11 | package: name=mapr-filemigrate state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-impala-catalog-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Impala Catalog 3 | - name: Stop Impala Catalog 4 | shell: "maprcli node services -name impalacatalog -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-impala-catalog 11 | package: name=mapr-impala-catalog state=absent 12 | -------------------------------------------------------------------------------- /sites/mapr-unprovision.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Uninstall MapR Core 3 | hosts: all:!mapr-core 4 | roles: 5 | - ../roles/mapr-core-uninstall 6 | - name: Unregister node vars 7 | hosts: all 8 | roles: 9 | - ../roles/mapr-unregister-node-vars 10 | - name: Unregister node vars 11 | hosts: mapr-cldb[0] 12 | vars: 13 | num_unprovision: "{{ groups['unprovision']|length }}" 14 | roles: 15 | - {role: ../roles/mapr-unregister-node, when: num_unprovision != '0'} -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | # Testing mapr-ansible roles 2 | 3 | ## Introduction 4 | 5 | Tests are based on molecule framework 6 | * https://molecule.readthedocs.io/ 7 | * https://github.com/metacloud/molecule 8 | 9 | ## Requirements 10 | 11 | * Docker 12 | 13 | ## Run linter 14 | 15 | docker run --rm -it \ 16 | -v "$(pwd)":/tmp/$(basename "${PWD}"):ro \ 17 | -v /var/run/docker.sock:/var/run/docker.sock \ 18 | -w /tmp/$(basename "${PWD}") \ 19 | retr0h/molecule:latest \ 20 | molecule test -s lint 21 | 22 | -------------------------------------------------------------------------------- /roles/mapr-impala-statestore-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Impala StateStore 3 | - name: Stop Impala StateStore 4 | shell: "maprcli node services -name impalastore -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-impala-statestore 11 | package: name=mapr-impala-statestore state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-patch-posix-client-basic-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Patches Client 3 | - name: Install mapr-patch-posix-client-basic 4 | package: name=mapr-patch-posix-client-basic state=present 5 | - name: Set /opt/mapr/conf/mapruserticket in fuse.conf 6 | lineinfile: dest=/opt/mapr/conf/fuse.conf regexp="^fuse\.ticketfile\.location" line="fuse.ticketfile.location=/opt/mapr/conf/maprfuseticket" 7 | - name: Start the mapr-posix-client-basic service 8 | service: name=mapr-posix-client-basic state=restarted enabled=yes -------------------------------------------------------------------------------- /roles/mapr-spark-historyserver-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Spark on YARN 3 | - name: Stop Spark Historyserver 4 | shell: "maprcli node services -name spark-historyserver -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-spark-historyserver 11 | package: name=mapr-spark-historyserver state=absent 12 | -------------------------------------------------------------------------------- /roles/mapr-configure-vars-hs/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure unsecure cluster 3 | 4 | - name: Get Job History Server 5 | set_fact: historyserver_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 6 | with_items: 7 | - "{{ groups['mapr-historyserver'] }}" 8 | register: historyserver_hosts 9 | 10 | - name: Make History Server String 11 | set_fact: historyserver_string={{ historyserver_hosts.results | map(attribute='ansible_facts.historyserver_item') | list | join(',')}} 12 | 13 | 14 | -------------------------------------------------------------------------------- /roles/mapr-historyserver-changes/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Retrieve all HistoryServer changes 2 | set_fact: historyserver_changes_item={{ hostvars[item]['historyserver_haschanged'] if 'historyserver_haschanged' in hostvars[item] else 0 }} 3 | with_items: 4 | - "{{ groups['all'] }}" 5 | register: historyserver_changes_hosts 6 | - name: Calculate number of History Server changes 7 | set_fact: historyserver_changes={{ historyserver_changes_hosts.results | map(attribute='ansible_facts.historyserver_changes_item') | map('int') | sum }} 8 | -------------------------------------------------------------------------------- /roles/mapr-log-monitoring-configure-vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR log monitoring configure 3 | 4 | - name: Get ElasticSearch FQDNs 5 | set_fact: elasticsearch_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 6 | with_items: 7 | - "{{ groups['mapr-elasticsearch'] }}" 8 | register: elasticsearch_hosts 9 | - name: Make ElasticSearch String 10 | set_fact: elasticsearch_string={{ elasticsearch_hosts.results | map(attribute='ansible_facts.elasticsearch_item') | list | join(',')}} 11 | -------------------------------------------------------------------------------- /roles/mapr-patch-posix-client-platinum-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Patches Client 3 | - name: Install mapr-patch-posix-client-platinum 4 | package: name=mapr-patch-posix-client-platinum state=present 5 | - name: Set /opt/mapr/conf/mapruserticket in fuse.conf 6 | lineinfile: dest=/opt/mapr/conf/fuse.conf regexp="^fuse\.ticketfile\.location" line="fuse.ticketfile.location=/opt/mapr/conf/maprfuseticket" 7 | - name: Start the mapr-posix-client-platinum service 8 | service: name=mapr-posix-client-platinum state=restarted enabled=yes -------------------------------------------------------------------------------- /roles/mapr-hbase-cli-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install HBase CLI 3 | - name: Install mapr-hbase 4 | package: name=mapr-hbase state=present 5 | - name: Set MAPR_HBASE_SERVER_OPTS using HYBRID_LOGIN_OPTS 6 | lineinfile: > 7 | dest=/opt/mapr/conf/env_override.sh 8 | line="export MAPR_HBASE_CLIENT_OPTS=\"${HYBRID_LOGIN_OPTS}\"" 9 | regexp="^export MAPR_HBASE_CLIENT_OPTS" 10 | mode=0644 11 | create=yes 12 | owner="{{ mapr_user }}" 13 | group="{{ mapr_group }}" 14 | when: hbase_thrift_rest_security == 'kerberos' -------------------------------------------------------------------------------- /roles/mapr-configure-env-override/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set MapR Subnet 3 | lineinfile: dest=/opt/mapr/conf/env_override.sh regexp="^export MAPR_SUBNETS" line="export MAPR_SUBNETS={{ mapr_subnets }}" mode=0644 create=yes owner="{{ mapr_user }}" group="{{ mapr_group }}" 4 | when: mapr_subnets is defined 5 | - name: Set MapR User 6 | lineinfile: dest=/opt/mapr/conf/env_override.sh regexp="^export MAPR_USER" line="export MAPR_USER={{ mapr_user }}" mode=0644 create=yes owner="{{ mapr_user }}" group="{{ mapr_group }}" 7 | when: mapr_user is defined 8 | -------------------------------------------------------------------------------- /roles/mapr-posix-client-basic-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall mapr-posix-client-basic 3 | - name: Check if Service Exists 4 | stat: path=/etc/init.d/mapr-posix-client-basic 5 | register: service_status 6 | - name: Stop the mapr-posix-client-basic service 7 | service: name=mapr-posix-client-basic state=stopped 8 | when: service_status.stat.exists 9 | - name: Uninstall mapr-posix-client-basic 10 | package: name=mapr-posix-client-basic state=absent 11 | # Other actions 12 | # - name: Remove /mapr dir 13 | # file: path=/mapr state=absent 14 | -------------------------------------------------------------------------------- /roles/mapr-unregister-node-vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure unsecure cluster 3 | - name: Get unprovisioned FQDNs 4 | set_fact: unprovision_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 5 | with_items: 6 | - "{{ groups['unprovision'] }}" 7 | register: unprovision_hosts 8 | - name: Make Unprovision String 9 | set_fact: unprovision_string={{ unprovision_hosts.results | map(attribute='ansible_facts.unprovision_item') | list | join(',')}} 10 | #- debug: msg="{{ unprovision_hosts }}" 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /sites/mapr-mep-hue.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Setup Hue Vars 3 | hosts: all 4 | vars: 5 | num_mysql_hosts: "{{ groups['ext-mysql']|length }}" 6 | roles: 7 | - ../roles/mapr-hue-vars 8 | - {role: ../roles/mapr-hue-mysql, when: hue_db_type == 'mysql' and hue_db_mysql_create_schema_and_user and num_mysql_hosts != '0' } 9 | - name: Install Hue 10 | hosts: mapr-hue 11 | roles: 12 | - ../roles/mapr-hue-install 13 | - name: Uninstall Hue 14 | hosts: all:!mapr-hue 15 | roles: 16 | - ../roles/mapr-hue-uninstall -------------------------------------------------------------------------------- /roles/ext-anaconda-python-spark/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Anaconda 3 | - name: Find Spark Path 4 | find: paths="/opt/mapr/spark" patterns="spark*" file_type=directory 5 | register: spark_path_result 6 | failed_when: spark_path_result.matched != 1 7 | - set_fact: spark_path="{{ spark_path_result.files[0].path }}" 8 | - name: Set new Python environment in Spark 9 | lineinfile: dest={{ spark_path }}/conf/spark-env.sh state=present regexp="^export\ PYSPARK_PYTHON" line="export PYSPARK_PYTHON={{ anaconda_install_path }}/envs/{{ anaconda_env_name }}/bin/python" 10 | -------------------------------------------------------------------------------- /sites/common.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Apply common configuration to all nodes 3 | hosts: common 4 | roles: 5 | - ../roles/common-prereq 6 | # - ../roles/common-selinux 7 | - {role: ../roles/common-oraclejdk18, when: jdk == 'oraclejdk18'} 8 | - {role: ../roles/common-openjdk18, when: jdk == 'openjdk18'} 9 | - {role: ../roles/common-mapruser, when: mapr_create_user} 10 | - ../roles/common-rpcbind 11 | - {role: ../roles/common-ntp, when: use_ntp} 12 | - ../roles/common-misc 13 | - ../roles/common-sysctl 14 | -------------------------------------------------------------------------------- /helper/set-mapruser-authkeys-sudo.yml: -------------------------------------------------------------------------------- 1 | - name: Update version in MapR config 2 | hosts: all 3 | tasks: 4 | - name: include global config 5 | include_vars: file="../group_vars/all" 6 | - name: Set mapr as sudoers 7 | lineinfile: 8 | path: /etc/sudoers 9 | state: present 10 | regexp: '^{{ mapr_user }}\s' 11 | line: '%{{ mapr_user }} ALL=(ALL) NOPASSWD: ALL' 12 | - name: Set authorized key taken from file 13 | authorized_key: 14 | user: "{{ mapr_user }}" 15 | state: present 16 | key: "{{ lookup('file', mapr_public_key_path ) }}" -------------------------------------------------------------------------------- /roles/common-mapruser/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Create MapR Group 4 | group: 5 | name: "{{ mapr_group }}" 6 | gid: "{{ mapr_group_gid }}" 7 | state: present 8 | - name: Create MapR User 9 | user: 10 | name: "{{ mapr_user }}" 11 | comment: "MapR default user" 12 | uid: "{{ mapr_user_uid }}" 13 | group: "{{mapr_group}}" 14 | groups: "{{ mapr_group }}" 15 | password: "{{ mapr_passwd }}" 16 | createhome: true 17 | update_password: on_create 18 | state: present 19 | append: true 20 | # groups={{ mapr_group }},root,wheel 21 | -------------------------------------------------------------------------------- /roles/mapr-posix-client-platinum-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall mapr-posix-client-platinum 3 | - name: Check if Service Exists 4 | stat: path=/etc/init.d/mapr-posix-client-platinum 5 | register: service_status 6 | - name: Stop the mapr-posix-client-platinum service 7 | service: name=mapr-posix-client-platinum state=stopped 8 | when: service_status.stat.exists 9 | - name: Uninstall mapr-posix-client-platinum 10 | package: name=mapr-posix-client-platinum state=absent 11 | 12 | # Other actions 13 | # - name: Remove /mapr dir 14 | # file: path=/mapr state=absent 15 | -------------------------------------------------------------------------------- /roles/ext-kerberos-keytabs-ad-generate/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # http://stackoverflow.com/questions/4322243/adding-a-user-with-a-password-in-active-directory-ldaps 3 | - name: Delete existing keytab 4 | file: path=/tmp/mapr.keytab state=absent 5 | - name: Copy ktutil script 6 | template: src=ktutil.txt dest=/tmp/ktutil.txt 7 | 8 | - name: Create mapr.keytab 9 | shell: cat /tmp/ktutil.txt | ktutil 10 | - fetch: src=/tmp/mapr.keytab dest="{{ mapr_kerberos_local_keytabs_dir }}/{{ ansible_fqdn }}.keytab" flat=yes 11 | 12 | - name: Clean up existing keytab 13 | file: path=/tmp/mapr.keytab state=absent 14 | -------------------------------------------------------------------------------- /roles/mapr-kibana-customssl/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Find Kibana Path 3 | find: paths="/opt/mapr/kibana/" patterns="kibana*" file_type=directory 4 | register: kibana_path_result 5 | failed_when: kibana_path_result.matched != 1 6 | - name: Extract key to Kibana directory 7 | shell: "openssl rsa -outform pem -in /opt/mapr/conf/ssl_keystore.pem -passin pass:mapr123 -out {{ kibana_path_result.files[0].path }}/config/key.pem" 8 | - name: Extract cert to Kibana directory 9 | shell: "openssl x509 -outform pem -in /opt/mapr/conf/ssl_keystore.pem -out {{ kibana_path_result.files[0].path }}/config/cert.pem" 10 | -------------------------------------------------------------------------------- /roles/mapr-gateway-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Gateway 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop Gateway 7 | shell: "maprcli node services -name gateway -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-gateway 15 | package: name=mapr-gateway state=absent 16 | -------------------------------------------------------------------------------- /helper/ssl/cert_tpl: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name = req_distinguished_name 3 | x509_extensions = v3_req 4 | req_extensions = v3_req 5 | prompt = no 6 | [req_distinguished_name] 7 | C = DE 8 | L = Munich 9 | O = MapR 10 | OU = MapR Professional Services 11 | CN = ip-10-0-0-${NODENUMBER}.ps.mapr.com 12 | 13 | [v3_req] 14 | keyUsage = keyEncipherment, dataEncipherment 15 | extendedKeyUsage = serverAuth 16 | subjectAltName = @alt_names 17 | 18 | [alt_names] 19 | DNS.1 = ip-10-0-0-${NODENUMBER}.ps.mapr.com 20 | DNS.2 = 10.0.0.${NODENUMBER} 21 | DNS.3 = ip-10-0-0-${NODENUMBER} 22 | DNS.4 = ip-10-0-0-${NODENUMBER}.eu-west-1.compute.internal -------------------------------------------------------------------------------- /roles/mapr-kafka-rest-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR mapr-kafka-rest 3 | - name: Stop Kafka REST 4 | shell: "maprcli node services -name kafka-rest -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-kafka-rest 11 | package: name=mapr-kafka-rest state=absent 12 | register: kafka_rest_result 13 | - name: Configure.sh -R 14 | shell: /opt/mapr/server/configure.sh -R 15 | when: kafka_rest_result.changed -------------------------------------------------------------------------------- /molecule/mapr-core-docker/plays/common.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # - import_playbook: mapr-include-vars.yml 3 | - name: Apply common configuration to all nodes 4 | hosts: common 5 | roles: 6 | - ../../roles/common-prereq 7 | # - ../../roles/common-selinux 8 | - {role: ../../roles/common-oraclejdk18, when: jdk == 'oraclejdk18'} 9 | - {role: ../../roles/common-openjdk18, when: jdk == 'openjdk18'} 10 | - {role: ../../roles/common-mapruser, when: mapr_create_user} 11 | - ../../roles/common-rpcbind 12 | - {role: ../../roles/common-ntp, when: use_ntp} 13 | - ../../roles/common-misc 14 | - ../../roles/common-sysctl 15 | -------------------------------------------------------------------------------- /roles/mapr-grafana-customssl/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Find Grafana Path 3 | find: paths="/opt/mapr/grafana/" patterns="grafana*" file_type=directory 4 | register: grafana_path_result 5 | failed_when: grafana_path_result.matched != 1 6 | - name: Extract key to Grafana directory 7 | shell: "openssl rsa -outform pem -in /opt/mapr/conf/ssl_keystore.pem -passin pass:mapr123 -out {{ grafana_path_result.files[0].path }}/etc/grafana/key.pem" 8 | - name: Extract cert to Grafana directory 9 | shell: "openssl x509 -outform pem -in /opt/mapr/conf/ssl_keystore.pem -out {{ grafana_path_result.files[0].path }}/etc/grafana/cert.pem" 10 | -------------------------------------------------------------------------------- /roles/mapr-configure-vars-rm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure unsecure cluster 3 | 4 | - name: Get Resource Manager 5 | set_fact: resourcemanager_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 6 | with_items: 7 | - "{{ groups['mapr-resourcemanager'] }}" 8 | register: resourcemanager_hosts 9 | when: groups['mapr-resourcemanager'] is defined 10 | - name: Make Resource Manager String 11 | set_fact: resourcemanager_string={{ resourcemanager_hosts.results | map(attribute='ansible_facts.resourcemanager_item') | list | join(',')}} 12 | when: groups['mapr-resourcemanager'] is defined 13 | 14 | -------------------------------------------------------------------------------- /roles/mapr-mcs-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR Control System (MCS) 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop MCS 7 | shell: "maprcli node services -name webserver -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-webserver 15 | package: name=mapr-webserver state=absent 16 | -------------------------------------------------------------------------------- /molecule/common/tests/test_mapruser.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_mapr_user(host): 10 | u = host.user('mapr') 11 | 12 | assert u.exists 13 | assert u.name == "mapr" 14 | assert u.group == "mapr" 15 | for g in ['mapr']: 16 | assert g in u.groups 17 | assert u.gid == 5000 18 | 19 | home = host.file("/home/mapr") 20 | 21 | assert home.is_directory 22 | assert home.user == "mapr" 23 | assert home.group == "mapr" 24 | -------------------------------------------------------------------------------- /roles/mapr-mastgateway-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MAST Gateway 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop MAST Gateway 7 | shell: "maprcli node services -name mastgateway -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-mastgateway 15 | package: name=mapr-mastgateway state=absent 16 | -------------------------------------------------------------------------------- /roles/mapr-nodemanager-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall YARN Node Manager 3 | - name: Check role nodemanager 4 | stat: path=/opt/mapr/roles/nodemanager 5 | register: role_nodemanager 6 | - name: Stop YARN Node Manager 7 | shell: "maprcli node services -name nodemanager -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: role_nodemanager.stat.exists 14 | - name: Uninstall mapr-nodemanager 15 | package: name=mapr-nodemanager state=absent 16 | -------------------------------------------------------------------------------- /sites/mapr-mep-oozie.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Setup Zookeeper String 3 | hosts: all 4 | roles: 5 | - ../roles/mapr-configure-vars-zk 6 | - name: Set Oozie Vars 7 | hosts: all 8 | vars: 9 | num_mysql_hosts: "{{ groups['ext-mysql']|length }}" 10 | roles: 11 | - {role: ../roles/mapr-oozie-mysql, when: oozie_db_type == 'mysql' and oozie_db_mysql_create_schema_and_user and num_mysql_hosts != '0' } 12 | - name: Install Oozie 13 | hosts: mapr-oozie 14 | roles: 15 | - ../roles/mapr-oozie-install 16 | - name: Uninstall Oozie 17 | hosts: all:!mapr-oozie 18 | roles: 19 | - ../roles/mapr-oozie-uninstall -------------------------------------------------------------------------------- /roles/mapr-fileserver-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR FileServer (MFS) 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop MapR Fileserver 7 | shell: "maprcli node services -name fileserver -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-fileserver 15 | package: name=mapr-fileserver state=absent 16 | -------------------------------------------------------------------------------- /helper/ssh_unsecure.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | tasks: 3 | - lineinfile: state=present dest=/etc/ssh/sshd_config regexp="^PermitRootLogin" line="PermitRootLogin yes" 4 | - lineinfile: state=present dest=/etc/ssh/sshd_config regexp="^PasswordAuthentication" line="PasswordAuthentication yes" 5 | - user: name=ec2-user password="$6$GzlcP5ei$E6P9QM40O6KJFLZ3wMcg3jzQBNxGXrf8yc77CbCmf0KtFSxK0mpCParhc3WVNEAm4fHKPAqyRV9rsiF6DAFlW0" createhome=yes state=present 6 | - user: name=root password="$6$GzlcP5ei$E6P9QM40O6KJFLZ3wMcg3jzQBNxGXrf8yc77CbCmf0KtFSxK0mpCParhc3WVNEAm4fHKPAqyRV9rsiF6DAFlW0" createhome=yes state=present 7 | - service: name=sshd state=restarted enabled=yes 8 | -------------------------------------------------------------------------------- /molecule/lint/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: docker 6 | lint: 7 | name: yamllint 8 | options: 9 | config-file: molecule/lint/yamllint.conf 10 | platforms: 11 | - name: instance 12 | image: centos:7 13 | provisioner: 14 | name: ansible 15 | lint: 16 | name: ansible-lint 17 | options: 18 | x: 19 | - ANSIBLE0002 20 | - ANSIBLE0006 21 | - ANSIBLE0011 22 | - ANSIBLE0012 23 | - ANSIBLE0013 24 | - ANSIBLE0016 25 | scenario: 26 | name: lint 27 | test_sequence: 28 | - lint 29 | verifier: 30 | name: testinfra 31 | lint: 32 | name: flake8 33 | -------------------------------------------------------------------------------- /roles/ext-anaconda-python/templates/condarc: -------------------------------------------------------------------------------- 1 | custom_channels: 2 | pkgs/main: {{ anaconda_repo_path }}/ 3 | pkgs/free: {{ anaconda_repo_path }}/ 4 | pkgs/r: {{ anaconda_repo_path }}/ 5 | pkgs/pro: {{ anaconda_repo_path }}/ 6 | custom_multichannels: 7 | defaults: ["{{ anaconda_repo_path }}/pkgs/main", "{{ anaconda_repo_path }}/pkgs/free", "{{ anaconda_repo_path }}/pkgs/r", "{{ anaconda_repo_path }}/pkgs/pro"] 8 | local: [] 9 | default_channels: 10 | - {{ anaconda_repo_path }}/pkgs/main 11 | - {{ anaconda_repo_path }}/pkgs/free 12 | - {{ anaconda_repo_path }}/pkgs/r 13 | - {{ anaconda_repo_path }}/pkgs/pro 14 | # - file:/{{ anaconda_custom_repo_path }} -------------------------------------------------------------------------------- /roles/ui-setup-nativesecurity-genkeys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - stat: path=/opt/mapr/conf/cldb.key 3 | register: cldb_key_status 4 | #- debug: msg={{cldb_key_status.stat.exists}} 5 | - name: Generate keys 6 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -secure -genkeys -Z {{ ansible_fqdn }} -C {{ ansible_fqdn }} -u {{ mapr_user }} -g {{mapr_group}}" 7 | when: cldb_key_status.stat.exists == false 8 | - name: Fetch generated files 9 | fetch: src="/opt/mapr/conf/{{ item }}" dest="{{ local_tmp_path }}/{{ item }}" flat=yes 10 | with_items: 11 | - cldb.key 12 | - maprserverticket 13 | - ssl_keystore 14 | - ssl_truststore 15 | 16 | -------------------------------------------------------------------------------- /roles/mapr-kafka-connect-jdbc-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR mapr-kafka-connect-jdbc 3 | - name: Stop Kafka Connect JDBC 4 | shell: "maprcli node services -name kafka-connect -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-kafka-connect-jdbc 11 | package: name=mapr-kafka-connect-jdbc state=absent 12 | register: connect_result 13 | - name: Configure.sh -R 14 | shell: /opt/mapr/server/configure.sh -R 15 | when: connect_result.changed -------------------------------------------------------------------------------- /roles/mapr-core-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR Core components for all machines 3 | - name: Stop Warden 4 | service: name=mapr-warden state=stopped 5 | - pause: seconds=30 6 | - name: Uninstall mapr-core 7 | vars: 8 | packages_Suse: ['mapr-compat-suse', 'mapr-core', 'mapr-core-internal'] 9 | packages_RedHat: ['mapr-core', 'mapr-core-internal'] 10 | packages_Debian: ['mapr-core', 'mapr-core-internal'] 11 | package: name={{ item }} state=absent 12 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 13 | - name: Clean /opt/mapr 14 | file: path=/opt/mapr state=absent 15 | - name: Clean /opt/cores 16 | file: path=/opt/cores state=absent -------------------------------------------------------------------------------- /roles/mapr-data-access-gateway-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall MapR mapr-kafka-connect-jdbc# 3 | - name: Stop Data Access Gateway 4 | shell: "maprcli node services -name data-access-gateway -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Uninstall mapr-data-access-gateway 11 | package: name=mapr-data-access-gateway state=absent 12 | register: gateway_result 13 | - name: Configure.sh -R 14 | shell: /opt/mapr/server/configure.sh -R 15 | when: gateway_result.changed -------------------------------------------------------------------------------- /roles/mapr-spark-thriftserver-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Gateway 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop Spark Thrift Server 7 | shell: "maprcli node services -name spark-thriftserver -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-spark-thriftserver 15 | package: name=mapr-spark-thriftserver state=absent 16 | -------------------------------------------------------------------------------- /roles/mapr-zookeeper-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Zookeeper 3 | - name: Check if Service Exists 4 | stat: path=/etc/init.d/mapr-zookeeper 5 | register: service_status 6 | - name: Stop Zookeeper 7 | service: name=mapr-zookeeper state=stopped 8 | when: service_status.stat.exists 9 | - name: Uninstall mapr-zookeeper 10 | package: name=mapr-zk-internal state=absent 11 | register: zookeeper_result 12 | - set_fact: zookeeper_haschanged={{ zookeeper_haschanged | int + 1 }} 13 | when: zookeeper_result.changed 14 | # disabled, because if it is a mount point it will fail 15 | #- name: Clean /opt/mapr/zkdata 16 | # file: path=/opt/mapr/zkdata state=absent 17 | 18 | -------------------------------------------------------------------------------- /roles/mapr-resourcemanager-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall YARN Resource Manager 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop YARN Resource Manager 7 | shell: "maprcli node services -name resourcemanager -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-resourcemanager 15 | package: name=mapr-resourcemanager state=absent 16 | -------------------------------------------------------------------------------- /roles/mapr-configure-unsecure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure unsecure cluster 3 | 4 | - name: Configure.sh 5 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -Z {{ zookeeper_string }} -C {{ cldb_string }} -u {{ mapr_user }} -g {{mapr_group}} -unsecure -HS {{ historyserver_string }} {{ '--isvm' if virtual_machine else '' }}" 6 | # mfs_heapsize_percent is optional, if required just remove # and set value, default is 35. 7 | - name: Set MFS memory percent 8 | lineinfile: dest=/opt/mapr/conf/warden.conf regexp="^service\.command\.mfs\.heapsize\.maxpercent=" line="service.command.mfs.heapsize.maxpercent={{ mfs_heapsize_maxpercent }}" 9 | when: mfs_heapsize_maxpercent is defined -------------------------------------------------------------------------------- /roles/mapr-spark-yarn-shuffle-distribute/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Copy JAR for MapR Spark YARN shuffle 3 | - name: Copy JAR for MapR Spark YARN shuffle 4 | copy: src="{{ local_tmp_path }}/spark-yarn-shuffle.jar" dest="/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/yarn/lib/" mode=0644 5 | register: copy_result 6 | 7 | - name: Restart Node Manager 8 | shell: "maprcli node services -name nodemanager -action restart -nodes {{ ansible_fqdn }}" 9 | register: task_result 10 | until: task_result.rc == 0 11 | retries: 10 12 | delay: 10 13 | ignore_errors: yes 14 | environment: 15 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 16 | when: copy_result.changed -------------------------------------------------------------------------------- /sites/mapr-mep-drill.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Setup Zookeeper String 3 | hosts: all 4 | roles: 5 | - ../roles/mapr-configure-vars-zk 6 | - name: Install Drill Standalone 7 | hosts: mapr-drill-standalone 8 | roles: 9 | - ../roles/mapr-drill-standalone-install 10 | - name: Uninstall Drill Standalone 11 | hosts: all:!mapr-drill-standalone 12 | roles: 13 | - ../roles/mapr-drill-standalone-uninstall 14 | - name: Install Drill on YARN 15 | hosts: mapr-drill-yarn 16 | roles: 17 | - ../roles/mapr-drill-yarn-install 18 | - name: Uninstall Drill on YARN 19 | hosts: all:!mapr-drill-yarn 20 | roles: 21 | - ../roles/mapr-drill-yarn-uninstall -------------------------------------------------------------------------------- /helper/windowsad/createkeytabs.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | set MAPR_HOSTS=hosts.txt 4 | set MAPR_TEC_USER=TEC-MAPR 5 | set REALM=PS.MAPR.COM 6 | 7 | for /F "tokens=*" %%A in (%MAPR_HOSTS%) do ( 8 | ktpass /princ mapr/%%A@%REALM% /mapuser %MAPR_TEC_USER% +rndPass /out out\mapr-%%A.keytab /crypto RC4-HMAC-NT /ptype KRB5_NT_PRINCIPAL 9 | echo mapr-%%A.keytab 10 | ktpass /princ cldb/%%A@%REALM% /mapuser %MAPR_TEC_USER% +rndPass /out out\cldb-%%A.keytab /crypto RC4-HMAC-NT /ptype KRB5_NT_PRINCIPAL 11 | echo mapr-%%A.keytab 12 | ktpass /princ HTTP/%%A@%REALM% /mapuser %MAPR_TEC_USER% +rndPass /out out\HTTP-%%A.keytab /crypto RC4-HMAC-NT /ptype KRB5_NT_PRINCIPAL 13 | echo mapr-%%A.keytab 14 | ) 15 | 16 | PAUSE -------------------------------------------------------------------------------- /molecule/common/tests/test_misc.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import testinfra.utils.ansible_runner 4 | 5 | testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 6 | os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') 7 | 8 | 9 | def test_mc_installed(host): 10 | p = host.package("mc") 11 | 12 | assert p.is_installed 13 | 14 | 15 | def test_wget_installed(host): 16 | p = host.package("wget") 17 | 18 | assert p.is_installed 19 | 20 | 21 | def test_mysql_python_installed(host): 22 | p = host.package("MySQL-python") 23 | 24 | assert p.is_installed 25 | 26 | 27 | def test_openssl_installed(host): 28 | p = host.package("openssl") 29 | 30 | assert p.is_installed 31 | -------------------------------------------------------------------------------- /roles/mapr-nfs-v3-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install NFS 3 | 4 | - name: Install mapr-nfs 5 | vars: 6 | packages_Suse: ['mapr-nfs', 'nfs-utils', 'rpcbind'] 7 | packages_RedHat: ['mapr-nfs', 'nfs-utils', 'rpcbind'] 8 | packages_Debian: ['mapr-nfs', 'nfs-common', 'rpcbind'] 9 | package: name={{ item }} state=present 10 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 11 | 12 | # Other actions 13 | - name: Create /mapr dir 14 | file: path=/mapr state=directory 15 | - name: Configure mapr_fstab for NFS 16 | template: src=mapr_fstab dest=/opt/mapr/conf/mapr_fstab 17 | - name: Configure exports for NFS 18 | template: src=exports dest=/opt/mapr/conf/exports 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /sites/mapr-mep-hbase.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Setup Zookeeper String 3 | hosts: all 4 | roles: 5 | - ../roles/mapr-configure-vars-zk 6 | - name: Install HBase CLI 7 | hosts: mapr-hbase-cli 8 | roles: 9 | - ../roles/mapr-hbase-cli-install 10 | - name: Install HBase REST and Thrift Server 11 | hosts: mapr-hbase-thrift-rest 12 | roles: 13 | - ../roles/mapr-hbase-thrift-rest-install 14 | - name: Uninstall HBase REST and Thrift Server 15 | hosts: all:!mapr-hbase-thrift-rest 16 | roles: 17 | - ../roles/mapr-hbase-thrift-rest-uninstall 18 | - name: Uninstall HBase CLI 19 | hosts: all:!mapr-hbase-cli 20 | roles: 21 | - ../roles/mapr-hbase-cli-uninstall -------------------------------------------------------------------------------- /roles/mapr-cldb-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall CLDB 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop CLDB 7 | shell: "maprcli node services -name cldb -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-cldb 15 | package: name=mapr-cldb state=absent 16 | register: cldb_result 17 | - set_fact: cldb_haschanged="{{ cldb_haschanged | int + 1 }}" 18 | when: cldb_result.changed 19 | -------------------------------------------------------------------------------- /roles/ext-rstudioserverpro/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Debian / Ubuntu 3 | - name: Install R Studio Server 4 | apt: deb={{ rstudio_server_pro_deb }} 5 | when: ansible_os_family == 'Debian' 6 | 7 | - name: Create projects folder in MapR-FS 8 | file: path="/mapr/{{ cluster_name }}/R-Shared/rprojects" state=directory mode=0777 9 | 10 | - name: Copy rserver.conf 11 | template: src=rserver.conf.ubuntu dest=/etc/rstudio/rserver.conf 12 | when: ansible_os_family == 'Debian' 13 | - name: Copy rsession.conf 14 | template: src=rsession.conf.ubuntu dest=/etc/rstudio/rsession.conf 15 | when: ansible_os_family == 'Debian' 16 | - name: Restart RStudio Server Pro and disable service startup 17 | service: name=rstudio-server state=restarted enabled=no -------------------------------------------------------------------------------- /roles/mapr-spark-yarn-shuffle-retrieve/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Copy JAR for MapR Spark YARN shuffle 3 | - name: Find Spark Path 4 | find: paths="/opt/mapr/spark" patterns="spark*" file_type=directory 5 | register: spark_path_result 6 | failed_when: spark_path_result.matched != 1 7 | - set_fact: spark_path="{{ spark_path_result.files[0].path }}" 8 | - name: Find Spark Shuffle on first Spark Node 9 | find: paths="{{ spark_path }}/yarn/" patterns="spark-*-shuffle.jar" file_type=file 10 | register: shuffle_jar_path_result 11 | failed_when: shuffle_jar_path_result.matched != 1 12 | 13 | - name: Fetch Spark Shuffle JAR 14 | fetch: src="{{ shuffle_jar_path_result.files[0].path }}" dest="{{ local_tmp_path }}/spark-yarn-shuffle.jar" flat=yes 15 | -------------------------------------------------------------------------------- /sites/mapr-mep-all.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: ext-mysql.yml 2 | - import_playbook: mapr-mep-kafka.yml 3 | - import_playbook: mapr-mep-drill.yml 4 | - import_playbook: mapr-mep-flume.yml 5 | - import_playbook: mapr-mep-hbase.yml 6 | - import_playbook: mapr-mep-hive.yml 7 | - import_playbook: mapr-mep-pig.yml 8 | - import_playbook: mapr-mep-impala.yml 9 | - import_playbook: mapr-mep-sqoop.yml 10 | - import_playbook: mapr-mep-httpfs.yml 11 | - import_playbook: mapr-mep-spark.yml 12 | - import_playbook: mapr-mep-oozie.yml 13 | - import_playbook: mapr-mep-filemigrate.yml 14 | - import_playbook: mapr-mep-data-access-gateway.yml 15 | - import_playbook: mapr-mep-hue.yml 16 | - import_playbook: mapr-mep-spyglass-metric.yml 17 | - import_playbook: mapr-mep-spyglass-log.yml -------------------------------------------------------------------------------- /roles/mapr-filemigrate-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install File Migrate Service 3 | - name: Install mapr-filemigrate 4 | package: name=mapr-filemigrate state=present 5 | register: filemigrate_result 6 | - name: Configure.sh -R 7 | shell: /opt/mapr/server/configure.sh -R 8 | when: filemigrate_result.changed 9 | - pause: seconds=10 10 | when: filemigrate_result.changed 11 | - name: Restart File Migrate Service 12 | shell: "maprcli node services -name filemigrate -action restart -nodes {{ ansible_fqdn }}" 13 | register: task_result 14 | until: task_result.rc == 0 15 | retries: 10 16 | delay: 10 17 | ignore_errors: yes 18 | environment: 19 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 20 | when: filemigrate_result.changed -------------------------------------------------------------------------------- /roles/mapr-configure-nativesecurity-secure-genkeys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure secure cluster 3 | - name: Enable custom security by creating /opt/mapr/conf/.customSecure 4 | file: path=/opt/mapr/conf/.customSecure state=touch mode=0644 5 | - name: Check if /opt/mapr/conf/cldb.key exists 6 | stat: path=/opt/mapr/conf/cldb.key 7 | register: cldb_key_status 8 | - name: Configure.sh to generate keys only run on first mapr-cldb node 9 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -Z {{ zookeeper_string }} -C {{ cldb_string }} -u {{ mapr_user }} -g {{ mapr_group }} -genkeys -secure {{ '-dare' if security_encryption_rest else '' }} -HS {{ historyserver_string }} {{ '--isvm' if virtual_machine else '' }}" 10 | when: not cldb_key_status.stat.exists 11 | -------------------------------------------------------------------------------- /roles/mapr-spark-thriftserver-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Spark SQL Server 3 | - name: Install mapr-spark-thriftserver 4 | package: name=mapr-spark-thriftserver state=present 5 | register: sparkthrift_result 6 | - name: Configure.sh -R 7 | shell: /opt/mapr/server/configure.sh -R 8 | when: sparkthrift_result.changed 9 | - pause: seconds=10 10 | when: sparkthrift_result.changed 11 | - name: Restart Spark Thrift Server 12 | shell: "maprcli node services -name spark-thriftserver -action restart -nodes {{ ansible_fqdn }}" 13 | register: task_result 14 | until: task_result.rc == 0 15 | retries: 10 16 | delay: 10 17 | ignore_errors: yes 18 | environment: 19 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 20 | when: sparkthrift_result.changed -------------------------------------------------------------------------------- /roles/common-sysctl/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Cannot use Ansible sysctl feature in docker, harder to test with molecule 3 | # https://docs.ansible.com/ansible/2.5/modules/sysctl_module.html?highlight=sysctl 4 | - name: Set vm.swappiness in /etc/sysctl.conf 5 | lineinfile: 6 | dest: /etc/sysctl.conf 7 | regexp: "{{item.regex}}" 8 | line: "{{item.line}}" 9 | create: true 10 | with_items: 11 | - regex: ^vm\.swappiness 12 | line: vm.swappiness = 1 13 | - regex: ^net\.ipv4\.tcp_retries2 14 | line: net.ipv4.tcp_retries2 = 5 15 | - regex: ^vm\.overcommit_memory 16 | line: vm.overcommit_memory = 0 17 | - regex: ^net\.ipv4\.tcp_fin_timeout 18 | line: net.ipv4.tcp_fin_timeout = 30 19 | register: sysctl_update 20 | notify: "Reload Sysctl" 21 | -------------------------------------------------------------------------------- /host_templates/hosts_dsr: -------------------------------------------------------------------------------- 1 | ###### These ones are not installed, just required for config generation ############ 2 | 3 | # Container Location DataBase 4 | [mapr-cldb] 5 | 10.0.0.153 ansible_user=ec2-user ansible_become=yes ansible_become_method=sudo 6 | 10.0.0.210 ansible_user=ec2-user ansible_become=yes ansible_become_method=sudo 7 | 10.0.0.213 ansible_user=ec2-user ansible_become=yes ansible_become_method=sudo 8 | 9 | [mapr-hive-server2] 10 | 10.0.0.153 ansible_user=ec2-user ansible_become=yes ansible_become_method=sudo 11 | 10.0.0.210 ansible_user=ec2-user ansible_become=yes ansible_become_method=sudo 12 | 13 | ###################################################################################### 14 | 15 | [mapr-dsr] 16 | 10.0.0.66 ansible_user=ec2-user ansible_become=yes ansible_become_method=sudo 17 | 18 | -------------------------------------------------------------------------------- /roles/mapr-configure-security-fetch-keys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure secure cluster 3 | - name: Copy MapR keys to local 4 | fetch: src="/opt/mapr/conf/{{ item }}" dest="{{ local_tmp_path }}/{{ item }}" flat=yes 5 | with_items: 6 | - maprserverticket 7 | - cldb.key 8 | - name: Copy MapR keystores and truststores to local 9 | fetch: src="/opt/mapr/conf/{{ item }}" dest="{{ local_tmp_path }}/{{ item }}" flat=yes 10 | with_items: 11 | - ssl_keystore 12 | - ssl_keystore.pem 13 | - ssl_keystore.p12 14 | - ssl_truststore 15 | - ssl_truststore.pem 16 | - ssl_truststore.p12 17 | when: not https_custom_certs_enabled 18 | - name: Copy DARE master key to local 19 | fetch: src=/opt/mapr/conf/dare.master.key dest="{{ local_tmp_path }}/dare.master.key" flat=yes 20 | when: security_encryption_rest 21 | -------------------------------------------------------------------------------- /roles/mapr-historyserver-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall History Server 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop Historyserver 7 | shell: "maprcli node services -name historyserver -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-historyserver 15 | package: name=mapr-historyserver state=absent 16 | register: historyserver_result 17 | - set_fact: historyserver_haschanged={{ historyserver_haschanged | int + 1 }} 18 | when: historyserver_result.changed -------------------------------------------------------------------------------- /roles/mapr-hue-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall Hue 3 | - name: Stop Hue 4 | shell: "maprcli node services -name hue -action stop -nodes {{ ansible_fqdn }}" 5 | register: maprcli_result 6 | failed_when: maprcli_result.rc > 1 7 | changed_when: maprcli_result.rc == 0 8 | environment: 9 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 10 | - name: Stop Hue-Livy 11 | shell: "maprcli node services -name livy -action stop -nodes {{ ansible_fqdn }}" 12 | register: maprcli_result 13 | failed_when: maprcli_result.rc > 1 14 | changed_when: maprcli_result.rc == 0 15 | environment: 16 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 17 | - name: Uninstall mapr-hue 18 | package: name="{{ item }}" state=absent 19 | with_items: 20 | - mapr-hue 21 | - mapr-hue-livy 22 | -------------------------------------------------------------------------------- /roles/common-prereq/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check operation system 3 | fail: msg="Only Suse, Debian, Ubuntu, RedHat and CentOS are supported" 4 | when: ansible_os_family not in ["Suse", "RedHat", "Debian"] 5 | 6 | - name: Ensure permissions for PAM config /etc/pam.d/system-auth-pc 7 | file: 8 | path: /etc/pam.d/system-auth-pc 9 | state: directory 10 | mode: 0644 11 | when: ansible_os_family == 'RedHat' 12 | 13 | - name: Retrieve info for zkdata directory 14 | stat: path=/opt/mapr/zkdata 15 | register: zkdataDir 16 | - name: Ensure owner mapr:mapr for /opt/mapr/zkdata if directory already exists 17 | file: 18 | path: /opt/mapr/zkdata 19 | state: directory 20 | owner: "{{ mapr_user }}" 21 | group: "{{ mapr_group }}" 22 | when: zkdataDir.stat.isdir is defined and zkdataDir.stat.isdir 23 | -------------------------------------------------------------------------------- /helper/haproxy/mapr-drill.cfg: -------------------------------------------------------------------------------- 1 | # Drill 2 | frontend mapr-drill-ui 3 | description "Drill UI" 4 | bind *:8047 5 | mode tcp 6 | default_backend mapr-drill-ui 7 | 8 | backend mapr-drill-ui 9 | description "Drill UI" 10 | mode tcp 11 | balance source 12 | server mapr-drill-ui1 10.0.0.63:8047 check 13 | server mapr-drill-ui2 10.0.0.110:8047 check 14 | server mapr-drill-ui3 10.0.0.232:8047 check 15 | 16 | frontend mapr-drill-sql 17 | description "Drill SQL" 18 | bind *:31010 19 | mode tcp 20 | default_backend mapr-drill-sql 21 | 22 | backend mapr-drill-sql 23 | description "Drill SQL" 24 | mode tcp 25 | balance source 26 | server mapr-drill-sql1 10.0.0.63:31010 check 27 | server mapr-drill-sql2 10.0.0.110:31010 check 28 | server mapr-drill-sql3 10.0.0.232:31010 check -------------------------------------------------------------------------------- /roles/common-misc/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install misc stuff - Midnight commander, wget, Python MySQL, openssl 3 | vars: 4 | packages_Suse: ['mc', 'wget', 'python-mysql', 'openssl'] 5 | packages_RedHat: ['mc', 'wget', 'MySQL-python', 6 | 'yum-utils', 'unzip', 'openssl', 7 | 'syslinux', 'sdparm', 'hdparm', 8 | 'redhat-lsb-core', 'lsof'] 9 | # syslinux-utils is for gethostip, libpython is required for collectd 10 | packages_Debian: ['mc', 11 | 'wget', 12 | 'python-mysqldb', 13 | 'openssl', 14 | 'syslinux-utils', 15 | 'libpython2.7'] 16 | package: name={{ item }} state=present 17 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 18 | -------------------------------------------------------------------------------- /roles/mapr-configure-disks/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Write disk tmp file 2 | shell: echo {{ disks }} | tr , '\n' > /tmp/disks.txt 3 | when: disks != "find_unused" 4 | - name: Copy find disks script 5 | copy: src="write_unused_disks.sh" dest="/tmp/write_unused_disks.sh" mode="0755" owner="{{ mapr_user }}" group="{{ mapr_group }}" 6 | when: disks == "find_unused" 7 | - name: Write unused disks tmp file 8 | shell: /tmp/write_unused_disks.sh 9 | when: disks == "find_unused" 10 | - name: Check if disks have been added 11 | stat: path=/opt/mapr/conf/disktab 12 | register: disks_added_status 13 | - name: Disk setup 14 | command: /opt/mapr/server/disksetup -F -W {{ disk_storage_pool_size }} /tmp/disks.txt 15 | when: not disks_added_status.stat.exists 16 | - set_fact: mfs_local_changes=1 17 | when: not disks_added_status.stat.exists -------------------------------------------------------------------------------- /roles/mapr-client-configure-nativesecurity-secure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure secure client 3 | # key store is required for Spark UI 4 | - name: Copy SSL keys to all nodes 5 | copy: src="{{ local_tmp_path }}/{{ item }}" dest="/opt/mapr/conf/{{ item }}" 6 | with_items: 7 | - ssl_truststore 8 | - ssl_keystore 9 | 10 | - name: Copy mapruserticket to all nodes (required for FUSE or nfsloopback) 11 | copy: src="{{ local_tmp_path }}/mapruserticket" dest="/opt/mapr/conf/mapruserticket" mode="0600" owner="{{ mapr_user }}" group="{{ mapr_group }}" 12 | # copy to other nodes 13 | # run configure on other nodes 14 | 15 | - name: Configure.sh for mapr-client - nativesecurity - secure 16 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -Z {{ zookeeper_string }} -C {{ cldb_string }} -u {{ mapr_user }} -g {{mapr_group}} -secure -c" -------------------------------------------------------------------------------- /roles/mapr-client-configure-nativesecurity-fetch/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Find Hadoop YARN server JARs 3 | find: paths="/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/yarn/" patterns="hadoop-yarn-server-*" file_type=file 4 | register: yarnserver_jar_path_result 5 | failed_when: yarnserver_jar_path_result.matched == 0 6 | - name: Transform to Hadoop YARN server JARs list 7 | set_fact: yarnserver_jars_list={{ yarnserver_jar_path_result.files | map(attribute='path') | list }} 8 | - debug: msg="Find Hadoop YARN server JARs {{ item | basename }}" 9 | with_items: "{{ yarnserver_jars_list }}" 10 | - name: Copy Jars to local, required for Hive Server and Spark 11 | fetch: src="{{ item }}" dest="{{ local_tmp_path }}/{{ item | basename }}" flat=yes 12 | with_items: "{{ yarnserver_jars_list }}" 13 | when: inventory_hostname == groups["mapr-cldb"][0] 14 | -------------------------------------------------------------------------------- /host_templates/hosts_anaconda: -------------------------------------------------------------------------------- 1 | [all:vars] 2 | ansible_user=ec2-user 3 | ansible_become=yes 4 | anaconda_install_binary=/mapr/demo.mapr.com/tools/anaconda/anaconda_installer/Anaconda3-5.0.1-Linux-x86_64.sh 5 | anaconda_install_path=/mapr/demo.mapr.com/tools/anaconda/anaconda 6 | anaconda_repo_path=file://mapr/demo.mapr.com/tools/anaconda/anaconda_repos/repo.continuum.io 7 | anaconda_custom_repo_path=/mapr/demo.mapr.com/tools/anaconda/anaconda_repos/repo.mapr.com 8 | mapr_user=mapr 9 | anaconda_env_name=spark_py35 10 | anaconda_python_version=3.5 11 | anaconda_packages=fbprophet colorama cycler dill fastparquet libxgboost numpy numba matplotlib mkl pandas patsy py pyodbc pytest pyparsing psutil scikit-learn scipy statsmodels thrift wheel jupyter 12 | 13 | [ext-anaconda-python] 14 | 10.0.0.70 15 | 16 | # Spark on YARN 17 | [mapr-spark-yarn] 18 | 10.0.0.70 19 | 10.0.0.229 20 | 10.0.0.246 21 | -------------------------------------------------------------------------------- /roles/ext-kerberos5-setup/templates/krb5.conf: -------------------------------------------------------------------------------- 1 | includedir /etc/krb5.conf.d/ 2 | 3 | [logging] 4 | default = FILE:/var/log/krb5libs.log 5 | kdc = FILE:/var/log/krb5kdc.log 6 | admin_server = FILE:/var/log/kadmind.log 7 | 8 | [libdefaults] 9 | default_realm = {{ mapr_kerberos_realm }} 10 | 11 | dns_lookup_realm = false 12 | ticket_lifetime = 7d 13 | renew_lifetime = 365d 14 | forwardable = true 15 | rdns = false 16 | # default_realm = EXAMPLE.COM 17 | # default_ccache_name = KEYRING:persistent:%{uid} 18 | 19 | [realms] 20 | {{ mapr_kerberos_realm }} = { 21 | kdc = {{ mapr_kerberos_admin_server }} 22 | admin_server = {{ mapr_kerberos_admin_server }} 23 | # default_domain = DEMO.MAPR.COM 24 | } 25 | 26 | [domain_realm] 27 | .{{ mapr_kerberos_domain }} = {{ mapr_kerberos_realm }} 28 | {{ mapr_kerberos_domain }} = {{ mapr_kerberos_realm }} 29 | -------------------------------------------------------------------------------- /roles/mapr-nfsloopback-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install NFS 3 | - name: Install mapr-nfs 4 | vars: 5 | packages_Suse: ['mapr-loopbacknfs', 'nfs-utils', 'rpcbind'] 6 | packages_RedHat: ['mapr-loopbacknfs', 'nfs-utils', 'rpcbind'] 7 | packages_Debian: ['mapr-loopbacknfs', 'nfs-common', 'rpcbind'] 8 | package: name={{ item }} state=present 9 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 10 | 11 | # Other actions 12 | - name: Copy mapr-clusters.conf 13 | copy: remote_src=True src=/opt/mapr/conf/mapr-clusters.conf dest=/usr/local/mapr-loopbacknfs/conf/mapr-clusters.conf 14 | - name: Start NFS Loopback 15 | service: name=mapr-loopbacknfs state=restarted enabled=yes 16 | - name: Create /mapr dir 17 | file: path=/mapr state=directory 18 | - name: Mount /mapr 19 | mount: name=/mapr state=mounted src=localhost:/mapr fstype=nfs opts=hard,nolock 20 | 21 | -------------------------------------------------------------------------------- /roles/mapr-impala-catalog-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install mapr-impala-catalog 3 | - name: Install mapr-impala-catalog 4 | package: name=mapr-impala-catalog state=present 5 | register: impala_result 6 | 7 | - find: paths="/opt/mapr/impala/" patterns="impala*" file_type=directory 8 | register: impala_path_result 9 | failed_when: impala_path_result.matched != 1 10 | - name: Configure.sh -R 11 | shell: /opt/mapr/server/configure.sh -R 12 | when: impala_result.changed 13 | - pause: seconds=10 14 | when: impala_result.changed 15 | - name: Restart Impala Catalog 16 | shell: "maprcli node services -name impalacatalog -action restart -nodes {{ ansible_fqdn }}" 17 | register: task_result 18 | until: task_result.rc == 0 19 | retries: 10 20 | delay: 10 21 | ignore_errors: yes 22 | environment: 23 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 24 | when: impala_result.changed -------------------------------------------------------------------------------- /sites/mapr-mep-sqoop.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Sqoop 3 | hosts: mapr-sqoop1 4 | roles: 5 | - ../roles/mapr-sqoop1-install 6 | - name: Uninstall Sqoop 7 | hosts: all:!mapr-sqoop1 8 | roles: 9 | - ../roles/mapr-sqoop1-uninstall 10 | - name: Set Sqoop 2 Server Vars 11 | hosts: all 12 | roles: 13 | - ../roles/mapr-sqoop2-server-vars 14 | - name: Install Sqoop 2 Server 15 | hosts: mapr-sqoop2-server 16 | roles: 17 | - ../roles/mapr-sqoop2-server-install 18 | - name: Uninstall Sqoop 2 Server 19 | hosts: all:!mapr-sqoop2-server 20 | roles: 21 | - ../roles/mapr-sqoop2-server-uninstall 22 | - name: Install Sqoop 2 Client 23 | hosts: mapr-sqoop2-client 24 | roles: 25 | - ../roles/mapr-sqoop2-client-install 26 | - name: Uninstall Sqoop 2 Client 27 | hosts: all:!mapr-sqoop2-client 28 | roles: 29 | - ../roles/mapr-sqoop2-client-uninstall -------------------------------------------------------------------------------- /helper/ssl/generate_certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CERT_PATH=~/testsslkeys 4 | CLUSTER_NAME=demo.mapr.com 5 | RESULT=$(ansible -i myhosts/hosts_3nodes -m shell -a 'echo "|host=$(hostname --ip-address)"' all) 6 | 7 | for i in $(echo $RESULT | tr "|" "\n") 8 | do 9 | if [[ "$i" =~ ^host ]] 10 | then 11 | export NODEIP=$(echo $i | cut -d "=" -f2-) 12 | export NODENUMBER=$(echo $NODEIP | cut -d "." -f4-) 13 | export NODENAME=ip-10-0-0-${NODENUMBER}.ps.mapr.com 14 | cat helper/ssl/cert_tpl | envsubst > $CERT_PATH/$NODENAME.conf 15 | openssl req -nodes -newkey rsa:4096 -keyout $CERT_PATH/key/$NODENAME.pem -out $CERT_PATH/$NODENAME.csr -config $CERT_PATH/$NODENAME.conf 16 | openssl x509 -req -in $CERT_PATH/$NODENAME.csr -CA $CERT_PATH/rootCA.pem -CAkey $CERT_PATH/rootCA-private.key -CAcreateserial -out $CERT_PATH/crt/$NODENAME.pem -days 1024 -sha256 17 | fi 18 | done 19 | -------------------------------------------------------------------------------- /roles/mapr-kibana-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Kibana 3 | - name: Install mapr-kibana 4 | package: name=mapr-kibana state=present 5 | #- name: Find Kibana Path 6 | # find: paths="/opt/mapr/kibana/" patterns="kibana*" file_type=directory 7 | # register: kibana_path_result 8 | # failed_when: kibana_path_result.matched != 1 9 | - name: Get Kibana version 10 | shell: cat /opt/mapr/kibana/kibanaversion 11 | register: version_output 12 | check_mode: no 13 | changed_when: no 14 | # Bugfix for Kibana custom secure 15 | #- name: Fix for Kibana custom secure 16 | # lineinfile: 17 | # dest: "/opt/mapr/kibana/kibana-{{ version_output.stdout }}/bin/configure.sh" 18 | # regexp: '\s*OPTS=\$\(getopt (.*)' 19 | # line: ' OPTS=$(getopt -a -o chk:ln:p:suvC:E:P:R -l help -l nodeCount: -l nodePort: -l ES: -l kibanaPort: -l secure -l customSecure -l unsecure -l password: -l EC: -l R -l verifySsl -- "$@")' 20 | -------------------------------------------------------------------------------- /roles/mapr-log-monitoring-configure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR log monitoring configure 3 | - name: Configure.sh 4 | # shell: /opt/mapr/server/configure.sh -R -ES {{ elasticsearch_string }} 5 | shell: "/opt/mapr/server/configure.sh -R -v -ES {{ elasticsearch_string }} -EPelasticsearch '-password {{ spyglass_log_password }}' -EPkibana '-password {{ spyglass_log_password }}' -EPfluentd '-password {{ spyglass_log_password }}'" 6 | environment: 7 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 8 | ES_ADMIN_PASSWORD: "{{ spyglass_log_password }}" 9 | - pause: seconds=10 10 | - name: Restart FluentD 11 | shell: "maprcli node services -name fluentd -action restart -nodes {{ ansible_fqdn }}" 12 | register: task_result 13 | until: task_result.rc == 0 14 | retries: 10 15 | delay: 10 16 | ignore_errors: yes 17 | environment: 18 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: minimal 3 | 4 | services: 5 | - docker 6 | 7 | before_install: 8 | - docker pull retr0h/molecule:latest 9 | 10 | script: 11 | - docker run --rm -it 12 | -v "$(pwd)":/tmp/$(basename "${PWD}"):ro 13 | -v /var/run/docker.sock:/var/run/docker.sock 14 | -w /tmp/$(basename "${PWD}") 15 | retr0h/molecule:latest 16 | molecule test -s lint 17 | - docker run --rm -it 18 | -v "$(pwd)":/tmp/$(basename "${PWD}"):ro 19 | -v /var/run/docker.sock:/var/run/docker.sock 20 | -w /tmp/$(basename "${PWD}") 21 | retr0h/molecule:latest 22 | sudo molecule test -s common 23 | - docker run --rm -it 24 | -v "$(pwd)":/tmp/$(basename "${PWD}"):ro 25 | -v /var/run/docker.sock:/var/run/docker.sock 26 | -w /tmp/$(basename "${PWD}") 27 | retr0h/molecule:latest 28 | sudo molecule test -s mapr-core-docker 29 | 30 | matrix: 31 | fast_finish: true 32 | -------------------------------------------------------------------------------- /roles/mapr-hbase-thrift-rest-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Stop HBase REST 3 | shell: "maprcli node services -name hbaserest -action stop -nodes {{ ansible_fqdn }}" 4 | register: maprcli_result 5 | failed_when: maprcli_result.rc > 1 6 | changed_when: maprcli_result.rc == 0 7 | environment: 8 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 9 | - name: Stop HBase Thrift 10 | shell: "maprcli node services -name hbasethrift -action stop -nodes {{ ansible_fqdn }}" 11 | register: maprcli_result 12 | failed_when: maprcli_result.rc > 1 13 | changed_when: maprcli_result.rc == 0 14 | environment: 15 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 16 | # Uninstall HBase REST 17 | - name: Uninstall mapr-hbase-rest 18 | package: name=mapr-hbase-rest state=absent 19 | # Uninstall HBase Thrift 20 | - name: Uninstall mapr-hbasethrift 21 | package: name=mapr-hbasethrift state=absent 22 | -------------------------------------------------------------------------------- /roles/mapr-impala-server-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install mapr-impala-catalog 3 | - name: Install mapr-impala-server 4 | package: name=mapr-impala-server state=present 5 | register: impala_result 6 | 7 | - find: paths="/opt/mapr/impala/" patterns="impala*" file_type=directory 8 | register: impala_path_result 9 | failed_when: impala_path_result.matched != 1 10 | 11 | # END secure 12 | - name: Configure.sh -R 13 | shell: /opt/mapr/server/configure.sh -R 14 | when: impala_result.changed 15 | - pause: seconds=10 16 | when: impala_result.changed 17 | - name: Restart Impala Server 18 | shell: "maprcli node services -name impalaserver -action restart -nodes {{ ansible_fqdn }}" 19 | register: task_result 20 | until: task_result.rc == 0 21 | retries: 10 22 | delay: 10 23 | ignore_errors: yes 24 | environment: 25 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 26 | when: impala_result.changed -------------------------------------------------------------------------------- /roles/mapr-impala-statestore-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install mapr-impala-statestore 3 | - name: Install mapr-impala-statestore 4 | package: name=mapr-impala-statestore state=present 5 | register: impala_result 6 | 7 | - find: paths="/opt/mapr/impala/" patterns="impala*" file_type=directory 8 | register: impala_path_result 9 | failed_when: impala_path_result.matched != 1 10 | 11 | - name: Configure.sh -R 12 | shell: /opt/mapr/server/configure.sh -R 13 | when: impala_result.changed 14 | - pause: seconds=10 15 | when: impala_result.changed 16 | - name: Restart Impala StateStore 17 | shell: "maprcli node services -name impalastore -action restart -nodes {{ ansible_fqdn }}" 18 | register: task_result 19 | until: task_result.rc == 0 20 | retries: 10 21 | delay: 10 22 | ignore_errors: yes 23 | environment: 24 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 25 | when: impala_result.changed -------------------------------------------------------------------------------- /roles/mapr-patch-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Check if patch is installed" 3 | shell: "rpm -qa | grep mapr-patch" 4 | register: patch_result # rc == 1, return code of grep, nothing found 5 | failed_when: false 6 | check_mode: no 7 | - name: Stop Warden 8 | service: name=mapr-warden state=stopped enabled=yes 9 | when: patch_result.rc == 1 10 | - name: Stop Zookeeper 11 | service: name=mapr-zookeeper state=stopped enabled=yes 12 | when: inventory_hostname in groups["mapr-zookeeper"] and patch_result.rc == 1 13 | # Install MapR Patches 14 | - name: Install mapr-patch 15 | package: name=mapr-patch state=present 16 | when: patch_result.rc == 1 17 | - name: Start Zookeeper 18 | service: name=mapr-zookeeper state=started enabled=yes 19 | when: inventory_hostname in groups["mapr-zookeeper"] patch_result.rc == 1 20 | - name: Start Warden 21 | service: name=mapr-warden state=started enabled=yes 22 | when: patch_result.rc == 1 23 | -------------------------------------------------------------------------------- /roles/mapr-nfs-v3-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Uninstall NFS 3 | - name: Check for mapr-clusters.conf 4 | stat: path=/opt/mapr/conf/mapr-clusters.conf 5 | register: clusters_conf 6 | - name: Stop NFS server 7 | shell: "maprcli node services -name nfs -action stop -nodes {{ ansible_fqdn }}" 8 | register: maprcli_result 9 | failed_when: maprcli_result.rc > 1 10 | changed_when: maprcli_result.rc == 0 11 | environment: 12 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 13 | when: clusters_conf.stat.exists 14 | - name: Uninstall mapr-nfs 15 | vars: 16 | packages_Suse: ['mapr-nfs'] 17 | packages_RedHat: ['mapr-nfs'] 18 | packages_Debian: ['mapr-nfs'] 19 | package: name="{{ item }}" state=absent 20 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 21 | 22 | # Other actions 23 | # conflicts with other posix clients 24 | #- name: Remove /mapr dir 25 | # file: path=/mapr state=absent 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /roles/mapr-patch-uninstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Check if patch is installed" 3 | shell: "rpm -qa | grep mapr-patch" 4 | register: patch_result # rc == 1, return code of grep, nothing found 5 | failed_when: false 6 | check_mode: no 7 | - name: Stop Warden 8 | service: name=mapr-warden state=stopped enabled=yes 9 | when: patch_result.rc == 0 10 | - name: Stop Zookeeper 11 | service: name=mapr-zookeeper state=stopped enabled=yes 12 | when: inventory_hostname in groups["mapr-zookeeper"] and patch_result.rc == 0 13 | # Uninstall MapR Patches 14 | - name: Uninstall mapr-patch 15 | package: name=mapr-patch state=absent 16 | when: patch_result.rc == 0 17 | - name: Start Zookeeper 18 | service: name=mapr-zookeeper state=started enabled=yes 19 | when: inventory_hostname in groups["mapr-zookeeper"] and patch_result.rc == 0 20 | - name: Start Warden 21 | service: name=mapr-warden state=started enabled=yes 22 | when: patch_result.rc == 0 23 | -------------------------------------------------------------------------------- /molecule/common/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: docker 6 | lint: 7 | name: yamllint 8 | options: 9 | config-file: molecule/common/yamllint.conf 10 | platforms: 11 | - name: instance 12 | image: centos:7 13 | command: /sbin/init 14 | volumes: 15 | - /sys/fs/cgroup:/sys/fs/cgroup:ro 16 | tmpfs: 17 | - /tmp 18 | - /run 19 | groups: 20 | - common 21 | provisioner: 22 | name: ansible 23 | inventory: 24 | links: 25 | group_vars: ../../group_vars/ 26 | lint: 27 | name: ansible-lint 28 | options: 29 | skip-tags: molecule-notest,notest,skip-docker 30 | scenario: 31 | name: common 32 | test_sequence: 33 | - lint 34 | - destroy 35 | - dependency 36 | - syntax 37 | - create 38 | - prepare 39 | - converge 40 | - idempotence 41 | - side_effect 42 | - verify 43 | - destroy 44 | verifier: 45 | name: testinfra 46 | lint: 47 | name: flake8 48 | -------------------------------------------------------------------------------- /roles/ext-sssd-ad-setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install SSSD 3 | vars: 4 | packages_RedHat: ['samba-winbind', 'sssd', 'samba-common', 'samba-client', 'authconfig', 'openldap-clients', 'oddjob', 'oddjob-mkhomedir', 'adcli'] 5 | package: name={{ item }} state=present 6 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 7 | - name: Configure sssd.conf 8 | template: src=sssd.conf dest=/etc/sssd/sssd.conf 9 | - name: Set /etc/sssd/sssd.conf to chmod 400 10 | file: path=/etc/sssd/sssd.conf mode=0400 11 | - name: Configure smb.conf 12 | template: src=smb.conf dest=/etc/samba/smb.conf 13 | - name: Join to Windows domain 14 | shell: echo '{{ mapr_kerberos_admin_bind_pw }}' | net ads join -U {{ mapr_kerberos_admin }} 15 | - name: Enable and start the SSSD service 16 | service: name=sssd state=restarted enabled=yes 17 | - name: Change PAM configuration with Redhat authconfig 18 | shell: authconfig --enablesssd --enablesssdauth --disablekrb5 --enablemkhomedir --update --enablecache 19 | -------------------------------------------------------------------------------- /roles/mapr-configure-vars-zk/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure unsecure cluster 3 | 4 | - name: Get Zookeeper FQDNs 5 | set_fact: zoo_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 6 | with_items: 7 | - "{{ groups['mapr-zookeeper'] }}" 8 | register: zookeeper_hosts 9 | - name: Make Zookeeper String 10 | set_fact: zookeeper_string={{ zookeeper_hosts.results | map(attribute='ansible_facts.zoo_item') | list | join(',')}} 11 | 12 | - name: Get Zookeeper FQDNs 13 | set_fact: zoo_item_withport={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}:5181 14 | with_items: 15 | - "{{ groups['mapr-zookeeper'] }}" 16 | register: zookeeper_hosts_withport 17 | - name: Make Zookeeper String 18 | set_fact: zookeeper_withport_string={{ zookeeper_hosts_withport.results | map(attribute='ansible_facts.zoo_item_withport') | list | join(',')}} 19 | 20 | - name: Debug Zookeeper 21 | debug: msg="{{ zookeeper_string }}" 22 | 23 | 24 | -------------------------------------------------------------------------------- /molecule/lint/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | # Molecule managed 2 | 3 | {% if item.registry is defined %} 4 | FROM {{ item.registry.url }}/{{ item.image }} 5 | {% else %} 6 | FROM {{ item.image }} 7 | {% endif %} 8 | 9 | RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ 10 | elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ 11 | elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ 12 | elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ 13 | elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ 14 | elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi 15 | -------------------------------------------------------------------------------- /roles/mapr-kibana-install-secure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #- name: Find Kibana Path 3 | # find: paths="/opt/mapr/kibana/" patterns="kibana*" file_type=directory 4 | # register: kibana_path_result 5 | # failed_when: kibana_path_result.matched != 1 6 | - name: Get Kibana version 7 | shell: cat /opt/mapr/kibana/kibanaversion 8 | register: version_output 9 | check_mode: no 10 | changed_when: no 11 | - name: Copy Kibana files for Security 12 | copy: src="{{ local_tmp_path }}/mapr_spyglass/es/{{ item.src }}" dest="/opt/mapr/kibana/kibana-{{ version_output.stdout }}/{{ item.dst }}" owner="{{ mapr_user }}" group="{{ mapr_group }}" mode="{{ item.mode }}" 13 | with_items: 14 | - src: sg/ca/es-root-ca.pem 15 | dst: config/ca/ 16 | mode: "0600" 17 | - src: sg/kibanaserver-usr-clientCombo.pem 18 | dst: config/certs/ 19 | mode: "0600" 20 | - src: sg/kibanaserver-usr-signed.pem 21 | dst: config/certs/ 22 | mode: "0600" 23 | - src: sg/kibanaserver-usr-private-key.pem 24 | dst: config/certs/ 25 | mode: "0600" 26 | -------------------------------------------------------------------------------- /sites/mapr-mep-kafka.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Kafka 3 | hosts: mapr-kafka 4 | roles: 5 | - ../roles/mapr-kafka-install 6 | - name: Uninstall Kafka 7 | hosts: all:!mapr-kafka 8 | roles: 9 | - ../roles/mapr-kafka-uninstall 10 | - name: Install Kafka Connect JDBC 11 | hosts: mapr-kafka-connect-jdbc 12 | roles: 13 | - ../roles/mapr-kafka-connect-jdbc-install 14 | - name: Uninstall Kafka Connect JDBC 15 | hosts: all:!mapr-kafka-connect-jdbc 16 | roles: 17 | - ../roles/mapr-kafka-connect-jdbc-uninstall 18 | - name: Install Kafka REST 19 | hosts: mapr-kafka-rest 20 | roles: 21 | - ../roles/mapr-kafka-rest-install 22 | - name: Uninstall Kafka REST 23 | hosts: all:!mapr-kafka-rest 24 | roles: 25 | - ../roles/mapr-kafka-rest-uninstall 26 | - name: Install Kafka KSQL 27 | hosts: mapr-kafka-ksql 28 | roles: 29 | - ../roles/mapr-kafka-ksql-install 30 | - name: Uninstall Kafka KSQL 31 | hosts: all:!mapr-kafka-ksql 32 | roles: 33 | - ../roles/mapr-kafka-ksql-uninstall -------------------------------------------------------------------------------- /roles/mapr-configure-kerberos-genkeys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure secure cluster 3 | - name: Enable customer security by creating /opt/mapr/conf/.customSecure 4 | file: path=/opt/mapr/conf/.customSecure state=touch mode=0644 5 | - name: Copy mapr.keytab to host 6 | copy: src="{{ mapr_kerberos_local_keytabs_dir }}/{{ ansible_fqdn }}.keytab" dest="/opt/mapr/conf/mapr.keytab" group="{{ mapr_group }}" owner="{{ mapr_user }}" mode=0600 7 | - name: Check if /opt/mapr/conf/cldb.key exists 8 | stat: path=/opt/mapr/conf/cldb.key 9 | register: cldb_key_status 10 | - name: Configure.sh to generate keys just run on first mapr-core node 11 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -Z {{ zookeeper_string }} -C {{ cldb_string }} -u {{ mapr_user }} -g {{ mapr_group }} -genkeys -secure {{ '-dare' if security_encryption_rest else '' }} -K -P '{{ mapr_user_cldb_principal }}/{{ cluster_name }}@{{ mapr_kerberos_realm }}' -HS {{ historyserver_string }} {{ '--isvm' if virtual_machine else '' }}" 12 | when: not cldb_key_status.stat.exists 13 | -------------------------------------------------------------------------------- /roles/mapr-client-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Client 3 | 4 | - name: Install mapr-client 5 | package: name=mapr-client state=present 6 | 7 | # Required for Spark 8 | - name: YARN proxy Jar 9 | copy: src="{{ local_tmp_path }}/{{ item | basename }}" dest="/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/yarn/" 10 | with_items: "{{ hostvars[groups['mapr-cldb'][0]].yarnserver_jars_list }}" # set in mapr-client-configure-nativesecurity-fetch 11 | 12 | #- name: YARN proxy Jar 13 | # copy: src=/tmp/hadoop-yarn-server-web-proxy-{{ hadoop_version }}-{{ mapr_build }}.jar dest=/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/yarn/ 14 | ## Required for Hive server 15 | #- name: YARN RM Jar 16 | # copy: src=/tmp/hadoop-yarn-server-common-{{ hadoop_version }}-{{ mapr_build }}.jar dest=/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/yarn/ 17 | #- name: YARN RM Jar 18 | # copy: src=/tmp/hadoop-yarn-server-resourcemanager-{{ hadoop_version }}-{{ mapr_build }}.jar dest=/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/yarn/ 19 | 20 | -------------------------------------------------------------------------------- /molecule/common/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | # Molecule managed 2 | 3 | {% if item.registry is defined %} 4 | FROM {{ item.registry.url }}/{{ item.image }} 5 | {% else %} 6 | FROM {{ item.image }} 7 | {% endif %} 8 | 9 | RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ 10 | elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ 11 | elif [ $(command -v yum) ]; then yum makecache fast && yum update -y && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ 12 | elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ 13 | elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ 14 | elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi 15 | -------------------------------------------------------------------------------- /host_templates/hosts_kerberos: -------------------------------------------------------------------------------- 1 | # Kerberos setup template 2 | [all:vars] 3 | cluster_name=demo.mapr.com 4 | mapr_kerberos_realm=PS.MAPR.COM 5 | mapr_kerberos_ad_workgroup=PS 6 | mapr_kerberos_admin_server=server01.ps.mapr.com 7 | mapr_kerberos_kdc=server01.ps.mapr.com 8 | mapr_kerberos_domain=ps.mapr.com 9 | mapr_kerberos_local_keytabs_dir=/mnt/c/Projects/keytabs/ 10 | mapr_kerberos_admin=Administrator 11 | mapr_kerberos_admin_bind_dn="CN=Administrator,CN=Users,DC=ps,DC=mapr,DC=com" 12 | mapr_kerberos_admin_bind_pw="mapr123" 13 | mapr_kerberos_ad_object_category="CN=Person,CN=Schema,CN=Configuration,DC=ps,DC=mapr,DC=com" 14 | mapr_kerberos_ad_user_cn="CN=Users,DC=ps,DC=mapr,DC=com" 15 | mapr_kerberos_keytab_passwd=mapr 16 | # This is a workaround, since CLDB uses always mapr-user 'mapr' in CLDB principal, don't change 17 | mapr_user_cldb_principal=mapr 18 | mapr_user=maprd 19 | mapr_group=maprdgrp 20 | ansible_user=ec2-user 21 | ansible_become=yes 22 | 23 | [ext-kerberos] 24 | node1 ansible_host=10.0.0.85 25 | node2 ansible_host=10.0.0.91 26 | node3 ansible_host=10.0.0.174 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /roles/ext-rlang/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Debian / Ubuntu 3 | - name: Install pre-required packages for R 4 | apt: name={{ item }} state=present update_cache=yes 5 | with_items: 6 | - unixodbc 7 | - unixodbc-dev 8 | when: ansible_os_family == 'Debian' 9 | - name: Install R Lang 10 | apt: name={{ item }} state=present update_cache=yes 11 | with_items: 12 | - r-base 13 | - r-base-core 14 | when: ansible_os_family == 'Debian' 15 | 16 | - name: Create lib folder in MapR-FS 17 | file: path="/mapr/{{ cluster_name }}/R-Shared/shared-libs" state=directory mode=0777 18 | - name: Find Hadoop Streaming Jar Path 19 | find: paths="/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/share/hadoop/tools/lib/" patterns="hadoop-streaming-*" file_type=file 20 | register: streaming_jar_path_result 21 | failed_when: streaming_jar_path_result.matched != 1 22 | - debug: msg="Streaming JAR Path {{ streaming_jar_path_result.files[0].path }}" 23 | # copy Renviron 24 | - name: Copy Renviron file 25 | template: src=Renviron.ubuntu dest=/usr/lib/R/etc/Renviron 26 | when: ansible_os_family == 'Debian' 27 | -------------------------------------------------------------------------------- /molecule/mapr-core-docker/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | # Molecule managed 2 | 3 | {% if item.registry is defined %} 4 | FROM {{ item.registry.url }}/{{ item.image }} 5 | {% else %} 6 | FROM {{ item.image }} 7 | {% endif %} 8 | 9 | RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ 10 | elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ 11 | elif [ $(command -v yum) ]; then yum makecache fast && yum update -y && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ 12 | elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ 13 | elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ 14 | elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi 15 | -------------------------------------------------------------------------------- /roles/mapr-posix-client-basic-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install mapr-posix-client-basic 3 | - name: Install mapr-posix-client-basic 4 | package: name={{ item }} state=present 5 | with_items: 6 | - mapr-posix-client-basic 7 | - rpcbind 8 | #- name: Check if maprfuseticket exists 9 | # stat: path=/opt/mapr/conf/maprfuseticket 10 | # register: fuse_maprticket 11 | #- name: Generate FUSE serviceticket with impersonation 12 | # shell: "maprlogin generateticket -type servicewithimpersonation -user {{ mapr_user }} -duration 100000:0:0 -out /opt/mapr/conf/maprfuseticket" 13 | # environment: 14 | # MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 15 | # when: not fuse_maprticket.stat.exists 16 | - name: Set /opt/mapr/conf/mapruserticket in fuse.conf 17 | lineinfile: dest=/opt/mapr/conf/fuse.conf regexp="^fuse\.ticketfile\.location" line="fuse.ticketfile.location=/opt/mapr/conf/mapruserticket" 18 | # Other actions 19 | - name: Create /mapr dir 20 | file: path=/mapr state=directory 21 | - name: Start the mapr-posix-client-basic service 22 | service: name=mapr-posix-client-basic state=restarted enabled=yes -------------------------------------------------------------------------------- /roles/ext-anaconda-python/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Anaconda 3 | - name: Install BZip2 4 | package: name=bzip2 state=present 5 | - name: Override Anaconda default config to access local repository. 6 | template: src=condarc dest=~/.condarc 7 | # become: true 8 | # become_user: "{{ mapr_user }}" 9 | - name: Install Anaconda (Python) 10 | shell: "{{ anaconda_install_binary }} -u -b -p {{ anaconda_install_path }}" 11 | # become: true 12 | # become_user: "{{ mapr_user }}" 13 | - name: Build index for custom repo 14 | shell: "{{ anaconda_install_path }}/bin/conda index {{ anaconda_custom_repo_path }}/linux-64/" 15 | # become: true 16 | # become_user: "{{ mapr_user }}" 17 | - name: Delete Python environment for re-creation 18 | file: dest="{{ anaconda_install_path }}/envs/{{ anaconda_env_name }}" state=absent 19 | # become: true 20 | # become_user: "{{ mapr_user }}" 21 | - name: Create Python environment 22 | shell: "{{ anaconda_install_path }}/bin/conda create --offline --name {{ anaconda_env_name }} python={{ anaconda_python_version }} {{ anaconda_packages }}" 23 | # become: true 24 | # become_user: "{{ mapr_user }}" 25 | -------------------------------------------------------------------------------- /roles/mapr-posix-client-platinum-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install mapr-posix-client-platinum 3 | - name: Install mapr-posix-client-platinum 4 | package: name={{ item }} state=present 5 | with_items: 6 | - mapr-posix-client-platinum 7 | - rpcbind 8 | #- name: Check if maprfuseticket exists 9 | # stat: path=/opt/mapr/conf/maprfuseticket 10 | # register: fuse_maprticket 11 | #- name: Generate FUSE serviceticket with impersonation 12 | # shell: "maprlogin generateticket -type servicewithimpersonation -user {{ mapr_user }} -duration 100000:0:0 -out /opt/mapr/conf/maprfuseticket" 13 | # environment: 14 | # MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 15 | # when: not fuse_maprticket.stat.exists 16 | - name: Set /opt/mapr/conf/mapruserticket in fuse.conf 17 | lineinfile: dest=/opt/mapr/conf/fuse.conf regexp="^fuse\.ticketfile\.location" line="fuse.ticketfile.location=/opt/mapr/conf/mapruserticket" 18 | # Other actions 19 | - name: Create /mapr dir 20 | file: path=/mapr state=directory 21 | - name: Start the mapr-posix-client-platinum service 22 | service: name=mapr-posix-client-platinum state=restarted enabled=yes -------------------------------------------------------------------------------- /molecule/mapr-core-docker/plays/prepare-one-node-with-mep.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # - import_playbook: ext-mysql.yml 3 | - name: Install MySQL 4 | hosts: all 5 | become: true 6 | roles: 7 | - ../../roles/ext-mysql-install 8 | 9 | 10 | # - import_playbook: ../../sites/mapr-mep-spark.yml 11 | - name: Install Spark on YARN 12 | hosts: all 13 | become: true 14 | roles: 15 | - ../../roles/mapr-spark-yarn-install 16 | 17 | - name: Install Spark History Server 18 | hosts: all 19 | become: true 20 | roles: 21 | - ../../roles/mapr-spark-historyserver-install 22 | 23 | # - import_playbook: mapr-mep-oozie.yml 24 | - name: Setup Zookeeper String 25 | hosts: all 26 | become: true 27 | roles: 28 | - ../../roles/mapr-configure-vars-zk 29 | - name: Set Oozie Vars 30 | hosts: all 31 | become: true 32 | vars: 33 | num_mysql_hosts: "{{ groups['ext-mysql']|length }}" 34 | roles: 35 | - {role: ../../roles/mapr-oozie-mysql, when: oozie_db_type == 'mysql' and oozie_db_mysql_create_schema_and_user and num_mysql_hosts != '0'} 36 | - name: Install Oozie 37 | hosts: all 38 | become: true 39 | roles: 40 | - ../../roles/mapr-oozie-install 41 | -------------------------------------------------------------------------------- /sites/mapr-mep-spyglass-metric.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install MapR collectd 3 | hosts: mapr-collectd 4 | roles: 5 | - ../roles/mapr-collectd-install 6 | - name: Uninstall MapR collectd 7 | hosts: all:!mapr-collectd 8 | roles: 9 | - ../roles/mapr-collectd-uninstall 10 | - name: Install Custom OpenTSDB 11 | hosts: mapr-opentsdb-custom 12 | roles: 13 | - ../roles/mapr-opentsdb-custom-install 14 | - name: Install MapR OpenTSDB 15 | hosts: mapr-opentsdb 16 | roles: 17 | - ../roles/mapr-opentsdb-install 18 | - name: Uninstall MapR OpenTSDB 19 | hosts: all:!mapr-opentsdb:!mapr-opentsdb-custom 20 | roles: 21 | - ../roles/mapr-opentsdb-uninstall 22 | - name: Install MapR Grafana 23 | hosts: mapr-grafana 24 | roles: 25 | - ../roles/mapr-grafana-install 26 | - { role: ../roles/mapr-grafana-customssl, when: https_custom_certs_enabled } 27 | - name: Uninstall MapR Grafana 28 | hosts: all:!mapr-grafana 29 | roles: 30 | - ../roles/mapr-grafana-uninstall 31 | - name: MapR Configure Metrics Monitoring 32 | hosts: mapr-collectd 33 | roles: 34 | - ../roles/mapr-metric-monitoring-configure 35 | -------------------------------------------------------------------------------- /roles/ext-mysql-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MySQL 3 | - name: Install MySQL / MariaDB 4 | vars: 5 | packages_Suse: ['mariadb', 'mariadb-client', 'python-mysql'] 6 | packages_RedHat: ['mariadb-server', 'mariadb', 'MySQL-python'] 7 | packages_Debian: ['mysql-server', 'mysql-client', 'python-mysqldb'] 8 | package: name={{ item }} state=present 9 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 10 | register: mysql_result 11 | - name: Remove localhost binding 12 | vars: 13 | os_major_version: "{{ '_16' if ansible_os_family == 'Debian' and ansible_distribution_major_version == '16' else '' }}" 14 | mysqlpath_Suse: /etc/my.cnf 15 | mysqlpath_RedHat: /etc/my.cnf 16 | mysqlpath_Debian: /etc/mysql/my.cnf 17 | mysqlpath_Debian_16: /etc/mysql/mysql.conf.d/mysqld.cnf 18 | lineinfile: dest={{ vars['mysqlpath_' + ansible_os_family + os_major_version] }} regexp="^bind-address" state=absent 19 | 20 | - name: Start the mysqld service 21 | vars: 22 | service_Suse: mysql 23 | service_RedHat: mariadb 24 | service_Debian: mysql 25 | service: name={{ vars['service_' + ansible_os_family] }} state=restarted enabled=yes 26 | when: mysql_result.changed 27 | 28 | -------------------------------------------------------------------------------- /roles/mapr-metric-monitoring-configure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR OpenTSDB configure 3 | 4 | - name: Get OpenTSDB FQDNs 5 | set_fact: opentsdb_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 6 | with_items: 7 | - "{{ groups['mapr-opentsdb'] }}" 8 | register: opentsdb_hosts 9 | - name: Make OpenTSDB String 10 | set_fact: opentsdb_string={{ opentsdb_hosts.results | map(attribute='ansible_facts.opentsdb_item') | list | join(',')}} 11 | - name: Configure.sh - secure 12 | shell: /opt/mapr/server/configure.sh -R -OT {{ opentsdb_string }} -EPgrafana '-password {{ spyglass_grafana_password }}' -EPcollectd {{ "-minimal" if spyglass_metrics_collect == "minimal" else "-all" }} 13 | environment: 14 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 15 | # Workaround for custom security bug 16 | GRAFANA_ADMIN_ID: "{{ mapr_user }}" 17 | GRAFANA_ADMIN_PASSWORD: mapr 18 | # when: mapr_security != 'none' 19 | 20 | # - name: Configure.sh 21 | # shell: /opt/mapr/server/configure.sh -R -OT {{ opentsdb_string }} -EPgrafana '-password {{ spyglass_grafana_password }}' 22 | # environment: 23 | # MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 24 | # when: mapr_security == 'none' 25 | -------------------------------------------------------------------------------- /sites/mapr-mep-impala.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install Impala Vars 3 | hosts: all 4 | roles: 5 | - ../roles/mapr-impala-vars 6 | - name: Install Impala 7 | hosts: mapr-impala-catalog:mapr-impala-statestore:mapr-impala-server 8 | roles: 9 | - ../roles/mapr-impala-core-install 10 | - name: Install Impala Catalog 11 | hosts: mapr-impala-catalog 12 | roles: 13 | - ../roles/mapr-impala-catalog-install 14 | - name: Uninstall Impala Catalog 15 | hosts: all:!mapr-impala-catalog 16 | roles: 17 | - ../roles/mapr-impala-catalog-uninstall 18 | - name: Install Impala StateStore 19 | hosts: mapr-impala-statestore 20 | roles: 21 | - ../roles/mapr-impala-statestore-install 22 | - name: Uninstall Impala StateStore 23 | hosts: all:!mapr-impala-statestore 24 | roles: 25 | - ../roles/mapr-impala-statestore-uninstall 26 | - name: Install Impala Server 27 | hosts: mapr-impala-server 28 | roles: 29 | - ../roles/mapr-impala-server-install 30 | - name: Uninstall Impala Server 31 | hosts: all:!mapr-impala-server 32 | roles: 33 | - ../roles/mapr-impala-server-uninstall 34 | - name: Uninstall Impala 35 | hosts: all:!(mapr-impala-catalog:&mapr-impala-statestore:&mapr-impala-server) 36 | roles: 37 | - ../roles/mapr-impala-core-uninstall -------------------------------------------------------------------------------- /roles/mapr-core-install/templates/sample-timelineserver.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Indicate to clients whether Timeline service is enabled or not. If enabled, the TimelineClient library used by end-users will post entities and events to the Timeline server. 5 | yarn.timeline-service.enabled 6 | true 7 | 8 | 9 | yarn.timeline-service.hostname 10 | ip-10-0-0-114.ps.mapr.com 11 | 12 | 13 | The setting that controls whether yarn system metrics is published on the timeline server or not by RM. 14 | yarn.resourcemanager.system-metrics-publisher.enabled 15 | true 16 | 17 | 18 | yarn.timeline-service.http-cross-origin.enabled 19 | true 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | yarn.timeline-service.http-authentication.type 29 | com.mapr.security.maprauth.MaprDelegationTokenAuthenticationHandler 30 | 31 | -------------------------------------------------------------------------------- /roles/mapr-hive-mysql/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get MySQL Host 3 | set_fact: mysql_item="{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}" 4 | with_items: 5 | - "{{ groups['ext-mysql'] }}" 6 | register: mysql_hosts 7 | - name: Make MySQL Host String 8 | set_fact: hive_db_host="{{ mysql_hosts.results | map(attribute='ansible_facts.mysql_item') | list | join(',')}}" 9 | - name: Create hive user localhost 10 | mysql_user: name="{{ hive_db_user }}" password="{{ hive_db_password }}" host=localhost priv="*.*:ALL,GRANT" state=present 11 | when: inventory_hostname == groups["ext-mysql"][0] 12 | - name: "Create hive user {{ ansible_fqdn }}" 13 | mysql_user: name="{{ hive_db_user }}" password="{{ hive_db_password }}" host="{{ ansible_fqdn }}" priv="*.*:ALL,GRANT" state=present 14 | when: inventory_hostname == groups["ext-mysql"][0] 15 | - name: Create hive user % 16 | mysql_user: name="{{ hive_db_user }}" password="{{ hive_db_password }}" host=% priv="*.*:ALL,GRANT" state=present 17 | when: inventory_hostname == groups["ext-mysql"][0] 18 | - name: Create hive mysql db 19 | mysql_db: name="{{ hive_db_schema }}" state=present login_host="{{ hive_db_host }}" login_user="{{ hive_db_user }}" login_password="{{ hive_db_password }}" 20 | when: inventory_hostname == groups["ext-mysql"][0] -------------------------------------------------------------------------------- /helper/create-user-ace.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | vars: 3 | mapr_passwd: $6$X4.bqoTK$RTAwnhg2BG/y.3Dugw9Vc31ZdXFsZAitwKtKuu3HgKSNutcQmZ7zLy4cXasivlv2gKD8QGouAjND5fU7H.HDW0 4 | 5 | tasks: 6 | - name: Create financegroup Group 7 | group: name=financegroup gid=5501 state=present 8 | - name: Create opsgroup Group 9 | group: name=opsgroup gid=5502 state=present 10 | - name: Create bobgroup Group 11 | group: name=bobgroup gid=5503 state=present 12 | 13 | - name: Create finance1 User 14 | user: name=finance1 uid=6001 groups=financegroup password={{ mapr_passwd }} createhome=yes state=present 15 | 16 | - name: Create finance2 User 17 | user: name=finance2 uid=6002 groups=financegroup password={{ mapr_passwd }} createhome=yes state=present 18 | 19 | - name: Create ops1 User 20 | user: name=ops1 uid=6003 groups=opsgroup password={{ mapr_passwd }} createhome=yes state=present 21 | 22 | - name: Create bob User 23 | user: name=bob uid=6004 groups=bobgroup,financegroup password={{ mapr_passwd }} createhome=yes state=present 24 | 25 | - name: Create bob User 26 | user: name=sam uid=6005 password={{ mapr_passwd }} createhome=yes state=present 27 | 28 | - name: Create test1 29 | user: name=test1 uid=300 password={{ mapr_passwd }} createhome=yes state=present 30 | -------------------------------------------------------------------------------- /roles/mapr-fluentd-install-secure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Is this a bug? It is not documented to do this. 3 | - name: Get Fluentd version 4 | shell: cat /opt/mapr/fluentd/fluentdversion 5 | register: version_output 6 | check_mode: no 7 | changed_when: no 8 | - name: Create CA dir 9 | file: path="/opt/mapr/fluentd/fluentd-{{ version_output.stdout }}/etc/fluentd/ca/" state=directory 10 | - name: Copy Kibana files for Security 11 | copy: src="{{ local_tmp_path }}/mapr_spyglass/es/{{ item.src }}" dest="/opt/mapr/fluentd/fluentd-{{ version_output.stdout }}/{{ item.dst }}" owner="{{ mapr_user }}" group="{{ mapr_group }}" mode="{{ item.mode }}" 12 | with_items: 13 | - src: sg/ca/es-root-ca.pem 14 | dst: etc/fluentd/ca/ 15 | mode: "0600" 16 | - src: sg/fluentd-usr-signed.pem 17 | dst: etc/fluentd/certs/ 18 | mode: "0600" 19 | - src: sg/fluentd-usr-private-key.pem 20 | dst: etc/fluentd/certs/ 21 | mode: "0600" 22 | - src: sg/fluentd-usr-clientCombo.pem 23 | dst: etc/fluentd/certs/ 24 | mode: "0600" 25 | # Workaround: Disable SSL cert check. For some reason validation always fails because of hostname. 26 | - name: Workaround - Disable SSL cert check. 27 | lineinfile: dest="/opt/mapr/fluentd/fluentd-{{ version_output.stdout }}/etc/fluentd/es_config.conf" regexp='(.*)ssl_verify(.*)' line=" ssl_verify false" 28 | -------------------------------------------------------------------------------- /roles/mapr-hue-mysql/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Get MySQL Host 4 | set_fact: mysql_item="{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}" 5 | with_items: 6 | - "{{ groups['ext-mysql'] }}" 7 | register: mysql_hosts 8 | - name: Make MySQL Host String 9 | set_fact: hue_db_host="{{ mysql_hosts.results | map(attribute='ansible_facts.mysql_item') | list | join(',')}}" 10 | when: hue_db_host is not defined 11 | - name: Create hue user localhost 12 | mysql_user: name="{{ hue_db_user }}" password="{{ hue_db_password }}" host=localhost priv="*.*:ALL,GRANT" state=present 13 | when: inventory_hostname == groups["ext-mysql"][0] 14 | - name: Create hue user {{ ansible_fqdn }} 15 | mysql_user: name="{{ hue_db_user }}" password="{{ hue_db_password }}" host="{{ ansible_fqdn }}" priv="*.*:ALL,GRANT" state=present 16 | when: inventory_hostname == groups["ext-mysql"][0] 17 | - name: Create hue user % 18 | mysql_user: name="{{ hue_db_user }}" password="{{ hue_db_password }}" host=% priv="*.*:ALL,GRANT" state=present 19 | when: inventory_hostname == groups["ext-mysql"][0] 20 | - name: Create Hue mysql db 21 | mysql_db: name="{{ hue_db_schema }}" state=present login_host="{{ hue_db_host }}" login_user="{{ hue_db_user }}" login_password="{{ hue_db_password }}" 22 | when: inventory_hostname == groups["ext-mysql"][0] 23 | -------------------------------------------------------------------------------- /roles/mapr-impala-core-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install mapr-impala 3 | - name: Install mapr-impala 4 | package: name=mapr-impala state=present 5 | 6 | - find: paths="/opt/mapr/impala/" patterns="impala*" file_type=directory 7 | register: impala_path_result 8 | failed_when: impala_path_result.matched != 1 9 | 10 | - name: Copy mapr.keytab to host 11 | copy: src="{{ mapr_kerberos_local_keytabs_dir }}/{{ ansible_fqdn }}.keytab" dest=/opt/mapr/conf/mapr.keytab group="{{ mapr_group }}" owner="{{ mapr_user }}" mode=0600 12 | when: impala_security == 'kerberos' 13 | 14 | - name: Configure env.sh 15 | template: src=env.sh dest="{{ impala_path_result.files[0].path }}/conf/env.sh" 16 | 17 | - name: Copy core-site.xml to Impala 18 | copy: remote_src=true src="/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/etc/hadoop/core-site.xml" dest="{{ impala_path_result.files[0].path }}/conf/" mode=0644 owner="{{ mapr_user }}" group="{{ mapr_group }}" 19 | 20 | - name: Get Hive version 21 | shell: cat /opt/mapr/hive/hiveversion 22 | register: version_output 23 | check_mode: no 24 | changed_when: no 25 | 26 | - name: Copy hive-site.xml to Impala 27 | copy: remote_src=true src="/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/hive-site.xml" dest="{{ impala_path_result.files[0].path }}/conf/" mode=0644 owner="{{ mapr_user }}" group="{{ mapr_group }}" 28 | -------------------------------------------------------------------------------- /roles/mapr-impala-vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get Impala Catalog 3 | set_fact: impala_catalog_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 4 | with_items: 5 | - "{{ groups['mapr-impala-catalog'] }}" 6 | register: impala_catalog_hosts 7 | - name: Make Impala Catalog String 8 | set_fact: impala_catalog_string={{ impala_catalog_hosts.results | map(attribute='ansible_facts.impala_catalog_item') | list | join(',')}} 9 | 10 | - name: Get Impala StateStore 11 | set_fact: impala_statestore_item={{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }} 12 | with_items: 13 | - "{{ groups['mapr-impala-statestore'] }}" 14 | register: impala_statestore_hosts 15 | - name: Make Impala StateStore String 16 | set_fact: impala_statestore_string={{ impala_statestore_hosts.results | map(attribute='ansible_facts.impala_statestore_item') | list | join(',')}} 17 | - name: Get Hive MetaStore server 18 | set_fact: metastore_item=thrift://{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}:9083 19 | with_items: 20 | - "{{ groups['mapr-hive-metastore'] }}" 21 | register: impala_hive_metastore_hosts 22 | - name: Make Hive MetaStore String 23 | set_fact: impala_hive_metastore_string={{ impala_hive_metastore_hosts.results | map(attribute='ansible_facts.metastore_item') | list | join(',')}} -------------------------------------------------------------------------------- /host_templates/hosts_client: -------------------------------------------------------------------------------- 1 | # If you do not want to install a component leave the block empty 2 | 3 | [all:vars] 4 | ansible_user=ec2-user 5 | ansible_become=yes 6 | 7 | ###### These ones are not installed, just required for config generation ############ 8 | 9 | # Zookeeper 10 | [mapr-zookeeper] 11 | 10.0.0.242 12 | 10.0.0.243 13 | 10.0.0.244 14 | 15 | # Container Location DataBase 16 | [mapr-cldb] 17 | 10.0.0.242 18 | 10.0.0.243 19 | 10.0.0.244 20 | 21 | # Hive Meta Store 22 | [mapr-hive-metastore] 23 | 10.0.0.243 24 | 25 | ###################################################################################### 26 | 27 | # sets up ntp, rpcbind, Java and mapr default user with standard password mapr123 28 | [common] 29 | 10.0.0.132 30 | 31 | [mapr-client] 32 | 10.0.0.132 33 | 34 | # Spark on YARN 35 | [mapr-spark-yarn] 36 | 10.0.0.132 37 | 38 | # NFS loopback, ATTENTION: install only on edge node (mapr-client)!!! 39 | [mapr-nfsloopback] 40 | #10.0.0.132 41 | 42 | # Flume 43 | [mapr-flume] 44 | 10.0.0.132 45 | 46 | # HBase CLI 47 | [mapr-hbase-cli] 48 | 10.0.0.132 49 | 50 | # Hive CLI 51 | [mapr-hive-cli] 52 | 10.0.0.132 53 | 54 | # Pig 55 | [mapr-pig] 56 | 10.0.0.132 57 | 58 | # MapR NFS loopback (only for MapR client!) 59 | [mapr-nfsloopback] 60 | 61 | # MapR Fuse Posix Client Basic 62 | [mapr-posix-client-basic] 63 | 64 | # MapR Fuse Posix Client Platinum 65 | [mapr-posix-client-platinum] 66 | -------------------------------------------------------------------------------- /molecule/mapr-core-docker/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: docker 6 | lint: 7 | name: yamllint 8 | 9 | platforms: 10 | 11 | - name: basic-core 12 | image: centos:7 13 | command: /sbin/init 14 | capabilities: 15 | - SYS_RESOURCE 16 | - SYS_NICE 17 | - SYS_ADMIN 18 | security_opts: 19 | - seccomp=unconfined 20 | - apparmor=unconfined 21 | volumes: 22 | - /sys/fs/cgroup:/sys/fs/cgroup:ro 23 | groups: 24 | - common 25 | - mapr-core 26 | - mapr-zookeeper 27 | - mapr-fileserver 28 | - mapr-cldb 29 | - mapr-mcs 30 | - mapr-spark-historyserver 31 | - mapr-hbase-cli 32 | - mapr-oozie 33 | - mapr-historyserver 34 | - mapr-resourcemanager 35 | - mapr-spark-yarn 36 | - mapr-nodemanager 37 | - ext-mysql 38 | 39 | provisioner: 40 | name: ansible 41 | lint: 42 | name: ansible-lint 43 | options: 44 | skip-tags: molecule-notest,notest,skip-docker 45 | 46 | scenario: 47 | name: mapr-core-docker 48 | test_sequence: 49 | - destroy 50 | - create 51 | - converge 52 | - verify 53 | - destroy 54 | 55 | verifier: 56 | name: testinfra 57 | additional_files_or_dirs: 58 | - ../../tests-mapr-core/test_mapr_core_*.py 59 | - ../roles/*/molecule/default/tests/*.py 60 | lint: 61 | name: flake8 62 | -------------------------------------------------------------------------------- /myhosts/hosts_client: -------------------------------------------------------------------------------- 1 | # If you do not want to install a component leave the block empty 2 | 3 | [all:vars] 4 | ansible_user=ec2-user 5 | ansible_become=yes 6 | 7 | ###### These ones are not installed, just required for config generation ############ 8 | 9 | # Zookeeper 10 | [mapr-zookeeper] 11 | 10.0.0.63 12 | 10.0.0.110 13 | 10.0.0.232 14 | 15 | # Container Location DataBase 16 | [mapr-cldb] 17 | 10.0.0.63 18 | 10.0.0.110 19 | 10.0.0.232 20 | 21 | 22 | # Hive Meta Store 23 | [mapr-hive-metastore] 24 | 10.0.0.63 25 | 10.0.0.110 26 | 27 | ###################################################################################### 28 | 29 | # sets up ntp, rpcbind, Java and mapr default user with standard password mapr123 30 | [common] 31 | 10.0.0.146 32 | 33 | # MapR-Client 34 | [mapr-client] 35 | 10.0.0.146 36 | 37 | # Spark on YARN, requires Hive! 38 | [mapr-spark-yarn] 39 | #10.0.0.146 40 | 41 | # NFS loopback, ATTENTION: install only on edge node (mapr-client)!!! 42 | [mapr-nfsloopback] 43 | 44 | # Flume 45 | [mapr-flume] 46 | 47 | # HBase CLI 48 | [mapr-hbase-cli] 49 | #10.0.0.146 50 | 51 | # Hive CLI 52 | [mapr-hive-cli] 53 | #10.0.0.146 54 | 55 | # Pig 56 | [mapr-pig] 57 | 58 | # MapR Fuse Posix Client 59 | [mapr-posix-client-basic] 60 | 61 | # MapR Fuse Posix Client Basic 62 | [mapr-posix-client-basic] 63 | #10.0.0.146 64 | 65 | # MapR Fuse Posix Client Platinum 66 | [mapr-posix-client-platinum] 67 | -------------------------------------------------------------------------------- /roles/common-oraclejdk18/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Oracle JDK playbook 3 | - name: Downloading Java JDK 4 | get_url: 5 | url: "{{oracle_jdk_url}}" 6 | force: true 7 | mode: 0440 8 | headers: "Cookie:oraclelicense=accept-securebackup-cookie" 9 | dest: /tmp/oracle-jdk.tar.gz 10 | timeout: 120 11 | 12 | - name: Create /opt/java 13 | file: 14 | path: /opt/java 15 | mode: 0755 16 | state: directory 17 | - name: Install Java JDK 18 | unarchive: 19 | src: /tmp/oracle-jdk.tar.gz 20 | dest: /opt/java 21 | copy: false 22 | - name: Create link /usr/bin/java 23 | file: 24 | src: "{{jdk_folder}}/bin/java" 25 | dest: /usr/bin/java 26 | state: link 27 | # Other actions 28 | - name: Copy Local Policy 29 | copy: 30 | src: "../files/UnlimitedJCEPolicyJDK8/local_policy.jar" 31 | dest: "{{jdk_folder}}/jre/lib/security/" 32 | mode: 0644 33 | # - name: Copy US Export Policy 34 | # copy: 35 | # src: "../files/UnlimitedJCEPolicyJDK8/US_export_policy.jar" 36 | # dest: "{{jdk_folder}}/jre/lib/security/" 37 | # mode: 0644 38 | - name: Add JAVA_HOME to /etc/profile 39 | lineinfile: 40 | dest: /etc/profile 41 | state: present 42 | line: "export JAVA_HOME={{jdk_folder}}" 43 | 44 | 45 | # - name: Copy /etc/environment with java and path settings 46 | # copy: 47 | # src: "../files/environment" 48 | # dest: "/etc/environment" 49 | -------------------------------------------------------------------------------- /roles/mapr-core-customssl-validate/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Copy rootCA to node 3 | copy: src="{{ https_custom_rootca }}" dest="/tmp/rootCA.pem" 4 | - name: Copy cert to node 5 | copy: src="{{ https_custom_certs_path }}/{{ ansible_fqdn }}.pem" dest="/tmp/{{ ansible_fqdn }}-crt.pem" 6 | - name: Copy private key to node 7 | copy: src="{{ https_custom_private_key_path }}/{{ ansible_fqdn }}.pem" dest="/tmp/{{ ansible_fqdn }}-key.pem" 8 | - name: Verify CN for certificate 9 | shell: "openssl x509 -noout -subject -in /tmp/{{ ansible_fqdn }}-crt.pem | grep ^subject= | grep CN={{ ansible_fqdn }}" 10 | - name: Validate that certificate is signed with rootCA.pem 11 | shell: "openssl verify -CAfile /tmp/rootCA.pem /tmp/{{ ansible_fqdn }}-crt.pem | grep OK" 12 | any_errors_fatal: yes 13 | - name: Get MD5 sum for certificate 14 | shell: "openssl x509 -noout -modulus -in /tmp/{{ ansible_fqdn }}-crt.pem | openssl md5" 15 | register: crt_md5 16 | - name: Get MD5 sum for private key 17 | shell: "openssl rsa -noout -modulus -in /tmp/{{ ansible_fqdn }}-key.pem | openssl md5" 18 | register: key_md5 19 | - name: MD5 sum for Key and CRT must be same 20 | assert: { that: "crt_md5.stdout == key_md5.stdout" } 21 | any_errors_fatal: yes 22 | - name: Cleanup files from certificates 23 | file: state=absent path="{{ item }}" 24 | with_items: 25 | - "/tmp/{{ ansible_fqdn }}-crt.pem" 26 | - "/tmp/{{ ansible_fqdn }}-key.pem" -------------------------------------------------------------------------------- /roles/mapr-core-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR Core components for all machines 3 | - name: Install mapr-core 4 | vars: 5 | packages_Suse: ['mapr-compat-suse', 'mapr-core', 'mapr-core-internal'] 6 | packages_RedHat: ['mapr-core', 'mapr-core-internal'] 7 | packages_Debian: ['mapr-core', 'mapr-core-internal'] 8 | package: name={{ item }} state=present 9 | with_items: "{{ vars['packages_' + ansible_os_family] }}" 10 | register: core_result 11 | - name: Modify configure-hadoop.sh - it should not touch yarn-site.xml 12 | lineinfile: 13 | path: /opt/mapr/server/configure-hadoop.sh 14 | insertafter: '^ConfigureYarnSiteXml' 15 | line: ' return 0' 16 | backup: yes 17 | - set_fact: core_local_changes="{{ core_result.results | map(attribute='changed') | map('int') | sum }}" 18 | - name: Configure core-site.xml 19 | template: src=core-site.xml dest=/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/etc/hadoop/core-site.xml 20 | - name: Configure yarn-site.xml 21 | template: src=yarn-site.xml dest=/opt/mapr/hadoop/hadoop-{{ hadoop_version }}/etc/hadoop/yarn-site.xml 22 | - name: Set MAPR_LOGIN_OPTS in env_override.sh to hybrid 23 | lineinfile: dest=/opt/mapr/conf/env_override.sh regexp="^export MAPR_LOGIN_OPTS" mode=0644 create=yes line="export MAPR_LOGIN_OPTS=\"-Dhadoop.login=hybrid ${MAPR_JAAS_CONFIG_OPTS} ${MAPR_ZOOKEEPER_OPTS}\"" 24 | when: mapr_security == 'kerberos' 25 | 26 | -------------------------------------------------------------------------------- /sites/mapr-mep-hive.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Setup Hive Variables 3 | hosts: all 4 | vars: 5 | num_mysql_hosts: "{{ groups['ext-mysql']|length }}" 6 | roles: 7 | - ../roles/mapr-hive-vars 8 | - ../roles/mapr-configure-vars-zk 9 | - {role: ../roles/mapr-hive-mysql, when: hive_db_type == 'mysql' and hive_db_mysql_create_schema_and_user and num_mysql_hosts != '0' } 10 | - name: Install Hive CLI 11 | hosts: mapr-hive-cli 12 | roles: 13 | - ../roles/mapr-hive-cli-install 14 | - name: Install Hive MetaStore 15 | hosts: mapr-hive-metastore 16 | roles: 17 | - ../roles/mapr-hive-metastore-install 18 | - name: Install Hive Server 2 19 | hosts: mapr-hive-server2 20 | roles: 21 | - ../roles/mapr-hive-server2-install 22 | - name: Install Hive WebHCat 23 | hosts: mapr-hive-webhcat 24 | roles: 25 | - ../roles/mapr-hive-webhcat-install 26 | - name: Uninstall Hive WebHCat 27 | hosts: all:!mapr-hive-webhcat 28 | roles: 29 | - ../roles/mapr-hive-webhcat-uninstall 30 | - name: Uninstall Hive Server 2 31 | hosts: all:!mapr-hive-server2 32 | roles: 33 | - ../roles/mapr-hive-server2-uninstall 34 | - name: Uninstall Hive MetaStore 35 | hosts: all:!mapr-hive-metastore 36 | roles: 37 | - ../roles/mapr-hive-metastore-uninstall 38 | - name: Uninstall Hive CLI 39 | hosts: all:!mapr-hive-cli 40 | roles: 41 | - ../roles/mapr-hive-cli-uninstall -------------------------------------------------------------------------------- /roles/mapr-nfs-v3-install/templates/exports: -------------------------------------------------------------------------------- 1 | # Sample Exports file 2 | 3 | # for /mapr exports 4 | # 5 | 6 | #access_control -> order is specific to default 7 | # list the hosts before specifying a default for all 8 | # a.b.c.d,1.2.3.4(ro) d.e.f.g(ro) (rw) 9 | # enforces ro for a.b.c.d & 1.2.3.4 and everybody else is rw 10 | 11 | # special path to export clusters in mapr-clusters.conf. To disable exporting, 12 | # comment it out. to restrict access use the exports_control 13 | # 14 | #/mapr (rw) 15 | /mapr localhost(rw) 16 | 17 | #to export only certain clusters, comment out the /mapr & uncomment. 18 | #/mapr/clustername (rw) 19 | 20 | #to export /mapr only to certain hosts (using exports_control) 21 | #/mapr a.b.c.d(rw),e.f.g.h(ro) 22 | 23 | # export /mapr/cluster1 rw to a.b.c.d & ro to e.f.g.h (denied for others) 24 | #/mapr/cluster1 a.b.c.d(rw),e.f.g.h(ro) 25 | 26 | # export /mapr/cluster2 only to e.f.g.h (denied for others) 27 | #/mapr/cluster2 e.f.g.h(rw) 28 | 29 | # export /mapr/cluster3 rw to e.f.g.h & ro to others 30 | #/mapr/cluster2 e.f.g.h(rw) (ro) 31 | 32 | #to export a certain cluster, volume or a subdirectory as an alias, 33 | #comment out /mapr & uncomment 34 | #/mapr/clustername /alias1 (rw) 35 | #/mapr/clustername/vol /alias2 (rw) 36 | #/mapr/clustername/vol/dir /alias3 (rw) 37 | #only the alias will be visible/exposed to the nfs client not the mapr path, host options as before -------------------------------------------------------------------------------- /roles/mapr-configure-warden-cldb-restart/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Restart Warden 3 | - name: Ensure warden.conf is owned by mapr user 4 | file: path=/opt/mapr/conf/warden.conf owner="{{ mapr_user }}" group="{{ mapr_group }}" 5 | - name: Stop Warden 6 | service: name=mapr-warden state=stopped enabled=yes 7 | - name: Delete /opt/mapr/logs/cldb.log 8 | file: path=/opt/mapr/logs/cldb.log state=absent 9 | - name: Start Warden 10 | service: name=mapr-warden state=started enabled=yes 11 | # Check on license is required, because when doing first installation, no license was applied 12 | - name: Wait for CLDB 13 | wait_for: path=/opt/mapr/logs/cldb.log state=present search_regex="CLDB State change \\:\ RUNNING|CLDBShutdown\\:\ license not found for CLDB HA|Becoming Slave Thread" timeout=1800 14 | - pause: seconds="{{ warden_restart_wait_time }}" 15 | when: ansible_check_mode is undefined or not ansible_check_mode 16 | - name: Check for CLDB not started, because of license 17 | shell: "cat /opt/mapr/logs/cldb.log | grep 'license not found for CLDB HA'" 18 | failed_when: false 19 | register: cldb_license_status 20 | check_mode: no 21 | - name: Check if CLDB process is running 22 | shell: "/opt/mapr/initscripts/mapr-cldb status" 23 | register: cldb_status 24 | failed_when: cldb_status.stdout.find('CLDB running') == -1 25 | when: cldb_license_status.rc == 1 # 1 means error from grep, no license error occured 26 | check_mode: no 27 | -------------------------------------------------------------------------------- /roles/ext-sssd-ad-setup/templates/sssd.conf: -------------------------------------------------------------------------------- 1 | [sssd] 2 | config_file_version = 2 3 | debug_level = 1 4 | domains = {{ mapr_kerberos_realm }} 5 | services = nss, pam, ssh 6 | 7 | [domain/{{ mapr_kerberos_realm }}] 8 | id_provider = ad 9 | auth_provider = ad 10 | access_provider = ad 11 | chpass_provider = ad 12 | 13 | ldap_id_mapping = true 14 | ldap_schema = rfc2307bis 15 | ad_server = {{ mapr_kerberos_kdc }} 16 | krb5_realm = {{ mapr_kerberos_realm }} 17 | 18 | # no dynamic dns update 19 | dyndns_update = false 20 | dyndns_refresh_interval = 43200 21 | dyndns_update_ptr = false 22 | dyndns_ttl = 3600 23 | 24 | ad_enable_gc = true 25 | 26 | ldap_sasl_mech = GSSAPI 27 | ldap_krb5_keytab = /etc/krb5.keytab 28 | ldap_krb5_init_creds = true 29 | 30 | #ldap_user_search_base = OU=People,DC=ps,DC=mapr,DC=com 31 | #ldap_group_search_base = OU=Groups,OU=_GVA,DC=ps,DC=mapr,DC=com 32 | #ad_access_filter = (memberOf=CN= replaceme,OU=Groups,OU=_GVA,DC=ps,DC=mapr,DC=com) 33 | #ldap_access_filter = (memberOf=CN= replaceme,OU=Groups,OU=_GVA,DC=ps,DC=mapr,DC=com) 34 | 35 | ad_gpo_access_control = disabled 36 | 37 | override_homedir = /home/%u 38 | override_shell = /bin/bash 39 | 40 | # Users and groups are not enumerated to reduce trafic 41 | enumerate = false 42 | ldap_access_order = expire 43 | ldap_account_expire_policy = ad 44 | ldap_force_upper_case_realm = true 45 | case_sensitive = false 46 | ldap_referrals = false 47 | 48 | # perf 49 | cache_credentials = true 50 | -------------------------------------------------------------------------------- /roles/mapr-hive-server2-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Hive Server 2 3 | - name: Install mapr-hiveserver2 4 | package: name=mapr-hiveserver2 state=present 5 | register: hs2_result 6 | #- name: Find Hive Path 7 | # find: paths="/opt/mapr/hive/" patterns="hive*" file_type=directory 8 | # register: hive_path_result 9 | # failed_when: hive_path_result.matched != 1 10 | - name: Get Hive version 11 | shell: cat /opt/mapr/hive/hiveversion 12 | register: version_output 13 | check_mode: no 14 | changed_when: no 15 | - name: Start all hive server instances 16 | lineinfile: dest="/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/warden.hs2.conf.template" regexp="^services=" line="services=hs2:all" 17 | - name: Copy to /opt/mapr/conf/conf.d/warden.hs2.conf 18 | copy: remote_src=true src="/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/warden.hs2.conf.template" dest=/opt/mapr/conf/conf.d/warden.hs2.conf mode=0644 owner="{{ mapr_user }}" group="{{ mapr_group }}" 19 | - name: Configure.sh -R 20 | shell: /opt/mapr/server/configure.sh -R 21 | when: hs2_result.changed 22 | - pause: seconds=10 23 | when: hs2_result.changed 24 | - name: Restart Hive Server2 25 | shell: "maprcli node services -name hs2 -action restart -nodes {{ ansible_fqdn }}" 26 | register: task_result 27 | until: task_result.rc == 0 28 | retries: 10 29 | delay: 10 30 | ignore_errors: yes 31 | environment: 32 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 33 | when: hs2_result.changed -------------------------------------------------------------------------------- /roles/mapr-oozie-mysql/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get MySQL Host 3 | set_fact: mysql_item="{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}" 4 | with_items: 5 | - "{{ groups['ext-mysql'] }}" 6 | register: mysql_hosts 7 | #- debug: var=mysql_hosts 8 | #- fail: msg="No MySQL / MariaDB was defined in inventory file." 9 | # when: "{{ mysql_hosts.skipped }}" 10 | - name: Make MySQL Host String 11 | set_fact: oozie_db_host="{{ mysql_hosts.results | map(attribute='ansible_facts.mysql_item') | list | join(',')}}" 12 | 13 | - name: Create oozie user localhost 14 | mysql_user: name="{{ oozie_db_user }}" password="{{ oozie_db_password }}" host=localhost priv="*.*:ALL,GRANT" state=present 15 | when: inventory_hostname == groups["ext-mysql"][0] 16 | - name: "Create oozie user {{ ansible_fqdn }}" 17 | mysql_user: name="{{ oozie_db_user }}" password="{{ oozie_db_password }}" host="{{ ansible_fqdn }}" priv="*.*:ALL,GRANT" state=present 18 | when: inventory_hostname == groups["ext-mysql"][0] 19 | - name: Create oozie user % 20 | mysql_user: name="{{ oozie_db_user }}" password="{{ oozie_db_password }}" host=% priv="*.*:ALL,GRANT" state=present 21 | when: inventory_hostname == groups["ext-mysql"][0] 22 | - name: Create oozie mysql db 23 | mysql_db: name="{{ oozie_db_schema }}" state=present login_host="{{ oozie_db_host }}" login_user="{{ oozie_db_user }}" login_password="{{ oozie_db_password }}" 24 | when: inventory_hostname == groups["ext-mysql"][0] 25 | -------------------------------------------------------------------------------- /helper/haproxy/haproxy-global.cfg: -------------------------------------------------------------------------------- 1 | global 2 | # to have these messages end up in /var/log/haproxy.log you will 3 | # need to: 4 | # 5 | # 1) configure syslog to accept network log events. This is done 6 | # by adding the '-r' option to the SYSLOGD_OPTIONS in 7 | # /etc/sysconfig/syslog 8 | # 9 | # 2) configure local2 events to go to the /var/log/haproxy.log 10 | # file. A line like the following can be added to 11 | # /etc/sysconfig/syslog 12 | # 13 | # local2.* /var/log/haproxy.log 14 | # 15 | log 127.0.0.1 local2 16 | 17 | chroot /var/lib/haproxy 18 | pidfile /var/run/haproxy.pid 19 | maxconn 4000 20 | user haproxy 21 | group haproxy 22 | daemon 23 | 24 | # turn on stats unix socket 25 | stats socket /var/lib/haproxy/stats 26 | 27 | defaults 28 | mode http 29 | log global 30 | # option httplog 31 | # option dontlognull 32 | # option http-server-close 33 | # option forwardfor except 127.0.0.0/8 34 | # option redispatch 35 | retries 3 36 | timeout http-request 10s 37 | timeout queue 1m 38 | timeout connect 10s 39 | timeout client 1m 40 | timeout server 1m 41 | # timeout http-keep-alive 10s 42 | timeout check 10s 43 | maxconn 3000 44 | -------------------------------------------------------------------------------- /roles/mapr-opentsdb-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR OpenTSDB 3 | 4 | - name: Install mapr-opentsdb 5 | package: name=mapr-opentsdb state=present 6 | - name: Get OpenTSDB version 7 | shell: cat /opt/mapr/opentsdb/opentsdbversion 8 | register: version_output 9 | check_mode: no 10 | changed_when: no 11 | 12 | - name: Register higher memory for OpenTSDB in Warden - service.heapsize.min 13 | lineinfile: dest="/opt/mapr/opentsdb/opentsdb-{{ version_output.stdout }}/etc/conf/warden.opentsdb.conf" regexp="^service\.heapsize\.min=" line="service.heapsize.min={{ spyglass_opentsdb_max_memory_in_mb }}" 14 | - name: Register higher memory for OpenTSDB in Warden - service.heapsize.max 15 | lineinfile: dest="/opt/mapr/opentsdb/opentsdb-{{ version_output.stdout }}/etc/conf/warden.opentsdb.conf" regexp="^service\.heapsize\.max=" line="service.heapsize.max={{ spyglass_opentsdb_max_memory_in_mb }}" 16 | - name: Copy to /opt/mapr/conf/conf.d/warden.opentsdb.conf 17 | copy: remote_src=true src="/opt/mapr/opentsdb/opentsdb-{{ version_output.stdout }}/etc/conf/warden.opentsdb.conf" dest=/opt/mapr/conf/conf.d/warden.opentsdb.conf mode=0644 owner="{{ mapr_user }}" group="{{ mapr_group }}" 18 | - name: Register higher memory for OpenTSDB in service script 19 | lineinfile: dest="/opt/mapr/opentsdb/opentsdb-{{ version_output.stdout }}/etc/init.d/opentsdb" regexp="^export JVMARGS" line="export JVMARGS='-enableassertions -enablesystemassertions -Xmx{{ spyglass_opentsdb_max_memory_in_mb }}m -Xss1m -XX:MaxMetaspaceSize=128m'" -------------------------------------------------------------------------------- /roles/mapr-spark-historyserver-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Spark on YARN 3 | - name: Install mapr-spark-historyserver 4 | package: name=mapr-spark-historyserver state=present 5 | register: sparkhs_result 6 | 7 | - name: Configure.sh -R 8 | shell: /opt/mapr/server/configure.sh -R 9 | when: sparkhs_result.changed 10 | - name: Find Spark Path 11 | find: paths="/opt/mapr/spark" patterns="spark*" file_type=directory 12 | register: spark_path_result 13 | failed_when: spark_path_result.matched != 1 14 | - set_fact: spark_path="{{ spark_path_result.files[0].path }}" 15 | - name: Security opts for Spark History Server 16 | lineinfile: regexp='^export SPARK_HISTORY_OPTS=' dest="{{ spark_path }}/conf/spark-env.sh" line='export SPARK_HISTORY_OPTS=" -Dspark.history.fs.logDirectory=maprfs:///apps/spark -Dspark.ui.acls.enable=true"' 17 | - name: Change Spark History Server port for SSL in Warden configuration 18 | lineinfile: regexp='^service.ui.port=' dest=/opt/mapr/conf/conf.d/warden.spark-historyserver.conf line='service.ui.port=18480' 19 | when: spark_ssl_encryption 20 | - pause: seconds=10 21 | when: sparkhs_result.changed 22 | - name: Restart Spark Historyserver 23 | shell: "maprcli node services -name spark-historyserver -action restart -nodes {{ ansible_fqdn }}" 24 | register: task_result 25 | until: task_result.rc == 0 26 | retries: 10 27 | delay: 10 28 | ignore_errors: yes 29 | environment: 30 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 31 | when: sparkhs_result.changed -------------------------------------------------------------------------------- /roles/mapr-hive-webhcat-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Hive WebHCat 3 | - name: Install mapr-hivewebhcat 4 | package: name=mapr-hivewebhcat state=present 5 | register: webhcat_result 6 | 7 | 8 | #- name: Find Hive Path 9 | # find: paths="/opt/mapr/hive/" patterns="hive*" file_type=directory 10 | # register: hive_path_result 11 | # failed_when: hive_path_result.matched != 1 12 | - name: Get Hive version 13 | shell: cat /opt/mapr/hive/hiveversion 14 | register: version_output 15 | check_mode: no 16 | changed_when: no 17 | #- name: Start all hcat instances 18 | # lineinfile: dest=/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/warden.hcat.conf.template regexp="^services=" line="services=hcat:all:cldb" 19 | #- name: Copy to /opt/mapr/conf/conf.d/warden.hcat.conf 20 | # copy: remote_src=true src=/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/warden.hcat.conf.template dest=/opt/mapr/conf/conf.d/warden.hcat.conf mode=0644 owner={{ mapr_user }} group={{ mapr_group }} 21 | 22 | - name: Configure.sh -R 23 | shell: /opt/mapr/server/configure.sh -R 24 | when: webhcat_result.changed 25 | - pause: seconds=10 26 | when: webhcat_result.changed 27 | - name: Restart Hive WebHCat 28 | shell: "maprcli node services -name hcat -action restart -nodes {{ ansible_fqdn }}" 29 | register: task_result 30 | until: task_result.rc == 0 31 | retries: 10 32 | delay: 10 33 | ignore_errors: yes 34 | environment: 35 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 36 | when: webhcat_result.changed -------------------------------------------------------------------------------- /helper/notes.md: -------------------------------------------------------------------------------- 1 | [[database]] 2 | engine=oracle 3 | host=node1 4 | port=1521 5 | user=hue 6 | password=hue_password 7 | name=MAPR 8 | options={'threaded':true} 9 | 10 | cd /opt/mapr/hue/hue- 11 | source ./build/env/bin/activate 12 | pip install cx_Oracle (geht auch mit pfad) 13 | hue syncdb --noinput 14 | hue migrate 15 | deactivate 16 | 17 | ## Create user and schema in Oracle 18 | DROP USER oozie CASCADE; 19 | CREATE USER oozie IDENTIFIED BY oozie; 20 | GRANT ALL PRIVILEGES TO oozie; 21 | GRANT CONNECT, RESOURCE TO oozie; 22 | DROP USER hue CASCADE; 23 | CREATE USER hue IDENTIFIED BY hue; 24 | GRANT ALL PRIVILEGES TO hue; 25 | GRANT CONNECT, RESOURCE TO hue; 26 | DROP USER hive CASCADE; 27 | CREATE USER hive IDENTIFIED BY hive; 28 | GRANT ALL PRIVILEGES TO hive; 29 | GRANT CONNECT, RESOURCE TO hive; 30 | 31 | 32 | 33 | CREATE USER oozie IDENTIFIED BY oozie; 34 | GRANT ALL PRIVILEGES TO oozie; 35 | GRANT CONNECT, RESOURCE TO oozie; 36 | QUIT; 37 | 38 | CREATE USER hue IDENTIFIED BY hue; 39 | GRANT ALL PRIVILEGES TO hue; 40 | GRANT CONNECT, RESOURCE TO hue; 41 | QUIT; 42 | 43 | CREATE USER hive IDENTIFIED BY hive; 44 | GRANT ALL PRIVILEGES TO hive; 45 | GRANT CONNECT, RESOURCE TO hive; 46 | QUIT; 47 | 48 | 49 | create user oozie identified by oozie; 50 | GRANT CONNECT,RESOURCE,DBA TO oozie; 51 | 52 | ## Other notes for copying 53 | 54 | export ANSIBLE_HOSTS=/Users/chufe/Documents/workspaces/mapr_ansible/hosts 55 | 56 | wget -r -np -nH --cut-dirs=3 -R index.html http://archive.mapr.com/releases/v5.2.0/suse/ 57 | -------------------------------------------------------------------------------- /roles/ext-docker-redhat/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add Docker GPG key 3 | rpm_key: key="{{ docker_gpg_key }}" state=present 4 | 5 | - name: Add Docker repository 6 | get_url: 7 | url: "{{ docker_yum_repo_url }}" 8 | dest: '/etc/yum.repos.d/docker-ce.repo' 9 | owner: root 10 | group: root 11 | mode: 0644 12 | 13 | # - name: Configure Docker Edge repo. 14 | # ini_file: 15 | # dest: '/etc/yum.repos.d/docker-ce.repo' 16 | # section: 'docker-{{ docker_edition }}-edge' 17 | # option: enabled 18 | # value: '{{ docker_yum_repo_enable_edge }}' 19 | 20 | - name: Install container-selinux from CentOS 21 | package: name="{{ container_selinux_url }}" 22 | when: ansible_distribution == 'Red Hat Enterprise Linux' 23 | 24 | - name: Install Docker 25 | package: name=docker-ce state=present 26 | 27 | - name: Enable Docker Service at boot 28 | service: name=docker state=started enabled=true 29 | 30 | - name: Add users to Docker group 31 | user: name="{{ item }}" groups=docker append=true 32 | with_items: "{{ docker_users }}" 33 | 34 | - name: Remove python-requests, because it conflicts with PIP 35 | package: name=python-requests state=absent 36 | 37 | - name: Download get-pip.py 38 | get_url: 39 | url: https://bootstrap.pypa.io/get-pip.py 40 | dest: /tmp/get-pip.py 41 | mode: 0755 42 | 43 | - name: Install PIP 44 | shell: "python /tmp/get-pip.py" 45 | 46 | - name: Install Python Docker module 47 | pip: name=docker 48 | 49 | - name: Restart docker 50 | service: name=docker state=restarted 51 | -------------------------------------------------------------------------------- /roles/mapr-hive-metastore-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Hive Metastore 3 | - name: Install mapr-hivemetastore 4 | package: name=mapr-hivemetastore state=present 5 | register: hivemeta_result 6 | #- name: Find Hive Path 7 | # find: paths="/opt/mapr/hive/" patterns="hive*" file_type=directory 8 | # register: hive_path_result 9 | # failed_when: hive_path_result.matched != 1 10 | - name: Get Hive version 11 | shell: cat /opt/mapr/hive/hiveversion 12 | register: version_output 13 | check_mode: no 14 | changed_when: no 15 | - name: Start all hive meta instances 16 | lineinfile: dest="/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/warden.hivemeta.conf.template" regexp="^services=" line="services=hivemeta:all" 17 | - name: Copy to /opt/mapr/conf/conf.d/warden.hivemeta.conf 18 | copy: remote_src=true src="/opt/mapr/hive/hive-{{ version_output.stdout }}/conf/warden.hivemeta.conf.template" dest=/opt/mapr/conf/conf.d/warden.hivemeta.conf mode=0644 owner="{{ mapr_user }}" group="{{ mapr_group }}" 19 | - name: Configure.sh -R 20 | shell: /opt/mapr/server/configure.sh -R 21 | when: hivemeta_result.changed 22 | - pause: seconds=10 23 | when: hivemeta_result.changed 24 | - name: Restart Hive Metastore 25 | shell: "maprcli node services -name hivemeta -action restart -nodes {{ ansible_fqdn }}" 26 | register: task_result 27 | until: task_result.rc == 0 28 | retries: 10 29 | delay: 10 30 | ignore_errors: yes 31 | environment: 32 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 33 | when: hivemeta_result.changed 34 | -------------------------------------------------------------------------------- /helper/windowsad/createsmbvolume/createvolume.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | YAMLFILE=$1 4 | parse_yaml() { 5 | local prefix=$2 6 | local s 7 | local w 8 | local fs 9 | s='[[:space:]]*' 10 | w='[a-zA-Z0-9_]*' 11 | fs="$(echo @|tr @ '\034')" 12 | sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ 13 | -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" "$1" | 14 | awk -F"$fs" '{ 15 | indent = length($1)/2; 16 | vname[indent] = $2; 17 | for (i in vname) {if (i > indent) {delete vname[i]}} 18 | if (length($3) > 0) { 19 | vn=""; for (i=0; i 0 21 | 22 | - name: Fetch MapR ticket 23 | hosts: mapr-cldb[0] 24 | tasks: 25 | - name: Copy keys to local 26 | fetch: src="/opt/mapr/conf/mapruserticket" dest="{{ local_tmp_path }}/mapruserticket_dsr" flat=yes 27 | - name: Create MapR user 28 | hosts: mapr-dsr 29 | roles: 30 | - ../roles/common-mapruser 31 | - name: Push mapr ticket to Docker node 32 | hosts: mapr-dsr 33 | tasks: 34 | - name: Copy mapruserticket to DSR nodes 35 | copy: src="{{ local_tmp_path }}/mapruserticket_dsr" dest="/etc/mapruserticket_dsr" mode="0600" owner="{{ mapr_user }}" group="{{ mapr_group }}" 36 | 37 | - name: Install Docker 38 | hosts: mapr-dsr 39 | roles: 40 | - ../roles/ext-docker-redhat 41 | - ../roles/mapr-dsr 42 | 43 | -------------------------------------------------------------------------------- /roles/mapr-core-install/templates/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 17 | 18 | hadoop.proxyuser.{{ mapr_user }}.groups 19 | * 20 | Allow the superuser mapr to impersonate any member of any group 21 | 22 | 23 | hadoop.proxyuser.{{ mapr_user }}.hosts 24 | * 25 | The superuser can connect from any host to impersonate a user 26 | 27 | 28 | hue.kerberos.principal.shortname 29 | mapr 30 | 31 | 32 | hbase.table.namespace.mappings 33 | *:/ 34 | 35 | {% if mapr_security != "none" %} 36 | 37 | hadoop.rpc.protection 38 | privacy 39 | 40 | {% endif %} 41 | 42 | 43 | -------------------------------------------------------------------------------- /roles/mapr-spark-yarn-install/templates/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | hbase.rootdir 7 | maprfs:///hbase 8 | 9 | 10 | 11 | hbase.cluster.distributed 12 | true 13 | 14 | 15 | 16 | hbase.zookeeper.quorum 17 | {{ zookeeper_string }} 18 | 19 | 20 | 21 | hbase.zookeeper.property.clientPort 22 | 5181 23 | 24 | 25 | 26 | dfs.support.append 27 | true 28 | 29 | 30 | 31 | hbase.fsutil.maprfs.impl 32 | org.apache.hadoop.hbase.util.FSMapRUtils 33 | 34 | 35 | 36 | hbase.regionserver.handler.count 37 | 30 38 | 39 | 40 | 41 | 47 | 48 | 49 | 50 | fs.mapr.threads 51 | 64 52 | 53 | 54 | 55 | mapr.hbase.default.db 56 | maprdb 57 | 58 | 59 | -------------------------------------------------------------------------------- /roles/mapr-sqoop2-server-install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install Sqoop2 Server 3 | - name: Install mapr-sqoop2-server 4 | package: name=mapr-sqoop2-server state=present 5 | register: sqoop2_result 6 | 7 | # Other actions 8 | - name: Set owner mapr-system user 9 | file: path=/opt/mapr/sqoop owner="{{ mapr_user }}" group="{{ mapr_group }}" state=directory recurse=yes 10 | 11 | #- name: Find Sqoop path 12 | # find: paths="/opt/mapr/sqoop/" patterns="sqoop*" file_type=directory 13 | # register: sqoop_path_result 14 | # failed_when: sqoop_path_result.matched != 1 15 | - name: Get Sqoop2 server version 16 | shell: cat /opt/mapr/sqoop/sqoopversion 17 | register: version_output 18 | check_mode: no 19 | changed_when: no 20 | - name: Configure sqoop.properties 21 | template: src=sqoop.properties dest="/opt/mapr/sqoop/sqoop-{{ version_output.stdout }}/conf/sqoop.properties" 22 | - name: Copy to /opt/mapr/conf/conf.d/warden.sqoop2.conf 23 | copy: remote_src=true src="/opt/mapr/sqoop/sqoop-{{ version_output.stdout }}/conf.d/warden.sqoop2.conf" dest=/opt/mapr/conf/conf.d/warden.sqoop2.conf mode=0644 owner="{{ mapr_user }}" group="{{ mapr_group }}" 24 | 25 | - name: Configure.sh -R 26 | shell: "/opt/mapr/server/configure.sh -R" 27 | when: sqoop2_result.changed 28 | - pause: seconds=10 29 | when: sqoop2_result.changed 30 | - name: Restart Sqoop2 Server 31 | shell: "maprcli node services -name sqoop2 -action restart -nodes {{ ansible_fqdn }}" 32 | register: task_result 33 | until: task_result.rc == 0 34 | retries: 10 35 | delay: 10 36 | ignore_errors: yes 37 | environment: 38 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 39 | when: sqoop2_result.changed -------------------------------------------------------------------------------- /roles/ext-kerberos-verify-keytabs/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Copy mapr.keytab 3 | copy: src="{{ mapr_kerberos_local_keytabs_dir }}/{{ ansible_fqdn }}.keytab" dest=/tmp/mapr.keytab 4 | 5 | - name: "Verify {{ mapr_user }}/ keytab" 6 | command: "kinit -k -t /tmp/mapr.keytab {{ mapr_user }}/{{ ansible_fqdn }}@{{ mapr_kerberos_realm }}" 7 | # when: inventory_hostname in groups["mapr-core"] 8 | - name: Display current ticket mapr 9 | command: klist 10 | register: klist_out 11 | # when: inventory_hostname in groups["mapr-core"] 12 | - debug: msg="{{ klist_out }}" 13 | # when: inventory_hostname in groups["mapr-core"] 14 | - name: Destroy current ticket 15 | command: kdestroy 16 | # when: inventory_hostname in groups["mapr-core"] 17 | 18 | - name: Verify HTTP/ keytab 19 | command: "kinit -k -t /tmp/mapr.keytab HTTP/{{ ansible_fqdn }}@{{ mapr_kerberos_realm }}" 20 | # when: inventory_hostname in groups["mapr-core"] 21 | - name: Display current ticket mapr 22 | command: klist 23 | register: klist_out 24 | # when: inventory_hostname in groups["mapr-core"] 25 | - debug: msg="{{ klist_out }}" 26 | # when: inventory_hostname in groups["mapr-core"] 27 | - name: Destroy current ticket 28 | command: kdestroy 29 | # when: inventory_hostname in groups["mapr-core"] 30 | 31 | - name: Verify Cluster keytab 32 | command: "kinit -k -t /tmp/mapr.keytab {{ mapr_user_cldb_principal }}/{{ cluster_name }}" 33 | - name: Display current ticket mapr 34 | command: klist 35 | register: klist_out 36 | - debug: msg="{{ klist_out }}" 37 | - name: Destroy current ticket 38 | command: kdestroy 39 | 40 | - name: Clean up existing keytab 41 | file: path=/tmp/mapr.keytab state=absent 42 | -------------------------------------------------------------------------------- /roles/ui-setup-nativesecurity-secure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get Zookeeper FQDNs 3 | set_fact: zoo_item="{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}" 4 | with_items: 5 | - "{{ groups['all'] }}" # TODO make new group for zookeeper 6 | register: zookeeper_hosts 7 | - name: Make Zookeeper String 8 | set_fact: zookeeper_string="{{ zookeeper_hosts.results | map(attribute='ansible_facts.zoo_item') | list | join(',')}}" 9 | 10 | - name: Get CLDB FQDNs 11 | set_fact: cldb_item="{{ hostvars[item]['ansible_fqdn'] if 'ansible_fqdn' in hostvars[item] else item }}" 12 | with_items: 13 | - "{{ groups['all'] }}" # TODO make new group for CLDB 14 | register: cldb_hosts 15 | - name: Make CLDB String 16 | set_fact: cldb_string="{{ cldb_hosts.results | map(attribute='ansible_facts.cldb_item') | list | join(',')}}" 17 | 18 | #- name: Debug Zookeeper 19 | # debug: msg="{{ zookeeper_string }}" 20 | #- name: Debug CLDB 21 | # debug: msg="{{ cldb_string }}" 22 | 23 | - name: Copy security files 24 | copy: src="{{ local_tmp_path }}/{{ item }}" dest="/opt/mapr/conf/{{ item }}" owner="{{ mapr_user }}" group="{{mapr_group}}" mode=0600 backup=yes 25 | with_items: 26 | - cldb.key 27 | - maprserverticket 28 | - ssl_keystore 29 | - ssl_truststore 30 | 31 | - name: Configure.sh -secure 32 | shell: "/opt/mapr/server/configure.sh -N {{ cluster_name }} -secure -Z {{ zookeeper_string }} -C {{ cldb_string }} -u {{ mapr_user }} -g {{mapr_group}}" 33 | 34 | - name: Restart Zookeeper 35 | service: name=mapr-zookeeper state=restarted enabled=yes 36 | 37 | - name: Restart MapR Warden 38 | service: name=mapr-warden state=restarted enabled=yes 39 | 40 | -------------------------------------------------------------------------------- /roles/mapr-log-monitoring-configure-genkeys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR log monitoring configure 3 | #- name: Find ElasticSearch Path 4 | # find: paths="/opt/mapr/elasticsearch/" patterns="elasticsearch*" file_type=directory 5 | # register: es_path_result 6 | # failed_when: es_path_result.matched != 1 7 | - name: Get Elasticsearch version 8 | shell: cat /opt/mapr/elasticsearch/elasticsearchversion 9 | register: version_output 10 | check_mode: no 11 | changed_when: no 12 | - name: Check if generated files exist 13 | stat: path="/opt/mapr/elasticsearch/elasticsearch-{{ version_output.stdout }}/etc/elasticsearch/sg/ca/es-root-ca.pem" 14 | register: es_root_ca 15 | - name: Configure.sh 16 | shell: "/opt/mapr/server/configure.sh -R -ES {{ elasticsearch_string }} -EPelasticsearch -genESKeys" 17 | when: not es_root_ca.stat.exists 18 | environment: 19 | MAPR_TICKETFILE_LOCATION: /opt/mapr/conf/mapruserticket 20 | - name: Find ElasticSearch SG files 21 | find: path="/opt/mapr/elasticsearch/elasticsearch-{{ version_output.stdout }}/etc/elasticsearch/sg" recurse=no 22 | register: files_to_copy 23 | - name: Fetch ES files SG 24 | fetch: src="{{ item.path }}" dest="{{ local_tmp_path }}/mapr_spyglass/es/sg/" flat=yes 25 | with_items: "{{ files_to_copy.files }}" 26 | - name: Fetch ES files 27 | fetch: src="/opt/mapr/elasticsearch/elasticsearch-{{ version_output.stdout }}{{ item.src_file }}" dest="{{ item.dst_file }}" flat=yes 28 | with_items: 29 | - src_file: "/etc/elasticsearch/.keystore_password" 30 | dst_file: "{{ local_tmp_path }}/mapr_spyglass/es/.keystore_password" 31 | - src_file: "/etc/elasticsearch/sg/ca/es-root-ca.pem" 32 | dst_file: "{{ local_tmp_path }}/mapr_spyglass/es/sg/ca/es-root-ca.pem" -------------------------------------------------------------------------------- /roles/mapr-spark-yarn-install/templates/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | {% if (groups['mapr-hive-metastore']|length) > 0 %} 22 | 23 | hive.metastore.uris 24 | {{ metastore_string }} 25 | 26 | {% if hive_metastore_security != 'none' %} 27 | 28 | hive.metastore.execute.setugi 29 | true 30 | 31 | 32 | hive.metastore.sasl.enabled 33 | true 34 | 35 | {% endif %} 36 | {% endif %} 37 | 38 | hive.exec.scratchdir 39 | maprfs:///tmp/hive 40 | Scratch space for Hive jobs 41 | 42 | -------------------------------------------------------------------------------- /roles/mapr-elasticsearch-install-secure/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install MapR elasticsearch 3 | #- name: Find ElasticSearch Path 4 | # find: paths="/opt/mapr/elasticsearch/" patterns="elasticsearch*" file_type=directory 5 | # register: es_path_result 6 | # failed_when: es_path_result.matched != 1 7 | - name: Get Elasticsearch version 8 | shell: cat /opt/mapr/elasticsearch/elasticsearchversion 9 | register: version_output 10 | check_mode: no 11 | changed_when: no 12 | - name: Copy ElasticSearch files for Security 13 | copy: src="{{ local_tmp_path }}/mapr_spyglass/es/{{ item.src }}" dest="/opt/mapr/elasticsearch/elasticsearch-{{ version_output.stdout }}/{{ item.dst }}" owner="{{ mapr_user }}" group="{{ mapr_group }}" mode="{{ item.mode }}" 14 | with_items: 15 | - src: sg/ca/es-root-ca.pem 16 | dst: etc/elasticsearch/ca/ 17 | mode: "0600" 18 | - src: sg/admin-usr-private-key.pem 19 | dst: etc/elasticsearch/certs/ 20 | mode: "0600" 21 | - src: sg/admin-usr-signed.pem 22 | dst: etc/elasticsearch/certs/ 23 | mode: "0600" 24 | - src: sg/truststore.jks 25 | dst: etc/elasticsearch/keystores/ 26 | mode: "0640" 27 | - src: sg/admin-usr-keystore.jks 28 | dst: etc/elasticsearch/keystores/ 29 | mode: "0640" 30 | - src: "sg/{{ ansible_fqdn }}-srvr-keystore.jks" 31 | dst: etc/elasticsearch/keystores/ 32 | mode: "0640" 33 | - src: sg/sg2.yml 34 | dst: etc/elasticsearch/ 35 | mode: "0600" 36 | - src: "sg/sg_http_{{ ansible_fqdn }}.yml" 37 | dst: etc/elasticsearch/ 38 | mode: "0600" 39 | - src: "sg/sg_ssl_{{ ansible_fqdn }}.yml" 40 | dst: etc/elasticsearch/ 41 | mode: "0600" 42 | - src: .keystore_password 43 | dst: etc/elasticsearch/ 44 | mode: "0600" 45 | -------------------------------------------------------------------------------- /roles/mapr-dsr/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create a docker container 3 | docker_container: 4 | name: mapr-dsr 5 | image: "{{ mapr_dsr_docker_image }}" 6 | pull: true 7 | detach: true 8 | tty: true 9 | interactive: true 10 | privileged: true 11 | published_ports: 12 | - "9995:9995" 13 | - "10000:10000" 14 | - "10001:10001" 15 | - "10002:10002" 16 | - "10003:10003" 17 | - "10004:10004" 18 | - "10005:10005" 19 | - "10006:10006" 20 | - "10007:10007" 21 | - "10008:10008" 22 | - "10009:10009" 23 | - "10010:10010" 24 | - "11000:11000" 25 | - "11001:11001" 26 | - "11002:11002" 27 | - "11003:11003" 28 | - "11004:11004" 29 | - "11005:11005" 30 | - "11006:11006" 31 | - "11007:11007" 32 | - "11008:11008" 33 | - "11009:11009" 34 | - "11010:11010" 35 | volumes: 36 | - "/etc/mapruserticket_dsr:/tmp/mapr_ticket:ro" 37 | capabilities: 38 | - SYS_ADMIN 39 | - SYS_RESOURCE 40 | devices: 41 | - "/dev/fuse" 42 | security_opts: 43 | - "apparmor:unconfined" 44 | env: 45 | HOST_IP: "{{ ansible_default_ipv4.address }}" 46 | MAPR_CLUSTER: "{{ cluster_name }}" 47 | MAPR_CLDB_HOSTS: "{{ cldb_string }}" 48 | MAPR_CONTAINER_USER: "{{ mapr_user }}" 49 | MAPR_CONTAINER_PASSWORD: "{{ mapr_user_container_password }}" 50 | MAPR_CONTAINER_GROUP: "{{ mapr_group }}" 51 | MAPR_CONTAINER_UID: "{{ mapr_user_uid }}" 52 | MAPR_CONTAINER_GID: "{{ mapr_group_gid }}" 53 | MAPR_TICKETFILE_LOCATION: /tmp/mapr_ticket 54 | MAPR_MOUNT_PATH: /mapr 55 | MAPR_HS_HOST: "{{ dsr_hiveserver2_string }}" 56 | ZEPPELIN_NOTEBOOK_DIR: "/mapr/{{ cluster_name }}/user/mapr/notebook" 57 | MAPR_TZ: US/Pacific 58 | -------------------------------------------------------------------------------- /sites/mapr-ssl.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: mapr-include-vars.yml 3 | - name: Backup conf 4 | hosts: mapr-core 5 | tasks: 6 | - name: Backup MapR configuration files 7 | archive: path=/opt/mapr/conf dest="/opt/maprconf-{{ ansible_date_time.iso8601_basic_short }}.tar.gz" owner="{{ mapr_user }}" group="{{ mapr_group }}" mode="0600" 8 | - name: Validate and import ssl_truststore 9 | hosts: mapr-core 10 | any_errors_fatal: yes 11 | roles: 12 | - ../roles/mapr-core-customssl-validate 13 | - name: Check if custom SSL is enabled 14 | hosts: mapr-core 15 | tasks: 16 | - fail: msg='https_custom_certs_enabled must be set to True to install certificates.' 17 | when: not https_custom_certs_enabled 18 | any_errors_fatal: yes 19 | - name: Stop Warden 20 | hosts: mapr-core 21 | tasks: 22 | - name: Stop MapR Warden 23 | service: name=mapr-warden state=stopped 24 | - name: Stop Zookeeper 25 | hosts: mapr-zookeeper 26 | tasks: 27 | - name: Stop MapR Zookeeper 28 | service: name=mapr-zookeeper state=stopped 29 | - name: Import ssl_truststore and ssl_keystore 30 | hosts: mapr-core 31 | roles: 32 | - ../roles/mapr-core-customssl 33 | 34 | - name: Grafana custom certs 35 | hosts: mapr-grafana 36 | roles: 37 | - ../roles/mapr-grafana-customssl 38 | 39 | # Kibana not configured anywhere? installed but not configured 40 | - name: Kibana custom certs 41 | hosts: mapr-kibana 42 | roles: 43 | - ../roles/mapr-kibana-customssl 44 | 45 | # Hue uses certificate from /opt/mapr/conf/ssl_keystore 46 | - name: Start Zookeeper 47 | hosts: mapr-zookeeper 48 | tasks: 49 | - name: Start MapR Zookeeper 50 | service: name=mapr-zookeeper state=started 51 | 52 | - name: Start Warden 53 | hosts: mapr-core 54 | tasks: 55 | - name: Start MapR Warden 56 | service: name=mapr-warden state=started 57 | -------------------------------------------------------------------------------- /sites/mapr-mep-spyglass-log.yml: -------------------------------------------------------------------------------- 1 | - import_playbook: mapr-include-vars.yml 2 | - name: Install MapR fluentd 3 | hosts: mapr-fluentd 4 | roles: 5 | - ../roles/mapr-fluentd-install 6 | - name: Uninstall MapR fluentd 7 | hosts: all:!mapr-fluentd 8 | roles: 9 | - ../roles/mapr-fluentd-uninstall 10 | - name: Install MapR ElasticSearch 11 | hosts: mapr-elasticsearch 12 | roles: 13 | - ../roles/mapr-elasticsearch-install 14 | - name: Uninstall MapR ElasticSearch 15 | hosts: all:!mapr-elasticsearch 16 | roles: 17 | - ../roles/mapr-elasticsearch-uninstall 18 | - name: Install MapR Kibana 19 | hosts: mapr-kibana 20 | roles: 21 | - ../roles/mapr-kibana-install 22 | - { role: ../roles/mapr-kibana-customssl, when: https_custom_certs_enabled } 23 | - name: Uninstall MapR Kibana 24 | hosts: all:!mapr-kibana 25 | roles: 26 | - ../roles/mapr-kibana-uninstall 27 | - name: Setup Spyglass ElasticSearch vars 28 | hosts: all 29 | roles: 30 | - ../roles/mapr-log-monitoring-configure-vars 31 | - name: Generate keys for Spyglass ElasticSearch 32 | hosts: mapr-elasticsearch[0] 33 | roles: 34 | - {role: ../roles/mapr-log-monitoring-configure-genkeys, when: spyglass_elasticsearch_secure} 35 | - name: Copy keys to Spyglass ElasticSearch nodes 36 | hosts: mapr-elasticsearch 37 | roles: 38 | - {role: ../roles/mapr-elasticsearch-install-secure, when: spyglass_elasticsearch_secure} 39 | - name: Copy keys to Kibana ElasticSearch nodes 40 | hosts: mapr-kibana 41 | roles: 42 | - {role: ../roles/mapr-kibana-install-secure, when: spyglass_elasticsearch_secure} 43 | - name: MapR Configure Log Monitoring 44 | hosts: mapr-fluentd 45 | roles: 46 | - ../roles/mapr-log-monitoring-configure 47 | serial: 1 48 | - name: Copy keys to FluentD nodes 49 | hosts: mapr-fluentd 50 | roles: 51 | - {role: ../roles/mapr-fluentd-install-secure, when: spyglass_elasticsearch_secure} --------------------------------------------------------------------------------