├── .gitignore ├── images └── one-script-all.gif ├── playbooks ├── pre_install │ ├── generate_certs │ │ └── templates │ │ │ ├── openssl-ext.conf │ │ │ └── openssl-ext-pvc.conf │ ├── freeipa_dbus_fix │ │ ├── files │ │ │ └── dbus_session.conf │ │ └── tasks │ │ │ └── main.yml │ ├── set_python3_platform_version │ │ └── tasks │ │ │ └── main.yml │ ├── license │ │ └── tasks │ │ │ └── main.yml │ ├── authorized_keys │ │ └── tasks │ │ │ └── main.yml │ ├── node_key │ │ └── tasks │ │ │ └── main.yml │ ├── etc_hosts_copy │ │ └── tasks │ │ │ └── main.yml │ ├── auth_key_for_all │ │ └── tasks │ │ │ └── main.yml │ ├── copy_certs │ │ └── tasks │ │ │ └── main.yml │ └── extra_vars.yml ├── pvc_setup │ ├── cml │ │ ├── create_cml_registry │ │ │ ├── templates │ │ │ │ ├── delete_cml_registry.json │ │ │ │ └── create_cml_registry.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── sign_cml_certs │ │ │ ├── templates │ │ │ │ ├── openssl-ext.conf │ │ │ │ └── check_cml_activation.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── create_cml │ │ │ ├── templates │ │ │ │ ├── check_cml_activation.json │ │ │ │ ├── delete_workspace.json │ │ │ │ ├── create_cml_1_5_5.json │ │ │ │ └── create_cml.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── upload_cml_certs │ │ │ ├── templates │ │ │ │ └── check_cml_activation.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── create_cml_inf │ │ │ ├── templates │ │ │ │ ├── delete_cml_inf.json │ │ │ │ └── create_cml_inf.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── setup_cmlutils │ │ │ ├── templates │ │ │ │ └── export-config.ini │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── setup_cdswctl │ │ │ └── meta │ │ │ │ └── main.yml │ │ └── enable_quota_management │ │ │ └── tasks │ │ │ └── main.yml │ ├── common │ │ ├── create_env │ │ │ ├── templates │ │ │ │ ├── deregister_cluster.json │ │ │ │ ├── delete_environment.json │ │ │ │ ├── register_cluster.json │ │ │ │ ├── create_environment.json │ │ │ │ └── create_environment_1_5_5.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── delete_env │ │ │ ├── templates │ │ │ │ ├── deregister_cluster.json │ │ │ │ └── delete_environment.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── get_cookie │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── meta │ │ │ │ └── main.yml │ │ └── get_ozone_creds │ │ │ └── meta │ │ │ └── main.yml │ ├── cde │ │ ├── setup_cde_cli │ │ │ ├── templates │ │ │ │ ├── gen_access_key.json │ │ │ │ ├── cde_credentials │ │ │ │ └── cde_config │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── upload_cde_certs │ │ │ ├── templates │ │ │ │ └── certs.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── sign_cde_certs │ │ │ ├── templates │ │ │ │ └── openssl-ext.conf │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── create_cde │ │ │ ├── templates │ │ │ │ ├── create_vc.json │ │ │ │ ├── create_vc_v1.4.1.json │ │ │ │ ├── activate_cde.json │ │ │ │ ├── create_vc_v1.5.1.json │ │ │ │ ├── create_vc_v1.5.2.json │ │ │ │ ├── create_vc_v1.5.5.json │ │ │ │ └── activate_cde_v1.4.1.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── yunikorn_ingress │ │ │ ├── templates │ │ │ │ └── yunikorn_ingress_ui.yaml │ │ │ └── tasks │ │ │ │ └── main.yml │ │ └── get_cde_token │ │ │ └── meta │ │ │ └── main.yml │ ├── cdw │ │ ├── create_cdw │ │ │ ├── templates │ │ │ │ ├── create_dbc.json │ │ │ │ ├── activate_env.json │ │ │ │ ├── activate_env_v1.5.5.json │ │ │ │ ├── activate_env_v1.5.2.json │ │ │ │ ├── create_hive_vw.json │ │ │ │ ├── create_impala_vw.json │ │ │ │ ├── create_hive_vw_v1.5.2.json │ │ │ │ └── create_hive_vw_v1.5.4.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── create_dataviz │ │ │ ├── templates │ │ │ │ ├── create-viz-v1_5_5.json │ │ │ │ ├── create-viz.json │ │ │ │ └── create-viz-v1_5_4.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ └── cdw_minimal_size │ │ │ ├── templates │ │ │ ├── viz_min_template.json │ │ │ └── dbc_min_template.json │ │ │ └── meta │ │ │ └── main.yml │ ├── tools │ │ ├── setup_cdpcli │ │ │ ├── templates │ │ │ │ ├── gen_access_key.json │ │ │ │ ├── cdp_config │ │ │ │ └── cdp_credentials │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── setup_aws_cli │ │ │ └── meta │ │ │ │ └── main.yml │ │ └── setup_k8s_usage │ │ │ └── tasks │ │ │ └── main.yaml │ ├── config │ │ ├── setup_cp_rights │ │ │ ├── templates │ │ │ │ ├── assign_role.json │ │ │ │ └── assign_env_role.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── setup_ds_rights │ │ │ ├── templates │ │ │ │ ├── assignment.json │ │ │ │ └── kerberos │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── setup_dedicated_nodes │ │ │ ├── templates │ │ │ │ └── taint-node.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── setup_classic_cluster │ │ │ ├── templates │ │ │ │ ├── classic-cluster.json │ │ │ │ └── register-cluster.json │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── restart_ecs │ │ │ └── meta │ │ │ │ └── main.yml │ │ ├── create_yunikorn_queues │ │ │ ├── meta │ │ │ │ └── main.yml │ │ │ └── templates │ │ │ │ ├── update_queue.json │ │ │ │ └── queue.json │ │ ├── setup_ldap_provider │ │ │ ├── meta │ │ │ │ └── main.yml │ │ │ └── templates │ │ │ │ ├── ldap.json │ │ │ │ └── ldap-1_5_5_SP1.json │ │ └── setup_yunikorn_root │ │ │ ├── meta │ │ │ └── main.yml │ │ │ ├── templates │ │ │ └── root_queue.json │ │ │ └── tasks │ │ │ └── main.yml │ └── prerequisites │ │ ├── prereqs │ │ └── templates │ │ │ ├── gen_credentials_ipa_pvc.p │ │ │ ├── gen_credentials_ipa_cm75.p │ │ │ └── gen_credentials_ipa_cm74.p │ │ └── patch_ecs_agent │ │ └── templates │ │ └── test_host.py.p ├── post_install │ ├── observability_restart │ │ └── templates │ │ │ └── restart_telemetry.json │ ├── push_knox_service_def │ │ └── files │ │ │ ├── service_ssb_lb.xml │ │ │ ├── rewrite_ssb_ws.xml │ │ │ ├── rewrite_ssb_lb_ws.xml │ │ │ ├── rewrite_ssb_lb_api.xml │ │ │ ├── service_ssb_lb_api.xml │ │ │ ├── service_livy.xml │ │ │ ├── service_ssb_ws.xml │ │ │ ├── service_ssb_ui.xml │ │ │ ├── service_ssb_lb_ws.xml │ │ │ └── service_ssb_lb_ui.xml │ ├── no-unlog │ │ └── templates │ │ │ └── unlog.json │ ├── add_ssb_knox │ │ └── templates │ │ │ ├── ssb_knox_dep.json │ │ │ ├── ssb_knox_url.json │ │ │ └── ssb_knox_url_api.json │ ├── add_livy3_knox │ │ └── templates │ │ │ ├── livy3_knox_url.json │ │ │ ├── livy3_knox_url_api.json │ │ │ └── livy3_knox_xforward.json │ ├── add_solr_host_knox │ │ └── templates │ │ │ ├── solr_knox_url.json │ │ │ └── solr_knox_url_api.json │ ├── observability_configure_telemetry │ │ └── templates │ │ │ ├── altus_key_config.json │ │ │ ├── add_telemetry.json │ │ │ ├── add_altus_key.json │ │ │ └── observability_config.json │ ├── fix_kms_tls │ │ ├── files │ │ │ ├── kms_tls_cdh_kms.json │ │ │ ├── kms_tls_cdh.json │ │ │ └── kms_tls.json │ │ └── default │ │ │ └── main.yml │ ├── setup_yarn_container_metrics │ │ └── templates │ │ │ └── yarn_c_metrics_config.json │ ├── fix_solr_plugin │ │ ├── templates │ │ │ └── solr_plugin.json │ │ └── default │ │ │ └── main.yml │ ├── restart_cluster │ │ └── default │ │ │ └── main.yml │ ├── add_solr_knox │ │ └── default │ │ │ └── main.yml │ ├── check_existing_services │ │ └── default │ │ │ └── main.yml │ ├── install_ssb_package │ │ └── tasks │ │ │ └── main.yml │ ├── get_cluster_root_cert │ │ └── tasks │ │ │ └── main.yml │ ├── restart_cm │ │ └── tasks │ │ │ └── main.yml │ ├── simple_tools │ │ └── tasks │ │ │ └── main.yml │ ├── ranger_extra_policies │ │ └── policies │ │ │ ├── solr-admin.json │ │ │ ├── solr.json │ │ │ ├── hdfs_user.json │ │ │ └── hdfs_hive_hwc.json │ └── fix_krb5_conf_freeipa │ │ └── tasks │ │ └── main.yml ├── pvc_post_install │ ├── set_namespace │ │ ├── defaults │ │ │ └── main.yml │ │ └── templates │ │ │ └── .dockerconfigjson │ ├── cert_manager │ │ ├── templates │ │ │ └── values.yaml │ │ └── defaults │ │ │ └── main.yml │ ├── cfm │ │ ├── operator │ │ │ └── templates │ │ │ │ └── values.yaml │ │ ├── nifi │ │ │ ├── tasks │ │ │ │ └── main.yml │ │ │ └── templates │ │ │ │ └── values.yaml │ │ └── nifiregistry │ │ │ ├── templates │ │ │ └── values.yaml │ │ │ └── tasks │ │ │ └── main.yml │ └── kafka │ │ ├── operator │ │ └── templates │ │ │ └── values.yaml │ │ └── kafka │ │ └── templates │ │ ├── values_broker.yaml │ │ ├── values_controller.yaml │ │ └── values.yaml ├── data_load │ ├── generate_data │ │ ├── add_service │ │ │ ├── templates │ │ │ │ ├── kudu_config.json │ │ │ │ ├── dependency.json │ │ │ │ ├── ranger-dependency.json │ │ │ │ ├── kafka_config.json │ │ │ │ ├── sr_config.json │ │ │ │ ├── solr_config.json │ │ │ │ ├── thresholds.json │ │ │ │ └── datagen_service.json │ │ │ └── files │ │ │ │ └── datagen_service_tls_configs.json │ │ └── main.yml │ ├── install_as_a_service │ │ ├── install_csd_and_parcel │ │ │ └── templates │ │ │ │ └── repos_url.json │ │ ├── main.yml │ │ └── add_public_certs │ │ │ └── tasks │ │ │ └── main.yml │ ├── ranger_policies │ │ ├── main.yml │ │ ├── extra_vars.yml │ │ └── push_policies │ │ │ ├── policies │ │ │ ├── nifi.json │ │ │ ├── yarn.json │ │ │ └── schema-registry.json │ │ │ └── tasks │ │ │ └── main.yml │ ├── main.yml │ ├── README.md │ └── auto-configure │ │ └── main.yml ├── ansible_install_preparation │ ├── centos_extra_packages │ │ └── templates │ │ │ └── centos-extras.repo │ ├── hdp_ansible_files │ │ └── tasks │ │ │ └── main.yml │ ├── extra_vars.yml │ ├── hdp_libs │ │ └── tasks │ │ │ └── main.yml │ ├── set_install_repo │ │ └── tasks │ │ │ └── main.yml │ └── main.yml ├── pvc_post_install_prereqs │ ├── install_python │ │ └── tasks │ │ │ └── main.yaml │ ├── copy_files │ │ └── tasks │ │ │ └── main.yml │ └── main.yml ├── requirements.yml ├── restart_paused_cluster │ ├── extra_vars.yml │ ├── all_clusters │ │ └── tasks │ │ │ └── main.yml │ └── main.yml └── user_creation │ ├── setup_user_auto_kinit │ └── tasks │ │ └── main.yml │ ├── push_users_keytab │ └── tasks │ │ └── main.yml │ ├── unix_users │ └── tasks │ │ └── main.yml │ ├── hdfs_paths │ └── tasks │ │ └── main.yml │ ├── users_directories │ └── tasks │ │ └── main.yml │ ├── ranger_policies │ └── policies │ │ └── admin │ │ ├── hive-storage.json │ │ ├── yarn.json │ │ ├── solr-admin.json │ │ └── solr.json │ └── extra_vars.yml ├── terraform └── aws │ ├── ids_hostname.tpl │ ├── hosts_internal.tpl │ ├── dns_records │ ├── hosts_eip.tpl │ └── outputs.tf │ ├── variables.tf │ └── providers.tf ├── ansible.cfg ├── ansible-cdp-73X ├── ansible-cdp-basic-enc │ └── hosts ├── ansible-cdp-basic │ └── hosts ├── ansible-cdp-pvc-oc │ └── hosts ├── ansible-cdp-pvc │ └── hosts ├── ansible-cdp-observability │ └── hosts ├── ansible-cdp-all-services │ └── hosts ├── ansible-cdp-streaming │ └── hosts ├── ansible-cdp-all-services-pvc-oc │ └── hosts ├── ansible-cdp-streaming-with-efm │ └── hosts ├── ansible-cdp │ └── hosts └── ansible-cdp-all-services-pvc │ └── hosts ├── ansible-legacy ├── ansible-cdh-5 │ └── hosts ├── ansible-cdh-6 │ └── hosts ├── ansible-cdh6-enc-stream │ └── hosts ├── ansible-hdp-2 │ └── hosts └── ansible-hdp-3 │ └── hosts ├── ansible-cdp-71X ├── ansible-cdp-basic-719 │ └── hosts ├── ansible-cdp-pvc-719-uat │ └── hosts ├── ansible-cdp-pvc-719 │ └── hosts ├── ansible-cdp-717 │ └── hosts ├── ansible-cdp-719 │ └── hosts ├── ansible-cdp-all-services-719 │ └── hosts └── ansible-cdp-streaming-719 │ └── hosts └── launch.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | customer-* 3 | customer-*/* 4 | launch_* 5 | terraform.tfstate -------------------------------------------------------------------------------- /images/one-script-all.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frischHWC/one-script-deploy/HEAD/images/one-script-all.gif -------------------------------------------------------------------------------- /playbooks/pre_install/generate_certs/templates/openssl-ext.conf: -------------------------------------------------------------------------------- 1 | subjectAltName=DNS:*.{{ item }},DNS:{{ item }} -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml_registry/templates/delete_cml_registry.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "{{ mlr_crn }}" 3 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/create_env/templates/deregister_cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "envNameOrCrn":"{{ env.name }}" 3 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/delete_env/templates/deregister_cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "envNameOrCrn":"{{ default_env }}" 3 | } -------------------------------------------------------------------------------- /playbooks/post_install/observability_restart/templates/restart_telemetry.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": ["mgmt-TELEMETRYPUBLISHER"] 3 | } -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_ssb_lb.xml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/sign_cml_certs/templates/openssl-ext.conf: -------------------------------------------------------------------------------- 1 | subjectAltName=DNS:*.{{ cml_endpoint }},DNS:{{ cml_endpoint }} -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/create_env/templates/delete_environment.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName":"{{ env.name }}" 3 | } 4 | -------------------------------------------------------------------------------- /terraform/aws/ids_hostname.tpl: -------------------------------------------------------------------------------- 1 | %{ for _, instance in instances ~} 2 | ${instance.id} ${instance.tags["Name"]} 3 | %{ endfor ~} -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/setup_cde_cli/templates/gen_access_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "user": "{{ user_crn }}", 3 | "type": "V2" 4 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/create_dbc.json: -------------------------------------------------------------------------------- 1 | { 2 | "warehouseName":"{{ warehouse.database_catalog_name }}" 3 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/delete_env/templates/delete_environment.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName":"{{ default_env }}" 3 | } 4 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/tools/setup_cdpcli/templates/gen_access_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "user": "{{ admin_crn }}", 3 | "type": "V2" 4 | } -------------------------------------------------------------------------------- /terraform/aws/hosts_internal.tpl: -------------------------------------------------------------------------------- 1 | %{ for _, instance in instances ~} 2 | ${instance.private_ip} ${instance.tags["Name"]} 3 | %{ endfor ~} -------------------------------------------------------------------------------- /playbooks/pvc_post_install/set_namespace/defaults/main.yml: -------------------------------------------------------------------------------- 1 | delete_previous_ns: true 2 | namespace: test 3 | secret_name: docker-registry-secret -------------------------------------------------------------------------------- /terraform/aws/dns_records/hosts_eip.tpl: -------------------------------------------------------------------------------- 1 | %{ for _, instance in instances ~} 2 | ${instance.public_ip} ${instance.tags["Name"]} 3 | %{ endfor ~} -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cert_manager/templates/values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | crds: 3 | enabled: true 4 | 5 | prometheus: 6 | enabled: false 7 | 8 | 9 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/tools/setup_cdpcli/templates/cdp_config: -------------------------------------------------------------------------------- 1 | [default] 2 | cdp_endpoint_url = {{ cdp_endpoint_url }} 3 | endpoint_url = {{ cdp_endpoint_url }} -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/setup_cde_cli/templates/cde_credentials: -------------------------------------------------------------------------------- 1 | [default] 2 | cdp_access_key_id = {{ user_access_key_id }} 3 | cdp_private_key = {{ user_private_key }} -------------------------------------------------------------------------------- /playbooks/pvc_setup/tools/setup_cdpcli/templates/cdp_credentials: -------------------------------------------------------------------------------- 1 | [default] 2 | cdp_access_key_id = {{ admin_access_key_id }} 3 | cdp_private_key = {{ admin_private_key }} -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/upload_cde_certs/templates/certs.json: -------------------------------------------------------------------------------- 1 | { 2 | "certificate_details": { 3 | "certificate": "{{ cde_crt }}", 4 | "key": "{{ cde_key }}" 5 | } 6 | } -------------------------------------------------------------------------------- /playbooks/post_install/no-unlog/templates/unlog.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "session_timeout", 5 | "value": 2592000 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml/templates/check_cml_activation.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ workspace.env_name }}", 3 | "workspaceName": "{{ workspace.name }}" 4 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_cp_rights/templates/assign_role.json: -------------------------------------------------------------------------------- 1 | { 2 | "role": "crn:altus:iam:us-west-1:altus:role:{{ user_role }}", 3 | "user": "{{ user_crn }}" 4 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/sign_cml_certs/templates/check_cml_activation.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ workspace.env_name }}", 3 | "workspaceName": "{{ workspace.name }}" 4 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/upload_cml_certs/templates/check_cml_activation.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ workspace.env_name }}", 3 | "workspaceName": "{{ workspace.name }}" 4 | } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/kudu_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : 3 | [ 4 | { 5 | "name": "kudu_url", 6 | "value": "{{ kudu_url }}" 7 | } 8 | ] 9 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/sign_cde_certs/templates/openssl-ext.conf: -------------------------------------------------------------------------------- 1 | subjectAltName=DNS:*.{{ cde_vc_endpoint_hostname }},DNS:{{ cde_vc_endpoint_hostname }},DNS:{{ cde_vc_endpoint_hostname_wildcard }} -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_ds_rights/templates/assignment.json: -------------------------------------------------------------------------------- 1 | { 2 | "assigneeCrn":"{{ user_crn }}", 3 | "resourceRoleCrn":"crn:altus:iam:us-west-1:altus:resourceRole:{{ user_role }}" 4 | } -------------------------------------------------------------------------------- /terraform/aws/dns_records/outputs.tf: -------------------------------------------------------------------------------- 1 | # For external /etc/hosts file use 2 | output "hosts_ips" { 3 | value = templatefile("hosts_eip.tpl", { 4 | instances = aws_eip.hosts_elastic_ips 5 | }) 6 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_ssb_knox/templates/ssb_knox_dep.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "knox_service", 5 | "value": "{{ knox_service_name }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /terraform/aws/variables.tf: -------------------------------------------------------------------------------- 1 | variable "create_ipa" { 2 | type = bool 3 | default = ${FREE_IPA} 4 | } 5 | 6 | variable "create_kts" { 7 | type = bool 8 | default = ${ENCRYPTION_ACTIVATED} 9 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml_inf/templates/delete_cml_inf.json: -------------------------------------------------------------------------------- 1 | { 2 | "appCrn":"crn:cdp:ml:us-west-1:0114a8af-bfca-4df7-bb3c-1a20a8ded14d:mlserving:17ecda3e-8123-47ae-a7cf-be697502c417", 3 | "force":false 4 | } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/dependency.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : 3 | [ 4 | { 5 | "name": "{{ dependency_service }}", 6 | "value": "{{ dependency_name }}" 7 | } 8 | ] 9 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_dedicated_nodes/templates/taint-node.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "node_taint", 5 | "value": "{{ node_taint_to_apply }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/ranger-dependency.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "ranger_service", 5 | "value": "{{ ranger_service_name }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml/templates/delete_workspace.json: -------------------------------------------------------------------------------- 1 | { 2 | "workspaceName":"{{ workspace.name }}", 3 | "environmentName":"{{ workspace.env_name }}", 4 | "removeStorage":false, 5 | "force":false 6 | } -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cfm/operator/templates/values.yaml: -------------------------------------------------------------------------------- 1 | installCRDs: true 2 | image: 3 | repository: "{{ docker_registry }}/cloudera/cfm-operator" 4 | tag: "{{ cfm_operator_version }}" 5 | licenseSecret: cfm-operator-license -------------------------------------------------------------------------------- /playbooks/pvc_post_install/set_namespace/templates/.dockerconfigjson: -------------------------------------------------------------------------------- 1 | {"auths":{"{{ docker_registry }}":{"username":"{{ docker_registry_user }}","password":"{{ docker_registry_password }}","auth":"{{ docker_registry_creds_based }}"}}} -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_cp_rights/templates/assign_env_role.json: -------------------------------------------------------------------------------- 1 | { 2 | "resourceCrn":"{{ env_crn }}", 3 | "resourceRoleCrn":"crn:altus:iam:us-west-1:altus:resourceRole:{{ env_role }}", 4 | "user":"{{ user_crn }}" 5 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_livy3_knox/templates/livy3_knox_url.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_descriptor_cdp_proxy", 5 | "value": "{{ new_gateway_descriptor_cdp_proxy }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_ssb_knox/templates/ssb_knox_url.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_descriptor_cdp_proxy", 5 | "value": "{{ new_gateway_descriptor_cdp_proxy }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/pre_install/generate_certs/templates/openssl-ext-pvc.conf: -------------------------------------------------------------------------------- 1 | subjectAltName=DNS:*.apps.{{ pvc_app_domain }},DNS:{{ pvc_app_domain }},DNS:*.{{ pvc_app_domain }},DNS:service.*.apps.{{ pvc_app_domain }},DNS:*.*.apps.{{ pvc_app_domain }} -------------------------------------------------------------------------------- /playbooks/post_install/add_solr_host_knox/templates/solr_knox_url.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_descriptor_cdp_proxy", 5 | "value": "{{ new_gateway_descriptor_cdp_proxy }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/post_install/observability_configure_telemetry/templates/altus_key_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "telemetry_altus_account", 5 | "value": "altus-key-for-observability" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/create_env/templates/register_cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "envNameOrCrn": "{{ env_crn }}", 3 | "clusterName": "{{ env.name }}-compute-cluster", 4 | "description": "", 5 | "kubeConfig": "{{ pvc_kube_config }}" 6 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_ssb_knox/templates/ssb_knox_url_api.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_descriptor_cdp_proxy_api", 5 | "value": "{{ new_gateway_descriptor_cdp_proxy_api }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_livy3_knox/templates/livy3_knox_url_api.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_descriptor_cdp_proxy_api", 5 | "value": "{{ new_gateway_descriptor_cdp_proxy_api }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_solr_host_knox/templates/solr_knox_url_api.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_descriptor_cdp_proxy_api", 5 | "value": "{{ new_gateway_descriptor_cdp_proxy_api }}" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/post_install/add_livy3_knox/templates/livy3_knox_xforward.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "gateway_xforwarded_header_context_append_servicename", 5 | "value": "LIVYSERVER,LIVY_FOR_SPARK3" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/post_install/fix_kms_tls/files/kms_tls_cdh_kms.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | 4 | { 5 | "name": "hadoop_kms_authentication_signer_secret_provider_zookeeper_auth_type", 6 | "value": "sasl" 7 | } 8 | 9 | ] 10 | } -------------------------------------------------------------------------------- /playbooks/data_load/install_as_a_service/install_csd_and_parcel/templates/repos_url.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": 3 | [ 4 | { 5 | "name": "remote_parcel_repo_urls", 6 | "value" :"{{ repos_url }},{{ datagen_parcel_repo }}" 7 | } 8 | ] 9 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/create_vc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.name }}", 3 | "config": { 4 | "properties": { 5 | "livy.ingress.enabled": "true", 6 | "spark.version":"{{ virtual_cluster.spark_version }}" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/setup_cmlutils/templates/export-config.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | url="{{ cml_endpoint_workspace }}" 3 | output_dir=~/cmlutils-export 4 | ca_path=/var/lib/cloudera-scm-agent/agent-cert/cm-auto-global_cacerts.pem 5 | username="{{ default_admin_user }}" 6 | apiv1_key= -------------------------------------------------------------------------------- /terraform/aws/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "~> 5.15" 6 | } 7 | } 8 | 9 | required_version = ">= 1.2.0" 10 | } 11 | 12 | provider "aws" { 13 | region = "${REGION}" 14 | } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/kafka_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : 3 | [ 4 | { 5 | "name": "kafka_url", 6 | "value": "{{ kafka_url }}" 7 | }, 8 | { 9 | "name": "kafka_protocol", 10 | "value": "{{ kafka_protocol }}" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/sr_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : 3 | [ 4 | { 5 | "name": "schema_registry_url", 6 | "value": "{{ sr_url }}" 7 | }, 8 | { 9 | "name": "schema_registry_tls_enabled", 10 | "value": {{ sr_tls }} 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/rewrite_ssb_ws.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/rewrite_ssb_lb_ws.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /playbooks/pre_install/freeipa_dbus_fix/files/dbus_session.conf: -------------------------------------------------------------------------------- 1 | 3 | 4 | 360000 5 | 360000 6 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | forks=12 3 | host_key_checking=False 4 | command_warnings=False 5 | action_warnings=False 6 | deprecation_warnings=False 7 | remote_tmp=/tmp/.ansible/tmp 8 | interpreter_python=auto_silent 9 | timeout=30 10 | 11 | [persistent_connection] 12 | command_timeout = 120 13 | 14 | [inventory] 15 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/setup_cde_cli/templates/cde_config: -------------------------------------------------------------------------------- 1 | # ~/.cde/config.yaml 2 | 3 | credentials-file: "~/.cde/credentials" 4 | cdp-endpoint: "{{ cdp_endpoint_url }}" 5 | tls-insecure: true 6 | 7 | profiles: 8 | {% for vc in cde_vcs.json %} 9 | - name: {{ vc.name }} 10 | vcluster-endpoint: {{ vc.appInfo.dexApiUrl }} 11 | {% endfor %} 12 | -------------------------------------------------------------------------------- /playbooks/post_install/observability_configure_telemetry/templates/add_telemetry.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : [ 3 | { 4 | "name" : "mgmt-TELEMETRYPUBLISHER", 5 | "type" : "TELEMETRYPUBLISHER", 6 | "hostRef" : { 7 | "hostId" : "{{ tp_host_id }}", 8 | "hostname" : "{{ tp_host }}" 9 | } 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/rewrite_ssb_lb_api.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml_registry/templates/create_cml_registry.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ ml_registry.env_name }}", 3 | "s3AccessKey": "{{ ozone_aws_access_key }}", 4 | "s3SecretKey": "{{ ozone_aws_secret_access_key }}", 5 | "s3Bucket": "{{ ml_registry.name }}", 6 | "s3Endpoint": "{{ ozone_s3_full_endpoint }}", 7 | "environmentCrn": "{{ env_crn }}" 8 | } -------------------------------------------------------------------------------- /playbooks/ansible_install_preparation/centos_extra_packages/templates/centos-extras.repo: -------------------------------------------------------------------------------- 1 | #additional packages that may be useful 2 | [extras] 3 | name=CentOS-{{ os_version }}- Extras 4 | #mirrorlist=http://vault.centos.org/?release={{ os_version[:1] }}&arch=$basearch&repo=extras 5 | baseurl=http://vault.centos.org/centos/{{ os_version[:1] }}/extras/\$basearch/ 6 | enabled=1 7 | gpgcheck=0 -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/solr_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : 3 | [ 4 | { 5 | "name": "solr_zk_quorum", 6 | "value": "{{ solr_zk_quorum }}" 7 | }, 8 | { 9 | "name": "solr_znode", 10 | "value": "{{ solr_znode }}" 11 | }, 12 | { 13 | "name": "solr_tls_enabled", 14 | "value": {{ solr_tls }} 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_classic_cluster/templates/classic-cluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "managerType": "cloudera-manager", 3 | "managerUri": "{{ classic_cluster.cm_url }}", 4 | "allowUntrusted": true, 5 | "withSingleSignOn": false, 6 | "behindGateway": false, 7 | "username": "{{ classic_cluster.cm_user }}", 8 | "password": "{{ classic_cluster.cm_password }}", 9 | "clusterType": "CDPDC" 10 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_dataviz/templates/create-viz-v1_5_5.json: -------------------------------------------------------------------------------- 1 | { 2 | "vizName":"{{ viz.name }}", 3 | "resourceTemplate": "{{ viz_template_id | default('viz-low') }}", 4 | "resourcePool": "{{ viz.queue | default('root.default') }}", 5 | "userGroups":["{{ viz.user_group | default('admin_group') }}"], 6 | "adminGroups":["{{ viz.admin_group | default('admin_group') }}"], 7 | "tags":[] 8 | } 9 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_dataviz/templates/create-viz.json: -------------------------------------------------------------------------------- 1 | { 2 | "vizName":"{{ viz.name }}", 3 | "template":"{{ viz.size }}-viz", 4 | "resourcePool": "{{ viz.queue | default('root.default') }}", 5 | "config":{ 6 | "ldapGroups":["{{ viz.user_group | default('admin_group') }}"], 7 | "ldapAdminGroups":["{{ viz.admin_group | default('admin_group') }}"] 8 | }, 9 | "tags":[] 10 | } 11 | -------------------------------------------------------------------------------- /playbooks/post_install/setup_yarn_container_metrics/templates/yarn_c_metrics_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "cm_yarn_enable_container_usage_aggregation", 5 | "value": "true" 6 | }, 7 | { 8 | "name": "cm_yarn_container_usage_job_user", 9 | "value": "{{ default_admin_user }}" 10 | }, 11 | { 12 | "name": "cm_yarn_container_usage_job_pool", 13 | "value": "root.default" 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_dataviz/templates/create-viz-v1_5_4.json: -------------------------------------------------------------------------------- 1 | { 2 | "vizName":"{{ viz.name }}", 3 | "template":"{{ viz.size }}-viz", 4 | "resourcePool": "{{ viz.queue | default('root.default') }}", 5 | "resourceTemplate": "{{ viz_template_id | default('viz-default') }}", 6 | "userGroups":["{{ viz.user_group | default('admin_group') }}"], 7 | "adminGroups":["{{ viz.admin_group | default('admin_group') }}"], 8 | "tags":[] 9 | } 10 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_ds_rights/templates/kerberos: -------------------------------------------------------------------------------- 1 | Request is: ------geckoformboundary7da202664fa94f7b3983ee7a10912094 2 | Content-Disposition: form-data; name="principal" 3 | 4 | {{ user.name }} 5 | ------geckoformboundary7da202664fa94f7b3983ee7a10912094 6 | Content-Disposition: form-data; name="file"; filename="{{ user.keytab }}" 7 | Content-Type: application/octet-stream 8 | 9 | {{ keytab }} 10 | ------geckoformboundary7da202664fa94f7b3983ee7a10912094-- -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/create_vc_v1.4.1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.name }}", 3 | "config": { 4 | "properties": { 5 | "livy.ingress.enabled": "true", 6 | "spark.version":"{{ virtual_cluster.spark_version }}" 7 | }, 8 | "resources": { 9 | "cpu_requests": "{{ virtual_cluster.vc_quota_cpu | default('12') }}", 10 | "mem_requests": "{{ virtual_cluster.vc_quota_mem | default('32Gi') }}" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/prerequisites/prereqs/templates/gen_credentials_ipa_pvc.p: -------------------------------------------------------------------------------- 1 | --- gen_credentials_ipa.sh 2023-01-06 11:00:38.040373706 -0500 2 | +++ gen_credentials_ipa.sh 2023-01-06 11:01:09.517430290 -0500 3 | @@ -37,6 +37,7 @@ 4 | 5 | # PRINCIPAL is in the full service/fqdn@REALM format. Parse to determine 6 | # principal name and host. 7 | +PRINCIPAL=$(echo $PRINCIPAL | sed 's/\.\./\./g') 8 | PRINC=${PRINCIPAL%%/*} 9 | HOST=`echo $PRINCIPAL | cut -d "/" -f 2 | cut -d "@" -f 1` -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/activate_env.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusterName": "{{ warehouse.env_name }}", 3 | {% if pvc_type == 'OC' %}"storageClassName": "{{ local_block_storage_class }}",{% endif %} 4 | "enableLowResourceMode": {{ warehouse.low_resource_mode | default('true') }}, 5 | "delegationUserName": "{{ delegation_username }}", 6 | "delegationUserPassword": "{{ delegation_password }}", 7 | "skipValidation":true, 8 | "envBootstrap":{"awsInfo":{},"azureInfo":{}} 9 | } -------------------------------------------------------------------------------- /playbooks/pvc_post_install/kafka/operator/templates/values.yaml: -------------------------------------------------------------------------------- 1 | watchAnyNamespace: true 2 | imagePullSecrets: 3 | - name: docker-registry-secret 4 | clouderaLicense: 5 | fileContent: "{{ license_file_content }}" 6 | resources: 7 | requests: 8 | cpu: "{{ kafka_operator.cpu_req | default('100m') }}" 9 | memory: "{{ kafka_operator.mem_req | default('256Mi') }}" 10 | limits: 11 | cpu: "{{ kafka_operator.cpu_lim | default('500m') }}" 12 | memory: "{{ kafka_operator.mem_lim | default('1Gi') }}" -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/cdw_minimal_size/templates/viz_min_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "min-viz", 3 | "name": "viz_minimal_resources", 4 | "version": 1, 5 | "entity-type": "viz", 6 | "description": "Default minimal resources for components inside Data Visualization.", 7 | "is-default": true, 8 | "resources": { 9 | "viz-webapp": { 10 | "cpu": "0.5", 11 | "memory": "4096MB" 12 | } 13 | }, 14 | "attachedEntities": null, 15 | "createdBy": "User" 16 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/prerequisites/prereqs/templates/gen_credentials_ipa_cm75.p: -------------------------------------------------------------------------------- 1 | --- gen_credentials_ipa.sh 2022-04-18 05:31:52.925032988 -0700 2 | +++ gen_credentials_ipa.sh 2022-04-18 05:36:05.430521111 -0700 3 | @@ -49,7 +49,11 @@ 4 | echo "Host $HOST exists" 5 | else 6 | echo "Adding new host: $HOST" 7 | - ipa host-add $HOST --force --no-reverse 8 | + if [[ $HOST =~ \. ]]; then 9 | + ipa host-add $HOST --force --no-reverse 10 | + else 11 | + ipa host-add $HOST.{{ cp_domain }} --force --no-reverse 12 | + fi 13 | fi 14 | 15 | set +e -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/yunikorn_ingress/templates/yunikorn_ingress_ui.yaml: -------------------------------------------------------------------------------- 1 | kind: Ingress 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: yunikorn-web 5 | namespace: yunikorn 6 | spec: 7 | rules: 8 | - host: "yunikorn-web-yunikorn.apps.{{ pvc_app_domain }}" 9 | http: 10 | paths: 11 | - path: / 12 | pathType: Prefix 13 | backend: 14 | service: 15 | name: yunikorn-service 16 | port: 17 | number: 9889 18 | status: 19 | loadBalancer: {} 20 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/activate_env_v1.5.5.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusterName": "{{ warehouse.env_name }}", 3 | {% if pvc_type == 'OC' %}"storageClassName": "{{ local_block_storage_class }}",{% endif %} 4 | "enableLowResourceMode": {{ warehouse.low_resource_mode | default('true') }}, 5 | "delegationUserName": "{{ delegation_username }}", 6 | "delegationUserPassword": "{{ delegation_password }}", 7 | "dedicatedExecutorNodes": {{ warehouse.use_dedicated_nodes | default('false') }}, 8 | "skipValidation":true, 9 | "envBootstrap":{"awsInfo":{},"azureInfo":{}} 10 | } -------------------------------------------------------------------------------- /playbooks/pvc_post_install/kafka/kafka/templates/values_broker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kafka.strimzi.io/v1beta2 2 | kind: KafkaNodePool 3 | metadata: 4 | name: "{{ kafka.name }}-broker" 5 | namespace: "{{ kafka.namespace }}" 6 | labels: 7 | strimzi.io/cluster: "{{ kafka.name }}" 8 | spec: 9 | replicas: {{ kafka.broker.replicas }} 10 | roles: 11 | - broker 12 | storage: 13 | type: jbod 14 | volumes: 15 | - id: 0 16 | type: persistent-claim 17 | size: "{{ kafka.broker.pvc_size | default('10Gi') }}" 18 | kraftMetadata: shared 19 | deleteClaim: false -------------------------------------------------------------------------------- /playbooks/post_install/observability_configure_telemetry/templates/add_altus_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "typeName":"ALTUS_ACCESS_KEY_AUTH", 3 | "displayName":"altus-key-for-observability", 4 | "name":"altus-key-for-observability", 5 | "accountConfigs": 6 | { 7 | "items": 8 | [ 9 | { 10 | "name":"access_key_id", 11 | "value":"{{ altus_key_id }}" 12 | }, 13 | { 14 | "name":"private_key", 15 | "value":"{{ altus_private_key_one_line }}" 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /playbooks/post_install/observability_configure_telemetry/templates/observability_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "telemetrypublisher_safety_valve", 5 | "value": "telemetry.upload.job.logs=true\ntelemetry.altus.url={{ observability_dbus_api_server_url }}\nautoactions.yarn.app.collector.enabled=true\nautoactions.impala.collector.enabled=true\nautoactions.collection.yarn.enabled=true\nautoactions.collection.mr.enabled=true\nautoactions.collection.spark.enabled=true\nautoactions.collection.hive.enabled=true\nautoactions.definition.cache.refresh.minutes=5" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/prerequisites/patch_ecs_agent/templates/test_host.py.p: -------------------------------------------------------------------------------- 1 | --- a/agents/cmf/src/cmf_test/monitor/test_host.py 2 | +++ b/agents/cmf/src/cmf_test/monitor/test_host.py 3 | @@ -493,7 +493,7 @@ 4 | local_filesystem_whitelist, 5 | fstypes_info_proc_file) 6 | 7 | - def get_all_mounted_partitions(self): 8 | + def get_all_mounted_partitions(self, monitored_nodev_fstypes): 9 | retlist = [] 10 | retlist.append(sdiskpart('/dev/sda1', '/', 'ext2', 'rw')) 11 | retlist.append(sdiskpart('/dev/sda2', '/windows', 'vfat', 'rw')) -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/create_env/templates/create_environment.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ env.name }}", 3 | "address": "{{ env.cm_address }}", 4 | "user": "{{ env.cm_user }}", 5 | "authenticationToken": "{{ env.cm_password }}", 6 | "clusterNames": [ 7 | "{{ env.cluster_names }}" 8 | ], 9 | "kubeConfig": "{{ pvc_kube_config | default('') }}", 10 | "storageClass": "", 11 | "authenticationTokenType": "CLEARTEXT_PASSWORD", 12 | "namespacePrefix": "cdp", 13 | "domain": "{{ env_domain | default('') }}", 14 | "dockerConfigJson": "", 15 | "description": "" 16 | } -------------------------------------------------------------------------------- /playbooks/pvc_post_install/kafka/kafka/templates/values_controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kafka.strimzi.io/v1beta2 2 | kind: KafkaNodePool 3 | metadata: 4 | name: "{{ kafka.name }}-controller" 5 | namespace: "{{ kafka.namespace }}" 6 | labels: 7 | strimzi.io/cluster: "{{ kafka.name }}" 8 | spec: 9 | replicas: {{ kafka.controller.replicas }} 10 | roles: 11 | - controller 12 | storage: 13 | type: jbod 14 | volumes: 15 | - id: 0 16 | type: persistent-claim 17 | size: "{{ kafka.controller.pvc_size | default('10Gi') }}" 18 | kraftMetadata: shared 19 | deleteClaim: false 20 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/cdw_minimal_size/templates/dbc_min_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "min-dbc", 3 | "name": "dbc_minimal_resources", 4 | "version": 1, 5 | "entity-type": "dbc", 6 | "description": "", 7 | "is-default": true, 8 | "resources": { 9 | "hue-query-processor": { 10 | "cpu": "0.25", 11 | "memory": "1024MB" 12 | }, 13 | "metastore": { 14 | "cpu": "0.5", 15 | "memory": "6192MB", 16 | "breakdown": { 17 | "xmx": "4500M" 18 | } 19 | } 20 | }, 21 | "attachedEntities": null, 22 | "createdBy": "User" 23 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/activate_cde.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.cde_service_name }}", 3 | "env": "{{ virtual_cluster.env_name }}", 4 | "config": { 5 | "properties": { 6 | "loadbalancer.internal": "true", 7 | "dbus-wxm-client.enabled": "false", 8 | "kubernetes.api.allowList": "", 9 | "mount.SSD": "false", 10 | "loadbalancer.allowList": "", 11 | "kubernetes.api.proxyCIDRList": "", 12 | "subnets": "", 13 | "tls.enabled": "false", 14 | "yunikorn.disabled": "false" 15 | }, 16 | "tags": {} 17 | }, 18 | "skipValidation": true 19 | } -------------------------------------------------------------------------------- /playbooks/post_install/fix_kms_tls/files/kms_tls_cdh.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "ssl_enabled", 5 | "value": "true" 6 | }, 7 | { 8 | "name": "ssl_server_keystore_location", 9 | "value": "{{CM_AUTO_TLS}}" 10 | }, 11 | { 12 | "name": "ssl_server_keystore_password", 13 | "value": "{{CM_AUTO_TLS}}" 14 | }, 15 | { 16 | "name": "ssl_client_truststore_location", 17 | "value": "{{CM_AUTO_TLS}}" 18 | }, 19 | { 20 | "name": "ssl_client_truststore_password", 21 | "value": "{{CM_AUTO_TLS}}" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_ssb_lb_api.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /ssb-mve-api-lb 5 | Streaming SQL Builder - Materialized View Engine API LB 6 | SSB - MVE API LB 7 | API 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /playbooks/pvc_post_install_prereqs/install_python/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install python 3.11 3 | package: 4 | name: python3.11 5 | state: present 6 | 7 | - name: Install python 3.11-pip 8 | package: 9 | name: python3.11-pip 10 | state: present 11 | 12 | - name: Install kubernetes python package 13 | ansible.builtin.pip: 14 | name: kubernetes 15 | executable: pip3.11 16 | 17 | - name: Install pyyaml python package 18 | ansible.builtin.pip: 19 | name: pyyaml 20 | executable: pip3.11 21 | 22 | - name: Install jsonpatch python package 23 | ansible.builtin.pip: 24 | name: jsonpatch 25 | executable: pip3.11 -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/activate_env_v1.5.2.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusterName": "{{ warehouse.env_name }}", 3 | {% if pvc_type == 'OC' %}"storageClassName": "{{ local_block_storage_class }}",{% endif %} 4 | "enableLowResourceMode": {{ warehouse.low_resource_mode | default('true') }}, 5 | "delegationUserName": "{{ delegation_username }}", 6 | "delegationUserPassword": "{{ delegation_password }}", 7 | "resourcePool": "{{ warehouse.dbc_queue | default('root.default') }}", 8 | "dedicatedExecutorNodes": {{ warehouse.use_dedicated_nodes | default('false') }}, 9 | "skipValidation":true, 10 | "envBootstrap":{"awsInfo":{},"azureInfo":{}} 11 | } -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-basic-enc/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KMS_SERVERS} 20 | 21 | ${CA_SERVERS} 22 | 23 | ${KRB5_SERVERS} 24 | 25 | [cluster:children] 26 | cluster_master_nodes 27 | cluster_worker_nodes 28 | cloudera_manager 29 | 30 | [all:vars] 31 | tls=False 32 | host_pattern_mismatch=ignore 33 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/create_hive_vw.json: -------------------------------------------------------------------------------- 1 | { 2 | "computeName": "{{ warehouse.name }}", 3 | "warehouseId": "{{ dwx_dbc_id }}", 4 | "template": "{{ warehouse.size | default(xsmall) }}", 5 | "autoScaling": { 6 | "minClusters": {{ warehouse.min_nodes | default('1') }}, 7 | "maxClusters": {{ warehouse.max_nodes | default('5') }}, 8 | "autoSuspendTimeoutSeconds": {{ warehouse.auto_suspend_timeout | default('300') }}, 9 | "waitTimeSeconds": {{ warehouse.wait_time | default('60') }}, 10 | "disableAutoSuspend": {{ warehouse.disable_auto_suspend | default('false') }} 11 | }, 12 | "config": { 13 | } 14 | } -------------------------------------------------------------------------------- /ansible-legacy/ansible-cdh-5/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [cluster:children] 28 | cluster_master_nodes 29 | cluster_worker_nodes 30 | cloudera_manager 31 | ${KTS_SERVERS_GROUP} 32 | 33 | [all:vars] 34 | tls=False 35 | host_pattern_mismatch=ignore 36 | -------------------------------------------------------------------------------- /ansible-legacy/ansible-cdh-6/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [cluster:children] 28 | cluster_master_nodes 29 | cluster_worker_nodes 30 | cloudera_manager 31 | ${KTS_SERVERS_GROUP} 32 | 33 | 34 | [all:vars] 35 | tls=False 36 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_livy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-basic-719/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [cluster:children] 28 | cluster_master_nodes 29 | cluster_worker_nodes 30 | cloudera_manager 31 | ${KTS_SERVERS_GROUP} 32 | 33 | [all:vars] 34 | tls=False 35 | host_pattern_mismatch=ignore 36 | -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-basic/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [cluster:children] 28 | cluster_master_nodes 29 | cluster_worker_nodes 30 | cloudera_manager 31 | ${KTS_SERVERS_GROUP} 32 | 33 | [all:vars] 34 | tls=False 35 | host_pattern_mismatch=ignore 36 | -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-pvc-oc/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [cluster:children] 28 | cluster_master_nodes 29 | cluster_worker_nodes 30 | cloudera_manager 31 | ${KTS_SERVERS_GROUP} 32 | 33 | [all:vars] 34 | tls=False 35 | host_pattern_mismatch=ignore 36 | -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/files/datagen_service_tls_configs.json: -------------------------------------------------------------------------------- 1 | { 2 | "items" : 3 | [ 4 | { 5 | "name": "ssl_enabled", 6 | "value": "true" 7 | }, 8 | { 9 | "name": "ssl_server_keystore_location", 10 | "value": "{{CM_AUTO_TLS}}" 11 | }, 12 | { 13 | "name": "ssl_server_keystore_password", 14 | "value": "{{CM_AUTO_TLS}}" 15 | }, 16 | { 17 | "name": "ssl_server_keystore_keypassword", 18 | "value": "{{CM_AUTO_TLS}}" 19 | }, 20 | { 21 | "name": "ssl_client_truststore_location", 22 | "value": "{{CM_AUTO_TLS}}" 23 | }, 24 | { 25 | "name": "ssl_client_truststore_password", 26 | "value": "{{CM_AUTO_TLS}}" 27 | } 28 | ] 29 | } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/thresholds.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "unexpected_exits_thresholds", 5 | "value": "{\"warning\":\"any\",\"critical\":\"any\"}" 6 | }, 7 | { 8 | "name": "log_directory_free_space_percentage_thresholds", 9 | "value": "{\"warning\":80,\"critical\":80}" 10 | }, 11 | { 12 | "name": "process_swap_memory_thresholds", 13 | "value": "{\"warning\":\"any\",\"critical\":\"never\"}" 14 | }{% if cloudera_manager_version is version_compare('7.4.4', '>') %}, 15 | { 16 | "name": "process_swap_memory_rate_thresholds", 17 | "value": "{\"warning\":\"any\",\"critical\":\"never\"}" 18 | } {% endif %} 19 | ] 20 | } -------------------------------------------------------------------------------- /playbooks/post_install/fix_solr_plugin/templates/solr_plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "isEnabled": true, 3 | "createdBy": "Admin", 4 | "updatedBy": "Admin", 5 | "type": "solr", 6 | "name": "cm_solr", 7 | "displayName": "cm_solr", 8 | "description": "", 9 | "tagService": "cm_tag", 10 | "configs": { 11 | "username": "admin", 12 | "password": "{{ default_password }}", 13 | "solr.url": "{{ solr_url }}/solr", 14 | "commonNameForCertificate": "", 15 | "tag.download.auth.users": "solr", 16 | "policy.download.auth.users": "solr", 17 | "ranger.plugin.audit.filters": "[{'accessResult':'DENIED','isAudited':true},{'users':['hive','hdfs','kafka','hbase','solr','rangerraz','knox','atlas'],'isAudited':false}]" 18 | } 19 | } -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-pvc/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [pvc_hosts] 28 | ${NODES_ECS_PRINTABLE} 29 | 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cloudera_manager 35 | ${KTS_SERVERS_GROUP} 36 | pvc_hosts 37 | 38 | 39 | [all:vars] 40 | tls=False 41 | host_pattern_mismatch=ignore 42 | -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-pvc-719-uat/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [pvc_hosts] 28 | ${NODES_ECS_PRINTABLE} 29 | 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cloudera_manager 35 | ${KTS_SERVERS_GROUP} 36 | pvc_hosts 37 | 38 | 39 | [all:vars] 40 | tls=False 41 | host_pattern_mismatch=ignore 42 | -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-pvc-719/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [db_server] 14 | ${NODE_0} 15 | 16 | [custom_repo] 17 | ${NODE_0} 18 | 19 | ${KTS_SERVERS} 20 | 21 | ${KMS_SERVERS} 22 | 23 | ${CA_SERVERS} 24 | 25 | ${KRB5_SERVERS} 26 | 27 | [pvc_hosts] 28 | ${NODES_ECS_PRINTABLE} 29 | 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cloudera_manager 35 | ${KTS_SERVERS_GROUP} 36 | pvc_hosts 37 | 38 | 39 | [all:vars] 40 | tls=False 41 | host_pattern_mismatch=ignore 42 | -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-observability/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=ObservabilityMaster1 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=ObservabilityMaster2 6 | ${NODE_2} host_template=ObservabilityExclusive 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=ObservabilityWorker 10 | ${NODE_4} host_template=ObservabilityWorker 11 | ${NODE_5} host_template=ObservabilityWorker 12 | 13 | 14 | [db_server] 15 | ${NODE_0} 16 | 17 | [custom_repo] 18 | ${NODE_0} 19 | 20 | ${KTS_SERVERS} 21 | 22 | ${KMS_SERVERS} 23 | 24 | ${CA_SERVERS} 25 | 26 | ${KRB5_SERVERS} 27 | 28 | [cluster:children] 29 | cluster_master_nodes 30 | cluster_worker_nodes 31 | cloudera_manager 32 | ${KTS_SERVERS_GROUP} 33 | 34 | [all:vars] 35 | tls=False 36 | host_pattern_mismatch=ignore 37 | -------------------------------------------------------------------------------- /playbooks/post_install/fix_kms_tls/files/kms_tls.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "name": "ssl_enabled", 5 | "value": "true" 6 | }, 7 | { 8 | "name": "ssl_server_keystore_location", 9 | "value": "{{CM_AUTO_TLS}}" 10 | }, 11 | { 12 | "name": "ssl_server_keystore_password", 13 | "value": "{{CM_AUTO_TLS}}" 14 | }, 15 | { 16 | "name": "ssl_client_truststore_location", 17 | "value": "{{CM_AUTO_TLS}}" 18 | }, 19 | { 20 | "name": "ssl_client_truststore_password", 21 | "value": "{{CM_AUTO_TLS}}" 22 | }, 23 | { 24 | "name": "hadoop_kms_authentication_signer_secret_provider_zookeeper_auth_type", 25 | "value": "kerberos" 26 | } 27 | 28 | ] 29 | } -------------------------------------------------------------------------------- /ansible-legacy/ansible-cdh6-enc-stream/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | 8 | [cluster_worker_nodes] 9 | ${NODE_3} host_template=Workers 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | 13 | [cluster_kafka_nodes] 14 | ${NODE_6} host_template=StreamWorker 15 | 16 | [db_server] 17 | ${NODE_0} 18 | 19 | [custom_repo] 20 | ${NODE_0} 21 | 22 | ${KTS_SERVERS} 23 | 24 | ${KMS_SERVERS} 25 | 26 | ${CA_SERVERS} 27 | 28 | ${KRB5_SERVERS} 29 | 30 | [cluster:children] 31 | cluster_master_nodes 32 | cluster_worker_nodes 33 | cluster_kafka_nodes 34 | cloudera_manager 35 | ${KTS_SERVERS_GROUP} 36 | 37 | 38 | [all:vars] 39 | tls=False 40 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/create_env/templates/create_environment_1_5_5.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ env.name }}", 3 | "environmentQuota":{ 4 | "cpuQuota": {{ env.cpu_quota | default('100') }}, 5 | "memoryQuota": {{ env.mem_quota | default('500') }}, 6 | "gpuQuota": {{ env.gpu_quota | default('10') }} 7 | }, 8 | "address": "{{ env.cm_address }}", 9 | "user": "{{ env.cm_user }}", 10 | "authenticationToken": "{{ env.cm_password }}", 11 | "clusterNames": [ 12 | "{{ env.cluster_names }}" 13 | ], 14 | "kubeConfig": "{{ pvc_kube_config | default('') }}", 15 | "storageClass": "", 16 | "authenticationTokenType": "CLEARTEXT_PASSWORD", 17 | "namespacePrefix": "cdp", 18 | "domain": "{{ env_domain | default('') }}", 19 | "dockerConfigJson": "", 20 | "description": "" 21 | } -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-all-services/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamWorkers 16 | ${NODE_8} host_template=StreamWorkers 17 | ${NODE_9} host_template=StreamWorkers 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KMS_SERVERS} 26 | 27 | ${CA_SERVERS} 28 | 29 | ${KRB5_SERVERS} 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cluster_worker_no_hadoop 35 | cloudera_manager 36 | 37 | [all:vars] 38 | tls=False 39 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-streaming/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamMaster 16 | ${NODE_8} host_template=StreamWorker 17 | ${NODE_9} host_template=StreamWorker 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KMS_SERVERS} 26 | 27 | ${CA_SERVERS} 28 | 29 | ${KRB5_SERVERS} 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cluster_worker_no_hadoop 35 | cloudera_manager 36 | 37 | [all:vars] 38 | tls=False 39 | host_pattern_mismatch=ignore 40 | -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-all-services-pvc-oc/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamWorkers 16 | ${NODE_8} host_template=StreamWorkers 17 | ${NODE_9} host_template=StreamWorkers 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KMS_SERVERS} 26 | 27 | ${CA_SERVERS} 28 | 29 | ${KRB5_SERVERS} 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cluster_worker_no_hadoop 35 | cloudera_manager 36 | 37 | [all:vars] 38 | tls=False 39 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-streaming-with-efm/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamMaster 16 | ${NODE_8} host_template=StreamWorker 17 | ${NODE_9} host_template=StreamWorker 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KMS_SERVERS} 26 | 27 | ${CA_SERVERS} 28 | 29 | ${KRB5_SERVERS} 30 | 31 | [cluster:children] 32 | cluster_master_nodes 33 | cluster_worker_nodes 34 | cluster_worker_no_hadoop 35 | cloudera_manager 36 | 37 | [all:vars] 38 | tls=False 39 | host_pattern_mismatch=ignore 40 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/prerequisites/prereqs/templates/gen_credentials_ipa_cm74.p: -------------------------------------------------------------------------------- 1 | --- /tmp/gen_credentials_ipa.sh 2022-02-03 03:05:50.102862982 -0800 2 | +++ gen_credentials_ipa.sh 2022-01-31 05:22:52.944775628 -0800 3 | @@ -49,7 +49,11 @@ 4 | echo "Host $HOST exists" 5 | else 6 | echo "Adding new host: $HOST" 7 | - ipa host-add $HOST 8 | + if [[ $HOST =~ \. ]]; then 9 | + ipa host-add $HOST --force --no-reverse 10 | + else 11 | + ipa host-add $HOST.{{ cp_domain }} --force --no-reverse 12 | + fi 13 | fi 14 | 15 | set +e 16 | @@ -62,9 +66,8 @@ 17 | else 18 | PRINC_EXISTS=no 19 | echo "Adding new principal: $PRINCIPAL" 20 | - ipa service-add $PRINCIPAL 21 | + ipa service-add $PRINCIPAL --force 22 | fi 23 | - 24 | # Set the maxrenewlife for the principal, if given. There is no interface 25 | # offered by the IPA to set it, so we use KADMIN as suggested in a few IPA 26 | # related forums. -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/create_vc_v1.5.1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.name }}", 3 | "config": { 4 | "properties": { 5 | "livy.ingress.enabled": "true", 6 | "spark.version":"{{ virtual_cluster.spark_version }}", 7 | "dexapp.appTier":"{{ virtual_cluster.tier | default('tier2') }}", 8 | "session.timeout":"{{ virtual_cluster.session_timeout | default('8h') }}" 9 | }, 10 | "resources": { 11 | "cpu_requests": "{{ virtual_cluster.vc_quota_cpu | default('12') }}", 12 | "mem_requests": "{{ virtual_cluster.vc_quota_mem | default('32Gi') }}", 13 | "gcpu_requests": "{{ virtual_cluster.vc_quota_min_cpu | default('8') }}", 14 | "gmem_requests": "{{ virtual_cluster.vc_quota_min_mem | default('24Gi') }}" 15 | }, 16 | "chartValueOverrides": 17 | { 18 | "dex-app": 19 | {"pipelines.enabled":"true"} 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=WorkersNoHadoop 16 | ${NODE_8} host_template=WorkersNoHadoop 17 | ${NODE_9} host_template=WorkersNoHadoop 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KTS_SERVERS} 26 | 27 | ${KMS_SERVERS} 28 | 29 | ${CA_SERVERS} 30 | 31 | ${KRB5_SERVERS} 32 | 33 | [cluster:children] 34 | cluster_master_nodes 35 | cluster_worker_nodes 36 | cluster_worker_no_hadoop 37 | cloudera_manager 38 | ${KTS_SERVERS_GROUP} 39 | 40 | [all:vars] 41 | tls=False 42 | host_pattern_mismatch=ignore 43 | -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-717/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=WorkersNoHadoop 16 | ${NODE_8} host_template=WorkersNoHadoop 17 | ${NODE_9} host_template=WorkersNoHadoop 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KTS_SERVERS} 26 | 27 | ${KMS_SERVERS} 28 | 29 | ${CA_SERVERS} 30 | 31 | ${KRB5_SERVERS} 32 | 33 | [cluster:children] 34 | cluster_master_nodes 35 | cluster_worker_nodes 36 | cluster_worker_no_hadoop 37 | cloudera_manager 38 | ${KTS_SERVERS_GROUP} 39 | 40 | [all:vars] 41 | tls=False 42 | host_pattern_mismatch=ignore 43 | -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-719/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=WorkersNoHadoop 16 | ${NODE_8} host_template=WorkersNoHadoop 17 | ${NODE_9} host_template=WorkersNoHadoop 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KTS_SERVERS} 26 | 27 | ${KMS_SERVERS} 28 | 29 | ${CA_SERVERS} 30 | 31 | ${KRB5_SERVERS} 32 | 33 | [cluster:children] 34 | cluster_master_nodes 35 | cluster_worker_nodes 36 | cluster_worker_no_hadoop 37 | cloudera_manager 38 | ${KTS_SERVERS_GROUP} 39 | 40 | [all:vars] 41 | tls=False 42 | host_pattern_mismatch=ignore 43 | -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-all-services-719/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamWorkers 16 | ${NODE_8} host_template=StreamWorkers 17 | ${NODE_9} host_template=StreamWorkers 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KTS_SERVERS} 26 | 27 | ${KMS_SERVERS} 28 | 29 | ${CA_SERVERS} 30 | 31 | ${KRB5_SERVERS} 32 | 33 | [cluster:children] 34 | cluster_master_nodes 35 | cluster_worker_nodes 36 | cluster_worker_no_hadoop 37 | cloudera_manager 38 | ${KTS_SERVERS_GROUP} 39 | 40 | [all:vars] 41 | tls=False 42 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /ansible-cdp-71X/ansible-cdp-streaming-719/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamMaster 16 | ${NODE_8} host_template=StreamWorker 17 | ${NODE_9} host_template=StreamWorker 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KTS_SERVERS} 26 | 27 | ${KMS_SERVERS} 28 | 29 | ${CA_SERVERS} 30 | 31 | ${KRB5_SERVERS} 32 | 33 | [cluster:children] 34 | cluster_master_nodes 35 | cluster_worker_nodes 36 | cluster_worker_no_hadoop 37 | cloudera_manager 38 | ${KTS_SERVERS_GROUP} 39 | 40 | [all:vars] 41 | tls=False 42 | host_pattern_mismatch=ignore 43 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/setup_cde_cli/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: get_cde_token -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /ansible-cdp-73X/ansible-cdp-all-services-pvc/hosts: -------------------------------------------------------------------------------- 1 | [cloudera_manager] 2 | ${NODE_0} host_template=Edge 3 | 4 | [cluster_master_nodes] 5 | ${NODE_1} host_template=Master1 6 | ${NODE_2} host_template=Master2 7 | ${NODE_3} host_template=Master3 8 | 9 | [cluster_worker_nodes] 10 | ${NODE_4} host_template=Workers 11 | ${NODE_5} host_template=Workers 12 | ${NODE_6} host_template=Workers 13 | 14 | [cluster_worker_no_hadoop] 15 | ${NODE_7} host_template=StreamWorkers 16 | ${NODE_8} host_template=StreamWorkers 17 | ${NODE_9} host_template=StreamWorkers 18 | 19 | [db_server] 20 | ${NODE_0} 21 | 22 | [custom_repo] 23 | ${NODE_0} 24 | 25 | ${KMS_SERVERS} 26 | 27 | ${CA_SERVERS} 28 | 29 | ${KRB5_SERVERS} 30 | 31 | [pvc_hosts] 32 | ${NODES_ECS_PRINTABLE} 33 | 34 | [cluster:children] 35 | cluster_master_nodes 36 | cluster_worker_nodes 37 | cluster_worker_no_hadoop 38 | cloudera_manager 39 | pvc_hosts 40 | 41 | [all:vars] 42 | tls=False 43 | host_pattern_mismatch=ignore -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/get_cde_token/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/sign_cde_certs/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/upload_cde_certs/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/cdw_minimal_size/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_dataviz/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/setup_cdswctl/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/setup_cmlutils/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/sign_cml_certs/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/upload_cml_certs/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/get_cookie/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | user_login: admin 21 | user_password: admin -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/get_cookie/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/restart_ecs/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config -------------------------------------------------------------------------------- /playbooks/pvc_setup/tools/setup_cdpcli/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_ssb_ws.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /ssb-sse-ws 5 | SSB-SSE-WS 6 | SSB-SSE-WS 7 | API 8 | 9 | 10 | 11 | responseExcludeHeaders 12 | CONTENT-LENGTH,WWW-AUTHENTICATE 13 | 14 | 15 | requestExcludeHeaders 16 | Cookie,Origin 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/get_ozone_creds/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_cp_rights/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_ds_rights/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/create_yunikorn_queues/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_classic_cluster/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_dedicated_nodes/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_ldap_provider/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_yunikorn_root/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/requirements.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | collections: 21 | - name: community.general 22 | - name: freeipa.ansible_freeipa 23 | -------------------------------------------------------------------------------- /playbooks/post_install/restart_cluster/default/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | cloudera_manager_user: admin 21 | cloudera_manager_password: admin -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cert_manager/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | namespace_cert_manager: cert_manager 21 | cert_manager_version: 1.17.0 -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/create_env/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config 22 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/common/delete_env/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config 22 | - role: common/get_cookie -------------------------------------------------------------------------------- /playbooks/pvc_setup/tools/setup_aws_cli/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/cm_config 22 | - role: common/get_ozone_creds -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_ssb_ui.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /ssb-sse-ui 5 | SSB-SSE-UI 6 | SSB-SSE-UI 7 | UI 8 | 9 | 10 | 11 | responseExcludeHeaders 12 | CONTENT-LENGTH,WWW-AUTHENTICATE 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /playbooks/data_load/ranger_policies/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Ranger Policies 21 | hosts: edge 22 | gather_facts: false 23 | roles: 24 | - push_policies -------------------------------------------------------------------------------- /playbooks/post_install/add_solr_knox/default/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | cloudera_manager_host: "localhost" 20 | cloudera_manager_user: "admin" 21 | cloudera_manager_password: "admin" -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_ssb_lb_ws.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | responseExcludeHeaders 6 | CONTENT-LENGTH,WWW-AUTHENTICATE 7 | 8 | 9 | requestExcludeHeaders 10 | Cookie,Origin 11 | 12 | 13 | 14 | /ssb-sse-ws-lb 15 | SSB-SSE-WS-LB 16 | SSB-SSE-WS-LB 17 | API 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml_inf/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie 22 | - role: common/cm_config 23 | - role: common/get_ozone_creds -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_yunikorn_root/templates/root_queue.json: -------------------------------------------------------------------------------- 1 | { 2 | "pool": { 3 | "path": "{{ yunikorn_root_queue.name }}", 4 | "quota": { 5 | "memory": "{{ yunikorn_root_queue.mem }}", 6 | "cpu": "{{ yunikorn_root_queue.cpu }}", 7 | "nvidia.com/gpu": "{{ yunikorn_root_queue.gpu }}" 8 | }, 9 | "guaranteedResources": { 10 | 11 | }, 12 | "owners": [ 13 | 14 | ], 15 | "admins": [ 16 | 17 | ], 18 | "policy": { 19 | "validity": "NEVER EXPIRES", 20 | "allocation": { 21 | "distribution": "ELASTIC", 22 | "order": "FIFO", 23 | "queueing": false 24 | } 25 | }, 26 | "metadata": { 27 | 28 | }, 29 | "scheduling": { 30 | "preemption": { 31 | "enabled": true, 32 | "fence": false, 33 | "delay": 30 34 | }, 35 | "priority": { 36 | "enabled": true, 37 | "fence": false, 38 | "offset": 0 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /playbooks/pre_install/set_python3_platform_version/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Set symbolic link of python 3 21 | shell: ln -sf /usr/libexec/platform-python /usr/bin/python3 22 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml_registry/meta/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | dependencies: 21 | - role: common/get_cookie 22 | - role: common/cm_config 23 | - role: common/get_ozone_creds -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_classic_cluster/templates/register-cluster.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "isWaiting": false, 4 | "connectivityVersion": null, 5 | "isUpgradeInProgress": false, 6 | "newKeyId": null, 7 | "belongsToMultiClusterManager": false, 8 | "dcName": "data-center", 9 | "managerUri": "{{ classic_cluster.cm_url }}", 10 | "locationId": 2612, 11 | "name": "{{ cluster_name }}", 12 | "description": "", 13 | "state": "TO_SYNC", 14 | "managerAddress": "{{ cluster_cm_ip }}", 15 | "allowUntrusted": true, 16 | "behindGateway": false, 17 | "clusterType": "CDPDC", 18 | "managerType": "cloudera-manager", 19 | "knoxEnabled": false, 20 | "createdBy": "{{ user_to_register_crn }}", 21 | "accountId": "{{ user_to_register_crn.split(':')[4] }}", 22 | "properties": { 23 | "tags": [] 24 | }, 25 | "username": "{{ classic_cluster.cm_user }}", 26 | "password": "{{ classic_cluster.cm_password }}" 27 | } 28 | ] -------------------------------------------------------------------------------- /playbooks/post_install/push_knox_service_def/files/service_ssb_lb_ui.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | responseExcludeHeaders 6 | CONTENT-LENGTH,WWW-AUTHENTICATE 7 | 8 | 9 | 10 | /ssb-sse-ui-lb 11 | SSB-SSE-UI-LB 12 | SSB-SSE-UI-LB 13 | UI 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_ldap_provider/templates/ldap.json: -------------------------------------------------------------------------------- 1 | { 2 | "ldapProviderName": "cm-ldap", 3 | "skipGroupSyncOnLogin": false, 4 | "url": "ldaps://{{ ipa_server }}:636", 5 | "bindDn": "uid=admin,cn=users,cn=accounts,{% for i in realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}", 6 | "bindPassword": "{{ ipa_bind_password | default('') }}", 7 | "userSearchBase": "cn=users,cn=accounts,{% for i in realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}", 8 | "userSearchFilter": "(&(uid={0})(objectClass=person))", 9 | "groupSearchBase": "cn=groups,cn=accounts,{% for i in realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}", 10 | "groupSearchFilter": "(&(|(member={0})(member={1}))(objectClass=posixgroup)(!(cn=admins)))", 11 | "syncGroupsOnLogin": true, 12 | "emailMappingAttribute": "mail", 13 | "showAdvanced": false, 14 | "tlsCaCertificates": [ 15 | "{{ ipa_crt | replace('\n', '\\n') }}" 16 | ] 17 | } -------------------------------------------------------------------------------- /playbooks/post_install/fix_kms_tls/default/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | cloudera_manager_host: "localhost" 20 | cloudera_manager_user: "admin" 21 | cloudera_manager_password: "admin" 22 | ranger_user: "admin" 23 | ranger_password: "admin" -------------------------------------------------------------------------------- /playbooks/post_install/fix_solr_plugin/default/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | cloudera_manager_host: "localhost" 20 | cloudera_manager_user: "admin" 21 | cloudera_manager_password: "admin" 22 | ranger_user: "admin" 23 | ranger_password: "admin" -------------------------------------------------------------------------------- /playbooks/pre_install/license/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy license key file 21 | ignore_errors: yes 22 | copy: 23 | src: "{{ license_file }}" 24 | dest: ~/license.txt 25 | when: license_file != "" -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/create_impala_vw.json: -------------------------------------------------------------------------------- 1 | { 2 | "impalaName": "{{ warehouse.name }}", 3 | "warehouseId": "{{ dwx_dbc_id }}", 4 | "template": "{{ warehouse.size | default(xsmall) }}-impala", 5 | "autoScaling": { 6 | "autoSuspendTimeoutSeconds": {{ warehouse.auto_suspend_timeout | default('300') }}, 7 | "triggerScaleUpDelay": {{ warehouse.scale_up_delay | default('20') }}, 8 | "triggerScaleDownDelay": {{ warehouse.scale_down_delay | default('20') }}, 9 | "enableHA": {{ warehouse.enable_ha | default('true') }}, 10 | "autoScaleMode": 0, 11 | "disableAutoSuspend": {{ warehouse.disable_auto_suspend | default('false') }}, 12 | "minClusters": {{ warehouse.min_nodes | default('1') }}, 13 | "maxClusters": {{ warehouse.max_nodes | default('5') }} 14 | }, 15 | "fengEnabled": false, 16 | "multithreading": { 17 | "useLegacyMultithreading": {{ warehouse.legacy_multithreading | default('true') }}, 18 | "multithreadingVersion": 1 19 | }, 20 | "config": { 21 | } 22 | } -------------------------------------------------------------------------------- /playbooks/post_install/check_existing_services/default/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | cloudera_manager_host: "localhost" 20 | cloudera_manager_user: "admin" 21 | cloudera_manager_password: "admin" 22 | ranger_user: "admin" 23 | ranger_password: "admin" -------------------------------------------------------------------------------- /playbooks/pvc_post_install_prereqs/copy_files/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy license key file 21 | ignore_errors: yes 22 | copy: 23 | src: "{{ license_file }}" 24 | dest: ~/license.txt 25 | when: license_file != "" -------------------------------------------------------------------------------- /playbooks/data_load/ranger_policies/extra_vars.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | # Use it for test 21 | # export CLUSTER_NAME=fri 22 | # ansible-playbook -i ~/hosts-${CLUSTER_NAME} main.yml --extra-vars "@extra_vars.yml" 23 | user: ${DEFAULT_ADMIN_USER} 24 | ranger_url: -------------------------------------------------------------------------------- /playbooks/pre_install/authorized_keys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Create authorized_keys file 21 | shell: "cat ~/authorized-key-* >> {{ authorized_keys }}" 22 | 23 | - name: Destroy authorized_keys 24 | shell: rm -rf ~/authorized-key-* -------------------------------------------------------------------------------- /playbooks/pre_install/node_key/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy node key file 21 | copy: 22 | src: "{{ node_key }}" 23 | dest: ~/node_key 24 | mode: 0400 25 | when: node_key is defined and node_key != "" 26 | ignore_errors: true -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Add service & Generate Data 21 | hosts: cloudera_manager 22 | gather_facts: false 23 | roles: 24 | - remove_service 25 | - add_service 26 | - { role: launch_command, when: launch_generation } -------------------------------------------------------------------------------- /playbooks/post_install/install_ssb_package/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Install openssl11-libs package 21 | package: 22 | name: openssl11-libs 23 | state: present 24 | when: cloudera_cdh_version is version('7.3.0','>=') 25 | ignore_errors: true 26 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml/templates/create_cml_1_5_5.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ workspace.env_name }}", 3 | "workspaceName": "{{ workspace.name }}" , 4 | "disableTLS": "{{ workspace.disable_tls | default(false) }}", 5 | "enableMonitoring": "{{ workspace.enable_monitoring | default(true) }}", 6 | "enableGovernance": "{{ workspace.enable_governance | default(false) }}", 7 | "enableModelMetrics": "{{ workspace.enable_model_metrics | default(false) }}", 8 | "existingDatabaseConfig": {}, 9 | "namespace": "{{ workspace.name }}", 10 | "resourcePoolConfig": { 11 | "workspaceQuota": { 12 | "cpuQuota": "{{ workspace.cpu_quota | default('50') }}", 13 | "memoryQuota": "{{ workspace.mem_quota | default('100') }} G" 14 | } 15 | }, 16 | "performCdswMigration": false, 17 | "mlGovernancePrincipal": "{{ workspace.ml_gov_principal | default('mlgov') }}", 18 | "nfsStorageClass": "{{ nfs_storage_class_to_use | default('longhorn') }}", 19 | "nfsDiskSize": "{{ workspace.nfs_disk_size | default('1000') }}", 20 | "staticSubdomain": "{{ workspace.static_sub_domain | default('') }}" 21 | } -------------------------------------------------------------------------------- /playbooks/restart_paused_cluster/extra_vars.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | cloudera_manager_host: ${CM_HOST} 21 | cloudera_manager_port: ${CM_PORT} 22 | cloudera_manager_user: ${CM_USER} 23 | cloudera_manager_password: ${CM_PASSWORD} 24 | cluster_name: ${CLUSTER_NAME} 25 | debug: ${DEBUG} 26 | -------------------------------------------------------------------------------- /playbooks/user_creation/setup_user_auto_kinit/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Kinit as default admin_user 21 | lineinfile: 22 | path: ~/.bashrc 23 | line: "kinit -kt /home/{{ default_admin_user }}/{{ default_admin_user }}.keytab {{ default_admin_user }}" 24 | create: yes 25 | -------------------------------------------------------------------------------- /playbooks/post_install/get_cluster_root_cert/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Adding enctypes and renewable ticket lifetime 21 | fetch: 22 | src: /var/lib/cloudera-scm-agent/agent-cert/cm-auto-global_cacerts.pem 23 | dest: "~/cluster-{{ cluster_name }}/global_cacerts.pem" 24 | flat: yes -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/yunikorn_ingress/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Template Ingress YAML file 21 | template: 22 | src: yunikorn_ingress_ui.yaml 23 | dest: ~/yunikorn_ingress_ui.yaml 24 | 25 | - name: Apply ingress 26 | shell: kubectl apply -f ~/yunikorn_ingress_ui.yaml 27 | 28 | 29 | -------------------------------------------------------------------------------- /playbooks/data_load/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | ### NEW RECOMMENDED WAY ### 21 | - import_playbook: auto-configure/main.yml 22 | when: datagen_as_a_service 23 | 24 | - import_playbook: install_as_a_service/main.yml 25 | when: datagen_as_a_service 26 | 27 | - import_playbook: generate_data/main.yml 28 | when: datagen_as_a_service -------------------------------------------------------------------------------- /playbooks/pre_install/etc_hosts_copy/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy /etc/hosts 21 | copy: 22 | src: "{{ etc_hosts_file }}" 23 | dest: ~/etc_hosts 24 | 25 | - name: Change /etc/hosts 26 | shell: cat ~/etc_hosts >> /etc/hosts 27 | 28 | - name: Destroy etc_hosts 29 | file: 30 | path: ~/etc_hosts 31 | state: absent -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml/templates/create_cml.json: -------------------------------------------------------------------------------- 1 | { 2 | "environmentName": "{{ workspace.env_name }}", 3 | "workspaceName": "{{ workspace.name }}" , 4 | "disableTLS": "{{ workspace.disable_tls | default(false) }}", 5 | "enableMonitoring": "{{ workspace.enable_monitoring | default(true) }}", 6 | "enableGovernance": "{{ workspace.enable_governance | default(false) }}", 7 | "enableModelMetrics": "{{ workspace.enable_model_metrics | default(false) }}", 8 | "existingDatabaseConfig": {}, 9 | "resourcePoolConfig": { 10 | "parentPoolName": "{{ workspace.queue | default('root.default') }}", 11 | "workspaceQuota": { 12 | "cpuQuota": "{{ workspace.cpu_quota | default('20') }}", 13 | "memoryQuota": "{{ workspace.mem_quota | default('50') }} G" 14 | } 15 | }, 16 | "namespace": "{{ workspace.name }}", 17 | "performCdswMigration": false, 18 | "mlGovernancePrincipal": "{{ workspace.ml_gov_principal | default('mlgov') }}", 19 | "nfsStorageClass": "{{ nfs_storage_class_to_use | default('longhorn') }}", 20 | "nfsDiskSize": "{{ workspace.nfs_disk_size | default('1000') }}", 21 | "staticSubdomain": "{{ workspace.static_sub_domain | default('') }}" 22 | } -------------------------------------------------------------------------------- /playbooks/user_creation/push_users_keytab/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy user's keytab 21 | copy: 22 | src: "~/cluster-{{ cluster_name }}/{{ user.name }}.keytab" 23 | dest: "/home/{{ user.name }}/{{ user.name }}.keytab" 24 | owner: "{{ user.name }}" 25 | group: "{{ user.name }}" 26 | mode: 0400 27 | -------------------------------------------------------------------------------- /playbooks/pvc_post_install/kafka/kafka/templates/values.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kafka.strimzi.io/v1beta2 2 | kind: Kafka 3 | metadata: 4 | name: "{{ kafka.name }}" 5 | namespace: "{{ kafka.namespace }}" 6 | annotations: 7 | strimzi.io/node-pools: enabled 8 | strimzi.io/kraft: enabled 9 | spec: 10 | cruiseControl: {} 11 | kafka: 12 | version: "{{ kafka.kafka_version | default('3.9.0.1.3') }}" 13 | listeners: 14 | - name: plain 15 | port: 9092 16 | type: internal 17 | tls: false 18 | - name: tls 19 | port: 9093 20 | type: internal 21 | tls: true 22 | - name: external 23 | port: 9094 24 | type: ingress 25 | tls: true 26 | authentication: 27 | type: tls 28 | configuration: 29 | bootstrap: 30 | host: "{{ kafka.name }}.apps.{{ pvc_app_domain }}" 31 | brokers: 32 | {% for i in range(kafka.broker.replicas) %} 33 | - broker: {{ i | int }} 34 | host: "{{ kafka.name }}-{{ i }}.apps.{{ pvc_app_domain }}" 35 | {% endfor %} 36 | class: nginx 37 | 38 | entityOperator: 39 | topicOperator: {} 40 | userOperator: {} 41 | -------------------------------------------------------------------------------- /playbooks/ansible_install_preparation/hdp_ansible_files/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy HDP hosts file 21 | copy: 22 | src: "{{ deploy_folder }}/hosts" 23 | dest: ~/deployment/ansible-repo/inventory/ 24 | 25 | - name: Copy HDP all file 26 | copy: 27 | src: "{{ deploy_folder }}/all" 28 | dest: ~/deployment/ansible-repo/playbooks/group_vars/ 29 | -------------------------------------------------------------------------------- /playbooks/post_install/restart_cm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Restart CM 21 | service: 22 | name: cloudera-scm-server 23 | state: restarted 24 | 25 | - name: wait cloudera-scm-server 26 | wait_for: 27 | host: "{{ cloudera_manager_host }}" 28 | port: "{{ cloudera_manager_port }}" 29 | delay: 20 30 | state: started 31 | timeout: 600 -------------------------------------------------------------------------------- /playbooks/data_load/ranger_policies/push_policies/policies/nifi.json: -------------------------------------------------------------------------------- 1 | { 2 | "policyType": "0", 3 | "name": "All for dev, admin and nifi", 4 | "isEnabled": true, 5 | "policyPriority": 0, 6 | "policyLabels": [ 7 | 8 | ], 9 | "description": "", 10 | "isAuditEnabled": true, 11 | "resources": { 12 | "nifi-resource": { 13 | "values": [ 14 | "/*" 15 | ], 16 | "isRecursive": false, 17 | "isExcludes": false 18 | } 19 | }, 20 | "isDenyAllElse": false, 21 | "policyItems": [ 22 | { 23 | "users": [ 24 | "nifi", 25 | "dev", 26 | "admin" 27 | ], 28 | "delegateAdmin": true, 29 | "accesses": [ 30 | { 31 | "type": "READ", 32 | "isAllowed": true 33 | }, 34 | { 35 | "type": "WRITE", 36 | "isAllowed": true 37 | } 38 | ] 39 | } 40 | ], 41 | "allowExceptions": [ 42 | 43 | ], 44 | "denyPolicyItems": [ 45 | 46 | ], 47 | "denyExceptions": [ 48 | 49 | ], 50 | "service": "{{ cluster_name | replace('-','_') }}_stream_nifi" 51 | } -------------------------------------------------------------------------------- /playbooks/pvc_post_install_prereqs/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Install Python 3.11 required for kubernetes commands 21 | hosts: "{{ pvc_ecs_server_host }}" 22 | gather_facts: false 23 | roles: 24 | - role: install_python 25 | 26 | - name: Copy required files 27 | hosts: "{{ pvc_ecs_server_host }}" 28 | gather_facts: false 29 | roles: 30 | - role: copy_files -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/create_yunikorn_queues/templates/update_queue.json: -------------------------------------------------------------------------------- 1 | { 2 | "pool": { 3 | "path": "{{ queue.name }}", 4 | "quota": { 5 | "memory": "{{ queue.mem_max | default('100G') }}", 6 | "cpu": "{{ queue.cpu_max | default('25') }}", 7 | "nvidia.com/gpu": "{{ queue.gpu_max | default(0) }}" 8 | }, 9 | "guaranteedResources": { 10 | "memory": "{{ queue.mem_guaranteed | default('100G') }}", 11 | "cpu": "{{ queue.cpu_guaranteed | default('25') }}", 12 | "nvidia.com/gpu": "{{ queue.gpu_guaranteed | default('0') }}" 13 | }, 14 | "owners": [ 15 | 16 | ], 17 | "admins": [ 18 | 19 | ], 20 | "policy": { 21 | "validity": "NEVER EXPIRES", 22 | "allocation": { 23 | "distribution": "ELASTIC", 24 | "order": "FIFO", 25 | "queueing": false 26 | } 27 | }, 28 | "metadata": { 29 | 30 | }, 31 | "scheduling": { 32 | "preemption": { 33 | "enabled": true, 34 | "fence": false, 35 | "delay": 30 36 | }, 37 | "priority": { 38 | "enabled": true, 39 | "fence": false, 40 | "offset": 0 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/tools/setup_k8s_usage/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Git clone k8s usage resource 21 | git: 22 | repo: "https://github.com/frischHWC/k8s-resources-usage.git" 23 | dest: "~/k8s-resources-usage/" 24 | force: true 25 | 26 | - name: Make it executable 27 | file: 28 | path: ~/k8s-resources-usage/compute_k8s_resources.sh 29 | mode: '0755' 30 | -------------------------------------------------------------------------------- /playbooks/user_creation/unix_users/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Create user's group 21 | group: 22 | name: "{{ user.group }}" 23 | state: present 24 | 25 | - name: Create User 26 | user: 27 | name: "{{ user.name }}" 28 | shell: /bin/bash 29 | home: "/home/{{ user.name }}" 30 | groups: "{{ user.group }}" 31 | password: "{{ user.password | password_hash('sha512') }}" 32 | -------------------------------------------------------------------------------- /playbooks/data_load/install_as_a_service/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Install CSD & Parcel 21 | hosts: cloudera_manager 22 | gather_facts: false 23 | roles: 24 | - install_csd_and_parcel 25 | 26 | # If required, import java certs to the truststore one 27 | - name: Add public Certs 28 | hosts: "{{ edge_host }}" 29 | gather_facts: false 30 | roles: 31 | - role: add_public_certs 32 | when: tls -------------------------------------------------------------------------------- /playbooks/pre_install/auth_key_for_all/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy authorized_keys file 21 | copy: 22 | src: "{{ authorized_keys }}" 23 | dest: ~/authorized_keys 24 | 25 | - name: Setup authorized_keys 26 | shell: cat ~/authorized_keys >> ~/.ssh/authorized_keys 27 | 28 | - name: Destroy authorized_keys 29 | file: 30 | path: ~/authorized_keys 31 | state: absent -------------------------------------------------------------------------------- /playbooks/data_load/install_as_a_service/add_public_certs/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | # Pre-requisites for DATAGEN (installation of jq) 21 | - name: Import public certs to internal truststore 22 | shell: "keytool -importkeystore -srckeystore /etc/pki/java/cacerts -destkeystore /var/lib/cloudera-scm-agent/agent-cert/cm-auto-global_truststore.jks -srcstorepass changeit -deststorepass Cloudera1234" 23 | ignore_errors: true 24 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_ldap_provider/templates/ldap-1_5_5_SP1.json: -------------------------------------------------------------------------------- 1 | { 2 | "ldapProviderName": "cm-ldap", 3 | "skipGroupSyncOnLogin": false, 4 | "url": "ldaps://{{ ipa_server }}:636", 5 | "bindDn": "uid=admin,cn=users,cn=accounts,{% for i in realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}", 6 | "bindPassword": "{{ ipa_bind_password | default('') }}", 7 | "userSearchBase": "cn=users,cn=accounts,{% for i in realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}", 8 | "userSearchFilter": "(&(uid={0})(objectClass=person))", 9 | "groupSearchBase": "cn=groups,cn=accounts,{% for i in realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}", 10 | "groupSearchFilter": "(&(|(member={0})(member={1}))(objectClass=posixgroup)(!(cn=admins)))", 11 | "syncGroupsOnLogin": true, 12 | "emailMappingAttribute": "mail", 13 | "usernameMappingAttribute": "uid", 14 | "showAdvanced": false, 15 | "userBindProperty": "", 16 | "groupDnProperty": "", 17 | "firstNameMappingAttribute": "", 18 | "lastNameMappingAttribute": "", 19 | "groupNameMappingAttribute": "", 20 | "tlsCaCertificates": [ 21 | "{{ ipa_crt | replace('\n', '\\n') }}" 22 | ] 23 | } -------------------------------------------------------------------------------- /playbooks/post_install/simple_tools/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Update /etc/profile with new alias for ll command to ls -l if not already there 21 | lineinfile: 22 | path: ~/.bashrc 23 | line: "alias ll='ls -l'" 24 | create: yes 25 | 26 | - name: Update /etc/profile with new alias for tailf command to tail -f if not already there 27 | lineinfile: 28 | path: ~/.bashrc 29 | line: "alias tailf='tail -f'" 30 | create: yes 31 | 32 | -------------------------------------------------------------------------------- /playbooks/ansible_install_preparation/extra_vars.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | distribution: ${DISTRIBUTION_TO_DEPLOY} 21 | install_repo_url: ${INSTALL_REPO_URL} 22 | pvc_type: ${PVC_TYPE} 23 | pvc: ${PVC} 24 | debug: ${DEBUG} 25 | deploy_folder: ${TO_DEPLOY_FOLDER} 26 | ansible_repo: "${ANSIBLE_REPO_DIR}" 27 | encryption_activated: ${ENCRYPTION_ACTIVATED} 28 | cdh6_kts_path: ${CDH6_KTS_PATH} 29 | cdh6_kts_kms_path: ${CDH6_KTS_KMS_PATH} 30 | os: ${OS} 31 | os_version: "${OS_VERSION}" 32 | database_type: ${DATABASE_TYPE} -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cfm/nifi/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Set up namespace 21 | include_role: 22 | name: ../set_namespace 23 | vars: 24 | namespace: "{{ nifi.namespace }}" 25 | 26 | - name: DEBUG - values.yaml 27 | debug: 28 | msg: "{{ lookup('template', 'values.yaml') | from_yaml }}" 29 | when: debug | default(false) 30 | 31 | - name: Create Nifi 32 | kubernetes.core.k8s: 33 | state: present 34 | definition: "{{ lookup('template', 'values.yaml') | from_yaml }}" -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cfm/nifiregistry/templates/values.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cfm.cloudera.com/v1alpha1 2 | kind: NifiRegistry 3 | metadata: 4 | name: "{{ nifireg.name }}" 5 | namespace: "{{ nifireg.namespace }}" 6 | spec: 7 | resources: 8 | nifiRegistry: 9 | requests: 10 | cpu: "{{ nifireg.requests_cpu | default('200m') }}" 11 | memory: "{{ nifireg.requests_mem | default('1G') }}" 12 | limits: 13 | cpu: "{{ nifireg.limits_cpu | default('500m') }}" 14 | memory: "{{ nifireg.limits_mem | default('2G') }}" 15 | 16 | image: 17 | repository: "{{ docker_registry }}/cloudera/cfm-nifiregistry-k8s" 18 | tag: "{{ nifireg.tag }}" 19 | tiniImage: 20 | repository: "{{ docker_registry }}/cloudera/cfm-tini" 21 | tag: "{{ cfm_operator_version }}" 22 | 23 | hostName: "{{ nifireg.name }}.apps.{{ pvc_app_domain }}" 24 | uiConnection: 25 | serviceConfig: 26 | sessionAffinity: ClientIP 27 | type: Ingress 28 | ingressConfig: 29 | ingressTLS: 30 | - hosts: 31 | - "{{ nifireg.name }}.apps.{{ pvc_app_domain }}" 32 | secretName: "{{ nifireg.name }}-ingress-cert" 33 | paths: 34 | - port: 18080 35 | path: "/" 36 | annotations: 37 | nginx.ingress.kubernetes.io/affinity: cookie 38 | nginx.ingress.kubernetes.io/affinity-mode: persistent 39 | -------------------------------------------------------------------------------- /ansible-legacy/ansible-hdp-2/hosts: -------------------------------------------------------------------------------- 1 | ###################################################### 2 | ## 3 | ## The static inventory pua the hosa in different groups as described here: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#hosa-and-groups 4 | ## Variables would be defined directly for each host, including the ansible_user, the login user with sudoers permissions. 5 | ## For authentication, either the password or the SSH Private Key can be used. 6 | ## 7 | ###################################################### 8 | 9 | [hdp-edge] 10 | master01 ansible_host=${NODE_0} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 11 | 12 | [hdp-master-1] 13 | master03 ansible_host=${NODE_1} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 14 | 15 | [hdp-master-2] 16 | master02 ansible_host=${NODE_2} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 17 | 18 | [hdp-worker] 19 | worker01 ansible_host=${NODE_3} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 20 | worker02 ansible_host=${NODE_4} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 21 | worker03 ansible_host=${NODE_5} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack -------------------------------------------------------------------------------- /ansible-legacy/ansible-hdp-3/hosts: -------------------------------------------------------------------------------- 1 | ###################################################### 2 | ## 3 | ## The static inventory pua the hosa in different groups as described here: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#hosa-and-groups 4 | ## Variables would be defined directly for each host, including the ansible_user, the login user with sudoers permissions. 5 | ## For authentication, either the password or the SSH Private Key can be used. 6 | ## 7 | ###################################################### 8 | 9 | [hdp-edge] 10 | master01 ansible_host=${NODE_0} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 11 | 12 | [hdp-master-1] 13 | master03 ansible_host=${NODE_1} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 14 | 15 | [hdp-master-2] 16 | master02 ansible_host=${NODE_2} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 17 | 18 | [hdp-worker] 19 | worker01 ansible_host=${NODE_3} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 20 | worker02 ansible_host=${NODE_4} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack 21 | worker03 ansible_host=${NODE_5} ansible_user=${NODE_USER} ansible_ssh_private_key_file="~/.ssh/id_rsa" rack=/default-rack -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cfm/nifiregistry/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Set up namespace 21 | include_role: 22 | name: ../set_namespace 23 | vars: 24 | namespace: "{{ nifireg.namespace }}" 25 | 26 | - name: DEBUG - values.yaml 27 | debug: 28 | msg: "{{ lookup('template', 'values.yaml') | from_yaml }}" 29 | when: debug | default(false) 30 | 31 | - name: Create Nifi Registry 32 | kubernetes.core.k8s: 33 | state: present 34 | definition: "{{ lookup('template', 'values.yaml') | from_yaml }}" -------------------------------------------------------------------------------- /playbooks/user_creation/hdfs_paths/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Kinit as default admin_user 21 | shell: "kinit -kt /home/{{ default_admin_user }}/{{ default_admin_user }}.keytab {{ default_admin_user }}" 22 | 23 | - name: Create HDFS paths 24 | shell: | 25 | hdfs dfs -mkdir -p /user/{{ user.name }}/ 26 | hdfs dfs -chmod 755 /user/{{ user.name }}/ 27 | hdfs dfs -chown {{ user.name }}:{{ user.group }} /user/{{ user.name }}/ 28 | ignore_errors: true 29 | 30 | - name: Kdestroy when finished 31 | shell: kdestroy -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/enable_quota_management/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Enable Quota Management (on ECS PvC 1.5.2-4 only) 21 | shell: /bin/bash -i -c "kubectl -n cdp set env deployment/dp-mlx-control-plane-app ENABLE_UMBRA_INTEGRATION=true" 22 | 23 | - name: Enable Quota Management (on ECS PvC 1.5.2-4 only) 24 | shell: /bin/bash -i -c "kubectl -n cdp set env deployment/dp-cadence-worker ENABLE_UMBRA_INTEGRATION=true" 25 | 26 | - name: Sleep for 15 seconds for letting quota management to be enabled 27 | wait_for: 28 | timeout: 15 29 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/create_vc_v1.5.2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.name }}", 3 | "config": { 4 | "properties": { 5 | "livy.ingress.enabled": "true", 6 | "iceberg.enabled": "{{ virtual_cluster.iceberg_enabled | default('false') }}", 7 | "spark.version":"{{ virtual_cluster.spark_version }}", 8 | "dexapp.appTier":"{{ virtual_cluster.tier | default('tier2') }}", 9 | "session.timeout":"{{ virtual_cluster.session_timeout | default('8h') }}" 10 | }, 11 | "resources": { 12 | "cpu_requests": "{{ virtual_cluster.vc_quota_cpu | default('12') }}", 13 | "mem_requests": "{{ virtual_cluster.vc_quota_mem | default('32Gi') }}", 14 | "gpu_requests": "{{ virtual_cluster.vc_quota_gpu | default('0') }}", 15 | "gcpu_requests": "{{ virtual_cluster.vc_quota_min_cpu | default('8') }}", 16 | "gmem_requests": "{{ virtual_cluster.vc_quota_min_mem | default('24Gi') }}", 17 | "ggpu_requests": "{{ virtual_cluster.vc_quota_min_gpu | default('0') }}" 18 | }, 19 | "logRetention": { 20 | "enabled": {{ virtual_cluster.log_retention_enabled | default('true') }}, 21 | "retentionPeriod": {{ virtual_cluster.log_retention_days | default('10') }}, 22 | "retentionUnit": "days" 23 | }, 24 | "chartValueOverrides": 25 | { 26 | "dex-app": 27 | {"pipelines.enabled":"true"} 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/create_vc_v1.5.5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.name }}", 3 | "config": { 4 | "properties": { 5 | "livy.ingress.enabled": "true", 6 | "iceberg.enabled": "{{ virtual_cluster.iceberg_enabled | default('false') }}", 7 | "spark.version":"{{ virtual_cluster.spark_version }}", 8 | "dexapp.appTier":"{{ virtual_cluster.tier | default('tier2') }}", 9 | "session.timeout":"{{ virtual_cluster.session_timeout | default('8h') }}" 10 | }, 11 | "resources": { 12 | "cpu_requests": "{{ virtual_cluster.vc_quota_cpu | default('12') }}", 13 | "mem_requests": "{{ virtual_cluster.vc_quota_mem | default('32Gi') }}", 14 | "gpu_requests": "{{ virtual_cluster.vc_quota_gpu | default('0') }}", 15 | "gcpu_requests": "{{ virtual_cluster.vc_quota_min_cpu | default('8') }}", 16 | "gmem_requests": "{{ virtual_cluster.vc_quota_min_mem | default('24Gi') }}", 17 | "ggpu_requests": "{{ virtual_cluster.vc_quota_min_gpu | default('0') }}" 18 | }, 19 | "logRetention": { 20 | "enabled": {{ virtual_cluster.log_retention_enabled | default('true') }}, 21 | "retentionPeriod": {{ virtual_cluster.log_retention_days | default('10') }}, 22 | "retentionUnit": "days" 23 | }, 24 | "chartValueOverrides": 25 | { 26 | "dex-app": 27 | {"pipelines.enabled":"true"} 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /launch.sh: -------------------------------------------------------------------------------- 1 | export PAYWALL_USER= 2 | export PAYWALL_PASSWORD= 3 | export LICENSE_FILE="/root/license.txt" 4 | export NODE_KEY="/root/frisch.pem" 5 | 6 | export REALM=FRISCH.CLOUDERA.COM 7 | export CLUSTER_NAME="bootcamp" 8 | 9 | 10 | 11 | ./setup-cluster.sh \ 12 | --cluster-name=${CLUSTER_NAME} \ 13 | --realm=${REALM} \ 14 | \ 15 | --license-file=${LICENSE_FILE} \ 16 | \ 17 | --paywall-username=${PAYWALL_USER} \ 18 | --paywall-password=${PAYWALL_PASSWORD} \ 19 | \ 20 | --node-user="root" \ 21 | --node-key=${NODE_KEY} \ 22 | \ 23 | --cluster-type="pvc" \ 24 | \ 25 | --pre-install=true \ 26 | --prepare-ansible-deployment=true \ 27 | --install=true \ 28 | --post-install=true \ 29 | --user-creation=true \ 30 | --install-pvc=true \ 31 | --configure-pvc=true \ 32 | --create-cdw=true \ 33 | --create-cml=true \ 34 | --create-cde=true \ 35 | --create-viz=true \ 36 | --data-load=true \ 37 | --free-ipa=true \ 38 | --debug=true \ 39 | \ 40 | --os="rhel" \ 41 | --os-version="8.6" \ 42 | \ 43 | --node-ipa="fri-ipa.cloudera.local" \ 44 | --nodes-base="fri-base-1.cloudera.local fri-base-2.cloudera.local fri-base-3.cloudera.local fri-base-4.cloudera.local fri-base-5.cloudera.local fri-base-6.cloudera.local" \ 45 | --nodes-ecs="fri-ecs-1.cloudera.local fri-ecs-2.cloudera.local fri-ecs-3.cloudera.local" -------------------------------------------------------------------------------- /playbooks/data_load/ranger_policies/push_policies/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Post Ranger policies declared in policies directory 21 | uri: 22 | url: "{{ ranger_url }}/service/public/v2/api/policy" 23 | method: POST 24 | user: "{{ ranger_user }}" 25 | password: "{{ ranger_password }}" 26 | return_content: yes 27 | body: "{{ lookup('template', '{{ item.src }}' ) }}" 28 | body_format: json 29 | status_code: 200 30 | validate_certs: no 31 | force_basic_auth: yes 32 | with_filetree: "{{ role_path }}/policies" 33 | ignore_errors: yes 34 | -------------------------------------------------------------------------------- /playbooks/restart_paused_cluster/all_clusters/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # 18 | --- 19 | - name: Get all clusters 20 | uri: 21 | url: "{{ cloudera_manager_api_url }}/clusters?clusterType=any" 22 | user: "{{ cloudera_manager_user }}" 23 | password: "{{ cloudera_manager_password }}" 24 | method: GET 25 | force_basic_auth: yes 26 | status_code: 200 27 | return_content: yes 28 | validate_certs: no 29 | body_format: json 30 | register: clusters_list 31 | 32 | - name: DEBUG - clusters_list 33 | debug: 34 | msg: "{{ clusters_list }}" 35 | when: debug | default(false) 36 | -------------------------------------------------------------------------------- /playbooks/data_load/README.md: -------------------------------------------------------------------------------- 1 | # Data generator and loading 2 | 3 | Intent of this project is to provide ansible scripts to automate the data generation and loading into different services of a CDP cluster (HDFS, HBase, Hive, Ozone, Kafka, Kudu, SolR). 4 | It is designed to be flexible by letting the user choose which data to generate, to which services, and user-friendly by setting a minimum of configuration while letting the user configure as much as wanted, in a single configuration file. 5 | 6 | ## Design 7 | 8 | All configurations are expected to be set in the extra_vars.yml file. 9 | 10 | There are three playbooks that could run independently but are in fact tied together by some variables used from one playbook to the other. 11 | 12 | ### Auto-configure 13 | 14 | Using CM, this playbook catches all required configurations to launch generation of data. 15 | Its goal is to facilitate configuration required by each service to generate by picking up as much as it can configurations. 16 | 17 | ### Ranger Policies 18 | 19 | This playbook aims at pushing required policies for data generation into Ranger, in order to avoid rights possible problems. 20 | 21 | ### Generate Data 22 | 23 | This playbook generates random data by launching multiple times a random-datagen program. 24 | 25 | 26 | ## How to launch it ? 27 | 28 | 29 | Run the below command: 30 | 31 | ansible-playbook -i main.yml --extra-vars "@extra_vars.yml" 32 | 33 | -------------------------------------------------------------------------------- /playbooks/pre_install/freeipa_dbus_fix/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Copy DBUS config file 21 | copy: 22 | src: dbus_session.conf 23 | dest: /etc/dbus-1/session-local.conf 24 | ignore_errors: true 25 | 26 | - name: Ensure dbus is enabled and unmasked 27 | systemd: 28 | name: dbus 29 | enabled: yes 30 | masked: no 31 | ignore_errors: true 32 | 33 | - name: Restart DBUS 34 | service: 35 | name: dbus 36 | state: restarted 37 | ignore_errors: true 38 | 39 | - name: Restart logind 40 | service: 41 | name: systemd-logind 42 | state: restarted 43 | ignore_errors: true 44 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cde/create_cde/templates/activate_cde_v1.4.1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "{{ virtual_cluster.cde_service_name }}", 3 | "env": "{{ virtual_cluster.env_name }}", 4 | "config": { 5 | "properties": { 6 | "loadbalancer.internal": "true", 7 | "dbus-wxm-client.enabled": "false", 8 | "kubernetes.api.allowList": "", 9 | "mount.SSD": "false", 10 | "loadbalancer.allowList": "", 11 | "kubernetes.api.proxyCIDRList": "", 12 | "subnets": "", 13 | "tls.enabled": "false", 14 | "yunikorn.disabled": "false", 15 | "shared.cluster.enabled":"true", 16 | "default-vc.create":"false", 17 | "wildcard.certificates.enabled":"false", 18 | "cluster.nfs.storageClass":"{{ virtual_cluster.storage_class | default('') }}" 19 | }, 20 | "resources": { 21 | "cluster_resource_pool_quota": { 22 | "cpu_requests": "{{ virtual_cluster.service_pool_quota_cpu | default('32') }}", 23 | "mem_requests": "{{ virtual_cluster.service_pool_quota_mem | default('100Gi') }}" 24 | }, 25 | "base_resource_pool_quota": { 26 | "cpu_requests": "{{ virtual_cluster.service_base_quota_cpu | default('8') }}", 27 | "mem_requests": "{{ virtual_cluster.service_base_quota_mem | default('20Gi') }}" 28 | }, 29 | "resource_pool": "root.default" 30 | }, 31 | 32 | "tags": {} 33 | }, 34 | "skipValidation": true 35 | } 36 | 37 | -------------------------------------------------------------------------------- /playbooks/user_creation/users_directories/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Change file ownership of home user 21 | file: 22 | path: "/home/{{ user.name }}/" 23 | owner: "{{ user.name }}" 24 | group: "{{ user.group }}" 25 | mode: '0755' 26 | state: directory 27 | 28 | - name: Create .ssh directory 29 | file: 30 | path: "/home/{{ user.name }}/.ssh/" 31 | state: directory 32 | mode: '0755' 33 | ignore_errors: true 34 | 35 | - name: Copy authorized_keys file to user's folder 36 | copy: 37 | src: ~/.ssh/authorized_keys 38 | dest: "/home/{{ user.name }}/.ssh/authorized_keys" 39 | ignore_errors: true 40 | -------------------------------------------------------------------------------- /playbooks/ansible_install_preparation/hdp_libs/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Install libs required by HDP 21 | package: 22 | name: 23 | - gcc 24 | - gcc-c++ 25 | - python-virtualenv 26 | - python-pip 27 | - python-devel 28 | - libffi-devel 29 | - openssl-devel 30 | - libyaml-devel 31 | - sshpass 32 | - vim-enhanced 33 | - epel-release 34 | state: present 35 | ignore_errors: yes 36 | 37 | - name: Install the epel-release rpm from fedora remote repo required by HDP 38 | yum: 39 | name: http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 40 | state: present 41 | ignore_errors: yes 42 | -------------------------------------------------------------------------------- /playbooks/restart_paused_cluster/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Check and wait for CM up 21 | hosts: cloudera_manager 22 | gather_facts: false 23 | roles: 24 | - role: cm_config 25 | 26 | - name: List all clusters 27 | hosts: cloudera_manager 28 | gather_facts: false 29 | roles: 30 | - role: all_clusters 31 | 32 | - name: Check and Heal cluster 33 | hosts: cloudera_manager 34 | gather_facts: false 35 | tasks: 36 | - name: cluster_check_and_heal 37 | include_role: 38 | name: cluster_check_and_heal 39 | with_items: "{{ clusters_list.json | community.general.json_query('items[]') }}" 40 | vars: 41 | cluster: "{{ item }}" 42 | 43 | -------------------------------------------------------------------------------- /playbooks/pvc_setup/cml/create_cml_inf/templates/create_cml_inf.json: -------------------------------------------------------------------------------- 1 | { 2 | "appName":"{{ ml_inf.name }}", 3 | "namespace":"{{ ml_inf.name }}", 4 | "skipValidation":true, 5 | "environmentCrn":"{{ env_crn }}", 6 | "clusterCrn":"{{ env_crn }}", 7 | "provisionK8sRequest":{ 8 | "instanceGroups":[ 9 | { 10 | "instanceType":"c6a.16xlarge", 11 | "instanceTier":"", 12 | "instanceCount":10, 13 | "name":"", 14 | "ingressRules":[ 15 | 16 | ], 17 | "autoscaling":{ 18 | "minInstances":10, 19 | "maxInstances":10, 20 | "enabled":true 21 | } 22 | } 23 | ], 24 | "environmentCrn":"{{ env_crn }}", 25 | "network":{ 26 | "plugin":"", 27 | "options":{ 28 | "encryption":"" 29 | }, 30 | "topology":{ 31 | "subnets":[ 32 | "" 33 | ] 34 | }, 35 | "serviceCidr":"", 36 | "inboundProxyCidrs":[ 37 | "" 38 | ], 39 | "podCidr":"" 40 | }, 41 | "xEntitlements":[ 42 | "" 43 | ] 44 | }, 45 | "usePublicLoadBalancer":false, 46 | "isPrivateCluster":true, 47 | "mlservingVersion":"", 48 | "ozoneS3Creds":{ 49 | "s3AccessID":"{{ ozone_aws_access_key }}", 50 | "s3SecretAccessKey":"{{ ozone_aws_secret_access_key }}", 51 | "s3EndPointUrl":"{{ ozone_s3_full_endpoint }}" 52 | } 53 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/setup_yunikorn_root/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Setup root Queue 21 | uri: 22 | url: "{{ pvc_api_call }}/compute/updateResourcePool" 23 | method: POST 24 | headers: 25 | Content-Type: application/json 26 | Accept: application/json 27 | Cookie: "{{ all_cookies }}" 28 | body: "{{ lookup('template', 'root_queue.json') }}" 29 | body_format: json 30 | status_code: [200, 500] 31 | return_content: yes 32 | validate_certs: no 33 | register: root_queue_creation 34 | ignore_errors: true 35 | 36 | - name: DEBUG - root_queue_creation 37 | debug: 38 | msg: "{{ root_queue_creation }}" 39 | when: debug | default(false) -------------------------------------------------------------------------------- /playbooks/user_creation/ranger_policies/policies/admin/hive-storage.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_hive", 4 | "name": "Hive Storage Policy for user {{ user.name }}", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "Admin Storage Policy for user {{ user.name }}", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "storage-type": { 11 | "values": [ 12 | "*" 13 | ], 14 | "isExcludes": false, 15 | "isRecursive": false 16 | }, 17 | "storage-url": { 18 | "values": [ 19 | "*", 20 | "{{ user.name }}*" 21 | ], 22 | "isExcludes": false, 23 | "isRecursive": false 24 | } 25 | }, 26 | "policyItems": [ 27 | { 28 | "accesses": [ 29 | { 30 | "type": "rwstorage", 31 | "isAllowed": true 32 | } 33 | ], 34 | "users": [ 35 | "{{ user.name }}" 36 | ], 37 | "groups": [ 38 | 39 | ], 40 | "roles": [ 41 | 42 | ], 43 | "conditions": [ 44 | 45 | ], 46 | "delegateAdmin": true 47 | } 48 | ], 49 | "denyPolicyItems": [ 50 | 51 | ], 52 | "allowExceptions": [ 53 | 54 | ], 55 | "denyExceptions": [ 56 | 57 | ], 58 | "dataMaskPolicyItems": [ 59 | 60 | ], 61 | "rowFilterPolicyItems": [ 62 | 63 | ], 64 | "serviceType": "hive", 65 | "options": { 66 | 67 | }, 68 | "validitySchedules": [ 69 | 70 | ], 71 | "policyLabels": [ 72 | 73 | ], 74 | "zoneName": "", 75 | "isDenyAllElse": false 76 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/create_hive_vw_v1.5.2.json: -------------------------------------------------------------------------------- 1 | { 2 | "computeName": "{{ warehouse.name }}", 3 | "warehouseId": "{{ dwx_dbc_id }}", 4 | "template": "{{ warehouse.size | default(xsmall) }}", 5 | "resourcePool": "{{ warehouse.vw_queue | default('root.default') }}", 6 | "enablePlatformJWTAuth": false, 7 | "autoScaling": { 8 | "minClusters": {{ warehouse.min_nodes | default('1') }}, 9 | "maxClusters": {{ warehouse.max_nodes | default('5') }}, 10 | "autoSuspendTimeoutSeconds": {{ warehouse.auto_suspend_timeout | default('300') }}, 11 | "waitTimeSeconds": {{ warehouse.wait_time | default('60') }}, 12 | "disableAutoSuspend": {{ warehouse.disable_auto_suspend | default('false') }}, 13 | "maxQueries": {{ warehouse.max_queries | default('2') }}, 14 | "maxNodesPerQuery": {{ warehouse.max_nodes_per_query | default('4') }}, 15 | "nodeCount": {{ warehouse.node_count | default('3') }}, 16 | "enableQueryIsolation": {{ warehouse.enable_query_isolation | default('false') }}, 17 | "autoScaleType": "{{ warehouse.auto_scale_type | default('Wait Time') }}" 18 | }, 19 | "queryIsolation": { 20 | "maxQueries": {{ warehouse.query_isolation_max_queries | default('0') }}, 21 | "maxNodesPerQuery": {{ warehouse.query_isolation_max_nodes_per_query | default('0') }} 22 | }, 23 | "config": { 24 | "ldapGroups": [ 25 | "{{ warehouse.allowed_group | default('') }}" 26 | ], 27 | "enableSSO": true 28 | }, 29 | "tags": [], 30 | } -------------------------------------------------------------------------------- /playbooks/ansible_install_preparation/set_install_repo/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Create ansible deployment directory 21 | file: 22 | path: ~/deployment/ 23 | state: directory 24 | mode: '0755' 25 | 26 | - name: Remove old ansible-repo 27 | file: 28 | name: ~/deployment/ansible-repo/ 29 | state: absent 30 | ignore_errors: yes 31 | 32 | - name: Get repository as a zip and unzip it 33 | unarchive: 34 | src: "{{ install_repo_url }}" 35 | dest: ~/deployment/ 36 | remote_src: yes 37 | 38 | - name: Copy content of repository to ansible-repo 39 | copy: 40 | src: "~/deployment/{{ ansible_repo }}/" 41 | dest: ~/deployment/ansible-repo/ 42 | remote_src: yes 43 | -------------------------------------------------------------------------------- /playbooks/pre_install/copy_certs/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Delete cert directory 21 | file: 22 | state: absent 23 | path: "/tmp/certs/" 24 | 25 | - name: Create cert directory 26 | file: 27 | path: "/tmp/certs/" 28 | state: directory 29 | owner: "root" 30 | group: "root" 31 | mode: '0775' 32 | recurse: yes 33 | 34 | - name: Copy of cert archive 35 | copy: 36 | src: "~/cluster-{{ cluster_name }}/cert.tgz" 37 | dest: /tmp/cert.tgz 38 | owner: "root" 39 | group: "root" 40 | mode: '0640' 41 | 42 | - name: Extract certs archive 43 | unarchive: 44 | src: /tmp/cert.tgz 45 | dest: /tmp/certs/ 46 | mode: '0640' 47 | remote_src: true 48 | -------------------------------------------------------------------------------- /playbooks/data_load/ranger_policies/push_policies/policies/yarn.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 1, 3 | "guid": "...", 4 | "isEnabled": true, 5 | "version": 1, 6 | "service": "cm_yarn", 7 | "name": "random-datagen {{ user }}", 8 | "policyType": 0, 9 | "policyPriority": 0, 10 | "description": "Policy for {{ user }} user", 11 | "isAuditEnabled": true, 12 | "resources": { 13 | "queue": { 14 | "values": [ 15 | "root.default" 16 | ], 17 | "isExcludes": false, 18 | "isRecursive": true 19 | } 20 | }, 21 | "policyItems": [ 22 | { 23 | "accesses": [ 24 | { 25 | "type": "submit-app", 26 | "isAllowed": true 27 | } 28 | ], 29 | "users": [ 30 | "{{ user }}" 31 | ], 32 | "groups": [ 33 | 34 | ], 35 | "roles": [ 36 | 37 | ], 38 | "conditions": [ 39 | 40 | ], 41 | "delegateAdmin": true 42 | } 43 | ], 44 | "denyPolicyItems": [ 45 | 46 | ], 47 | "allowExceptions": [ 48 | 49 | ], 50 | "denyExceptions": [ 51 | 52 | ], 53 | "dataMaskPolicyItems": [ 54 | 55 | ], 56 | "rowFilterPolicyItems": [ 57 | 58 | ], 59 | "serviceType": "yarn", 60 | "options": { 61 | 62 | }, 63 | "validitySchedules": [ 64 | 65 | ], 66 | "policyLabels": [ 67 | 68 | ], 69 | "zoneName": "", 70 | "isDenyAllElse": false 71 | } -------------------------------------------------------------------------------- /playbooks/pvc_post_install/cfm/nifi/templates/values.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cfm.cloudera.com/v1alpha1 2 | kind: Nifi 3 | metadata: 4 | name: "{{ nifi.name }}" 5 | namespace: "{{ nifi.namespace }}" 6 | spec: 7 | replicas: {{ nifi.replicas }} 8 | resources: 9 | nifi: 10 | requests: 11 | cpu: "{{ nifi.requests_cpu | default('200m') }}" 12 | memory: "{{ nifi.requests_mem | default('2G') }}" 13 | limits: 14 | cpu: "{{ nifi.limits_cpu | default('1') }}" 15 | memory: "{{ nifi.limits_mem | default('4G') }}" 16 | image: 17 | repository: "{{ docker_registry }}/cloudera/cfm-nifi-k8s" 18 | tag: "{{ nifi.tag }}" 19 | tiniImage: 20 | repository: "{{ docker_registry }}/cloudera/cfm-tini" 21 | tag: "{{ cfm_operator_version }}" 22 | hostName: "{{ nifi.name }}.apps.{{ pvc_app_domain }}" 23 | uiConnection: 24 | serviceConfig: 25 | sessionAffinity: ClientIP 26 | type: Ingress 27 | ingressConfig: 28 | ingressTLS: 29 | - hosts: 30 | - "{{ nifi.name }}.apps.{{ pvc_app_domain }}" 31 | secretName: "{{ nifi.name }}-ingress-cert" 32 | paths: 33 | - port: 8080 34 | path: "/" 35 | annotations: 36 | nginx.ingress.kubernetes.io/affinity: cookie 37 | nginx.ingress.kubernetes.io/affinity-mode: persistent 38 | 39 | configOverride: 40 | nifiProperties: 41 | upsert: 42 | nifi.sensitive.props.algorithm: "NIFI_PBKDF2_AES_GCM_256" 43 | bootstrapConf: 44 | upsert: 45 | java.arg.2: -Xms2g 46 | java.arg.3: -Xmx4g 47 | -------------------------------------------------------------------------------- /playbooks/data_load/auto-configure/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Auto-Configure 21 | hosts: edge 22 | gather_facts: false 23 | roles: 24 | - cm-config 25 | - all-config 26 | - { role: ranger-config, when: config_ranger } 27 | - { role: hive-config, when: config_hive } 28 | - { role: hbase-config, when: config_hbase } 29 | - { role: zookeeper-config, when: config_zookeeper } 30 | - { role: solr-config, when: config_solr } 31 | - { role: kudu-config, when: config_kudu } 32 | - { role: ozone-config, when: config_ozone } 33 | - { role: hdfs-config, when: config_hdfs } 34 | - { role: schema-registry-config, when: config_schema_registry } 35 | - { role: kafka-config, when: config_kafka } -------------------------------------------------------------------------------- /playbooks/data_load/generate_data/add_service/templates/datagen_service.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": 3 | [ 4 | { 5 | "tags" : [ ], 6 | "name" : "datagen", 7 | "type" : "DATAGEN", 8 | "serviceVersion" : "CDH {{ cdp_version[:5] }}", 9 | "displayName" : "Datagen", 10 | "roles": [ 11 | { 12 | "name" : "DATAGEN_SERVER", 13 | "type" : "DATAGEN_SERVER", 14 | "hostRef" : { 15 | "hostId" : "{{ edge_host_id }}", 16 | "hostname" : "{{ edge_host }}" 17 | }, 18 | "config" : { 19 | "items" : 20 | [ 21 | { 22 | "name" : "threads", 23 | "value" : 10 24 | }, 25 | { 26 | "name" : "ranger_url", 27 | "value" : "{{ ranger_url }}" 28 | }, 29 | { 30 | "name" : "ranger_password", 31 | "value" : "{{ ranger_password }}" 32 | } 33 | ] 34 | } 35 | } 36 | ] 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /playbooks/user_creation/extra_vars.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | default_password: ${DEFAULT_PASSWORD} 21 | free_ipa: ${FREE_IPA} 22 | cloudera_manager_host: ${NODE_0} 23 | cloudera_manager_user: admin 24 | cloudera_manager_password: admin 25 | ranger_user: admin 26 | ranger_password: ${DEFAULT_PASSWORD} 27 | auto_tls: ${TLS} 28 | ipa_server: ${NODE_IPA} 29 | cluster_name: ${CLUSTER_NAME} 30 | debug: ${DEBUG} 31 | default_admin_user: ${DEFAULT_ADMIN_USER} 32 | kerberos: ${KERBEROS} 33 | 34 | users: 35 | - name: ${DEFAULT_ADMIN_USER} 36 | password: ${DEFAULT_PASSWORD} 37 | group: admin_group 38 | is_admin: true 39 | - name: superman 40 | password: ${DEFAULT_PASSWORD} 41 | group: hero 42 | - name: batman 43 | password: ${DEFAULT_PASSWORD} 44 | group: hero -------------------------------------------------------------------------------- /playbooks/pvc_setup/cdw/create_cdw/templates/create_hive_vw_v1.5.4.json: -------------------------------------------------------------------------------- 1 | { 2 | "computeName": "{{ warehouse.name }}", 3 | "warehouseId": "{{ dwx_dbc_id }}", 4 | "template": "{{ warehouse.size | default(xsmall) }}", 5 | "resourcePool": "{{ warehouse.vw_queue | default('root.default') }}", 6 | "resourceTemplate": "{{ hive_template_id | default('hive-default') }}", 7 | "enablePlatformJWTAuth": false, 8 | "autoScaling": { 9 | "minClusters": {{ warehouse.min_nodes | default('1') }}, 10 | "maxClusters": {{ warehouse.max_nodes | default('5') }}, 11 | "autoSuspendTimeoutSeconds": {{ warehouse.auto_suspend_timeout | default('300') }}, 12 | "waitTimeSeconds": {{ warehouse.wait_time | default('60') }}, 13 | "disableAutoSuspend": {{ warehouse.disable_auto_suspend | default('false') }}, 14 | "maxQueries": {{ warehouse.max_queries | default('2') }}, 15 | "maxNodesPerQuery": {{ warehouse.max_nodes_per_query | default('4') }}, 16 | "nodeCount": {{ warehouse.node_count | default('3') }}, 17 | "enableQueryIsolation": {{ warehouse.enable_query_isolation | default('false') }}, 18 | "autoScaleType": "{{ warehouse.auto_scale_type | default('Wait Time') }}" 19 | }, 20 | "queryIsolation": { 21 | "maxQueries": {{ warehouse.query_isolation_max_queries | default('0') }}, 22 | "maxNodesPerQuery": {{ warehouse.query_isolation_max_nodes_per_query | default('0') }} 23 | }, 24 | "config": { 25 | "ldapGroups": [ 26 | "{{ warehouse.allowed_group | default('') }}" 27 | ], 28 | "enableSSO": true 29 | }, 30 | "tags": [], 31 | } -------------------------------------------------------------------------------- /playbooks/post_install/ranger_extra_policies/policies/solr-admin.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_solr", 4 | "name": "SolR all collections", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "Policy for admin users on collections", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "admin": { 11 | "values": [ 12 | "*", 13 | "tmp_*" 14 | ], 15 | "isExcludes": false, 16 | "isRecursive": false 17 | } 18 | }, 19 | "policyItems": [ 20 | { 21 | "accesses": [ 22 | { 23 | "type": "query", 24 | "isAllowed": true 25 | }, 26 | { 27 | "type": "update", 28 | "isAllowed": true 29 | } 30 | ], 31 | "users": [ 32 | "solr", 33 | "hue" 34 | ], 35 | "groups": [ 36 | 37 | ], 38 | "roles": [ 39 | 40 | ], 41 | "conditions": [ 42 | 43 | ], 44 | "delegateAdmin": true 45 | } 46 | ], 47 | "denyPolicyItems": [ 48 | 49 | ], 50 | "allowExceptions": [ 51 | 52 | ], 53 | "denyExceptions": [ 54 | 55 | ], 56 | "dataMaskPolicyItems": [ 57 | 58 | ], 59 | "rowFilterPolicyItems": [ 60 | 61 | ], 62 | "serviceType": "solr", 63 | "options": { 64 | 65 | }, 66 | "validitySchedules": [ 67 | 68 | ], 69 | "policyLabels": [ 70 | 71 | ], 72 | "zoneName": "", 73 | "isDenyAllElse": false 74 | } -------------------------------------------------------------------------------- /playbooks/post_install/ranger_extra_policies/policies/solr.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_solr", 4 | "name": "SolR all collections", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "Policy for admin users on collections", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "collection": { 11 | "values": [ 12 | "*", 13 | "tmp_*" 14 | ], 15 | "isExcludes": false, 16 | "isRecursive": false 17 | } 18 | }, 19 | "policyItems": [ 20 | { 21 | "accesses": [ 22 | { 23 | "type": "query", 24 | "isAllowed": true 25 | }, 26 | { 27 | "type": "update", 28 | "isAllowed": true 29 | } 30 | ], 31 | "users": [ 32 | "solr", 33 | "hue" 34 | ], 35 | "groups": [ 36 | 37 | ], 38 | "roles": [ 39 | 40 | ], 41 | "conditions": [ 42 | 43 | ], 44 | "delegateAdmin": true 45 | } 46 | ], 47 | "denyPolicyItems": [ 48 | 49 | ], 50 | "allowExceptions": [ 51 | 52 | ], 53 | "denyExceptions": [ 54 | 55 | ], 56 | "dataMaskPolicyItems": [ 57 | 58 | ], 59 | "rowFilterPolicyItems": [ 60 | 61 | ], 62 | "serviceType": "solr", 63 | "options": { 64 | 65 | }, 66 | "validitySchedules": [ 67 | 68 | ], 69 | "policyLabels": [ 70 | 71 | ], 72 | "zoneName": "", 73 | "isDenyAllElse": false 74 | } -------------------------------------------------------------------------------- /playbooks/user_creation/ranger_policies/policies/admin/yarn.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_yarn", 4 | "name": "YARN Policy for user {{ user.name }}", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "Admin Policy for user {{ user.name }}", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "queue": { 11 | "values": [ 12 | "*", 13 | "{{ user.name }}*" 14 | ], 15 | "isExcludes": false, 16 | "isRecursive": true 17 | } 18 | }, 19 | "policyItems": [ 20 | { 21 | "accesses": [ 22 | { 23 | "type": "submit-app", 24 | "isAllowed": true 25 | }, 26 | { 27 | "type": "admin-queue", 28 | "isAllowed": true 29 | } 30 | ], 31 | "users": [ 32 | "{{ user.name }}" 33 | ], 34 | "groups": [ 35 | ], 36 | "roles": [ 37 | 38 | ], 39 | "conditions": [ 40 | 41 | ], 42 | "delegateAdmin": true 43 | } 44 | ], 45 | "denyPolicyItems": [ 46 | 47 | ], 48 | "allowExceptions": [ 49 | 50 | ], 51 | "denyExceptions": [ 52 | 53 | ], 54 | "dataMaskPolicyItems": [ 55 | 56 | ], 57 | "rowFilterPolicyItems": [ 58 | 59 | ], 60 | "serviceType": "yarn", 61 | "options": { 62 | 63 | }, 64 | "validitySchedules": [ 65 | 66 | ], 67 | "policyLabels": [ 68 | 69 | ], 70 | "zoneName": "", 71 | "isDenyAllElse": false 72 | } -------------------------------------------------------------------------------- /playbooks/ansible_install_preparation/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Install required packages for ansible installation 21 | hosts: main 22 | gather_facts: false 23 | become: true 24 | roles: 25 | - ansible_install 26 | - { role: hdp_libs, when: distribution == "HDP" } 27 | 28 | - name: Install required package repository for Postgres DB 29 | hosts: all 30 | gather_facts: false 31 | become: true 32 | roles: 33 | - { role: centos_extra_packages, when: os == 'centos' and database_type == 'postgresql' } 34 | 35 | - name: Copy files for installation 36 | hosts: main 37 | gather_facts: false 38 | roles: 39 | - set_install_repo 40 | - { role: cdp_ansible_files, when: distribution != "HDP" } 41 | - { role: hdp_ansible_files, when: distribution == "HDP" } 42 | 43 | -------------------------------------------------------------------------------- /playbooks/post_install/fix_krb5_conf_freeipa/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | - name: Adding enctypes and renewable ticket lifetime 21 | blockinfile: 22 | dest: "/etc/krb5.conf" 23 | insertafter: 'ticket_lifetime = 24h' 24 | block: | 25 | default_tgs_enctypes= aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des-cbc-md5 26 | default_tkt_enctypes = aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des-cbc-md5 27 | renew_lifetime = 7d 28 | max_life = 365d 29 | max_renewable_life = 365d 30 | ignore_errors: true 31 | 32 | - name: Comment default_ccache_name in krb5.conf 33 | replace: 34 | dest: /etc/krb5.conf 35 | regexp: 'default_ccache_name = KEYRING:persistent:%{uid}' 36 | replace: '#default_ccache_name = KEYRING:persistent:%{uid}' 37 | ignore_errors: true 38 | -------------------------------------------------------------------------------- /playbooks/user_creation/ranger_policies/policies/admin/solr-admin.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_solr", 4 | "name": "SolR Admin Policy for {{ user.name }}", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "Admin Admin Policy for user {{ user.name }}", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "admin": { 11 | "values": [ 12 | "collections" 13 | ], 14 | "isExcludes": false, 15 | "isRecursive": false 16 | } 17 | }, 18 | "conditions": [], 19 | "policyItems": [ 20 | { 21 | "accesses": [ 22 | { 23 | "type": "query", 24 | "isAllowed": true 25 | }, 26 | { 27 | "type": "update", 28 | "isAllowed": true 29 | } 30 | ], 31 | "users": [ 32 | "{{ user.name }}" 33 | ], 34 | "groups": [ 35 | 36 | ], 37 | "roles": [ 38 | 39 | ], 40 | "conditions": [ 41 | 42 | ], 43 | "delegateAdmin": true 44 | } 45 | ], 46 | "denyPolicyItems": [ 47 | 48 | ], 49 | "allowExceptions": [ 50 | 51 | ], 52 | "denyExceptions": [ 53 | 54 | ], 55 | "dataMaskPolicyItems": [ 56 | 57 | ], 58 | "rowFilterPolicyItems": [ 59 | 60 | ], 61 | "serviceType": "solr", 62 | "options": { 63 | 64 | }, 65 | "validitySchedules": [ 66 | 67 | ], 68 | "policyLabels": [ 69 | 70 | ], 71 | "zoneName": "", 72 | "isDenyAllElse": false 73 | } -------------------------------------------------------------------------------- /playbooks/pre_install/extra_vars.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | distribution: ${DISTRIBUTION_TO_DEPLOY} 21 | debug: ${DEBUG} 22 | free_ipa: ${FREE_IPA} 23 | cluster_name: ${CLUSTER_NAME} 24 | etc_hosts_file: ${HOSTS_ETC} 25 | set_etc_hosts: ${SETUP_ETC_HOSTS} 26 | known_hosts: ${KNOWN_HOSTS} 27 | license_file: ${LICENSE_FILE} 28 | authorized_keys: ${AUTHORIZED_KEYS} 29 | set_hosts_key: ${SETUP_HOSTS_KEYS} 30 | node_key: ${NODE_KEY} 31 | use_root_ca: ${USE_ROOT_CA} 32 | root_ca_cert: ${ROOT_CA_CERT} 33 | root_ca_key: ${ROOT_CA_KEY} 34 | ca_server_attrs: /C=FR/ST=Paris/L=Paris/O=Cloudera/OU=PS/ 35 | certs_dir: "~/cluster-{{ cluster_name }}/certs/" 36 | keystore_password: ${DEFAULT_PASSWORD} 37 | truststore_password: ${DEFAULT_PASSWORD} 38 | pvc: ${PVC} 39 | pvc_app_domain: ${PVC_APP_DOMAIN} 40 | set_python3_link: ${SET_PYTHON_3_LINK} 41 | -------------------------------------------------------------------------------- /playbooks/post_install/ranger_extra_policies/policies/hdfs_user.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_hdfs", 4 | "name": "Policy fo Users on their Home Directory", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "path": { 11 | "values": [ 12 | "/user/{USER}" 13 | ], 14 | "isExcludes": false, 15 | "isRecursive": true 16 | } 17 | }, 18 | "policyItems": [ 19 | { 20 | "accesses": [ 21 | { 22 | "type": "read", 23 | "isAllowed": true 24 | }, 25 | { 26 | "type": "write", 27 | "isAllowed": true 28 | }, 29 | { 30 | "type": "execute", 31 | "isAllowed": true 32 | } 33 | ], 34 | "users": [ 35 | "{USER}" 36 | ], 37 | "groups": [ 38 | ], 39 | "roles": [ 40 | 41 | ], 42 | "conditions": [ 43 | 44 | ], 45 | "delegateAdmin": true 46 | } 47 | ], 48 | "denyPolicyItems": [ 49 | 50 | ], 51 | "allowExceptions": [ 52 | 53 | ], 54 | "denyExceptions": [ 55 | 56 | ], 57 | "dataMaskPolicyItems": [ 58 | 59 | ], 60 | "rowFilterPolicyItems": [ 61 | 62 | ], 63 | "serviceType": "hdfs", 64 | "options": { 65 | 66 | }, 67 | "validitySchedules": [ 68 | 69 | ], 70 | "policyLabels": [ 71 | 72 | ], 73 | "zoneName": "", 74 | "isDenyAllElse": false 75 | } -------------------------------------------------------------------------------- /playbooks/user_creation/ranger_policies/policies/admin/solr.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_solr", 4 | "name": "SolR Policy for {{ user.name }}", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "Admin Policy for user {{ user.name }}", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "collection": { 11 | "values": [ 12 | "*", 13 | "tmp_*", 14 | "{{ user.name }}_*" 15 | ], 16 | "isExcludes": false, 17 | "isRecursive": false 18 | } 19 | }, 20 | "policyItems": [ 21 | { 22 | "accesses": [ 23 | { 24 | "type": "query", 25 | "isAllowed": true 26 | }, 27 | { 28 | "type": "update", 29 | "isAllowed": true 30 | } 31 | ], 32 | "users": [ 33 | "{{ user.name }}" 34 | ], 35 | "groups": [ 36 | 37 | ], 38 | "roles": [ 39 | 40 | ], 41 | "conditions": [ 42 | 43 | ], 44 | "delegateAdmin": true 45 | } 46 | ], 47 | "denyPolicyItems": [ 48 | 49 | ], 50 | "allowExceptions": [ 51 | 52 | ], 53 | "denyExceptions": [ 54 | 55 | ], 56 | "dataMaskPolicyItems": [ 57 | 58 | ], 59 | "rowFilterPolicyItems": [ 60 | 61 | ], 62 | "serviceType": "solr", 63 | "options": { 64 | 65 | }, 66 | "validitySchedules": [ 67 | 68 | ], 69 | "policyLabels": [ 70 | 71 | ], 72 | "zoneName": "", 73 | "isDenyAllElse": false 74 | } -------------------------------------------------------------------------------- /playbooks/post_install/ranger_extra_policies/policies/hdfs_hive_hwc.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "service": "cm_hdfs", 4 | "name": "Policy fo Users on HWC staging Directory", 5 | "policyType": 0, 6 | "policyPriority": 0, 7 | "description": "", 8 | "isAuditEnabled": true, 9 | "resources": { 10 | "path": { 11 | "values": [ 12 | "/tmp/staging/hwc/*/{USER}" 13 | ], 14 | "isExcludes": false, 15 | "isRecursive": true 16 | } 17 | }, 18 | "policyItems": [ 19 | { 20 | "accesses": [ 21 | { 22 | "type": "read", 23 | "isAllowed": true 24 | }, 25 | { 26 | "type": "write", 27 | "isAllowed": true 28 | }, 29 | { 30 | "type": "execute", 31 | "isAllowed": true 32 | } 33 | ], 34 | "users": [ 35 | "{USER}" 36 | ], 37 | "groups": [ 38 | ], 39 | "roles": [ 40 | 41 | ], 42 | "conditions": [ 43 | 44 | ], 45 | "delegateAdmin": true 46 | } 47 | ], 48 | "denyPolicyItems": [ 49 | 50 | ], 51 | "allowExceptions": [ 52 | 53 | ], 54 | "denyExceptions": [ 55 | 56 | ], 57 | "dataMaskPolicyItems": [ 58 | 59 | ], 60 | "rowFilterPolicyItems": [ 61 | 62 | ], 63 | "serviceType": "hdfs", 64 | "options": { 65 | 66 | }, 67 | "validitySchedules": [ 68 | 69 | ], 70 | "policyLabels": [ 71 | 72 | ], 73 | "zoneName": "", 74 | "isDenyAllElse": false 75 | } -------------------------------------------------------------------------------- /playbooks/data_load/ranger_policies/push_policies/policies/schema-registry.json: -------------------------------------------------------------------------------- 1 | { 2 | "policyType": "0", 3 | "name": "random-datagen {{ user }}", 4 | "isEnabled": true, 5 | "policyPriority": 0, 6 | "policyLabels": [ 7 | 8 | ], 9 | "description": "", 10 | "isAuditEnabled": true, 11 | "resources": { 12 | "schema-group": { 13 | "values": [ 14 | "random-datagen*" 15 | ], 16 | "isRecursive": false, 17 | "isExcludes": false 18 | }, 19 | "schema-metadata": { 20 | "values": [ 21 | "*" 22 | ], 23 | "isRecursive": false, 24 | "isExcludes": false 25 | }, 26 | "schema-branch": { 27 | "values": [ 28 | "*" 29 | ], 30 | "isRecursive": false, 31 | "isExcludes": false 32 | }, 33 | "schema-version": { 34 | "values": [ 35 | "*" 36 | ], 37 | "isRecursive": false, 38 | "isExcludes": false 39 | } 40 | }, 41 | "isDenyAllElse": false, 42 | "policyItems": [ 43 | { 44 | "users": [ 45 | "{{ user }}" 46 | ], 47 | "accesses": [ 48 | { 49 | "type": "create", 50 | "isAllowed": true 51 | }, 52 | { 53 | "type": "read", 54 | "isAllowed": true 55 | }, 56 | { 57 | "type": "update", 58 | "isAllowed": true 59 | }, 60 | { 61 | "type": "delete", 62 | "isAllowed": true 63 | } 64 | ] 65 | } 66 | ], 67 | "allowExceptions": [ 68 | 69 | ], 70 | "denyPolicyItems": [ 71 | 72 | ], 73 | "denyExceptions": [ 74 | 75 | ], 76 | "service": "cm_schema-registry" 77 | } -------------------------------------------------------------------------------- /playbooks/pvc_setup/config/create_yunikorn_queues/templates/queue.json: -------------------------------------------------------------------------------- 1 | { 2 | "pool": { 3 | "path": "{{ queue.name }}", 4 | "quota": { 5 | "memory": "{{ queue.mem_max | default('100G') }}", 6 | "cpu": "{{ queue.cpu_max | default('25') }}", 7 | "nvidia.com/gpu": "{{ queue.gpu_max | default(0) }}" 8 | }, 9 | "guaranteedResources": { 10 | "memory": "{{ queue.mem_guaranteed | default('100G') }}", 11 | "cpu": "{{ queue.cpu_guaranteed | default('25') }}", 12 | "nvidia.com/gpu": "{{ queue.gpu_guaranteed | default('0') }}" 13 | }, 14 | "owners": [], 15 | "admins": [], 16 | "policy": { 17 | "validity": "{{ queue.validity | default('NEVER EXPIRES') }}", 18 | "allocation": { 19 | "distribution": "{{ queue.elasticity | default('INELASTIC') }}", 20 | "order": "FIFO", 21 | "queueing": false 22 | } 23 | }, 24 | "metadata": { 25 | "tags": { 26 | {% if queue.tags is defined %} 27 | {% for value in queue.tags %}"{{ value.split(':')[0] }}": "{{ value.split(':')[1] }}"{% if not loop.last %},{% endif %}{% endfor %} 28 | {% endif %} 29 | } 30 | }, 31 | "scheduling": { 32 | "preemption": { 33 | "enabled": {{ queue.preemption | default('true') }}, 34 | "fence": {{ queue.preemption_fencing | default('false') }}, 35 | "delay": {{ queue.preemption_delay | default('30') }} 36 | }, 37 | "priority": { 38 | "enabled": {{ queue.priority | default('true') }}, 39 | "fence": {{ queue.priority_fencing | default('false') }}, 40 | "offset": {{ queue.priority_offset | default('0') }} 41 | } 42 | } 43 | } 44 | } --------------------------------------------------------------------------------