├── README.md ├── amazon-linux-llap ├── README-INSTRUCTOR.md └── setup.sh ├── amazon-linux ├── README-INSTRUCTOR.md └── setup.sh ├── bin ├── cloudformation-status.sh ├── clusters-create.sh ├── clusters-report-flat.sh ├── clusters-report.sh ├── clusters-status.sh ├── clusters-terminate.sh ├── hdc-clusters-create.sh └── hdc-clusters-terminate.sh ├── cloud └── artifacts │ ├── geolocation.csv │ └── trucks.csv ├── generic ├── README.md ├── TEST.md ├── aws-cloudformation-in-existing-vpc.py ├── cloudformation.json └── setup.sh ├── governance ├── README.md ├── instances │ └── 20150915 ├── labs │ ├── falconChurnDemo │ │ ├── cleanseEmailProcess.xml │ │ ├── cleansedEmailFeed.xml │ │ ├── emailIngestProcess.xml │ │ ├── oregonCluster.xml │ │ ├── rawEmailFeed.xml │ │ └── virginiaCluster.xml │ └── hadoop-data-pipeline.sh └── setup.sh ├── hdc ├── README.md └── templates │ ├── cluster-analytics.json │ ├── cluster-datascience.json │ └── hdp26-data-science-spark2.json ├── hdf ├── README-INSTRUCTOR.md └── README.md ├── operations ├── README.txt ├── files │ ├── create-clouds-loopy.sh │ ├── report-clouds.sh │ └── terminate-clouds-loopy.sh └── lab-notes.txt ├── prepare ├── activedirectory │ ├── activedirectory-temp.sh │ └── activedirectory.sh ├── azure │ └── azure-hostname-fix.sh └── google │ ├── deploy-ad-lab.sh │ ├── prepare-account.sh │ └── scripts │ ├── deploy-hdp-ipa.sh │ ├── deploy-ipa.sh │ └── ipautil.patch ├── ranger-atlas ├── Data │ ├── claim_savings.csv.bz2 │ ├── claims_provider_summary_data.csv.bz2 │ ├── eu_countries.csv.bz2 │ ├── pos_txt.csv.bz2 │ ├── tax_2015.csv.bz2 │ ├── us_customers_data.csv.bz2 │ └── ww_customers_data.csv.bz2 ├── Notebooks │ ├── HortoniaBank - Compliance Admin View.json │ ├── HortoniaBank - Ivana EU HR.json │ └── HortoniaBank - Joe Analyst.json ├── README-INSTRUCTOR.md ├── README.md ├── Scripts │ ├── create-secgovdemo-hortoniabank-tables.ddl │ ├── create-secgovdemo-hortoniabank-tables.sh │ ├── create-secgovdemo-hortoniabank-userfolders.sh │ ├── hive_example.json │ ├── load-secgovdemo-hortoniabank-files.sh │ ├── ranger-policies-enabled.json │ ├── ranger-policies-tags.json │ └── ranger-policies.json ├── media │ ├── screenshot-atlas-create-tag-expireson.png │ ├── screenshot-atlas-tax2015-search.png │ ├── screenshot-atlas-tax2015-tag.png │ ├── screenshot-ranger-add-tag-condition1.png │ ├── screenshot-ranger-add-tag-condition2.png │ ├── screenshot-ranger-add-tag-service.png │ └── screenshot-ranger-configure-hive-tag-service.png └── setup.sh ├── reference └── setup.sh ├── security-advanced ├── .DS_Store ├── README-212.md ├── README-INSTRUCTOR.md ├── README.md ├── extras │ ├── active-directory.md │ ├── ambari.keytab │ ├── ca.crt │ ├── ca.key │ ├── ca.srl │ ├── wildcard-lab-hortonworks-net.crt │ ├── wildcard-lab-hortonworks-net.csr │ ├── wildcard-lab-hortonworks-net.key │ └── wildcard-lab-hortonworks-net.p12 ├── screenshots │ ├── .DS_Store │ ├── AD-corpusers.png │ ├── AD-hadoopnodes.png │ ├── AD-hadoopservices.png │ ├── AD-serviceusers.png │ ├── Ambari-KMS-enhancedconfig1.png │ ├── Ambari-KMS-enhancedconfig2.png │ ├── Ambari-KMS-enhancedconfig3.png │ ├── Ambari-KMS-proxy.png │ ├── Ambari-Knox-install.png │ ├── Ambari-RM-public-host.png │ ├── Ambari-YARN-internal-host.png │ ├── Ambari-add-KMS.png │ ├── Ambari-configureidentities.png │ ├── Ambari-host-add-tez.png │ ├── Ambari-kerberos-wizard-1.png │ ├── Ambari-kerberos-wizard-2.png │ ├── Ambari-kerberos-wizard-3.png │ ├── Ambari-kerberos-wizard-4.png │ ├── Ambari-kerberos-wizard-5.png │ ├── Ambari-kerberos-wizard-6.png │ ├── Ambari-kerberos-wizard-7.png │ ├── Ambari-kerberos-wizard-8.png │ ├── Ambari-make-user-admin.png │ ├── Ambari-multiple-KMS.png │ ├── Ambari-pig-nodes.png │ ├── Ambari-proxyuser.png │ ├── Ambari-restart-services.png │ ├── Ambari-run-LDAPsync.png │ ├── Ambari-setup-LDAP.png │ ├── Ambari-start-kerberos-wizard.png │ ├── Banana-audits.png │ ├── Banana-empty.png │ ├── Files-view.png │ ├── Hive-view-viz.png │ ├── Hive-view.png │ ├── Pig-view.png │ ├── Ranger-AccessManager.png │ ├── Ranger-HBASE-create-policy.png │ ├── Ranger-HBASE-edit-policy.png │ ├── Ranger-HBASE-plugin-config.png │ ├── Ranger-HBASE-policy.png │ ├── Ranger-HDFS-create-policy.png │ ├── Ranger-HDFS-edit-policy-add-hadoopadmin.png │ ├── Ranger-HDFS-edit-policy.png │ ├── Ranger-HDFS-plugin-config.png │ ├── Ranger-HDFS-policy.png │ ├── Ranger-HIVE-audit-persons.png │ ├── Ranger-HIVE-create-policy-persons.png │ ├── Ranger-HIVE-create-policy.png │ ├── Ranger-HIVE-edit-policy.png │ ├── Ranger-HIVE-plugin-config.png │ ├── Ranger-HIVE-policy-ambari.png │ ├── Ranger-HIVE-policy.png │ ├── Ranger-KMS-audit.png │ ├── Ranger-KMS-config-audit.png │ ├── Ranger-KMS-create-policy-testkey.png │ ├── Ranger-KMS-createkey.png │ ├── Ranger-KMS-edit-policy.png │ ├── Ranger-KMS-edit-repo.png │ ├── Ranger-KMS-policy-add-nn.png │ ├── Ranger-KMS-policy.png │ ├── Ranger-KNOX-plugin-config.png │ ├── Ranger-YARN-plugin-config.png │ ├── Ranger-add-nn.png │ ├── Ranger-ambari-user.png │ ├── Ranger-audit-HBASE-allowed.png │ ├── Ranger-audit-HBASE-denied.png │ ├── Ranger-audit-HBASE-policy-details.png │ ├── Ranger-audit-HBASE-summary.png │ ├── Ranger-audit-HDFS-allowed.png │ ├── Ranger-audit-HDFS-denied.png │ ├── Ranger-audit-HDFS-summary.png │ ├── Ranger-audit-HIVE-allowed.png │ ├── Ranger-audit-HIVE-denied.png │ ├── Ranger-audit-HIVE-policy-details.png │ ├── Ranger-audit-HIVE-summary.png │ ├── Ranger-audit-KNOX-hive-summary.png │ ├── Ranger-audit-banana.png │ ├── Ranger-audit-policy-details.png │ ├── Ranger-audits.png │ ├── Ranger-install-configure-components.png │ ├── Ranger-knox-hive-policy.png │ ├── Ranger-knox-webhdfs-audit.png │ ├── Ranger-knox-webhdfs-policy.png │ ├── Ranger-plugins.png │ ├── Ranger-policy-kms-audit.png │ ├── Ranger-user-HTTP.png │ ├── Ranger-user-groups.png │ ├── Ranger-user-hive.png │ ├── Ranger-user-permissions-audits.png │ ├── Ranger-user-permissions.png │ ├── Tez-view-viz.png │ ├── Tez-view.png │ ├── chrome-untrusted.png │ ├── clustername.png │ ├── firefox-untrusted.png │ ├── knox-location.png │ ├── knox-webhdfs-browser1.png │ ├── knox-webhdfs-browser2.png │ ├── knox-webhdfs-browser3.png │ ├── putty.png │ ├── solr-cloud.png │ ├── solr-dashboard-collection.png │ ├── solr-service-configs.png │ └── solr-service-placement.png ├── setup.sh ├── training-keypair.pem └── training-keypair.ppk ├── security-official └── containers │ ├── ambari │ └── Dockerfile │ └── desktop │ └── hortonworks.jpg ├── security ├── README.md └── setup.sh ├── single-view ├── LABS.md ├── README-INSTRUCTOR.md ├── files │ ├── banana.json │ ├── nifi_single-view-demo.xml │ ├── zeppelin_notebook_10_edw.json │ ├── zeppelin_notebook_20_nifi.json │ └── zeppelin_notebook_30_analytics.json └── scripts │ ├── 00_prep_host.sh │ ├── 10_deploy_hdp.sh │ └── 20_setup_demo.sh ├── spark ├── README-INSTRUCTOR.md └── README.md └── sql ├── LABS-copy-paste.txt ├── LABS.md ├── Lab2 ├── customers.txt └── prepare.sh ├── Lab3 ├── prepare.sh └── tweets.tgz ├── Lab4 ├── json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar ├── prepare.sh └── sample_twitter_data.txt ├── Lab5 └── prepare.sh ├── Lab6 └── prepare.sh ├── README-INSTRUCTOR.md ├── change-port.sh ├── cloudformation-generate-template.py ├── cloudformation-generate-template_m3-2xlarge.py ├── cloudformation-status.sh ├── cloudformation.json ├── clusters-create.sh ├── clusters-report.sh ├── clusters-status.sh ├── clusters-terminate.sh ├── deploy.sh ├── labs-setup.sh ├── restart-service.sh └── setup.sh /README.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | ## Pre-reqs 5 | ----------- 6 | 7 | - AWS CLI installed: http://docs.aws.amazon.com/cli/latest/userguide/installing.html 8 | - Configure AWS CLI: `aws configure` 9 | - Details on creating credentials: http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-set-up.html 10 | - These AWS IAM permissions are needed: EC2, AutoScaling, CloudFormation, Marketplace 11 | 12 | ## How this works 13 | ----------------- 14 | 15 | Under each class directory (e.g. generic, sql, ...) there are several scripts used to make this work: 16 | 17 | For AWS CloudFormation: 18 | 19 | - cloudformation-generate-template.py: generates the AWS Cloudformation template 20 | - cloudformation.json: the CloudFormation template which deploys a cluster 21 | 22 | For managing multiple clusters (each takes variables for naming & the number of clusters): 23 | 24 | - ../bin/clusters-create.sh: calls cloudformation to create clusters 25 | - ../bin/clusters-report.sh: list cluster details 26 | - ../bin/clusters-terminate.sh: calls cloudformation to terminate clusters 27 | - ../bin/cloudformation-status.sh: list all cloudformation stacks in the region 28 | 29 | For preparing the host(s): 30 | 31 | - setup.sh: This is called by CloudFormation. It shoudl be usable without CloudFormation in any environment 32 | 33 | ## Deploy, report & terminate clusters on AWS 34 | ------------------------- 35 | 36 | 1. Get this repo: 37 | 38 | ``` 39 | git clone https://github.com/seanorama/masterclass 40 | ``` 41 | 42 | 1. Switch to the directory of the class, or use 'generic': 43 | 44 | ``` 45 | cd masterclass/generic 46 | ``` 47 | 48 | 1. Check for conflicting/existing stacks (same name as what you plan to deploy): 49 | - Open the CloudFormat Web UI (ensure you have chosen the correct region in the top right) 50 | - Or with the command-line: `..bin/cloudformation-status.sh` 51 | 52 | 1. Set variables to define the naming & number of clusters to deploy: 53 | - **each class has specific instructions for this**. Below is a generic example. 54 | - this deploys 2 clusters (mc-generic100, mc-generic101) in AWS region us-west-2 55 | - update 'lab_count' to the number of clusters you want 56 | 57 | ```sh 58 | export AWS_DEFAULT_REGION=us-west-2 ## region to deploy in 59 | export lab_prefix=mc-generic ## template for naming the cloudformation stacks 60 | export lab_first=100 ## number to start at in naming 61 | export lab_count=2 ## number of clusters to create 62 | ``` 63 | 64 | 2. Set parameters which are passed to CloudFormation: 65 | - **each class has specific instructions for this**. Below is a generic example. 66 | - KeyName: The key (added on the EC2 page) to access the cluster. 67 | - AmbariServices: Which HDP services to deploy. 68 | - AdditionalInstanceCount: How many additional nodes to deploy. (Setting to 2 will deploy 3 nodes total) 69 | - SubnetId & SecurityGroups: This CloudFormation deploys in an existing Subnet & Security Group. **You must update this to your environment**. 70 | 71 | ```sh 72 | export cfn_parameters=' 73 | [ 74 | {"ParameterKey":"KeyName","ParameterValue":"training-keypair"}, 75 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER"}, 76 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-02edac67"}, 77 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-a02d17c4"}] 78 | ' 79 | ``` 80 | 81 | 1. Provision your clusters 82 | 83 | ``` 84 | ../bin/clusters-create.sh 85 | ``` 86 | 87 | 1. Check the build status 88 | - From the CloudFormation Web UI 89 | - Or from the command-line: 90 | 91 | ``` 92 | ../bin/cloudformation-status.sh 93 | ``` 94 | 95 | 1. Once your clusters are ready, get list of clusters nodes for providing to students: 96 | 97 | ``` 98 | ..bin/clusters-report.sh 99 | ``` 100 | 101 | 1. Use the clusters: 102 | - `ssh centos@ipFromReportAbove` ## use the key which was specified during the build 103 | 104 | 1. Terminate clusters 105 | 106 | ``` 107 | ..bin/clusters-terminate.sh 108 | ``` 109 | 110 | 1. Verify that all clusters are terminated 111 | - From the AWS CloudFormation Web UI 112 | - Or from the CLI 113 | 114 | ``` 115 | ../bin/cloudformation-status.sh 116 | ``` 117 | 118 | ######## 119 | 120 | ## Running sessions 121 | 122 | It's recommended to use an "etherpad" to share: 123 | 124 | - the cluster details (from above) 125 | - instructions to students 126 | 127 | You can create your own, or use a hosted version such as TitanPad. You should create this account in advance. 128 | 129 | ######## 130 | 131 | ## Issues: Deployment 132 | 133 | #### Creation 134 | 135 | - Some instances will fail their creation and time out, being rolled back, this is a nature of deploying large volumes of instances 136 | - Those that fail should simply be manually deleted from the cloudformations web ui 137 | 138 | #### Deleting cloudformations 139 | 140 | - Occasionally cloudformations will fail to delete due to timing issues, in which case, it’s probably the VPC or InternetGateway, just switch to the VPC service window within the AWS site, delete the specific VPC that is being complained about in the cloudformation and then once the cloudformation delete has failed, retry the delete, deletion should complete this time. 141 | - Once you’ve done the VPC deletion you can also do an AWS CLI call instead: 142 | - `aws cloudformation delete-stack --stack-name ` 143 | 144 | #### AWS Website 145 | 146 | If you suddenly notice that your instances/cloudformations/etc have vanished from the AWS control panel, you may have to re-login. 147 | 148 | 149 | 150 | ######## 151 | 152 | ## Issues: Other 153 | 154 | #### Run commands in bulk on all nodes 155 | 156 | * There are several options, such as pdsh, cssh, ... 157 | 158 | * Example using cssh, csshX or tmux-cssh (you'll need to install it) 159 | 160 | ``` 161 | ## for Ambari nodes only: 162 | ../bin/clusters-report.sh | awk '/^AmbariNode:/ {print $2}' | xargs echo tmux-cssh -u student 163 | ``` 164 | 165 | * After executing you will get a terminal with small windows to all of the clusters. 166 | * Anything you type will go to all hosts. 167 | 168 | #### Venue Internet blocks Ambari Server (port 8080) 169 | 170 | * Change Ambari to port 8081 171 | 172 | ``` 173 | export TERM=xterm 174 | echo "client.api.port=8081" | sudo tee -a /etc/ambari-server/conf/ambari.properties 175 | sudo ambari-server restart 176 | sudo ambari-agent restart 177 | ``` 178 | -------------------------------------------------------------------------------- /amazon-linux-llap/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | 1. Get the repo and switch to the 'generic' directory 5 | 6 | ``` 7 | git clone https://github.com/seanorama/masterclass 8 | cd masterclass/amazon-linux 9 | ``` 10 | 11 | 2. Set these variables, updating the values as appropriate: 12 | 13 | ```sh 14 | export AWS_DEFAULT_REGION=us-west-2 ## region to deploy in 15 | export lab_prefix=${USER} ## template for naming the cloudformation stacks 16 | export lab_first=100 ## number to start at in naming 17 | export lab_count=1 ## number of clusters to create 18 | 19 | export cfn_parameters=' 20 | [ 21 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 22 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-76f4222f"}, 23 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-1cb3f678"}, 24 | {"ParameterKey":"HDPStack","ParameterValue":"2.5"}, 25 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG HIVE YARN ZOOKEEPER SPARK AMBARI_METRICS SQOOP TEZ FALCON KAFKA STORM OOZIE SLIDER"}, 26 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"3"}, 27 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/amazon-linux-llap/setup.sh | bash"}, 28 | {"ParameterKey":"InstanceType","ParameterValue":"m4.2xlarge"}, 29 | {"ParameterKey":"BootDiskSize","ParameterValue":"100"} 30 | ] 31 | ' 32 | ``` 33 | 34 | 3. You can then execute ../bin/clusters-create.sh and the other cluster scripts as explained in ../README.md 35 | 36 | ## REMEMBER to terminate the clusters immediately after the class is over, or be prepared to pay $$$! 37 | 38 | Further, you should verify deletion of the CloudFormations & EC2 instances from the AWS Console. 39 | 40 | ## Issues: See ../README.md 41 | 42 | ## Advanced usage 43 | 44 | 1. Only deploy the infrastructure by setting PostCommand to /bin/true 45 | -------------------------------------------------------------------------------- /amazon-linux-llap/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o xtrace 3 | 4 | export HOME=${HOME:-/root} 5 | export TERM=xterm 6 | 7 | : ${install_ambari_server:=true} 8 | : ${ambari_pass:="BadPass#1"} 9 | ambari_password="${ambari_pass}" 10 | : ${host_count:=skip} 11 | : ${ambari_services:="HDFS MAPREDUCE2 PIG HIVE YARN ZOOKEEPER TEZ SLIDER"} 12 | cluster_name=${stack:-mycluster} 13 | 14 | export install_ambari_server ambari_pass host_count ambari_services 15 | export ambari_password cluster_name 16 | 17 | #export ambari_repo=http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.1.0/ambari.repo 18 | #export ambari_repo=http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-2/centos6/ambari.repo 19 | export recommendation_strategy="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES" 20 | 21 | cd 22 | 23 | yum makecache 24 | yum -y -q install git 25 | 26 | git clone http://github.com/seanorama/ambari-bootstrap 27 | cd ambari-bootstrap 28 | ./ambari-bootstrap.sh 29 | 30 | ## Ambari Server specific tasks 31 | if [ "${install_ambari_server}" = "true" ]; then 32 | bash -c "nohup ambari-server restart" || true 33 | 34 | sleep 60 35 | 36 | ambari_pass=admin source ~/ambari-bootstrap/extras/ambari_functions.sh 37 | ambari_change_pass admin admin ${ambari_pass} 38 | 39 | # alias curl="curl -L -H X-Requested-By:blah -u admin:${ambari_pass}" 40 | #cat > /tmp/repo.json <<-'EOF' 41 | #{ 42 | #"Repositories": { 43 | #"base_url": "http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6", 44 | #"verify_base_url": true 45 | #} 46 | #} 47 | #EOF 48 | #url="http://localhost:8080/api/v1/stacks/HDP/versions/2.5/operating_systems/redhat6/repositories/HDP-UTILS-1.1.0.21" 49 | #curl -X PUT "${url}" -d @/tmp/repo.json 50 | #curl "${url}" 51 | #rm -f /tmp/repo.json 52 | 53 | ## register HDP repo 54 | #cat > /tmp/repo.json <<-'EOF' 55 | #{ 56 | #"Repositories": { 57 | #"base_url": "http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/2.5.0.0-2/centos6", 58 | #"verify_base_url": true 59 | #} 60 | #} 61 | #EOF 62 | #url="http://localhost:8080/api/v1/stacks/HDP/versions/2.5/operating_systems/redhat6/repositories/HDP-2.5" 63 | #curl -X PUT "${url}" -d @/tmp/repo.json 64 | #curl "${url}" 65 | #rm -f /tmp/repo.json 66 | #unalias curl 67 | 68 | if [ "${deploy}" = "true" ]; then 69 | 70 | cd ~/ambari-bootstrap/deploy 71 | 72 | cat << EOF > configuration-custom.json 73 | { 74 | "configurations" : { 75 | "core-site": { 76 | "hadoop.proxyuser.HTTP.groups" : "users,hadoop-users", 77 | "hadoop.proxyuser.HTTP.hosts" : "*", 78 | "hadoop.proxyuser.hbase.groups" : "users,hadoop-users", 79 | "hadoop.proxyuser.hbase.hosts" : "*", 80 | "hadoop.proxyuser.hcat.groups" : "users,hadoop-users", 81 | "hadoop.proxyuser.hcat.hosts" : "*", 82 | "hadoop.proxyuser.hive.groups" : "users,hadoop-users", 83 | "hadoop.proxyuser.hive.hosts" : "*", 84 | "hadoop.proxyuser.knox.groups" : "users,hadoop-users", 85 | "hadoop.proxyuser.knox.hosts" : "*", 86 | "hadoop.proxyuser.oozie.groups" : "users", 87 | "hadoop.proxyuser.oozie.hosts" : "*", 88 | "hadoop.proxyuser.root.groups" : "users,hadoop-users", 89 | "hadoop.proxyuser.root.hosts" : "*", 90 | "fs.trash.interval": "4320" 91 | }, 92 | "hive-interactive-env": { 93 | "enable_hive_interactive": "true", 94 | "llap_queue_capacity": "75" 95 | }, 96 | "yarn-site": { 97 | "yarn.acl.enable" : "true" 98 | }, 99 | "hdfs-site": { 100 | "dfs.namenode.safemode.threshold-pct": "0.99" 101 | }, 102 | "hive-site": { 103 | "hive.exec.compress.output": "true", 104 | "hive.merge.mapfiles": "true", 105 | "hive.server2.tez.initialize.default.sessions": "true" 106 | }, 107 | "mapred-site": { 108 | "mapreduce.job.reduce.slowstart.completedmaps": "0.7", 109 | "mapreduce.map.output.compress": "true", 110 | "mapreduce.output.fileoutputformat.compress": "true" 111 | } 112 | } 113 | } 114 | EOF 115 | ./deploy-recommended-cluster.bash 116 | 117 | source ~/ambari-bootstrap/extras/ambari_functions.sh 118 | ambari_configs 119 | ambari_wait_request_complete 1 120 | 121 | cd ~ 122 | sleep 10 123 | 124 | useradd -G users admin 125 | echo "${ambari_pass}" | passwd admin --stdin 126 | sudo -u hdfs bash -c " 127 | hadoop fs -mkdir /user/admin; 128 | hadoop fs -chown admin /user/admin; 129 | hdfs dfsadmin -refreshUserToGroupsMappings" 130 | fi 131 | fi 132 | 133 | 134 | -------------------------------------------------------------------------------- /amazon-linux/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | 1. Get the repo and switch to the 'generic' directory 5 | 6 | ``` 7 | git clone https://github.com/seanorama/masterclass 8 | cd masterclass/amazon-linux 9 | ``` 10 | 11 | 2. Set these variables, updating the values as appropriate: 12 | 13 | ```sh 14 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 15 | export lab_prefix=${USER} ## template for naming the cloudformation stacks 16 | export lab_first=100 ## number to start at in naming 17 | export lab_count=1 ## number of clusters to create 18 | 19 | export cfn_parameters=' 20 | [ 21 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 22 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 23 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 24 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG HIVE YARN ZOOKEEPER SPARK AMBARI_METRICS SQOOP TEZ"}, 25 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"0"}, 26 | {"ParameterKey":"AmbariVersion","ParameterValue":"2.4.1.0"}, 27 | {"ParameterKey":"HDPStack","ParameterValue":"2.5"}, 28 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/amazon-linux/setup.sh | bash"}, 29 | {"ParameterKey":"InstanceType","ParameterValue":"m4.xlarge"}, 30 | {"ParameterKey":"BootDiskSize","ParameterValue":"100"} 31 | ] 32 | ' 33 | 34 | export AWS_DEFAULT_REGION=us-west-2 ## region to deploy in 35 | export lab_prefix=${USER} ## template for naming the cloudformation stacks 36 | export lab_first=100 ## number to start at in naming 37 | export lab_count=1 ## number of clusters to create 38 | 39 | export cfn_parameters=' 40 | [ 41 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 42 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-76f4222f"}, 43 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-1cb3f678"}, 44 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG HIVE YARN ZOOKEEPER SPARK AMBARI_INFRA AMBARI_METRICS TEZ ZEPPELIN SLIDER"}, 45 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/generic/setup.sh | bash"}, 46 | {"ParameterKey":"AmbariVersion","ParameterValue":"2.4.2.0"}, 47 | {"ParameterKey":"HDPStack","ParameterValue":"2.5"}, 48 | {"ParameterKey":"DeployCluster","ParameterValue":"true"}, 49 | {"ParameterKey":"InstanceType","ParameterValue":"m4.2xlarge"}, 50 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"1"}, 51 | {"ParameterKey":"BootDiskSize","ParameterValue":"100"} 52 | ] 53 | ' 54 | ``` 55 | 56 | 3. You can then execute ../bin/clusters-create.sh and the other cluster scripts as explained in ../README.md 57 | 58 | ## REMEMBER to terminate the clusters immediately after the class is over, or be prepared to pay $$$! 59 | 60 | Further, you should verify deletion of the CloudFormations & EC2 instances from the AWS Console. 61 | 62 | ## Issues: See ../README.md 63 | 64 | ## Advanced usage 65 | 66 | 1. Only deploy the infrastructure by setting PostCommand to /bin/true 67 | -------------------------------------------------------------------------------- /amazon-linux/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o xtrace 3 | 4 | export HOME=${HOME:-/root} 5 | export TERM=xterm 6 | : ${ambari_pass:="BadPass#1"} 7 | ambari_password="${ambari_pass}" 8 | : ${ambari_services:="HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER AMBARI_METRICS SLIDER AMBARI_INFRA LOGSEARCH TEZ"} 9 | : ${install_ambari_server:=true} 10 | : ${ambari_stack_version:=2.5} 11 | cluster_name=${stack:-mycluster} 12 | 13 | : ${install_ambari_server:=true} 14 | : ${ambari_stack_version:=2.5} 15 | : ${host_count:=skip} 16 | 17 | : ${recommendation_strategy:="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"} 18 | 19 | export install_ambari_server ambari_pass host_count ambari_services 20 | export ambari_password cluster_name recommendation_strategy 21 | 22 | cd 23 | 24 | yum makecache 25 | yum -y -q install git epel-release ntpd screen mysql-connector-java jq python-argparse python-configobj ack 26 | 27 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 28 | 29 | ~/ambari-bootstrap/ambari-bootstrap.sh 30 | 31 | ## Ambari Server specific tasks 32 | if [ "${install_ambari_server}" = "true" ]; then 33 | bash -c "nohup ambari-server restart" || true 34 | 35 | sleep 60 36 | 37 | ambari_pass=admin source ~/ambari-bootstrap/extras/ambari_functions.sh 38 | ambari_change_pass admin admin ${ambari_pass} 39 | sleep 1 40 | 41 | if [ "${deploy}" = "true" ]; then 42 | 43 | cd ~/ambari-bootstrap/deploy 44 | 45 | ## various configuration changes for demo environments, and fixes to defaults 46 | cat << EOF > configuration-custom.json 47 | { 48 | "configurations" : { 49 | "core-site": { 50 | "hadoop.proxyuser.root.users" : "admin", 51 | "fs.trash.interval": "4320" 52 | }, 53 | "hdfs-site": { 54 | "dfs.namenode.safemode.threshold-pct": "0.99" 55 | }, 56 | "hive-interactive-env": { 57 | "enable_hive_interactive": "true", 58 | "llap_queue_capacity": "75" 59 | }, 60 | "hive-site": { 61 | "hive.exec.compress.output": "true", 62 | "hive.merge.mapfiles": "true", 63 | "hive.server2.tez.initialize.default.sessions": "true" 64 | }, 65 | "mapred-site": { 66 | "mapreduce.job.reduce.slowstart.completedmaps": "0.7", 67 | "mapreduce.map.output.compress": "true", 68 | "mapreduce.output.fileoutputformat.compress": "true" 69 | }, 70 | "yarn-site": { 71 | "yarn.acl.enable" : "true" 72 | } 73 | } 74 | } 75 | EOF 76 | 77 | ./deploy-recommended-cluster.bash 78 | cd ~ 79 | sleep 5 80 | 81 | source ~/ambari-bootstrap/extras/ambari_functions.sh 82 | ambari_configs 83 | ambari_wait_request_complete 1 84 | cd ~ 85 | sleep 10 86 | 87 | usermod -a -G users ${USER} 88 | useradd -a -G users admin 89 | echo "${ambari_pass}" | passwd admin --stdin 90 | sudo sudo -u hdfs bash -c " 91 | hadoop fs -mkdir /user/admin; 92 | hadoop fs -chown admin /user/admin; 93 | hdfs dfsadmin -refreshUserToGroupsMappings" 94 | fi 95 | fi 96 | 97 | -------------------------------------------------------------------------------- /bin/cloudformation-status.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "########################################" 4 | echo "## Status of all CloudFormation stacks in this region:" 5 | aws cloudformation describe-stacks --query 'Stacks[*].[StackName, StackStatus]' --output text 6 | echo "########################################" 7 | -------------------------------------------------------------------------------- /bin/clusters-create.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | ## Create Cloudformations in batches 4 | 5 | ## Note: 6 | ## Your default AWS region will be used. Override by executing this before the scripts: 7 | ## export AWS_DEFAULT_REGION=eu-west-1 8 | ## 9 | 10 | #### 11 | lab_location="${lab_location:-0.0.0.0/0}" 12 | lab_prefix=${lab_prefix:-$USER} 13 | lab_first=${lab_first:-100} 14 | lab_count=${lab_count:-1} 15 | lab_batch=${lab_batch:-20} ## how many cluster to deploy at a time 16 | lab_batch_delay=${lab_batch_delay:-300} ## seconds to wait between batches 17 | cfn_parameters=${cfn_parameters:-} 18 | cfn_switches=${cfn_switches:-} 19 | cfn_file="${cfn_file:-cloudformation.json}" 20 | 21 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 22 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 23 | 24 | echo "########################################################" 25 | echo "## Clusters to be created:" 26 | echo 27 | echo ${clusters} 28 | echo 29 | echo "########################################################" 30 | 31 | # quick check to ensure pre-reqs are setup 32 | while true; do 33 | read -p "Proceed with creating these ${lab_count} clusters? (y/n) " yn 34 | case $yn in 35 | [Yy]* ) break;; 36 | [Nn]* ) exit;; 37 | * ) echo "Please answer yes or no.";; 38 | esac 39 | done 40 | 41 | batchcounter=0 42 | for cluster in ${clusters} 43 | do 44 | ((batchcounter++)) 45 | aws cloudformation create-stack --stack-name ${cluster} \ 46 | --capabilities CAPABILITY_IAM \ 47 | --template-body "file://./${cfn_file}" \ 48 | --parameters "${cfn_parameters}" \ 49 | ${cfn_switches} 50 | 51 | ######################################################################## 52 | echo Initiated creation of $cluster cluster 53 | 54 | sleep 5 55 | if [ $batchcounter -eq ${lab_batch} ]; then 56 | sleep ${lab_batch_delay} 57 | batchcounter=0 58 | fi 59 | done 60 | -------------------------------------------------------------------------------- /bin/clusters-report-flat.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-mc-test} 4 | lab_first=${lab_first:-1} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | for cluster in ${clusters} 11 | do 12 | printf "\n$cluster" 13 | for nodetype in AmbariNode AdditionalNodes; do 14 | printf " "$(aws ec2 describe-instances \ 15 | --query \ 16 | 'Reservations[*].Instances[*].[PublicIpAddress]' \ 17 | --filters Name=instance-state-name,Values=running \ 18 | Name=tag:aws:cloudformation:stack-name,Values=$cluster \ 19 | Name=tag:aws:cloudformation:logical-id,Values=${nodetype} --output text | xargs echo) 20 | done 21 | done 22 | 23 | #aws ec2 describe-instances \ 24 | #--query \ 25 | #'Reservations[*].Instances[*].[ 26 | #Tags[?Key==`aws:cloudformation:logical-id`].Value[] 27 | #,PublicIpAddress]' \ 28 | #--filters Name=instance-state-name,Values=running \ 29 | #Name=tag:aws:cloudformation:stack-name,Values=$cluster \ 30 | #Name=tag:aws:cloudformation:logical-id,Values=${nodetype} --output text | xargs echo 31 | -------------------------------------------------------------------------------- /bin/clusters-report.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-${USER}} 4 | lab_first=${lab_first:-100} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | for cluster in ${clusters} 11 | do 12 | echo "####################################################" 13 | echo "## $cluster is operated by: " 14 | for nodetype in AmbariNode AdditionalNodes; do 15 | printf "${nodetype}: "; aws ec2 describe-instances \ 16 | --query \ 17 | 'Reservations[*].Instances[*].[PublicIpAddress]' \ 18 | --filters Name=instance-state-name,Values=running \ 19 | Name=tag:aws:cloudformation:stack-name,Values=$cluster \ 20 | Name=tag:aws:cloudformation:logical-id,Values=${nodetype} --output text | xargs echo 21 | done 22 | done 23 | 24 | #aws ec2 describe-instances \ 25 | #--query \ 26 | #'Reservations[*].Instances[*].[ 27 | #Tags[?Key==`aws:cloudformation:logical-id`].Value[] 28 | #,PublicIpAddress]' \ 29 | #--filters Name=instance-state-name,Values=running \ 30 | #Name=tag:aws:cloudformation:stack-name,Values=$cluster \ 31 | #Name=tag:aws:cloudformation:logical-id,Values=${nodetype} --output text | xargs echo 32 | -------------------------------------------------------------------------------- /bin/clusters-status.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-mc-test} 4 | lab_first=${lab_first:-1} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | echo "########################################" 11 | echo "## Status of the cluster's CloudFormation Stacks:" 12 | for cluster in ${clusters} 13 | do 14 | aws cloudformation describe-stacks \ 15 | --query 'Stacks[*].[StackName, StackStatus]' \ 16 | --stack-name ${cluster} \ 17 | --output text 18 | done 19 | echo "########################################" 20 | 21 | -------------------------------------------------------------------------------- /bin/clusters-terminate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-${USER}} 4 | lab_first=${lab_first:-100} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | echo "########################################################" 11 | echo "## Clusters to be terminated:" 12 | echo 13 | echo ${clusters} 14 | echo 15 | echo "########################################################" 16 | 17 | # quick safety check to ensure termination is required 18 | while true; do 19 | read -p "Do you wish terminate all of these clusters? (y/n) " yn 20 | case $yn in 21 | [Yy]* ) break;; 22 | [Nn]* ) exit;; 23 | * ) echo "Please answer yes or no.";; 24 | esac 25 | done 26 | 27 | # Don't change this, change the batchcount test lower down the end of script 28 | batchcount=0 29 | 30 | # change the list of fruit to vary the number of clusters deleted 31 | for cluster in ${clusters} 32 | do 33 | ((batchcount++)) 34 | aws cloudformation delete-stack --stack-name $cluster 35 | echo Initiated deletion of $cluster cluster 36 | # this is the sleep interval between instances 37 | sleep 5 38 | ######################################################## 39 | # change the value below to increase/decrease batch size 40 | ######################################################## 41 | if [ $batchcount -eq 20 ] 42 | then 43 | # this is the sleep interval between batches of instances 44 | sleep 20 45 | # Don't change this, change the batchcount variable above 46 | batchcount=0 47 | fi 48 | done 49 | -------------------------------------------------------------------------------- /bin/hdc-clusters-create.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | ## Create Cloudformations in batches 4 | 5 | ## Note: 6 | ## Your default AWS region will be used. Override by executing this before the scripts: 7 | ## export AWS_DEFAULT_REGION=eu-west-1 8 | ## 9 | 10 | #### 11 | lab_prefix=${lab_prefix:-$USER} 12 | lab_first=${lab_first:-100} 13 | lab_count=${lab_count:-1} 14 | lab_batch=${lab_batch:-5} ## how many cluster to deploy at a time 15 | lab_batch_delay=${lab_batch_delay:-150} ## seconds to wait between batches 16 | 17 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 18 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 19 | 20 | echo "########################################################" 21 | echo "## Clusters to be created:" 22 | echo 23 | echo ${clusters} 24 | echo 25 | echo "########################################################" 26 | 27 | # quick check to ensure pre-reqs are setup 28 | while true; do 29 | read -p "Proceed with creating these ${lab_count} clusters? (y/n) " yn 30 | case $yn in 31 | [Yy]* ) break;; 32 | [Nn]* ) exit;; 33 | * ) echo "Please answer yes or no.";; 34 | esac 35 | done 36 | 37 | batchcounter=0 38 | for cluster in ${clusters} 39 | do 40 | ((batchcounter++)) 41 | sed -i "s/\(\"ClusterName\": \"\).*\(\",\)/\1${cluster}\2/" ./templates/${cluster_type}.json 42 | hdc create-cluster --cli-input-json ./templates/${cluster_type}.json --input-json-param-ClusterAndAmbariPassword ${ambari_pass} 43 | 44 | 45 | ######################################################################## 46 | echo Initiated creation of $cluster cluster 47 | 48 | sleep 5 49 | if [ $batchcounter -eq ${lab_batch} ]; then 50 | sleep ${lab_batch_delay} 51 | batchcounter=0 52 | fi 53 | done 54 | -------------------------------------------------------------------------------- /bin/hdc-clusters-terminate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-${USER}} 4 | lab_first=${lab_first:-100} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | echo "########################################################" 11 | echo "## Clusters to be terminated:" 12 | echo 13 | echo ${clusters} 14 | echo 15 | echo "########################################################" 16 | 17 | # quick safety check to ensure termination is required 18 | while true; do 19 | read -p "Do you wish terminate all of these clusters? (y/n) " yn 20 | case $yn in 21 | [Yy]* ) break;; 22 | [Nn]* ) exit;; 23 | * ) echo "Please answer yes or no.";; 24 | esac 25 | done 26 | 27 | # Don't change this, change the batchcount test lower down the end of script 28 | batchcount=0 29 | 30 | # change the list of fruit to vary the number of clusters deleted 31 | for cluster in ${clusters} 32 | do 33 | ((batchcount++)) 34 | hdc terminate-cluster --cluster-name ${cluster} 35 | echo Initiated deletion of $cluster cluster 36 | # this is the sleep interval between instances 37 | sleep 5 38 | ######################################################## 39 | # change the value below to increase/decrease batch size 40 | ######################################################## 41 | if [ $batchcount -eq 5 ] 42 | then 43 | # this is the sleep interval between batches of instances 44 | sleep 20 45 | # Don't change this, change the batchcount variable above 46 | batchcount=0 47 | fi 48 | done 49 | -------------------------------------------------------------------------------- /generic/README.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | 3 | ## Before you start 4 | 5 | See ../README.md for instructions on using the cluster management scripts. 6 | 7 | ### a) Deploy cluster(s) 8 | 9 | 1. Get the repo and switch to the 'generic' directory 10 | 11 | ``` 12 | git clone https://github.com/seanorama/masterclass 13 | cd masterclass/generic 14 | ``` 15 | 16 | 2. Set these variables, updating the values as appropriate: 17 | 18 | ```sh 19 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 20 | export lab_prefix="${USER}" ## template for naming the cloudformation stacks 21 | export lab_first=100 ## number to start at in naming 22 | export lab_count=1 ## number of clusters to create 23 | 24 | export cfn_parameters=' 25 | [ 26 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 27 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 28 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 29 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"2"}, 30 | {"ParameterKey":"KeyName","ParameterValue":"ey-student"}, 31 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-c10a11a5"}, 32 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-3a408c43"}, 33 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG HIVE YARN ZOOKEEPER SPARK AMBARI_INFRA LOGSEARCH AMBARI_METRICS SQOOP TEZ ZEPPELIN SLIDER"}, 34 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"0"}, 35 | {"ParameterKey":"DeployCluster","ParameterValue":"true"}, 36 | {"ParameterKey":"AmbariVersion","ParameterValue":"2.4.2.0"}, 37 | {"ParameterKey":"HDPStack","ParameterValue":"2.5"}, 38 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/generic/setup.sh | bash"}, 39 | {"ParameterKey":"InstanceType","ParameterValue":"m4.xlarge"}, 40 | {"ParameterKey":"BootDiskSize","ParameterValue":"80"} 41 | ] 42 | ' 43 | ``` 44 | 45 | 3. Create the cluster(s): `../bin/clusters-create.sh` 46 | 47 | 4. List cluster host(s): `../bin/clusters-report.sh` 48 | 49 | 5. Terminate cluster(s): `../bin/clusters-terminate.sh` 50 | 51 | ## REMEMBER to terminate the clusters immediately after the class is over, or be prepared to pay $$$! 52 | 53 | Further, you should verify deletion of the CloudFormations & EC2 instances from the AWS Console. 54 | 55 | ## Issues: See ../README.md 56 | 57 | ## Advanced usage 58 | 59 | 1. Only deploy the infrastructure by setting PostCommand to /bin/true 60 | -------------------------------------------------------------------------------- /generic/TEST.md: -------------------------------------------------------------------------------- 1 | # Commands for various testing use cases 2 | ======================================== 3 | 4 | 5 | 1. Deploy infrastructure only 6 | 7 | ```sh 8 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 9 | export lab_prefix=${USER} ## template for naming the cloudformation stacks 10 | export lab_first=100 ## number to start at in naming 11 | export lab_count=1 ## number of clusters to create 12 | 13 | export cfn_parameters=' 14 | [ 15 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 16 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 17 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 18 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"0"}, 19 | {"ParameterKey":"PostCommand","ParameterValue":"/bin/true"}, 20 | {"ParameterKey":"InstanceType","ParameterValue":"m4.xlarge"}, 21 | {"ParameterKey":"BootDiskSize","ParameterValue":"30"} 22 | ] 23 | ' 24 | ``` 25 | -------------------------------------------------------------------------------- /generic/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o xtrace 3 | 4 | export HOME=${HOME:-/root} 5 | export TERM=xterm 6 | : ${ambari_pass:="BadPass#1"} 7 | ambari_password="${ambari_pass}" 8 | : ${cluster_name:="mycluster"} 9 | : ${ambari_services:="HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER AMBARI_METRICS SLIDER AMBARI_INFRA TEZ"} 10 | : ${install_ambari_server:=true} 11 | : ${ambari_stack_version:=2.5} 12 | : ${host_count:=skip} 13 | : ${recommendation_strategy:="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"} 14 | 15 | : ${install_nifi:=false} 16 | nifi_version=1.1.2 17 | 18 | export install_ambari_server ambari_pass host_count ambari_services 19 | export ambari_password cluster_name recommendation_strategy 20 | 21 | cd 22 | 23 | yum makecache 24 | yum -y -q install git epel-release ntpd screen mysql-connector-java jq python-argparse python-configobj ack 25 | 26 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 27 | 28 | ~/ambari-bootstrap/extras/deploy/prep-hosts.sh 29 | 30 | ~/ambari-bootstrap/ambari-bootstrap.sh 31 | 32 | ## Ambari Server specific tasks 33 | if [ "${install_ambari_server}" = "true" ]; then 34 | bash -c "nohup ambari-server restart" || true 35 | 36 | sleep 60 37 | 38 | ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar 39 | ambari_pass=admin source ~/ambari-bootstrap/extras/ambari_functions.sh 40 | ambari_change_pass admin admin ${ambari_pass} 41 | sleep 1 42 | 43 | if [ "${deploy}" = "true" ]; then 44 | 45 | cd ~/ambari-bootstrap/deploy 46 | 47 | ## various configuration changes for demo environments, and fixes to defaults 48 | cat << EOF > configuration-custom.json 49 | { 50 | "configurations" : { 51 | "core-site": { 52 | "hadoop.proxyuser.root.users" : "admin", 53 | "fs.trash.interval": "4320" 54 | }, 55 | "hdfs-site": { 56 | "dfs.namenode.safemode.threshold-pct": "0.99" 57 | }, 58 | "hive-site": { 59 | "hive.server2.transport.mode": "http", 60 | "hive.exec.compress.output": "true", 61 | "hive.merge.mapfiles": "true", 62 | "hive.server2.tez.initialize.default.sessions": "true" 63 | }, 64 | "mapred-site": { 65 | "mapreduce.job.reduce.slowstart.completedmaps": "0.7", 66 | "mapreduce.map.output.compress": "true", 67 | "mapreduce.output.fileoutputformat.compress": "true" 68 | }, 69 | "yarn-site": { 70 | "yarn.acl.enable" : "true" 71 | } 72 | } 73 | } 74 | EOF 75 | 76 | ./deploy-recommended-cluster.bash 77 | 78 | if [ "${install_nifi}" = "true" ]; then 79 | cd /opt 80 | curl -ssLO http://mirrors.ukfast.co.uk/sites/ftp.apache.org/nifi/${nifi_version}/nifi-${nifi_version}-bin.tar.gz 81 | tar -xzvf nifi-${nifi_version}-bin.tar.gz 82 | sed -i 's/^\(nifi.web.http.port=\).*/\19090/' nifi-${nifi_version}/conf/nifi.properties 83 | /opt/nifi-${nifi_version}/bin/nifi.sh start 84 | fi 85 | 86 | cd ~ 87 | sleep 5 88 | source ~/ambari-bootstrap/extras/ambari_functions.sh 89 | ambari_configs 90 | ambari_wait_request_complete 1 91 | cd ~ 92 | sleep 10 93 | 94 | usermod -a -G users ${USER} 95 | echo "${ambari_pass}" | passwd admin --stdin 96 | sudo sudo -u hdfs bash -c " 97 | hadoop fs -mkdir /user/admin; 98 | hadoop fs -chown admin /user/admin; 99 | hdfs dfsadmin -refreshUserToGroupsMappings" 100 | fi 101 | fi 102 | 103 | -------------------------------------------------------------------------------- /governance/instances/20150915: -------------------------------------------------------------------------------- 1 | export lab_count=40 2 | export lab_first=100 3 | export lab_prefix=par 4 | source ~/src/ambari-bootstrap/providers/google/create-google-hosts.sh 5 | create=true ~/src/ambari-bootstrap/providers/google/create-google-hosts.sh 6 | 7 | exit 8 | 9 | command="echo OK"; pdsh -w ${hosts_all} "${command}" 10 | pdsh -w ${hosts_all} "${command}" 11 | 12 | exit 13 | 14 | read -r -d '' command < 2 | 5 | 6 | 7 | pipeline=churnAnalysisDataPipeline,owner=ETLGroup 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 1 17 | FIFO 18 | hours(1) 19 | GMT+02:00 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 31 | 32 | 33 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /governance/labs/falconChurnDemo/cleansedEmailFeed.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 7 | 8 | owner=USMarketing,classification=Secure,externalSource=USProdEmailServers,externalTarget=BITools 9 | churnAnalysisDataPipeline 10 | 11 | hours(1) 12 | GMT+02:00 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /governance/labs/falconChurnDemo/emailIngestProcess.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | pipeline=churnAnalysisDataPipeline,owner=ETLGroup,externalSystem=USWestEmailServers 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 1 17 | FIFO 18 | hours(1) 19 | GMT+02:00 20 | 21 | 22 | 23 | 24 | 25 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /governance/labs/falconChurnDemo/oregonCluster.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /governance/labs/falconChurnDemo/rawEmailFeed.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 7 | 8 | externalSystem=USWestEmailServers,classification=secure 9 | churnAnalysisDataPipeline 10 | hours(1) 11 | GMT+02:00 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /governance/labs/falconChurnDemo/virginiaCluster.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /governance/labs/hadoop-data-pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env 2 | 3 | function pause(){ read -p "$*" } 4 | 5 | echo ## Creating Directories & Fetching Lab Files 6 | sudo mkdir -p /tmp/data_pipeline_demo/input; chmod 777 /tmp/data_pipeline_demo/input 7 | sudo mkdir /app; chown ${USER}:users /app; chmod 777 /app; cd /app 8 | git clone -q https://github.com/seanorama/hadoop-data-pipeline/ 9 | cd hadoop-data-pipeline/scripts/ 10 | 11 | echo ## Preparing application environment 12 | set+x; sudo ./setupAppOnHDFS.sh; set -x 13 | 14 | pause "Press [Enter] to continue ..." 15 | 16 | echo ## Updating feed & process dates 17 | sudo ./changeValidityForFeed.sh 18 | sudo ./changeValidityForProcess.sh 19 | 20 | echo ## Inspect the feed: 21 | set +x; cat ../falcon/feeds/inputFeed.xml; set -x 22 | pause "Press [Enter] to continue ..." 23 | 24 | echo ## Inspect the process: 25 | set +x; cat ../falcon/process/processData.xml; set -x 26 | pause "Press [Enter] to continue ..." 27 | 28 | echo ## Submit & schedule the entities: 29 | set+x 30 | sudo ./submitEntities.sh 31 | sudo ./scheduleEntities.sh 32 | set -x 33 | pause "Press [Enter] to continue ..." 34 | 35 | echo ## Loading sample data to Flume 36 | cp -a /app/hadoop-data-pipeline/input_data/SV-sample-1.xml /tmp/data_pipeline_demo/input/ -------------------------------------------------------------------------------- /governance/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | ## for prepping a 1-node cluster for the governance masterclass 4 | 5 | sudo yum -y -q install git 6 | cd 7 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 8 | source ~/ambari-bootstrap/extras/ambari_functions.sh 9 | 10 | ${__dir}/deploy/prep-hosts.sh 11 | 12 | export ambari_services="YARN ZOOKEEPER TEZ OOZIE FLUME PIG SLIDER MAPREDUCE2 HIVE HDFS FALCON ATLAS SQOOP" 13 | "${__dir}/deploy/deploy-hdp.sh" 14 | sleep 30 15 | 16 | source ${__dir}/ambari_functions.sh 17 | source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_change_pass admin admin BadPass#1 18 | echo "export ambari_pass=BadPass#1" > ~/ambari-bootstrap/extras/.ambari.conf; chmod 660 ~/ambari-bootstrap/extras/.ambari.conf 19 | source ${__dir}/ambari_functions.sh 20 | ambari_configs 21 | ambari_wait_request_complete 1 22 | 23 | ## Generic setup 24 | sudo chkconfig mysqld on; sudo service mysqld start 25 | ${__dir}/add-trusted-ca.sh 26 | ${__dir}/onboarding.sh 27 | #${__dir}/ambari-views/create-views.sh 28 | config_proxyuser=true ${__dir}/ambari-views/create-views.sh 29 | #${__dir}/samples/sample-data.sh 30 | ${__dir}/configs/proxyusers.sh 31 | ${__dir}/ranger/prep-mysql.sh 32 | proxyusers="oozie" ${__dir}/configs/proxyusers.sh 33 | ## centos6 only #${__dir}/oozie/replace-mysql-connector.sh 34 | 35 | sudo mkdir -p /app; sudo chown ${USER}:users /app; sudo chmod g+wx /app 36 | 37 | mirror_host="${mirror_host:-mc-teacher1.$(hostname -d)}" 38 | mirror_host_ip=$(ping -w 1 ${mirror_host} | awk 'NR==1 {print $3}' | sed 's/[()]//g') 39 | echo "${mirror_host_ip} mirror.hortonworks.com ${mirror_host} mirror admin admin.hortonworks.com" | sudo tee -a /etc/hosts 40 | 41 | ${ambari_config_set} capacity-scheduler yarn.scheduler.capacity.root.default.maximum-am-resource-percent 0.5 42 | ${ambari_config_set} capacity-scheduler yarn.scheduler.capacity.maximum-am-resource-percent 0.5 43 | ${ambari_config_set} yarn-site yarn.scheduler.minimum-allocation-mb 250 44 | #${ambari_config_set} yarn-site yarn.scheduler.maximum-allocation-mb 8704 45 | ${ambari_config_set} yarn-site "yarn.resourcemanager.webapp.proxyuser.hcat.groups" "*" 46 | ${ambari_config_set} yarn-site "yarn.resourcemanager.webapp.proxyuser.hcat.hosts" "*" 47 | ${ambari_config_set} yarn-site "yarn.resourcemanager.webapp.proxyuser.oozie.groups" "*" 48 | ${ambari_config_set} yarn-site "yarn.resourcemanager.webapp.proxyuser.oozie.hosts" "*" 49 | ${ambari_config_set} yarn-site yarn.scheduler.minimum-allocation-vcores 1 50 | 51 | ## Governance specific setup 52 | sudo usermod -a -G hadoop admin 53 | ${__dir}/atlas/atlas-hive-enable.sh 54 | proxyusers="falcon flume" ${__dir}/configs/proxyusers.sh 55 | proxyusers="falcon flume" ${__dir}/oozie/proxyusers.sh 56 | ${__dir}/falcon/bugfix_oozie-site_elexpression.sh 57 | ${ambari_config_set} oozie-site oozie.service.AuthorizationService.security.enabled "false" 58 | 59 | ##### atlas client tutorial 60 | ## install atlas client 61 | curl -ssLO https://github.com/seanorama/atlas/releases/download/0.1/atlas-client.tar.bz2 62 | sudo yum -y -q install bzip2 63 | tar -xf atlas-client.tar.bz2 64 | sudo mv atlas-client /opt 65 | sudo ln -sf /opt/atlas-client/bin/atlas-client /usr/local/bin/ 66 | sudo touch /application.log /audit.log; sudo chown ${USER} /application.log /audit.log 67 | 68 | ## setup source DRIVERS & TIMESHEET database in MySQL 69 | cd 70 | git clone https://github.com/seanorama/atlas 71 | cd atlas/tutorial 72 | mysql -u root < MySQLSourceSystem.sql 73 | #### 74 | 75 | 76 | ## setup falcon churn demo 77 | 78 | mkdir /tmp/falcon-churn; cd /tmp/falcon-churn 79 | curl -sSL -O http://hortonassets.s3.amazonaws.com/tutorial/falcon/falcon.zip 80 | unzip falcon.zip 81 | sudo su - hdfs -c "hadoop fs -mkdir -p /shared/falcon/demo/primary/processed/enron; hadoop fs -chmod -R 777 /shared" 82 | sudo sudo -u admin hadoop fs -copyFromLocal demo /shared/falcon/ 83 | sudo sudo -u hdfs hadoop fs -chown -R admin:hadoop /shared/falcon 84 | sudo sudo -u hdfs hadoop fs -chmod -R g+w /shared/falcon 85 | 86 | ( 87 | sudo mkdir -p /opt/hadoop/samples 88 | sudo chmod 777 /opt/hadoop/samples 89 | cd /opt/hadoop/samples 90 | 91 | dfs_cmd="sudo sudo -u hdfs hadoop fs" 92 | dfs_cmd_admin="sudo sudo -u admin hadoop fs" 93 | 94 | ${dfs_cmd} -mkdir /public 95 | ${dfs_cmd} -mkdir -p /public/samples /public/secured/dir1 96 | ${dfs_cmd} -chmod -R 777 /public 97 | 98 | ## Sandbox data sets 99 | curl -sSL -O https://raw.githubusercontent.com/abajwa-hw/security-workshops/master/data/sample_07.csv 100 | curl -sSL -O https://raw.githubusercontent.com/abajwa-hw/security-workshops/master/data/sample_08.csv 101 | ${dfs_cmd_admin} -put sample_07.csv sample_08.csv /public/samples 102 | ) 103 | 104 | -------------------------------------------------------------------------------- /hdc/README.md: -------------------------------------------------------------------------------- 1 | # Hortonworks Data Cloud 2 | 3 | ## Fetch this repo 4 | ``` 5 | sudo yum -y install git 6 | git clone https://github.com/seanorama/masterclass 7 | cd masterclass/hdc 8 | ``` 9 | 10 | ## Demo HDCloud 11 | 12 | ### Demo: Show deployment options and integrations 13 | 1. Create HDCloud Controller 14 | 2. From HDCloud: 15 | - Register Authentication (LDAP) 16 | - Register Hive MetaStore (Postgresql on AWS RDS) 17 | 3. From HDCloud: 18 | - Create Shared Data Lake Services 19 | 5. From HDCloud: 20 | - Create Clusters 21 | 22 | ### Demo: Show CLI 23 | 24 | 1. SSH to Cloud controller 25 | 2. Use `hdc` cli 26 | 27 | 28 | ``` 29 | ## show options 30 | hdc 31 | 32 | ## list clusters 33 | hdc list-clusters -output table 34 | 35 | ## create a cluster 36 | hdc create-cluster --cli-input-json ./templates/cluster-datascience.json --input-json-param-ClusterAndAmbariPassword ${pass} 37 | ``` 38 | 39 | -------- 40 | 41 | # Automated Deployment of many clusters 42 | 43 | ## Deploy clusters 44 | 45 | Set below before executing other commands: 46 | ``` 47 | export lab_prefix="crashcourse" ## template for naming the cloudformation stacks 48 | export lab_first=10 ## number to start at in naming 49 | export lab_count=3 ## number of clusters to create 50 | export cluster_type=hdp26-data-science-spark2 51 | export ambari_pass=BadPass#1 52 | ``` 53 | 54 | Commands to manage clusters: 55 | ``` 56 | ## create clusters 57 | ../bin/hdc-clusters-create.sh 58 | 59 | ## list clusters and check status 60 | hdc list-clusters -output table 61 | 62 | ## get name & master's IP for all available clusters 63 | clusters=$(hdc list-clusters | jq -r '.[]|select(.Status=="AVAILABLE")|.ClusterName') 64 | for cluster in ${clusters}; do 65 | printf "${cluster}\t$(hdc describe-cluster instances --cluster-name ${cluster} | jq -r '.[] | select(.Type =="master - ambari server")|.PublicIP')\n" 66 | done 67 | 68 | ## terminate the above clusters 69 | ../bin/hdc-clusters-terminate.sh 70 | 71 | ## terminate ALL clusters managed by this HDCloud controller 72 | ## WARNING: THIS WILL TERMINATE ALL CLUSTERS WITHOUT PROMPTING YOU!!!! 73 | clusters=$(hdc list-clusters | jq -r '.[].ClusterName') 74 | for cluster in ${clusters}; do 75 | hdc terminate-cluster --cluster-name ${cluster} 76 | done 77 | ``` 78 | 79 | -------------------------------------------------------------------------------- /hdc/templates/cluster-analytics.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "ClusterName": "sroberts-analytics", 4 | "HDPVersion": "2.6", 5 | "ClusterType": "EDW-Analytics: Apache Hive 2 LLAP, Apache Zeppelin 0.7.0", 6 | "Master": { 7 | "InstanceType": "m4.4xlarge", 8 | "VolumeType": "gp2", 9 | "VolumeSize": 32, 10 | "VolumeCount": 1 11 | }, 12 | "Worker": { 13 | "InstanceType": "m3.xlarge", 14 | "VolumeType": "ephemeral", 15 | "VolumeSize": 40, 16 | "VolumeCount": 2, 17 | "InstanceCount": 3, 18 | "RecoveryMode": "AUTO" 19 | }, 20 | "Compute": { 21 | "InstanceType": "m3.xlarge", 22 | "VolumeType": "ephemeral", 23 | "VolumeSize": 40, 24 | "VolumeCount": 2, 25 | "InstanceCount": 1, 26 | "RecoveryMode": "AUTO" 27 | }, 28 | "SSHKeyName": "secloud", 29 | "RemoteAccess": "0.0.0.0/0", 30 | "WebAccess": true, 31 | "HiveJDBCAccess": true, 32 | "ClusterComponentAccess": true, 33 | "ClusterAndAmbariUser": "admin", 34 | "ClusterAndAmbariPassword": "", 35 | "Tags": { 36 | "Owner": "sroberts", 37 | "Project": "summit" 38 | }, 39 | "ClusterInputs": { 40 | "LDAP_URL": "ldap://52.28.61.126:389", 41 | "LDAP_BIND_PASSWORD": "BadPass#1", 42 | "LDAP_SYNC_SEARCH_BASE": "ou=CorpUsers,dc=lab,dc=hortonworks,dc=net", 43 | "LDAP_DOMAIN": "lab.hortonworks.net", 44 | "RANGER_ADMIN_USERNAME": "amb_ranger_admin", 45 | "REMOTE_CLUSTER_NAME": "lake0", 46 | "ATLAS_KAFKA_BOOTSTRAP_SERVERS": "ip-172-31-29-241.eu-central-1.compute.internal:6667,ip-172-31-22-240.eu-central-1.compute.internal:6667,ip-172-31-28-91.eu-central-1.compute.internal:6667", 47 | "LDAP_BIND_DN": "cn=ldap-reader,ou=ServiceUsers,dc=lab,dc=hortonworks,dc=net", 48 | "ADMIN_USERNAME": "admin", 49 | "S3_BUCKET": "hdc-datalake/lake", 50 | "ATLAS_KAFKA_HOOK_GROUP_ID": "atlas", 51 | "LDAP_GROUP_SEARCH_BASE": "dc=lab,dc=hortonworks,dc=net", 52 | "SOLR_ZOOKEPERS_URL": "ip-172-31-22-240.eu-central-1.compute.internal:2181,ip-172-31-28-91.eu-central-1.compute.internal:2181,ip-172-31-29-241.eu-central-1.compute.internal:2181/infra-solr", 53 | "RANGER_ADMIN_PASSWORD": "Shio1dae", 54 | "RANGER_REST_ADDRESS": "http://ip-172-31-28-91.eu-central-1.compute.internal:6080" 55 | }, 56 | "InstanceRole": "CREATE", 57 | "HiveMetastore": { 58 | "Name": "hive0" 59 | }, 60 | "Network": { 61 | "VpcId": "vpc-56b8713f", 62 | "SubnetId": "subnet-025cae79" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /hdc/templates/cluster-datascience.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "ClusterName": "sroberts-datascience", 4 | "HDPVersion": "2.6", 5 | "ClusterType": "Data Science: Apache Spark 2.1, Apache Zeppelin 0.6.2", 6 | "Master": { 7 | "InstanceType": "m4.4xlarge", 8 | "VolumeType": "gp2", 9 | "VolumeSize": 32, 10 | "VolumeCount": 1 11 | }, 12 | "Worker": { 13 | "InstanceType": "m3.xlarge", 14 | "VolumeType": "ephemeral", 15 | "VolumeSize": 40, 16 | "VolumeCount": 2, 17 | "InstanceCount": 3, 18 | "RecoveryMode": "AUTO" 19 | }, 20 | "Compute": { 21 | "InstanceType": "m3.xlarge", 22 | "VolumeType": "ephemeral", 23 | "VolumeSize": 40, 24 | "VolumeCount": 2, 25 | "InstanceCount": 0, 26 | "RecoveryMode": "AUTO" 27 | }, 28 | "SSHKeyName": "secloud", 29 | "RemoteAccess": "0.0.0.0/0", 30 | "WebAccess": true, 31 | "HiveJDBCAccess": true, 32 | "ClusterComponentAccess": true, 33 | "ClusterAndAmbariUser": "admin", 34 | "ClusterAndAmbariPassword": "", 35 | "Tags": { 36 | "Owner": "sroberts", 37 | "Project": "summit" 38 | }, 39 | "ClusterInputs": { 40 | "REMOTE_CLUSTER_NAME": "lake0", 41 | "LDAP_BIND_PASSWORD": "BadPass#1", 42 | "LDAP_SYNC_SEARCH_BASE": "ou=CorpUsers,dc=lab,dc=hortonworks,dc=net", 43 | "ATLAS_KAFKA_BOOTSTRAP_SERVERS": "ip-172-31-29-241.eu-central-1.compute.internal:6667,ip-172-31-22-240.eu-central-1.compute.internal:6667,ip-172-31-28-91.eu-central-1.compute.internal:6667", 44 | "ADMIN_USERNAME": "admin", 45 | "LDAP_URL": "ldap://52.28.61.126:389", 46 | "RANGER_REST_ADDRESS": "http://ip-172-31-28-91.eu-central-1.compute.internal:6080", 47 | "SOLR_ZOOKEPERS_URL": "ip-172-31-22-240.eu-central-1.compute.internal:2181,ip-172-31-28-91.eu-central-1.compute.internal:2181,ip-172-31-29-241.eu-central-1.compute.internal:2181/infra-solr", 48 | "LDAP_GROUP_SEARCH_BASE": "dc=lab,dc=hortonworks,dc=net", 49 | "LDAP_DOMAIN": "lab.hortonworks.net", 50 | "RANGER_ADMIN_USERNAME": "amb_ranger_admin", 51 | "LDAP_BIND_DN": "cn=ldap-reader,ou=ServiceUsers,dc=lab,dc=hortonworks,dc=net", 52 | "S3_BUCKET": "hdc-datalake/lake", 53 | "ATLAS_KAFKA_HOOK_GROUP_ID": "atlas", 54 | "RANGER_ADMIN_PASSWORD": "Shio1dae" 55 | }, 56 | "InstanceRole": "CREATE", 57 | "HiveMetastore": { 58 | "Name": "hive0" 59 | }, 60 | "Network": { 61 | "VpcId": "vpc-56b8713f", 62 | "SubnetId": "subnet-025cae79" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /hdc/templates/hdp26-data-science-spark2.json: -------------------------------------------------------------------------------- 1 | { 2 | "ClusterName": "crashcourse11", 3 | "HDPVersion": "2.6", 4 | "ClusterType": "Data Science: Apache Spark 2.1, Apache Zeppelin 0.7.0", 5 | "Master": { 6 | "InstanceType": "m4.4xlarge", 7 | "VolumeType": "gp2", 8 | "VolumeSize": 32, 9 | "VolumeCount": 1 10 | }, 11 | "Worker": { 12 | "InstanceType": "m3.xlarge", 13 | "VolumeType": "ephemeral", 14 | "VolumeSize": 40, 15 | "VolumeCount": 2, 16 | "InstanceCount": 1, 17 | "RecoveryMode": "AUTO" 18 | }, 19 | "Compute": { 20 | "InstanceType": "m3.xlarge", 21 | "VolumeType": "ephemeral", 22 | "VolumeSize": 40, 23 | "VolumeCount": 2, 24 | "InstanceCount": 0, 25 | "RecoveryMode": "AUTO" 26 | }, 27 | "SSHKeyName": "field", 28 | "RemoteAccess": "0.0.0.0/0", 29 | "WebAccess": true, 30 | "HiveJDBCAccess": true, 31 | "ClusterComponentAccess": false, 32 | "ClusterAndAmbariUser": "admin", 33 | "ClusterAndAmbariPassword": "", 34 | "Tags": {}, 35 | "Autoscaling": { 36 | "Configurations": { 37 | "CooldownTime": 30, 38 | "ClusterMinSize": 3, 39 | "ClusterMaxSize": 100 40 | } 41 | }, 42 | "InstanceRole": "CREATE", 43 | "Network": { 44 | "VpcId": "vpc-7a2b2813", 45 | "SubnetId": "subnet-e719e68f" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /hdf/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | These instructions will create lab environments for: 5 | 6 | - a. HDP Cluster: 1 node with HDFS, YARN, HIVE, NIFI, SOLR, SPARK, ZEPPELIN 7 | - b. Nifi: 1 node with nifi 8 | 9 | You should **open 2 separate shells** for deploying each environment. This is due to the management scripts depending on environment variables. 10 | 11 | ## Before you start 12 | 13 | See ../README.md for instructions on using the cluster management scripts (step 3 in the processes below): 14 | 15 | - ../bin/cluster-create.sh 16 | - ../bin/cluster-report.sh 17 | - ../bin/cluster-terminate.sh 18 | 19 | ## Deploy clusters 20 | Open 2 clean shell environments on a host where the AWS CLI has been configured. 21 | 22 | ### a) Deploy the HDP node 23 | 24 | 1. Get the repo and switch to the 'generic' directory 25 | 26 | ``` 27 | git clone https://github.com/seanorama/masterclass 28 | cd masterclass/generic 29 | ``` 30 | 31 | 2. Set these variables, updating the values as appropriate: 32 | 33 | ```sh 34 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 35 | export lab_prefix=mc-hdf-hdp ## template for naming the cloudformation stacks 36 | export lab_first=100 ## number to start at in naming 37 | export lab_count=1 ## number of clusters to create 38 | 39 | export cfn_parameters=' 40 | [ 41 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 42 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 43 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 44 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER NIFI SOLR KAFKA"}, 45 | {"ParameterKey":"InstanceType","ParameterValue":"m4.xlarge"}, 46 | {"ParameterKey":"BootDiskSize","ParameterValue":"80"} 47 | ] 48 | ' 49 | ``` 50 | 51 | 3. You can then execute ../bin/clusters-create.sh and the other cluster scripts as explained in ../README.md 52 | 53 | ### b) Deploy the NiFi node 54 | 55 | In the 2nd shell: 56 | 57 | 1. Switch to the 'generic' directory 58 | 59 | ``` 60 | cd masterclass/generic 61 | ``` 62 | 63 | 2. Set these variables, updating the values as appropriate: 64 | 65 | ```sh 66 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 67 | export lab_prefix=mc-hdf-nifi ## template for naming the cloudformation stacks 68 | export lab_first=100 ## number to start at in naming 69 | export lab_count=1 ## number of clusters to create 70 | 71 | export cfn_parameters=' 72 | [ 73 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 74 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 75 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 76 | {"ParameterKey":"AmbariServices","ParameterValue":"NIFI"}, 77 | {"ParameterKey":"InstanceType","ParameterValue":"t2.medium"}, 78 | {"ParameterKey":"BootDiskSize","ParameterValue":"80"} 79 | ] 80 | ' 81 | ``` 82 | 83 | 3. You can then execute ../bin/clusters-create.sh and the other cluster scripts as explained in ../README.md 84 | 85 | ## REMEMBER to terminate the clusters immediately after the class is over, or be prepared to pay $$$! 86 | 87 | Further, you should verify deletion of the CloudFormations & EC2 instances from the AWS Console. 88 | 89 | ## Issues: See ../README.md 90 | -------------------------------------------------------------------------------- /hdf/README.md: -------------------------------------------------------------------------------- 1 | Demos & Labs available at https://github.com/simonellistonball/masterclass-hdf 2 | -------------------------------------------------------------------------------- /operations/README.txt: -------------------------------------------------------------------------------- 1 | # Operations Masterclass materials 2 | 3 | From Dave Russell with collaboration from @seanorama 4 | -------------------------------------------------------------------------------- /operations/files/create-clouds-loopy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ############################################################################## 3 | # Note the loops and pause reduce overloading the AWS region during 4 | # provisioning. Number of instances spawned per batch is controlled by the 5 | # variable "batchcount", first sleep is in between each instance spawn, second 6 | # sleep is between each batch of instances. 7 | ############################################################################## 8 | 9 | #### 10 | #export clusters="apple apricot banana blackberry blackcurrant blueberry coconut cherry clementine cranberry elderberry fig gooseberry grape guava huckleberry lemon lime lychee mango melon nectarine orange passionfruit peach pear persimmon plum prune pineapple pomegranate raspberry satsuma strawberry tangerine" 11 | clusters=${clusters:-apple} 12 | 13 | # quick check to ensure pre-reqs are setup 14 | while true; do 15 | read -p "Is the proxy hostname correctly setup in the cfn template and proxy is running? (y/n) " yn 16 | case $yn in 17 | [Yy]* ) break;; 18 | [Nn]* ) exit;; 19 | * ) echo "Please answer yes or no.";; 20 | esac 21 | done 22 | 23 | # Don't change this, change the batchcount variable towards the end of script 24 | batchcount=0 25 | 26 | # change the list of fruit to vary the number of clusters deployed 27 | for fruitycluster in ${clusters} 28 | do 29 | ((batchcount++)) 30 | aws cloudformation create-stack --stack-name $fruitycluster --template-body file://./cfn-ambari-opsmasterclass.template-24.json --parameters ParameterKey=KeyName,ParameterValue=secloud --capabilities CAPABILITY_IAM 31 | echo Initiated creation of $fruitycluster cluster 32 | # this is the sleep interval between instances 33 | sleep 5 34 | ######################################################## 35 | # change the value below to increase/decrease batch size 36 | ######################################################## 37 | if [ $batchcount -eq 6 ] 38 | then 39 | # this is the sleep interval between batches of instances 40 | sleep 600 41 | # Don't change this, change the batchcount variable above 42 | batchcount=0 43 | fi 44 | done 45 | -------------------------------------------------------------------------------- /operations/files/report-clouds.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #export clusters="apple apricot banana blackberry blackcurrant blueberry coconut cherry clementine cranberry damson elderberry fig gooseberry grape guava huckleberry lemon lime lychee mango melon nectarine orange passionfruit peach pear persimmon plum prune pineapple pomegranate raspberry satsuma strawberry tangerine" 4 | clusters=${clusters:-apple} 5 | 6 | for fruitycluster in ${clusters} 7 | do 8 | 9 | echo "####################################################" 10 | echo "This is the $fruitycluster cluster" 11 | echo "####################################################" 12 | echo "This will be operated by: <1st PERSON NAME HERE>" 13 | echo " and: <2nd PERSON NAME HERE>" 14 | echo "####################################################" 15 | echo "" 16 | echo "This is the $fruitycluster cluster Ambari Node" 17 | echo "public name:" 18 | aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PublicDnsName]' --filters Name=instance-state-name,Values=running Name=tag:aws:cloudformation:stack-name,Values=$fruitycluster Name=tag:aws:cloudformation:logical-id,Values=AmbariNode --output text 19 | echo "" 20 | echo "private name:" 21 | aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PrivateDnsName]' --filters Name=instance-state-name,Values=running Name=tag:aws:cloudformation:stack-name,Values=$fruitycluster Name=tag:aws:cloudformation:logical-id,Values=AmbariNode --output text 22 | echo "" 23 | echo "These are the $fruitycluster cluster Master Nodes" 24 | aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PrivateDnsName]' --filters Name=instance-state-name,Values=running Name=tag:aws:cloudformation:stack-name,Values=$fruitycluster Name=tag:aws:cloudformation:logical-id,Values=MasterNodes --output text 25 | echo "" 26 | echo "These are the $fruitycluster cluster Data Nodes" 27 | aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PrivateDnsName]' --filters Name=instance-state-name,Values=running Name=tag:aws:cloudformation:stack-name,Values=$fruitycluster Name=tag:aws:cloudformation:logical-id,Values=WorkerNodes --output text 28 | echo "" 29 | done 30 | -------------------------------------------------------------------------------- /operations/files/terminate-clouds-loopy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################## 3 | # Note the loops and pause reduce overloading the AWS region during 4 | # deletion. Number of instances deleted per batch is controlled by the 5 | # variable "batchcount", first sleep is in between each instance deletion, second 6 | # sleep is between each batch of instances. 7 | ############################################################################## 8 | 9 | # full set of 35 fruits, use as many as necessary 10 | # apple apricot banana blackberry blackcurrant blueberry coconut cherry clementine cranberry damson elderberry fig gooseberry grape guava huckleberry lemon lime lychee mango melon nectarine orange passionfruit peach pear plum prune pineapple pomegranate raspberry satsuma strawberry tangerine 11 | 12 | #export clusters="apple apricot banana blackberry blackcurrant blueberry coconut cherry clementine cranberry damson elderberry fig gooseberry grape guava huckleberry lemon lime lychee mango melon nectarine orange passionfruit peach pear persimmon plum prune pineapple pomegranate raspberry satsuma strawberry tangerine" 13 | clusters=${clusters:-apple} 14 | echo ${clusters} 15 | 16 | 17 | # quick safety check to ensure termination is required 18 | while true; do 19 | read -p "Do you wish terminate all masterclass instances? (y/n) " yn 20 | case $yn in 21 | [Yy]* ) break;; 22 | [Nn]* ) exit;; 23 | * ) echo "Please answer yes or no.";; 24 | esac 25 | done 26 | 27 | # Don't change this, change the batchcount test lower down the end of script 28 | batchcount=0 29 | 30 | # change the list of fruit to vary the number of clusters deleted 31 | for fruitycluster in ${clusters} 32 | do 33 | ((batchcount++)) 34 | aws cloudformation delete-stack --stack-name $fruitycluster 35 | echo Initiated deletion of $fruitycluster cluster 36 | # this is the sleep interval between instances 37 | sleep 5 38 | ######################################################## 39 | # change the value below to increase/decrease batch size 40 | ######################################################## 41 | if [ $batchcount -eq 7 ] 42 | then 43 | # this is the sleep interval between batches of instances 44 | sleep 20 45 | # Don't change this, change the batchcount variable above 46 | batchcount=0 47 | fi 48 | done 49 | -------------------------------------------------------------------------------- /prepare/activedirectory/activedirectory-temp.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## for active directory only since we won't be syncing users 4 | ldap_user=ldap-connect@hortonworks.com 5 | ldap_pass="BadPass#1" 6 | users=$(ldapsearch -w ${ldap_pass} -D ${ldap_user} "(UnixHomeDirectory=/home/*)" sAMAccountName | awk '/^sAMAccountName: / {print $2}') 7 | for user in ${users}; do 8 | if ! id -u ${user}; then 9 | sudo useradd -G users ${user} 10 | printf "BadPass#1\nBadPass#1" | sudo passwd ${user} 11 | fi 12 | done 13 | 14 | 15 | exit 16 | 17 | sudo tee /etc/samba/smb.conf > /dev/null <<-EOF 18 | [Global] 19 | netbios name = $(hostname -s) 20 | workgroup = HORTONWORKS 21 | realm = HORTONWORKS.COM 22 | server string = %h HDP Host 23 | security = ads 24 | encrypt passwords = yes 25 | password server = activedirectory.hortonworks.com 26 | 27 | kerberos method = secrets and keytab 28 | 29 | idmap config * : backend = rid 30 | idmap config * : range = 10000-20000 31 | 32 | winbind use default domain = Yes 33 | winbind enum users = Yes 34 | winbind enum groups = Yes 35 | winbind nested groups = Yes 36 | winbind separator = + 37 | winbind refresh tickets = yes 38 | 39 | template shell = /bin/bash 40 | template homedir = /home/%U 41 | 42 | preferred master = no 43 | dns proxy = no 44 | wins server = activedirectory.hortonworks.com 45 | wins proxy = no 46 | 47 | inherit acls = Yes 48 | map acl inherit = Yes 49 | acl group control = yes 50 | 51 | load printers = no 52 | debug level = 3 53 | use sendfile = no 54 | EOF 55 | 56 | ## 57 | sudo yum -y install sssd 58 | 59 | sudo tee /etc/sssd/sssd.conf > /dev/null <<-'EOF' 60 | [sssd] 61 | config_file_version = 2 62 | domains = hortonworks.com 63 | services = nss, pam 64 | debug_level = 0 65 | 66 | [nss] 67 | 68 | [pam] 69 | 70 | [domain/hortonworks.com] 71 | 72 | ldap_uri = ldap://activedirectory.hortonworks.com 73 | ldap_default_bind_dn = CN=ldap-connect,cn=Users,dc=hortonworks,dc=com 74 | ldap_default_authtok_type = password 75 | ldap_default_authtok = "BadPass#1" 76 | ldap_group_search_base = dc=hortonworks,dc=com 77 | ldap_user_search_base = dc=hortonworks,dc=com 78 | 79 | access_provider = ldap 80 | id_provider = ldap 81 | cache_credentials = True 82 | 83 | ldap_id_use_start_tls = True 84 | ldap_tls_cacertdir = /etc/pki/tls/certs 85 | #ldap_tls_cacert = /etc/pki/ca-trust/source/anchors/activedirectory.pem 86 | ldap_tls_cacert = /etc/pki/tls/certs/ca-bundle.crt 87 | ldap_referrals = false 88 | ldap_schema = rfc2307bis 89 | ldap_access_order = expire 90 | ldap_account_expire_policy = ad 91 | 92 | ldap_group_object_class = group 93 | 94 | ldap_user_object_class = user 95 | ldap_user_home_directory = unixHomeDirectory 96 | ldap_user_principal = userPrincipalName 97 | ldap_user_shell = loginShell 98 | EOF 99 | 100 | sudo chmod 0600 /etc/sssd/sssd.conf 101 | sudo chkconfig sssd on 102 | sudo service sssd restart 103 | 104 | sudo authconfig \ 105 | --enablesssd \ 106 | --enablesssdauth \ 107 | --enablelocauthorize \ 108 | --enableldap \ 109 | --enableldapauth \ 110 | --ldapserver=ldaps://activedirectory.hortonworks.com \ 111 | --disableldaptls \ 112 | --ldapbasedn=dc=hortonworks,dc=com \ 113 | --enablerfc2307bis \ 114 | --enablemkhomedir \ 115 | --enablecachecreds \ 116 | --update 117 | 118 | 119 | -------------------------------------------------------------------------------- /prepare/activedirectory/activedirectory.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | sudo yum -y install openldap-clients bind-utils krb5-workstation ca-certificates 4 | 5 | ad_host="activedirectory.$(hostname -d)" 6 | ad_host_ip=$(ping -w 1 ${ad_host} | awk 'NR==1 {print $3}' | sed 's/[()]//g') 7 | echo "${ad_host_ip} activedirectory.hortonworks.com ${ad_host} activedirectory" | sudo tee -a /etc/hosts 8 | 9 | ad_cert=/etc/pki/ca-trust/source/anchors/activedirectory.pem 10 | sudo tee ${ad_cert} > /dev/null <<-'EOF' 11 | -----BEGIN CERTIFICATE----- 12 | MIIDrTCCApWgAwIBAgIQFNSgEcmw1r9OP9AicCaGdDANBgkqhkiG9w0BAQUFADBd 13 | MRMwEQYKCZImiZPyLGQBGRYDY29tMRswGQYKCZImiZPyLGQBGRYLaG9ydG9ud29y 14 | a3MxKTAnBgNVBAMTIGhvcnRvbndvcmtzLUFDVElWRURJUkVDVE9SWS1DQS0xMB4X 15 | DTE1MDcxNzIxMDI0N1oXDTIwMDcxNzIxMTI0N1owXTETMBEGCgmSJomT8ixkARkW 16 | A2NvbTEbMBkGCgmSJomT8ixkARkWC2hvcnRvbndvcmtzMSkwJwYDVQQDEyBob3J0 17 | b253b3Jrcy1BQ1RJVkVESVJFQ1RPUlktQ0EtMTCCASIwDQYJKoZIhvcNAQEBBQAD 18 | ggEPADCCAQoCggEBAL6hKjn05Wr2/seMjrIYjdTG03sAeK1U5htzIJTZr7YWCJi0 19 | fpj9NyIxYnQ58jxCnvq5dHVCExSzXCGvYhxoqk/VlA2puT1olrCPdakkBldj5S6l 20 | 0zUgauP4TQhBGVOnnBnfGOunfp2LP7yFj/QNCvQfKFeDeFptDiEEGyKO02Vif0Hp 21 | IpS4Qqsk34kgPZ7Qy52frXIWLpndGQKsRFik3dNKcayP47ld4kRlrjvWJ0U5QZsj 22 | Y1wIPf+VMCyp+npUraKO9wMguBbjwzU6PpsiLe1I/s11aq4lhCDaJoh9qOHXwBY5 23 | YV9nl3bxJ79Xi1t23QG4+yUEgGGPkroCLHHWXN0CAwEAAaNpMGcwEwYJKwYBBAGC 24 | NxQCBAYeBABDAEEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYD 25 | VR0OBBYEFNG0iZwgi9cFiSMBVirEZ49fLYKiMBAGCSsGAQQBgjcVAQQDAgEAMA0G 26 | CSqGSIb3DQEBBQUAA4IBAQBmb05AMnh7PFisKLsQnz3US/hMf1RM1WZl8DmS3sSx 27 | rh5tYsg2UTp3mbm8VG6aFMYYzhazIt1u2JLN5r5DwTVHvG44r5hOQ37LImUr2W5W 28 | ZnTeGdLU7uEV6RUSN5RtjIN7AHosIQik+dXt2BhdNyoYY9GsLvjfNO7IhBSGlmz9 29 | ZP1AFD4f0oMDCETkSSvb1nW2sCjrMkA5ttxO/IKq8kaim/nPsXA4yO6Fbbrd/Y8C 30 | e4XWhj04QLBbncIvRmyQMHD1DGLlLVeS2/LPMGR8ThYOk9Kh9MhZqx56eV7bPiU1 31 | vxeIATwrsETUfRhyKA2oM1X0DJ1ZjxSDNIXJH6HNd4TA 32 | -----END CERTIFICATE----- 33 | EOF 34 | 35 | sudo update-ca-trust enable 36 | sudo update-ca-trust extract; sudo update-ca-trust check 37 | sudo keytool -import -trustcacerts -noprompt -storepass changeit \ 38 | -file ${ad_cert} -keystore /etc/pki/java/cacerts 39 | sudo keytool -importcert -noprompt -storepass changeit \ 40 | -file ${ad_cert} -keystore /etc/pki/java/cacerts 41 | sudo keytool -import -trustcacerts -noprompt -storepass changeit \ 42 | -file ${ad_cert} -keystore /var/lib/ambari-server/keys/ldaps-keystore.jks 43 | 44 | sudo tee /etc/openldap/ldap.conf > /dev/null <<-EOF 45 | SASL_NOCANON on 46 | URI ldaps://activedirectory.hortonworks.com 47 | BASE dc=hortonworks,dc=com 48 | TLS_CACERTDIR /etc/pki/tls/certs 49 | TLS_CACERT /etc/pki/tls/certs/ca-bundle.crt 50 | EOF 51 | ## can test with: 52 | #ldapsearch -W -D user@domain.com 53 | 54 | exit 55 | 56 | ## for active directory only since we won't be syncing users 57 | ldap_user=ldap-connect@hortonworks.com 58 | ldap_pass="BadPass#1" 59 | users=$(ldapsearch -w ${ldap_pass} -D ${ldap_user} "(UnixHomeDirectory=/home/*)" sAMAccountName | awk '/^sAMAccountName: / {print $2}') 60 | for user in ${users}; do 61 | if ! id -u ${user}; then 62 | sudo useradd -G users ${user} 63 | printf "BadPass#1\nBadPass#1" | sudo passwd ${user} 64 | fi 65 | done 66 | -------------------------------------------------------------------------------- /prepare/azure/azure-hostname-fix.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | sudo yum -y install screen 4 | 5 | ip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) 6 | echo "${ip} $(hostname -s).cloudapp.net $(hostname -s)" | sudo tee -a /etc/hosts 7 | 8 | el_version=$(sed 's/^.\+ release \([.0-9]\+\).*/\1/' /etc/redhat-release | cut -d. -f1) 9 | case ${el_version} in 10 | "6") 11 | echo "HOSTNAME=$(hostname -s).cloudapp.net" | sudo tee -a /etc/sysconfig/network 12 | sudo hostname $(hostname -s).cloudapp.net 13 | screen -dmS restart-networking bash 14 | screen -S restart-networking -X stuff "sleep 2; sudo service network restart; exit\n" 15 | ;; 16 | "7") 17 | sudo hostnamectl set-hostname $(hostname -s).cloudapp.net 18 | sudo systemctl restart systemd-hostnamed 19 | ;; 20 | esac 21 | 22 | exit 23 | -------------------------------------------------------------------------------- /prepare/google/deploy-ad-lab.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export PDSH_SSH_ARGS_APPEND="-l student -i ${HOME}/.ssh/student.pri.key -o ConnectTimeout=5 -o CheckHostIP=no -o StrictHostKeyChecking=no -o RequestTTY=force" 4 | export domain="europe-west1-b.siq-haas" 5 | export lab_prefix=p-test 6 | export labs=$(echo {005..005}) 7 | 8 | type="hdp" hosts_hdp=$(for i in ${labs}; do printf "${lab_prefix}${i}-${type}.${domain},"; done) 9 | export hosts_all=${hosts_hdp} 10 | 11 | echo "Creating these hosts:" 12 | for host in ${hosts_all}; do printf " %s\n" "${host}"; done 13 | read -p "Continue?" 14 | 15 | cd ~/src/masterclass/prepare/google 16 | for i in $labs; do lab=$i ./create-lab.sh & sleep 5; done 17 | sleep 60 18 | gcloud compute config-ssh 19 | 20 | command="uptime" 21 | pdsh -w ${hosts_all} "${command}" 22 | read -p "Press [Enter] key to continue" 23 | command="uptime" 24 | pdsh -w ${hosts_all} "${command}" 25 | read -p "Press [Enter] key to continue" 26 | 27 | #### 4. all hosts: set passwords, install shellinaboxd, grow root partition & reboot 28 | command="curl https://raw.githubusercontent.com/seanorama/masterclass/master/prepare/google/scripts/deploy-all.sh | bash \ 29 | ; sudo /opt/ambari-bootstrap/providers/growroot.sh; sudo reboot" 30 | pdsh -w ${hosts_all} "${command}" 31 | sleep 60 32 | 33 | command="uptime" 34 | pdsh -w ${hosts_all} "${command}" 35 | read -p "Press [Enter] key to continue" 36 | command="uptime" 37 | pdsh -w ${hosts_all} "${command}" 38 | read -p "Press [Enter] key to continue" 39 | 40 | 41 | #### 5) deploy HDP (a few minutes after above to account for the reboot) 42 | command="curl https://raw.githubusercontent.com/seanorama/masterclass/master/prepare/google/scripts/deploy-hdp.sh | bash" 43 | time pdsh -w ${hosts_hdp} "${command}" 44 | command="source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_change_pass admin admin BadPass#1" 45 | time pdsh -w ${hosts_hdp} "${command}" 46 | #command="echo export ambari_pass=BadPass#1 > ~/.ambari.conf; chmod 600 ~/.ambari.conf" 47 | #time pdsh -w ${hosts_hdp} "${command}" 48 | for dest in $(echo ${hosts_hdp} | tr ',' ' '); do 49 | scp -o "RequestTTY=no" -i ~/.ssh/student.pri.key ~/src/masterclass/prepare/activedirectory/ambari.keytab student@${dest}:~/ 50 | done 51 | 52 | sleep 600 53 | command='source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_get_cluster; ${ambari_curl}/clusters/${ambari_cluster}/requests/1 | grep request_status' 54 | time pdsh -w ${hosts_hdp} "${command}" 55 | read -p "Press [Enter] key to continue" 56 | command='source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_get_cluster; ${ambari_curl}/clusters/${ambari_cluster}/requests/1 | grep request_status' 57 | time pdsh -w ${hosts_hdp} "${command}" 58 | read -p "Press [Enter] key to continue" 59 | command='source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_get_cluster; ${ambari_curl}/clusters/${ambari_cluster}/requests/1 | grep request_status' 60 | time pdsh -w ${hosts_hdp} "${command}" 61 | read -p "Press [Enter] key to continue" 62 | command='source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_get_cluster; ${ambari_curl}/clusters/${ambari_cluster}/requests/1 | grep request_status' 63 | time pdsh -w ${hosts_hdp} "${command}" 64 | read -p "Press [Enter] key to continue" 65 | 66 | 67 | command="sudo chkconfig mysqld on; sudo service mysqld start" 68 | time pdsh -w ${hosts_hdp} "${command}" 69 | command="~/ambari-bootstrap/extras/add-trusted-ca.sh" 70 | time pdsh -w ${hosts_all} "${command}" 71 | command="~/ambari-bootstrap/extras/samples/sample-data.sh; ~/ambari-bootstrap/extras/configs/proxyusers.sh" 72 | time pdsh -w ${hosts_hdp} "${command}" 73 | 74 | 75 | exit 76 | 77 | -------------------------------------------------------------------------------- /prepare/google/prepare-account.sh: -------------------------------------------------------------------------------- 1 | 2 | ## Setup 3 | 4 | #### Set zone & project 5 | gcloud config set project siq-haas 6 | gcloud config set compute/zone europe-west1-b 7 | 8 | #### Create network 9 | gcloud compute --project "siq-haas" networks create "hdp-partner-workshop" --range "10.240.0.0/16" 10 | 11 | gcloud preview --project "siq-haas" instance-groups --zone "europe-west1-b" \ 12 | create "hdp-partner-workshop" --network "hdp-partner-workshop" 13 | 14 | 15 | -------------------------------------------------------------------------------- /prepare/google/scripts/deploy-hdp-ipa.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | pass=hortonworks 4 | 5 | sudo yum makecache 6 | sudo yum -y install ipa-client openldap-clients patch 7 | sudo service ntpd restart 8 | 9 | curl -sSL -O https://raw.githubusercontent.com/seanorama/masterclass/master/prepare/google/scripts/ipautil.patch 10 | sudo patch -f -b /usr/lib/python2.6/site-packages/ipapython/ipautil.py < ipautil.patch 11 | 12 | echo ${pass} | sudo ipa-client-install -U --domain=hortonworks.com \ 13 | --server="$(hostname -s|sed 's/-hdp/-ipa/').$(hostname -d)" \ 14 | -p admin@HORTONWORKS.COM --mkhomedir --fixed-primary -N -W \ 15 | --hostname=$(hostname -f) 16 | 17 | echo ${pass} | kinit admin 18 | echo ${pass} | sudo kinit admin 19 | -------------------------------------------------------------------------------- /prepare/google/scripts/deploy-ipa.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #exec 3>&1 4>&2 4 | #trap 'exec 2>&4 1>&3' 0 1 2 3 5 | #exec 1>prep-hdp.out 2>&1 6 | 7 | pass="hortonworks" 8 | 9 | sudo yum makecache 10 | sudo yum -y install *ipa-server bind bind-dyndb-ldap epel-release ntpd screen patch 11 | sudo yum -y install haveged 12 | sudo service haveged start; sudo chkconfig haveged on 13 | sudo service ntpd restart 14 | 15 | curl -sSL -O https://raw.githubusercontent.com/seanorama/masterclass/master/prepare/google/scripts/ipautil.patch 16 | sudo patch -b /usr/lib/python2.7/site-packages/ipapython/ipautil.py < ipautil.patch 17 | 18 | ip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) 19 | 20 | sudo ipa-server-install --domain=hortonworks.com \ 21 | --realm=HORTONWORKS.COM --ds-password=${pass} \ 22 | --master-password=${pass} --admin-password=${pass} \ 23 | --zonemgr 'sroberts+workshop@hortonworks.com' \ 24 | --hostname=$(hostname -f) --ip-address=${ip} \ 25 | --unattended --mkhomedir 26 | #--setup-dns --forwarder=8.8.8.8 \ 27 | 28 | echo ${pass} | kinit admin 29 | echo ${pass} | sudo kinit admin 30 | -------------------------------------------------------------------------------- /prepare/google/scripts/ipautil.patch: -------------------------------------------------------------------------------- 1 | --- /usr/lib/python2.6/site-packages/ipapython/ipautil.py 2014-10-16 15:02:34.000000000 +0000 2 | +++ /home/sroberts/ipautil.py 2015-07-13 13:55:15.247400058 +0000 3 | @@ -163,11 +163,6 @@ 4 | elif addr.version == 6: 5 | net = netaddr.IPNetwork(str(addr) + '/64') 6 | 7 | - if not allow_network and addr == net.network: 8 | - raise ValueError("cannot use IP network address") 9 | - if not allow_broadcast and addr.version == 4 and addr == net.broadcast: 10 | - raise ValueError("cannot use broadcast IP address") 11 | - 12 | super(CheckedIPAddress, self).__init__(addr, flags=self.netaddr_ip_flags) 13 | self.prefixlen = net.prefixlen 14 | self.defaultnet = defnet 15 | -------------------------------------------------------------------------------- /ranger-atlas/Data/claim_savings.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/claim_savings.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/Data/claims_provider_summary_data.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/claims_provider_summary_data.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/Data/eu_countries.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/eu_countries.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/Data/pos_txt.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/pos_txt.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/Data/tax_2015.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/tax_2015.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/Data/us_customers_data.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/us_customers_data.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/Data/ww_customers_data.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/Data/ww_customers_data.csv.bz2 -------------------------------------------------------------------------------- /ranger-atlas/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | ## Before you start 5 | 6 | See ../README.md for instructions on using the cluster management scripts. 7 | 8 | ### a) Deploy cluster(s) 9 | 10 | 1. Get the repo and switch to the 'generic' directory 11 | 12 | ``` 13 | git clone https://github.com/seanorama/masterclass 14 | cd masterclass/generic 15 | ``` 16 | 17 | 2. Set these variables, updating the values as appropriate: 18 | 19 | ```sh 20 | export AWS_DEFAULT_REGION=eu-central-1 ## region to deploy in 21 | export lab_prefix="${USER}" ## template for naming the cloudformation stacks 22 | export lab_first=100 ## number to start at in naming 23 | export lab_count=1 ## number of clusters to create 24 | 25 | export cfn_parameters=' 26 | [ 27 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 28 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-025cae79"}, 29 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-d6c31dbd"}, 30 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"0"}, 31 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER AMBARI_INFRA AMBARI_METRICS SQOOP TEZ ZEPPELIN SLIDER SPARK RANGER ATLAS KAFKA"}, 32 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"0"}, 33 | {"ParameterKey":"DeployCluster","ParameterValue":"true"}, 34 | {"ParameterKey":"AmbariVersion","ParameterValue":"2.5.0.3"}, 35 | {"ParameterKey":"HDPStack","ParameterValue":"2.6"}, 36 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/ranger-atlas/setup.sh | bash"}, 37 | {"ParameterKey":"InstanceType","ParameterValue":"m4.2xlarge"}, 38 | {"ParameterKey":"BootDiskSize","ParameterValue":"80"} 39 | ] 40 | ' 41 | ``` 42 | 43 | 3. Create the cluster(s): `../bin/clusters-create.sh` 44 | 45 | 4. List cluster host(s): `../bin/clusters-report.sh` 46 | 47 | 5. Terminate cluster(s): `../bin/clusters-terminate.sh` 48 | 49 | ## REMEMBER to terminate the clusters immediately after the class is over, or be prepared to pay $$$! 50 | 51 | Further, you should verify deletion of the CloudFormations & EC2 instances from the AWS Console. 52 | 53 | ## Extra commands needed 54 | 55 | These additional steps are needed once the clusters are deployed. Use a cluster shell to execute. 56 | 57 | ``` 58 | ## this will give you a command to open tmux-cssh to all hosts: 59 | echo ${hosts} | xargs echo tmux-cssh -sa \"-o StrictHostKeyChecking=no\" -u centos blah 60 | ``` 61 | 62 | ``` 63 | ad_host="ad01.lab.hortonworks.net" 64 | ad_root="ou=CorpUsers,dc=lab,dc=hortonworks,dc=net" 65 | ad_user="cn=ldap-reader,ou=ServiceUsers,dc=lab,dc=hortonworks,dc=net" 66 | 67 | sudo ambari-server setup-ldap \ 68 | --ldap-url=${ad_host}:389 \ 69 | --ldap-secondary-url= \ 70 | --ldap-ssl=false \ 71 | --ldap-base-dn=${ad_root} \ 72 | --ldap-manager-dn=${ad_user} \ 73 | --ldap-bind-anonym=false \ 74 | --ldap-dn=distinguishedName \ 75 | --ldap-member-attr=member \ 76 | --ldap-group-attr=cn \ 77 | --ldap-group-class=group \ 78 | --ldap-user-class=user \ 79 | --ldap-user-attr=sAMAccountName \ 80 | --ldap-save-settings \ 81 | --ldap-bind-anonym=false \ 82 | --ldap-sync-username-collisions-behavior=convert \ 83 | --ldap-referral= 84 | 85 | sudo ambari-server restart 86 | echo hadoop-users,hr,sales,legal,hadoop-admins,compliance,analyst,eu_employees,us_employees > groups.txt 87 | sudo ambari-server sync-ldap --groups groups.txt 88 | 89 | 90 | ## set role permissions in ambari 91 | ambari_pass=BadPass#1 92 | source ~/ambari-bootstrap/extras/ambari_functions.sh 93 | ambari_get_cluster 94 | : ${cluster_name:=${ambari_cluster}} 95 | read -r -d '' body < Tag Based Policies 28 | - Click the + icon and create a service named 'tags' 29 | - ![](./media/screenshot-ranger-add-tag-service.png) 30 | 31 | - [ ] Configure Hive for Tag based Policies 32 | - Open Ranger 33 | - Click Access Manager -> Resources Based Policies 34 | - Click ‘edit/pen’ icon next to the service’ 35 | - [ ] Set ‘Select Tag Service’ to ‘tags’ 36 | - ![](./media/screenshot-ranger-configure-hive-tag-service.png) 37 | 38 | ## Lab 04: Ranger: Allow 'compliance' group to see expired data 39 | 40 | - [ ] Update 'EXPIRES_ON' tag based policy 41 | - Open Ranger 42 | - Click Access Manager -> Tag Based Policies 43 | - Open the 'tags' service 44 | - Edit the 'EXPIRES_ON' policy 45 | - Notice the condition which denies access to 'public' 46 | - Add these conditions: 47 | - 1. 'Allow Conditions': Give the group 'compliance' access to component 'hive' 48 | - ![](./media/screenshot-ranger-add-tag-condition1.png) 49 | - 2. 'Exclude from Deny Conditions': Give the group 'compliance' access to component 'hive' 50 | - ![](./media/screenshot-ranger-add-tag-condition1.png) 51 | 52 | ## Lab 05: Atlas: Create 'EXPIRES_ON' tag in Atlas 53 | 54 | - [ ] Create tag in Atlas 55 | - Open Atlas 56 | - Click ‘Tags’ -> ‘Create Tag’ 57 | - Name: "EXPIRES_ON" 58 | - Attributes: "expiry_date" as type "date" 59 | - ![](./media/screenshot-atlas-create-tag-expireson.png) 60 | 61 | ## Lab 06: Atlas: Add 'EXPIRES_ON' tag to table 'tax_2015' 62 | 63 | 1. [ ] Find the 'tax_2015' table 64 | - Open Atlas 65 | - Click ‘Search’: 66 | - Type: hive_table 67 | - Query: tax_2015 68 | - ![](media/screenshot-atlas-tax2015-search.png) 69 | 2. [ ] Add tag to table 70 | - Click blue "+" sign next to 'tax_2015' 71 | - Tag: EXPIRES_ON 72 | - expiry_date: 2016-12-31T00:00:00.000Z 73 | - ![](media/screenshot-atlas-tax2015-tag.png) 74 | 75 | ## Lab 07: Test policies from Ambari 76 | 77 | For each task below: 78 | - Login as the user specified 79 | - Execute the queries listed. 80 | 81 | ### Login as ivana-eu-hr 82 | - [ ] Execute this query which shows EU country row filtering 83 | ``` 84 | SELECT surname, streetaddress, country, countryfull, age, password, nationalid, ccnumber, mrn, birthday 85 | FROM hortoniabank.ww_customers LIMIT 50; 86 | ``` 87 | 88 | ### Login as joe-analyst 89 | - [ ] Execute this query which shows data masking 90 | ``` 91 | SELECT surname, streetaddress, age, password, nationalid, ccnumber, mrn, birthday 92 | FROM hortoniabank.us_customers LIMIT 20; 93 | ``` 94 | - [ ] Execute this query 95 | ``` 96 | select gender, title, givenname, streetaddress, city, state, country from hortoniabank.ww_customers 97 | limit 100; 98 | ``` 99 | - [ ] Execute this query to test denied row combinations 100 | ``` 101 | select givenname, surname, emailaddress, zipcode, mrn, bloodtype 102 | from hortoniabank.ww_customers 103 | limit 50; 104 | ``` 105 | - [ ] Execute this query to confirm access denied for tax_2015 106 | ``` 107 | select * from finance.tax_2015; 108 | ``` 109 | 110 | ### Login as kate-hr 111 | - [ ] Execute this query to show no data masking 112 | ``` 113 | SELECT surname, streetaddress, age, password, nationalid, ccnumber, mrn, birthday 114 | FROM hortoniabank.us_customers LIMIT 20; 115 | ``` 116 | - [ ] Execute this query to test that you only get US data 117 | ``` 118 | select givenname, surname, emailaddress, zipcode, mrn, bloodtype 119 | from hortoniabank.ww_customers 120 | limit 50; 121 | ``` 122 | 123 | ### Login as compliance-admin 124 | - [ ] Execute this query to confirm access to expired data 125 | ``` 126 | select * from finance.tax_2015; 127 | ``` 128 | 129 | ## Lab 08: Review Ranger audits 130 | 131 | - [ ] Check Ranger audits 132 | - Open Ranger 133 | - Click Audit 134 | - Notice, in the far right column, the '[EXPIRES_ON]' tag policy being used 135 | - Review the other audit logs for row level filtering and row masking. 136 | 137 | ## Lab 09: Import Zeppelin Notebooks 138 | 139 | - [ ] Import Notebooks 140 | - Login to Zeppelin 141 | - Import each of these notebooks using URL 142 | - [ ] [HortoniaBank - compliance-admin Notebook](https://raw.githubusercontent.com/seanorama/masterclass/master/ranger-atlas/Notebooks/HortoniaBank%20-%20Compliance%20Admin%20View.json) 143 | - [ ] [HortoniaBank - ivana-eu-ur Notebook](https://raw.githubusercontent.com/seanorama/masterclass/master/ranger-atlas/Notebooks/HortoniaBank%20-%20Ivana%20EU%20HR.json) 144 | - [ ] [HortoniaBank - joe-analyst Notebook](https://raw.githubusercontent.com/seanorama/masterclass/master/ranger-atlas/Notebooks/HortoniaBank%20-%20Joe%20Analyst.json) 145 | 146 | - [ ] Or view Notebooks: https://www.zeppelinhub.com/viewer/ 147 | -------------------------------------------------------------------------------- /ranger-atlas/Scripts/create-secgovdemo-hortoniabank-tables.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | beeline -n hive -u "jdbc:hive2://localhost:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2" -f create-secgovdemo-hortoniabank-tables.ddl 3 | 4 | -------------------------------------------------------------------------------- /ranger-atlas/Scripts/create-secgovdemo-hortoniabank-userfolders.sh: -------------------------------------------------------------------------------- 1 | sudo sudo -u hdfs bash -c " 2 | # kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs-hortoniabank@FIELD.HORTONWORKS.COM 3 | hadoop fs -mkdir /user/admin 4 | hadoop fs -chown admin:hdfs /user/admin 5 | hadoop fs -mkdir /user/joe-analyst 6 | hadoop fs -chown joe-analyst:analyst /user/joe-analyst 7 | hadoop fs -mkdir /user/kate-hr 8 | hadoop fs -chown kate-hr:hr /user/kate-hr 9 | hadoop fs -mkdir /user/ivanna-eu-hr 10 | hadoop fs -chown ivanna-eu-hr:hr /user/ivanna-eu-hr 11 | hadoop fs -mkdir /user/compliance-admin 12 | hadoop fs -chown compliance-admin:compliance /user/compliance-admin 13 | hadoop fs -mkdir /user/hadoopadmin 14 | hadoop fs -chown hadoopadmin:hdfs /user/hadoopadmin 15 | " 16 | 17 | -------------------------------------------------------------------------------- /ranger-atlas/Scripts/load-secgovdemo-hortoniabank-files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #sudo su - 4 | #kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs-hortoniabank@FIELD.HORTONWORKS.COM 5 | 6 | cp -a ../Data /tmp 7 | bunzip2 /tmp/Data/*.bz2 8 | 9 | 10 | sudo sudo -u hdfs bash -c " 11 | hdfs dfs -mkdir -p /user/admin/hortoniabank_data/us_customers 12 | hdfs dfs -mkdir -p /user/admin/hortoniabank_data/ww_customers 13 | hdfs dfs -mkdir -p /user/admin/hortoniabank_data/eu_countries 14 | hdfs dfs -mkdir -p /user/admin/hortoniabank_data/tax_2015 15 | hdfs dfs -mkdir -p /user/admin/hortoniabank_data/claim_savings 16 | hdfs dfs -mkdir -p /user/admin/hortoniabank_data/provider_summary 17 | 18 | hdfs dfs -put /tmp/Data/us_customers_data.csv /user/admin/hortoniabank_data/us_customers 19 | hdfs dfs -put /tmp/Data/ww_customers_data.csv /user/admin/hortoniabank_data/ww_customers 20 | hdfs dfs -put /tmp/Data/eu_countries.csv /user/admin/hortoniabank_data/eu_countries 21 | hdfs dfs -put /tmp/Data/tax_2015.csv /user/admin/hortoniabank_data/tax_2015 22 | hdfs dfs -put /tmp/Data/claim_savings.csv /user/admin/hortoniabank_data/claim_savings 23 | hdfs dfs -put /tmp/Data/claims_provider_summary_data.csv /user/admin/hortoniabank_data/provider_summary 24 | hdfs dfs -chown -R admin:hadoop /user/admin/hortoniabank_data 25 | hdfs dfs -chmod -R g+wX /user/admin/hortoniabank_data 26 | " 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-atlas-create-tag-expireson.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-atlas-create-tag-expireson.png -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-atlas-tax2015-search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-atlas-tax2015-search.png -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-atlas-tax2015-tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-atlas-tax2015-tag.png -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-ranger-add-tag-condition1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-ranger-add-tag-condition1.png -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-ranger-add-tag-condition2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-ranger-add-tag-condition2.png -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-ranger-add-tag-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-ranger-add-tag-service.png -------------------------------------------------------------------------------- /ranger-atlas/media/screenshot-ranger-configure-hive-tag-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/ranger-atlas/media/screenshot-ranger-configure-hive-tag-service.png -------------------------------------------------------------------------------- /reference/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o xtrace 3 | 4 | ######################################################################## 5 | ######################################################################## 6 | ## variables 7 | 8 | export HOME=${HOME:-/root} 9 | export TERM=xterm 10 | : ${ambari_pass:="BadPass#1"} 11 | ambari_password="${ambari_pass}" 12 | : ${stack:="mycluster"} 13 | : ${cluster_name:=${stack}} 14 | : ${ambari_services:="HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER AMBARI_METRICS SLIDER AMBARI_INFRA TEZ KAFKA SPARK ZEPPELIN HBASE SMARTSENSE"} 15 | : ${install_ambari_server:=true} 16 | : ${ambari_stack_version:=2.6} 17 | : ${deploy:=true} 18 | : ${host_count:=skip} 19 | : ${recommendation_strategy:="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"} 20 | 21 | ## overrides 22 | #export ambari_repo=https://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.5.0.3/ambari.repo 23 | 24 | export install_ambari_server ambari_pass host_count ambari_services 25 | export ambari_password cluster_name recommendation_strategy 26 | export ambari_stack_version 27 | 28 | ######################################################################## 29 | ######################################################################## 30 | ## 31 | cd 32 | 33 | yum makecache 34 | yum -y -q install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 35 | yum clean cache 36 | yum -y -q install git ntpd screen mysql-connector-java jq python-argparse python-configobj ack postgresql-jdbc 37 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 38 | 39 | ######################################################################## 40 | ######################################################################## 41 | ~/ambari-bootstrap/extras/deploy/prep-hosts.sh 42 | ~/ambari-bootstrap/ambari-bootstrap.sh 43 | 44 | ## Ambari Server specific tasks 45 | if [ "${install_ambari_server}" = "true" ]; then 46 | 47 | ## bug workaround: 48 | sed -i.bak "s/\(^ total_sinks_count = \)0$/\11/" /var/lib/ambari-server/resources/stacks/HDP/2.0.6/services/stack_advisor.py 49 | bash -c "nohup ambari-server restart" || true 50 | 51 | sleep 60 52 | 53 | ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar 54 | ambari_pass=admin source ~/ambari-bootstrap/extras/ambari_functions.sh 55 | ambari_change_pass admin admin ${ambari_pass} 56 | sleep 1 57 | 58 | cd ~/ambari-bootstrap/deploy 59 | 60 | ## various configuration changes for demo environments, and fixes to defaults 61 | cat << EOF > configuration-custom.json 62 | { 63 | "configurations" : { 64 | "core-site": { 65 | "fs.trash.interval": "4320" 66 | }, 67 | "hdfs-site": { 68 | "dfs.namenode.safemode.threshold-pct": "0.99" 69 | }, 70 | "hive-site": { 71 | "hive.server2.enable.doAs" : "true", 72 | "hive.server2.transport.mode": "http", 73 | "hive.exec.compress.output": "true", 74 | "hive.merge.mapfiles": "true", 75 | "hive.server2.tez.initialize.default.sessions": "true" 76 | }, 77 | "mapred-site": { 78 | "mapreduce.job.reduce.slowstart.completedmaps": "0.7", 79 | "mapreduce.map.output.compress": "true", 80 | "mapreduce.output.fileoutputformat.compress": "true" 81 | }, 82 | "yarn-site": { 83 | "yarn.acl.enable" : "true" 84 | }, 85 | "ams-site": { 86 | "timeline.metrics.cache.size": "100" 87 | } 88 | } 89 | } 90 | EOF 91 | 92 | ./deploy-recommended-cluster.bash 93 | 94 | if [ "${deploy}" = "true" ]; then 95 | 96 | cd ~ 97 | sleep 5 98 | source ~/ambari-bootstrap/extras/ambari_functions.sh 99 | ambari_configs 100 | ambari_wait_request_complete 1 101 | cd ~ 102 | sleep 10 103 | 104 | usermod -a -G users ${USER} 105 | usermod -a -G users admin 106 | echo "${ambari_pass}" | passwd admin --stdin 107 | sudo -u hdfs bash -c " 108 | hadoop fs -mkdir /user/admin; 109 | hadoop fs -chown admin /user/admin; 110 | hdfs dfsadmin -refreshUserToGroupsMappings" 111 | 112 | UID_MIN=$(awk '$1=="UID_MIN" {print $2}' /etc/login.defs) 113 | users="$(getent passwd|awk -v UID_MIN="${UID_MIN}" -F: '$3>=UID_MIN{print $1}')" 114 | for user in ${users}; do usermod -a -G users ${user}; done 115 | for user in ${users}; do usermod -a -G hadoop-users ${user}; done 116 | ~/ambari-bootstrap/extras/onboarding.sh 117 | fi 118 | fi 119 | 120 | -------------------------------------------------------------------------------- /security-advanced/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/.DS_Store -------------------------------------------------------------------------------- /security-advanced/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | ## Pre-reqs 5 | ----------- 6 | 7 | - AWS CLI installed: http://docs.aws.amazon.com/cli/latest/userguide/installing.html 8 | - Configure AWS CLI: `aws configure` 9 | - Details on creating credentials: http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-set-up.html 10 | - These AWS IAM permissions are needed: EC2, AutoScaling, CloudFormation, Marketplace 11 | 12 | ## How this works 13 | ----------------- 14 | 15 | There are several scripts used to make this work: 16 | 17 | For AWS CloudFormation: 18 | 19 | - cloudformation-generate-template.py: generates the AWS Cloudformation template 20 | - cloudformation.json: the CloudFormation template which deploys a cluster 21 | 22 | For managing multiple clusters (each takes variables for naming & the number of clusters): 23 | 24 | - ../bin/clusters-create.sh: calls cloudformation to create clusters 25 | - ../bin/clusters-report.sh: list cluster details 26 | - ../bin/clusters-terminate.sh: calls cloudformation to terminate clusters 27 | - ../bin/cloudformation-status.sh: list all cloudformation stacks in the region 28 | 29 | ## Deploy, report & terminate clusters on AWS 30 | ------------------------- 31 | 32 | 1. Get this repo: 33 | 34 | ``` 35 | git clone https://github.com/seanorama/masterclass 36 | cd masterclass/generic 37 | ``` 38 | 39 | 1. Check for conflicting/existing stacks (same name as what you plan to deploy): 40 | - Open the CloudFormat Web UI 41 | - Or with the command-line: `../bin/cloudformation-status.sh` 42 | 43 | 1. Open a 'screen' so the commands continue if you lose connectivity: 44 | 45 | ``` 46 | screen 47 | 48 | ## Note: If you get disconnected, SSH back to the host and execute: `screen -x` 49 | ``` 50 | 51 | 52 | 1. Set variables to define the naming & number of clusters to deploy: 53 | - the following will deploy 2 clusters with name sec100,sec101 54 | - update 'lab_count' to the number of clusters you want 55 | 56 | ```sh 57 | export AWS_DEFAULT_REGION=us-west-2 ## region to deploy in 58 | export lab_prefix=${USER} ## template for naming the cloudformation stacks 59 | export lab_first=100 ## number to start at in naming 60 | export lab_count=1 ## number of clusters to create 61 | ``` 62 | 63 | 2. Set parameters which are passed to CloudFormation: 64 | - KeyName: The key (added on the EC2 page) to access the cluster. 65 | - AmbariServices: Which HDP services to deploy. 66 | - AdditionalInstanceCount: How many additional nodes to deploy. (Setting to 2 will deploy 3 nodes total) 67 | - SubnetId & SecurityGroups: This CloudFormation deploys in an existing Subnet & Security Group. **You must update this to your environment**. 68 | 69 | ```sh 70 | ## Update with your keypair name, subnet, securitygroups and the number of instances you want 71 | export cfn_parameters=' 72 | [ 73 | {"ParameterKey":"KeyName","ParameterValue":"training-keypair"}, 74 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-02edac67"}, 75 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-a02d17c4"}, 76 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG YARN ZOOKEEPER AMBARI_METRICS SLIDER AMBARI_INFRA LOGSEARCH TEZ KNOX"}, 77 | {"ParameterKey":"DeployCluster","ParameterValue":"true"}, 78 | {"ParameterKey":"AmbariVersion","ParameterValue":"2.4.1.0"}, 79 | {"ParameterKey":"HDPStack","ParameterValue":"2.5"}, 80 | {"ParameterKey":"BootDiskSize","ParameterValue":"100"}, 81 | {"ParameterKey":"InstanceType","ParameterValue":"m4.xlarge"}, 82 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"3"}, 83 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/security-advanced/setup.sh | bash"} 84 | ] 85 | ' 86 | ``` 87 | 88 | 1. Provision your clusters 89 | 90 | ``` 91 | ../bin/clusters-create.sh 92 | ``` 93 | 94 | 1. Check the build status 95 | - From the CloudFormation Web UI 96 | - Or from the command-line: 97 | 98 | ``` 99 | ../bin/cloudformation-status.sh 100 | ``` 101 | 102 | 1. Once your clusters are ready, get list of clusters nodes for providing to students: 103 | 104 | ``` 105 | ..bin/clusters-report.sh 106 | ``` 107 | 108 | 1. Use the clusters: 109 | - `ssh centos@ipFromReportAbove` ## use the key which was specified during the build 110 | 111 | 1. Terminate clusters 112 | 113 | ``` 114 | ..bin/clusters-terminate.sh 115 | ``` 116 | 117 | 1. Verify that all clusters are terminated 118 | - From the AWS CloudFormation Web UI 119 | - Or from the CLI 120 | 121 | ``` 122 | ../bin/cloudformation-status.sh 123 | ``` 124 | 125 | ######## 126 | 127 | ## Running sessions 128 | 129 | It's recommended to use an "etherpad" to share: 130 | 131 | - the cluster details (from above) 132 | - instructions to students 133 | 134 | You can create your own, or use a hosted version such as TitanPad. You should create this account in advance. 135 | 136 | ######## 137 | 138 | ## Issues: Deployment 139 | 140 | #### Creation 141 | 142 | - Some instances will fail their creation and time out, being rolled back, this is a nature of deploying large volumes of instances 143 | - Those that fail should simply be manually deleted from the cloudformations web ui 144 | 145 | #### Deleting cloudformations 146 | 147 | - Occasionally cloudformations will fail to delete due to timing issues, in which case, it’s probably the VPC or InternetGateway, just switch to the VPC service window within the AWS site, delete the specific VPC that is being complained about in the cloudformation and then once the cloudformation delete has failed, retry the delete, deletion should complete this time. 148 | - Once you’ve done the VPC deletion you can also do an AWS CLI call instead: 149 | - `aws cloudformation delete-stack --stack-name ` 150 | 151 | #### AWS Website 152 | 153 | If you suddenly notice that your instances/cloudformations/etc have vanished from the AWS control panel, you may have to re-login. 154 | 155 | 156 | 157 | ######## 158 | 159 | ## Issues: Other 160 | 161 | #### Run commands in bulk on all nodes 162 | 163 | * There are several options, such as pdsh, cssh, ... 164 | 165 | * Example using cssh, csshX or tmux-cssh (you'll need to install it) 166 | 167 | ``` 168 | ../bin/clusters-report.sh | grep "^[0-9]" | xargs echo tmux-cssh -u masterclass 169 | ``` 170 | 171 | * After executing you will get a terminal with small windows to all of the clusters. 172 | * Anything you type will go to all hosts. 173 | 174 | #### Venue Internet blocks Ambari Server (port 8080) 175 | 176 | * Change Ambari to port 8081 177 | 178 | ``` 179 | export TERM=xterm 180 | echo "client.api.port=8081" | sudo tee -a /etc/ambari-server/conf/ambari.properties 181 | sudo ambari-server restart 182 | sudo ambari-agent restart 183 | ``` 184 | -------------------------------------------------------------------------------- /security-advanced/extras/ambari.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/extras/ambari.keytab -------------------------------------------------------------------------------- /security-advanced/extras/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFwTCCA6mgAwIBAgIJALpre8sh6EgSMA0GCSqGSIb3DQEBBQUAMEkxHDAaBgNV 3 | BAMTE2xhYi5ob3J0b253b3Jrcy5uZXQxHDAaBgNVBAoTE0hvcnRvbndvcmtzIFRl 4 | c3RpbmcxCzAJBgNVBAYTAlVTMB4XDTE1MTIxODEzNTAwN1oXDTI1MTIxNTEzNTAw 5 | N1owSTEcMBoGA1UEAxMTbGFiLmhvcnRvbndvcmtzLm5ldDEcMBoGA1UEChMTSG9y 6 | dG9ud29ya3MgVGVzdGluZzELMAkGA1UEBhMCVVMwggIiMA0GCSqGSIb3DQEBAQUA 7 | A4ICDwAwggIKAoICAQC8rk9q+WC4Nrk+JoGkusE0ynEC/VOIES8eXbaaczhac59m 8 | BxL6Uj1zC5ITf4qx2Nu+71JO3YcOYblmSw/kvs6Ijv+UcGKAmSHryUPkhRQdn5X/ 9 | bXBfU/A8ANYke089bv7MUSdBwvew7kq8+uUx0lHSiM1P4x8nR4k5E8RvCD8JP+iK 10 | nc6hCgHoYiQkr4DWzrvvjvo13UbWPVNuPoE+saQqFEr+wQxmyUuaZ80TJMYOqtSx 11 | XFpqHVh2PKTW+AX4PLOqJU+NUZY6IomtXrK0fbR1dCjCwoTQMzROeNPFtsZkD7RY 12 | /vgKx5bURV25chwzCcj9MhH4WFA3AOxiUlHnWYQWFXy2Pw4LbgHmcC7Vt/EvN+zW 13 | f/wI45aJpt679OBYjFObj+Fr/e59Ypmx8gGu3x9hl6SKDfA55oGcAIuwYrUHML/1 14 | wyFjTIhRxHBNEXS3D92w2yCmqbjD5PIZ8GrbNTBkizzuMWnLZip9KOz94lFXiWdm 15 | D43H3UHah6ki7GasA3bbWJzdXkWkCoele+ju0SA6/jTAZ6nCygmNjko+Hf0mjHb2 16 | TmlLDlmB7/pGXRyY/AqZy2qxKkl9lL7X0IJpyyrsznRjjxgd2QjscKEiCUPR9vcz 17 | g7HlrqEJKeQZgru0RlM8q3BrHkRnC0Jxu6kicd1X1HjfzwoQc4Zk7IX1JJ++8wID 18 | AQABo4GrMIGoMB0GA1UdDgQWBBTpDScMxcw4hv9b7JimEJ/hbhStRTB5BgNVHSME 19 | cjBwgBTpDScMxcw4hv9b7JimEJ/hbhStRaFNpEswSTEcMBoGA1UEAxMTbGFiLmhv 20 | cnRvbndvcmtzLm5ldDEcMBoGA1UEChMTSG9ydG9ud29ya3MgVGVzdGluZzELMAkG 21 | A1UEBhMCVVOCCQC6a3vLIehIEjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUA 22 | A4ICAQBLeVhtXYJ45Jx35RKZ/BH6WGvMObLX/K6Bx+St3jz+MoVPdKeiynmtCWJE 23 | vXPvhz6zLCBdRY65/ih8eaCFto0OphmIAhNShxErfcMxDoYrPMAfDKqtNuMgZJOW 24 | JKagBDlodw/i9ZUSWOtGtd1iGI+Th3f7zKqYNSUgf22bBLJL0Mw45sYghl70CvXP 25 | +jYI+b3M0P2YHhRNTqH74vBfMWhifl7ri8TC1OeD2p6MvuxQPR+5Nu+Qr7jW5EE1 26 | uZE01l7zRKsabQcjyel14NoL1Z0WMe68kxYSeNK9j01eMJeu9sedjxraNgNtoO+a 27 | A3lbeurqo8AiVzfwp/aPCBO7J/n/Jilk1jSynj/PbXMZyhsdhGUqIsI4YLY/aATY 28 | lqsZYz2OZ9QYBB7d55LYKil8p38YDv99BAHY2AsQq+9fWNylyp9KiDsz6b/sCLXk 29 | wo/MqsLwByNUFL4ea0WTNcEnFQV2F8xoxCM0t3jBUGlXDWifPYnEWDPFBbMfJXsu 30 | SvyYQ/gvAAGdLqCG4qz5yxLuNxzK9qBARNjQHPVr0CueyPVCamSjL7YEfT5ym9f1 31 | SGG41GpOGQODBeKi7d14KRwp3TfNaFSkU9Z0uFr8BLW9PvIsVnmRJRf7Q1SW7Y6j 32 | jcEK9de4PzvRHcfTwS+5tGxQ+k3hEil8JRdVXXbi5UjC+Qj4fw== 33 | -----END CERTIFICATE----- 34 | -------------------------------------------------------------------------------- /security-advanced/extras/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKQIBAAKCAgEAvK5PavlguDa5PiaBpLrBNMpxAv1TiBEvHl22mnM4WnOfZgcS 3 | +lI9cwuSE3+Ksdjbvu9STt2HDmG5ZksP5L7OiI7/lHBigJkh68lD5IUUHZ+V/21w 4 | X1PwPADWJHtPPW7+zFEnQcL3sO5KvPrlMdJR0ojNT+MfJ0eJORPEbwg/CT/oip3O 5 | oQoB6GIkJK+A1s677476Nd1G1j1Tbj6BPrGkKhRK/sEMZslLmmfNEyTGDqrUsVxa 6 | ah1Ydjyk1vgF+DyzqiVPjVGWOiKJrV6ytH20dXQowsKE0DM0TnjTxbbGZA+0WP74 7 | CseW1EVduXIcMwnI/TIR+FhQNwDsYlJR51mEFhV8tj8OC24B5nAu1bfxLzfs1n/8 8 | COOWiabeu/TgWIxTm4/ha/3ufWKZsfIBrt8fYZekig3wOeaBnACLsGK1BzC/9cMh 9 | Y0yIUcRwTRF0tw/dsNsgpqm4w+TyGfBq2zUwZIs87jFpy2YqfSjs/eJRV4lnZg+N 10 | x91B2oepIuxmrAN221ic3V5FpAqHpXvo7tEgOv40wGepwsoJjY5KPh39Jox29k5p 11 | Sw5Zge/6Rl0cmPwKmctqsSpJfZS+19CCacsq7M50Y48YHdkI7HChIglD0fb3M4Ox 12 | 5a6hCSnkGYK7tEZTPKtwax5EZwtCcbupInHdV9R4388KEHOGZOyF9SSfvvMCAwEA 13 | AQKCAgADFVoo5mBlQkeus1RlC8RCh5hnMxqdW6KIfYGRbpaaK8+MDEuMGbH1YlgF 14 | x4zNsTyOFz0iGs6tNn2gJVPiKhgX9TZKB7gkX9xFDSn13xLgFQ6PeQtgE3AWmvDk 15 | 8bruxxSU5DKfCX+gvZQtGHCMauJ5bVDloFGPA8VDUIZiRiE5lnjHYyLQUOiNEqWC 16 | j/esCv7ZQF3oaXBR6fimEZFtI9wJ8AD1vtPkDiAVKuMYD/7ImJQXPeogV9BtRjpv 17 | ZOg8EUniWu+84HtzAcN8ONL0ziUX0kkkPaeL+3o8Wg8j6QO++N66oZgOHQri9ys7 18 | 75h0YRGOybWveEn/TB2Q0cWS4UyeRJaOdLKKjhi8F7WFStuS8RjvksmeZiiBBXfB 19 | h7G/TMaSS3ssDLNgv/lgaOw5P8Jc+YejdjdQXAvAtAC8yqTJJF+mbv5jv5miWGNS 20 | KQ1ECwLALKUC9p3jfJ1RMUZk/aEb3AN7kLVahG+ipmxcF55ODERPW2gqrAiJ5FvZ 21 | 6qJRGX5NY9kKY09RJhTZiiWCqSlaR1CNId4YmNGevwWCc7Fm0SBti6uM5bWdXDYt 22 | TjuZauy5KJAVwdaveAtsxfH2wDI+8FVW97qtlyUQfcRte8VANVZCHjTvDFlDK+Bi 23 | ZQBbS+7R7eC1tA+uqyHdxp32wplZeHg+ckdZYR4KIEH4KVpRUQKCAQEA3pHlkyAq 24 | C0GptSSknZPmDsNBkyfJyQYBonO03g4sLwjbTHkE8bL1HMUq0T93WKy+kDuJwEix 25 | Rm+l6F8s6rbuoVpepm6RB5JlbiCZLcafxMgwrDjgQnwszTuOZDDp+TlX3VwNEZd+ 26 | 595kWJfthjgPtXexFbSM8wbLzlju7V3n147nsbhmfgzk4YpF1jwI+k2oMYCUGBAr 27 | IELHhwu5NXEzeYdk9r2QpSyjeSFb3gX8J8MyzNwFzaF8UquuWse4KD+CWWRatDq8 28 | uyGLwPhJuynm1iBe2QsxRHtNMeYI+sbEwThz0cMYbtnom481E/kJiCxtxb9TMwhM 29 | kEbApjNT5ftk+QKCAQEA2QVWOmQjru6QSOBchAwQWbbtMWJskpSNSFsDZWUtbz/6 30 | VbOR3Ufap1Lh4y1Cw34sfwRfO4q2bs2otRxPB7//5u5cZWfT8e0XmJomDl1Gy0C4 31 | f+RIhhtXJtGZ/Ksts+8LdG4B01oDZpjN+9VMmaxc9vzxhJlj+umRBOhmE4jzEQnr 32 | 95TPtfvsevoeJ3TFpg60Gqr7sBzyg7gOx1zDPxwzfEn/wBqAgPSuOM94YbXjshBn 33 | 4dDNzOTiP7W+w7cuXwlDHjqCgRMVoEHRTxx5mi0LvBixqD+fUATc6+h8uA+iuPIu 34 | v7WM0vBqKcaex8erx/Y3Tp0f9yA0HqqrIq9F2876SwKCAQEAm0CkJnuyOEXmboAQ 35 | AAdR3Axhqpd/jyUMcNhLWvYB5hFEf9SPGVYe2qHCW1HtG0LsbsnJLgLG/f4VecEi 36 | INvYG19hVLEdo22AKVC96Qt7GRYXtZq6/EWILuUTT1o7a2iPnM6AIzxM/srHrnDT 37 | 8+MZOhkywpQ+Gaws8FRi3dq6RUfe4L5jaBvwcE82pdkc3hTbdSReqWZl3swWSGJq 38 | IBhFssk6KU9UHZPEuFMHtRjxqn7j6bi0p5q0kSpGpfZYxGKAt4BkMQIj8ljVFpdb 39 | Px5cchMEdlvzUvS7hUoUC3Ipfyq0x6ZSXcjNE+3SNYtrFb0AGYd7LsbkOWPRMF3v 40 | cSZqaQKCAQAwlB8/JtdN+xzVE1PiQwkN7AKgQVXoYdRWU/koo0NOWdXhhTarf8AU 41 | IFtsDN+a/w1ayVG/9Ebt+p8/Gz0dkP7pZIXONTWeO+JCSI8oO/zy1mP2TNVnH4uc 42 | puJ/SpoKYwH/Z/iEpOuJ9b52FsL0yz5sEybE9ckJi2ro3FStHsS6Fax6WRc9t0yy 43 | InbJgjwT9cqfBZMdUv1+Op8kRUF3SQrFUZMsH0KQmzS8hsR9qsdzPnTieWj36mwy 44 | sq4nBUQeBea+R36bAfifTNxbhxY5SsgLrMNn1AYHVW9WZ04zrVxRZfTM+Qiy1xoM 45 | i+dS9QEg2555vHZBTTYdfzqbovxSPbyNAoIBAQC0uB8vK6jRgXlNnZPE/yfy+jf3 46 | eAdiinVyl5AuWaw4igWNCYn0Xej7hEG3b4tLBSKMn256HutOG+sxwBe9MBtGawwd 47 | frF5AaRKYRy2teaHEsQlilR0r/SxoAfoJy2v57mKJiUu/u4puGk1z3fmplmZvim5 48 | Yvbofinoz34mjHjq9+AJuJ4lSZcl9th9KJDxHYdleo7Zc1OKheaWSWCEkXzF95sd 49 | w4vyqiM8ZYuHvjeVP/FowFnC191JDQZkszSFuNYi7C54YUAbQqclng6gLE9oH/tX 50 | J4ndx4kOVtEoBWEdK3KEgoYdoGT9CH0Z/z7gflpkiQ0PYxZa5Ea7tExkDazV 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /security-advanced/extras/ca.srl: -------------------------------------------------------------------------------- 1 | E3A462A7BB198138 2 | -------------------------------------------------------------------------------- /security-advanced/extras/wildcard-lab-hortonworks-net.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEEDCCAfgCCQDjpGKnuxmBODANBgkqhkiG9w0BAQUFADBJMRwwGgYDVQQDExNs 3 | YWIuaG9ydG9ud29ya3MubmV0MRwwGgYDVQQKExNIb3J0b253b3JrcyBUZXN0aW5n 4 | MQswCQYDVQQGEwJVUzAeFw0xNTEyMTgxMzUxNDZaFw0yNTEyMTUxMzUxNDZaMEsx 5 | HjAcBgNVBAMUFSoubGFiLmhvcnRvbndvcmtzLm5ldDEcMBoGA1UEChMTSG9ydG9u 6 | d29ya3MgVGVzdGluZzELMAkGA1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IB 7 | DwAwggEKAoIBAQDZlFBgCBvClv5Y9ykGZ8QEFw8y4ZWVFYxf8ALpO701Y04w2RGR 8 | nzAdmJmjrORrhXC88R2xGXRvhSeAO6N6z8GrYPm49LxLjOj4QiQagfR8W0oOuBZW 9 | iwqv6zTY/F2uGe4a+1UGxYHegNgvk0wzAoh3q2ucdhGk0PwaDuG61NzbhdEzKCy0 10 | 8H/rZCXwsG1Pb8C9kd21sPW5MH9kmK2DfDulIy4gMQ92yNcbiIjlT/0FFdRI+flp 11 | fhIrBQAJa1WNTIyGEwdI8dL467LQJ0uU/aCabKbhEXniEQgtjQZnBhmjnVNpgFKV 12 | GCcMSpsd+p5R6vLhyRtGP5Nbilw6EfjVtMy9AgMBAAEwDQYJKoZIhvcNAQEFBQAD 13 | ggIBABvDxfTuy7xi2JirBrpL2R6HGR4FBlPgTU09dieHNXqnaRpVILIkMDPr1T3l 14 | 3PkcxVLhMDVf7Yu6KfjK2f8Y5VUholNswEzxUUOg39fY0miNXmKrwbMrSCogI/Al 15 | UiZpvN1/udTDuioZmHEf6D6P7P4DGMglpWWB4iS64aOyih8aVg30yKxMseheLibu 16 | nX729fpK+9xaOXsNVq9dn0gmZ9x3f34DW8Kde1YjSpf5Tm1WHr/OgpvwZudSXjmI 17 | QhjhaCaCvBQAgQxI+oXYawHI3DKS7b8ph2yS24UgF6OiBa62m2NdTrFqnA8Wr1Mf 18 | Mzlv5IHe9sD3K3GO32x+3gNbWLZIl33XsABv3TAIM7r9JEgf1UBWZFj9NKaGuOvr 19 | Nqx3498L+TWRRBh2A8V8ExDkavAWvVlFzmQ5yeG4p7DkZH9OM6HVCmO2P00bgWFM 20 | MpV8OW0+LS0SGIIx1WVmG5fzsLDr9EtmmQ9kQzgZXyhKpc4ndUKZByPleDjaS3lr 21 | pUyEE4sYoUjslt/xPe8skVh5vyqPvL4fEP3jUotjyHg4k7MOtjoS25AIr8Mj/cEI 22 | w7pP2C9oQUTYa0mKh3vyu/nXoLSnyvJHsurkmCkb8Y91Ks0P51YANw/exLBKp6vO 23 | NSX4lDWHAtTpSkRIrof5ioVT1B8pvZROvpyBry3oLXbhwYRU 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /security-advanced/extras/wildcard-lab-hortonworks-net.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICkDCCAXgCAQAwSzEeMBwGA1UEAxQVKi5sYWIuaG9ydG9ud29ya3MubmV0MRww 3 | GgYDVQQKExNIb3J0b253b3JrcyBUZXN0aW5nMQswCQYDVQQGEwJVUzCCASIwDQYJ 4 | KoZIhvcNAQEBBQADggEPADCCAQoCggEBANmUUGAIG8KW/lj3KQZnxAQXDzLhlZUV 5 | jF/wAuk7vTVjTjDZEZGfMB2YmaOs5GuFcLzxHbEZdG+FJ4A7o3rPwatg+bj0vEuM 6 | 6PhCJBqB9HxbSg64FlaLCq/rNNj8Xa4Z7hr7VQbFgd6A2C+TTDMCiHera5x2EaTQ 7 | /BoO4brU3NuF0TMoLLTwf+tkJfCwbU9vwL2R3bWw9bkwf2SYrYN8O6UjLiAxD3bI 8 | 1xuIiOVP/QUV1Ej5+Wl+EisFAAlrVY1MjIYTB0jx0vjrstAnS5T9oJpspuEReeIR 9 | CC2NBmcGGaOdU2mAUpUYJwxKmx36nlHq8uHJG0Y/k1uKXDoR+NW0zL0CAwEAAaAA 10 | MA0GCSqGSIb3DQEBBQUAA4IBAQCXLwzSremUaS5CfrC+d2QxGzUs+/5pR4NrkErW 11 | 9V+xnS0eTkzhqPPjqpiLmYCkYme7yUrR5n4z9hEOA9R9hBUOeWVFeDa1tsCZwjGI 12 | g3kMuBVjwoGF7ON0NZaintVyhx3PNgbZf2SqPcY3dGPdPC5tkNtn1oPPNVlMi0tK 13 | lWF8kE4JtzgGThBxFLhldr9vnF8eApb85Ice3kNl9fUTDXcmZhDfX0aw/HJ6WfbW 14 | vxVecn14GdUKSR9mIKCZALuuc3rOxHNqAHp9sV+UT+X9EybX1w5OKow4AxyHUyHl 15 | piuBo5QOolZ51HaL9byCJGTjsU8A466FhJptpd2b0Bo/l0U9 16 | -----END CERTIFICATE REQUEST----- 17 | -------------------------------------------------------------------------------- /security-advanced/extras/wildcard-lab-hortonworks-net.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEA2ZRQYAgbwpb+WPcpBmfEBBcPMuGVlRWMX/AC6Tu9NWNOMNkR 3 | kZ8wHZiZo6zka4VwvPEdsRl0b4UngDujes/Bq2D5uPS8S4zo+EIkGoH0fFtKDrgW 4 | VosKr+s02PxdrhnuGvtVBsWB3oDYL5NMMwKId6trnHYRpND8Gg7hutTc24XRMygs 5 | tPB/62Ql8LBtT2/AvZHdtbD1uTB/ZJitg3w7pSMuIDEPdsjXG4iI5U/9BRXUSPn5 6 | aX4SKwUACWtVjUyMhhMHSPHS+Ouy0CdLlP2gmmym4RF54hEILY0GZwYZo51TaYBS 7 | lRgnDEqbHfqeUery4ckbRj+TW4pcOhH41bTMvQIDAQABAoIBAAqamdg0TPYM2dSI 8 | Jy2yvPDeFtWJ/PpJFJBQaDPpv2zsrBDhY2PPeE7Y8xPxByjeVgI/Vg0dinks6iHr 9 | xinkGjI6BbGn681azrfasVvBXaL56AATpLMPD7RHKEoy8nXbuakC+r/bvFqVuPOI 10 | oDwUMYZzikkT0ARSUxcKBqJOYX0pT2h1FXb0tJwsOAqiwBgP48kqlSlnLlUTFu1e 11 | GHgbvVWvijLEjbYiDfCj/W+Ct+AEWdh6H/HGqj94KB+r/4n99QVAm3ipyjmK2gzJ 12 | a227feVfFvI/w/HKRrs8+1290wv93iizC4UCjN4cUj9Dp2Ac17jAzOXrWJn8euBs 13 | VjmLj5kCgYEA8L5c8kGvUUs4ZvIrx/qUNfCUf7AwqAjBtYYK/kAn8UeqEDTGL759 14 | EoX6vQ/DVH92rgsPl5WJh4WX/3X9FM18ipNTyk2Sj9qkRc5QbvwnpbSLf1QG2f95 15 | h6Yaw+5x8piD/8UecZBD6NaR+fzt5Vdl9eN2z5/bE0FFaOVMD9ydoFMCgYEA514m 16 | 7S6h4M87mUFMzYB17H/bCWJJqYVbtm+0K3rGyiu1uEOuzY0ImJsrhp/WIAwYggcY 17 | WshdYFDqOsdbuKelxHTH1iCe/Znx26S/R77p4xqzH2250UsXDzxJ0MKKyrmXyrMx 18 | nGfjpESxtTMr7g6KIZMJ3htkdWp+RNoLv8kEfK8CgYAgxEgyUO5jyjErEdsTpO93 19 | R/9uQbTdqhNzeYMVh/XnyTFEc2pmJS8wzjuKcBLN9WtZ1PeSUjAd2fkONCj2I+HH 20 | mrXU2yoPmlqxhZs8/+UFkMVL10v3b+ZCqEEMWe/4bPnKXjH+kvwrPMkxlVNr27Fs 21 | /kC1kbRnsoKX/F2V6xkpMQKBgAMpxl53xL7OFoKB4grtNq+FD/mQ3nDy4c4cuNnf 22 | U9hvwIxEYJ9uzi9H79BnXwYRJESi1DJpit1GPe2PzDE6GYoWDqVezM2qzSOznOne 23 | MlRoiQpk9aVm4WnT6S5xenRLFUgwKv14biRQzB0uFeurRzguSnQ+66CvVAqc8Iwu 24 | s2EBAoGAfMPlmbRVYfT91/NAUqTTioafCFBc3Iw1u0a6XCFhgUOWl2/QN7ZFIpEO 25 | cLvY8zvC0c4Y6acGP+zZoCJPDfXD4yvfGgKpSwdP2AdBkJZHPqnHC9Vee96w+4bl 26 | /HNT05WjFa18hykS10Y8v8Y+b4sriMLEPWSIaVaQWv6zeSERWNE= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /security-advanced/extras/wildcard-lab-hortonworks-net.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/extras/wildcard-lab-hortonworks-net.p12 -------------------------------------------------------------------------------- /security-advanced/screenshots/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/.DS_Store -------------------------------------------------------------------------------- /security-advanced/screenshots/AD-corpusers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/AD-corpusers.png -------------------------------------------------------------------------------- /security-advanced/screenshots/AD-hadoopnodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/AD-hadoopnodes.png -------------------------------------------------------------------------------- /security-advanced/screenshots/AD-hadoopservices.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/AD-hadoopservices.png -------------------------------------------------------------------------------- /security-advanced/screenshots/AD-serviceusers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/AD-serviceusers.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-KMS-enhancedconfig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-KMS-enhancedconfig1.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-KMS-enhancedconfig2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-KMS-enhancedconfig2.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-KMS-enhancedconfig3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-KMS-enhancedconfig3.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-KMS-proxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-KMS-proxy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-Knox-install.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-Knox-install.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-RM-public-host.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-RM-public-host.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-YARN-internal-host.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-YARN-internal-host.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-add-KMS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-add-KMS.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-configureidentities.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-configureidentities.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-host-add-tez.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-host-add-tez.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-1.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-2.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-3.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-4.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-5.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-6.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-7.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-kerberos-wizard-8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-kerberos-wizard-8.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-make-user-admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-make-user-admin.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-multiple-KMS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-multiple-KMS.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-pig-nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-pig-nodes.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-proxyuser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-proxyuser.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-restart-services.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-restart-services.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-run-LDAPsync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-run-LDAPsync.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-setup-LDAP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-setup-LDAP.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ambari-start-kerberos-wizard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ambari-start-kerberos-wizard.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Banana-audits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Banana-audits.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Banana-empty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Banana-empty.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Files-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Files-view.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Hive-view-viz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Hive-view-viz.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Hive-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Hive-view.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Pig-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Pig-view.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-AccessManager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-AccessManager.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HBASE-create-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HBASE-create-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HBASE-edit-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HBASE-edit-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HBASE-plugin-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HBASE-plugin-config.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HBASE-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HBASE-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HDFS-create-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HDFS-create-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HDFS-edit-policy-add-hadoopadmin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HDFS-edit-policy-add-hadoopadmin.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HDFS-edit-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HDFS-edit-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HDFS-plugin-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HDFS-plugin-config.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HDFS-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HDFS-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-audit-persons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-audit-persons.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-create-policy-persons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-create-policy-persons.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-create-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-create-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-edit-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-edit-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-plugin-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-plugin-config.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-policy-ambari.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-policy-ambari.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-HIVE-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-HIVE-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-audit.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-config-audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-config-audit.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-create-policy-testkey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-create-policy-testkey.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-createkey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-createkey.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-edit-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-edit-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-edit-repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-edit-repo.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-policy-add-nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-policy-add-nn.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KMS-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KMS-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-KNOX-plugin-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-KNOX-plugin-config.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-YARN-plugin-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-YARN-plugin-config.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-add-nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-add-nn.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-ambari-user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-ambari-user.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HBASE-allowed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HBASE-allowed.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HBASE-denied.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HBASE-denied.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HBASE-policy-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HBASE-policy-details.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HBASE-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HBASE-summary.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HDFS-allowed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HDFS-allowed.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HDFS-denied.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HDFS-denied.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HDFS-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HDFS-summary.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HIVE-allowed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HIVE-allowed.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HIVE-denied.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HIVE-denied.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HIVE-policy-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HIVE-policy-details.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-HIVE-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-HIVE-summary.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-KNOX-hive-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-KNOX-hive-summary.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-banana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-banana.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audit-policy-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audit-policy-details.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-audits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-audits.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-install-configure-components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-install-configure-components.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-knox-hive-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-knox-hive-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-knox-webhdfs-audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-knox-webhdfs-audit.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-knox-webhdfs-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-knox-webhdfs-policy.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-plugins.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-plugins.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-policy-kms-audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-policy-kms-audit.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-user-HTTP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-user-HTTP.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-user-groups.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-user-groups.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-user-hive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-user-hive.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-user-permissions-audits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-user-permissions-audits.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Ranger-user-permissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Ranger-user-permissions.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Tez-view-viz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Tez-view-viz.png -------------------------------------------------------------------------------- /security-advanced/screenshots/Tez-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/Tez-view.png -------------------------------------------------------------------------------- /security-advanced/screenshots/chrome-untrusted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/chrome-untrusted.png -------------------------------------------------------------------------------- /security-advanced/screenshots/clustername.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/clustername.png -------------------------------------------------------------------------------- /security-advanced/screenshots/firefox-untrusted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/firefox-untrusted.png -------------------------------------------------------------------------------- /security-advanced/screenshots/knox-location.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/knox-location.png -------------------------------------------------------------------------------- /security-advanced/screenshots/knox-webhdfs-browser1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/knox-webhdfs-browser1.png -------------------------------------------------------------------------------- /security-advanced/screenshots/knox-webhdfs-browser2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/knox-webhdfs-browser2.png -------------------------------------------------------------------------------- /security-advanced/screenshots/knox-webhdfs-browser3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/knox-webhdfs-browser3.png -------------------------------------------------------------------------------- /security-advanced/screenshots/putty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/putty.png -------------------------------------------------------------------------------- /security-advanced/screenshots/solr-cloud.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/solr-cloud.png -------------------------------------------------------------------------------- /security-advanced/screenshots/solr-dashboard-collection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/solr-dashboard-collection.png -------------------------------------------------------------------------------- /security-advanced/screenshots/solr-service-configs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/solr-service-configs.png -------------------------------------------------------------------------------- /security-advanced/screenshots/solr-service-placement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-advanced/screenshots/solr-service-placement.png -------------------------------------------------------------------------------- /security-advanced/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o xtrace 3 | 4 | export HOME=${HOME:-/root} 5 | export TERM=xterm 6 | : ${ambari_pass:="BadPass#1"} 7 | ambari_password="${ambari_pass}" 8 | : ${ambari_services:="HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER AMBARI_METRICS SLIDER AMBARI_INFRA LOGSEARCH TEZ"} 9 | : ${install_ambari_server:=true} 10 | : ${ambari_stack_version:=2.5} 11 | cluster_name=${stack:-mycluster} 12 | 13 | : ${install_ambari_server:=true} 14 | : ${ambari_stack_version:=2.5} 15 | : ${host_count:=skip} 16 | 17 | : ${recommendation_strategy:="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"} 18 | 19 | export install_ambari_server ambari_pass host_count ambari_services 20 | export ambari_password cluster_name recommendation_strategy 21 | 22 | cd 23 | 24 | yum makecache 25 | yum -y -q install git epel-release ntpd screen mysql-connector-java jq python-argparse python-configobj ack 26 | 27 | ######## 28 | ######## 29 | ## Install rackerlabs/recap to gather metrics 30 | yum -y -q install git bc elinks net-tools sysstat iotop 31 | git clone https://github.com/rackerlabs/recap.git 32 | cd recap; make install; cd 33 | 34 | opts="USESAR USESARR USESARQ USEPSTREE USENETSTATSUM USEDF USESLAB USEFDISK" 35 | for opt in ${opts} 36 | do 37 | sed -i "s/${opt}=no/${opt}=yes/" /etc/recap 38 | done 39 | unset opts 40 | 41 | recap 42 | echo '*/5 * * * * root /usr/sbin/recap' >> /etc/cron.d/recap 43 | ######## 44 | ######## 45 | 46 | #ad_ip=172.30.0.78 47 | #echo "${ad_ip} ad01.lab.hortonworks.net ad01" | sudo tee -a /etc/hosts 48 | 49 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 50 | 51 | ~/ambari-bootstrap/ambari-bootstrap.sh 52 | 53 | ## Ambari Server specific tasks 54 | if [ "${install_ambari_server}" = "true" ]; then 55 | bash -c "nohup ambari-server restart" || true 56 | 57 | sleep 60 58 | 59 | ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar 60 | ambari_pass=admin source ~/ambari-bootstrap/extras/ambari_functions.sh 61 | ambari_change_pass admin admin ${ambari_pass} 62 | sleep 1 63 | 64 | if [ "${deploy}" = "true" ]; then 65 | 66 | cd ~/ambari-bootstrap/deploy 67 | 68 | ## various configuration changes for demo environments, and fixes to defaults 69 | cat << EOF > configuration-custom.json 70 | { 71 | "configurations" : { 72 | "core-site": { 73 | "hadoop.proxyuser.root.users" : "admin", 74 | "fs.trash.interval": "4320" 75 | }, 76 | "hdfs-site": { 77 | "dfs.namenode.safemode.threshold-pct": "0.99" 78 | }, 79 | "hive-interactive-env": { 80 | "enable_hive_interactive": "true", 81 | "llap_queue_capacity": "75" 82 | }, 83 | "hive-site": { 84 | "hive.exec.compress.output": "true", 85 | "hive.merge.mapfiles": "true", 86 | "hive.server2.tez.initialize.default.sessions": "true" 87 | }, 88 | "mapred-site": { 89 | "mapreduce.job.reduce.slowstart.completedmaps": "0.7", 90 | "mapreduce.map.output.compress": "true", 91 | "mapreduce.output.fileoutputformat.compress": "true" 92 | }, 93 | "yarn-site": { 94 | "yarn.acl.enable" : "true" 95 | } 96 | } 97 | } 98 | EOF 99 | 100 | ./deploy-recommended-cluster.bash 101 | cd ~ 102 | sleep 5 103 | 104 | source ~/ambari-bootstrap/extras/ambari_functions.sh 105 | ambari_configs 106 | ambari_wait_request_complete 1 107 | fi 108 | fi 109 | 110 | exit 0 111 | 112 | -------------------------------------------------------------------------------- /security-advanced/training-keypair.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAgvWXfY06APncXqGcVHCBAjqOpC9NSOsVsKvKLNfXyJFERG6wUiBNp/0Vmi6S 3 | +u+F9hvjATBv9x0LnAcVlS+pWRL/sH5460exRR/5EEuMSTkUQoBvBwUxg8UZAsuzbPbGVcY1T8kC 4 | g50elo/EV7cLw/JXsVl5Ze1MDIVq2D8aGkOw6rHaia+WtYfodTlJaMXzBeRIkTOAvSByX1KXaUEU 5 | CYq41dBqzNTntUGIeDfbBFBGTTJLe4omCs4O+V34JzQpCkhNswDXgNBAkVynF9I/EjmD15ihTKeg 6 | hvEPs47dXYFlDUFvl7JPKJ+Gmwbzhwr+1OxNY4xnUi7hAn2o2Xr0PQIDAQABAoIBAHo4EBC5e9er 7 | WJHsi8R2VsAovDTYfG1AELcYLJAbRKRHhonDL8UFptPPq5iAPOX6InCkaAAVH3llXw9XwTnBCHA7 8 | /erOrbUO7I8RntUpXJx9q73AtV286BANIyFr0GEyEbm9SMXUWAdS3FgxhlFCYLWXd2XplhWdF+tb 9 | nHKCua6l9+CHIKEVHrjTgtSwRPZfwY5zzCVa69UDcpTPvEsOCbovASUsIeUOVRmvQSN9ZZUBGb8T 10 | wmcGhqZlmd+qLc7Qle4Ry6WyHQ3kENNsN0xfVfZGRVK+9/L0HWv6HxCGTzVy+Tb9EYLAP/DXCRm8 11 | VSBhxnWLLqZa075y3LMvktUNJIECgYEA9FeiGirtaSj4yJt/fzxMaTlEcZpW0MCOL9+8C5INwiej 12 | Xy3vn1EjlMN4dnoC91CuPoJC5Lh39zJ4AK9BS340CKDXqHfPrrHaBRXlcklQXAhsF14VKh7E506h 13 | sj0FBuxMgUM14+EA9m20HQbuh+9OJS5KVbshKLGysfyO71NngaECgYEAiTUc6g+4Sc9eHg3W5b29 14 | OzPSTV5KY7wZrzu+TeIMSVSYz7mPkwC0MuVPoK7mpzGiuv/7BLmdvzRd346O0PYa1NbZ4jIZDUA3 15 | wQHazWNROoWWtkpszmsw4OPdUlOpuFL4Ml1OzzbtEkIFO7XMubVmqHlEY/tFBq17mv9NLfzlJR0C 16 | gYEAyVf9J/r/r4wnlYctRLd3b63VDJfVuicyA5kl55Lu63KmWSQNNQczfr6kYVyhXRKKws+sN7Ua 17 | hgR8hnPkJkVUs1d23kf8R0QfF9m3PLnVEkGOcKMCZbb9Im3VKH7HybCmSk4L9j/O/0HfDZxMqo8z 18 | ocfHUWXmKRZk3e30Cpp7zIECgYBoD9FbMAr8Z/sg3iwzWGFxyk7hAPd4hwwP3EbsONpKr4+qVDkx 19 | K+J/i22GF8C1465hF44hOkoK30OnLy7gCxe0fzMHHCnl4+OlTavOBFpyASmaHf5B3AhZOFj+fJeH 20 | Rd4aDKyd+jF5nHrtm2enHY67jiWxkrv6YY8Ssjit/VUpfQKBgHV25TXiD3mYYbtNJ1wkl830Bpx0 21 | wB7HIPXg5lpMlTUHUVYnz8+Mv+ICKQABjDcFuqZlt/rx0jH6regghOYpa7Y5N0irBHcC+GPxLg24 22 | QYbbEsKiX2i9Cc3zhyifRQJZ7WvaSuiA6HyUzprEvQ6Qgcn3Sg6t2EnpDgrN6p+PKc1J 23 | -----END RSA PRIVATE KEY----- 24 | -------------------------------------------------------------------------------- /security-advanced/training-keypair.ppk: -------------------------------------------------------------------------------- 1 | PuTTY-User-Key-File-2: ssh-rsa 2 | Encryption: none 3 | Comment: imported-openssh-key 4 | Public-Lines: 6 5 | AAAAB3NzaC1yc2EAAAADAQABAAABAQCC9Zd9jToA+dxeoZxUcIECOo6kL01I6xWw 6 | q8os19fIkUREbrBSIE2n/RWaLpL674X2G+MBMG/3HQucBxWVL6lZEv+wfnjrR7FF 7 | H/kQS4xJORRCgG8HBTGDxRkCy7Ns9sZVxjVPyQKDnR6Wj8RXtwvD8lexWXll7UwM 8 | hWrYPxoaQ7DqsdqJr5a1h+h1OUloxfMF5EiRM4C9IHJfUpdpQRQJirjV0GrM1Oe1 9 | QYh4N9sEUEZNMkt7iiYKzg75XfgnNCkKSE2zANeA0ECRXKcX0j8SOYPXmKFMp6CG 10 | 8Q+zjt1dgWUNQW+Xsk8on4abBvOHCv7U7E1jjGdSLuECfajZevQ9 11 | Private-Lines: 14 12 | AAABAHo4EBC5e9erWJHsi8R2VsAovDTYfG1AELcYLJAbRKRHhonDL8UFptPPq5iA 13 | POX6InCkaAAVH3llXw9XwTnBCHA7/erOrbUO7I8RntUpXJx9q73AtV286BANIyFr 14 | 0GEyEbm9SMXUWAdS3FgxhlFCYLWXd2XplhWdF+tbnHKCua6l9+CHIKEVHrjTgtSw 15 | RPZfwY5zzCVa69UDcpTPvEsOCbovASUsIeUOVRmvQSN9ZZUBGb8TwmcGhqZlmd+q 16 | Lc7Qle4Ry6WyHQ3kENNsN0xfVfZGRVK+9/L0HWv6HxCGTzVy+Tb9EYLAP/DXCRm8 17 | VSBhxnWLLqZa075y3LMvktUNJIEAAACBAPRXohoq7Wko+Mibf388TGk5RHGaVtDA 18 | ji/fvAuSDcIno18t759RI5TDeHZ6AvdQrj6CQuS4d/cyeACvQUt+NAig16h3z66x 19 | 2gUV5XJJUFwIbBdeFSoexOdOobI9BQbsTIFDNePhAPZttB0G7ofvTiUuSlW7ISix 20 | srH8ju9TZ4GhAAAAgQCJNRzqD7hJz14eDdblvb07M9JNXkpjvBmvO75N4gxJVJjP 21 | uY+TALQy5U+gruanMaK6//sEuZ2/NF3fjo7Q9hrU1tniMhkNQDfBAdrNY1E6hZa2 22 | SmzOazDg491SU6m4UvgyXU7PNu0SQgU7tcy5tWaoeURj+0UGrXua/00t/OUlHQAA 23 | AIB1duU14g95mGG7TSdcJJfN9AacdMAexyD14OZaTJU1B1FWJ8/PjL/iAikAAYw3 24 | BbqmZbf68dIx+q3oIITmKWu2OTdIqwR3Avhj8S4NuEGG2xLCol9ovQnN84con0UC 25 | We1r2krogOh8lM6axL0OkIHJ90oOrdhJ6Q4KzeqfjynNSQ== 26 | Private-MAC: 095e18fef7b9459f870aa622c32667aacdc8e14c 27 | -------------------------------------------------------------------------------- /security-official/containers/ambari/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sequenceiq/ambari:latest 2 | MAINTAINER seanorama 3 | 4 | RUN yum install -y screen git openssh-server openssh-clients unzip bind-utils sudo createrepo 5 | 6 | ## optional packages for students 7 | RUN yum -y install screen jq tmux 8 | 9 | RUN chkconfig sshd on 10 | RUN service sshd restart 11 | 12 | RUN useradd student 13 | RUN usermod -aG wheel student 14 | RUN echo 'student:BadPass#1' | chpasswd 15 | RUN sed -i -e 's/^# \(%wheel\)/\1/' /etc/sudoers 16 | 17 | RUN cp -a /etc/ambari-agent/conf/internal-hostname.sh /etc/ambari-agent/conf/public-hostname.sh 18 | RUN chkconfig ambari-agent on 19 | RUN ambari-agent restart 20 | 21 | ENV PS1 "[\u@\h \W]"\#" " 22 | RUN echo 'export PS1="[\u@\h \W]"\#" "' >> /root/.bash_profile 23 | 24 | EXPOSE 22 25 | EXPOSE 8443 26 | -------------------------------------------------------------------------------- /security-official/containers/desktop/hortonworks.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/security-official/containers/desktop/hortonworks.jpg -------------------------------------------------------------------------------- /security/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## for prepping a 1-node cluster for the security masterclass 4 | 5 | sudo yum -y -q install git 6 | cd 7 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 8 | source ~/ambari-bootstrap/extras/ambari_functions.sh 9 | 10 | ${__dir}/deploy/prep-hosts.sh 11 | 12 | export ambari_services="KNOX YARN ZOOKEEPER TEZ PIG SLIDER MAPREDUCE2 HIVE HDFS HBASE SQOOP FLUME OOZIE" 13 | export custom_repos=true 14 | "${__dir}/deploy/deploy-hdp.sh" 15 | sleep 30 16 | 17 | source ${__dir}/ambari_functions.sh 18 | source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_change_pass admin admin BadPass#1 19 | echo "export ambari_pass=BadPass#1" > ~/ambari-bootstrap/extras/.ambari.conf; chmod 660 ~/ambari-bootstrap/extras/.ambari.conf 20 | source ${__dir}/ambari_functions.sh 21 | ambari_configs 22 | ambari_wait_request_complete 1 23 | 24 | ## details of my ad host 25 | ad_host="${ad_host:-activedirectory.$(hostname -d)}" 26 | ad_host_ip=$(ping -w 1 ${ad_host} | awk 'NR==1 {print $3}' | sed 's/[()]//g') 27 | echo "${ad_host_ip} activedirectory.hortonworks.com ${ad_host} activedirectory" | sudo tee -a /etc/hosts 28 | 29 | ## Generic setup 30 | sudo chkconfig mysqld on; sudo service mysqld start 31 | ${__dir}/add-trusted-ca.sh 32 | ${__dir}/onboarding.sh 33 | ${__dir}/ambari-views/create-views.sh 34 | #config_proxyuser=true ${__dir}/ambari-views/create-views.sh 35 | ${__dir}/samples/sample-data.sh 36 | ${__dir}/configs/proxyusers.sh 37 | ${__dir}/ranger/prep-mysql.sh 38 | #proxyusers="oozie falcon" ${__dir}/configs/proxyusers.sh 39 | ##centos6 only #${__dir}/oozie/replace-mysql-connector.sh 40 | 41 | mirror_host="${mirror_host:-mc-teacher1.$(hostname -d)}" 42 | mirror_host_ip=$(ping -w 1 ${mirror_host} | awk 'NR==1 {print $3}' | sed 's/[()]//g') 43 | echo "${mirror_host_ip} mirror.hortonworks.com ${mirror_host} mirror admin admin.hortonworks.com" | sudo tee -a /etc/hosts 44 | sudo mkdir -p /app; sudo chown ${USER}:users /app; sudo chmod g+wx /app 45 | 46 | -------------------------------------------------------------------------------- /single-view/LABS.md: -------------------------------------------------------------------------------- 1 | 2 | ## 0. Prep hosts 3 | 4 | This is already done on our automated clusters. 5 | 6 | ``` 7 | ~/masterclass/single-view/scripts/00_prep_host.sh 8 | ``` 9 | 10 | ## 1. Deploy HDP 11 | 12 | ``` 13 | ~/masterclass/single-view/scripts/10_deploy_hdp.sh 14 | ``` 15 | 16 | ## 2. Open Ambari to monitor cluster deployment 17 | 18 | Don't proceed to next steps until Ambari is done and all services are up. 19 | 20 | ## 3. Setup demo 21 | 22 | ``` 23 | ~/masterclass/single-view/scripts/20_setup_demo.sh 24 | ``` 25 | 26 | ## 4. Open Zeppelin 27 | 28 | ## 5. Work through the EDW Notebook: 29 | 30 | https://raw.githubusercontent.com/seanorama/masterclass/master/single-view/files/zeppelin_notebook_10_edw.json 31 | 32 | ## 6. Work through the NiFi Notebook 33 | 34 | https://raw.githubusercontent.com/seanorama/masterclass/master/single-view/files/zeppelin_notebook_20_nifi.json 35 | 36 | ##7. Work through the Hive Analytics Notebook: 37 | 38 | https://raw.githubusercontent.com/seanorama/masterclass/master/single-view/files/zeppelin_notebook_30_analytics.json 39 | -------------------------------------------------------------------------------- /single-view/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | ## Deploy clusters on AWS 5 | 6 | See ../generic/README-INSTRUCTOR.md for more details 7 | 8 | ``` 9 | 10 | cd ../generic 11 | 12 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 13 | export lab_prefix=summit ## template for naming the cloudformation stacks 14 | export lab_first=100 ## number to start at in naming 15 | export lab_count=1 ## number of clusters to create 16 | 17 | export cfn_parameters=' 18 | [ 19 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 20 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 21 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 22 | {"ParameterKey":"AdditionalInstanceCount","ParameterValue":"0"}, 23 | {"ParameterKey":"PostCommand","ParameterValue":"curl -ksSL https://raw.githubusercontent.com/seanorama/masterclass/master/single-view/scripts/00_prep_host.sh | bash"}, 24 | {"ParameterKey":"InstanceType","ParameterValue":"m4.2xlarge"}, 25 | {"ParameterKey":"BootDiskSize","ParameterValue":"100"} 26 | ] 27 | ' 28 | 29 | ``` 30 | 31 | ## Distribute cluster details to participants: 32 | 33 | Populate lab list: 34 | 35 | 1. Execute: 36 | ``` 37 | ../bin/clusters-report-flat.sh 38 | ``` 39 | 40 | 2. Paste into a shared notepad. My personal one is: http://j.mp/summit-labs 41 | 3. Distribute link to participants and ask them to claim their custers 42 | 43 | ## Follow labs in LABS.md 44 | -------------------------------------------------------------------------------- /single-view/files/zeppelin_notebook_20_nifi.json: -------------------------------------------------------------------------------- 1 | {"paragraphs":[{"text":"%md\n## Ingest tweets and weblogs into HDFS using Nifi\n\nIn this section we will:\n\n1. Create a script which generates weblogs\n2. Setup Solr for twitter\n3. Create a Twitter App for accessing the Twitter API\n4. Use NiFi to load tweets & weblogs into HDFS & SOLR\n","dateUpdated":"Apr 10, 2016 6:18:54 PM","config":{"colWidth":12,"editorMode":"ace/mode/markdown","editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334513_658662481","id":"20160410-181854_1856674203","result":{"code":"SUCCESS","type":"HTML","msg":"

Ingest tweets and weblogs into HDFS using Nifi

\n

In this section we will:

\n
    \n
  1. Create a script which generates weblogs
    2. \n
  2. Setup Solr for twitter
    3. \n
  3. Create a Twitter App for accessing the Twitter API
    4. \n
  4. Use NiFi to load tweets & weblogs into HDFS & SOLR
    5. \n
\n"},"dateCreated":"Apr 10, 2016 6:18:54 PM","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:396"},{"title":"1. Script \"weblog generator\" script","text":"%sh\ngit clone https://github.com/abajwa-hw/single-view-demo.git\ncp /home/zeppelin/single-view-demo/createlog-psql.sh /tmp\ncp /home/zeppelin/single-view-demo/DimCustomer.csv /tmp\n\necho \"/tmp/createlog-psql.sh /tmp/data/DimCustomer.csv 20 >> /tmp/webtraffic.log\" >> /tmp/generate-weblog.sh\n\nchmod 777 /tmp/createlog-psql.sh\nchmod 777 /tmp/generate-weblog.sh\nchmod 444 /tmp/DimCustomer.csv\n","dateUpdated":"Apr 10, 2016 6:20:36 PM","config":{"colWidth":12,"editorMode":"ace/mode/sh","editorHide":false,"title":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334514_659816728","id":"20160410-181854_1276584461","dateCreated":"Apr 10, 2016 6:18:54 PM","status":"FINISHED","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:397","dateFinished":"Apr 10, 2016 6:20:52 PM","dateStarted":"Apr 10, 2016 6:20:36 PM","focus":true},{"title":"2. Configure Solr for Twitter","text":"%sh\nsudo curl -ksSL https://raw.githubusercontent.com/seanorama/masterclass/master/single-view/files/banana.json -o /opt/lucidworks-hdpsearch/solr/server/solr-webapp/webapp/banana/app/dashboards/default.json\nsudo curl -L https://git.io/vVRPz -o /opt/lucidworks-hdpsearch/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml\n","dateUpdated":"Apr 10, 2016 7:13:22 PM","config":{"colWidth":12,"editorMode":"ace/mode/sh","title":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334514_659816728","id":"20160410-181854_1594798780","dateCreated":"Apr 10, 2016 6:18:54 PM","status":"FINISHED","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:398","focus":true,"dateFinished":"Apr 10, 2016 7:13:23 PM","dateStarted":"Apr 10, 2016 7:13:22 PM"},{"text":"%md\n#### Restart Solr from Ambari\n\nDo not proceed to the next step until Solr is up","dateUpdated":"Apr 10, 2016 6:18:54 PM","config":{"colWidth":12,"editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334514_659816728","id":"20160410-181854_878270341","result":{"code":"SUCCESS","type":"HTML","msg":"

Restart Solr from Ambari

\n

Do not proceed to the next step until Solr is up

\n"},"dateCreated":"Apr 10, 2016 6:18:54 PM","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:399"},{"title":"Create Tweets collection in Solr","text":"%sh\nsudo -i /opt/lucidworks-hdpsearch/solr/bin/solr create -c tweets \\\n -d data_driven_schema_configs \\\n -s 1 \\\n -rf 1 >> /tmp/zeppelin.out 2>&1","dateUpdated":"Apr 10, 2016 6:28:47 PM","config":{"colWidth":12,"title":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/sh"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334515_659431979","id":"20160410-181854_243623636","result":{"code":"SUCCESS","type":"TEXT","msg":""},"dateCreated":"Apr 10, 2016 6:18:54 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:400","dateFinished":"Apr 10, 2016 6:28:51 PM","dateStarted":"Apr 10, 2016 6:28:47 PM","focus":true},{"text":"%md\n#### 3. Create Twitter app & access tokens\n\nTo monitor tweets, you will need to do the following on your Twitter account:\n- Create a Twitter account and app and get your consumer key/token and access keys/tokens:\n - Open https://apps.twitter.com \n - sign in\n - create new app\n - the \"Application Details\" do not matter. Put whatever you want.\n - create access tokens\n\n#### 4. Load our prebuilt Tempalte into NiFI\n\n1. Download prebuilt Single_view_demo.xml template to your laptop from [here](https://github.com/abajwa-hw/single-view-demo/raw/master/template/Single_view_demo.xml)\n2. Open Nifi webui (http://:9090)\n3. Import the Single_view_demo.xml template info Nifi\n\n#### 5. Update Twitter credentials in the Twitter Process Group\n\n#### 6. Start the Flows","dateUpdated":"Apr 10, 2016 6:18:54 PM","config":{"colWidth":12,"editorMode":"ace/mode/markdown","editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334515_659431979","id":"20160410-181854_2071407171","result":{"code":"SUCCESS","type":"HTML","msg":"

3. Create Twitter app & access tokens

\n

To monitor tweets, you will need to do the following on your Twitter account:

\n
    \n
  • Create a Twitter account and app and get your consumer key/token and access keys/tokens:
    • \n
  • Open https://apps.twitter.com
    • \n
  • sign in
    • \n
  • create new app
    • \n
  • the “Application Details” do not matter. Put whatever you want.
    • \n
  • create access tokens
    • \n
\n

4. Load our prebuilt Tempalte into NiFI

\n
    \n
  1. Download prebuilt Single_view_demo.xml template to your laptop from here
    2. \n
  2. Open Nifi webui (http://:9090)
    3. \n
  3. Import the Single_view_demo.xml template info Nifi
    4. \n
\n

5. Update Twitter credentials in the Twitter Process Group

\n

6. Start the Flows

\n"},"dateCreated":"Apr 10, 2016 6:18:54 PM","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:401"},{"text":"%md\n## Done! NiFi is now:\n\n - ingesting the tweets under /tmp/weblog_staging in HDFS\n - ingest the weblogs under /tmp/tweets_staging in HDFS","dateUpdated":"Apr 10, 2016 6:18:54 PM","config":{"colWidth":12,"editorMode":"ace/mode/markdown","editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334515_659431979","id":"20160410-181854_1371995331","result":{"code":"SUCCESS","type":"HTML","msg":"

Done! NiFi is now:

\n
    \n
  • ingesting the tweets under /tmp/weblog_staging in HDFS
    • \n
  • ingest the weblogs under /tmp/tweets_staging in HDFS
    • \n
\n"},"dateCreated":"Apr 10, 2016 6:18:54 PM","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:402"},{"dateUpdated":"Apr 10, 2016 6:18:54 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460312334515_659431979","id":"20160410-181854_794655745","dateCreated":"Apr 10, 2016 6:18:54 PM","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:403"}],"name":"Single View - NiFi","id":"2BJYETW55","angularObjects":{"2BHF1YQU4":[],"2BFUFECZ7":[],"2BFS78D7X":[],"2BJ3XSZGS":[],"2BG5E2RDQ":[],"2BFGEHQMK":[],"2BGKHY6PM":[],"2BHU77U7J":[],"2BGE9XXA9":[],"2BJ55Z5ZP":[],"2BHEH6UEK":[],"2BJJJE6HG":[],"2BFX55QVZ":[],"2BJE2ZWVQ":[],"2BGEDT8GC":[]},"config":{"looknfeel":"default"},"info":{}} -------------------------------------------------------------------------------- /single-view/scripts/00_prep_host.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o xtrace 3 | 4 | cd /root 5 | export TERM=xterm 6 | export ambari_pass=${ambari_pass:-BadPass#1} 7 | 8 | yum makecache 9 | yum -y -q install epel-release 10 | yum -y -q install autoconf python-crypto python-devel unzip gcc-c++ git python-argparse 11 | 12 | git clone -b develop https://github.com/seanorama/ambari-bootstrap 13 | ~/ambari-bootstrap/extras/deploy/prep-hosts.sh 14 | 15 | git clone https://github.com/seanorama/masterclass 16 | 17 | #( 18 | #cd /tmp 19 | #wget http://hortonworks-masterclass.s3.amazonaws.com/single-view/data.zip 20 | #unzip data.zip 21 | #) 22 | 23 | #yum -y -q install postgresql-server 24 | #echo "host all all 127.0.0.1/32 md5" >> /var/lib/pgsql/data/pg_hba.conf 25 | 26 | #service postgresql start 27 | #chkconfig postgresql on 28 | #sleep 5 29 | 30 | #git clone https://github.com/abajwa-hw/single-view-demo 31 | #sudo -i -u postgres psql -c "create database contoso;" 32 | #sudo -i -u postgres psql -c "CREATE USER zeppelin WITH PASSWORD 'zeppelin';" 33 | #sudo -i -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE contoso to zeppelin;" 34 | #sudo -i -u postgres psql -c "\du" 35 | #export PGPASSWORD=zeppelin 36 | #psql -U zeppelin -d contoso -h localhost -f ~/single-view-demo/contoso-psql.sql 37 | #psql -U zeppelin -d contoso -h localhost -c "\dt" 38 | 39 | #echo "zeppelin ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers 40 | 41 | -------------------------------------------------------------------------------- /single-view/scripts/10_deploy_hdp.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd ~ 4 | 5 | test -f ~/masterclass.env && source ~/masterclass.env 6 | export ambari_pass="${ambari_pass:-BadPass#1}" 7 | export cluster_name="${cluster_name:-mycluster}" 8 | 9 | ## Install Python in virtualenv 10 | VERSION=15.0.1 11 | curl -sSL https://pypi.python.org/packages/source/v/virtualenv/virtualenv-$VERSION.tar.gz | tar xz 12 | $(which python) virtualenv-$VERSION/virtualenv.py ~/python 13 | rm -rf virtualenv-$VERSION; unset VERSION 14 | set +u ; source ~/python/bin/activate; set -u 15 | pip install ansible boto six ruamel.yaml 16 | 17 | ## ansible-hadoop 18 | git clone -b develop https://github.com/seanorama/ansible-hadoop 19 | cd ansible-hadoop 20 | 21 | printf "[master-nodes]\nlocalhost ansible_ssh_host=localhost\n" \ 22 | > inventory/static 23 | 24 | sed -i.bak \ 25 | -e "s/^\(cluster_name: \).*/\1'${cluster_name}'/" \ 26 | -e "s/^\(admin_password: \).*/\1'${ambari_pass}'/" \ 27 | -e "s/^\(update_etc_hosts: \).*/\1false/" \ 28 | -e "s/^\(java_type: \).*/\1open/" \ 29 | -e "s/^\(tech_preview: \).*/\1true/" \ 30 | -e "s/^\(install_nifi: \).*/\1true/" \ 31 | -e "s/^\(install_solr: \).*/\1true/" \ 32 | -e "s/^\(install_ams: \).*/\1true/" \ 33 | -e "s/^\(install_zeppelin: \).*/\1true/" \ 34 | -e "s/^\(install_falcon: \).*/\1false/" \ 35 | -e "s/^\(install_flume: \).*/\1false/" \ 36 | -e "s/^\(install_oozie: \).*/\1false/" \ 37 | -e "s/^\(install_kafka: \).*/\1false/" \ 38 | -e "s/^\(install_slider: \).*/\1false/" \ 39 | -e "s/^\(install_storm: \).*/\1false/" \ 40 | -e "s/^\(deploy_hadoop: \).*/\1true/" \ 41 | playbooks/group_vars/all 42 | 43 | ansible-playbook -i inventory/static -c local playbooks/bootstrap.yml 44 | ansible-playbook -i inventory/static -c local playbooks/hortonworks.yml 45 | 46 | sleep 60 47 | 48 | ## Wait until cluster is built 49 | echo "export ambari_pass=BadPass#1" >> ~/.ambari.conf 50 | chmod 600 ~/.ambari.conf 51 | source ~/ambari-bootstrap/extras/ambari_functions.sh 52 | ambari_configs 53 | ambari_wait_request_complete 1 54 | 55 | -------------------------------------------------------------------------------- /single-view/scripts/20_setup_demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd ~ 4 | 5 | wget https://jdbc.postgresql.org/download/postgresql-9.4.1207.jar -P /usr/hdp/current/sqoop-client/lib 6 | 7 | UID_MIN=$(awk '$1=="UID_MIN" {print $2}' /etc/login.defs) 8 | users="${users:-$(getent passwd|awk -v UID_MIN="${UID_MIN}" -F: '$3>=UID_MIN{print $1}')}" 9 | dfs_cmd="sudo sudo -u hdfs hadoop fs" 10 | for user in ${users}; do 11 | usermod -a -G users "${user}" 12 | if ! ${dfs_cmd} -stat /user/${user} 2> /dev/null; then 13 | ${dfs_cmd} -mkdir -p "/user/${user}" 14 | ${dfs_cmd} -chown "${user}" "/user/${user}" & 15 | fi 16 | done 17 | 18 | ( 19 | cd /tmp 20 | wget http://hortonworks-masterclass.s3.amazonaws.com/single-view/data.zip 21 | unzip data.zip 22 | ) 23 | 24 | echo "host all all 127.0.0.1/32 md5" >> /var/lib/pgsql/data/pg_hba.conf 25 | service postgresql reload 26 | 27 | git clone https://github.com/abajwa-hw/single-view-demo 28 | sudo -i -u postgres psql -c "create database contoso;" 29 | sudo -i -u postgres psql -c "CREATE USER zeppelin WITH PASSWORD 'zeppelin';" 30 | sudo -i -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE contoso to zeppelin;" 31 | sudo -i -u postgres psql -c "\du" 32 | export PGPASSWORD=zeppelin 33 | psql -U zeppelin -d contoso -h localhost -f ~/single-view-demo/contoso-psql.sql 34 | psql -U zeppelin -d contoso -h localhost -c "\dt" 35 | 36 | echo "zeppelin ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers 37 | 38 | wait 39 | 40 | -------------------------------------------------------------------------------- /spark/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Instructor notes 2 | ======================================== 3 | 4 | These instructions will create lab environments for: 5 | 6 | - a. HDP Cluster: 1 node with HDFS, YARN, HIVE, KAFKA, SPARK, ZEPPELIN 7 | 8 | ## Before you start 9 | 10 | See ../README.md for instructions on using the cluster management scripts (step 3 in the processes below): 11 | 12 | - ../bin/cluster-create.sh 13 | - ../bin/cluster-report.sh 14 | - ../bin/cluster-terminate.sh 15 | 16 | ## Deploy clusters 17 | 18 | ### Deploy the HDP node 19 | 20 | 1. Get the repo and switch to the 'generic' directory 21 | 22 | ``` 23 | git clone https://github.com/seanorama/masterclass 24 | cd masterclass/generic 25 | ``` 26 | 27 | 1. Set these variables, updating the values as appropriate: 28 | 29 | ```sh 30 | export AWS_DEFAULT_REGION=eu-west-1 ## region to deploy in 31 | export lab_prefix=mc-spark ## template for naming the cloudformation stacks 32 | export lab_first=100 ## number to start at in naming 33 | export lab_count=1 ## number of clusters to create 34 | 35 | export cfn_parameters=' 36 | [ 37 | {"ParameterKey":"KeyName","ParameterValue":"secloud"}, 38 | {"ParameterKey":"SubnetId","ParameterValue":"subnet-7e49641b"}, 39 | {"ParameterKey":"SecurityGroups","ParameterValue":"sg-f915bc9d"}, 40 | {"ParameterKey":"AmbariServices","ParameterValue":"HDFS MAPREDUCE2 PIG YARN HIVE ZOOKEEPER KAFKA SPARK ZEPPELIN"}, 41 | {"ParameterKey":"PostCommand","ParameterValue":"curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/generic/setup.sh | bash ; curl -sSL https://raw.githubusercontent.com/bernhard-42/Spark-Masterclass/master/prepare.sh | bash"}, 42 | {"ParameterKey":"InstanceType","ParameterValue":"m4.xlarge"}, 43 | {"ParameterKey":"BootDiskSize","ParameterValue":"80"} 44 | ] 45 | ' 46 | ``` 47 | 48 | 1. You can then execute ../bin/clusters-create.sh and the other cluster scripts as explained in ../README.md 49 | 50 | ## REMEMBER to terminate the clusters immediately after the class is over, or be prepared to pay $$$! 51 | 52 | Further, you should verify deletion of the CloudFormations & EC2 instances from the AWS Console. 53 | 54 | ## Issues: See [../README.md](../README.md) 55 | -------------------------------------------------------------------------------- /spark/README.md: -------------------------------------------------------------------------------- 1 | Demos & Labs available at https://github.com/bernhard-42/Spark-Masterclass 2 | -------------------------------------------------------------------------------- /sql/Lab2/customers.txt: -------------------------------------------------------------------------------- 1 | LeopoldDickensFlat 59YCremin harborsWest BridgettshireB1R 5TELeopold.Dickens@mraz.net00408 43349 2 | HarlanBartolettiFlat 72Sandy laneZiemannfurtR2A 4AVHarlan.Bartoletti@gulgowskiernser.info(00964) 38955 3 | SerenaBalistreriStudio 50Powlowski squareDianeburySE0 8DTSerena.Balistreri@dibbertzemlak.com0183658159 4 | ColemanMayert404 Haley parkwaySouth BostonB88 2PEColeman.Mayert@gmail.com05836 66923 5 | YoshioDietrichStudio 66Algernon pointFlemboroughB98 2GBYoshio.Dietrich@gutkowski.net+44(0)4323 073480 6 | MarylynBahringerStudio 19Fielding extensionNorth ShaneH12 9LFMarylyn.Bahringer@reingermoen.com0777032224 7 | JermaineZiemannStudio 6Paisley courseBrocksideN5F 4ZKJermaine.Ziemann@yahoo.com+44(0)2249930455 8 | PollieDickens468 Towne harborSouth RhodaMG2H 5AVPollie.Dickens@donnelly.info+44(0)2206 811796 9 | KeaganKlockoStudio 9Vandervort overpassOndrickalandV62 1ZSKeagan.Klocko@hotmail.com0635716718 10 | KyanKshlerin498 Emmaline roadsRoseannetonB3R 1HCKyan.Kshlerin@crooksmurphy.biz03274 90737 11 | HildredRathStudio 2Harris overpassNew BrittchesterR1 8IAHildred.Rath@hueldach.com01548 48730 12 | KenneyWhite87 Dietrich summitTruemouthCE30 5XVKenney.White@yahoo.com+44(0)6713 059823 13 | ChanaWolf498 Torp groveHernansideYI96 3HAChana.Wolf@gmail.com+44(0)3145967661 14 | JairoKiehnStudio 99NBraulio islandsNorth VirginiaburyA4I 6RGJairo.Kiehn@gmail.com05269 891135 15 | LouannCarroll43 Deven damEast DeyanirashireA8N 6RGLouann.Carroll@flatley.com+44(0)1030 52838 16 | ElleHettinger638 Neveah springsNew ChaunceybergPF1R 0XKElle.Hettinger@rippin.biz+44(0)273434337 17 | SquireStehr355 Ethelyn crossroadPort FarahL5 3LNSquire.Stehr@yahoo.com+44(0)124309456 18 | IceyWyman61 Leandra spursPalliebergC4 6VSIcey.Wyman@yahoo.com+44(0)647833158 19 | DelmaWeberFlat 91Herzog turnpikeSouth MarlieburghOA5 5HIDelma.Weber@bauch.org+44(0)5351489174 20 | LawyerJacobi446 Howe grovesJayciehavenF80 3RKLawyer.Jacobi@yahoo.com(07099) 309323 21 | -------------------------------------------------------------------------------- /sql/Lab2/prepare.sh: -------------------------------------------------------------------------------- 1 | echo -e "\n==> Copy customer data to HDFS\n" 2 | 3 | sudo -u hdfs hdfs dfs -mkdir /masterclass/ 4 | sudo -u hdfs hdfs dfs -mkdir /masterclass/lab2/ 5 | sudo -u hdfs hdfs dfs -mkdir /masterclass/lab2/customers 6 | sudo -u hdfs hdfs dfs -chmod 777 /masterclass/lab2/customers 7 | cp ./customers.txt /tmp 8 | sudo -u hdfs hdfs dfs -put /tmp/customers.txt /masterclass/lab2/customers/data1.txt 9 | sudo -u hdfs hdfs dfs -chmod 666 /masterclass/lab2/customers/data1.txt 10 | 11 | 12 | echo -e "\n==> Create customer schema in Hive\n" 13 | 14 | cat < /tmp/customers.sql 15 | CREATE EXTERNAL TABLE customers( 16 | first_name STRING, 17 | last_name STRING, 18 | address ARRAY, 19 | email STRING, 20 | phone STRING 21 | ) 22 | LOCATION "/masterclass/lab2/customers"; 23 | EOF 24 | 25 | sudo -u hdfs hive -f /tmp/customers.sql 26 | -------------------------------------------------------------------------------- /sql/Lab3/prepare.sh: -------------------------------------------------------------------------------- 1 | echo -e "\n==> Copy tweets data to HDFS\n" 2 | 3 | tar -C / -zxf ./tweets.tgz 4 | 5 | sudo -u hdfs hdfs dfs -mkdir /masterclass/lab3 6 | sudo -u hdfs hdfs dfs -mkdir /masterclass/lab3/tweets 7 | sudo -u hdfs hdfs dfs -chmod 777 /masterclass/lab3/tweets 8 | sudo -u hdfs hdfs dfs -put /tmp/tweets/* /masterclass/lab3/tweets/ 9 | sudo -u hdfs hdfs dfs -chmod 666 /masterclass/lab3/tweets/* 10 | 11 | -------------------------------------------------------------------------------- /sql/Lab3/tweets.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/sql/Lab3/tweets.tgz -------------------------------------------------------------------------------- /sql/Lab4/json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seanorama/masterclass/0cbd9b93a9d305a30bdf7f6b4e4da250ee8a9cee/sql/Lab4/json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar -------------------------------------------------------------------------------- /sql/Lab4/prepare.sh: -------------------------------------------------------------------------------- 1 | echo -e "\n==> Copy sample_twitter_data to HDFS\n" 2 | 3 | mkdir -p /tmp/Lab4 4 | cp -a * /tmp/Lab4/ 5 | 6 | sudo -u hdfs hdfs dfs -mkdir -p /masterclass/lab4 7 | sudo -u hdfs hdfs dfs -mkdir /masterclass/lab4/twitter 8 | sudo -u hdfs hdfs dfs -chmod 777 /masterclass/lab4/twitter 9 | sudo -u hdfs hdfs dfs -put /tmp/Lab4/sample_twitter_data.txt /masterclass/lab4/twitter/ 10 | sudo -u hdfs hdfs dfs -chmod 666 /masterclass/lab4/twitter/sample_twitter_data.txt 11 | 12 | 13 | echo -e "\n==> Copy JSON serde to /var/lib/hive\n" 14 | 15 | sudo -u hdfs hdfs dfs -mkdir -p /lib/hive 16 | sudo -u hdfs hdfs dfs -put /tmp/Lab4/json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar /lib/hive/ 17 | sudo cp /tmp/Lab4/json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar /var/lib/hive/ 18 | sudo chmod 644 /var/lib/hive/json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar 19 | 20 | if [ ! -d ~/ambari-bootstrap ]; then 21 | cd 22 | git clone https://github.com/seanorama/ambari-bootstrap 23 | fi 24 | 25 | source ~/ambari-bootstrap/extras/ambari_functions.sh 26 | ambari_configs 27 | 28 | ${ambari_config_get} hive-env \ 29 | | sed -e '1,3d' \ 30 | -e '/^"content" : / s#",$#\\n\\nHIVE_AUX_JARS_PATH=$HIVE_AUX_JARS_PATH:/var/lib/hive/json-serde-1.1.9.9-Hive13-jar-with-dependencies.jar",#' \ 31 | > /tmp/hive-env.json 32 | ${ambari_config_set} hive-env /tmp/hive-env.json 33 | 34 | -------------------------------------------------------------------------------- /sql/Lab4/sample_twitter_data.txt: -------------------------------------------------------------------------------- 1 | {"user":{"userlocation":"Cinderford, Gloucestershire","id":230231618,"name":"Aimee","screenname":"Aimee_Cottle","geoenabled":true},"tweetmessage":"Gastroenteritis has pretty much killed me this week :( off work for a few days whilst I recover!","createddate":"2013-06-20T12:08:14","geolocation":null} 2 | {"user":{"userlocation":"Garena ID : NuraBlazee","id":635239939,"name":"Axyraf.","screenname":"Asyraf_Fauzi","geoenabled":false},"tweetmessage":"RT @abhigyantweets: Can't stop a natural disaster but so many lives in U'khand wouldn't have been lost if there was disaster preparedness. …","createddate":"2013-06-20T12:08:16","geolocation":null} 3 | {"user":{"userlocation":"Gemert,Netherlands","id":21418083,"name":"Ad van Steenbruggen","screenname":"torment00","geoenabled":true},"tweetmessage":"♫ Listening to 'The Constant' by 'Anthrax' from 'Worship Music","createddate":"2013-06-20T12:08:20","geolocation":null} 4 | {"user":{"userlocation":"","id":494156410,"name":"Balistik KartikaSari","screenname":"Balistik_Akas","geoenabled":false},"tweetmessage":"NEXT MATCH : PERSIBA BALIKPAPAN vs Persiwa Wamena | 26-06-2013 | Std.Parikesit | 15.30 Wita | NOT LIVE !!!","createddate":"2013-06-20T12:08:28","geolocation":null} 5 | {"user":{"userlocation":"","id":1183221613,"name":"Gabrielle Moser","screenname":"binhamoser","geoenabled":true},"tweetmessage":"Meu que merda, meu médico me proibiu de ir na passeata por que eu posso ficar com pneumonia, ah se foder","createddate":"2013-06-20T12:08:29","geolocation":null} 6 | {"user":{"userlocation":"Cheltenham, England","id":377932072,"name":"Sarah Daly","screenname":"sarahmygreeneye","geoenabled":false},"tweetmessage":"RT @healthpoverty: Interesting article from @foodpolicynews on the relationship between agriculture and #malaria in Africa...","createddate":"2013-06-20T12:08:39","geolocation":null} 7 | {"user":{"userlocation":"","id":1514008171,"name":"Auzzie Jet","screenname":"metalheadgrunge","geoenabled":false},"tweetmessage":"Anthrax - Black - Sunglasses hell yah\n http://t.co/qCNjba57Dm","createddate":"2013-06-20T12:08:44","geolocation":null} 8 | {"user":{"userlocation":"BANDUNG-INDONESIA","id":236827129,"name":"Powerpuff Girls","screenname":"TasyootReyoot","geoenabled":true},"tweetmessage":"RT @viddyVR: Kok gitu sih RT @TasyootReyoot: #HelloKitty itu aneh pala gede bekumis pula badan kecil kayak org kena polio\"","createddate":"2013-06-20T12:08:45","geolocation":null} 9 | {"user":{"userlocation":"Bolton, UK","id":14141159,"name":"Chris Beckett","screenname":"ChrisBeckett","geoenabled":true},"tweetmessage":"vCOps people - Does Advanced Edition == 5 VMs? I know Std has UI and analytics VM, but what does rest use? Hyperic etc? #vmware #vcops","createddate":"2013-06-20T12:08:46","geolocation":null} -------------------------------------------------------------------------------- /sql/Lab5/prepare.sh: -------------------------------------------------------------------------------- 1 | # Create employees database 2 | 3 | # echo -e "\n==> Install git\n" 4 | 5 | # sudo yum install -y git 6 | 7 | echo -e "\n==> Clone sample database\n" 8 | 9 | cd /tmp 10 | 11 | git clone https://github.com/datacharmer/test_db 12 | 13 | echo -e "\n==> Create sample database\n" 14 | 15 | cd test_db 16 | mysql -u root < employees.sql 17 | 18 | # Convert to tsv 19 | 20 | echo -e "\n==> Convert employees table to tsv\n" 21 | 22 | mkdir /tmp/employees_tsv 23 | chmod 777 /tmp/employees_tsv/ 24 | sudo mysqldump --tab /tmp/employees_tsv/ employees 25 | 26 | # Import into HDFS 27 | 28 | echo -e "\n==> Import employees table into HDFS as tsv\n" 29 | 30 | sudo -u hdfs hdfs dfs -mkdir /tmp/employees_tsv 31 | sudo -u hdfs hdfs dfs -put /tmp/employees_tsv/*.txt /tmp/employees_tsv 32 | sudo -u hdfs hdfs dfs -chmod 666 /tmp/employees_tsv/*.txt 33 | 34 | # Create HBase database 35 | 36 | echo -e "\n==> Create HBase database employees\n" 37 | 38 | cat < /tmp/hbase.txt 39 | create "employees", {NAME => "f"}, {SPLITS => ["200000", "300000", "400000"]} 40 | enable "employees" 41 | exit 42 | EOF 43 | 44 | hbase shell /tmp/hbase.txt 45 | 46 | 47 | # Import data from tsv file 48 | 49 | echo -e "\n==> Import tsv from HDFS into employees\n" 50 | 51 | sudo -u hdfs hbase org.apache.hadoop.hbase.mapreduce.ImportTsv \ 52 | -Dimporttsv.columns=HBASE_ROW_KEY,f:birth_date,f:first_name,f:last_name,f:gender,f:hire_date \ 53 | employees /tmp/employees_tsv/employees.txt 54 | 55 | -------------------------------------------------------------------------------- /sql/Lab6/prepare.sh: -------------------------------------------------------------------------------- 1 | cat < /tmp/db.sql 2 | CREATE DATABASE employees; 3 | EOF 4 | 5 | echo -e "\n==> Creating database employees in hive\n" 6 | 7 | beeline -u jdbc:hive2://localhost:10000/default -n student -f /tmp/db.sql 8 | 9 | 10 | for table in departments dept_emp dept_manager employees salaries titles; do 11 | 12 | echo -e "\n==> Importing table $table into hive\n" 13 | 14 | sudo -u hive sqoop import --connect jdbc:mysql://localhost/employees \ 15 | --username root --table $table \ 16 | --hive-import --hive-table employees.$table --direct --m 1; 17 | done 18 | -------------------------------------------------------------------------------- /sql/README-INSTRUCTOR.md: -------------------------------------------------------------------------------- 1 | # Masterclass: Instructor Notes 2 | =================================== 3 | 4 | ## Important 5 | ------------ 6 | 7 | - Provisioning can take up to an hour, so start in advance 8 | - Some clusters may fail (due to AWS issues) so make a few extra 9 | - TERMINATE clusters as soon as the session finishes! 10 | - Or prepare to pay ... 11 | 12 | 13 | ## Pre-reqs 14 | ----------- 15 | 16 | ### If not using our existing jump host: 17 | 18 | - Choose where you will trigger these builds from 19 | - A Cloud machine is recommended such that you can trigger the build without needing to keep your laptop open 20 | - Install the AWS CLI on that machine 21 | - http://docs.aws.amazon.com/cli/latest/userguide/installing.html 22 | - Configure AWS CLI: 23 | - `aws configure` 24 | - Details on creating credentials for that: http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-set-up.html 25 | - Hortonworkers 26 | - Login to EC2 from Okta (file a helpdesk with your manager on cc if it is not present) 27 | - Choose "EU (Ireland)" as the region (top right of page) 28 | - Create a User for yourself from the "IAM" page 29 | - Clone this repo 30 | - `git clone https://github.com/seanorama/masterclass` 31 | 32 | ### If using our jump host: 33 | 34 | - See internal documentation for the hostname & how to access 35 | - Proceed as usual below. 36 | 37 | ## How this works 38 | ----------------- 39 | 40 | There are several scripts used to make this work: 41 | 42 | For AWS CloudFormation: 43 | 44 | - cloudformation-generate-template.py: generates the AWS Cloudformation template 45 | - cloudformation.json: the CloudFormation template which deploys a cluster 46 | 47 | For managing multiple clusters (each takes variables for naming & the number of clusters): 48 | 49 | - clusters-create.sh: calls cloudformation to create clusters 50 | - clusters-report.sh: list cluster details 51 | - clusters-terminate.sh: calls cloudformation to terminate clusters 52 | 53 | If not using CloudFormation: 54 | 55 | - setup.sh 56 | - preps the deployed instances, installs Ambari, installs HDP, ... 57 | - triggered directly by CloudFormation to setup each cluster 58 | - if not using CloudFormation, you can trigger this directly on any CentOS/RHEL7 instance 59 | 60 | ## Deploy, report & terminate clusters on AWS 61 | ------------------------- 62 | 63 | 1. Check for conflicting/existing stacks (same name as what you plan to deploy): 64 | - Open the CloudFormat Web UI 65 | - Ensure 'EU - Ireland' is the region 66 | - Open the CloudFormation paged 67 | - Or with the command-line: 68 | 69 | ``` 70 | ./cloudformation-status.sh 71 | ``` 72 | 73 | 1. Open a 'screen' so the commands continue if you lose connectivity: 74 | 75 | ``` 76 | screen 77 | 78 | ## Note: If you get disconnected, SSH back to the host and execute: `screen -x` 79 | ``` 80 | 81 | 82 | 1. Set variables for the number of labs & their naming 83 | - the following will deploy only 1 cluster. 84 | - update 'lab_count' to the number of clusters you want 85 | 86 | ``` 87 | export lab_count=1 88 | export lab_first=1 89 | export lab_prefix="mc-sql" 90 | ``` 91 | 92 | 1. Provision your clusters 93 | 94 | ``` 95 | ./clusters-create.sh 96 | ``` 97 | 98 | 1. Check the build status 99 | - From the CloudFormation Web UI 100 | - Or from the command-line: 101 | 102 | ``` 103 | ./clusters-status.sh 104 | ``` 105 | 106 | 1. Once your clusters are ready, get list of clusters for pasting into Etherpad 107 | - ensure the same variables are used from above 108 | 109 | ``` 110 | ./clusters-report.sh 111 | ``` 112 | 113 | 1. Terminate clusters 114 | - ensure the same variables are used from above 115 | 116 | ``` 117 | ./clusters-terminate.sh 118 | ``` 119 | 120 | 1. Verify that all clusters are terminated 121 | - From the AWS CloudFormation Web UI 122 | - Or from the CLI 123 | 124 | ``` 125 | ./clusters-status.sh 126 | ``` 127 | 128 | ######## 129 | 130 | ## Running sessions 131 | 132 | It's recommended to use an "etherpad" to share: 133 | 134 | - the cluster details (from above) 135 | - instructions to students 136 | 137 | You can create your own, or use a hosted version such as TitanPad. You should create this account in advance. 138 | 139 | ######## 140 | 141 | ## Issues: Deployment 142 | 143 | #### Creation 144 | 145 | - Some instances will fail their creation and time out, being rolled back, this is a nature of deploying large volumes of instances 146 | - Those that fail should simply be manually deleted from the cloudformations web ui 147 | 148 | #### Deleting cloudformations 149 | 150 | - Occasionally cloudformations will fail to delete due to timing issues, in which case, it’s probably the VPC or InternetGateway, just switch to the VPC service window within the AWS site, delete the specific VPC that is being complained about in the cloudformation and then once the cloudformation delete has failed, retry the delete, deletion should complete this time. 151 | - Once you’ve done the VPC deletion you can also do an AWS CLI call instead: 152 | - `aws cloudformation delete-stack --stack-name ` 153 | 154 | #### AWS Website 155 | 156 | If you suddenly notice that your instances/cloudformations/etc have vanished from the AWS control panel, you may have to re-login (from Okta if a Hortonworker) 157 | 158 | 159 | 160 | ######## 161 | 162 | ## Issues: Other 163 | 164 | #### Run commands in bulk on all nodes 165 | 166 | * There are several options, such as pdsh, cssh, ... 167 | 168 | * Example using tmux-cssh *(which is installed on the jump box)* 169 | 170 | ``` 171 | ./clusters-report.sh | grep "^[0-9]" | xargs echo tmux-cssh -u masterclass 172 | ``` 173 | 174 | * After executing you will get a terminal with small windows to all of the clusters. 175 | * Anything you type will go to all hosts. 176 | 177 | #### Venue Internet blocks Ambari Server (port 8080) 178 | 179 | * Change Ambari to port 8081 180 | 181 | ``` 182 | export TERM=xterm 183 | echo "client.api.port=8081" | sudo tee -a /etc/ambari-server/conf/ambari.properties 184 | sudo ambari-server restart 185 | sudo ambari-agent restart 186 | ``` 187 | -------------------------------------------------------------------------------- /sql/change-port.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PORT=$1 4 | shift 5 | 6 | cat < /tmp/doit.sh 7 | #!/bin/bash 8 | 9 | sudo sh -c "echo 'client.api.port=${PORT}' >> /etc/ambari-server/conf/ambari.properties" 10 | sudo TERM=xterm ambari-server restart 11 | sudo TERM=xterm ambari-agent restart 12 | EOF 13 | 14 | for IP in $@; do 15 | echo "===> $IP <====" 16 | scp /tmp/doit.sh centos@$IP:/tmp 17 | ssh centos@$IP chmod 777 /tmp/doit.sh 18 | ssh -f centos@$IP '/tmp/doit.sh' 19 | echo "waiting ..." 20 | sleep 30 21 | echo "... done" 22 | done -------------------------------------------------------------------------------- /sql/cloudformation-status.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "########################################" 4 | echo "## Status of all CloudFormation stacks in this region:" 5 | aws cloudformation describe-stacks --query 'Stacks[*].[StackName, StackStatus]' --output text 6 | echo "########################################" 7 | -------------------------------------------------------------------------------- /sql/clusters-create.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ############################################################################## 3 | # Note the loops and pause reduce overloading the AWS region during 4 | # provisioning. Number of instances spawned per batch is controlled by the 5 | # variable "batchcount", first sleep is in between each instance spawn, second 6 | # sleep is between each batch of instances. 7 | ############################################################################## 8 | 9 | #### 10 | export lab_prefix=${lab_prefix:-mc-sql} 11 | lab_first=${lab_first:-1} 12 | lab_count=${lab_count:-1} 13 | 14 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 15 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 16 | 17 | echo "########################################################" 18 | echo "## Clusters to be created:" 19 | echo 20 | echo ${clusters} 21 | echo 22 | echo "########################################################" 23 | 24 | # quick check to ensure pre-reqs are setup 25 | while true; do 26 | read -p "Proceed with creating these ${lab_count} clusters? (y/n) " yn 27 | case $yn in 28 | [Yy]* ) break;; 29 | [Nn]* ) exit;; 30 | * ) echo "Please answer yes or no.";; 31 | esac 32 | done 33 | 34 | # Don't change this, change the batchcount variable towards the end of script 35 | batchcount=0 36 | 37 | # change the list of fruit to vary the number of clusters deployed 38 | for cluster in ${clusters} 39 | do 40 | ((batchcount++)) 41 | aws cloudformation create-stack --stack-name ${cluster} \ 42 | --capabilities CAPABILITY_IAM \ 43 | --template-body file://./cloudformation.json \ 44 | --parameters ParameterKey=KeyName,ParameterValue=secloud \ 45 | ParameterKey=OpenLocation,ParameterValue=0.0.0.0/0 46 | 47 | echo Initiated creation of $cluster cluster 48 | # this is the sleep interval between instances 49 | sleep 5 50 | ######################################################## 51 | # change the value below to increase/decrease batch size 52 | ######################################################## 53 | if [ $batchcount -eq 20 ] 54 | then 55 | # this is the sleep interval between batches of instances 56 | sleep 600 57 | # Don't change this, change the batchcount variable above 58 | batchcount=0 59 | fi 60 | done 61 | -------------------------------------------------------------------------------- /sql/clusters-report.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-mc-sql} 4 | lab_first=${lab_first:-1} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | for cluster in ${clusters} 11 | do 12 | echo "####################################################" 13 | echo "## The $cluster operated by: <1st PERSON NAME HERE>" 14 | echo "## and: <2nd PERSON NAME HERE>" 15 | aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PublicIpAddress]' --filters Name=instance-state-name,Values=running Name=tag:aws:cloudformation:stack-name,Values=$cluster Name=tag:aws:cloudformation:logical-id,Values=AmbariNode --output text 16 | echo "####################################################" 17 | done 18 | -------------------------------------------------------------------------------- /sql/clusters-status.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-mc-sql} 4 | lab_first=${lab_first:-1} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | echo "########################################" 11 | echo "## Status of the cluster's CloudFormation Stacks:" 12 | for cluster in ${clusters} 13 | do 14 | aws cloudformation describe-stacks \ 15 | --query 'Stacks[*].[StackName, StackStatus]' \ 16 | --stack-name ${cluster} \ 17 | --output text 18 | done 19 | echo "########################################" 20 | 21 | -------------------------------------------------------------------------------- /sql/clusters-terminate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export lab_prefix=${lab_prefix:-mc-sql} 4 | lab_first=${lab_first:-1} 5 | lab_count=${lab_count:-1} 6 | 7 | clusters=$(seq -w ${lab_first} $((lab_first+lab_count-1))) 8 | clusters=$(for cluster in ${clusters}; do echo ${lab_prefix}${cluster}; done) 9 | 10 | echo "########################################################" 11 | echo "## Clusters to be terminated:" 12 | echo 13 | echo ${clusters} 14 | echo 15 | echo "########################################################" 16 | 17 | # quick safety check to ensure termination is required 18 | while true; do 19 | read -p "Do you wish terminate all of these clusters? (y/n) " yn 20 | case $yn in 21 | [Yy]* ) break;; 22 | [Nn]* ) exit;; 23 | * ) echo "Please answer yes or no.";; 24 | esac 25 | done 26 | 27 | # Don't change this, change the batchcount test lower down the end of script 28 | batchcount=0 29 | 30 | # change the list of fruit to vary the number of clusters deleted 31 | for cluster in ${clusters} 32 | do 33 | ((batchcount++)) 34 | aws cloudformation delete-stack --stack-name $cluster 35 | echo Initiated deletion of $cluster cluster 36 | # this is the sleep interval between instances 37 | sleep 5 38 | ######################################################## 39 | # change the value below to increase/decrease batch size 40 | ######################################################## 41 | if [ $batchcount -eq 8 ] 42 | then 43 | # this is the sleep interval between batches of instances 44 | sleep 20 45 | # Don't change this, change the batchcount variable above 46 | batchcount=0 47 | fi 48 | done 49 | -------------------------------------------------------------------------------- /sql/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | aws cloudformation create-stack --stack-name sqltest \ 4 | --capabilities CAPABILITY_IAM \ 5 | --template-body file://./cloudformation.json \ 6 | --parameters ParameterKey=KeyName,ParameterValue=secloud \ 7 | ParameterKey=PostScript,ParameterValue='export cluster_name=$stack; curl -sSL https://raw.githubusercontent.com/seanorama/masterclass/master/sql/setup.sh | bash' 8 | 9 | 10 | lab_count=${lab_count:-2} 11 | lab_prefix=${lab_prefix:-test} 12 | 13 | ip=$(curl -s4 icanhazip.com)/32 14 | aws ec2 authorize-security-group-ingress \ 15 | --group-id sg-f915bc9d \ 16 | --ip-permissions \ 17 | '[{"IpProtocol": "icmp", "FromPort": -1, "ToPort": -1, "IpRanges": [{"CidrIp": "'${ip}'"}]}, 18 | {"IpProtocol": "tcp", "FromPort": 0, "ToPort": 65535, "IpRanges": [{"CidrIp": "'${ip}'"}]}, 19 | {"IpProtocol": "udp", "FromPort": 0, "ToPort": 65535, "IpRanges": [{"CidrIp": "'${ip}'"}]}]' 20 | 21 | 22 | m4.xlarge 23 | 24 | resources=$(aws ec2 run-instances \ 25 | --count 1 \ 26 | --image-id ami-25158352 \ 27 | --instance-type m4.xlarge \ 28 | --key-name masterclass \ 29 | --subnet-id subnet-7e49641b \ 30 | --associate-public-ip-address \ 31 | | jq -r ".Instances[0].InstanceId") 32 | 33 | # --user-data file://userdata.sh 34 | 35 | aws ec2 create-tags --resources "${resources}" ` --tags "Key=Name,Value=development_webserver" 36 | 37 | 38 | exit 39 | 40 | export lab_count=40 41 | export lab_first=101 42 | export lab_prefix=sto 43 | source ~/src/ambari-bootstrap/providers/google/create-google-hosts.sh 44 | create=true ~/src/ambari-bootstrap/providers/google/create-google-hosts.sh 45 | 46 | exit 47 | 48 | 49 | command="echo OK"; pdsh -w ${hosts_all} "${command}" 50 | 51 | command="df -h /" 52 | pdsh -w ${hosts_all} "${command}" 53 | 54 | 55 | read -r -d '' command < Trigger stopping of $SERVICE" 102 | stop_service $HOST $PORT $PASSWORD $NAME $SERVICE 103 | else 104 | echo "==> Trigger starting of $SERVICE" 105 | start_service $HOST $PORT $PASSWORD $NAME $SERVICE 106 | fi 107 | sleep 1 108 | LAST_SERVICE=$SERVICE 109 | done 110 | fi 111 | 112 | echo "" 113 | echo -n "Waiting " 114 | 115 | while [ $WAIT -eq 1 ]; do 116 | echo -n "." 117 | 118 | STATE=$(get_state $HOST $PORT $PASSWORD $NAME $LAST_SERVICE) 119 | if [ "$STATE" == "$CRIT" ]; then 120 | echo -e "\nDone" 121 | WAIT=0 122 | fi 123 | sleep 5 124 | done 125 | -------------------------------------------------------------------------------- /sql/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## for prepping a 1-node cluster for the security masterclass 4 | 5 | yum makecache 6 | yum -y -q install git epel-release ntpd 7 | yum -y -q install jq python-argparse python-configobj 8 | 9 | ## get mysql community on el/centos7 10 | el_version=$(sed 's/^.\+ release \([.0-9]\+\).*/\1/' /etc/redhat-release | cut -d. -f1) 11 | case ${el_version} in 12 | "6") 13 | true 14 | ;; 15 | "7") 16 | rpm -Uvh http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm 17 | ;; 18 | esac 19 | 20 | cd 21 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/extras/deploy/install-ambari-bootstrap.sh | bash 22 | source ~/ambari-bootstrap/extras/ambari_functions.sh 23 | 24 | #mypass=masterclass 25 | ${__dir}/deploy/prep-hosts.sh 26 | ${__dir}/../ambari-bootstrap.sh 27 | 28 | cd ${__dir}/../deploy/ 29 | 30 | cat << EOF > configuration-custom.json 31 | { 32 | "configurations" : { 33 | "hdfs-site": { 34 | "dfs.replication": "1", 35 | "dfs.datanode.data.dir" : "/mnt/dev/xvdb/dn", 36 | "dfs.namenode.name.dir" : "/mnt/dev/xvdb/nn" 37 | }, 38 | "yarn-site": { 39 | "yarn.scheduler.minimum-allocation-vcores": "1", 40 | "yarn.scheduler.maximum-allocation-vcores": "1", 41 | "yarn.scheduler.minimum-allocation-mb": "512", 42 | "yarn.scheduler.maximum-allocation-mb": "2048", 43 | "yarn.nodemanager.resource.memory-mb": "32768" 44 | }, 45 | "capacity-scheduler": { 46 | "yarn.scheduler.capacity.maximum-am-resource-percent": "0.25" 47 | }, 48 | "mapred-site": { 49 | "mapreduce.map.memory.mb": "2048", 50 | "mapreduce.reduce.memory.mb": "2048", 51 | "mapreduce.reduce.java.opts": "-Xmx1228m", 52 | "mapreduce.map.java.opts": "-Xmx1228m", 53 | "mapreduce.task.io.sort.mb": "859", 54 | "yarn.app.mapreduce.am.resource.mb" : "2048" 55 | }, 56 | "hive-site": { 57 | "hive.tez.container.size": "2048", 58 | "hive.auto.convert.join.noconditionaltask.size": "536870912" 59 | }, 60 | "tez-site": { 61 | "tez.task.resource.memory.mb": "2048", 62 | "tez.am.resource.memory.mb": "2048" 63 | }, 64 | "core-site": { 65 | "hadoop.proxyuser.HTTP.groups" : "users,hadoop-users", 66 | "hadoop.proxyuser.HTTP.hosts" : "*", 67 | "hadoop.proxyuser.hbase.groups" : "users,hadoop-users", 68 | "hadoop.proxyuser.hbase.hosts" : "*", 69 | "hadoop.proxyuser.hcat.groups" : "users,hadoop-users", 70 | "hadoop.proxyuser.hcat.hosts" : "*", 71 | "hadoop.proxyuser.hive.groups" : "users,hadoop-users", 72 | "hadoop.proxyuser.hive.hosts" : "*", 73 | "hadoop.proxyuser.knox.groups" : "users,hadoop-users", 74 | "hadoop.proxyuser.knox.hosts" : "*", 75 | "hadoop.proxyuser.oozie.groups" : "users", 76 | "hadoop.proxyuser.oozie.hosts" : "*", 77 | "hadoop.proxyuser.root.groups" : "users,hadoop-users", 78 | "hadoop.proxyuser.root.hosts" : "*" 79 | } 80 | } 81 | } 82 | EOF 83 | 84 | export ambari_services="KNOX YARN ZOOKEEPER TEZ PIG SLIDER MAPREDUCE2 HIVE HDFS HBASE SQOOP FLUME OOZIE SPARK" 85 | export host_count=skip 86 | ./deploy-recommended-cluster.bash 87 | sleep 5 88 | 89 | source ~/ambari-bootstrap/extras/ambari_functions.sh; ambari_change_pass admin admin BadPass#1 90 | echo "export ambari_pass=BadPass#1" >> ~/ambari-bootstrap/extras/.ambari.conf; chmod 660 ~/ambari-bootstrap/extras/.ambari.conf 91 | source ${__dir}/ambari_functions.sh 92 | ambari_configs 93 | ambari_wait_request_complete 1 94 | 95 | ## Generic setup 96 | usermod -a -G users ${USER} 97 | chkconfig mysqld on; service mysqld start 98 | ${__dir}/onboarding.sh 99 | config_proxyuser=true ${__dir}/ambari-views/create-views.sh 100 | ${__dir}/configs/proxyusers.sh 101 | 102 | cd /opt 103 | git clone https://github.com/seanorama/masterclass 104 | cd masterclass/sql 105 | ./labs-setup.sh 106 | 107 | exit 0 108 | --------------------------------------------------------------------------------