├── .gitignore ├── .travis.yml ├── .travis └── github_tags_push_deploy_key.enc ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── Makefile ├── README.md ├── bin ├── depend.sh ├── flag.sh ├── label-parent.sh ├── push.sh └── test.sh ├── etc └── compose │ ├── cdh5.12-hive │ └── docker-compose.yml │ ├── cdh5.15-hive │ └── docker-compose.yml │ ├── hdp2.5-hive │ └── docker-compose.yml │ ├── hdp2.6-hive │ └── docker-compose.yml │ ├── hive3.1-hive │ └── docker-compose.yml │ └── iop4.2-hive │ └── docker-compose.yml └── prestodb ├── cdh5.12-hive-kerberized ├── Dockerfile └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── container-executor.cfg │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── taskcontroller.cfg │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ ├── hive-site.xml │ │ │ └── hiveserver2-site.xml │ ├── krb5.conf │ └── supervisord.d │ │ └── kdc.conf │ └── var │ └── kerberos │ └── krb5kdc │ ├── kadm5.acl │ └── kdc.conf ├── cdh5.12-hive ├── Dockerfile └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── capacity-scheduler.xml │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ ├── hive-env.sh │ │ │ └── hive-site.xml │ ├── supervisord.conf │ └── supervisord.d │ │ ├── hdfs-datanode.conf │ │ ├── hdfs-namenode.conf │ │ ├── hive-metastore.conf │ │ ├── hive-server2.conf │ │ ├── mysql-metastore.conf │ │ ├── socks-proxy.conf │ │ ├── sshd.conf │ │ ├── yarn-nodemanager.conf │ │ └── yarn-resourcemanager.conf │ └── root │ └── setup.sh ├── cdh5.15-hive-kerberized ├── Dockerfile ├── README.md └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── container-executor.cfg │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── taskcontroller.cfg │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ ├── hive-site.xml │ │ │ └── hiveserver2-site.xml │ ├── krb5.conf │ └── supervisord.d │ │ └── kdc.conf │ └── var │ └── kerberos │ └── krb5kdc │ ├── kadm5.acl │ └── kdc.conf ├── cdh5.15-hive ├── Dockerfile ├── README.md └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── capacity-scheduler.xml │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ ├── hive-env.sh │ │ │ └── hive-site.xml │ ├── supervisord.conf │ └── supervisord.d │ │ ├── hdfs-datanode.conf │ │ ├── hdfs-namenode.conf │ │ ├── hive-metastore.conf │ │ ├── hive-server2.conf │ │ ├── mysql-metastore.conf │ │ ├── socks-proxy.conf │ │ ├── sshd.conf │ │ ├── yarn-nodemanager.conf │ │ └── yarn-resourcemanager.conf │ └── root │ └── setup.sh ├── centos6-oj8-openldap ├── Dockerfile ├── README.md └── files │ └── etc │ ├── openldap │ ├── certs │ │ ├── active-directory-certificate.crt │ │ ├── openldap-certificate.pem │ │ ├── private.pem │ │ └── serial.txt │ └── setup │ │ ├── createOU.ldif │ │ ├── ldap_disable_bind_anon.ldif │ │ ├── memberof.ldif │ │ ├── modify_server.ldif │ │ └── refint.ldif │ ├── supervisord.conf │ └── supervisord.d │ └── slapd.conf ├── centos6-oj8 ├── Dockerfile └── README.md ├── centos6-presto-admin-tests-build ├── Dockerfile └── README.md ├── centos6-presto-admin-tests-runtime ├── Dockerfile ├── README.md └── vagrant_insecure_rsa ├── centos7-oj8 ├── Dockerfile └── files │ ├── opt │ └── prestodb │ │ └── site-override.xslt │ └── usr │ └── local │ └── bin │ ├── apply-all-site-xml-overrides │ └── apply-site-xml-override ├── dns ├── Dockerfile └── README.md ├── hdp2.5-base ├── Dockerfile ├── README.md └── files │ └── etc │ └── supervisord.conf ├── hdp2.5-hive-kerberized ├── Dockerfile ├── README.md └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ ├── hive-site.xml │ │ │ └── hiveserver2-site.xml │ ├── krb5.conf │ └── supervisord.d │ │ └── kdc.conf │ └── var │ └── kerberos │ └── krb5kdc │ ├── kadm5.acl │ └── kdc.conf ├── hdp2.5-hive ├── Dockerfile ├── README.md └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── capacity-scheduler.xml │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ └── hive-site.xml │ ├── supervisord.d │ │ ├── hdfs-datanode.conf │ │ ├── hdfs-namenode.conf │ │ ├── hive-metastore.conf │ │ ├── hive-server2.conf │ │ ├── mapreduce-historyserver.conf │ │ ├── mysql-metastore.conf │ │ ├── socks-proxy.conf │ │ ├── sshd.conf │ │ ├── yarn-nodemanager.conf │ │ ├── yarn-resourcemanager.conf │ │ └── zookeeper.conf │ └── zookeeper │ │ └── conf │ │ ├── log4j.properties │ │ ├── zoo.cfg │ │ └── zookeeper-env.sh │ └── root │ └── setup.sh ├── hdp2.6-hive-kerberized ├── Dockerfile ├── README.md └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ ├── hive-site.xml │ │ │ └── hiveserver2-site.xml │ ├── krb5.conf │ └── supervisord.d │ │ └── kdc.conf │ └── var │ └── kerberos │ └── krb5kdc │ ├── kadm5-other.acl │ ├── kadm5.acl │ └── kdc.conf ├── hdp2.6-hive ├── Dockerfile ├── README.md └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── capacity-scheduler.xml │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ └── hive-site.xml │ ├── supervisord.conf │ └── supervisord.d │ │ ├── hdfs-datanode.conf │ │ ├── hdfs-namenode.conf │ │ ├── hive-metastore.conf │ │ ├── hive-server2.conf │ │ ├── mysql-metastore.conf │ │ ├── socks-proxy.conf │ │ ├── sshd.conf │ │ ├── yarn-nodemanager.conf │ │ └── yarn-resourcemanager.conf │ └── root │ └── setup.sh ├── hive3.1-hive ├── Dockerfile └── files │ ├── etc │ ├── hadoop-init.d │ │ ├── init-hdfs.sh │ │ └── set-aws-creds.sh │ ├── supervisord.conf │ └── supervisord.d │ │ ├── hdfs-datanode.conf │ │ ├── hdfs-namenode.conf │ │ ├── hive-metastore.conf │ │ ├── hive-server2.conf │ │ ├── mysql-metastore.conf │ │ ├── socks-proxy.conf │ │ └── sshd.conf │ ├── opt │ ├── hadoop │ │ └── etc │ │ │ └── hadoop │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ └── hdfs-site.xml │ └── hive │ │ └── conf │ │ ├── hive-env.sh │ │ └── hive-site.xml │ └── root │ ├── entrypoint.sh │ └── setup.sh ├── iop4.2-base ├── Dockerfile ├── README.md └── files │ └── etc │ └── supervisord.conf ├── iop4.2-hive ├── Dockerfile ├── README.md ├── capabilities.txt └── files │ ├── etc │ ├── hadoop │ │ └── conf │ │ │ ├── capacity-scheduler.xml │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hdfs-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ ├── hive │ │ └── conf │ │ │ └── hive-site.xml │ └── supervisord.d │ │ ├── hdfs-datanode.conf │ │ ├── hdfs-namenode.conf │ │ ├── hive-metastore.conf │ │ ├── hive-server2.conf │ │ ├── mysql-metastore.conf │ │ ├── socks-proxy.conf │ │ ├── sshd.conf │ │ ├── yarn-nodemanager.conf │ │ └── yarn-resourcemanager.conf │ └── root │ └── setup.sh └── spark ├── build-images.sh ├── spark-base ├── Dockerfile └── README.md ├── spark-master ├── Dockerfile ├── README.md └── start-master.sh ├── spark-submit ├── Dockerfile ├── README.md └── spark-submit.sh └── spark-worker ├── Dockerfile ├── README.md └── start-worker.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | .DS_Store 5 | .idea 6 | depends 7 | flags 8 | graphviz 9 | dependency_graph.svg 10 | *.swp 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: generic 2 | dist: trusty 3 | 4 | env: 5 | - IMAGE_TO_BUILD=prestodb/hdp2.5-hive-kerberized IMAGE_TO_TEST=hdp2.5-hive 6 | - IMAGE_TO_BUILD=prestodb/hdp2.6-hive-kerberized IMAGE_TO_TEST=hdp2.6-hive 7 | - IMAGE_TO_BUILD=prestodb/cdh5.12-hive-kerberized IMAGE_TO_TEST=cdh5.12-hive 8 | - IMAGE_TO_BUILD=prestodb/cdh5.15-hive-kerberized IMAGE_TO_TEST=cdh5.15-hive 9 | - IMAGE_TO_BUILD=prestodb/iop4.2-hive IMAGE_TO_TEST=iop4.2-hive 10 | - IMAGE_TO_BUILD=prestodb/hive3.1-hive IMAGE_TO_TEST=hive3.1-hive 11 | 12 | # Prevent duplicate builds on tag pushes. 13 | # See https://github.com/mockito/mockito/commit/500519aac4cc13770aa47c1eb7d28e905c27e7aa 14 | branches: 15 | except: 16 | - /^release.*/ 17 | - /^latest-snapshot.*/ 18 | 19 | services: 20 | - docker 21 | 22 | install: make ${IMAGE_TO_BUILD} 23 | 24 | script: make test IMAGE_TO_TEST=${IMAGE_TO_TEST} 25 | 26 | before_deploy: 27 | # decrypt the github deploy key 28 | - openssl aes-256-cbc -K $encrypted_cd2beb64619c_key -iv $encrypted_cd2beb64619c_iv 29 | -in .travis/github_tags_push_deploy_key.enc -out .travis/github_tags_push_deploy_key -d 30 | - chmod 600 .travis/github_tags_push_deploy_key 31 | - eval "$(ssh-agent -s)" 32 | - ssh-add .travis/github_tags_push_deploy_key 33 | - docker login --username=${DOCKERHUB_LOGIN} --password=${DOCKERHUB_PASSWORD} 34 | 35 | # Publish snapshots to Dockerhub for commits tagged `latest-snapshot-*` 36 | deploy: 37 | on: 38 | all_branches: true 39 | condition: $( git tag --points-at HEAD | grep '^latest-snapshot' ) != '' 40 | skip_cleanup: true 41 | provider: script 42 | script: make snapshot 43 | -------------------------------------------------------------------------------- /.travis/github_tags_push_deploy_key.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prestodb/docker-images/3f7be0efd4d420065ed5f813f32267b453ec6174/.travis/github_tags_push_deploy_key.enc -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.fb.com/codeofconduct) so that you can understand what actions will and will not be tolerated. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to docker-images 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `master`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need to do this once, so if you've done this for one repository in the [prestodb](https://github.com/prestodb) organization, you're good to go. If you are submitting a pull request for the first time, the communitybridge-easycla bot will notify you if you haven't signed, and will provide you with a link. If you are contributing on behalf of a company, you might want to let the person who manages your corporate CLA whitelist know they will be receiving a request from you. 17 | 18 | ## Issues 19 | We use GitHub issues to track public bugs. Please ensure your description is 20 | clear and has sufficient instructions to be able to reproduce the issue. 21 | 22 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 23 | disclosure of security bugs. In those cases, please go through the process 24 | outlined on that page and do not file a public issue. 25 | 26 | ## License 27 | By contributing to docker-images, you agree that your contributions will be licensed 28 | under the LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /bin/flag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | usage() { 4 | echo "$0 {target image}" >&2 5 | } 6 | 7 | # 8 | # This doesn't handle ARGs with a default supplied in the Dockerfile. 9 | # Feel free to add that functionality if needed. For now, YAGNI. 10 | # 11 | find_args() { 12 | local target_image=$(dirname "$target_dockerfile") 13 | awk -v image="$target_image" ' 14 | BEGIN { 15 | ARG_PATTERN = "^\\s*ARG"; 16 | print "DBFLAGS_" image " :="; 17 | } 18 | 19 | $1 == "ARG" { 20 | key = $2; 21 | print "DBFLAGS_" image " += --build-arg " key "=$(" key ")"; 22 | }' "$1" 23 | } 24 | 25 | if [ $# -lt 1 ]; then 26 | usage 27 | exit 1 28 | fi 29 | 30 | target_dockerfile=$1 31 | 32 | find_args "$target_dockerfile" 33 | -------------------------------------------------------------------------------- /bin/label-parent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Requires that the top layer is a LABEL layer. If it is, return the checksum 5 | # of its parent layer and exit 0. If the top layer isn't a LABEL layer, exit 1. 6 | # 7 | docker history --no-trunc --format "{{.ID}}\t{{.CreatedBy}}" "$1" | awk ' 8 | BEGIN { 9 | ec = 1; 10 | } 11 | $5 == "LABEL" && NR == 1 { 12 | getline; 13 | ec = 0; 14 | exit; 15 | } 16 | END { 17 | if (ec == 0) { 18 | print $1 19 | } 20 | exit ec 21 | }' 22 | -------------------------------------------------------------------------------- /bin/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | while [ -n "$1" ]; do 4 | docker push "$1" 5 | shift 6 | done 7 | -------------------------------------------------------------------------------- /etc/compose/cdh5.12-hive/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | services: 3 | hadoop-master: 4 | hostname: hadoop-master 5 | image: prestodb/cdh5.12-hive:latest 6 | 7 | -------------------------------------------------------------------------------- /etc/compose/cdh5.15-hive/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | services: 3 | hadoop-master: 4 | hostname: hadoop-master 5 | image: prestodb/cdh5.15-hive:latest 6 | 7 | -------------------------------------------------------------------------------- /etc/compose/hdp2.5-hive/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | services: 3 | hadoop-master: 4 | hostname: hadoop-master 5 | image: prestodb/hdp2.5-hive:latest 6 | 7 | -------------------------------------------------------------------------------- /etc/compose/hdp2.6-hive/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | services: 3 | hadoop-master: 4 | hostname: hadoop-master 5 | image: prestodb/hdp2.6-hive:latest 6 | 7 | -------------------------------------------------------------------------------- /etc/compose/hive3.1-hive/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | services: 3 | hadoop-master: 4 | hostname: hadoop-master 5 | image: prestodb/hive3.1-hive:latest 6 | -------------------------------------------------------------------------------- /etc/compose/iop4.2-hive/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.0' 2 | services: 3 | hadoop-master: 4 | hostname: hadoop-master 5 | image: prestodb/iop4.2-hive:latest 6 | 7 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/container-executor.cfg: -------------------------------------------------------------------------------- 1 | yarn.nodemanager.local-dirs=/var/lib/hadoop-yarn/cache/yarn/nm-local-dir 2 | yarn.nodemanager.linux-container-executor.group=yarn 3 | yarn.nodemanager.log-dirs=/var/log/hadoop-yarn/containers 4 | banned.users=yarn,mapred,bin 5 | min.user.id=400 6 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | 30 | dfs.block.access.token.enable 31 | true 32 | 33 | 34 | 35 | 36 | dfs.namenode.keytab.file 37 | /etc/hadoop/conf/hdfs.keytab 38 | 39 | 40 | dfs.namenode.kerberos.principal 41 | hdfs/hadoop-master@LABS.TERADATA.COM 42 | 43 | 44 | dfs.namenode.kerberos.internal.spnego.principal 45 | HTTP/hadoop-master@LABS.TERADATA.COM 46 | 47 | 48 | 49 | 50 | dfs.secondary.namenode.keytab.file 51 | /etc/hadoop/conf/hdfs.keytab 52 | 53 | 54 | dfs.secondary.namenode.kerberos.principal 55 | hdfs/hadoop-master@LABS.TERADATA.COM 56 | 57 | 58 | dfs.secondary.namenode.kerberos.internal.spnego.principal 59 | HTTP/hadoop-master@LABS.TERADATA.COM 60 | 61 | 62 | 63 | 64 | dfs.datanode.keytab.file 65 | /etc/hadoop/conf/hdfs.keytab 66 | 67 | 68 | dfs.datanode.kerberos.principal 69 | hdfs/hadoop-master@LABS.TERADATA.COM 70 | 71 | 72 | 73 | 74 | dfs.webhdfs.enabled 75 | true 76 | 77 | 78 | 79 | 80 | dfs.web.authentication.kerberos.principal 81 | HTTP/hadoop-master@LABS.TERADATA.COM 82 | 83 | 84 | 85 | dfs.web.authentication.kerberos.keytab 86 | /etc/hadoop/conf/HTTP.keytab 87 | 88 | 89 | 90 | ignore.secure.ports.for.testing 91 | true 92 | 93 | 94 | 95 | dfs.http.policy 96 | HTTP_ONLY 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.jobhistory.address 27 | hadoop-master:10020 28 | 29 | 30 | 31 | mapreduce.jobhistory.webapp.address 32 | hadoop-master:19888 33 | 34 | 35 | 36 | To set the value of tmp directory for map and reduce tasks. 37 | mapreduce.task.tmp.dir 38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 39 | 40 | 41 | 42 | 43 | mapreduce.jobhistory.keytab 44 | /etc/hadoop/conf/mapred.keytab 45 | 46 | 47 | 48 | mapreduce.jobhistory.principal 49 | mapred/hadoop-master@LABS.TERADATA.COM 50 | 51 | 52 | 53 | 54 | mapreduce.jobtracker.kerberos.principal 55 | mapred/hadoop-master@LABS.TERADATA.COM 56 | 57 | 58 | 59 | mapreduce.jobtracker.keytab.file 60 | /etc/hadoop/conf/mapred.keytab 61 | 62 | 63 | 64 | 65 | mapreduce.tasktracker.kerberos.principal 66 | mapred/hadoop-master@LABS.TERADATA.COM 67 | 68 | 69 | 70 | mapreduce.tasktracker.keytab.file 71 | /etc/hadoop/conf/mapred.keytab 72 | 73 | 74 | 75 | mapred.task.tracker.task-controller 76 | org.apache.hadoop.mapred.LinuxTaskController 77 | 78 | 79 | 80 | mapreduce.tasktracker.group 81 | mapred 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/taskcontroller.cfg: -------------------------------------------------------------------------------- 1 | hadoop.log.dir=/var/log/hadoop-mapreduce 2 | mapreduce.tasktracker.group=mapred 3 | banned.users=mapred,bin 4 | min.user.id=400 5 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | hive.security.authenticator.manager 23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator 24 | 25 | 26 | 27 | 28 | hive.security.authorization.manager 29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory 30 | SQL standards based Hive authorization 31 | 32 | 33 | 34 | hive.security.authorization.enabled 35 | true 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/krb5.conf: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = LABS.TERADATA.COM 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | LABS.TERADATA.COM = { 16 | kdc = hadoop-master 17 | admin_server = hadoop-master 18 | } 19 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/etc/supervisord.d/kdc.conf: -------------------------------------------------------------------------------- 1 | [program:krb5kdc] 2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -r LABS.TERADATA.COM -P /var/run/krb5kdc.pid -n" 3 | autostart=true 4 | autorestart=true 5 | redirect_stderr=true 6 | stdout_logfile=/dev/stdout 7 | stdout_logfile_maxbytes=0 8 | 9 | [program:kadmind] 10 | command=/bin/bash -c "exec /usr/sbin/kadmind -r LABS.TERADATA.COM -P /var/run/kadmind.pid -nofork" 11 | autostart=true 12 | autorestart=true 13 | redirect_stderr=true 14 | stdout_logfile=/dev/stdout 15 | stdout_logfile_maxbytes=0 16 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl: -------------------------------------------------------------------------------- 1 | */admin@LABS.TERADATA.COM * 2 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | LABS.TERADATA.COM = { 7 | #master_key_type = aes256-cts 8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 9 | dict_file = /usr/share/dict/words 10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 12 | } 13 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/centos6-oj8:unlabelled 14 | MAINTAINER Presto community 15 | 16 | # Change default timezone 17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone 18 | 19 | # Setup CDH repo, pin the CDH distribution to a concrete version 20 | RUN wget -nv https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo -P /etc/yum.repos.d \ 21 | && rpm --import https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera \ 22 | && sed -i '/^baseurl=/c\baseurl=https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.12.2/' /etc/yum.repos.d/cloudera-cdh5.repo 23 | 24 | # Install Hadoop, Hive (w/ MySQL) 25 | RUN \ 26 | set -xeu && \ 27 | yum install -y \ 28 | hadoop-hdfs-namenode \ 29 | hadoop-hdfs-secondarynamenode \ 30 | hadoop-hdfs-datanode \ 31 | \ 32 | hadoop-mapreduce \ 33 | \ 34 | hadoop-yarn-resourcemanager \ 35 | hadoop-yarn-nodemanager \ 36 | \ 37 | hadoop-client \ 38 | hadoop-conf-pseudo \ 39 | \ 40 | hive \ 41 | hive-metastore \ 42 | hive-server2 \ 43 | \ 44 | mysql-server mysql-connector-java \ 45 | \ 46 | # Cleanup 47 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 48 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/mysql-connector-java.jar 49 | 50 | # Copy configuration files 51 | COPY ./files / 52 | 53 | # Run setup script 54 | RUN /root/setup.sh \ 55 | && rm -rf /tmp/* /var/tmp/* 56 | 57 | # Setup sock proxy 58 | RUN yum install -y openssh openssh-clients openssh-server && yum -y clean all 59 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ 60 | && ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key \ 61 | && ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key \ 62 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 63 | RUN chmod 755 /root && chmod 700 /root/.ssh 64 | RUN passwd --unlock root 65 | 66 | # HDFS ports 67 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 68 | 69 | # YARN ports 70 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 71 | 72 | # HIVE port 73 | EXPOSE 9083 10000 74 | 75 | # SOCKS port 76 | EXPOSE 1180 77 | 78 | CMD supervisord -c /etc/supervisord.conf 79 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://hadoop-master:9000 23 | 24 | 25 | 26 | 27 | hadoop.proxyuser.oozie.hosts 28 | * 29 | 30 | 31 | hadoop.proxyuser.oozie.groups 32 | * 33 | 34 | 35 | 36 | 37 | hadoop.proxyuser.httpfs.hosts 38 | * 39 | 40 | 41 | hadoop.proxyuser.httpfs.groups 42 | * 43 | 44 | 45 | 46 | 47 | hadoop.proxyuser.llama.hosts 48 | * 49 | 50 | 51 | hadoop.proxyuser.llama.groups 52 | * 53 | 54 | 55 | 56 | 57 | hadoop.proxyuser.hue.hosts 58 | * 59 | 60 | 61 | hadoop.proxyuser.hue.groups 62 | * 63 | 64 | 65 | 66 | 67 | hadoop.proxyuser.mapred.hosts 68 | * 69 | 70 | 71 | hadoop.proxyuser.mapred.groups 72 | * 73 | 74 | 75 | 76 | 77 | hadoop.proxyuser.hive.hosts 78 | * 79 | 80 | 81 | 82 | hadoop.proxyuser.hive.groups 83 | * 84 | 85 | 86 | 87 | 88 | hadoop.proxyuser.hdfs.groups 89 | * 90 | 91 | 92 | 93 | hadoop.proxyuser.hdfs.hosts 94 | * 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hadoop/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Set Hadoop-specific environment variables here. 17 | # Forcing YARN-based mapreduce implementaion. 18 | # Make sure to comment out if you want to go back to the default or 19 | # if you want this to be tweakable on a per-user basis 20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce 21 | 22 | # The maximum amount of heap to use, in MB. Default is 1000. 23 | export HADOOP_HEAPSIZE=512 24 | 25 | # Extra Java runtime options. Empty by default. 26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m" 27 | export YARN_OPTS="$YARN_OPTS -Xmx256m" 28 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.jobhistory.address 27 | hadoop-master:10020 28 | 29 | 30 | 31 | mapreduce.jobhistory.webapp.address 32 | hadoop-master:19888 33 | 34 | 35 | 36 | To set the value of tmp directory for map and reduce tasks. 37 | mapreduce.task.tmp.dir 38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hadoop/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | yarn.nodemanager.aux-services 22 | mapreduce_shuffle 23 | 24 | 25 | 26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 27 | org.apache.hadoop.mapred.ShuffleHandler 28 | 29 | 30 | 31 | yarn.log-aggregation-enable 32 | true 33 | 34 | 35 | 36 | yarn.dispatcher.exit-on-error 37 | true 38 | 39 | 40 | 41 | List of directories to store localized files in. 42 | yarn.nodemanager.local-dirs 43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 44 | 45 | 46 | 47 | Where to store container logs. 48 | yarn.nodemanager.log-dirs 49 | /var/log/hadoop-yarn/containers 50 | 51 | 52 | 53 | Where to aggregate logs to. 54 | yarn.nodemanager.remote-app-log-dir 55 | /var/log/hadoop-yarn/apps 56 | 57 | 58 | 59 | Classpath for typical applications. 60 | yarn.application.classpath 61 | 62 | /etc/hadoop/conf, 63 | /usr/lib/hadoop/*, 64 | /usr/lib/hadoop/lib/*, 65 | /usr/lib/hadoop-hdfs/*, 66 | /usr/lib/hadoop-hdfs/lib/*, 67 | /usr/lib/hadoop-yarn/*, 68 | /usr/lib/hadoop-yarn/lib/*, 69 | /usr/lib/hadoop-mapreduce/*, 70 | /usr/lib/hadoop-mapreduce/lib/* 71 | 72 | 73 | 74 | 75 | yarn.resourcemanager.hostname 76 | hadoop-master 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hive/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | export HADOOP_OPTS="$HADOOP_OPTS -Dhive.root.logger=INFO,console" 2 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/hive/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.security.authorization.createtable.owner.grants 58 | ALL 59 | The set of privileges automatically granted to the owner whenever a table gets created. 60 | 61 | 62 | 63 | hive.users.in.admin.role 64 | hdfs,hive 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | command=hive --service metastore 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-metastore.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-nodemanager] 2 | command=yarn nodemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-nodemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/etc/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-resourcemanager] 2 | command=yarn resourcemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-resourcemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.12-hive/files/root/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 0 make file system hostname resolvable 4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts 5 | 6 | # 1 format namenode 7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 8 | 9 | # workaround for 'could not open session' bug as suggested here: 10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 11 | rm -f /etc/security/limits.d/hdfs.conf 12 | su -c "echo 'N' | hdfs namenode -format" hdfs 13 | 14 | # 2 start hdfs 15 | su -c "hdfs datanode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-datanode.log" hdfs& 16 | su -c "hdfs namenode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-namenode.log" hdfs& 17 | 18 | # 3 wait for process starting 19 | sleep 10 20 | 21 | # remove a broken symlink created by cdh installer so that init-hdfs.sh does no blow up on it 22 | # (hbase-annotations.jar seems not needed in our case) 23 | rm /usr/lib/hive/lib/hbase-annotations.jar 24 | 25 | # 4 exec cloudera hdfs init script 26 | /usr/lib/hadoop/libexec/init-hdfs.sh 27 | 28 | # 5 init hive directories 29 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 30 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 31 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 32 | 33 | # 6 stop hdfs 34 | killall java 35 | 36 | # 7 setup metastore 37 | mysql_install_db 38 | 39 | /usr/bin/mysqld_safe & 40 | sleep 10s 41 | 42 | cd /usr/lib/hive/scripts/metastore/upgrade/mysql/ 43 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 44 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.1.0.mysql.sql;" | mysql 45 | /usr/bin/mysqladmin -u root password 'root' 46 | 47 | killall mysqld 48 | sleep 10s 49 | mkdir /var/log/mysql/ 50 | chown mysql:mysql /var/log/mysql/ -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/README.md: -------------------------------------------------------------------------------- 1 | # cdh5.15-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/cdh5.15-hive-kerberized.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/cdh5.15-hive-kerberized 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/cdh5.15-hive-kerberized.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/cdh5.15-hive-kerberized 7 | 8 | A pseudo-distributed Hadoop image with Kerberos authentication enabled, based on the CDH distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/container-executor.cfg: -------------------------------------------------------------------------------- 1 | yarn.nodemanager.local-dirs=/var/lib/hadoop-yarn/cache/yarn/nm-local-dir 2 | yarn.nodemanager.linux-container-executor.group=yarn 3 | yarn.nodemanager.log-dirs=/var/log/hadoop-yarn/containers 4 | banned.users=yarn,mapred,bin 5 | min.user.id=400 6 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | 30 | dfs.block.access.token.enable 31 | true 32 | 33 | 34 | 35 | 36 | dfs.namenode.keytab.file 37 | /etc/hadoop/conf/hdfs.keytab 38 | 39 | 40 | dfs.namenode.kerberos.principal 41 | hdfs/hadoop-master@LABS.TERADATA.COM 42 | 43 | 44 | dfs.namenode.kerberos.internal.spnego.principal 45 | HTTP/hadoop-master@LABS.TERADATA.COM 46 | 47 | 48 | 49 | 50 | dfs.secondary.namenode.keytab.file 51 | /etc/hadoop/conf/hdfs.keytab 52 | 53 | 54 | dfs.secondary.namenode.kerberos.principal 55 | hdfs/hadoop-master@LABS.TERADATA.COM 56 | 57 | 58 | dfs.secondary.namenode.kerberos.internal.spnego.principal 59 | HTTP/hadoop-master@LABS.TERADATA.COM 60 | 61 | 62 | 63 | 64 | dfs.datanode.keytab.file 65 | /etc/hadoop/conf/hdfs.keytab 66 | 67 | 68 | dfs.datanode.kerberos.principal 69 | hdfs/hadoop-master@LABS.TERADATA.COM 70 | 71 | 72 | 73 | 74 | dfs.webhdfs.enabled 75 | true 76 | 77 | 78 | 79 | 80 | dfs.web.authentication.kerberos.principal 81 | HTTP/hadoop-master@LABS.TERADATA.COM 82 | 83 | 84 | 85 | dfs.web.authentication.kerberos.keytab 86 | /etc/hadoop/conf/HTTP.keytab 87 | 88 | 89 | 90 | ignore.secure.ports.for.testing 91 | true 92 | 93 | 94 | 95 | dfs.http.policy 96 | HTTP_ONLY 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.jobhistory.address 27 | hadoop-master:10020 28 | 29 | 30 | 31 | mapreduce.jobhistory.webapp.address 32 | hadoop-master:19888 33 | 34 | 35 | 36 | To set the value of tmp directory for map and reduce tasks. 37 | mapreduce.task.tmp.dir 38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 39 | 40 | 41 | 42 | 43 | mapreduce.jobhistory.keytab 44 | /etc/hadoop/conf/mapred.keytab 45 | 46 | 47 | 48 | mapreduce.jobhistory.principal 49 | mapred/hadoop-master@LABS.TERADATA.COM 50 | 51 | 52 | 53 | 54 | mapreduce.jobtracker.kerberos.principal 55 | mapred/hadoop-master@LABS.TERADATA.COM 56 | 57 | 58 | 59 | mapreduce.jobtracker.keytab.file 60 | /etc/hadoop/conf/mapred.keytab 61 | 62 | 63 | 64 | 65 | mapreduce.tasktracker.kerberos.principal 66 | mapred/hadoop-master@LABS.TERADATA.COM 67 | 68 | 69 | 70 | mapreduce.tasktracker.keytab.file 71 | /etc/hadoop/conf/mapred.keytab 72 | 73 | 74 | 75 | mapred.task.tracker.task-controller 76 | org.apache.hadoop.mapred.LinuxTaskController 77 | 78 | 79 | 80 | mapreduce.tasktracker.group 81 | mapred 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/taskcontroller.cfg: -------------------------------------------------------------------------------- 1 | hadoop.log.dir=/var/log/hadoop-mapreduce 2 | mapreduce.tasktracker.group=mapred 3 | banned.users=mapred,bin 4 | min.user.id=400 5 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | hive.security.authenticator.manager 23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator 24 | 25 | 26 | 27 | 28 | hive.security.authorization.manager 29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory 30 | SQL standards based Hive authorization 31 | 32 | 33 | 34 | hive.security.authorization.enabled 35 | true 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/krb5.conf: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = LABS.TERADATA.COM 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | LABS.TERADATA.COM = { 16 | kdc = hadoop-master 17 | admin_server = hadoop-master 18 | } 19 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/etc/supervisord.d/kdc.conf: -------------------------------------------------------------------------------- 1 | [program:krb5kdc] 2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -r LABS.TERADATA.COM -P /var/run/krb5kdc.pid -n" 3 | autostart=true 4 | autorestart=true 5 | redirect_stderr=true 6 | stdout_logfile=/dev/stdout 7 | stdout_logfile_maxbytes=0 8 | 9 | [program:kadmind] 10 | command=/bin/bash -c "exec /usr/sbin/kadmind -r LABS.TERADATA.COM -P /var/run/kadmind.pid -nofork" 11 | autostart=true 12 | autorestart=true 13 | redirect_stderr=true 14 | stdout_logfile=/dev/stdout 15 | stdout_logfile_maxbytes=0 16 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl: -------------------------------------------------------------------------------- 1 | */admin@LABS.TERADATA.COM * 2 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | LABS.TERADATA.COM = { 7 | #master_key_type = aes256-cts 8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 9 | dict_file = /usr/share/dict/words 10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 12 | } 13 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/centos6-oj8:unlabelled 14 | MAINTAINER Presto community 15 | 16 | # Change default timezone 17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone 18 | 19 | # Setup CDH repo, pin the CDH distribution to a concrete version 20 | RUN wget -nv https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo -P /etc/yum.repos.d \ 21 | && rpm --import https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera \ 22 | && sed -i '/^baseurl=/c\baseurl=https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.15.1/' /etc/yum.repos.d/cloudera-cdh5.repo 23 | 24 | # Install Hadoop, Hive (w/ MySQL) 25 | RUN \ 26 | set -xeu && \ 27 | yum install -y \ 28 | hadoop-hdfs-namenode \ 29 | hadoop-hdfs-secondarynamenode \ 30 | hadoop-hdfs-datanode \ 31 | \ 32 | hadoop-mapreduce \ 33 | \ 34 | hadoop-yarn-resourcemanager \ 35 | hadoop-yarn-nodemanager \ 36 | \ 37 | hadoop-client \ 38 | hadoop-conf-pseudo \ 39 | \ 40 | hive \ 41 | hive-metastore \ 42 | hive-server2 \ 43 | \ 44 | mysql-server mysql-connector-java \ 45 | \ 46 | # Cleanup 47 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 48 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/mysql-connector-java.jar 49 | 50 | # Copy configuration files 51 | COPY ./files / 52 | 53 | # Run setup script 54 | RUN /root/setup.sh \ 55 | && rm -rf /tmp/* /var/tmp/* 56 | 57 | # Setup sock proxy 58 | RUN yum install -y openssh openssh-clients openssh-server && yum -y clean all 59 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ 60 | && ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key \ 61 | && ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key \ 62 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 63 | RUN chmod 755 /root && chmod 700 /root/.ssh 64 | RUN passwd --unlock root 65 | 66 | # HDFS ports 67 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 68 | 69 | # YARN ports 70 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 71 | 72 | # HIVE port 73 | EXPOSE 9083 10000 74 | 75 | # SOCKS port 76 | EXPOSE 1180 77 | 78 | CMD supervisord -c /etc/supervisord.conf 79 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/README.md: -------------------------------------------------------------------------------- 1 | # cdh5.15-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/cdh5.15-hive.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/cdh5.15-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/cdh5.15-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/cdh5.15-hive 7 | 8 | Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. 9 | For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | ## Run 14 | 15 | ``` 16 | $ docker run -d --name hadoop-master -h hadoop-master prestodb/cdh5.15-hive 17 | ``` 18 | 19 | ## Oracle license 20 | 21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 23 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://hadoop-master:9000 23 | 24 | 25 | 26 | 27 | hadoop.proxyuser.oozie.hosts 28 | * 29 | 30 | 31 | hadoop.proxyuser.oozie.groups 32 | * 33 | 34 | 35 | 36 | 37 | hadoop.proxyuser.httpfs.hosts 38 | * 39 | 40 | 41 | hadoop.proxyuser.httpfs.groups 42 | * 43 | 44 | 45 | 46 | 47 | hadoop.proxyuser.llama.hosts 48 | * 49 | 50 | 51 | hadoop.proxyuser.llama.groups 52 | * 53 | 54 | 55 | 56 | 57 | hadoop.proxyuser.hue.hosts 58 | * 59 | 60 | 61 | hadoop.proxyuser.hue.groups 62 | * 63 | 64 | 65 | 66 | 67 | hadoop.proxyuser.mapred.hosts 68 | * 69 | 70 | 71 | hadoop.proxyuser.mapred.groups 72 | * 73 | 74 | 75 | 76 | 77 | hadoop.proxyuser.hive.hosts 78 | * 79 | 80 | 81 | 82 | hadoop.proxyuser.hive.groups 83 | * 84 | 85 | 86 | 87 | 88 | hadoop.proxyuser.hdfs.groups 89 | * 90 | 91 | 92 | 93 | hadoop.proxyuser.hdfs.hosts 94 | * 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hadoop/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Set Hadoop-specific environment variables here. 17 | # Forcing YARN-based mapreduce implementaion. 18 | # Make sure to comment out if you want to go back to the default or 19 | # if you want this to be tweakable on a per-user basis 20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce 21 | 22 | # The maximum amount of heap to use, in MB. Default is 1000. 23 | export HADOOP_HEAPSIZE=512 24 | 25 | # Extra Java runtime options. Empty by default. 26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m" 27 | export YARN_OPTS="$YARN_OPTS -Xmx256m" 28 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.jobhistory.address 27 | hadoop-master:10020 28 | 29 | 30 | 31 | mapreduce.jobhistory.webapp.address 32 | hadoop-master:19888 33 | 34 | 35 | 36 | To set the value of tmp directory for map and reduce tasks. 37 | mapreduce.task.tmp.dir 38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hadoop/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | yarn.nodemanager.aux-services 22 | mapreduce_shuffle 23 | 24 | 25 | 26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 27 | org.apache.hadoop.mapred.ShuffleHandler 28 | 29 | 30 | 31 | yarn.log-aggregation-enable 32 | true 33 | 34 | 35 | 36 | yarn.dispatcher.exit-on-error 37 | true 38 | 39 | 40 | 41 | List of directories to store localized files in. 42 | yarn.nodemanager.local-dirs 43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 44 | 45 | 46 | 47 | Where to store container logs. 48 | yarn.nodemanager.log-dirs 49 | /var/log/hadoop-yarn/containers 50 | 51 | 52 | 53 | Where to aggregate logs to. 54 | yarn.nodemanager.remote-app-log-dir 55 | /var/log/hadoop-yarn/apps 56 | 57 | 58 | 59 | Classpath for typical applications. 60 | yarn.application.classpath 61 | 62 | /etc/hadoop/conf, 63 | /usr/lib/hadoop/*, 64 | /usr/lib/hadoop/lib/*, 65 | /usr/lib/hadoop-hdfs/*, 66 | /usr/lib/hadoop-hdfs/lib/*, 67 | /usr/lib/hadoop-yarn/*, 68 | /usr/lib/hadoop-yarn/lib/*, 69 | /usr/lib/hadoop-mapreduce/*, 70 | /usr/lib/hadoop-mapreduce/lib/* 71 | 72 | 73 | 74 | 75 | yarn.resourcemanager.hostname 76 | hadoop-master 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hive/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | export HADOOP_OPTS="$HADOOP_OPTS -Dhive.root.logger=INFO,console" 2 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/hive/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.security.authorization.createtable.owner.grants 58 | ALL 59 | The set of privileges automatically granted to the owner whenever a table gets created. 60 | 61 | 62 | 63 | hive.users.in.admin.role 64 | hdfs,hive 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | command=hive --service metastore 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-metastore.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-nodemanager] 2 | command=yarn nodemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-nodemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/etc/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-resourcemanager] 2 | command=yarn resourcemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-resourcemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/cdh5.15-hive/files/root/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 0 make file system hostname resolvable 4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts 5 | 6 | # 1 format namenode 7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 8 | 9 | # workaround for 'could not open session' bug as suggested here: 10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 11 | rm -f /etc/security/limits.d/hdfs.conf 12 | su -c "echo 'N' | hdfs namenode -format" hdfs 13 | 14 | # 2 start hdfs 15 | su -c "hdfs datanode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-datanode.log" hdfs& 16 | su -c "hdfs namenode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-namenode.log" hdfs& 17 | 18 | # 3 wait for process starting 19 | sleep 10 20 | 21 | # remove a broken symlink created by cdh installer so that init-hdfs.sh does no blow up on it 22 | # (hbase-annotations.jar seems not needed in our case) 23 | rm /usr/lib/hive/lib/hbase-annotations.jar 24 | 25 | # 4 exec cloudera hdfs init script 26 | /usr/lib/hadoop/libexec/init-hdfs.sh 27 | 28 | # 5 init hive directories 29 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 30 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 31 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 32 | 33 | # 6 stop hdfs 34 | killall java 35 | 36 | # 7 setup metastore 37 | mysql_install_db 38 | 39 | /usr/bin/mysqld_safe & 40 | sleep 10s 41 | 42 | cd /usr/lib/hive/scripts/metastore/upgrade/mysql/ 43 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 44 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.1.0.mysql.sql;" | mysql 45 | /usr/bin/mysqladmin -u root password 'root' 46 | 47 | killall mysqld 48 | sleep 10s 49 | mkdir /var/log/mysql/ 50 | chown mysql:mysql /var/log/mysql/ -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/README.md: -------------------------------------------------------------------------------- 1 | # centos6-java8-oracle-openldap [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/centos6-oj8-openldap.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/centos6-oj8-openldap 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/centos6-oj8-openldap.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/centos6-oj8-openldap 7 | 8 | Docker image of CentOS 6 with Oracle JDK 8 installed. This has OpenLdap needed for 9 | Presto when running product-tests with front-end LDAP authentication. 10 | 11 | ## Oracle license 12 | 13 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 14 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 15 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/certs/active-directory-certificate.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIGJTCCBQ2gAwIBAgITHQAAAAUM8gMo/ohRwwAAAAAABTANBgkqhkiG9w0BAQUF 3 | ADBlMRMwEQYKCZImiZPyLGQBGRYDY29tMRgwFgYKCZImiZPyLGQBGRYIdGVzdGxk 4 | YXAxFjAUBgoJkiaJk/IsZAEZFgZwcmVzdG8xHDAaBgNVBAMTE3ByZXN0by1BRC1U 5 | RVNULUNBLTEwHhcNMTcwNjI3MDA0NTM1WhcNMTgwNjI3MDA0NTM1WjAAMIIBIjAN 6 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoFyafCrwVdNqQ1Y1D1HMQVScNAc9 7 | m5l0nBbezmtbWExSgKXKE0DOIHLLZB38F8TKJZoa3m3J7Oin+GISrzWgyhoW6+jg 8 | KKGD9PZR5y4Xhu23Fv8IVlN9wf08vDploq34uzRFoBAwvjUcW42FPe+6MoKeMkat 9 | KgrfhCkcXAGhg0SOYmdCAHcPA63HBCNmpgYAZ38iqwiZugnraIXgn0VGhcmLlj7c 10 | S5Sv1GYOUtEd0MXELepAZPUXx8HivPvnz09tz++W6aKAkLsYvQShijPVnFinOAa9 11 | 8ZNlHyz8v5UiUbS/9kyQOY86mM+1U+qK9BQVIbRVvBSkLXPY2ZAvyZ9pCwIDAQAB 12 | o4IDMTCCAy0wPQYJKwYBBAGCNxUHBDAwLgYmKwYBBAGCNxUIhdXlWYazpGSH5Z0V 13 | hczUNYOhlVGBRKj2FYbb3DICAWQCAQEwMgYDVR0lBCswKQYHKwYBBQIDBQYKKwYB 14 | BAGCNxQCAgYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB/wQEAwIFoDBABgkr 15 | BgEEAYI3FQoEMzAxMAkGBysGAQUCAwUwDAYKKwYBBAGCNxQCAjAKBggrBgEFBQcD 16 | ATAKBggrBgEFBQcDAjAdBgNVHQ4EFgQU74Oc6DMfI847F2OplR+db0pIQJEwHwYD 17 | VR0jBBgwFoAUYVLOSj8PMaLbsTkN+P/R5RwEvMIwgdwGA1UdHwSB1DCB0TCBzqCB 18 | y6CByIaBxWxkYXA6Ly8vQ049cHJlc3RvLUFELVRFU1QtQ0EtMSxDTj1hZC10ZXN0 19 | LENOPUNEUCxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD 20 | Tj1Db25maWd1cmF0aW9uLERDPXByZXN0byxEQz10ZXN0bGRhcCxEQz1jb20/Y2Vy 21 | dGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENsYXNzPWNSTERpc3Ry 22 | aWJ1dGlvblBvaW50MIHQBggrBgEFBQcBAQSBwzCBwDCBvQYIKwYBBQUHMAKGgbBs 23 | ZGFwOi8vL0NOPXByZXN0by1BRC1URVNULUNBLTEsQ049QUlBLENOPVB1YmxpYyUy 24 | MEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9 25 | cHJlc3RvLERDPXRlc3RsZGFwLERDPWNvbT9jQUNlcnRpZmljYXRlP2Jhc2U/b2Jq 26 | ZWN0Q2xhc3M9Y2VydGlmaWNhdGlvbkF1dGhvcml0eTB0BgNVHREBAf8EajBooCwG 27 | CisGAQQBgjcUAgOgHgwcQUQtVEVTVCRAcHJlc3RvLnRlc3RsZGFwLmNvbYIbYWQt 28 | dGVzdC5wcmVzdG8udGVzdGxkYXAuY29tghNwcmVzdG8udGVzdGxkYXAuY29tggZQ 29 | UkVTVE8wDQYJKoZIhvcNAQEFBQADggEBAAtuksGPz4cwkIB7g8nqI1ysi0uaPxBO 30 | DSdq3wXPA9jS+W6bR39iiflugaeOW1pfQsZhMJadVCMkvQ/sYC4y1WmD+j6VSVs2 31 | 1znc4Gh+41teF3x1XRHgvQ1Vum+7GJIeKhZEUuB1F9SpuYxQT3wOIX2VnKq+1Z3/ 32 | 7TlRyZ9bAG8tmSeYXK4/dL5q+L7o1UgRU47V7RzhOJp/bmQFQd0gtXRpSH5LJZhk 33 | +lIlmCxgsjkawBBvXHWd9K6R/5SIr/S/RfOIVdU/WbIcOzzAzZr9YqhmB3POc4aL 34 | f4IY0N7Rxc/dBW1bO833+lNOI4dJJgtdZoOxCe3BiHsM2EbZzUiu1HA= 35 | -----END CERTIFICATE----- 36 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/certs/openldap-certificate.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDgTCCAmmgAwIBAgIJAKvPRrEwaoF0MA0GCSqGSIb3DQEBBQUAMFcxCzAJBgNV 3 | BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg 4 | Q29tcGFueSBMdGQxEzARBgNVBAMMCmxkYXBzZXJ2ZXIwHhcNMTYwODEwMTczOTIx 5 | WhcNMTkwODEwMTczOTIxWjBXMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVs 6 | dCBDaXR5MRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRkMRMwEQYDVQQDDAps 7 | ZGFwc2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA29sHE6G4 8 | 8F0MmHEuir/ZWMpIMjabWtMCp6q2Vbp52tqxSGVWqeVAKW+1qXmvNNMMlHzngmjT 9 | 0EIBoWk9LCKFeXkDCjwzASvjmGU1p9L0tFo9yEclDNlHo1B/Dq81ZnuaXZyBA7vd 10 | sbkcXqsES0m1736j9TP4Yfl915jrzma8JF92b0p479fH3xf/ZVTO3VNjtFYp186a 11 | ImIzeWDeNZk5ujPB6sJcKLm0l4q+wD4qpUbfcOqaT2YuauCsQGtqYAIf5F2uxyDA 12 | FdQt18zddlRocvuU/uPuF3bBNYxNICPrirqxsIJYudjdzmPjG+V1rDdQsl02c27e 13 | brbfq9d2togWDQIDAQABo1AwTjAdBgNVHQ4EFgQUfSaAXBYaUUlZ8myIwFJxmZI8 14 | EqIwHwYDVR0jBBgwFoAUfSaAXBYaUUlZ8myIwFJxmZI8EqIwDAYDVR0TBAUwAwEB 15 | /zANBgkqhkiG9w0BAQUFAAOCAQEAbTtHWLNJHhJUiZznPUTcAcX0YN+xcimcSquD 16 | JBKT6jqRgj+gOFXzG0gXVtATvXsvtXk/3wkbXEnnE6mBG/RMaP3pmsQ8BW1pUx1g 17 | bZofvPeEv8vRxd2Cs/dKhYv3MbTSAUPhuYub8oFB44TKS8uPmykIrTa9xG2uOYj/ 18 | LdRPATE6W8KcCw55OUMfUdFuJB9fkiDqTYNT/hscBeuBrRJeDOuKSvhPXDYb1+ie 19 | 8rL6EM5YOlT9MNAYohMb9re1ry4u4jwrWuO8X461NdFhCHxwIXYH/p2JW4cHl0yt 20 | JvmE/yFsEsUu21RtL798p2uV8ADwpX7VwcjxkTSjyS6ktTyI/A== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/certs/private.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEA29sHE6G48F0MmHEuir/ZWMpIMjabWtMCp6q2Vbp52tqxSGVW 3 | qeVAKW+1qXmvNNMMlHzngmjT0EIBoWk9LCKFeXkDCjwzASvjmGU1p9L0tFo9yEcl 4 | DNlHo1B/Dq81ZnuaXZyBA7vdsbkcXqsES0m1736j9TP4Yfl915jrzma8JF92b0p4 5 | 79fH3xf/ZVTO3VNjtFYp186aImIzeWDeNZk5ujPB6sJcKLm0l4q+wD4qpUbfcOqa 6 | T2YuauCsQGtqYAIf5F2uxyDAFdQt18zddlRocvuU/uPuF3bBNYxNICPrirqxsIJY 7 | udjdzmPjG+V1rDdQsl02c27ebrbfq9d2togWDQIDAQABAoIBAQDBip4/NQYWawN0 8 | i2Z6dItEc8Z8lb/Lg6h5q3maHG+7LvwUDEnvhEYodBPK3rlTWltfuFkhX2Dfm8vN 9 | qVRVrworNlFryDTqWhWgibBjTf1na22qCp9eIUTES0+ewLTUxAjPUb/tZouyMY/8 10 | 8rX0BS6pZ6W0bQzCSxzyF8e2SpBjvxhA+MYleMtxqc4rsn2lASu+ITNvgRIHXlQT 11 | ApD7IW3oG8DsSXE9UTAum/UoFKiv4VVqtKi5dAeSPRAabV/fo/Qtf8suQAvf4JxV 12 | tyUJtyMUvpRHV6KP6B1F0S+UYaXCWrD1r9xm8uA2WFtA4XvFFVVikreoos1soU7+ 13 | VJuOQlxNAoGBAPHHiYOWKl83+UrR94Mmjb9NJ7s4CKau9Y1Iu18ct+tt2rKihktE 14 | OkZk/duOKmW2oXxTuruz16d3+WGDErqbCM+hJS7rUs7Go9qPq+umjSbQHtOWUDwg 15 | 5osPorAR+9h/+xu9CPwaibT/9C/THa/gh0FsBEFw6ViHK5045NRDs7QLAoGBAOjJ 16 | Yj5/bhWyIdWawimZEvsNjG7zCDlM85yM/gd0xjvdBuPXTL9xY23TXHbodqw+x174 17 | K72w/6/B34eV/FLCcOK+esataod8J215O+OKvgul+E1SG1/Sfn4e/0rTVuHzzRvz 18 | aIbALMdU6PbPTVc73t7y5ALzQVpVOK723GwLLNVHAoGAIvGz7lDFzv1ICohOPaZU 19 | 1lFQbL6sgWrIEqNwcU61RlGaSjczMa2qvoHaLuFdP1YmANIZKwfTI2EzqeRH0LCX 20 | lWq09RMN5NTQ2cXqEyT/QZzS18Hh1s7qgy6/VLNEswXMRTYNFmK/hGl54W+h2smY 21 | wtvZwZuQHjtrEk4mxIcEcgECgYAoA12ownyorArWzZgsUEyL7MND2vCB8PE66p1L 22 | qM3eQlrJ1RszGQlvOlJ9ee3UI6Jfd6cFHySEEH4crzJFA3kl8HnoJqzWBwg78AfU 23 | T+fk1q18lO1xphvqD+fv863Ofvq1oDxKVUQdK0I0EmQH55milvuMakXEgjuRD4vJ 24 | 4o3dRwKBgQC5JH3iqCNhrS5jOlXsLnXiZQVvO4viWMFgyJi56dMkDM/dhhYhrr7r 25 | i5h/6LJOufhUfmvvoAaOG4mBkNwWfZImCGrj7VxQWMlwgIUFb5atgnx0f/SkOrqn 26 | +V7Dmj05PwtnYTpltKgCLjsoQGyzLeXV3bFXfSCt9Xmcyp1qRKrxEg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/certs/serial.txt: -------------------------------------------------------------------------------- 1 | 1234 -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/setup/createOU.ldif: -------------------------------------------------------------------------------- 1 | dn: dc=presto,dc=testldap,dc=com 2 | dc: presto 3 | o: teradata 4 | objectClass: organization 5 | objectClass: dcObject 6 | 7 | dn: ou=America,dc=presto,dc=testldap,dc=com 8 | objectClass: organizationalUnit 9 | ou: America 10 | 11 | dn: ou=Asia,dc=presto,dc=testldap,dc=com 12 | objectClass: organizationalUnit 13 | ou: Asia 14 | 15 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/setup/ldap_disable_bind_anon.ldif: -------------------------------------------------------------------------------- 1 | dn: cn=config 2 | changetype: modify 3 | add: olcDisallows 4 | olcDisallows: bind_anon 5 | 6 | dn: cn=config 7 | changetype: modify 8 | add: olcRequires 9 | olcRequires: authc 10 | 11 | dn: olcDatabase={-1}frontend,cn=config 12 | changetype: modify 13 | add: olcRequires 14 | olcRequires: authc 15 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/setup/memberof.ldif: -------------------------------------------------------------------------------- 1 | # This overlay helps in determing which group a given user belongs to 2 | dn: cn=module,cn=config 3 | cn: module 4 | objectClass: olcModuleList 5 | olcModuleLoad: memberof.la 6 | olcModulePath: /usr/lib64/openldap 7 | 8 | dn: olcOverlay={0}memberof,olcDatabase={2}bdb,cn=config 9 | objectClass: olcConfig 10 | objectClass: olcMemberOf 11 | objectClass: olcOverlayConfig 12 | objectClass: top 13 | olcOverlay: memberof 14 | olcMemberOfDangling: ignore 15 | olcMemberOfRefInt: TRUE 16 | olcMemberOfGroupOC: groupOfNames 17 | olcMemberOfMemberAD: member 18 | olcMemberOfMemberOfAD: memberOf 19 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/setup/modify_server.ldif: -------------------------------------------------------------------------------- 1 | dn: olcDatabase={2}bdb,cn=config 2 | changetype: modify 3 | replace: olcRootDN 4 | olcRootDN: cn=admin,dc=presto,dc=testldap,dc=com 5 | - 6 | replace: olcSuffix 7 | olcSuffix: dc=presto,dc=testldap,dc=com 8 | - 9 | replace: olcRootPW 10 | olcRootPW: admin 11 | - 12 | replace: olcAccess 13 | olcAccess: {0}to attrs=userPassword by self write by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" write by anonymous auth by * none 14 | olcAccess: {1}to * by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" write by self write by * read 15 | 16 | dn: olcDatabase={1}monitor,cn=config 17 | changetype: modify 18 | replace: olcAccess 19 | olcAccess: {0}to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=exter 20 | nal,cn=auth" read by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" read by * n 21 | one 22 | 23 | dn: cn=config 24 | changetype: modify 25 | replace: olcTLSCACertificateFile 26 | olcTLSCACertificateFile: /etc/openldap/certs/openldap-certificate.pem 27 | - 28 | replace: olcTLSCertificateFile 29 | olcTLSCertificateFile: /etc/openldap/certs/openldap-certificate.pem 30 | - 31 | replace: olcTLSCertificateKeyFile 32 | olcTLSCertificateKeyFile: /etc/openldap/certs/private.pem 33 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/openldap/setup/refint.ldif: -------------------------------------------------------------------------------- 1 | # This overlay ensures that when user accounts are modified or deleted, 2 | # the corresponding member attribute in groups is also handled accordingly. 3 | dn: cn=module{1},cn=config 4 | cn: module 5 | objectclass: olcModuleList 6 | objectclass: top 7 | olcmoduleload: refint.la 8 | 9 | dn: olcOverlay={1}refint,olcDatabase={2}bdb,cn=config 10 | objectClass: olcConfig 11 | objectClass: olcOverlayConfig 12 | objectClass: olcRefintConfig 13 | objectClass: top 14 | olcOverlay: {1}refint 15 | olcRefintAttribute: memberof member manager owner 16 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8-openldap/files/etc/supervisord.d/slapd.conf: -------------------------------------------------------------------------------- 1 | [program:slapd] 2 | command=/usr/bin/pidproxy /var/run/openldap/slapd.pid /usr/sbin/slapd -d 1 -F /etc/openldap/slapd.d/ -h 'ldaps://0.0.0.0:636 ldap://0.0.0.0:389' 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/slapd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM library/centos:6.9 14 | MAINTAINER Presto community 15 | 16 | ARG JDK_URL 17 | ARG JDK_PATH 18 | 19 | # Install Oracle Java and presto-admin dependences (already has python 2.6) 20 | RUN \ 21 | set -xeu && \ 22 | yum install -y wget && \ 23 | # install and trim Oracle JDK 24 | wget -nv --header "Cookie: oraclelicense=accept-securebackup-cookie" $JDK_URL -O jdk.rpm && \ 25 | rpm -ivh jdk.rpm && rm jdk.rpm && \ 26 | rm -r $JDK_PATH/*src.zip \ 27 | $JDK_PATH/lib/missioncontrol \ 28 | $JDK_PATH/lib/visualvm \ 29 | $JDK_PATH/lib/*javafx* \ 30 | $JDK_PATH/jre/lib/plugin.jar \ 31 | $JDK_PATH/jre/lib/ext/jfxrt.jar \ 32 | $JDK_PATH/jre/bin/javaws \ 33 | $JDK_PATH/jre/lib/javaws.jar \ 34 | $JDK_PATH/jre/lib/desktop \ 35 | $JDK_PATH/jre/plugin \ 36 | $JDK_PATH/jre/lib/deploy* \ 37 | $JDK_PATH/jre/lib/*javafx* \ 38 | $JDK_PATH/jre/lib/*jfx* \ 39 | $JDK_PATH/jre/lib/amd64/libdecora_sse.so \ 40 | $JDK_PATH/jre/lib/amd64/libprism_*.so \ 41 | $JDK_PATH/jre/lib/amd64/libfxplugins.so \ 42 | $JDK_PATH/jre/lib/amd64/libglass.so \ 43 | $JDK_PATH/jre/lib/amd64/libgstreamer-lite.so \ 44 | $JDK_PATH/jre/lib/amd64/libjavafx*.so \ 45 | $JDK_PATH/jre/lib/amd64/libjfx*.so && \ 46 | \ 47 | # make java fixed (so further layers don't accidentally override it) 48 | update-alternatives --set java $JDK_PATH/jre/bin/java && \ 49 | \ 50 | # install supervisor 51 | yum install -y setuptools epel-release && \ 52 | yum install -y python-pip && \ 53 | pip install --upgrade pip==9.0.3 `# latest version for Python 2.6` && \ 54 | pip install --upgrade setuptools==36.8.0 `# latest version for Python 2.6` && \ 55 | pip install supervisor && \ 56 | \ 57 | # install tools helpful when troubleshooting product tests 58 | yum install -y vim && \ 59 | # cleanup 60 | yum -y clean all && rm -rf /tmp/* /var/tmp/* 61 | 62 | ENV JAVA_HOME $JDK_PATH 63 | -------------------------------------------------------------------------------- /prestodb/centos6-oj8/README.md: -------------------------------------------------------------------------------- 1 | # centos6-oj8 [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/centos6-oj8.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/centos6-oj8 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/centos6-oj8.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/centos6-oj8 7 | 8 | Docker image of CentOS 6 with Oracle JDK 8 installed. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/centos6-presto-admin-tests-build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM prestodb/centos6-presto-admin-tests-runtime:unlabelled 2 | MAINTAINER Presto community 3 | 4 | RUN yum install -y \ 5 | wget \ 6 | gcc \ 7 | python-devel \ 8 | python-devel \ 9 | python-pip \ 10 | openssl-devel \ 11 | libffi-devel && \ 12 | pip install --upgrade pip==9.0.3 `# latest version for Python 2.6` && \ 13 | # cleanup 14 | yum -y clean all && rm -rf /tmp/* /var/tmp/* 15 | -------------------------------------------------------------------------------- /prestodb/centos6-presto-admin-tests-build/README.md: -------------------------------------------------------------------------------- 1 | Centos 6 with presto-admin build dependences 2 | -------------------------------------------------------------------------------- /prestodb/centos6-presto-admin-tests-runtime/README.md: -------------------------------------------------------------------------------- 1 | # centos6-presto-adminn-tests-runtime 2 | 3 | Docker image of CentOS 6 with Oracle JDK 8 installed, and with sshd 4 | running. Passwordless ssh for the user app-admin is also set up. 5 | 6 | ## Further documentation 7 | For more documentation on this image, see the documentation for the 8 | base centos6-ssh image it uses: 9 | https://registry.hub.docker.com/u/jdeathe/centos-ssh/ 10 | 11 | ## Oracle license 12 | 13 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 14 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 15 | -------------------------------------------------------------------------------- /prestodb/centos6-presto-admin-tests-runtime/vagrant_insecure_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI 3 | w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP 4 | kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 5 | hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO 6 | Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW 7 | yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd 8 | ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 9 | Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf 10 | TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK 11 | iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A 12 | sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf 13 | 4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP 14 | cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk 15 | EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN 16 | CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX 17 | 3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG 18 | YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj 19 | 3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ 20 | dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz 21 | 6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC 22 | P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF 23 | llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ 24 | kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH 25 | +vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ 26 | NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /prestodb/centos7-oj8/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM library/centos:7 14 | MAINTAINER Presto community 15 | 16 | COPY ./files / 17 | 18 | # Install Java and presto-admin dependences 19 | RUN \ 20 | set -xeu && \ 21 | yum install -y \ 22 | java-1.8.0-openjdk-devel \ 23 | nc \ 24 | wget \ 25 | && \ 26 | \ 27 | # install Zulu JDK (will not be the default `java`) 28 | rpm -i https://cdn.azul.com/zulu/bin/zulu11.39.15-ca-jdk11.0.7-linux.x86_64.rpm && \ 29 | \ 30 | # install supervisor 31 | yum --enablerepo=extras install -y setuptools epel-release && \ 32 | yum install -y python-pip && \ 33 | pip install supervisor && \ 34 | \ 35 | # install commonly needed packages 36 | yum install -y \ 37 | less `# helpful when troubleshooting product tests` \ 38 | net-tools `# netstat is required by run_on_docker.sh` \ 39 | sudo \ 40 | telnet `# helpful when troubleshooting product tests` \ 41 | vim `# helpful when troubleshooting product tests` \ 42 | && \ 43 | # cleanup 44 | yum -y clean all && rm -rf /tmp/* /var/tmp/* 45 | 46 | ENV PATH="/usr/local/bin:${PATH}" 47 | ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk 48 | ENV LANG=en_US.UTF-8 49 | -------------------------------------------------------------------------------- /prestodb/centos7-oj8/files/opt/prestodb/site-override.xslt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /prestodb/centos7-oj8/files/usr/local/bin/apply-all-site-xml-overrides: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | fail() { 6 | echo "$(basename "$0"): $*" >&2 7 | exit 1 8 | } 9 | 10 | if [ $# -ne 1 ]; then 11 | fail "Usage: $0 " >&2 12 | fi 13 | 14 | overrides_dir="$1" 15 | 16 | for file in $(find $overrides_dir -name '*.xml'); do 17 | target_filename="${file#"$overrides_dir"}" 18 | echo "Applying configuration override from $file to $target_filename" 19 | apply-site-xml-override "$target_filename" "$file" 20 | done 21 | -------------------------------------------------------------------------------- /prestodb/centos7-oj8/files/usr/local/bin/apply-site-xml-override: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | fail() { 6 | echo "$(basename "$0"): $*" >&2 7 | exit 1 8 | } 9 | 10 | if [ $# -ne 2 ]; then 11 | fail "Usage: $0 " >&2 12 | fi 13 | 14 | site_xml="$1" 15 | overrides="$2" 16 | site_xml_new="$1.new" 17 | 18 | test -f "${site_xml}" || fail "${site_xml} does not exist or is not a file" 19 | test -f "${overrides}" || fail "${overrides} does not exist or is not a file" 20 | test ! -e "${site_xml_new}" || fail "${site_xml_new} already exists" 21 | 22 | xsltproc --param override-path "'${overrides}'" "/opt/prestodb/site-override.xslt" "${site_xml}" > "${site_xml_new}" 23 | cat "${site_xml_new}" > "${site_xml}" # Preserve file owner & permissions 24 | rm "${site_xml_new}" 25 | -------------------------------------------------------------------------------- /prestodb/dns/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM alpine:3.3 14 | MAINTAINER Presto community 15 | RUN apk --no-cache add dnsmasq 16 | EXPOSE 53 53/udp 17 | ENTRYPOINT ["dnsmasq", "-k"] 18 | -------------------------------------------------------------------------------- /prestodb/dns/README.md: -------------------------------------------------------------------------------- 1 | # docker-dnsmasq [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/dns.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/dns 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/dns.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/dns 7 | 8 | Based on https://github.com/andyshinn/docker-dnsmasq 9 | 10 | It's a [dnsmasq][dnsmasq] Docker image. It is only 6 MB in size. It is just an `ENTRYPOINT` to the `dnsmasq` binary. Can you smell what the rock is cookin'? 11 | 12 | ## Usage 13 | 14 | It is usually a good idea to use a tag other than `latest` if you are using this image in a production setting. 15 | 16 | [dnsmasq][dnsmasq] requires `NET_ADMIN` capabilities to run correctly. Start it with something like `docker run -p 53:53/tcp -p 53:53/udp --cap-add=NET_ADMIN prestodb/dns:2.75`. 17 | 18 | The configuration is all handled on the command line (no wrapper scripts here). The `ENTRYPOINT` is `dnsmasq -k` to keep it running in the foreground. If you wanted to send requests for an internal domain (such as Consul) you can forward the requests upstream using something like `docker run -p 53:53/tcp -p 53:53/udp --cap-add=NET_ADMIN prestodb/dns:2.75 -S /consul/10.17.0.2`. This will send a request for `redis.service.consul` to `10.17.0.2` 19 | 20 | As this is a very barebones entrypoint with just enough to run in the foreground, there is no logging enabled by default. To send logging to stdout you can add `--log-facility=-` as an option. 21 | 22 | [dnsmasq]: http://www.thekelleys.org.uk/dnsmasq/doc.html 23 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-base/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/centos6-oj8:unlabelled 14 | MAINTAINER Presto community 15 | 16 | RUN \ 17 | wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo -P /etc/yum.repos.d \ 18 | 19 | && yum install -y \ 20 | hadoop-hdfs-namenode \ 21 | hadoop-hdfs-secondarynamenode \ 22 | hadoop-hdfs-datanode \ 23 | 24 | hadoop-mapreduce-historyserver \ 25 | 26 | hadoop-yarn-resourcemanager \ 27 | hadoop-yarn-nodemanager \ 28 | 29 | zookeeper \ 30 | zookeeper-server \ 31 | 32 | hive \ 33 | hive-metastore \ 34 | hive-server2 \ 35 | 36 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 37 | COPY ./files / 38 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-base/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.5-base.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.5-base 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.5-base.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.5-base 7 | 8 | A Hadoop base image for creating images testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-base/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.5-hive-kerberized.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.5-hive-kerberized 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.5-hive-kerberized.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.5-hive-kerberized 7 | 8 | A pseudo-distributed Hadoop image with Kerberos authentication enabled, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | 30 | dfs.block.access.token.enable 31 | true 32 | 33 | 34 | 35 | 36 | dfs.namenode.keytab.file 37 | /etc/hadoop/conf/hdfs.keytab 38 | 39 | 40 | dfs.namenode.kerberos.principal 41 | hdfs/hadoop-master@LABS.TERADATA.COM 42 | 43 | 44 | dfs.namenode.kerberos.internal.spnego.principal 45 | HTTP/hadoop-master@LABS.TERADATA.COM 46 | 47 | 48 | 49 | 50 | dfs.secondary.namenode.keytab.file 51 | /etc/hadoop/conf/hdfs.keytab 52 | 53 | 54 | dfs.secondary.namenode.kerberos.principal 55 | hdfs/hadoop-master@LABS.TERADATA.COM 56 | 57 | 58 | dfs.secondary.namenode.kerberos.internal.spnego.principal 59 | HTTP/hadoop-master@LABS.TERADATA.COM 60 | 61 | 62 | 63 | 64 | dfs.datanode.keytab.file 65 | /etc/hadoop/conf/hdfs.keytab 66 | 67 | 68 | dfs.datanode.kerberos.principal 69 | hdfs/hadoop-master@LABS.TERADATA.COM 70 | 71 | 72 | 73 | 74 | dfs.webhdfs.enabled 75 | true 76 | 77 | 78 | 79 | 80 | dfs.web.authentication.kerberos.principal 81 | HTTP/hadoop-master@LABS.TERADATA.COM 82 | 83 | 84 | 85 | dfs.web.authentication.kerberos.keytab 86 | /etc/hadoop/conf/HTTP.keytab 87 | 88 | 89 | 90 | ignore.secure.ports.for.testing 91 | true 92 | 93 | 94 | 95 | dfs.http.policy 96 | HTTP_ONLY 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.framework.name 27 | yarn 28 | 29 | 30 | 31 | mapreduce.jobhistory.address 32 | hadoop-master:10020 33 | 34 | 35 | 36 | mapreduce.jobhistory.webapp.address 37 | hadoop-master:19888 38 | 39 | 40 | 41 | To set the value of tmp directory for map and reduce tasks. 42 | mapreduce.task.tmp.dir 43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 44 | 45 | 46 | 47 | 48 | mapreduce.jobhistory.keytab 49 | /etc/hadoop/conf/mapred.keytab 50 | 51 | 52 | 53 | mapreduce.jobhistory.principal 54 | mapred/hadoop-master@LABS.TERADATA.COM 55 | 56 | 57 | 58 | 59 | mapreduce.jobtracker.kerberos.principal 60 | mapred/hadoop-master@LABS.TERADATA.COM 61 | 62 | 63 | 64 | mapreduce.jobtracker.keytab.file 65 | /etc/hadoop/conf/mapred.keytab 66 | 67 | 68 | 69 | 70 | mapreduce.tasktracker.kerberos.principal 71 | mapred/hadoop-master@LABS.TERADATA.COM 72 | 73 | 74 | 75 | mapreduce.tasktracker.keytab.file 76 | /etc/hadoop/conf/mapred.keytab 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | hive.security.authenticator.manager 23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator 24 | 25 | 26 | 27 | 28 | hive.security.authorization.manager 29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory 30 | SQL standards based Hive authorization 31 | 32 | 33 | 34 | hive.security.authorization.enabled 35 | true 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/etc/krb5.conf: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = LABS.TERADATA.COM 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | LABS.TERADATA.COM = { 16 | kdc = hadoop-master 17 | admin_server = hadoop-master 18 | } 19 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/etc/supervisord.d/kdc.conf: -------------------------------------------------------------------------------- 1 | [program:krb5kdc] 2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -r LABS.TERADATA.COM -P /var/run/krb5kdc.pid -n" 3 | autostart=true 4 | autorestart=true 5 | redirect_stderr=true 6 | stdout_logfile=/dev/stdout 7 | stdout_logfile_maxbytes=0 8 | 9 | [program:kadmind] 10 | command=/bin/bash -c "exec /usr/sbin/kadmind -r LABS.TERADATA.COM -P /var/run/kadmind.pid -nofork" 11 | autostart=true 12 | autorestart=true 13 | redirect_stderr=true 14 | stdout_logfile=/dev/stdout 15 | stdout_logfile_maxbytes=0 16 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl: -------------------------------------------------------------------------------- 1 | */admin@LABS.TERADATA.COM * 2 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | LABS.TERADATA.COM = { 7 | #master_key_type = aes256-cts 8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 9 | dict_file = /usr/share/dict/words 10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 12 | } 13 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/hdp2.5-base:unlabelled 14 | MAINTAINER Presto community 15 | 16 | # CHANGE DEFAULT TIMEZONE 17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone 18 | 19 | # INSTALL MYSQL (Hive Metastore) 20 | RUN yum install -y mysql-server mysql-connector-java \ 21 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 22 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/hdp/current/hive-metastore/lib/mysql-connector-java.jar 23 | 24 | # Create needed dirs 25 | RUN mkdir /var/lib/zookeeper 26 | RUN chown zookeeper:hadoop /var/lib/zookeeper 27 | 28 | # DELETE ORIGINAL CONFIGURATION 29 | RUN rm -r /etc/hadoop/conf/* 30 | RUN rm -r /etc/zookeeper/conf/* 31 | RUN rm -r /etc/hive/conf/* 32 | 33 | # COPY CONFIGURATION FILES 34 | COPY ./files / 35 | 36 | # RUN SETUP 37 | RUN /root/setup.sh 38 | 39 | # DO CLEANUP 40 | RUN rm -rf /tmp/* /var/tmp/* 41 | 42 | # SETUP SOCKS PROXY 43 | RUN yum install -y openssh openssh-clients openssh-server 44 | RUN ssh-keygen -t rsa -b 4096 -C "automation@prestodb.io" -N "" -f /root/.ssh/id_rsa 45 | RUN ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key 46 | RUN ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key 47 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 48 | RUN chmod 755 /root && chmod 700 /root/.ssh 49 | RUN passwd --unlock root 50 | 51 | # HDFS PORTS 52 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 53 | 54 | # YARN PORTS 55 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 56 | 57 | # HIVE PORT 58 | EXPOSE 9083 10000 59 | 60 | # SOCKS PORT 61 | EXPOSE 1180 62 | 63 | CMD supervisord -c /etc/supervisord.conf 64 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.5-hive.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.5-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.5-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.5-hive 7 | 8 | A pseudo-distributed Hadoop image for testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://hadoop-master:9000 23 | 24 | 25 | 26 | 27 | hadoop.proxyuser.oozie.hosts 28 | * 29 | 30 | 31 | hadoop.proxyuser.oozie.groups 32 | * 33 | 34 | 35 | 36 | 37 | hadoop.proxyuser.httpfs.hosts 38 | * 39 | 40 | 41 | hadoop.proxyuser.httpfs.groups 42 | * 43 | 44 | 45 | 46 | 47 | hadoop.proxyuser.llama.hosts 48 | * 49 | 50 | 51 | hadoop.proxyuser.llama.groups 52 | * 53 | 54 | 55 | 56 | 57 | hadoop.proxyuser.hue.hosts 58 | * 59 | 60 | 61 | hadoop.proxyuser.hue.groups 62 | * 63 | 64 | 65 | 66 | 67 | hadoop.proxyuser.mapred.hosts 68 | * 69 | 70 | 71 | hadoop.proxyuser.mapred.groups 72 | * 73 | 74 | 75 | 76 | 77 | hadoop.proxyuser.hive.hosts 78 | * 79 | 80 | 81 | 82 | hadoop.proxyuser.hive.groups 83 | * 84 | 85 | 86 | 87 | 88 | hadoop.proxyuser.hdfs.groups 89 | * 90 | 91 | 92 | 93 | hadoop.proxyuser.hdfs.hosts 94 | * 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/hadoop/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Set Hadoop-specific environment variables here. 17 | # Forcing YARN-based mapreduce implementaion. 18 | # Make sure to comment out if you want to go back to the default or 19 | # if you want this to be tweakable on a per-user basis 20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce 21 | 22 | # The maximum amount of heap to use, in MB. Default is 1000. 23 | export HADOOP_HEAPSIZE=256 24 | 25 | # Extra Java runtime options. Empty by default. 26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m" 27 | export YARN_OPTS="$YARN_OPTS -Xmx256m" 28 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.framework.name 27 | yarn 28 | 29 | 30 | 31 | mapreduce.jobhistory.address 32 | hadoop-master:10020 33 | 34 | 35 | 36 | mapreduce.jobhistory.webapp.address 37 | hadoop-master:19888 38 | 39 | 40 | 41 | To set the value of tmp directory for map and reduce tasks. 42 | mapreduce.task.tmp.dir 43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/hadoop/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | yarn.nodemanager.aux-services 23 | mapreduce_shuffle 24 | 25 | 26 | 27 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 28 | org.apache.hadoop.mapred.ShuffleHandler 29 | 30 | 31 | 32 | yarn.log-aggregation-enable 33 | true 34 | 35 | 36 | 37 | yarn.dispatcher.exit-on-error 38 | true 39 | 40 | 41 | 42 | List of directories to store localized files in. 43 | yarn.nodemanager.local-dirs 44 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 45 | 46 | 47 | 48 | Where to store container logs. 49 | yarn.nodemanager.log-dirs 50 | /var/log/hadoop/yarn/containers 51 | 52 | 53 | 54 | Where to aggregate logs to. 55 | yarn.nodemanager.remote-app-log-dir 56 | /var/log/hadoop/yarn/apps 57 | 58 | 59 | 60 | Classpath for typical applications. 61 | yarn.application.classpath 62 | 63 | /etc/hadoop/conf, 64 | /usr/hdp/current/hadoop-client/*, 65 | /usr/hdp/current/hadoop-client/lib/*, 66 | /usr/hdp/current/hadoop-hdfs-client/*, 67 | /usr/hdp/current/hadoop-hdfs-client/lib/*, 68 | /usr/hdp/current/hadoop-yarn-client/*, 69 | /usr/hdp/current/hadoop-yarn-client/lib/*, 70 | /usr/hdp/current/hadoop-mapreduce-client/*, 71 | /usr/hdp/current/hadoop-mapreduce-client/lib/* 72 | 73 | 74 | 75 | 76 | yarn.resourcemanager.hostname 77 | hadoop-master 78 | 79 | 80 | 81 | yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage 82 | 100 83 | 84 | 85 | 86 | yarn.log.server.url 87 | http://hadoop-master:19888/jobhistory/logs 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/hive/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.security.authorization.createtable.owner.grants 58 | ALL 59 | The set of privileges automatically granted to the owner whenever a table gets created. 60 | 61 | 62 | 63 | hive.users.in.admin.role 64 | hdfs,hive 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-datanode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-namenode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | command=hive --service metastore 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-metastore.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | [program:mapreduce-historyserver] 2 | command=mapred historyserver 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mapred 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/mapreduce/hadoop-mapreduce-historyserver.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-nodemanager] 2 | command=yarn nodemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-nodemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-resourcemanager] 2 | command=yarn resourcemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-resourcemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | [program:zookeeper] 2 | command=/usr/bin/zookeeper-server start-foreground 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=zookeeper 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/zookeeper/zookeeper.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/zookeeper/conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | zookeeper.root.logger=INFO, CONSOLE 3 | zookeeper.console.threshold=INFO 4 | zookeeper.log.threshold=DEBUG 5 | log4j.rootLogger=INFO,CONSOLE 6 | 7 | # Example with rolling log file 8 | #log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE 9 | 10 | # Example with rolling log file and tracing 11 | #log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE 12 | 13 | # 14 | # Log INFO level and above messages to the console 15 | # 16 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 17 | log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} 18 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n 20 | 21 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/zookeeper/conf/zoo.cfg: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | maxClientCnxns=50 17 | # The number of milliseconds of each tick 18 | tickTime=2000 19 | # The number of ticks that the initial 20 | # synchronization phase can take 21 | initLimit=10 22 | # The number of ticks that can pass between 23 | # sending a request and getting an acknowledgement 24 | syncLimit=5 25 | # the directory where the snapshot is stored. 26 | dataDir=/var/lib/zookeeper 27 | # the port at which the clients will connect 28 | clientPort=2181 29 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/etc/zookeeper/conf/zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | export JAVA_HOME=${JAVA_HOME} 16 | export ZOO_LOG_DIR=${LOG_DIR} 17 | -------------------------------------------------------------------------------- /prestodb/hdp2.5-hive/files/root/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 0 make file system hostname resolvable 4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts 5 | 6 | # 1 format namenode 7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 8 | 9 | # workaround for 'could not open session' bug as suggested here: 10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 11 | rm -f /etc/security/limits.d/hdfs.conf 12 | su -c "echo 'N' | hdfs namenode -format" hdfs 13 | 14 | # 2 start hdfs 15 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 16 | 17 | # 3 wait for process starting 18 | sleep 10 19 | 20 | # 4 init basic hdfs directories 21 | /usr/hdp/2.5.0.0-1245/hadoop/libexec/init-hdfs.sh 22 | 23 | # 4.1 Create an hdfs home directory for the yarn user. For some reason, init-hdfs doesn't do so. 24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/yarn && /usr/bin/hadoop fs -chown yarn:yarn /user/yarn' 25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn' 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging' 27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging/history' 28 | 29 | 30 | 31 | # 5 init hive directories 32 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 33 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 34 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 35 | 36 | # 6 stop hdfs 37 | killall java 38 | 39 | # 7 setup metastore 40 | mysql_install_db 41 | 42 | /usr/bin/mysqld_safe & 43 | sleep 10s 44 | 45 | cd /usr/hdp/2.5.*/hive/scripts/metastore/upgrade/mysql/ 46 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 47 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.2.1000.mysql.sql;" | mysql 48 | /usr/bin/mysqladmin -u root password 'root' 49 | 50 | killall mysqld 51 | sleep 10s 52 | mkdir /var/log/mysql/ 53 | chown mysql:mysql /var/log/mysql/ 54 | 55 | exit 0 56 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.6-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.6-hive-kerberized.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.6-hive-kerberized 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.6-hive-kerberized.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.6-hive-kerberized 7 | 8 | A pseudo-distributed Hadoop image with Kerberos authentication enabled, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.framework.name 27 | yarn 28 | 29 | 30 | 31 | mapreduce.jobhistory.address 32 | hadoop-master:10020 33 | 34 | 35 | 36 | mapreduce.jobhistory.webapp.address 37 | hadoop-master:19888 38 | 39 | 40 | 41 | To set the value of tmp directory for map and reduce tasks. 42 | mapreduce.task.tmp.dir 43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 44 | 45 | 46 | 47 | 48 | mapreduce.jobhistory.keytab 49 | /etc/hadoop/conf/mapred.keytab 50 | 51 | 52 | 53 | mapreduce.jobhistory.principal 54 | mapred/hadoop-master@LABS.TERADATA.COM 55 | 56 | 57 | 58 | 59 | mapreduce.jobtracker.kerberos.principal 60 | mapred/hadoop-master@LABS.TERADATA.COM 61 | 62 | 63 | 64 | mapreduce.jobtracker.keytab.file 65 | /etc/hadoop/conf/mapred.keytab 66 | 67 | 68 | 69 | 70 | mapreduce.tasktracker.kerberos.principal 71 | mapred/hadoop-master@LABS.TERADATA.COM 72 | 73 | 74 | 75 | mapreduce.tasktracker.keytab.file 76 | /etc/hadoop/conf/mapred.keytab 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | hive.security.authenticator.manager 23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator 24 | 25 | 26 | 27 | 28 | hive.security.authorization.manager 29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory 30 | SQL standards based Hive authorization 31 | 32 | 33 | 34 | hive.security.authorization.enabled 35 | true 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/etc/krb5.conf: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = LABS.TERADATA.COM 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | LABS.TERADATA.COM = { 16 | kdc = hadoop-master:88 17 | admin_server = hadoop-master 18 | } 19 | OTHERLABS.TERADATA.COM = { 20 | kdc = hadoop-master:89 21 | admin_server = hadoop-master 22 | } 23 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/etc/supervisord.d/kdc.conf: -------------------------------------------------------------------------------- 1 | [program:krb5kdc] 2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid -n -r LABS.TERADATA.COM -n -d /var/kerberos/krb5kdc/principal-other -r OTHERLABS.TERADATA.COM" 3 | autostart=true 4 | autorestart=true 5 | redirect_stderr=true 6 | stdout_logfile=/dev/stdout 7 | stdout_logfile_maxbytes=0 8 | 9 | [program:kadmind] 10 | command=/bin/bash -c "exec /usr/sbin/kadmind -P /var/run/kadmind.pid -nofork -r LABS.TERADATA.COM" 11 | autostart=true 12 | autorestart=true 13 | redirect_stderr=true 14 | stdout_logfile=/dev/stdout 15 | stdout_logfile_maxbytes=0 16 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/var/kerberos/krb5kdc/kadm5-other.acl: -------------------------------------------------------------------------------- 1 | */admin@OTHERLABS.TERADATA.COM * 2 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl: -------------------------------------------------------------------------------- 1 | */admin@LABS.TERADATA.COM * 2 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | LABS.TERADATA.COM = { 7 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 8 | dict_file = /usr/share/dict/words 9 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 10 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 11 | } 12 | 13 | OTHERLABS.TERADATA.COM = { 14 | acl_file = /var/kerberos/krb5kdc/kadm5-other.acl 15 | dict_file = /usr/share/dict/words 16 | admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab 17 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 18 | kdc_listen = 89 19 | kdc_tcp_listen = 89 20 | kdc_ports = 89 21 | kdc_tcp_ports = 89 22 | } 23 | 24 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/centos6-oj8:unlabelled 14 | MAINTAINER Presto community 15 | 16 | # Change default timezone 17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone 18 | 19 | # Install HDP repo 20 | RUN wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0/hdp.repo -P /etc/yum.repos.d 21 | 22 | # Install Hadoop, Hive (w/ MySQL) 23 | RUN yum install -y \ 24 | hadoop-hdfs-namenode \ 25 | hadoop-hdfs-secondarynamenode \ 26 | hadoop-hdfs-datanode \ 27 | 28 | hadoop-yarn-resourcemanager \ 29 | hadoop-yarn-nodemanager \ 30 | 31 | hive \ 32 | hive-metastore \ 33 | hive-server2 \ 34 | 35 | mysql-server mysql-connector-java \ 36 | 37 | # Cleanup 38 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 39 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/hdp/current/hive-metastore/lib/mysql-connector-java.jar 40 | 41 | # Delete original configuration 42 | RUN rm -r /etc/hadoop/conf/* \ 43 | && rm -r /etc/hive/conf/* 44 | 45 | # Copy configuration files 46 | COPY ./files / 47 | 48 | # Run setup script 49 | RUN /root/setup.sh \ 50 | && rm -rf /tmp/* /var/tmp/* 51 | 52 | # Setup sock proxy 53 | RUN yum install -y openssh openssh-clients openssh-server && yum -y clean all 54 | RUN ssh-keygen -t rsa -b 4096 -C "automation@prestodb.io" -N "" -f /root/.ssh/id_rsa \ 55 | && ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key \ 56 | && ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key \ 57 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 58 | RUN chmod 755 /root && chmod 700 /root/.ssh 59 | RUN passwd --unlock root 60 | 61 | # HDFS ports 62 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 63 | 64 | # YARN ports 65 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 66 | 67 | # HIVE ports 68 | EXPOSE 9083 10000 69 | 70 | # SOCKS port 71 | EXPOSE 1180 72 | 73 | CMD supervisord -c /etc/supervisord.conf 74 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.6-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.6-hive.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.6-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.6-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.6-hive 7 | 8 | A pseudo-distributed Hadoop image for testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://hadoop-master:9000 23 | 24 | 25 | 26 | 27 | hadoop.proxyuser.oozie.hosts 28 | * 29 | 30 | 31 | hadoop.proxyuser.oozie.groups 32 | * 33 | 34 | 35 | 36 | 37 | hadoop.proxyuser.httpfs.hosts 38 | * 39 | 40 | 41 | hadoop.proxyuser.httpfs.groups 42 | * 43 | 44 | 45 | 46 | 47 | hadoop.proxyuser.llama.hosts 48 | * 49 | 50 | 51 | hadoop.proxyuser.llama.groups 52 | * 53 | 54 | 55 | 56 | 57 | hadoop.proxyuser.hue.hosts 58 | * 59 | 60 | 61 | hadoop.proxyuser.hue.groups 62 | * 63 | 64 | 65 | 66 | 67 | hadoop.proxyuser.mapred.hosts 68 | * 69 | 70 | 71 | hadoop.proxyuser.mapred.groups 72 | * 73 | 74 | 75 | 76 | 77 | hadoop.proxyuser.hive.hosts 78 | * 79 | 80 | 81 | 82 | hadoop.proxyuser.hive.groups 83 | * 84 | 85 | 86 | 87 | 88 | hadoop.proxyuser.hdfs.groups 89 | * 90 | 91 | 92 | 93 | hadoop.proxyuser.hdfs.hosts 94 | * 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/hadoop/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Set Hadoop-specific environment variables here. 17 | # Forcing YARN-based mapreduce implementaion. 18 | # Make sure to comment out if you want to go back to the default or 19 | # if you want this to be tweakable on a per-user basis 20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce 21 | 22 | # The maximum amount of heap to use, in MB. Default is 1000. 23 | export HADOOP_HEAPSIZE=256 24 | 25 | # Extra Java runtime options. Empty by default. 26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m" 27 | export YARN_OPTS="$YARN_OPTS -Xmx256m" 28 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.framework.name 27 | yarn 28 | 29 | 30 | 31 | mapreduce.jobhistory.address 32 | hadoop-master:10020 33 | 34 | 35 | 36 | mapreduce.jobhistory.webapp.address 37 | hadoop-master:19888 38 | 39 | 40 | 41 | To set the value of tmp directory for map and reduce tasks. 42 | mapreduce.task.tmp.dir 43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/hadoop/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | yarn.nodemanager.aux-services 23 | mapreduce_shuffle 24 | 25 | 26 | 27 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 28 | org.apache.hadoop.mapred.ShuffleHandler 29 | 30 | 31 | 32 | yarn.log-aggregation-enable 33 | true 34 | 35 | 36 | 37 | yarn.dispatcher.exit-on-error 38 | true 39 | 40 | 41 | 42 | List of directories to store localized files in. 43 | yarn.nodemanager.local-dirs 44 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 45 | 46 | 47 | 48 | Where to store container logs. 49 | yarn.nodemanager.log-dirs 50 | /var/log/hadoop-yarn/containers 51 | 52 | 53 | 54 | Where to aggregate logs to. 55 | yarn.nodemanager.remote-app-log-dir 56 | /var/log/hadoop-yarn/apps 57 | 58 | 59 | 60 | Classpath for typical applications. 61 | yarn.application.classpath 62 | 63 | /etc/hadoop/conf, 64 | /usr/hdp/current/hadoop-client/*, 65 | /usr/hdp/current/hadoop-client/lib/*, 66 | /usr/hdp/current/hadoop-hdfs-client/*, 67 | /usr/hdp/current/hadoop-hdfs-client/lib/*, 68 | /usr/hdp/current/hadoop-yarn-client/*, 69 | /usr/hdp/current/hadoop-yarn-client/lib/*, 70 | /usr/hdp/current/hadoop-mapreduce-client/*, 71 | /usr/hdp/current/hadoop-mapreduce-client/lib/* 72 | 73 | 74 | 75 | 76 | yarn.resourcemanager.hostname 77 | hadoop-master 78 | 79 | 80 | 81 | yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage 82 | 100 83 | 84 | 85 | 86 | yarn.log.server.url 87 | http://hadoop-master:19888/jobhistory/logs 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/hive/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.security.authorization.createtable.owner.grants 58 | ALL 59 | The set of privileges automatically granted to the owner whenever a table gets created. 60 | 61 | 62 | 63 | hive.users.in.admin.role 64 | hdfs,hive 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | command=hive --service metastore 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-metastore.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-nodemanager] 2 | command=yarn nodemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-nodemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/etc/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-resourcemanager] 2 | command=yarn resourcemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-resourcemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hdp2.6-hive/files/root/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 0 make file system hostname resolvable 4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts 5 | 6 | # 1 format namenode 7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 8 | 9 | # workaround for 'could not open session' bug as suggested here: 10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 11 | rm -f /etc/security/limits.d/hdfs.conf 12 | su -c "echo 'N' | hdfs namenode -format" hdfs 13 | 14 | # 2 start hdfs 15 | su -c "hdfs namenode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-namenode.log" hdfs& 16 | 17 | # 3 wait for process starting 18 | sleep 15 19 | 20 | # 4 init basic hdfs directories 21 | /usr/hdp/current/hadoop-client/libexec/init-hdfs.sh 22 | 23 | # 4.1 Create an hdfs home directory for the yarn user. For some reason, init-hdfs doesn't do so. 24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/yarn && /usr/bin/hadoop fs -chown yarn:yarn /user/yarn' 25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn' 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging' 27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging/history' 28 | 29 | 30 | 31 | # 5 init hive directories 32 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 33 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 34 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 35 | 36 | # 6 stop hdfs 37 | killall java 38 | 39 | # 7 setup metastore 40 | mysql_install_db 41 | 42 | /usr/bin/mysqld_safe & 43 | sleep 10s 44 | 45 | cd /usr/hdp/current/hive-metastore/scripts/metastore/upgrade/mysql/ 46 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 47 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.2.1000.mysql.sql;" | mysql 48 | /usr/bin/mysqladmin -u root password 'root' 49 | 50 | killall mysqld 51 | sleep 10s 52 | mkdir /var/log/mysql/ 53 | chown mysql:mysql /var/log/mysql/ 54 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/centos7-oj8:latest 14 | MAINTAINER Presto community 15 | 16 | RUN yum install -y \ 17 | mariadb-server \ 18 | openssh \ 19 | openssh-clients \ 20 | openssh-server \ 21 | psmisc \ 22 | which \ 23 | && yum -q clean all && rm -rf /var/cache/yum 24 | 25 | ARG HADOOP_VERSION=3.1.2 26 | ARG HIVE_VERSION=3.1.2 27 | 28 | # TODO Apache Archive is rate limited -- these should probably go in S3 29 | ARG HADOOP_BINARY_PATH=https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz 30 | ARG HIVE_BINARY_PATH=https://apache.mivzakim.net/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz 31 | 32 | RUN curl -o /tmp/hadoop.tar.gz --url $HADOOP_BINARY_PATH && \ 33 | tar xzf /tmp/hadoop.tar.gz --directory /opt && mv /opt/hadoop-$HADOOP_VERSION /opt/hadoop 34 | 35 | RUN curl -o /tmp/hive.tar.gz --url $HIVE_BINARY_PATH && \ 36 | tar xzf /tmp/hive.tar.gz --directory /opt && mv /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive 37 | 38 | ARG MYSQL_CONNECTOR_VERSION=8.0.13 39 | ARG AWS_SDK_VERSION=1.11.906 40 | RUN mkdir /opt/hive/auxlib && \ 41 | curl -o /opt/hive/auxlib/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar && \ 42 | curl -o /opt/hive/auxlib/aws-java-sdk-core-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/$AWS_SDK_VERSION/aws-java-sdk-core-$AWS_SDK_VERSION.jar && \ 43 | curl -o /opt/hive/auxlib/aws-java-sdk-s3-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/$AWS_SDK_VERSION/aws-java-sdk-s3-$AWS_SDK_VERSION.jar 44 | 45 | ENV HADOOP_HOME=/opt/hadoop 46 | ENV HIVE_HOME=/opt/hive 47 | ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/* 48 | ENV PATH=${HIVE_HOME}/bin:${HADOOP_HOME}/bin:${PATH} 49 | 50 | RUN ssh-keygen -t rsa -b 4096 -C "automation@prestodb.io" -N "" -f /root/.ssh/id_rsa && \ 51 | ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key && \ 52 | cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 53 | RUN chmod 755 /root && chmod 700 /root/.ssh 54 | RUN passwd --unlock root 55 | 56 | # Copy configuration files 57 | COPY ./files / 58 | 59 | # Run setup script 60 | RUN /root/setup.sh 61 | 62 | # HDFS port 63 | EXPOSE 9000 9870 64 | 65 | # HIVE Metastore port 66 | EXPOSE 9083 10000 67 | 68 | EXPOSE 1180 69 | 70 | CMD /root/entrypoint.sh 71 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/hadoop-init.d/init-hdfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | echo 'N' | hdfs namenode -format 4 | sed -i -e "s|hdfs://localhost|hdfs://$(hostname)|g" /opt/hadoop/etc/hadoop/core-site.xml 5 | hdfs namenode & 6 | sleep 10 && hdfs dfs -mkdir -p /user/hive/warehouse && killall java 7 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/hadoop-init.d/set-aws-creds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | if [[ -n "${AWS_ACCESS_KEY_ID}" ]] 4 | then 5 | echo "Setting AWS keys" 6 | sed -i -e "s|\"Use AWS_ACCESS_KEY_ID .*\"|${AWS_ACCESS_KEY_ID}|g" \ 7 | -e "s|\"Use AWS_SECRET_ACCESS_KEY .*\"|${AWS_SECRET_ACCESS_KEY}|g" \ 8 | /opt/hive/conf/hive-site.xml 9 | fi 10 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=root 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=root 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | # Add `--debug:port=5006` for debugging 3 | command=hive --service metastore 4 | startsecs=2 5 | stopwaitsecs=10 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/hive/hive-metastore.log 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=root 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/etc/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/opt/hadoop/etc/hadoop/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | fs.defaultFS 8 | hdfs://localhost:9000 9 | 10 | 11 | 12 | hadoop.proxyuser.root.hosts 13 | * 14 | 15 | 16 | 17 | hadoop.proxyuser.root.groups 18 | * 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/opt/hadoop/etc/hadoop/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | export HADOOP_HEAPSIZE=256 2 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/opt/hadoop/etc/hadoop/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | dfs.permissions.enabled 23 | false 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/opt/hive/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS} -Xmx256m -Djava.io.tmpdir=/tmp" 2 | export HADOOP_HEAPSIZE=256 3 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/opt/hive/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | hive.metastore.uris 7 | thrift://localhost:9083 8 | 9 | 10 | 11 | javax.jdo.option.ConnectionURL 12 | jdbc:mysql://localhost:3306/metastore?useSSL=false 13 | 14 | 15 | 16 | javax.jdo.option.ConnectionDriverName 17 | com.mysql.cj.jdbc.Driver 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionUserName 22 | root 23 | 24 | 25 | 26 | javax.jdo.option.ConnectionPassword 27 | root 28 | 29 | 30 | 31 | hive.metastore.connect.retries 32 | 15 33 | 34 | 35 | 36 | hive.metastore.disallow.incompatible.col.type.changes 37 | false 38 | 39 | 40 | 41 | 42 | metastore.storage.schema.reader.impl 43 | org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader 44 | 45 | 46 | 47 | hive.support.concurrency 48 | true 49 | 50 | 51 | 52 | hive.txn.manager 53 | org.apache.hadoop.hive.ql.lockmgr.DbTxnManager 54 | 55 | 56 | 57 | hive.compactor.initiator.on 58 | true 59 | 60 | 61 | 62 | hive.compactor.worker.threads 63 | 1 64 | 65 | 66 | 67 | fs.s3.awsAccessKeyId 68 | "Use AWS_ACCESS_KEY_ID environment variable to set this value" 69 | 70 | 71 | 72 | fs.s3.awsSecretAccessKey 73 | "Use AWS_SECRET_ACCESS_KEY environment variable to set this value" 74 | 75 | 76 | 77 | fs.s3a.access.key 78 | "Use AWS_ACCESS_KEY_ID environment variable to set this value" 79 | 80 | 81 | 82 | fs.s3a.secret.key 83 | "Use AWS_SECRET_ACCESS_KEY environment variable to set this value" 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/root/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | for init_script in /etc/hadoop-init.d/*; do 4 | "${init_script}" 5 | done 6 | 7 | supervisord -c /etc/supervisord.conf 8 | -------------------------------------------------------------------------------- /prestodb/hive3.1-hive/files/root/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | ln -s /usr/bin/re solveip /usr/libexec # mariadb-server installs resolveip in /usr/bin but mysql_install_db expects it in /usr/libexec 4 | mkdir /var/log/mysql /var/log/hive /var/log/hadoop-hdfs 5 | 6 | mysql_install_db 7 | 8 | chown -R mysql:mysql /var/lib/mysql 9 | 10 | /usr/bin/mysqld_safe & 11 | sleep 10s 12 | 13 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 14 | echo "CREATE DATABASE metastore;" | mysql 15 | /usr/bin/mysqladmin -u root password 'root' 16 | /opt/hive/bin/schematool -dbType mysql -initSchema 17 | 18 | killall mysqld 19 | sleep 10s 20 | chown -R mysql:mysql /var/log/mysql/ 21 | rm -rf /tmp/* /var/tmp/* 22 | 23 | adduser hive 24 | 25 | -------------------------------------------------------------------------------- /prestodb/iop4.2-base/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/centos6-oj8:unlabelled 14 | MAINTAINER Presto community 15 | 16 | RUN \ 17 | # Add IBM IOP repo 18 | wget -nv https://ibm-open-platform.ibm.com/repos/IOP/rhel/6/x86_64/4.2.x/GA/IOP.repo -P /etc/yum.repos.d \ 19 | 20 | # Install hive 21 | && yum install -y hive_4_2_0_0 \ 22 | 23 | # Alias log directories so that files in `/common` can use `/var/log/hadoop/*` format 24 | && mkdir -p /var/log/hadoop \ 25 | && ln -s /var/log/hadoop-hdfs /var/log/hadoop/hdfs \ 26 | && ln -s /var/log/hadoop-mapreduce /var/log/hadoop/mapreduce \ 27 | && ln -s /var/log/hadoop-yarn /var/log/hadoop/yarn \ 28 | 29 | # Cleanup 30 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 31 | 32 | COPY ./files / 33 | -------------------------------------------------------------------------------- /prestodb/iop4.2-base/README.md: -------------------------------------------------------------------------------- 1 | # iop4.2-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/iop4.2-base.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/iop4.2-base 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/iop4.2-base.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/iop4.2-base 7 | 8 | A Hadoop base image for creating images testing Presto, based on the IOP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /prestodb/iop4.2-base/files/etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | 20 | [include] 21 | files = /etc/supervisord.d/*.conf 22 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM prestodb/iop4.2-base:unlabelled 14 | MAINTAINER Presto community 15 | 16 | # CHANGE DEFAULT TIMEZONE 17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone 18 | 19 | # INSTALL MYSQL (Hive Metastore) 20 | RUN yum install -y mysql-server mysql-connector-java \ 21 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 22 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/iop/4.2.0.0/hive/lib/mysql-connector-java.jar 23 | 24 | # COPY CONFIGURATION FILES 25 | COPY ./files / 26 | 27 | # RUN SETUP 28 | RUN /root/setup.sh 29 | 30 | # DO CLEANUP 31 | RUN rm -rf /tmp/* /var/tmp/* 32 | 33 | # SETUP SOCKS PROXY 34 | RUN yum install -y openssh openssh-clients openssh-server 35 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa 36 | RUN ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key 37 | RUN ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key 38 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 39 | RUN chmod 755 /root && chmod 700 /root/.ssh 40 | RUN passwd --unlock root 41 | 42 | # HDFS PORTS 43 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 44 | 45 | # YARN PORTS 46 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 47 | 48 | # HIVE PORT 49 | EXPOSE 9083 10000 50 | 51 | # SOCKS PORT 52 | EXPOSE 1180 53 | 54 | CMD supervisord -c /etc/supervisord.conf 55 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/README.md: -------------------------------------------------------------------------------- 1 | # iop4.2-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/iop4.2-hive.svg 4 | [layers-link]: https://microbadger.com/images/prestodb/iop4.2-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/iop4.2-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/iop4.2-hive 7 | 8 | Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. 9 | For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | ## Run 14 | 15 | ``` 16 | $ docker run -d --name hadoop-master -h hadoop-master prestodb/iop4.2-hive 17 | ``` 18 | 19 | ## Oracle license 20 | 21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 23 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/capabilities.txt: -------------------------------------------------------------------------------- 1 | exposes_hive 2 | allows_creating_a_table_in_hive 3 | allows_selecting_from_the_table 4 | exposes_socks_proxy 5 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://hadoop-master:9000 23 | 24 | 25 | 26 | 27 | hadoop.proxyuser.oozie.hosts 28 | * 29 | 30 | 31 | hadoop.proxyuser.oozie.groups 32 | * 33 | 34 | 35 | 36 | 37 | hadoop.proxyuser.httpfs.hosts 38 | * 39 | 40 | 41 | hadoop.proxyuser.httpfs.groups 42 | * 43 | 44 | 45 | 46 | 47 | hadoop.proxyuser.llama.hosts 48 | * 49 | 50 | 51 | hadoop.proxyuser.llama.groups 52 | * 53 | 54 | 55 | 56 | 57 | hadoop.proxyuser.hue.hosts 58 | * 59 | 60 | 61 | hadoop.proxyuser.hue.groups 62 | * 63 | 64 | 65 | 66 | 67 | hadoop.proxyuser.mapred.hosts 68 | * 69 | 70 | 71 | hadoop.proxyuser.mapred.groups 72 | * 73 | 74 | 75 | 76 | 77 | hadoop.proxyuser.hive.hosts 78 | * 79 | 80 | 81 | 82 | hadoop.proxyuser.hive.groups 83 | * 84 | 85 | 86 | 87 | 88 | hadoop.proxyuser.hdfs.groups 89 | * 90 | 91 | 92 | 93 | hadoop.proxyuser.hdfs.hosts 94 | * 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/hadoop/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Set Hadoop-specific environment variables here. 17 | # Forcing YARN-based mapreduce implementaion. 18 | # Make sure to comment out if you want to go back to the default or 19 | # if you want this to be tweakable on a per-user basis 20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce 21 | 22 | # The maximum amount of heap to use, in MB. Default is 1000. 23 | export HADOOP_HEAPSIZE=256 24 | 25 | # Extra Java runtime options. Empty by default. 26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m" 27 | export YARN_OPTS="$YARN_OPTS -Xmx256m" 28 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | dfs.namenode.name.dir 20 | /var/lib/hadoop-hdfs/cache/name/ 21 | 22 | 23 | 24 | dfs.datanode.data.dir 25 | /var/lib/hadoop-hdfs/cache/data/ 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | mapred.job.tracker 22 | hadoop-master:8021 23 | 24 | 25 | 26 | mapreduce.framework.name 27 | yarn 28 | 29 | 30 | 31 | mapreduce.jobhistory.address 32 | hadoop-master:10020 33 | 34 | 35 | 36 | mapreduce.jobhistory.webapp.address 37 | hadoop-master:19888 38 | 39 | 40 | 41 | To set the value of tmp directory for map and reduce tasks. 42 | mapreduce.task.tmp.dir 43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/hadoop/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | yarn.nodemanager.aux-services 21 | mapreduce_shuffle 22 | 23 | 24 | 25 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 26 | org.apache.hadoop.mapred.ShuffleHandler 27 | 28 | 29 | 30 | yarn.log-aggregation-enable 31 | true 32 | 33 | 34 | 35 | yarn.dispatcher.exit-on-error 36 | true 37 | 38 | 39 | 40 | List of directories to store localized files in. 41 | yarn.nodemanager.local-dirs 42 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 43 | 44 | 45 | 46 | Where to store container logs. 47 | yarn.nodemanager.log-dirs 48 | /var/log/hadoop/yarn/containers 49 | 50 | 51 | 52 | Where to aggregate logs to. 53 | yarn.nodemanager.remote-app-log-dir 54 | /var/log/hadoop/yarn/apps 55 | 56 | 57 | 58 | Classpath for typical applications. 59 | yarn.application.classpath 60 | 61 | /etc/hadoop/conf, 62 | /usr/iop/4.2.0.0/hadoop/*, 63 | /usr/iop/4.2.0.0/hadoop/lib/*, 64 | /usr/iop/4.2.0.0/hadoop-hdfs/*, 65 | /usr/iop/4.2.0.0/hadoop-hdfs/lib/*, 66 | /usr/iop/4.2.0.0/hadoop-yarn/*, 67 | /usr/iop/4.2.0.0/hadoop-yarn/lib/*, 68 | /usr/iop/4.2.0.0/hadoop-mapreduce/*, 69 | /usr/iop/4.2.0.0/hadoop-mapreduce/lib/* 70 | 71 | 72 | 73 | 74 | yarn.resourcemanager.hostname 75 | hadoop-master 76 | 77 | 78 | 79 | yarn.log.server.url 80 | http://hadoop-master:19888/jobhistory/logs 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/hive/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.security.authorization.createtable.owner.grants 58 | ALL 59 | The set of privileges automatically granted to the owner whenever a table gets created. 60 | 61 | 62 | 63 | hive.users.in.admin.role 64 | hdfs,hive 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-datanode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-namenode.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | command=hive --service metastore 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-metastore.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=true 10 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-nodemanager] 2 | command=yarn nodemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-nodemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/etc/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-resourcemanager] 2 | command=yarn resourcemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-resourcemanager.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /prestodb/iop4.2-hive/files/root/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 0 make file system hostname resolvable 4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts 5 | 6 | # 1 format namenode 7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 8 | 9 | # workaround for 'could not open session' bug as suggested here: 10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 11 | rm -f /etc/security/limits.d/hdfs.conf 12 | su -c "echo 'N' | hdfs namenode -format" hdfs 13 | 14 | # 2 start hdfs 15 | su -c "hdfs datanode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-datanode.log" hdfs& 16 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 17 | 18 | # 3 wait for process starting 19 | sleep 10 20 | 21 | # 4 exec hdfs init script 22 | /usr/iop/4.2.0.0/hadoop/libexec/init-hdfs.sh 23 | 24 | # 5 init hive directories 25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 28 | 29 | # 6 stop hdfs 30 | killall java 31 | 32 | # 7 setup metastore 33 | mysql_install_db 34 | 35 | /usr/bin/mysqld_safe & 36 | sleep 10s 37 | 38 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 39 | echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/iop/4.2.0.0/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql 40 | /usr/bin/mysqladmin -u root password 'root' 41 | 42 | killall mysqld 43 | sleep 10s 44 | mkdir /var/log/mysql/ 45 | chown mysql:mysql /var/log/mysql/ 46 | -------------------------------------------------------------------------------- /prestodb/spark/build-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | VERSION=3.5.1 6 | 7 | docker build -t prestodb/spark-base:$VERSION ./spark-base 8 | docker build -t prestodb/spark-master:$VERSION ./spark-master 9 | docker build -t prestodb/spark-worker:$VERSION ./spark-worker 10 | docker build -t prestodb/spark-submit:$VERSION ./spark-submit 11 | -------------------------------------------------------------------------------- /prestodb/spark/spark-base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM eclipse-temurin:11-jdk 2 | MAINTAINER Presto community 3 | LABEL version="0.2" 4 | 5 | ENV DAEMON_RUN=true 6 | ENV SPARK_VERSION=3.5.1 7 | ENV HADOOP_VERSION=3 8 | ENV SCALA_VERSION=2.13.14 9 | ENV SCALA_HOME=/usr/share/scala 10 | ENV SPARK_HOME=/spark 11 | 12 | RUN apt-get update && apt-get install -y curl vim wget software-properties-common ssh net-tools ca-certificates jq 13 | 14 | RUN cd "/tmp" && \ 15 | wget --no-verbose "https://downloads.typesafe.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.tgz" && \ 16 | tar xzf "scala-${SCALA_VERSION}.tgz" && \ 17 | mkdir "${SCALA_HOME}" && \ 18 | rm "/tmp/scala-${SCALA_VERSION}/bin/"*.bat && \ 19 | mv "/tmp/scala-${SCALA_VERSION}/bin" "/tmp/scala-${SCALA_VERSION}/lib" "${SCALA_HOME}" && \ 20 | ln -s "${SCALA_HOME}/bin/"* "/usr/bin/" && \ 21 | rm -rf "/tmp/"* 22 | 23 | # Add Dependencies for PySpark 24 | RUN apt-get install -y python3 python3-pip python3-numpy python3-matplotlib python3-scipy python3-pandas python3-simpy 25 | RUN update-alternatives --install "/usr/bin/python" "python" "$(which python3)" 1 26 | 27 | #Scala instalation 28 | RUN export PATH="/usr/local/sbt/bin:$PATH" && apt update && apt install ca-certificates wget tar && mkdir -p "/usr/local/sbt" && wget -qO - --no-check-certificate "https://github.com/sbt/sbt/releases/download/v1.2.8/sbt-1.2.8.tgz" | tar xz -C /usr/local/sbt --strip-components=1 && sbt sbtVersion 29 | 30 | RUN wget --no-verbose http://apache.mirror.iphh.net/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ 31 | && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark \ 32 | && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz 33 | 34 | # Fix the value of PYTHONHASHSEED 35 | # Note: this is needed when you use Python 3.3 or greater 36 | ENV PYTHONHASHSEED 1 37 | -------------------------------------------------------------------------------- /prestodb/spark/spark-base/README.md: -------------------------------------------------------------------------------- 1 | # spark-base 2 | 3 | Docker image with a Spark 3 environment installed on JDK 11. Used in the 4 | spark-master, spark-submit, and spark-work images. 5 | 6 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster). 7 | -------------------------------------------------------------------------------- /prestodb/spark/spark-master/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_VERSION=3.5.1 2 | 3 | FROM prestodb/spark-base:${BASE_VERSION} 4 | MAINTAINER Presto community 5 | 6 | COPY start-master.sh / 7 | 8 | ENV SPARK_MASTER_PORT 7077 9 | ENV SPARK_MASTER_WEBUI_PORT 8080 10 | ENV SPARK_MASTER_LOG /spark/logs 11 | 12 | EXPOSE 8080 7077 6066 13 | 14 | CMD ["/bin/bash", "/start-master.sh"] 15 | -------------------------------------------------------------------------------- /prestodb/spark/spark-master/README.md: -------------------------------------------------------------------------------- 1 | # spark-master 2 | 3 | Spark master image used in Presto Spark integration tests (presto-spark-testing). 4 | 5 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster). 6 | -------------------------------------------------------------------------------- /prestodb/spark/spark-master/start-master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export SPARK_MASTER_HOST=`hostname` 4 | 5 | . "/spark/sbin/spark-config.sh" 6 | 7 | . "/spark/bin/load-spark-env.sh" 8 | 9 | mkdir -p $SPARK_MASTER_LOG 10 | 11 | export SPARK_HOME=/spark 12 | 13 | ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out 14 | 15 | cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out 16 | -------------------------------------------------------------------------------- /prestodb/spark/spark-submit/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_VERSION=3.5.1 2 | 3 | FROM prestodb/spark-base:${BASE_VERSION} 4 | MAINTAINER Presto community 5 | 6 | COPY spark-submit.sh / 7 | 8 | ENV SPARK_MASTER_URL="spark://spark-master:7077" 9 | ENV SPARK_SUBMIT_ARGS="" 10 | ENV SPARK_APPLICATION_ARGS "" 11 | #ENV SPARK_APPLICATION_JAR_LOCATION /opt/spark-apps/myjar.jar 12 | #ENV SPARK_APPLICATION_MAIN_CLASS my.main.Application 13 | 14 | CMD ["/bin/bash", "/spark-submit.sh"] 15 | -------------------------------------------------------------------------------- /prestodb/spark/spark-submit/README.md: -------------------------------------------------------------------------------- 1 | # spark-submit 2 | 3 | Spark submit image used in Presto Spark integration tests (presto-spark-testing). 4 | 5 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster). 6 | -------------------------------------------------------------------------------- /prestodb/spark/spark-submit/spark-submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /spark/bin/spark-submit \ 4 | --class ${SPARK_APPLICATION_MAIN_CLASS} \ 5 | --master ${SPARK_MASTER_URL} \ 6 | --deploy-mode cluster \ 7 | --total-executor-cores 1 \ 8 | ${SPARK_SUBMIT_ARGS} \ 9 | ${SPARK_APPLICATION_JAR_LOCATION} \ 10 | ${SPARK_APPLICATION_ARGS} \ 11 | -------------------------------------------------------------------------------- /prestodb/spark/spark-worker/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_VERSION=3.5.1 2 | 3 | FROM prestodb/spark-base:${BASE_VERSION} 4 | MAINTAINER Presto community 5 | 6 | COPY start-worker.sh / 7 | 8 | ENV SPARK_WORKER_WEBUI_PORT 8081 9 | ENV SPARK_WORKER_LOG /spark/logs 10 | ENV SPARK_MASTER "spark://spark-master:7077" 11 | 12 | EXPOSE 8081 13 | 14 | CMD ["/bin/bash", "/start-worker.sh"] 15 | -------------------------------------------------------------------------------- /prestodb/spark/spark-worker/README.md: -------------------------------------------------------------------------------- 1 | # spark-worker 2 | 3 | Spark worker image used in Presto Spark integration tests (presto-spark-testing). 4 | 5 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster). 6 | -------------------------------------------------------------------------------- /prestodb/spark/spark-worker/start-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . "/spark/sbin/spark-config.sh" 4 | . "/spark/bin/load-spark-env.sh" 5 | 6 | mkdir -p $SPARK_WORKER_LOG 7 | 8 | export SPARK_HOME=/spark 9 | 10 | ln -sf /dev/stdout $SPARK_WORKER_LOG/spark-worker.out 11 | 12 | /spark/sbin/../bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port $SPARK_WORKER_WEBUI_PORT $SPARK_MASTER >> $SPARK_WORKER_LOG/spark-worker.out 13 | --------------------------------------------------------------------------------