├── .gitignore
├── .travis.yml
├── .travis
└── github_tags_push_deploy_key.enc
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE.txt
├── Makefile
├── README.md
├── bin
├── depend.sh
├── flag.sh
├── label-parent.sh
├── push.sh
└── test.sh
├── etc
└── compose
│ ├── cdh5.12-hive
│ └── docker-compose.yml
│ ├── cdh5.15-hive
│ └── docker-compose.yml
│ ├── hdp2.5-hive
│ └── docker-compose.yml
│ ├── hdp2.6-hive
│ └── docker-compose.yml
│ ├── hive3.1-hive
│ └── docker-compose.yml
│ └── iop4.2-hive
│ └── docker-compose.yml
└── prestodb
├── cdh5.12-hive-kerberized
├── Dockerfile
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── container-executor.cfg
│ │ │ ├── core-site.xml
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ ├── taskcontroller.cfg
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ ├── hive-site.xml
│ │ │ └── hiveserver2-site.xml
│ ├── krb5.conf
│ └── supervisord.d
│ │ └── kdc.conf
│ └── var
│ └── kerberos
│ └── krb5kdc
│ ├── kadm5.acl
│ └── kdc.conf
├── cdh5.12-hive
├── Dockerfile
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── capacity-scheduler.xml
│ │ │ ├── core-site.xml
│ │ │ ├── hadoop-env.sh
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ ├── hive-env.sh
│ │ │ └── hive-site.xml
│ ├── supervisord.conf
│ └── supervisord.d
│ │ ├── hdfs-datanode.conf
│ │ ├── hdfs-namenode.conf
│ │ ├── hive-metastore.conf
│ │ ├── hive-server2.conf
│ │ ├── mysql-metastore.conf
│ │ ├── socks-proxy.conf
│ │ ├── sshd.conf
│ │ ├── yarn-nodemanager.conf
│ │ └── yarn-resourcemanager.conf
│ └── root
│ └── setup.sh
├── cdh5.15-hive-kerberized
├── Dockerfile
├── README.md
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── container-executor.cfg
│ │ │ ├── core-site.xml
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ ├── taskcontroller.cfg
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ ├── hive-site.xml
│ │ │ └── hiveserver2-site.xml
│ ├── krb5.conf
│ └── supervisord.d
│ │ └── kdc.conf
│ └── var
│ └── kerberos
│ └── krb5kdc
│ ├── kadm5.acl
│ └── kdc.conf
├── cdh5.15-hive
├── Dockerfile
├── README.md
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── capacity-scheduler.xml
│ │ │ ├── core-site.xml
│ │ │ ├── hadoop-env.sh
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ ├── hive-env.sh
│ │ │ └── hive-site.xml
│ ├── supervisord.conf
│ └── supervisord.d
│ │ ├── hdfs-datanode.conf
│ │ ├── hdfs-namenode.conf
│ │ ├── hive-metastore.conf
│ │ ├── hive-server2.conf
│ │ ├── mysql-metastore.conf
│ │ ├── socks-proxy.conf
│ │ ├── sshd.conf
│ │ ├── yarn-nodemanager.conf
│ │ └── yarn-resourcemanager.conf
│ └── root
│ └── setup.sh
├── centos6-oj8-openldap
├── Dockerfile
├── README.md
└── files
│ └── etc
│ ├── openldap
│ ├── certs
│ │ ├── active-directory-certificate.crt
│ │ ├── openldap-certificate.pem
│ │ ├── private.pem
│ │ └── serial.txt
│ └── setup
│ │ ├── createOU.ldif
│ │ ├── ldap_disable_bind_anon.ldif
│ │ ├── memberof.ldif
│ │ ├── modify_server.ldif
│ │ └── refint.ldif
│ ├── supervisord.conf
│ └── supervisord.d
│ └── slapd.conf
├── centos6-oj8
├── Dockerfile
└── README.md
├── centos6-presto-admin-tests-build
├── Dockerfile
└── README.md
├── centos6-presto-admin-tests-runtime
├── Dockerfile
├── README.md
└── vagrant_insecure_rsa
├── centos7-oj8
├── Dockerfile
└── files
│ ├── opt
│ └── prestodb
│ │ └── site-override.xslt
│ └── usr
│ └── local
│ └── bin
│ ├── apply-all-site-xml-overrides
│ └── apply-site-xml-override
├── dns
├── Dockerfile
└── README.md
├── hdp2.5-base
├── Dockerfile
├── README.md
└── files
│ └── etc
│ └── supervisord.conf
├── hdp2.5-hive-kerberized
├── Dockerfile
├── README.md
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── core-site.xml
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ ├── hive-site.xml
│ │ │ └── hiveserver2-site.xml
│ ├── krb5.conf
│ └── supervisord.d
│ │ └── kdc.conf
│ └── var
│ └── kerberos
│ └── krb5kdc
│ ├── kadm5.acl
│ └── kdc.conf
├── hdp2.5-hive
├── Dockerfile
├── README.md
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── capacity-scheduler.xml
│ │ │ ├── core-site.xml
│ │ │ ├── hadoop-env.sh
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ └── hive-site.xml
│ ├── supervisord.d
│ │ ├── hdfs-datanode.conf
│ │ ├── hdfs-namenode.conf
│ │ ├── hive-metastore.conf
│ │ ├── hive-server2.conf
│ │ ├── mapreduce-historyserver.conf
│ │ ├── mysql-metastore.conf
│ │ ├── socks-proxy.conf
│ │ ├── sshd.conf
│ │ ├── yarn-nodemanager.conf
│ │ ├── yarn-resourcemanager.conf
│ │ └── zookeeper.conf
│ └── zookeeper
│ │ └── conf
│ │ ├── log4j.properties
│ │ ├── zoo.cfg
│ │ └── zookeeper-env.sh
│ └── root
│ └── setup.sh
├── hdp2.6-hive-kerberized
├── Dockerfile
├── README.md
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── core-site.xml
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ ├── hive-site.xml
│ │ │ └── hiveserver2-site.xml
│ ├── krb5.conf
│ └── supervisord.d
│ │ └── kdc.conf
│ └── var
│ └── kerberos
│ └── krb5kdc
│ ├── kadm5-other.acl
│ ├── kadm5.acl
│ └── kdc.conf
├── hdp2.6-hive
├── Dockerfile
├── README.md
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── capacity-scheduler.xml
│ │ │ ├── core-site.xml
│ │ │ ├── hadoop-env.sh
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ └── hive-site.xml
│ ├── supervisord.conf
│ └── supervisord.d
│ │ ├── hdfs-datanode.conf
│ │ ├── hdfs-namenode.conf
│ │ ├── hive-metastore.conf
│ │ ├── hive-server2.conf
│ │ ├── mysql-metastore.conf
│ │ ├── socks-proxy.conf
│ │ ├── sshd.conf
│ │ ├── yarn-nodemanager.conf
│ │ └── yarn-resourcemanager.conf
│ └── root
│ └── setup.sh
├── hive3.1-hive
├── Dockerfile
└── files
│ ├── etc
│ ├── hadoop-init.d
│ │ ├── init-hdfs.sh
│ │ └── set-aws-creds.sh
│ ├── supervisord.conf
│ └── supervisord.d
│ │ ├── hdfs-datanode.conf
│ │ ├── hdfs-namenode.conf
│ │ ├── hive-metastore.conf
│ │ ├── hive-server2.conf
│ │ ├── mysql-metastore.conf
│ │ ├── socks-proxy.conf
│ │ └── sshd.conf
│ ├── opt
│ ├── hadoop
│ │ └── etc
│ │ │ └── hadoop
│ │ │ ├── core-site.xml
│ │ │ ├── hadoop-env.sh
│ │ │ └── hdfs-site.xml
│ └── hive
│ │ └── conf
│ │ ├── hive-env.sh
│ │ └── hive-site.xml
│ └── root
│ ├── entrypoint.sh
│ └── setup.sh
├── iop4.2-base
├── Dockerfile
├── README.md
└── files
│ └── etc
│ └── supervisord.conf
├── iop4.2-hive
├── Dockerfile
├── README.md
├── capabilities.txt
└── files
│ ├── etc
│ ├── hadoop
│ │ └── conf
│ │ │ ├── capacity-scheduler.xml
│ │ │ ├── core-site.xml
│ │ │ ├── hadoop-env.sh
│ │ │ ├── hdfs-site.xml
│ │ │ ├── mapred-site.xml
│ │ │ └── yarn-site.xml
│ ├── hive
│ │ └── conf
│ │ │ └── hive-site.xml
│ └── supervisord.d
│ │ ├── hdfs-datanode.conf
│ │ ├── hdfs-namenode.conf
│ │ ├── hive-metastore.conf
│ │ ├── hive-server2.conf
│ │ ├── mysql-metastore.conf
│ │ ├── socks-proxy.conf
│ │ ├── sshd.conf
│ │ ├── yarn-nodemanager.conf
│ │ └── yarn-resourcemanager.conf
│ └── root
│ └── setup.sh
└── spark
├── build-images.sh
├── spark-base
├── Dockerfile
└── README.md
├── spark-master
├── Dockerfile
├── README.md
└── start-master.sh
├── spark-submit
├── Dockerfile
├── README.md
└── spark-submit.sh
└── spark-worker
├── Dockerfile
├── README.md
└── start-worker.sh
/.gitignore:
--------------------------------------------------------------------------------
1 | *.iml
2 | *.ipr
3 | *.iws
4 | .DS_Store
5 | .idea
6 | depends
7 | flags
8 | graphviz
9 | dependency_graph.svg
10 | *.swp
11 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: generic
2 | dist: trusty
3 |
4 | env:
5 | - IMAGE_TO_BUILD=prestodb/hdp2.5-hive-kerberized IMAGE_TO_TEST=hdp2.5-hive
6 | - IMAGE_TO_BUILD=prestodb/hdp2.6-hive-kerberized IMAGE_TO_TEST=hdp2.6-hive
7 | - IMAGE_TO_BUILD=prestodb/cdh5.12-hive-kerberized IMAGE_TO_TEST=cdh5.12-hive
8 | - IMAGE_TO_BUILD=prestodb/cdh5.15-hive-kerberized IMAGE_TO_TEST=cdh5.15-hive
9 | - IMAGE_TO_BUILD=prestodb/iop4.2-hive IMAGE_TO_TEST=iop4.2-hive
10 | - IMAGE_TO_BUILD=prestodb/hive3.1-hive IMAGE_TO_TEST=hive3.1-hive
11 |
12 | # Prevent duplicate builds on tag pushes.
13 | # See https://github.com/mockito/mockito/commit/500519aac4cc13770aa47c1eb7d28e905c27e7aa
14 | branches:
15 | except:
16 | - /^release.*/
17 | - /^latest-snapshot.*/
18 |
19 | services:
20 | - docker
21 |
22 | install: make ${IMAGE_TO_BUILD}
23 |
24 | script: make test IMAGE_TO_TEST=${IMAGE_TO_TEST}
25 |
26 | before_deploy:
27 | # decrypt the github deploy key
28 | - openssl aes-256-cbc -K $encrypted_cd2beb64619c_key -iv $encrypted_cd2beb64619c_iv
29 | -in .travis/github_tags_push_deploy_key.enc -out .travis/github_tags_push_deploy_key -d
30 | - chmod 600 .travis/github_tags_push_deploy_key
31 | - eval "$(ssh-agent -s)"
32 | - ssh-add .travis/github_tags_push_deploy_key
33 | - docker login --username=${DOCKERHUB_LOGIN} --password=${DOCKERHUB_PASSWORD}
34 |
35 | # Publish snapshots to Dockerhub for commits tagged `latest-snapshot-*`
36 | deploy:
37 | on:
38 | all_branches: true
39 | condition: $( git tag --points-at HEAD | grep '^latest-snapshot' ) != ''
40 | skip_cleanup: true
41 | provider: script
42 | script: make snapshot
43 |
--------------------------------------------------------------------------------
/.travis/github_tags_push_deploy_key.enc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prestodb/docker-images/3f7be0efd4d420065ed5f813f32267b453ec6174/.travis/github_tags_push_deploy_key.enc
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Code of Conduct
2 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.fb.com/codeofconduct) so that you can understand what actions will and will not be tolerated.
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing to docker-images
2 | We want to make contributing to this project as easy and transparent as
3 | possible.
4 |
5 | ## Pull Requests
6 | We actively welcome your pull requests.
7 |
8 | 1. Fork the repo and create your branch from `master`.
9 | 2. If you've added code that should be tested, add tests.
10 | 3. If you've changed APIs, update the documentation.
11 | 4. Ensure the test suite passes.
12 | 5. Make sure your code lints.
13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA").
14 |
15 | ## Contributor License Agreement ("CLA")
16 | In order to accept your pull request, we need you to submit a CLA. You only need to do this once, so if you've done this for one repository in the [prestodb](https://github.com/prestodb) organization, you're good to go. If you are submitting a pull request for the first time, the communitybridge-easycla bot will notify you if you haven't signed, and will provide you with a link. If you are contributing on behalf of a company, you might want to let the person who manages your corporate CLA whitelist know they will be receiving a request from you.
17 |
18 | ## Issues
19 | We use GitHub issues to track public bugs. Please ensure your description is
20 | clear and has sufficient instructions to be able to reproduce the issue.
21 |
22 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
23 | disclosure of security bugs. In those cases, please go through the process
24 | outlined on that page and do not file a public issue.
25 |
26 | ## License
27 | By contributing to docker-images, you agree that your contributions will be licensed
28 | under the LICENSE file in the root directory of this source tree.
--------------------------------------------------------------------------------
/bin/flag.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | usage() {
4 | echo "$0 {target image}" >&2
5 | }
6 |
7 | #
8 | # This doesn't handle ARGs with a default supplied in the Dockerfile.
9 | # Feel free to add that functionality if needed. For now, YAGNI.
10 | #
11 | find_args() {
12 | local target_image=$(dirname "$target_dockerfile")
13 | awk -v image="$target_image" '
14 | BEGIN {
15 | ARG_PATTERN = "^\\s*ARG";
16 | print "DBFLAGS_" image " :=";
17 | }
18 |
19 | $1 == "ARG" {
20 | key = $2;
21 | print "DBFLAGS_" image " += --build-arg " key "=$(" key ")";
22 | }' "$1"
23 | }
24 |
25 | if [ $# -lt 1 ]; then
26 | usage
27 | exit 1
28 | fi
29 |
30 | target_dockerfile=$1
31 |
32 | find_args "$target_dockerfile"
33 |
--------------------------------------------------------------------------------
/bin/label-parent.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | #
4 | # Requires that the top layer is a LABEL layer. If it is, return the checksum
5 | # of its parent layer and exit 0. If the top layer isn't a LABEL layer, exit 1.
6 | #
7 | docker history --no-trunc --format "{{.ID}}\t{{.CreatedBy}}" "$1" | awk '
8 | BEGIN {
9 | ec = 1;
10 | }
11 | $5 == "LABEL" && NR == 1 {
12 | getline;
13 | ec = 0;
14 | exit;
15 | }
16 | END {
17 | if (ec == 0) {
18 | print $1
19 | }
20 | exit ec
21 | }'
22 |
--------------------------------------------------------------------------------
/bin/push.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | while [ -n "$1" ]; do
4 | docker push "$1"
5 | shift
6 | done
7 |
--------------------------------------------------------------------------------
/etc/compose/cdh5.12-hive/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2.0'
2 | services:
3 | hadoop-master:
4 | hostname: hadoop-master
5 | image: prestodb/cdh5.12-hive:latest
6 |
7 |
--------------------------------------------------------------------------------
/etc/compose/cdh5.15-hive/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2.0'
2 | services:
3 | hadoop-master:
4 | hostname: hadoop-master
5 | image: prestodb/cdh5.15-hive:latest
6 |
7 |
--------------------------------------------------------------------------------
/etc/compose/hdp2.5-hive/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2.0'
2 | services:
3 | hadoop-master:
4 | hostname: hadoop-master
5 | image: prestodb/hdp2.5-hive:latest
6 |
7 |
--------------------------------------------------------------------------------
/etc/compose/hdp2.6-hive/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2.0'
2 | services:
3 | hadoop-master:
4 | hostname: hadoop-master
5 | image: prestodb/hdp2.6-hive:latest
6 |
7 |
--------------------------------------------------------------------------------
/etc/compose/hive3.1-hive/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2.0'
2 | services:
3 | hadoop-master:
4 | hostname: hadoop-master
5 | image: prestodb/hive3.1-hive:latest
6 |
--------------------------------------------------------------------------------
/etc/compose/iop4.2-hive/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2.0'
2 | services:
3 | hadoop-master:
4 | hostname: hadoop-master
5 | image: prestodb/iop4.2-hive:latest
6 |
7 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/container-executor.cfg:
--------------------------------------------------------------------------------
1 | yarn.nodemanager.local-dirs=/var/lib/hadoop-yarn/cache/yarn/nm-local-dir
2 | yarn.nodemanager.linux-container-executor.group=yarn
3 | yarn.nodemanager.log-dirs=/var/log/hadoop-yarn/containers
4 | banned.users=yarn,mapred,bin
5 | min.user.id=400
6 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
30 | dfs.block.access.token.enable
31 | true
32 |
33 |
34 |
35 |
36 | dfs.namenode.keytab.file
37 | /etc/hadoop/conf/hdfs.keytab
38 |
39 |
40 | dfs.namenode.kerberos.principal
41 | hdfs/hadoop-master@LABS.TERADATA.COM
42 |
43 |
44 | dfs.namenode.kerberos.internal.spnego.principal
45 | HTTP/hadoop-master@LABS.TERADATA.COM
46 |
47 |
48 |
49 |
50 | dfs.secondary.namenode.keytab.file
51 | /etc/hadoop/conf/hdfs.keytab
52 |
53 |
54 | dfs.secondary.namenode.kerberos.principal
55 | hdfs/hadoop-master@LABS.TERADATA.COM
56 |
57 |
58 | dfs.secondary.namenode.kerberos.internal.spnego.principal
59 | HTTP/hadoop-master@LABS.TERADATA.COM
60 |
61 |
62 |
63 |
64 | dfs.datanode.keytab.file
65 | /etc/hadoop/conf/hdfs.keytab
66 |
67 |
68 | dfs.datanode.kerberos.principal
69 | hdfs/hadoop-master@LABS.TERADATA.COM
70 |
71 |
72 |
73 |
74 | dfs.webhdfs.enabled
75 | true
76 |
77 |
78 |
79 |
80 | dfs.web.authentication.kerberos.principal
81 | HTTP/hadoop-master@LABS.TERADATA.COM
82 |
83 |
84 |
85 | dfs.web.authentication.kerberos.keytab
86 | /etc/hadoop/conf/HTTP.keytab
87 |
88 |
89 |
90 | ignore.secure.ports.for.testing
91 | true
92 |
93 |
94 |
95 | dfs.http.policy
96 | HTTP_ONLY
97 |
98 |
99 |
100 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.jobhistory.address
27 | hadoop-master:10020
28 |
29 |
30 |
31 | mapreduce.jobhistory.webapp.address
32 | hadoop-master:19888
33 |
34 |
35 |
36 | To set the value of tmp directory for map and reduce tasks.
37 | mapreduce.task.tmp.dir
38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
39 |
40 |
41 |
42 |
43 | mapreduce.jobhistory.keytab
44 | /etc/hadoop/conf/mapred.keytab
45 |
46 |
47 |
48 | mapreduce.jobhistory.principal
49 | mapred/hadoop-master@LABS.TERADATA.COM
50 |
51 |
52 |
53 |
54 | mapreduce.jobtracker.kerberos.principal
55 | mapred/hadoop-master@LABS.TERADATA.COM
56 |
57 |
58 |
59 | mapreduce.jobtracker.keytab.file
60 | /etc/hadoop/conf/mapred.keytab
61 |
62 |
63 |
64 |
65 | mapreduce.tasktracker.kerberos.principal
66 | mapred/hadoop-master@LABS.TERADATA.COM
67 |
68 |
69 |
70 | mapreduce.tasktracker.keytab.file
71 | /etc/hadoop/conf/mapred.keytab
72 |
73 |
74 |
75 | mapred.task.tracker.task-controller
76 | org.apache.hadoop.mapred.LinuxTaskController
77 |
78 |
79 |
80 | mapreduce.tasktracker.group
81 | mapred
82 |
83 |
84 |
85 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/hadoop/conf/taskcontroller.cfg:
--------------------------------------------------------------------------------
1 | hadoop.log.dir=/var/log/hadoop-mapreduce
2 | mapreduce.tasktracker.group=mapred
3 | banned.users=mapred,bin
4 | min.user.id=400
5 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 | hive.security.authenticator.manager
23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator
24 |
25 |
26 |
27 |
28 | hive.security.authorization.manager
29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory
30 | SQL standards based Hive authorization
31 |
32 |
33 |
34 | hive.security.authorization.enabled
35 | true
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/krb5.conf:
--------------------------------------------------------------------------------
1 | [logging]
2 | default = FILE:/var/log/krb5libs.log
3 | kdc = FILE:/var/log/krb5kdc.log
4 | admin_server = FILE:/var/log/kadmind.log
5 |
6 | [libdefaults]
7 | default_realm = LABS.TERADATA.COM
8 | dns_lookup_realm = false
9 | dns_lookup_kdc = false
10 | ticket_lifetime = 24h
11 | renew_lifetime = 7d
12 | forwardable = true
13 |
14 | [realms]
15 | LABS.TERADATA.COM = {
16 | kdc = hadoop-master
17 | admin_server = hadoop-master
18 | }
19 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/etc/supervisord.d/kdc.conf:
--------------------------------------------------------------------------------
1 | [program:krb5kdc]
2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -r LABS.TERADATA.COM -P /var/run/krb5kdc.pid -n"
3 | autostart=true
4 | autorestart=true
5 | redirect_stderr=true
6 | stdout_logfile=/dev/stdout
7 | stdout_logfile_maxbytes=0
8 |
9 | [program:kadmind]
10 | command=/bin/bash -c "exec /usr/sbin/kadmind -r LABS.TERADATA.COM -P /var/run/kadmind.pid -nofork"
11 | autostart=true
12 | autorestart=true
13 | redirect_stderr=true
14 | stdout_logfile=/dev/stdout
15 | stdout_logfile_maxbytes=0
16 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl:
--------------------------------------------------------------------------------
1 | */admin@LABS.TERADATA.COM *
2 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf:
--------------------------------------------------------------------------------
1 | [kdcdefaults]
2 | kdc_ports = 88
3 | kdc_tcp_ports = 88
4 |
5 | [realms]
6 | LABS.TERADATA.COM = {
7 | #master_key_type = aes256-cts
8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl
9 | dict_file = /usr/share/dict/words
10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
12 | }
13 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/centos6-oj8:unlabelled
14 | MAINTAINER Presto community
15 |
16 | # Change default timezone
17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone
18 |
19 | # Setup CDH repo, pin the CDH distribution to a concrete version
20 | RUN wget -nv https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo -P /etc/yum.repos.d \
21 | && rpm --import https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera \
22 | && sed -i '/^baseurl=/c\baseurl=https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.12.2/' /etc/yum.repos.d/cloudera-cdh5.repo
23 |
24 | # Install Hadoop, Hive (w/ MySQL)
25 | RUN \
26 | set -xeu && \
27 | yum install -y \
28 | hadoop-hdfs-namenode \
29 | hadoop-hdfs-secondarynamenode \
30 | hadoop-hdfs-datanode \
31 | \
32 | hadoop-mapreduce \
33 | \
34 | hadoop-yarn-resourcemanager \
35 | hadoop-yarn-nodemanager \
36 | \
37 | hadoop-client \
38 | hadoop-conf-pseudo \
39 | \
40 | hive \
41 | hive-metastore \
42 | hive-server2 \
43 | \
44 | mysql-server mysql-connector-java \
45 | \
46 | # Cleanup
47 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
48 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/mysql-connector-java.jar
49 |
50 | # Copy configuration files
51 | COPY ./files /
52 |
53 | # Run setup script
54 | RUN /root/setup.sh \
55 | && rm -rf /tmp/* /var/tmp/*
56 |
57 | # Setup sock proxy
58 | RUN yum install -y openssh openssh-clients openssh-server && yum -y clean all
59 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \
60 | && ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key \
61 | && ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key \
62 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
63 | RUN chmod 755 /root && chmod 700 /root/.ssh
64 | RUN passwd --unlock root
65 |
66 | # HDFS ports
67 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
68 |
69 | # YARN ports
70 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888
71 |
72 | # HIVE port
73 | EXPOSE 9083 10000
74 |
75 | # SOCKS port
76 | EXPOSE 1180
77 |
78 | CMD supervisord -c /etc/supervisord.conf
79 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hadoop/conf/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | fs.defaultFS
22 | hdfs://hadoop-master:9000
23 |
24 |
25 |
26 |
27 | hadoop.proxyuser.oozie.hosts
28 | *
29 |
30 |
31 | hadoop.proxyuser.oozie.groups
32 | *
33 |
34 |
35 |
36 |
37 | hadoop.proxyuser.httpfs.hosts
38 | *
39 |
40 |
41 | hadoop.proxyuser.httpfs.groups
42 | *
43 |
44 |
45 |
46 |
47 | hadoop.proxyuser.llama.hosts
48 | *
49 |
50 |
51 | hadoop.proxyuser.llama.groups
52 | *
53 |
54 |
55 |
56 |
57 | hadoop.proxyuser.hue.hosts
58 | *
59 |
60 |
61 | hadoop.proxyuser.hue.groups
62 | *
63 |
64 |
65 |
66 |
67 | hadoop.proxyuser.mapred.hosts
68 | *
69 |
70 |
71 | hadoop.proxyuser.mapred.groups
72 | *
73 |
74 |
75 |
76 |
77 | hadoop.proxyuser.hive.hosts
78 | *
79 |
80 |
81 |
82 | hadoop.proxyuser.hive.groups
83 | *
84 |
85 |
86 |
87 |
88 | hadoop.proxyuser.hdfs.groups
89 | *
90 |
91 |
92 |
93 | hadoop.proxyuser.hdfs.hosts
94 | *
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hadoop/conf/hadoop-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Set Hadoop-specific environment variables here.
17 | # Forcing YARN-based mapreduce implementaion.
18 | # Make sure to comment out if you want to go back to the default or
19 | # if you want this to be tweakable on a per-user basis
20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
21 |
22 | # The maximum amount of heap to use, in MB. Default is 1000.
23 | export HADOOP_HEAPSIZE=512
24 |
25 | # Extra Java runtime options. Empty by default.
26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m"
27 | export YARN_OPTS="$YARN_OPTS -Xmx256m"
28 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.jobhistory.address
27 | hadoop-master:10020
28 |
29 |
30 |
31 | mapreduce.jobhistory.webapp.address
32 | hadoop-master:19888
33 |
34 |
35 |
36 | To set the value of tmp directory for map and reduce tasks.
37 | mapreduce.task.tmp.dir
38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
39 |
40 |
41 |
42 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hadoop/conf/yarn-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | yarn.nodemanager.aux-services
22 | mapreduce_shuffle
23 |
24 |
25 |
26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class
27 | org.apache.hadoop.mapred.ShuffleHandler
28 |
29 |
30 |
31 | yarn.log-aggregation-enable
32 | true
33 |
34 |
35 |
36 | yarn.dispatcher.exit-on-error
37 | true
38 |
39 |
40 |
41 | List of directories to store localized files in.
42 | yarn.nodemanager.local-dirs
43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir
44 |
45 |
46 |
47 | Where to store container logs.
48 | yarn.nodemanager.log-dirs
49 | /var/log/hadoop-yarn/containers
50 |
51 |
52 |
53 | Where to aggregate logs to.
54 | yarn.nodemanager.remote-app-log-dir
55 | /var/log/hadoop-yarn/apps
56 |
57 |
58 |
59 | Classpath for typical applications.
60 | yarn.application.classpath
61 |
62 | /etc/hadoop/conf,
63 | /usr/lib/hadoop/*,
64 | /usr/lib/hadoop/lib/*,
65 | /usr/lib/hadoop-hdfs/*,
66 | /usr/lib/hadoop-hdfs/lib/*,
67 | /usr/lib/hadoop-yarn/*,
68 | /usr/lib/hadoop-yarn/lib/*,
69 | /usr/lib/hadoop-mapreduce/*,
70 | /usr/lib/hadoop-mapreduce/lib/*
71 |
72 |
73 |
74 |
75 | yarn.resourcemanager.hostname
76 | hadoop-master
77 |
78 |
79 |
80 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hive/conf/hive-env.sh:
--------------------------------------------------------------------------------
1 | export HADOOP_OPTS="$HADOOP_OPTS -Dhive.root.logger=INFO,console"
2 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/hive/conf/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | javax.jdo.option.ConnectionURL
22 | jdbc:mysql://localhost/metastore
23 | the URL of the MySQL database
24 |
25 |
26 |
27 | javax.jdo.option.ConnectionDriverName
28 | com.mysql.jdbc.Driver
29 |
30 |
31 |
32 | javax.jdo.option.ConnectionUserName
33 | root
34 |
35 |
36 |
37 | javax.jdo.option.ConnectionPassword
38 | root
39 |
40 |
41 |
42 | datanucleus.autoCreateSchema
43 | false
44 |
45 |
46 |
47 | datanucleus.fixedDatastore
48 | true
49 |
50 |
51 |
52 | datanucleus.autoStartMechanism
53 | SchemaTable
54 |
55 |
56 |
57 | hive.security.authorization.createtable.owner.grants
58 | ALL
59 | The set of privileges automatically granted to the owner whenever a table gets created.
60 |
61 |
62 |
63 | hive.users.in.admin.role
64 | hdfs,hive
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/hdfs-datanode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-datanode]
2 | command=hdfs datanode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/hdfs-namenode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-namenode]
2 | command=hdfs namenode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/hive-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:hive-metastore]
2 | command=hive --service metastore
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-metastore.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/hive-server2.conf:
--------------------------------------------------------------------------------
1 | [program:hive-server2]
2 | command=hive --service hiveserver2
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-server2.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/mysql-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:mysql-metastore]
2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mysql
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/mysql/mysql.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/socks-proxy.conf:
--------------------------------------------------------------------------------
1 | [program:socks-proxy]
2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/socks-proxy
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/sshd.conf:
--------------------------------------------------------------------------------
1 | [program:sshd]
2 | command=/usr/sbin/sshd -D
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/sshd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/yarn-nodemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-nodemanager]
2 | command=yarn nodemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-nodemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/etc/supervisord.d/yarn-resourcemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-resourcemanager]
2 | command=yarn resourcemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-resourcemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.12-hive/files/root/setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ex
2 |
3 | # 0 make file system hostname resolvable
4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts
5 |
6 | # 1 format namenode
7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/
8 |
9 | # workaround for 'could not open session' bug as suggested here:
10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610
11 | rm -f /etc/security/limits.d/hdfs.conf
12 | su -c "echo 'N' | hdfs namenode -format" hdfs
13 |
14 | # 2 start hdfs
15 | su -c "hdfs datanode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-datanode.log" hdfs&
16 | su -c "hdfs namenode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-namenode.log" hdfs&
17 |
18 | # 3 wait for process starting
19 | sleep 10
20 |
21 | # remove a broken symlink created by cdh installer so that init-hdfs.sh does no blow up on it
22 | # (hbase-annotations.jar seems not needed in our case)
23 | rm /usr/lib/hive/lib/hbase-annotations.jar
24 |
25 | # 4 exec cloudera hdfs init script
26 | /usr/lib/hadoop/libexec/init-hdfs.sh
27 |
28 | # 5 init hive directories
29 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse'
30 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse'
31 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse'
32 |
33 | # 6 stop hdfs
34 | killall java
35 |
36 | # 7 setup metastore
37 | mysql_install_db
38 |
39 | /usr/bin/mysqld_safe &
40 | sleep 10s
41 |
42 | cd /usr/lib/hive/scripts/metastore/upgrade/mysql/
43 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
44 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.1.0.mysql.sql;" | mysql
45 | /usr/bin/mysqladmin -u root password 'root'
46 |
47 | killall mysqld
48 | sleep 10s
49 | mkdir /var/log/mysql/
50 | chown mysql:mysql /var/log/mysql/
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/README.md:
--------------------------------------------------------------------------------
1 | # cdh5.15-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/cdh5.15-hive-kerberized.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/cdh5.15-hive-kerberized
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/cdh5.15-hive-kerberized.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/cdh5.15-hive-kerberized
7 |
8 | A pseudo-distributed Hadoop image with Kerberos authentication enabled, based on the CDH distro.
9 |
10 | ## Oracle license
11 |
12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/container-executor.cfg:
--------------------------------------------------------------------------------
1 | yarn.nodemanager.local-dirs=/var/lib/hadoop-yarn/cache/yarn/nm-local-dir
2 | yarn.nodemanager.linux-container-executor.group=yarn
3 | yarn.nodemanager.log-dirs=/var/log/hadoop-yarn/containers
4 | banned.users=yarn,mapred,bin
5 | min.user.id=400
6 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
30 | dfs.block.access.token.enable
31 | true
32 |
33 |
34 |
35 |
36 | dfs.namenode.keytab.file
37 | /etc/hadoop/conf/hdfs.keytab
38 |
39 |
40 | dfs.namenode.kerberos.principal
41 | hdfs/hadoop-master@LABS.TERADATA.COM
42 |
43 |
44 | dfs.namenode.kerberos.internal.spnego.principal
45 | HTTP/hadoop-master@LABS.TERADATA.COM
46 |
47 |
48 |
49 |
50 | dfs.secondary.namenode.keytab.file
51 | /etc/hadoop/conf/hdfs.keytab
52 |
53 |
54 | dfs.secondary.namenode.kerberos.principal
55 | hdfs/hadoop-master@LABS.TERADATA.COM
56 |
57 |
58 | dfs.secondary.namenode.kerberos.internal.spnego.principal
59 | HTTP/hadoop-master@LABS.TERADATA.COM
60 |
61 |
62 |
63 |
64 | dfs.datanode.keytab.file
65 | /etc/hadoop/conf/hdfs.keytab
66 |
67 |
68 | dfs.datanode.kerberos.principal
69 | hdfs/hadoop-master@LABS.TERADATA.COM
70 |
71 |
72 |
73 |
74 | dfs.webhdfs.enabled
75 | true
76 |
77 |
78 |
79 |
80 | dfs.web.authentication.kerberos.principal
81 | HTTP/hadoop-master@LABS.TERADATA.COM
82 |
83 |
84 |
85 | dfs.web.authentication.kerberos.keytab
86 | /etc/hadoop/conf/HTTP.keytab
87 |
88 |
89 |
90 | ignore.secure.ports.for.testing
91 | true
92 |
93 |
94 |
95 | dfs.http.policy
96 | HTTP_ONLY
97 |
98 |
99 |
100 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.jobhistory.address
27 | hadoop-master:10020
28 |
29 |
30 |
31 | mapreduce.jobhistory.webapp.address
32 | hadoop-master:19888
33 |
34 |
35 |
36 | To set the value of tmp directory for map and reduce tasks.
37 | mapreduce.task.tmp.dir
38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
39 |
40 |
41 |
42 |
43 | mapreduce.jobhistory.keytab
44 | /etc/hadoop/conf/mapred.keytab
45 |
46 |
47 |
48 | mapreduce.jobhistory.principal
49 | mapred/hadoop-master@LABS.TERADATA.COM
50 |
51 |
52 |
53 |
54 | mapreduce.jobtracker.kerberos.principal
55 | mapred/hadoop-master@LABS.TERADATA.COM
56 |
57 |
58 |
59 | mapreduce.jobtracker.keytab.file
60 | /etc/hadoop/conf/mapred.keytab
61 |
62 |
63 |
64 |
65 | mapreduce.tasktracker.kerberos.principal
66 | mapred/hadoop-master@LABS.TERADATA.COM
67 |
68 |
69 |
70 | mapreduce.tasktracker.keytab.file
71 | /etc/hadoop/conf/mapred.keytab
72 |
73 |
74 |
75 | mapred.task.tracker.task-controller
76 | org.apache.hadoop.mapred.LinuxTaskController
77 |
78 |
79 |
80 | mapreduce.tasktracker.group
81 | mapred
82 |
83 |
84 |
85 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/hadoop/conf/taskcontroller.cfg:
--------------------------------------------------------------------------------
1 | hadoop.log.dir=/var/log/hadoop-mapreduce
2 | mapreduce.tasktracker.group=mapred
3 | banned.users=mapred,bin
4 | min.user.id=400
5 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 | hive.security.authenticator.manager
23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator
24 |
25 |
26 |
27 |
28 | hive.security.authorization.manager
29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory
30 | SQL standards based Hive authorization
31 |
32 |
33 |
34 | hive.security.authorization.enabled
35 | true
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/krb5.conf:
--------------------------------------------------------------------------------
1 | [logging]
2 | default = FILE:/var/log/krb5libs.log
3 | kdc = FILE:/var/log/krb5kdc.log
4 | admin_server = FILE:/var/log/kadmind.log
5 |
6 | [libdefaults]
7 | default_realm = LABS.TERADATA.COM
8 | dns_lookup_realm = false
9 | dns_lookup_kdc = false
10 | ticket_lifetime = 24h
11 | renew_lifetime = 7d
12 | forwardable = true
13 |
14 | [realms]
15 | LABS.TERADATA.COM = {
16 | kdc = hadoop-master
17 | admin_server = hadoop-master
18 | }
19 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/etc/supervisord.d/kdc.conf:
--------------------------------------------------------------------------------
1 | [program:krb5kdc]
2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -r LABS.TERADATA.COM -P /var/run/krb5kdc.pid -n"
3 | autostart=true
4 | autorestart=true
5 | redirect_stderr=true
6 | stdout_logfile=/dev/stdout
7 | stdout_logfile_maxbytes=0
8 |
9 | [program:kadmind]
10 | command=/bin/bash -c "exec /usr/sbin/kadmind -r LABS.TERADATA.COM -P /var/run/kadmind.pid -nofork"
11 | autostart=true
12 | autorestart=true
13 | redirect_stderr=true
14 | stdout_logfile=/dev/stdout
15 | stdout_logfile_maxbytes=0
16 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl:
--------------------------------------------------------------------------------
1 | */admin@LABS.TERADATA.COM *
2 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf:
--------------------------------------------------------------------------------
1 | [kdcdefaults]
2 | kdc_ports = 88
3 | kdc_tcp_ports = 88
4 |
5 | [realms]
6 | LABS.TERADATA.COM = {
7 | #master_key_type = aes256-cts
8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl
9 | dict_file = /usr/share/dict/words
10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
12 | }
13 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/centos6-oj8:unlabelled
14 | MAINTAINER Presto community
15 |
16 | # Change default timezone
17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone
18 |
19 | # Setup CDH repo, pin the CDH distribution to a concrete version
20 | RUN wget -nv https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo -P /etc/yum.repos.d \
21 | && rpm --import https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera \
22 | && sed -i '/^baseurl=/c\baseurl=https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.15.1/' /etc/yum.repos.d/cloudera-cdh5.repo
23 |
24 | # Install Hadoop, Hive (w/ MySQL)
25 | RUN \
26 | set -xeu && \
27 | yum install -y \
28 | hadoop-hdfs-namenode \
29 | hadoop-hdfs-secondarynamenode \
30 | hadoop-hdfs-datanode \
31 | \
32 | hadoop-mapreduce \
33 | \
34 | hadoop-yarn-resourcemanager \
35 | hadoop-yarn-nodemanager \
36 | \
37 | hadoop-client \
38 | hadoop-conf-pseudo \
39 | \
40 | hive \
41 | hive-metastore \
42 | hive-server2 \
43 | \
44 | mysql-server mysql-connector-java \
45 | \
46 | # Cleanup
47 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
48 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/mysql-connector-java.jar
49 |
50 | # Copy configuration files
51 | COPY ./files /
52 |
53 | # Run setup script
54 | RUN /root/setup.sh \
55 | && rm -rf /tmp/* /var/tmp/*
56 |
57 | # Setup sock proxy
58 | RUN yum install -y openssh openssh-clients openssh-server && yum -y clean all
59 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \
60 | && ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key \
61 | && ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key \
62 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
63 | RUN chmod 755 /root && chmod 700 /root/.ssh
64 | RUN passwd --unlock root
65 |
66 | # HDFS ports
67 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
68 |
69 | # YARN ports
70 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888
71 |
72 | # HIVE port
73 | EXPOSE 9083 10000
74 |
75 | # SOCKS port
76 | EXPOSE 1180
77 |
78 | CMD supervisord -c /etc/supervisord.conf
79 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/README.md:
--------------------------------------------------------------------------------
1 | # cdh5.15-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/cdh5.15-hive.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/cdh5.15-hive
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/cdh5.15-hive.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/cdh5.15-hive
7 |
8 | Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set.
9 | For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file.
10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal
11 | for general correctness testing.
12 |
13 | ## Run
14 |
15 | ```
16 | $ docker run -d --name hadoop-master -h hadoop-master prestodb/cdh5.15-hive
17 | ```
18 |
19 | ## Oracle license
20 |
21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
23 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hadoop/conf/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | fs.defaultFS
22 | hdfs://hadoop-master:9000
23 |
24 |
25 |
26 |
27 | hadoop.proxyuser.oozie.hosts
28 | *
29 |
30 |
31 | hadoop.proxyuser.oozie.groups
32 | *
33 |
34 |
35 |
36 |
37 | hadoop.proxyuser.httpfs.hosts
38 | *
39 |
40 |
41 | hadoop.proxyuser.httpfs.groups
42 | *
43 |
44 |
45 |
46 |
47 | hadoop.proxyuser.llama.hosts
48 | *
49 |
50 |
51 | hadoop.proxyuser.llama.groups
52 | *
53 |
54 |
55 |
56 |
57 | hadoop.proxyuser.hue.hosts
58 | *
59 |
60 |
61 | hadoop.proxyuser.hue.groups
62 | *
63 |
64 |
65 |
66 |
67 | hadoop.proxyuser.mapred.hosts
68 | *
69 |
70 |
71 | hadoop.proxyuser.mapred.groups
72 | *
73 |
74 |
75 |
76 |
77 | hadoop.proxyuser.hive.hosts
78 | *
79 |
80 |
81 |
82 | hadoop.proxyuser.hive.groups
83 | *
84 |
85 |
86 |
87 |
88 | hadoop.proxyuser.hdfs.groups
89 | *
90 |
91 |
92 |
93 | hadoop.proxyuser.hdfs.hosts
94 | *
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hadoop/conf/hadoop-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Set Hadoop-specific environment variables here.
17 | # Forcing YARN-based mapreduce implementaion.
18 | # Make sure to comment out if you want to go back to the default or
19 | # if you want this to be tweakable on a per-user basis
20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
21 |
22 | # The maximum amount of heap to use, in MB. Default is 1000.
23 | export HADOOP_HEAPSIZE=512
24 |
25 | # Extra Java runtime options. Empty by default.
26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m"
27 | export YARN_OPTS="$YARN_OPTS -Xmx256m"
28 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.jobhistory.address
27 | hadoop-master:10020
28 |
29 |
30 |
31 | mapreduce.jobhistory.webapp.address
32 | hadoop-master:19888
33 |
34 |
35 |
36 | To set the value of tmp directory for map and reduce tasks.
37 | mapreduce.task.tmp.dir
38 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
39 |
40 |
41 |
42 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hadoop/conf/yarn-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | yarn.nodemanager.aux-services
22 | mapreduce_shuffle
23 |
24 |
25 |
26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class
27 | org.apache.hadoop.mapred.ShuffleHandler
28 |
29 |
30 |
31 | yarn.log-aggregation-enable
32 | true
33 |
34 |
35 |
36 | yarn.dispatcher.exit-on-error
37 | true
38 |
39 |
40 |
41 | List of directories to store localized files in.
42 | yarn.nodemanager.local-dirs
43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir
44 |
45 |
46 |
47 | Where to store container logs.
48 | yarn.nodemanager.log-dirs
49 | /var/log/hadoop-yarn/containers
50 |
51 |
52 |
53 | Where to aggregate logs to.
54 | yarn.nodemanager.remote-app-log-dir
55 | /var/log/hadoop-yarn/apps
56 |
57 |
58 |
59 | Classpath for typical applications.
60 | yarn.application.classpath
61 |
62 | /etc/hadoop/conf,
63 | /usr/lib/hadoop/*,
64 | /usr/lib/hadoop/lib/*,
65 | /usr/lib/hadoop-hdfs/*,
66 | /usr/lib/hadoop-hdfs/lib/*,
67 | /usr/lib/hadoop-yarn/*,
68 | /usr/lib/hadoop-yarn/lib/*,
69 | /usr/lib/hadoop-mapreduce/*,
70 | /usr/lib/hadoop-mapreduce/lib/*
71 |
72 |
73 |
74 |
75 | yarn.resourcemanager.hostname
76 | hadoop-master
77 |
78 |
79 |
80 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hive/conf/hive-env.sh:
--------------------------------------------------------------------------------
1 | export HADOOP_OPTS="$HADOOP_OPTS -Dhive.root.logger=INFO,console"
2 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/hive/conf/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | javax.jdo.option.ConnectionURL
22 | jdbc:mysql://localhost/metastore
23 | the URL of the MySQL database
24 |
25 |
26 |
27 | javax.jdo.option.ConnectionDriverName
28 | com.mysql.jdbc.Driver
29 |
30 |
31 |
32 | javax.jdo.option.ConnectionUserName
33 | root
34 |
35 |
36 |
37 | javax.jdo.option.ConnectionPassword
38 | root
39 |
40 |
41 |
42 | datanucleus.autoCreateSchema
43 | false
44 |
45 |
46 |
47 | datanucleus.fixedDatastore
48 | true
49 |
50 |
51 |
52 | datanucleus.autoStartMechanism
53 | SchemaTable
54 |
55 |
56 |
57 | hive.security.authorization.createtable.owner.grants
58 | ALL
59 | The set of privileges automatically granted to the owner whenever a table gets created.
60 |
61 |
62 |
63 | hive.users.in.admin.role
64 | hdfs,hive
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/hdfs-datanode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-datanode]
2 | command=hdfs datanode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/hdfs-namenode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-namenode]
2 | command=hdfs namenode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/hive-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:hive-metastore]
2 | command=hive --service metastore
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-metastore.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/hive-server2.conf:
--------------------------------------------------------------------------------
1 | [program:hive-server2]
2 | command=hive --service hiveserver2
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-server2.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/mysql-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:mysql-metastore]
2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mysql
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/mysql/mysql.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/socks-proxy.conf:
--------------------------------------------------------------------------------
1 | [program:socks-proxy]
2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/socks-proxy
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/sshd.conf:
--------------------------------------------------------------------------------
1 | [program:sshd]
2 | command=/usr/sbin/sshd -D
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/sshd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/yarn-nodemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-nodemanager]
2 | command=yarn nodemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-nodemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/etc/supervisord.d/yarn-resourcemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-resourcemanager]
2 | command=yarn resourcemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-resourcemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/cdh5.15-hive/files/root/setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ex
2 |
3 | # 0 make file system hostname resolvable
4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts
5 |
6 | # 1 format namenode
7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/
8 |
9 | # workaround for 'could not open session' bug as suggested here:
10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610
11 | rm -f /etc/security/limits.d/hdfs.conf
12 | su -c "echo 'N' | hdfs namenode -format" hdfs
13 |
14 | # 2 start hdfs
15 | su -c "hdfs datanode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-datanode.log" hdfs&
16 | su -c "hdfs namenode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-namenode.log" hdfs&
17 |
18 | # 3 wait for process starting
19 | sleep 10
20 |
21 | # remove a broken symlink created by cdh installer so that init-hdfs.sh does no blow up on it
22 | # (hbase-annotations.jar seems not needed in our case)
23 | rm /usr/lib/hive/lib/hbase-annotations.jar
24 |
25 | # 4 exec cloudera hdfs init script
26 | /usr/lib/hadoop/libexec/init-hdfs.sh
27 |
28 | # 5 init hive directories
29 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse'
30 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse'
31 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse'
32 |
33 | # 6 stop hdfs
34 | killall java
35 |
36 | # 7 setup metastore
37 | mysql_install_db
38 |
39 | /usr/bin/mysqld_safe &
40 | sleep 10s
41 |
42 | cd /usr/lib/hive/scripts/metastore/upgrade/mysql/
43 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
44 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.1.0.mysql.sql;" | mysql
45 | /usr/bin/mysqladmin -u root password 'root'
46 |
47 | killall mysqld
48 | sleep 10s
49 | mkdir /var/log/mysql/
50 | chown mysql:mysql /var/log/mysql/
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/README.md:
--------------------------------------------------------------------------------
1 | # centos6-java8-oracle-openldap [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/centos6-oj8-openldap.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/centos6-oj8-openldap
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/centos6-oj8-openldap.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/centos6-oj8-openldap
7 |
8 | Docker image of CentOS 6 with Oracle JDK 8 installed. This has OpenLdap needed for
9 | Presto when running product-tests with front-end LDAP authentication.
10 |
11 | ## Oracle license
12 |
13 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
14 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
15 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/certs/active-directory-certificate.crt:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIGJTCCBQ2gAwIBAgITHQAAAAUM8gMo/ohRwwAAAAAABTANBgkqhkiG9w0BAQUF
3 | ADBlMRMwEQYKCZImiZPyLGQBGRYDY29tMRgwFgYKCZImiZPyLGQBGRYIdGVzdGxk
4 | YXAxFjAUBgoJkiaJk/IsZAEZFgZwcmVzdG8xHDAaBgNVBAMTE3ByZXN0by1BRC1U
5 | RVNULUNBLTEwHhcNMTcwNjI3MDA0NTM1WhcNMTgwNjI3MDA0NTM1WjAAMIIBIjAN
6 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoFyafCrwVdNqQ1Y1D1HMQVScNAc9
7 | m5l0nBbezmtbWExSgKXKE0DOIHLLZB38F8TKJZoa3m3J7Oin+GISrzWgyhoW6+jg
8 | KKGD9PZR5y4Xhu23Fv8IVlN9wf08vDploq34uzRFoBAwvjUcW42FPe+6MoKeMkat
9 | KgrfhCkcXAGhg0SOYmdCAHcPA63HBCNmpgYAZ38iqwiZugnraIXgn0VGhcmLlj7c
10 | S5Sv1GYOUtEd0MXELepAZPUXx8HivPvnz09tz++W6aKAkLsYvQShijPVnFinOAa9
11 | 8ZNlHyz8v5UiUbS/9kyQOY86mM+1U+qK9BQVIbRVvBSkLXPY2ZAvyZ9pCwIDAQAB
12 | o4IDMTCCAy0wPQYJKwYBBAGCNxUHBDAwLgYmKwYBBAGCNxUIhdXlWYazpGSH5Z0V
13 | hczUNYOhlVGBRKj2FYbb3DICAWQCAQEwMgYDVR0lBCswKQYHKwYBBQIDBQYKKwYB
14 | BAGCNxQCAgYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB/wQEAwIFoDBABgkr
15 | BgEEAYI3FQoEMzAxMAkGBysGAQUCAwUwDAYKKwYBBAGCNxQCAjAKBggrBgEFBQcD
16 | ATAKBggrBgEFBQcDAjAdBgNVHQ4EFgQU74Oc6DMfI847F2OplR+db0pIQJEwHwYD
17 | VR0jBBgwFoAUYVLOSj8PMaLbsTkN+P/R5RwEvMIwgdwGA1UdHwSB1DCB0TCBzqCB
18 | y6CByIaBxWxkYXA6Ly8vQ049cHJlc3RvLUFELVRFU1QtQ0EtMSxDTj1hZC10ZXN0
19 | LENOPUNEUCxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD
20 | Tj1Db25maWd1cmF0aW9uLERDPXByZXN0byxEQz10ZXN0bGRhcCxEQz1jb20/Y2Vy
21 | dGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENsYXNzPWNSTERpc3Ry
22 | aWJ1dGlvblBvaW50MIHQBggrBgEFBQcBAQSBwzCBwDCBvQYIKwYBBQUHMAKGgbBs
23 | ZGFwOi8vL0NOPXByZXN0by1BRC1URVNULUNBLTEsQ049QUlBLENOPVB1YmxpYyUy
24 | MEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9
25 | cHJlc3RvLERDPXRlc3RsZGFwLERDPWNvbT9jQUNlcnRpZmljYXRlP2Jhc2U/b2Jq
26 | ZWN0Q2xhc3M9Y2VydGlmaWNhdGlvbkF1dGhvcml0eTB0BgNVHREBAf8EajBooCwG
27 | CisGAQQBgjcUAgOgHgwcQUQtVEVTVCRAcHJlc3RvLnRlc3RsZGFwLmNvbYIbYWQt
28 | dGVzdC5wcmVzdG8udGVzdGxkYXAuY29tghNwcmVzdG8udGVzdGxkYXAuY29tggZQ
29 | UkVTVE8wDQYJKoZIhvcNAQEFBQADggEBAAtuksGPz4cwkIB7g8nqI1ysi0uaPxBO
30 | DSdq3wXPA9jS+W6bR39iiflugaeOW1pfQsZhMJadVCMkvQ/sYC4y1WmD+j6VSVs2
31 | 1znc4Gh+41teF3x1XRHgvQ1Vum+7GJIeKhZEUuB1F9SpuYxQT3wOIX2VnKq+1Z3/
32 | 7TlRyZ9bAG8tmSeYXK4/dL5q+L7o1UgRU47V7RzhOJp/bmQFQd0gtXRpSH5LJZhk
33 | +lIlmCxgsjkawBBvXHWd9K6R/5SIr/S/RfOIVdU/WbIcOzzAzZr9YqhmB3POc4aL
34 | f4IY0N7Rxc/dBW1bO833+lNOI4dJJgtdZoOxCe3BiHsM2EbZzUiu1HA=
35 | -----END CERTIFICATE-----
36 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/certs/openldap-certificate.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIDgTCCAmmgAwIBAgIJAKvPRrEwaoF0MA0GCSqGSIb3DQEBBQUAMFcxCzAJBgNV
3 | BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg
4 | Q29tcGFueSBMdGQxEzARBgNVBAMMCmxkYXBzZXJ2ZXIwHhcNMTYwODEwMTczOTIx
5 | WhcNMTkwODEwMTczOTIxWjBXMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVs
6 | dCBDaXR5MRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRkMRMwEQYDVQQDDAps
7 | ZGFwc2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA29sHE6G4
8 | 8F0MmHEuir/ZWMpIMjabWtMCp6q2Vbp52tqxSGVWqeVAKW+1qXmvNNMMlHzngmjT
9 | 0EIBoWk9LCKFeXkDCjwzASvjmGU1p9L0tFo9yEclDNlHo1B/Dq81ZnuaXZyBA7vd
10 | sbkcXqsES0m1736j9TP4Yfl915jrzma8JF92b0p479fH3xf/ZVTO3VNjtFYp186a
11 | ImIzeWDeNZk5ujPB6sJcKLm0l4q+wD4qpUbfcOqaT2YuauCsQGtqYAIf5F2uxyDA
12 | FdQt18zddlRocvuU/uPuF3bBNYxNICPrirqxsIJYudjdzmPjG+V1rDdQsl02c27e
13 | brbfq9d2togWDQIDAQABo1AwTjAdBgNVHQ4EFgQUfSaAXBYaUUlZ8myIwFJxmZI8
14 | EqIwHwYDVR0jBBgwFoAUfSaAXBYaUUlZ8myIwFJxmZI8EqIwDAYDVR0TBAUwAwEB
15 | /zANBgkqhkiG9w0BAQUFAAOCAQEAbTtHWLNJHhJUiZznPUTcAcX0YN+xcimcSquD
16 | JBKT6jqRgj+gOFXzG0gXVtATvXsvtXk/3wkbXEnnE6mBG/RMaP3pmsQ8BW1pUx1g
17 | bZofvPeEv8vRxd2Cs/dKhYv3MbTSAUPhuYub8oFB44TKS8uPmykIrTa9xG2uOYj/
18 | LdRPATE6W8KcCw55OUMfUdFuJB9fkiDqTYNT/hscBeuBrRJeDOuKSvhPXDYb1+ie
19 | 8rL6EM5YOlT9MNAYohMb9re1ry4u4jwrWuO8X461NdFhCHxwIXYH/p2JW4cHl0yt
20 | JvmE/yFsEsUu21RtL798p2uV8ADwpX7VwcjxkTSjyS6ktTyI/A==
21 | -----END CERTIFICATE-----
22 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/certs/private.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN RSA PRIVATE KEY-----
2 | MIIEpAIBAAKCAQEA29sHE6G48F0MmHEuir/ZWMpIMjabWtMCp6q2Vbp52tqxSGVW
3 | qeVAKW+1qXmvNNMMlHzngmjT0EIBoWk9LCKFeXkDCjwzASvjmGU1p9L0tFo9yEcl
4 | DNlHo1B/Dq81ZnuaXZyBA7vdsbkcXqsES0m1736j9TP4Yfl915jrzma8JF92b0p4
5 | 79fH3xf/ZVTO3VNjtFYp186aImIzeWDeNZk5ujPB6sJcKLm0l4q+wD4qpUbfcOqa
6 | T2YuauCsQGtqYAIf5F2uxyDAFdQt18zddlRocvuU/uPuF3bBNYxNICPrirqxsIJY
7 | udjdzmPjG+V1rDdQsl02c27ebrbfq9d2togWDQIDAQABAoIBAQDBip4/NQYWawN0
8 | i2Z6dItEc8Z8lb/Lg6h5q3maHG+7LvwUDEnvhEYodBPK3rlTWltfuFkhX2Dfm8vN
9 | qVRVrworNlFryDTqWhWgibBjTf1na22qCp9eIUTES0+ewLTUxAjPUb/tZouyMY/8
10 | 8rX0BS6pZ6W0bQzCSxzyF8e2SpBjvxhA+MYleMtxqc4rsn2lASu+ITNvgRIHXlQT
11 | ApD7IW3oG8DsSXE9UTAum/UoFKiv4VVqtKi5dAeSPRAabV/fo/Qtf8suQAvf4JxV
12 | tyUJtyMUvpRHV6KP6B1F0S+UYaXCWrD1r9xm8uA2WFtA4XvFFVVikreoos1soU7+
13 | VJuOQlxNAoGBAPHHiYOWKl83+UrR94Mmjb9NJ7s4CKau9Y1Iu18ct+tt2rKihktE
14 | OkZk/duOKmW2oXxTuruz16d3+WGDErqbCM+hJS7rUs7Go9qPq+umjSbQHtOWUDwg
15 | 5osPorAR+9h/+xu9CPwaibT/9C/THa/gh0FsBEFw6ViHK5045NRDs7QLAoGBAOjJ
16 | Yj5/bhWyIdWawimZEvsNjG7zCDlM85yM/gd0xjvdBuPXTL9xY23TXHbodqw+x174
17 | K72w/6/B34eV/FLCcOK+esataod8J215O+OKvgul+E1SG1/Sfn4e/0rTVuHzzRvz
18 | aIbALMdU6PbPTVc73t7y5ALzQVpVOK723GwLLNVHAoGAIvGz7lDFzv1ICohOPaZU
19 | 1lFQbL6sgWrIEqNwcU61RlGaSjczMa2qvoHaLuFdP1YmANIZKwfTI2EzqeRH0LCX
20 | lWq09RMN5NTQ2cXqEyT/QZzS18Hh1s7qgy6/VLNEswXMRTYNFmK/hGl54W+h2smY
21 | wtvZwZuQHjtrEk4mxIcEcgECgYAoA12ownyorArWzZgsUEyL7MND2vCB8PE66p1L
22 | qM3eQlrJ1RszGQlvOlJ9ee3UI6Jfd6cFHySEEH4crzJFA3kl8HnoJqzWBwg78AfU
23 | T+fk1q18lO1xphvqD+fv863Ofvq1oDxKVUQdK0I0EmQH55milvuMakXEgjuRD4vJ
24 | 4o3dRwKBgQC5JH3iqCNhrS5jOlXsLnXiZQVvO4viWMFgyJi56dMkDM/dhhYhrr7r
25 | i5h/6LJOufhUfmvvoAaOG4mBkNwWfZImCGrj7VxQWMlwgIUFb5atgnx0f/SkOrqn
26 | +V7Dmj05PwtnYTpltKgCLjsoQGyzLeXV3bFXfSCt9Xmcyp1qRKrxEg==
27 | -----END RSA PRIVATE KEY-----
28 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/certs/serial.txt:
--------------------------------------------------------------------------------
1 | 1234
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/setup/createOU.ldif:
--------------------------------------------------------------------------------
1 | dn: dc=presto,dc=testldap,dc=com
2 | dc: presto
3 | o: teradata
4 | objectClass: organization
5 | objectClass: dcObject
6 |
7 | dn: ou=America,dc=presto,dc=testldap,dc=com
8 | objectClass: organizationalUnit
9 | ou: America
10 |
11 | dn: ou=Asia,dc=presto,dc=testldap,dc=com
12 | objectClass: organizationalUnit
13 | ou: Asia
14 |
15 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/setup/ldap_disable_bind_anon.ldif:
--------------------------------------------------------------------------------
1 | dn: cn=config
2 | changetype: modify
3 | add: olcDisallows
4 | olcDisallows: bind_anon
5 |
6 | dn: cn=config
7 | changetype: modify
8 | add: olcRequires
9 | olcRequires: authc
10 |
11 | dn: olcDatabase={-1}frontend,cn=config
12 | changetype: modify
13 | add: olcRequires
14 | olcRequires: authc
15 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/setup/memberof.ldif:
--------------------------------------------------------------------------------
1 | # This overlay helps in determing which group a given user belongs to
2 | dn: cn=module,cn=config
3 | cn: module
4 | objectClass: olcModuleList
5 | olcModuleLoad: memberof.la
6 | olcModulePath: /usr/lib64/openldap
7 |
8 | dn: olcOverlay={0}memberof,olcDatabase={2}bdb,cn=config
9 | objectClass: olcConfig
10 | objectClass: olcMemberOf
11 | objectClass: olcOverlayConfig
12 | objectClass: top
13 | olcOverlay: memberof
14 | olcMemberOfDangling: ignore
15 | olcMemberOfRefInt: TRUE
16 | olcMemberOfGroupOC: groupOfNames
17 | olcMemberOfMemberAD: member
18 | olcMemberOfMemberOfAD: memberOf
19 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/setup/modify_server.ldif:
--------------------------------------------------------------------------------
1 | dn: olcDatabase={2}bdb,cn=config
2 | changetype: modify
3 | replace: olcRootDN
4 | olcRootDN: cn=admin,dc=presto,dc=testldap,dc=com
5 | -
6 | replace: olcSuffix
7 | olcSuffix: dc=presto,dc=testldap,dc=com
8 | -
9 | replace: olcRootPW
10 | olcRootPW: admin
11 | -
12 | replace: olcAccess
13 | olcAccess: {0}to attrs=userPassword by self write by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" write by anonymous auth by * none
14 | olcAccess: {1}to * by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" write by self write by * read
15 |
16 | dn: olcDatabase={1}monitor,cn=config
17 | changetype: modify
18 | replace: olcAccess
19 | olcAccess: {0}to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=exter
20 | nal,cn=auth" read by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" read by * n
21 | one
22 |
23 | dn: cn=config
24 | changetype: modify
25 | replace: olcTLSCACertificateFile
26 | olcTLSCACertificateFile: /etc/openldap/certs/openldap-certificate.pem
27 | -
28 | replace: olcTLSCertificateFile
29 | olcTLSCertificateFile: /etc/openldap/certs/openldap-certificate.pem
30 | -
31 | replace: olcTLSCertificateKeyFile
32 | olcTLSCertificateKeyFile: /etc/openldap/certs/private.pem
33 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/openldap/setup/refint.ldif:
--------------------------------------------------------------------------------
1 | # This overlay ensures that when user accounts are modified or deleted,
2 | # the corresponding member attribute in groups is also handled accordingly.
3 | dn: cn=module{1},cn=config
4 | cn: module
5 | objectclass: olcModuleList
6 | objectclass: top
7 | olcmoduleload: refint.la
8 |
9 | dn: olcOverlay={1}refint,olcDatabase={2}bdb,cn=config
10 | objectClass: olcConfig
11 | objectClass: olcOverlayConfig
12 | objectClass: olcRefintConfig
13 | objectClass: top
14 | olcOverlay: {1}refint
15 | olcRefintAttribute: memberof member manager owner
16 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8-openldap/files/etc/supervisord.d/slapd.conf:
--------------------------------------------------------------------------------
1 | [program:slapd]
2 | command=/usr/bin/pidproxy /var/run/openldap/slapd.pid /usr/sbin/slapd -d 1 -F /etc/openldap/slapd.d/ -h 'ldaps://0.0.0.0:636 ldap://0.0.0.0:389'
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/slapd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM library/centos:6.9
14 | MAINTAINER Presto community
15 |
16 | ARG JDK_URL
17 | ARG JDK_PATH
18 |
19 | # Install Oracle Java and presto-admin dependences (already has python 2.6)
20 | RUN \
21 | set -xeu && \
22 | yum install -y wget && \
23 | # install and trim Oracle JDK
24 | wget -nv --header "Cookie: oraclelicense=accept-securebackup-cookie" $JDK_URL -O jdk.rpm && \
25 | rpm -ivh jdk.rpm && rm jdk.rpm && \
26 | rm -r $JDK_PATH/*src.zip \
27 | $JDK_PATH/lib/missioncontrol \
28 | $JDK_PATH/lib/visualvm \
29 | $JDK_PATH/lib/*javafx* \
30 | $JDK_PATH/jre/lib/plugin.jar \
31 | $JDK_PATH/jre/lib/ext/jfxrt.jar \
32 | $JDK_PATH/jre/bin/javaws \
33 | $JDK_PATH/jre/lib/javaws.jar \
34 | $JDK_PATH/jre/lib/desktop \
35 | $JDK_PATH/jre/plugin \
36 | $JDK_PATH/jre/lib/deploy* \
37 | $JDK_PATH/jre/lib/*javafx* \
38 | $JDK_PATH/jre/lib/*jfx* \
39 | $JDK_PATH/jre/lib/amd64/libdecora_sse.so \
40 | $JDK_PATH/jre/lib/amd64/libprism_*.so \
41 | $JDK_PATH/jre/lib/amd64/libfxplugins.so \
42 | $JDK_PATH/jre/lib/amd64/libglass.so \
43 | $JDK_PATH/jre/lib/amd64/libgstreamer-lite.so \
44 | $JDK_PATH/jre/lib/amd64/libjavafx*.so \
45 | $JDK_PATH/jre/lib/amd64/libjfx*.so && \
46 | \
47 | # make java fixed (so further layers don't accidentally override it)
48 | update-alternatives --set java $JDK_PATH/jre/bin/java && \
49 | \
50 | # install supervisor
51 | yum install -y setuptools epel-release && \
52 | yum install -y python-pip && \
53 | pip install --upgrade pip==9.0.3 `# latest version for Python 2.6` && \
54 | pip install --upgrade setuptools==36.8.0 `# latest version for Python 2.6` && \
55 | pip install supervisor && \
56 | \
57 | # install tools helpful when troubleshooting product tests
58 | yum install -y vim && \
59 | # cleanup
60 | yum -y clean all && rm -rf /tmp/* /var/tmp/*
61 |
62 | ENV JAVA_HOME $JDK_PATH
63 |
--------------------------------------------------------------------------------
/prestodb/centos6-oj8/README.md:
--------------------------------------------------------------------------------
1 | # centos6-oj8 [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/centos6-oj8.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/centos6-oj8
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/centos6-oj8.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/centos6-oj8
7 |
8 | Docker image of CentOS 6 with Oracle JDK 8 installed.
9 |
10 | ## Oracle license
11 |
12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/centos6-presto-admin-tests-build/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM prestodb/centos6-presto-admin-tests-runtime:unlabelled
2 | MAINTAINER Presto community
3 |
4 | RUN yum install -y \
5 | wget \
6 | gcc \
7 | python-devel \
8 | python-devel \
9 | python-pip \
10 | openssl-devel \
11 | libffi-devel && \
12 | pip install --upgrade pip==9.0.3 `# latest version for Python 2.6` && \
13 | # cleanup
14 | yum -y clean all && rm -rf /tmp/* /var/tmp/*
15 |
--------------------------------------------------------------------------------
/prestodb/centos6-presto-admin-tests-build/README.md:
--------------------------------------------------------------------------------
1 | Centos 6 with presto-admin build dependences
2 |
--------------------------------------------------------------------------------
/prestodb/centos6-presto-admin-tests-runtime/README.md:
--------------------------------------------------------------------------------
1 | # centos6-presto-adminn-tests-runtime
2 |
3 | Docker image of CentOS 6 with Oracle JDK 8 installed, and with sshd
4 | running. Passwordless ssh for the user app-admin is also set up.
5 |
6 | ## Further documentation
7 | For more documentation on this image, see the documentation for the
8 | base centos6-ssh image it uses:
9 | https://registry.hub.docker.com/u/jdeathe/centos-ssh/
10 |
11 | ## Oracle license
12 |
13 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
14 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
15 |
--------------------------------------------------------------------------------
/prestodb/centos6-presto-admin-tests-runtime/vagrant_insecure_rsa:
--------------------------------------------------------------------------------
1 | -----BEGIN RSA PRIVATE KEY-----
2 | MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
3 | w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
4 | kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
5 | hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
6 | Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
7 | yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
8 | ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
9 | Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
10 | TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
11 | iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
12 | sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
13 | 4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
14 | cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
15 | EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
16 | CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
17 | 3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
18 | YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
19 | 3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
20 | dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
21 | 6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
22 | P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
23 | llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
24 | kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
25 | +vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
26 | NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
27 | -----END RSA PRIVATE KEY-----
28 |
--------------------------------------------------------------------------------
/prestodb/centos7-oj8/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM library/centos:7
14 | MAINTAINER Presto community
15 |
16 | COPY ./files /
17 |
18 | # Install Java and presto-admin dependences
19 | RUN \
20 | set -xeu && \
21 | yum install -y \
22 | java-1.8.0-openjdk-devel \
23 | nc \
24 | wget \
25 | && \
26 | \
27 | # install Zulu JDK (will not be the default `java`)
28 | rpm -i https://cdn.azul.com/zulu/bin/zulu11.39.15-ca-jdk11.0.7-linux.x86_64.rpm && \
29 | \
30 | # install supervisor
31 | yum --enablerepo=extras install -y setuptools epel-release && \
32 | yum install -y python-pip && \
33 | pip install supervisor && \
34 | \
35 | # install commonly needed packages
36 | yum install -y \
37 | less `# helpful when troubleshooting product tests` \
38 | net-tools `# netstat is required by run_on_docker.sh` \
39 | sudo \
40 | telnet `# helpful when troubleshooting product tests` \
41 | vim `# helpful when troubleshooting product tests` \
42 | && \
43 | # cleanup
44 | yum -y clean all && rm -rf /tmp/* /var/tmp/*
45 |
46 | ENV PATH="/usr/local/bin:${PATH}"
47 | ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
48 | ENV LANG=en_US.UTF-8
49 |
--------------------------------------------------------------------------------
/prestodb/centos7-oj8/files/opt/prestodb/site-override.xslt:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/prestodb/centos7-oj8/files/usr/local/bin/apply-all-site-xml-overrides:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -euo pipefail
4 |
5 | fail() {
6 | echo "$(basename "$0"): $*" >&2
7 | exit 1
8 | }
9 |
10 | if [ $# -ne 1 ]; then
11 | fail "Usage: $0 " >&2
12 | fi
13 |
14 | overrides_dir="$1"
15 |
16 | for file in $(find $overrides_dir -name '*.xml'); do
17 | target_filename="${file#"$overrides_dir"}"
18 | echo "Applying configuration override from $file to $target_filename"
19 | apply-site-xml-override "$target_filename" "$file"
20 | done
21 |
--------------------------------------------------------------------------------
/prestodb/centos7-oj8/files/usr/local/bin/apply-site-xml-override:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -euo pipefail
4 |
5 | fail() {
6 | echo "$(basename "$0"): $*" >&2
7 | exit 1
8 | }
9 |
10 | if [ $# -ne 2 ]; then
11 | fail "Usage: $0 " >&2
12 | fi
13 |
14 | site_xml="$1"
15 | overrides="$2"
16 | site_xml_new="$1.new"
17 |
18 | test -f "${site_xml}" || fail "${site_xml} does not exist or is not a file"
19 | test -f "${overrides}" || fail "${overrides} does not exist or is not a file"
20 | test ! -e "${site_xml_new}" || fail "${site_xml_new} already exists"
21 |
22 | xsltproc --param override-path "'${overrides}'" "/opt/prestodb/site-override.xslt" "${site_xml}" > "${site_xml_new}"
23 | cat "${site_xml_new}" > "${site_xml}" # Preserve file owner & permissions
24 | rm "${site_xml_new}"
25 |
--------------------------------------------------------------------------------
/prestodb/dns/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM alpine:3.3
14 | MAINTAINER Presto community
15 | RUN apk --no-cache add dnsmasq
16 | EXPOSE 53 53/udp
17 | ENTRYPOINT ["dnsmasq", "-k"]
18 |
--------------------------------------------------------------------------------
/prestodb/dns/README.md:
--------------------------------------------------------------------------------
1 | # docker-dnsmasq [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/dns.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/dns
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/dns.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/dns
7 |
8 | Based on https://github.com/andyshinn/docker-dnsmasq
9 |
10 | It's a [dnsmasq][dnsmasq] Docker image. It is only 6 MB in size. It is just an `ENTRYPOINT` to the `dnsmasq` binary. Can you smell what the rock is cookin'?
11 |
12 | ## Usage
13 |
14 | It is usually a good idea to use a tag other than `latest` if you are using this image in a production setting.
15 |
16 | [dnsmasq][dnsmasq] requires `NET_ADMIN` capabilities to run correctly. Start it with something like `docker run -p 53:53/tcp -p 53:53/udp --cap-add=NET_ADMIN prestodb/dns:2.75`.
17 |
18 | The configuration is all handled on the command line (no wrapper scripts here). The `ENTRYPOINT` is `dnsmasq -k` to keep it running in the foreground. If you wanted to send requests for an internal domain (such as Consul) you can forward the requests upstream using something like `docker run -p 53:53/tcp -p 53:53/udp --cap-add=NET_ADMIN prestodb/dns:2.75 -S /consul/10.17.0.2`. This will send a request for `redis.service.consul` to `10.17.0.2`
19 |
20 | As this is a very barebones entrypoint with just enough to run in the foreground, there is no logging enabled by default. To send logging to stdout you can add `--log-facility=-` as an option.
21 |
22 | [dnsmasq]: http://www.thekelleys.org.uk/dnsmasq/doc.html
23 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-base/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/centos6-oj8:unlabelled
14 | MAINTAINER Presto community
15 |
16 | RUN \
17 | wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo -P /etc/yum.repos.d \
18 |
19 | && yum install -y \
20 | hadoop-hdfs-namenode \
21 | hadoop-hdfs-secondarynamenode \
22 | hadoop-hdfs-datanode \
23 |
24 | hadoop-mapreduce-historyserver \
25 |
26 | hadoop-yarn-resourcemanager \
27 | hadoop-yarn-nodemanager \
28 |
29 | zookeeper \
30 | zookeeper-server \
31 |
32 | hive \
33 | hive-metastore \
34 | hive-server2 \
35 |
36 | && yum -y clean all && rm -rf /tmp/* /var/tmp/*
37 | COPY ./files /
38 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-base/README.md:
--------------------------------------------------------------------------------
1 | # hdp2.5-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.5-base.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.5-base
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.5-base.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.5-base
7 |
8 | A Hadoop base image for creating images testing Presto, based on the HDP distro.
9 |
10 | ## Oracle license
11 |
12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-base/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/README.md:
--------------------------------------------------------------------------------
1 | # hdp2.5-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.5-hive-kerberized.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.5-hive-kerberized
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.5-hive-kerberized.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.5-hive-kerberized
7 |
8 | A pseudo-distributed Hadoop image with Kerberos authentication enabled, based on the HDP distro.
9 |
10 | ## Oracle license
11 |
12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
30 | dfs.block.access.token.enable
31 | true
32 |
33 |
34 |
35 |
36 | dfs.namenode.keytab.file
37 | /etc/hadoop/conf/hdfs.keytab
38 |
39 |
40 | dfs.namenode.kerberos.principal
41 | hdfs/hadoop-master@LABS.TERADATA.COM
42 |
43 |
44 | dfs.namenode.kerberos.internal.spnego.principal
45 | HTTP/hadoop-master@LABS.TERADATA.COM
46 |
47 |
48 |
49 |
50 | dfs.secondary.namenode.keytab.file
51 | /etc/hadoop/conf/hdfs.keytab
52 |
53 |
54 | dfs.secondary.namenode.kerberos.principal
55 | hdfs/hadoop-master@LABS.TERADATA.COM
56 |
57 |
58 | dfs.secondary.namenode.kerberos.internal.spnego.principal
59 | HTTP/hadoop-master@LABS.TERADATA.COM
60 |
61 |
62 |
63 |
64 | dfs.datanode.keytab.file
65 | /etc/hadoop/conf/hdfs.keytab
66 |
67 |
68 | dfs.datanode.kerberos.principal
69 | hdfs/hadoop-master@LABS.TERADATA.COM
70 |
71 |
72 |
73 |
74 | dfs.webhdfs.enabled
75 | true
76 |
77 |
78 |
79 |
80 | dfs.web.authentication.kerberos.principal
81 | HTTP/hadoop-master@LABS.TERADATA.COM
82 |
83 |
84 |
85 | dfs.web.authentication.kerberos.keytab
86 | /etc/hadoop/conf/HTTP.keytab
87 |
88 |
89 |
90 | ignore.secure.ports.for.testing
91 | true
92 |
93 |
94 |
95 | dfs.http.policy
96 | HTTP_ONLY
97 |
98 |
99 |
100 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.framework.name
27 | yarn
28 |
29 |
30 |
31 | mapreduce.jobhistory.address
32 | hadoop-master:10020
33 |
34 |
35 |
36 | mapreduce.jobhistory.webapp.address
37 | hadoop-master:19888
38 |
39 |
40 |
41 | To set the value of tmp directory for map and reduce tasks.
42 | mapreduce.task.tmp.dir
43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
44 |
45 |
46 |
47 |
48 | mapreduce.jobhistory.keytab
49 | /etc/hadoop/conf/mapred.keytab
50 |
51 |
52 |
53 | mapreduce.jobhistory.principal
54 | mapred/hadoop-master@LABS.TERADATA.COM
55 |
56 |
57 |
58 |
59 | mapreduce.jobtracker.kerberos.principal
60 | mapred/hadoop-master@LABS.TERADATA.COM
61 |
62 |
63 |
64 | mapreduce.jobtracker.keytab.file
65 | /etc/hadoop/conf/mapred.keytab
66 |
67 |
68 |
69 |
70 | mapreduce.tasktracker.kerberos.principal
71 | mapred/hadoop-master@LABS.TERADATA.COM
72 |
73 |
74 |
75 | mapreduce.tasktracker.keytab.file
76 | /etc/hadoop/conf/mapred.keytab
77 |
78 |
79 |
80 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 | hive.security.authenticator.manager
23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator
24 |
25 |
26 |
27 |
28 | hive.security.authorization.manager
29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory
30 | SQL standards based Hive authorization
31 |
32 |
33 |
34 | hive.security.authorization.enabled
35 | true
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/etc/krb5.conf:
--------------------------------------------------------------------------------
1 | [logging]
2 | default = FILE:/var/log/krb5libs.log
3 | kdc = FILE:/var/log/krb5kdc.log
4 | admin_server = FILE:/var/log/kadmind.log
5 |
6 | [libdefaults]
7 | default_realm = LABS.TERADATA.COM
8 | dns_lookup_realm = false
9 | dns_lookup_kdc = false
10 | ticket_lifetime = 24h
11 | renew_lifetime = 7d
12 | forwardable = true
13 |
14 | [realms]
15 | LABS.TERADATA.COM = {
16 | kdc = hadoop-master
17 | admin_server = hadoop-master
18 | }
19 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/etc/supervisord.d/kdc.conf:
--------------------------------------------------------------------------------
1 | [program:krb5kdc]
2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -r LABS.TERADATA.COM -P /var/run/krb5kdc.pid -n"
3 | autostart=true
4 | autorestart=true
5 | redirect_stderr=true
6 | stdout_logfile=/dev/stdout
7 | stdout_logfile_maxbytes=0
8 |
9 | [program:kadmind]
10 | command=/bin/bash -c "exec /usr/sbin/kadmind -r LABS.TERADATA.COM -P /var/run/kadmind.pid -nofork"
11 | autostart=true
12 | autorestart=true
13 | redirect_stderr=true
14 | stdout_logfile=/dev/stdout
15 | stdout_logfile_maxbytes=0
16 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl:
--------------------------------------------------------------------------------
1 | */admin@LABS.TERADATA.COM *
2 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf:
--------------------------------------------------------------------------------
1 | [kdcdefaults]
2 | kdc_ports = 88
3 | kdc_tcp_ports = 88
4 |
5 | [realms]
6 | LABS.TERADATA.COM = {
7 | #master_key_type = aes256-cts
8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl
9 | dict_file = /usr/share/dict/words
10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
12 | }
13 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/hdp2.5-base:unlabelled
14 | MAINTAINER Presto community
15 |
16 | # CHANGE DEFAULT TIMEZONE
17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone
18 |
19 | # INSTALL MYSQL (Hive Metastore)
20 | RUN yum install -y mysql-server mysql-connector-java \
21 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
22 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/hdp/current/hive-metastore/lib/mysql-connector-java.jar
23 |
24 | # Create needed dirs
25 | RUN mkdir /var/lib/zookeeper
26 | RUN chown zookeeper:hadoop /var/lib/zookeeper
27 |
28 | # DELETE ORIGINAL CONFIGURATION
29 | RUN rm -r /etc/hadoop/conf/*
30 | RUN rm -r /etc/zookeeper/conf/*
31 | RUN rm -r /etc/hive/conf/*
32 |
33 | # COPY CONFIGURATION FILES
34 | COPY ./files /
35 |
36 | # RUN SETUP
37 | RUN /root/setup.sh
38 |
39 | # DO CLEANUP
40 | RUN rm -rf /tmp/* /var/tmp/*
41 |
42 | # SETUP SOCKS PROXY
43 | RUN yum install -y openssh openssh-clients openssh-server
44 | RUN ssh-keygen -t rsa -b 4096 -C "automation@prestodb.io" -N "" -f /root/.ssh/id_rsa
45 | RUN ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key
46 | RUN ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key
47 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
48 | RUN chmod 755 /root && chmod 700 /root/.ssh
49 | RUN passwd --unlock root
50 |
51 | # HDFS PORTS
52 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
53 |
54 | # YARN PORTS
55 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888
56 |
57 | # HIVE PORT
58 | EXPOSE 9083 10000
59 |
60 | # SOCKS PORT
61 | EXPOSE 1180
62 |
63 | CMD supervisord -c /etc/supervisord.conf
64 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/README.md:
--------------------------------------------------------------------------------
1 | # hdp2.5-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.5-hive.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.5-hive
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.5-hive.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.5-hive
7 |
8 | A pseudo-distributed Hadoop image for testing Presto, based on the HDP distro.
9 |
10 | ## Oracle license
11 |
12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/hadoop/conf/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | fs.defaultFS
22 | hdfs://hadoop-master:9000
23 |
24 |
25 |
26 |
27 | hadoop.proxyuser.oozie.hosts
28 | *
29 |
30 |
31 | hadoop.proxyuser.oozie.groups
32 | *
33 |
34 |
35 |
36 |
37 | hadoop.proxyuser.httpfs.hosts
38 | *
39 |
40 |
41 | hadoop.proxyuser.httpfs.groups
42 | *
43 |
44 |
45 |
46 |
47 | hadoop.proxyuser.llama.hosts
48 | *
49 |
50 |
51 | hadoop.proxyuser.llama.groups
52 | *
53 |
54 |
55 |
56 |
57 | hadoop.proxyuser.hue.hosts
58 | *
59 |
60 |
61 | hadoop.proxyuser.hue.groups
62 | *
63 |
64 |
65 |
66 |
67 | hadoop.proxyuser.mapred.hosts
68 | *
69 |
70 |
71 | hadoop.proxyuser.mapred.groups
72 | *
73 |
74 |
75 |
76 |
77 | hadoop.proxyuser.hive.hosts
78 | *
79 |
80 |
81 |
82 | hadoop.proxyuser.hive.groups
83 | *
84 |
85 |
86 |
87 |
88 | hadoop.proxyuser.hdfs.groups
89 | *
90 |
91 |
92 |
93 | hadoop.proxyuser.hdfs.hosts
94 | *
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/hadoop/conf/hadoop-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Set Hadoop-specific environment variables here.
17 | # Forcing YARN-based mapreduce implementaion.
18 | # Make sure to comment out if you want to go back to the default or
19 | # if you want this to be tweakable on a per-user basis
20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
21 |
22 | # The maximum amount of heap to use, in MB. Default is 1000.
23 | export HADOOP_HEAPSIZE=256
24 |
25 | # Extra Java runtime options. Empty by default.
26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m"
27 | export YARN_OPTS="$YARN_OPTS -Xmx256m"
28 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.framework.name
27 | yarn
28 |
29 |
30 |
31 | mapreduce.jobhistory.address
32 | hadoop-master:10020
33 |
34 |
35 |
36 | mapreduce.jobhistory.webapp.address
37 | hadoop-master:19888
38 |
39 |
40 |
41 | To set the value of tmp directory for map and reduce tasks.
42 | mapreduce.task.tmp.dir
43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
44 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/hadoop/conf/yarn-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 | yarn.nodemanager.aux-services
23 | mapreduce_shuffle
24 |
25 |
26 |
27 | yarn.nodemanager.aux-services.mapreduce_shuffle.class
28 | org.apache.hadoop.mapred.ShuffleHandler
29 |
30 |
31 |
32 | yarn.log-aggregation-enable
33 | true
34 |
35 |
36 |
37 | yarn.dispatcher.exit-on-error
38 | true
39 |
40 |
41 |
42 | List of directories to store localized files in.
43 | yarn.nodemanager.local-dirs
44 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir
45 |
46 |
47 |
48 | Where to store container logs.
49 | yarn.nodemanager.log-dirs
50 | /var/log/hadoop/yarn/containers
51 |
52 |
53 |
54 | Where to aggregate logs to.
55 | yarn.nodemanager.remote-app-log-dir
56 | /var/log/hadoop/yarn/apps
57 |
58 |
59 |
60 | Classpath for typical applications.
61 | yarn.application.classpath
62 |
63 | /etc/hadoop/conf,
64 | /usr/hdp/current/hadoop-client/*,
65 | /usr/hdp/current/hadoop-client/lib/*,
66 | /usr/hdp/current/hadoop-hdfs-client/*,
67 | /usr/hdp/current/hadoop-hdfs-client/lib/*,
68 | /usr/hdp/current/hadoop-yarn-client/*,
69 | /usr/hdp/current/hadoop-yarn-client/lib/*,
70 | /usr/hdp/current/hadoop-mapreduce-client/*,
71 | /usr/hdp/current/hadoop-mapreduce-client/lib/*
72 |
73 |
74 |
75 |
76 | yarn.resourcemanager.hostname
77 | hadoop-master
78 |
79 |
80 |
81 | yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage
82 | 100
83 |
84 |
85 |
86 | yarn.log.server.url
87 | http://hadoop-master:19888/jobhistory/logs
88 |
89 |
90 |
91 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/hive/conf/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | javax.jdo.option.ConnectionURL
22 | jdbc:mysql://localhost/metastore
23 | the URL of the MySQL database
24 |
25 |
26 |
27 | javax.jdo.option.ConnectionDriverName
28 | com.mysql.jdbc.Driver
29 |
30 |
31 |
32 | javax.jdo.option.ConnectionUserName
33 | root
34 |
35 |
36 |
37 | javax.jdo.option.ConnectionPassword
38 | root
39 |
40 |
41 |
42 | datanucleus.autoCreateSchema
43 | false
44 |
45 |
46 |
47 | datanucleus.fixedDatastore
48 | true
49 |
50 |
51 |
52 | datanucleus.autoStartMechanism
53 | SchemaTable
54 |
55 |
56 |
57 | hive.security.authorization.createtable.owner.grants
58 | ALL
59 | The set of privileges automatically granted to the owner whenever a table gets created.
60 |
61 |
62 |
63 | hive.users.in.admin.role
64 | hdfs,hive
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/hdfs-datanode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-datanode]
2 | command=hdfs datanode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-datanode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/hdfs-namenode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-namenode]
2 | command=hdfs namenode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-namenode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/hive-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:hive-metastore]
2 | command=hive --service metastore
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-metastore.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/hive-server2.conf:
--------------------------------------------------------------------------------
1 | [program:hive-server2]
2 | command=hive --service hiveserver2
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-server2.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/mapreduce-historyserver.conf:
--------------------------------------------------------------------------------
1 | [program:mapreduce-historyserver]
2 | command=mapred historyserver
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mapred
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/mapreduce/hadoop-mapreduce-historyserver.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/mysql-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:mysql-metastore]
2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mysql
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/mysql/mysql.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/socks-proxy.conf:
--------------------------------------------------------------------------------
1 | [program:socks-proxy]
2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/socks-proxy
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/sshd.conf:
--------------------------------------------------------------------------------
1 | [program:sshd]
2 | command=/usr/sbin/sshd -D
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/sshd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/yarn-nodemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-nodemanager]
2 | command=yarn nodemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-nodemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/yarn-resourcemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-resourcemanager]
2 | command=yarn resourcemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-resourcemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/supervisord.d/zookeeper.conf:
--------------------------------------------------------------------------------
1 | [program:zookeeper]
2 | command=/usr/bin/zookeeper-server start-foreground
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=zookeeper
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/zookeeper/zookeeper.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/zookeeper/conf/log4j.properties:
--------------------------------------------------------------------------------
1 | # Define some default values that can be overridden by system properties
2 | zookeeper.root.logger=INFO, CONSOLE
3 | zookeeper.console.threshold=INFO
4 | zookeeper.log.threshold=DEBUG
5 | log4j.rootLogger=INFO,CONSOLE
6 |
7 | # Example with rolling log file
8 | #log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
9 |
10 | # Example with rolling log file and tracing
11 | #log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
12 |
13 | #
14 | # Log INFO level and above messages to the console
15 | #
16 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
17 | log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold}
18 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
19 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
20 |
21 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/zookeeper/conf/zoo.cfg:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | maxClientCnxns=50
17 | # The number of milliseconds of each tick
18 | tickTime=2000
19 | # The number of ticks that the initial
20 | # synchronization phase can take
21 | initLimit=10
22 | # The number of ticks that can pass between
23 | # sending a request and getting an acknowledgement
24 | syncLimit=5
25 | # the directory where the snapshot is stored.
26 | dataDir=/var/lib/zookeeper
27 | # the port at which the clients will connect
28 | clientPort=2181
29 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/etc/zookeeper/conf/zookeeper-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | export JAVA_HOME=${JAVA_HOME}
16 | export ZOO_LOG_DIR=${LOG_DIR}
17 |
--------------------------------------------------------------------------------
/prestodb/hdp2.5-hive/files/root/setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ex
2 |
3 | # 0 make file system hostname resolvable
4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts
5 |
6 | # 1 format namenode
7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/
8 |
9 | # workaround for 'could not open session' bug as suggested here:
10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610
11 | rm -f /etc/security/limits.d/hdfs.conf
12 | su -c "echo 'N' | hdfs namenode -format" hdfs
13 |
14 | # 2 start hdfs
15 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs&
16 |
17 | # 3 wait for process starting
18 | sleep 10
19 |
20 | # 4 init basic hdfs directories
21 | /usr/hdp/2.5.0.0-1245/hadoop/libexec/init-hdfs.sh
22 |
23 | # 4.1 Create an hdfs home directory for the yarn user. For some reason, init-hdfs doesn't do so.
24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/yarn && /usr/bin/hadoop fs -chown yarn:yarn /user/yarn'
25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn'
26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging'
27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging/history'
28 |
29 |
30 |
31 | # 5 init hive directories
32 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse'
33 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse'
34 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse'
35 |
36 | # 6 stop hdfs
37 | killall java
38 |
39 | # 7 setup metastore
40 | mysql_install_db
41 |
42 | /usr/bin/mysqld_safe &
43 | sleep 10s
44 |
45 | cd /usr/hdp/2.5.*/hive/scripts/metastore/upgrade/mysql/
46 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
47 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.2.1000.mysql.sql;" | mysql
48 | /usr/bin/mysqladmin -u root password 'root'
49 |
50 | killall mysqld
51 | sleep 10s
52 | mkdir /var/log/mysql/
53 | chown mysql:mysql /var/log/mysql/
54 |
55 | exit 0
56 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/README.md:
--------------------------------------------------------------------------------
1 | # hdp2.6-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.6-hive-kerberized.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.6-hive-kerberized
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.6-hive-kerberized.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.6-hive-kerberized
7 |
8 | A pseudo-distributed Hadoop image with Kerberos authentication enabled, based on the HDP distro.
9 |
10 | ## Oracle license
11 |
12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.framework.name
27 | yarn
28 |
29 |
30 |
31 | mapreduce.jobhistory.address
32 | hadoop-master:10020
33 |
34 |
35 |
36 | mapreduce.jobhistory.webapp.address
37 | hadoop-master:19888
38 |
39 |
40 |
41 | To set the value of tmp directory for map and reduce tasks.
42 | mapreduce.task.tmp.dir
43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
44 |
45 |
46 |
47 |
48 | mapreduce.jobhistory.keytab
49 | /etc/hadoop/conf/mapred.keytab
50 |
51 |
52 |
53 | mapreduce.jobhistory.principal
54 | mapred/hadoop-master@LABS.TERADATA.COM
55 |
56 |
57 |
58 |
59 | mapreduce.jobtracker.kerberos.principal
60 | mapred/hadoop-master@LABS.TERADATA.COM
61 |
62 |
63 |
64 | mapreduce.jobtracker.keytab.file
65 | /etc/hadoop/conf/mapred.keytab
66 |
67 |
68 |
69 |
70 | mapreduce.tasktracker.kerberos.principal
71 | mapred/hadoop-master@LABS.TERADATA.COM
72 |
73 |
74 |
75 | mapreduce.tasktracker.keytab.file
76 | /etc/hadoop/conf/mapred.keytab
77 |
78 |
79 |
80 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/etc/hive/conf/hiveserver2-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 | hive.security.authenticator.manager
23 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator
24 |
25 |
26 |
27 |
28 | hive.security.authorization.manager
29 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory
30 | SQL standards based Hive authorization
31 |
32 |
33 |
34 | hive.security.authorization.enabled
35 | true
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/etc/krb5.conf:
--------------------------------------------------------------------------------
1 | [logging]
2 | default = FILE:/var/log/krb5libs.log
3 | kdc = FILE:/var/log/krb5kdc.log
4 | admin_server = FILE:/var/log/kadmind.log
5 |
6 | [libdefaults]
7 | default_realm = LABS.TERADATA.COM
8 | dns_lookup_realm = false
9 | dns_lookup_kdc = false
10 | ticket_lifetime = 24h
11 | renew_lifetime = 7d
12 | forwardable = true
13 |
14 | [realms]
15 | LABS.TERADATA.COM = {
16 | kdc = hadoop-master:88
17 | admin_server = hadoop-master
18 | }
19 | OTHERLABS.TERADATA.COM = {
20 | kdc = hadoop-master:89
21 | admin_server = hadoop-master
22 | }
23 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/etc/supervisord.d/kdc.conf:
--------------------------------------------------------------------------------
1 | [program:krb5kdc]
2 | command=/bin/bash -c "exec /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid -n -r LABS.TERADATA.COM -n -d /var/kerberos/krb5kdc/principal-other -r OTHERLABS.TERADATA.COM"
3 | autostart=true
4 | autorestart=true
5 | redirect_stderr=true
6 | stdout_logfile=/dev/stdout
7 | stdout_logfile_maxbytes=0
8 |
9 | [program:kadmind]
10 | command=/bin/bash -c "exec /usr/sbin/kadmind -P /var/run/kadmind.pid -nofork -r LABS.TERADATA.COM"
11 | autostart=true
12 | autorestart=true
13 | redirect_stderr=true
14 | stdout_logfile=/dev/stdout
15 | stdout_logfile_maxbytes=0
16 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/var/kerberos/krb5kdc/kadm5-other.acl:
--------------------------------------------------------------------------------
1 | */admin@OTHERLABS.TERADATA.COM *
2 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/var/kerberos/krb5kdc/kadm5.acl:
--------------------------------------------------------------------------------
1 | */admin@LABS.TERADATA.COM *
2 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive-kerberized/files/var/kerberos/krb5kdc/kdc.conf:
--------------------------------------------------------------------------------
1 | [kdcdefaults]
2 | kdc_ports = 88
3 | kdc_tcp_ports = 88
4 |
5 | [realms]
6 | LABS.TERADATA.COM = {
7 | acl_file = /var/kerberos/krb5kdc/kadm5.acl
8 | dict_file = /usr/share/dict/words
9 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
10 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
11 | }
12 |
13 | OTHERLABS.TERADATA.COM = {
14 | acl_file = /var/kerberos/krb5kdc/kadm5-other.acl
15 | dict_file = /usr/share/dict/words
16 | admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab
17 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
18 | kdc_listen = 89
19 | kdc_tcp_listen = 89
20 | kdc_ports = 89
21 | kdc_tcp_ports = 89
22 | }
23 |
24 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/centos6-oj8:unlabelled
14 | MAINTAINER Presto community
15 |
16 | # Change default timezone
17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone
18 |
19 | # Install HDP repo
20 | RUN wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0/hdp.repo -P /etc/yum.repos.d
21 |
22 | # Install Hadoop, Hive (w/ MySQL)
23 | RUN yum install -y \
24 | hadoop-hdfs-namenode \
25 | hadoop-hdfs-secondarynamenode \
26 | hadoop-hdfs-datanode \
27 |
28 | hadoop-yarn-resourcemanager \
29 | hadoop-yarn-nodemanager \
30 |
31 | hive \
32 | hive-metastore \
33 | hive-server2 \
34 |
35 | mysql-server mysql-connector-java \
36 |
37 | # Cleanup
38 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
39 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/hdp/current/hive-metastore/lib/mysql-connector-java.jar
40 |
41 | # Delete original configuration
42 | RUN rm -r /etc/hadoop/conf/* \
43 | && rm -r /etc/hive/conf/*
44 |
45 | # Copy configuration files
46 | COPY ./files /
47 |
48 | # Run setup script
49 | RUN /root/setup.sh \
50 | && rm -rf /tmp/* /var/tmp/*
51 |
52 | # Setup sock proxy
53 | RUN yum install -y openssh openssh-clients openssh-server && yum -y clean all
54 | RUN ssh-keygen -t rsa -b 4096 -C "automation@prestodb.io" -N "" -f /root/.ssh/id_rsa \
55 | && ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key \
56 | && ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key \
57 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
58 | RUN chmod 755 /root && chmod 700 /root/.ssh
59 | RUN passwd --unlock root
60 |
61 | # HDFS ports
62 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
63 |
64 | # YARN ports
65 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888
66 |
67 | # HIVE ports
68 | EXPOSE 9083 10000
69 |
70 | # SOCKS port
71 | EXPOSE 1180
72 |
73 | CMD supervisord -c /etc/supervisord.conf
74 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/README.md:
--------------------------------------------------------------------------------
1 | # hdp2.6-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/hdp2.6-hive.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/hdp2.6-hive
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/hdp2.6-hive.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/hdp2.6-hive
7 |
8 | A pseudo-distributed Hadoop image for testing Presto, based on the HDP distro.
9 |
10 | ## Oracle license
11 |
12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/hadoop/conf/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | fs.defaultFS
22 | hdfs://hadoop-master:9000
23 |
24 |
25 |
26 |
27 | hadoop.proxyuser.oozie.hosts
28 | *
29 |
30 |
31 | hadoop.proxyuser.oozie.groups
32 | *
33 |
34 |
35 |
36 |
37 | hadoop.proxyuser.httpfs.hosts
38 | *
39 |
40 |
41 | hadoop.proxyuser.httpfs.groups
42 | *
43 |
44 |
45 |
46 |
47 | hadoop.proxyuser.llama.hosts
48 | *
49 |
50 |
51 | hadoop.proxyuser.llama.groups
52 | *
53 |
54 |
55 |
56 |
57 | hadoop.proxyuser.hue.hosts
58 | *
59 |
60 |
61 | hadoop.proxyuser.hue.groups
62 | *
63 |
64 |
65 |
66 |
67 | hadoop.proxyuser.mapred.hosts
68 | *
69 |
70 |
71 | hadoop.proxyuser.mapred.groups
72 | *
73 |
74 |
75 |
76 |
77 | hadoop.proxyuser.hive.hosts
78 | *
79 |
80 |
81 |
82 | hadoop.proxyuser.hive.groups
83 | *
84 |
85 |
86 |
87 |
88 | hadoop.proxyuser.hdfs.groups
89 | *
90 |
91 |
92 |
93 | hadoop.proxyuser.hdfs.hosts
94 | *
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/hadoop/conf/hadoop-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Set Hadoop-specific environment variables here.
17 | # Forcing YARN-based mapreduce implementaion.
18 | # Make sure to comment out if you want to go back to the default or
19 | # if you want this to be tweakable on a per-user basis
20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
21 |
22 | # The maximum amount of heap to use, in MB. Default is 1000.
23 | export HADOOP_HEAPSIZE=256
24 |
25 | # Extra Java runtime options. Empty by default.
26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m"
27 | export YARN_OPTS="$YARN_OPTS -Xmx256m"
28 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.framework.name
27 | yarn
28 |
29 |
30 |
31 | mapreduce.jobhistory.address
32 | hadoop-master:10020
33 |
34 |
35 |
36 | mapreduce.jobhistory.webapp.address
37 | hadoop-master:19888
38 |
39 |
40 |
41 | To set the value of tmp directory for map and reduce tasks.
42 | mapreduce.task.tmp.dir
43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
44 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/hadoop/conf/yarn-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 | yarn.nodemanager.aux-services
23 | mapreduce_shuffle
24 |
25 |
26 |
27 | yarn.nodemanager.aux-services.mapreduce_shuffle.class
28 | org.apache.hadoop.mapred.ShuffleHandler
29 |
30 |
31 |
32 | yarn.log-aggregation-enable
33 | true
34 |
35 |
36 |
37 | yarn.dispatcher.exit-on-error
38 | true
39 |
40 |
41 |
42 | List of directories to store localized files in.
43 | yarn.nodemanager.local-dirs
44 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir
45 |
46 |
47 |
48 | Where to store container logs.
49 | yarn.nodemanager.log-dirs
50 | /var/log/hadoop-yarn/containers
51 |
52 |
53 |
54 | Where to aggregate logs to.
55 | yarn.nodemanager.remote-app-log-dir
56 | /var/log/hadoop-yarn/apps
57 |
58 |
59 |
60 | Classpath for typical applications.
61 | yarn.application.classpath
62 |
63 | /etc/hadoop/conf,
64 | /usr/hdp/current/hadoop-client/*,
65 | /usr/hdp/current/hadoop-client/lib/*,
66 | /usr/hdp/current/hadoop-hdfs-client/*,
67 | /usr/hdp/current/hadoop-hdfs-client/lib/*,
68 | /usr/hdp/current/hadoop-yarn-client/*,
69 | /usr/hdp/current/hadoop-yarn-client/lib/*,
70 | /usr/hdp/current/hadoop-mapreduce-client/*,
71 | /usr/hdp/current/hadoop-mapreduce-client/lib/*
72 |
73 |
74 |
75 |
76 | yarn.resourcemanager.hostname
77 | hadoop-master
78 |
79 |
80 |
81 | yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage
82 | 100
83 |
84 |
85 |
86 | yarn.log.server.url
87 | http://hadoop-master:19888/jobhistory/logs
88 |
89 |
90 |
91 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/hive/conf/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | javax.jdo.option.ConnectionURL
22 | jdbc:mysql://localhost/metastore
23 | the URL of the MySQL database
24 |
25 |
26 |
27 | javax.jdo.option.ConnectionDriverName
28 | com.mysql.jdbc.Driver
29 |
30 |
31 |
32 | javax.jdo.option.ConnectionUserName
33 | root
34 |
35 |
36 |
37 | javax.jdo.option.ConnectionPassword
38 | root
39 |
40 |
41 |
42 | datanucleus.autoCreateSchema
43 | false
44 |
45 |
46 |
47 | datanucleus.fixedDatastore
48 | true
49 |
50 |
51 |
52 | datanucleus.autoStartMechanism
53 | SchemaTable
54 |
55 |
56 |
57 | hive.security.authorization.createtable.owner.grants
58 | ALL
59 | The set of privileges automatically granted to the owner whenever a table gets created.
60 |
61 |
62 |
63 | hive.users.in.admin.role
64 | hdfs,hive
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/hdfs-datanode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-datanode]
2 | command=hdfs datanode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/hdfs-namenode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-namenode]
2 | command=hdfs namenode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/hive-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:hive-metastore]
2 | command=hive --service metastore
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-metastore.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/hive-server2.conf:
--------------------------------------------------------------------------------
1 | [program:hive-server2]
2 | command=hive --service hiveserver2
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-server2.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/mysql-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:mysql-metastore]
2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mysql
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/mysql/mysql.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/socks-proxy.conf:
--------------------------------------------------------------------------------
1 | [program:socks-proxy]
2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/socks-proxy
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/sshd.conf:
--------------------------------------------------------------------------------
1 | [program:sshd]
2 | command=/usr/sbin/sshd -D
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/sshd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/yarn-nodemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-nodemanager]
2 | command=yarn nodemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-nodemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/etc/supervisord.d/yarn-resourcemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-resourcemanager]
2 | command=yarn resourcemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-yarn/hadoop-yarn-resourcemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hdp2.6-hive/files/root/setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ex
2 |
3 | # 0 make file system hostname resolvable
4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts
5 |
6 | # 1 format namenode
7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/
8 |
9 | # workaround for 'could not open session' bug as suggested here:
10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610
11 | rm -f /etc/security/limits.d/hdfs.conf
12 | su -c "echo 'N' | hdfs namenode -format" hdfs
13 |
14 | # 2 start hdfs
15 | su -c "hdfs namenode 2>&1 > /var/log/hadoop-hdfs/hadoop-hdfs-namenode.log" hdfs&
16 |
17 | # 3 wait for process starting
18 | sleep 15
19 |
20 | # 4 init basic hdfs directories
21 | /usr/hdp/current/hadoop-client/libexec/init-hdfs.sh
22 |
23 | # 4.1 Create an hdfs home directory for the yarn user. For some reason, init-hdfs doesn't do so.
24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/yarn && /usr/bin/hadoop fs -chown yarn:yarn /user/yarn'
25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn'
26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging'
27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chown mapred:mapred /tmp/hadoop-yarn/staging/history && /usr/bin/hadoop fs -chmod -R 1777 /tmp/hadoop-yarn/staging/history'
28 |
29 |
30 |
31 | # 5 init hive directories
32 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse'
33 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse'
34 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse'
35 |
36 | # 6 stop hdfs
37 | killall java
38 |
39 | # 7 setup metastore
40 | mysql_install_db
41 |
42 | /usr/bin/mysqld_safe &
43 | sleep 10s
44 |
45 | cd /usr/hdp/current/hive-metastore/scripts/metastore/upgrade/mysql/
46 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
47 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.2.1000.mysql.sql;" | mysql
48 | /usr/bin/mysqladmin -u root password 'root'
49 |
50 | killall mysqld
51 | sleep 10s
52 | mkdir /var/log/mysql/
53 | chown mysql:mysql /var/log/mysql/
54 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/centos7-oj8:latest
14 | MAINTAINER Presto community
15 |
16 | RUN yum install -y \
17 | mariadb-server \
18 | openssh \
19 | openssh-clients \
20 | openssh-server \
21 | psmisc \
22 | which \
23 | && yum -q clean all && rm -rf /var/cache/yum
24 |
25 | ARG HADOOP_VERSION=3.1.2
26 | ARG HIVE_VERSION=3.1.2
27 |
28 | # TODO Apache Archive is rate limited -- these should probably go in S3
29 | ARG HADOOP_BINARY_PATH=https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
30 | ARG HIVE_BINARY_PATH=https://apache.mivzakim.net/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz
31 |
32 | RUN curl -o /tmp/hadoop.tar.gz --url $HADOOP_BINARY_PATH && \
33 | tar xzf /tmp/hadoop.tar.gz --directory /opt && mv /opt/hadoop-$HADOOP_VERSION /opt/hadoop
34 |
35 | RUN curl -o /tmp/hive.tar.gz --url $HIVE_BINARY_PATH && \
36 | tar xzf /tmp/hive.tar.gz --directory /opt && mv /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive
37 |
38 | ARG MYSQL_CONNECTOR_VERSION=8.0.13
39 | ARG AWS_SDK_VERSION=1.11.906
40 | RUN mkdir /opt/hive/auxlib && \
41 | curl -o /opt/hive/auxlib/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar && \
42 | curl -o /opt/hive/auxlib/aws-java-sdk-core-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/$AWS_SDK_VERSION/aws-java-sdk-core-$AWS_SDK_VERSION.jar && \
43 | curl -o /opt/hive/auxlib/aws-java-sdk-s3-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/$AWS_SDK_VERSION/aws-java-sdk-s3-$AWS_SDK_VERSION.jar
44 |
45 | ENV HADOOP_HOME=/opt/hadoop
46 | ENV HIVE_HOME=/opt/hive
47 | ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/*
48 | ENV PATH=${HIVE_HOME}/bin:${HADOOP_HOME}/bin:${PATH}
49 |
50 | RUN ssh-keygen -t rsa -b 4096 -C "automation@prestodb.io" -N "" -f /root/.ssh/id_rsa && \
51 | ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key && \
52 | cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
53 | RUN chmod 755 /root && chmod 700 /root/.ssh
54 | RUN passwd --unlock root
55 |
56 | # Copy configuration files
57 | COPY ./files /
58 |
59 | # Run setup script
60 | RUN /root/setup.sh
61 |
62 | # HDFS port
63 | EXPOSE 9000 9870
64 |
65 | # HIVE Metastore port
66 | EXPOSE 9083 10000
67 |
68 | EXPOSE 1180
69 |
70 | CMD /root/entrypoint.sh
71 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/hadoop-init.d/init-hdfs.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -x
2 |
3 | echo 'N' | hdfs namenode -format
4 | sed -i -e "s|hdfs://localhost|hdfs://$(hostname)|g" /opt/hadoop/etc/hadoop/core-site.xml
5 | hdfs namenode &
6 | sleep 10 && hdfs dfs -mkdir -p /user/hive/warehouse && killall java
7 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/hadoop-init.d/set-aws-creds.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -x
2 |
3 | if [[ -n "${AWS_ACCESS_KEY_ID}" ]]
4 | then
5 | echo "Setting AWS keys"
6 | sed -i -e "s|\"Use AWS_ACCESS_KEY_ID .*\"|${AWS_ACCESS_KEY_ID}|g" \
7 | -e "s|\"Use AWS_SECRET_ACCESS_KEY .*\"|${AWS_SECRET_ACCESS_KEY}|g" \
8 | /opt/hive/conf/hive-site.xml
9 | fi
10 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/hdfs-datanode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-datanode]
2 | command=hdfs datanode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=root
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/hdfs-namenode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-namenode]
2 | command=hdfs namenode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=root
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/hive-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:hive-metastore]
2 | # Add `--debug:port=5006` for debugging
3 | command=hive --service metastore
4 | startsecs=2
5 | stopwaitsecs=10
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/hive/hive-metastore.log
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/hive-server2.conf:
--------------------------------------------------------------------------------
1 | [program:hive-server2]
2 | command=hive --service hiveserver2
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=root
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-server2.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/mysql-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:mysql-metastore]
2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mysql
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/mysql/mysql.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/socks-proxy.conf:
--------------------------------------------------------------------------------
1 | [program:socks-proxy]
2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/socks-proxy
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/etc/supervisord.d/sshd.conf:
--------------------------------------------------------------------------------
1 | [program:sshd]
2 | command=/usr/sbin/sshd -D
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/sshd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/opt/hadoop/etc/hadoop/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | fs.defaultFS
8 | hdfs://localhost:9000
9 |
10 |
11 |
12 | hadoop.proxyuser.root.hosts
13 | *
14 |
15 |
16 |
17 | hadoop.proxyuser.root.groups
18 | *
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/opt/hadoop/etc/hadoop/hadoop-env.sh:
--------------------------------------------------------------------------------
1 | export HADOOP_HEAPSIZE=256
2 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/opt/hadoop/etc/hadoop/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
16 |
17 |
18 |
19 |
20 |
21 |
22 | dfs.permissions.enabled
23 | false
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/opt/hive/conf/hive-env.sh:
--------------------------------------------------------------------------------
1 | export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS} -Xmx256m -Djava.io.tmpdir=/tmp"
2 | export HADOOP_HEAPSIZE=256
3 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/opt/hive/conf/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | hive.metastore.uris
7 | thrift://localhost:9083
8 |
9 |
10 |
11 | javax.jdo.option.ConnectionURL
12 | jdbc:mysql://localhost:3306/metastore?useSSL=false
13 |
14 |
15 |
16 | javax.jdo.option.ConnectionDriverName
17 | com.mysql.cj.jdbc.Driver
18 |
19 |
20 |
21 | javax.jdo.option.ConnectionUserName
22 | root
23 |
24 |
25 |
26 | javax.jdo.option.ConnectionPassword
27 | root
28 |
29 |
30 |
31 | hive.metastore.connect.retries
32 | 15
33 |
34 |
35 |
36 | hive.metastore.disallow.incompatible.col.type.changes
37 | false
38 |
39 |
40 |
41 |
42 | metastore.storage.schema.reader.impl
43 | org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
44 |
45 |
46 |
47 | hive.support.concurrency
48 | true
49 |
50 |
51 |
52 | hive.txn.manager
53 | org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
54 |
55 |
56 |
57 | hive.compactor.initiator.on
58 | true
59 |
60 |
61 |
62 | hive.compactor.worker.threads
63 | 1
64 |
65 |
66 |
67 | fs.s3.awsAccessKeyId
68 | "Use AWS_ACCESS_KEY_ID environment variable to set this value"
69 |
70 |
71 |
72 | fs.s3.awsSecretAccessKey
73 | "Use AWS_SECRET_ACCESS_KEY environment variable to set this value"
74 |
75 |
76 |
77 | fs.s3a.access.key
78 | "Use AWS_ACCESS_KEY_ID environment variable to set this value"
79 |
80 |
81 |
82 | fs.s3a.secret.key
83 | "Use AWS_SECRET_ACCESS_KEY environment variable to set this value"
84 |
85 |
86 |
87 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/root/entrypoint.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -x
2 |
3 | for init_script in /etc/hadoop-init.d/*; do
4 | "${init_script}"
5 | done
6 |
7 | supervisord -c /etc/supervisord.conf
8 |
--------------------------------------------------------------------------------
/prestodb/hive3.1-hive/files/root/setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ex
2 |
3 | ln -s /usr/bin/re solveip /usr/libexec # mariadb-server installs resolveip in /usr/bin but mysql_install_db expects it in /usr/libexec
4 | mkdir /var/log/mysql /var/log/hive /var/log/hadoop-hdfs
5 |
6 | mysql_install_db
7 |
8 | chown -R mysql:mysql /var/lib/mysql
9 |
10 | /usr/bin/mysqld_safe &
11 | sleep 10s
12 |
13 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
14 | echo "CREATE DATABASE metastore;" | mysql
15 | /usr/bin/mysqladmin -u root password 'root'
16 | /opt/hive/bin/schematool -dbType mysql -initSchema
17 |
18 | killall mysqld
19 | sleep 10s
20 | chown -R mysql:mysql /var/log/mysql/
21 | rm -rf /tmp/* /var/tmp/*
22 |
23 | adduser hive
24 |
25 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-base/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/centos6-oj8:unlabelled
14 | MAINTAINER Presto community
15 |
16 | RUN \
17 | # Add IBM IOP repo
18 | wget -nv https://ibm-open-platform.ibm.com/repos/IOP/rhel/6/x86_64/4.2.x/GA/IOP.repo -P /etc/yum.repos.d \
19 |
20 | # Install hive
21 | && yum install -y hive_4_2_0_0 \
22 |
23 | # Alias log directories so that files in `/common` can use `/var/log/hadoop/*` format
24 | && mkdir -p /var/log/hadoop \
25 | && ln -s /var/log/hadoop-hdfs /var/log/hadoop/hdfs \
26 | && ln -s /var/log/hadoop-mapreduce /var/log/hadoop/mapreduce \
27 | && ln -s /var/log/hadoop-yarn /var/log/hadoop/yarn \
28 |
29 | # Cleanup
30 | && yum -y clean all && rm -rf /tmp/* /var/tmp/*
31 |
32 | COPY ./files /
33 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-base/README.md:
--------------------------------------------------------------------------------
1 | # iop4.2-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/iop4.2-base.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/iop4.2-base
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/iop4.2-base.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/iop4.2-base
7 |
8 | A Hadoop base image for creating images testing Presto, based on the IOP distro.
9 |
10 | ## Oracle license
11 |
12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here:
13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
14 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-base/files/etc/supervisord.conf:
--------------------------------------------------------------------------------
1 | [supervisord]
2 | logfile = /var/log/supervisord.log
3 | logfile_maxbytes = 50MB
4 | logfile_backups=10
5 | loglevel = info
6 | pidfile = /var/run/supervisord.pid
7 | nodaemon = true
8 | directory = /tmp
9 | strip_ansi = false
10 |
11 | [unix_http_server]
12 | file = /tmp/supervisor.sock
13 |
14 | [rpcinterface:supervisor]
15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16 |
17 | [supervisorctl]
18 | serverurl = unix:///tmp/supervisor.sock
19 |
20 | [include]
21 | files = /etc/supervisord.d/*.conf
22 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/Dockerfile:
--------------------------------------------------------------------------------
1 | # Licensed under the Apache License, Version 2.0 (the "License");
2 | # you may not use this file except in compliance with the License.
3 | # You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | FROM prestodb/iop4.2-base:unlabelled
14 | MAINTAINER Presto community
15 |
16 | # CHANGE DEFAULT TIMEZONE
17 | RUN ln -snf "/usr/share/zoneinfo/Asia/Kathmandu" /etc/localtime && echo "Asia/Kathmandu" > /etc/timezone
18 |
19 | # INSTALL MYSQL (Hive Metastore)
20 | RUN yum install -y mysql-server mysql-connector-java \
21 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
22 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/iop/4.2.0.0/hive/lib/mysql-connector-java.jar
23 |
24 | # COPY CONFIGURATION FILES
25 | COPY ./files /
26 |
27 | # RUN SETUP
28 | RUN /root/setup.sh
29 |
30 | # DO CLEANUP
31 | RUN rm -rf /tmp/* /var/tmp/*
32 |
33 | # SETUP SOCKS PROXY
34 | RUN yum install -y openssh openssh-clients openssh-server
35 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa
36 | RUN ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key
37 | RUN ssh-keygen -t dsa -b 1024 -N "" -f /etc/ssh/ssh_host_dsa_key
38 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
39 | RUN chmod 755 /root && chmod 700 /root/.ssh
40 | RUN passwd --unlock root
41 |
42 | # HDFS PORTS
43 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
44 |
45 | # YARN PORTS
46 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888
47 |
48 | # HIVE PORT
49 | EXPOSE 9083 10000
50 |
51 | # SOCKS PORT
52 | EXPOSE 1180
53 |
54 | CMD supervisord -c /etc/supervisord.conf
55 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/README.md:
--------------------------------------------------------------------------------
1 | # iop4.2-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link]
2 |
3 | [layers-badge]: https://images.microbadger.com/badges/image/prestodb/iop4.2-hive.svg
4 | [layers-link]: https://microbadger.com/images/prestodb/iop4.2-hive
5 | [version-badge]: https://images.microbadger.com/badges/version/prestodb/iop4.2-hive.svg
6 | [dockerhub-link]: https://hub.docker.com/r/prestodb/iop4.2-hive
7 |
8 | Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set.
9 | For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file.
10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal
11 | for general correctness testing.
12 |
13 | ## Run
14 |
15 | ```
16 | $ docker run -d --name hadoop-master -h hadoop-master prestodb/iop4.2-hive
17 | ```
18 |
19 | ## Oracle license
20 |
21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)
23 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/capabilities.txt:
--------------------------------------------------------------------------------
1 | exposes_hive
2 | allows_creating_a_table_in_hive
3 | allows_selecting_from_the_table
4 | exposes_socks_proxy
5 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/hadoop/conf/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | fs.defaultFS
22 | hdfs://hadoop-master:9000
23 |
24 |
25 |
26 |
27 | hadoop.proxyuser.oozie.hosts
28 | *
29 |
30 |
31 | hadoop.proxyuser.oozie.groups
32 | *
33 |
34 |
35 |
36 |
37 | hadoop.proxyuser.httpfs.hosts
38 | *
39 |
40 |
41 | hadoop.proxyuser.httpfs.groups
42 | *
43 |
44 |
45 |
46 |
47 | hadoop.proxyuser.llama.hosts
48 | *
49 |
50 |
51 | hadoop.proxyuser.llama.groups
52 | *
53 |
54 |
55 |
56 |
57 | hadoop.proxyuser.hue.hosts
58 | *
59 |
60 |
61 | hadoop.proxyuser.hue.groups
62 | *
63 |
64 |
65 |
66 |
67 | hadoop.proxyuser.mapred.hosts
68 | *
69 |
70 |
71 | hadoop.proxyuser.mapred.groups
72 | *
73 |
74 |
75 |
76 |
77 | hadoop.proxyuser.hive.hosts
78 | *
79 |
80 |
81 |
82 | hadoop.proxyuser.hive.groups
83 | *
84 |
85 |
86 |
87 |
88 | hadoop.proxyuser.hdfs.groups
89 | *
90 |
91 |
92 |
93 | hadoop.proxyuser.hdfs.hosts
94 | *
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/hadoop/conf/hadoop-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Set Hadoop-specific environment variables here.
17 | # Forcing YARN-based mapreduce implementaion.
18 | # Make sure to comment out if you want to go back to the default or
19 | # if you want this to be tweakable on a per-user basis
20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
21 |
22 | # The maximum amount of heap to use, in MB. Default is 1000.
23 | export HADOOP_HEAPSIZE=256
24 |
25 | # Extra Java runtime options. Empty by default.
26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m"
27 | export YARN_OPTS="$YARN_OPTS -Xmx256m"
28 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/hadoop/conf/hdfs-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
18 |
19 | dfs.namenode.name.dir
20 | /var/lib/hadoop-hdfs/cache/name/
21 |
22 |
23 |
24 | dfs.datanode.data.dir
25 | /var/lib/hadoop-hdfs/cache/data/
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/hadoop/conf/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | mapred.job.tracker
22 | hadoop-master:8021
23 |
24 |
25 |
26 | mapreduce.framework.name
27 | yarn
28 |
29 |
30 |
31 | mapreduce.jobhistory.address
32 | hadoop-master:10020
33 |
34 |
35 |
36 | mapreduce.jobhistory.webapp.address
37 | hadoop-master:19888
38 |
39 |
40 |
41 | To set the value of tmp directory for map and reduce tasks.
42 | mapreduce.task.tmp.dir
43 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks
44 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/hadoop/conf/yarn-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 | yarn.nodemanager.aux-services
21 | mapreduce_shuffle
22 |
23 |
24 |
25 | yarn.nodemanager.aux-services.mapreduce_shuffle.class
26 | org.apache.hadoop.mapred.ShuffleHandler
27 |
28 |
29 |
30 | yarn.log-aggregation-enable
31 | true
32 |
33 |
34 |
35 | yarn.dispatcher.exit-on-error
36 | true
37 |
38 |
39 |
40 | List of directories to store localized files in.
41 | yarn.nodemanager.local-dirs
42 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir
43 |
44 |
45 |
46 | Where to store container logs.
47 | yarn.nodemanager.log-dirs
48 | /var/log/hadoop/yarn/containers
49 |
50 |
51 |
52 | Where to aggregate logs to.
53 | yarn.nodemanager.remote-app-log-dir
54 | /var/log/hadoop/yarn/apps
55 |
56 |
57 |
58 | Classpath for typical applications.
59 | yarn.application.classpath
60 |
61 | /etc/hadoop/conf,
62 | /usr/iop/4.2.0.0/hadoop/*,
63 | /usr/iop/4.2.0.0/hadoop/lib/*,
64 | /usr/iop/4.2.0.0/hadoop-hdfs/*,
65 | /usr/iop/4.2.0.0/hadoop-hdfs/lib/*,
66 | /usr/iop/4.2.0.0/hadoop-yarn/*,
67 | /usr/iop/4.2.0.0/hadoop-yarn/lib/*,
68 | /usr/iop/4.2.0.0/hadoop-mapreduce/*,
69 | /usr/iop/4.2.0.0/hadoop-mapreduce/lib/*
70 |
71 |
72 |
73 |
74 | yarn.resourcemanager.hostname
75 | hadoop-master
76 |
77 |
78 |
79 | yarn.log.server.url
80 | http://hadoop-master:19888/jobhistory/logs
81 |
82 |
83 |
84 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/hive/conf/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 | javax.jdo.option.ConnectionURL
22 | jdbc:mysql://localhost/metastore
23 | the URL of the MySQL database
24 |
25 |
26 |
27 | javax.jdo.option.ConnectionDriverName
28 | com.mysql.jdbc.Driver
29 |
30 |
31 |
32 | javax.jdo.option.ConnectionUserName
33 | root
34 |
35 |
36 |
37 | javax.jdo.option.ConnectionPassword
38 | root
39 |
40 |
41 |
42 | datanucleus.autoCreateSchema
43 | false
44 |
45 |
46 |
47 | datanucleus.fixedDatastore
48 | true
49 |
50 |
51 |
52 | datanucleus.autoStartMechanism
53 | SchemaTable
54 |
55 |
56 |
57 | hive.security.authorization.createtable.owner.grants
58 | ALL
59 | The set of privileges automatically granted to the owner whenever a table gets created.
60 |
61 |
62 |
63 | hive.users.in.admin.role
64 | hdfs,hive
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/hdfs-datanode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-datanode]
2 | command=hdfs datanode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-datanode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/hdfs-namenode.conf:
--------------------------------------------------------------------------------
1 | [program:hdfs-namenode]
2 | command=hdfs namenode
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hdfs
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-namenode.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/hive-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:hive-metastore]
2 | command=hive --service metastore
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-metastore.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/hive-server2.conf:
--------------------------------------------------------------------------------
1 | [program:hive-server2]
2 | command=hive --service hiveserver2
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=hive
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hive/hive-server2.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/mysql-metastore.conf:
--------------------------------------------------------------------------------
1 | [program:mysql-metastore]
2 | command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=mysql
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/mysql/mysql.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/socks-proxy.conf:
--------------------------------------------------------------------------------
1 | [program:socks-proxy]
2 | command=/usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/socks-proxy
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/sshd.conf:
--------------------------------------------------------------------------------
1 | [program:sshd]
2 | command=/usr/sbin/sshd -D
3 | startsecs=2
4 | stopwaitsecs=10
5 | startretries=30
6 | user=root
7 | redirect_stderr=true
8 | stdout_logfile=/var/log/sshd
9 | autostart=true
10 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/yarn-nodemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-nodemanager]
2 | command=yarn nodemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-nodemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/etc/supervisord.d/yarn-resourcemanager.conf:
--------------------------------------------------------------------------------
1 | [program:yarn-resourcemanager]
2 | command=yarn resourcemanager
3 | startsecs=2
4 | stopwaitsecs=10
5 | user=yarn
6 | redirect_stderr=true
7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-resourcemanager.log
8 | autostart=true
9 |
--------------------------------------------------------------------------------
/prestodb/iop4.2-hive/files/root/setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ex
2 |
3 | # 0 make file system hostname resolvable
4 | echo "127.0.0.1 hadoop-master" >> /etc/hosts
5 |
6 | # 1 format namenode
7 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/
8 |
9 | # workaround for 'could not open session' bug as suggested here:
10 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610
11 | rm -f /etc/security/limits.d/hdfs.conf
12 | su -c "echo 'N' | hdfs namenode -format" hdfs
13 |
14 | # 2 start hdfs
15 | su -c "hdfs datanode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-datanode.log" hdfs&
16 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs&
17 |
18 | # 3 wait for process starting
19 | sleep 10
20 |
21 | # 4 exec hdfs init script
22 | /usr/iop/4.2.0.0/hadoop/libexec/init-hdfs.sh
23 |
24 | # 5 init hive directories
25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse'
26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse'
27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse'
28 |
29 | # 6 stop hdfs
30 | killall java
31 |
32 | # 7 setup metastore
33 | mysql_install_db
34 |
35 | /usr/bin/mysqld_safe &
36 | sleep 10s
37 |
38 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
39 | echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/iop/4.2.0.0/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql
40 | /usr/bin/mysqladmin -u root password 'root'
41 |
42 | killall mysqld
43 | sleep 10s
44 | mkdir /var/log/mysql/
45 | chown mysql:mysql /var/log/mysql/
46 |
--------------------------------------------------------------------------------
/prestodb/spark/build-images.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -e
4 |
5 | VERSION=3.5.1
6 |
7 | docker build -t prestodb/spark-base:$VERSION ./spark-base
8 | docker build -t prestodb/spark-master:$VERSION ./spark-master
9 | docker build -t prestodb/spark-worker:$VERSION ./spark-worker
10 | docker build -t prestodb/spark-submit:$VERSION ./spark-submit
11 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-base/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM eclipse-temurin:11-jdk
2 | MAINTAINER Presto community
3 | LABEL version="0.2"
4 |
5 | ENV DAEMON_RUN=true
6 | ENV SPARK_VERSION=3.5.1
7 | ENV HADOOP_VERSION=3
8 | ENV SCALA_VERSION=2.13.14
9 | ENV SCALA_HOME=/usr/share/scala
10 | ENV SPARK_HOME=/spark
11 |
12 | RUN apt-get update && apt-get install -y curl vim wget software-properties-common ssh net-tools ca-certificates jq
13 |
14 | RUN cd "/tmp" && \
15 | wget --no-verbose "https://downloads.typesafe.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.tgz" && \
16 | tar xzf "scala-${SCALA_VERSION}.tgz" && \
17 | mkdir "${SCALA_HOME}" && \
18 | rm "/tmp/scala-${SCALA_VERSION}/bin/"*.bat && \
19 | mv "/tmp/scala-${SCALA_VERSION}/bin" "/tmp/scala-${SCALA_VERSION}/lib" "${SCALA_HOME}" && \
20 | ln -s "${SCALA_HOME}/bin/"* "/usr/bin/" && \
21 | rm -rf "/tmp/"*
22 |
23 | # Add Dependencies for PySpark
24 | RUN apt-get install -y python3 python3-pip python3-numpy python3-matplotlib python3-scipy python3-pandas python3-simpy
25 | RUN update-alternatives --install "/usr/bin/python" "python" "$(which python3)" 1
26 |
27 | #Scala instalation
28 | RUN export PATH="/usr/local/sbt/bin:$PATH" && apt update && apt install ca-certificates wget tar && mkdir -p "/usr/local/sbt" && wget -qO - --no-check-certificate "https://github.com/sbt/sbt/releases/download/v1.2.8/sbt-1.2.8.tgz" | tar xz -C /usr/local/sbt --strip-components=1 && sbt sbtVersion
29 |
30 | RUN wget --no-verbose http://apache.mirror.iphh.net/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
31 | && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark \
32 | && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
33 |
34 | # Fix the value of PYTHONHASHSEED
35 | # Note: this is needed when you use Python 3.3 or greater
36 | ENV PYTHONHASHSEED 1
37 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-base/README.md:
--------------------------------------------------------------------------------
1 | # spark-base
2 |
3 | Docker image with a Spark 3 environment installed on JDK 11. Used in the
4 | spark-master, spark-submit, and spark-work images.
5 |
6 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster).
7 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-master/Dockerfile:
--------------------------------------------------------------------------------
1 | ARG BASE_VERSION=3.5.1
2 |
3 | FROM prestodb/spark-base:${BASE_VERSION}
4 | MAINTAINER Presto community
5 |
6 | COPY start-master.sh /
7 |
8 | ENV SPARK_MASTER_PORT 7077
9 | ENV SPARK_MASTER_WEBUI_PORT 8080
10 | ENV SPARK_MASTER_LOG /spark/logs
11 |
12 | EXPOSE 8080 7077 6066
13 |
14 | CMD ["/bin/bash", "/start-master.sh"]
15 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-master/README.md:
--------------------------------------------------------------------------------
1 | # spark-master
2 |
3 | Spark master image used in Presto Spark integration tests (presto-spark-testing).
4 |
5 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster).
6 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-master/start-master.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | export SPARK_MASTER_HOST=`hostname`
4 |
5 | . "/spark/sbin/spark-config.sh"
6 |
7 | . "/spark/bin/load-spark-env.sh"
8 |
9 | mkdir -p $SPARK_MASTER_LOG
10 |
11 | export SPARK_HOME=/spark
12 |
13 | ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out
14 |
15 | cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out
16 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-submit/Dockerfile:
--------------------------------------------------------------------------------
1 | ARG BASE_VERSION=3.5.1
2 |
3 | FROM prestodb/spark-base:${BASE_VERSION}
4 | MAINTAINER Presto community
5 |
6 | COPY spark-submit.sh /
7 |
8 | ENV SPARK_MASTER_URL="spark://spark-master:7077"
9 | ENV SPARK_SUBMIT_ARGS=""
10 | ENV SPARK_APPLICATION_ARGS ""
11 | #ENV SPARK_APPLICATION_JAR_LOCATION /opt/spark-apps/myjar.jar
12 | #ENV SPARK_APPLICATION_MAIN_CLASS my.main.Application
13 |
14 | CMD ["/bin/bash", "/spark-submit.sh"]
15 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-submit/README.md:
--------------------------------------------------------------------------------
1 | # spark-submit
2 |
3 | Spark submit image used in Presto Spark integration tests (presto-spark-testing).
4 |
5 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster).
6 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-submit/spark-submit.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | /spark/bin/spark-submit \
4 | --class ${SPARK_APPLICATION_MAIN_CLASS} \
5 | --master ${SPARK_MASTER_URL} \
6 | --deploy-mode cluster \
7 | --total-executor-cores 1 \
8 | ${SPARK_SUBMIT_ARGS} \
9 | ${SPARK_APPLICATION_JAR_LOCATION} \
10 | ${SPARK_APPLICATION_ARGS} \
11 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-worker/Dockerfile:
--------------------------------------------------------------------------------
1 | ARG BASE_VERSION=3.5.1
2 |
3 | FROM prestodb/spark-base:${BASE_VERSION}
4 | MAINTAINER Presto community
5 |
6 | COPY start-worker.sh /
7 |
8 | ENV SPARK_WORKER_WEBUI_PORT 8081
9 | ENV SPARK_WORKER_LOG /spark/logs
10 | ENV SPARK_MASTER "spark://spark-master:7077"
11 |
12 | EXPOSE 8081
13 |
14 | CMD ["/bin/bash", "/start-worker.sh"]
15 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-worker/README.md:
--------------------------------------------------------------------------------
1 | # spark-worker
2 |
3 | Spark worker image used in Presto Spark integration tests (presto-spark-testing).
4 |
5 | Based on the work from [Spydernaz](https://github.com/Spydernaz/docker-spark-cluster).
6 |
--------------------------------------------------------------------------------
/prestodb/spark/spark-worker/start-worker.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | . "/spark/sbin/spark-config.sh"
4 | . "/spark/bin/load-spark-env.sh"
5 |
6 | mkdir -p $SPARK_WORKER_LOG
7 |
8 | export SPARK_HOME=/spark
9 |
10 | ln -sf /dev/stdout $SPARK_WORKER_LOG/spark-worker.out
11 |
12 | /spark/sbin/../bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port $SPARK_WORKER_WEBUI_PORT $SPARK_MASTER >> $SPARK_WORKER_LOG/spark-worker.out
13 |
--------------------------------------------------------------------------------