├── teradatalabs ├── centos6-java8-oracle-ldap │ ├── files │ │ ├── serial.txt │ │ ├── openldap-certificate.pem │ │ └── active-directory-certificate.crt │ ├── README.md │ └── Dockerfile ├── cdh5-base │ ├── files │ │ ├── startup.sh │ │ ├── supervisord.conf │ │ ├── supervisord.d │ │ │ └── bootstrap.conf │ │ └── change_timezone.sh │ ├── README.md │ └── Dockerfile ├── hdp2.5-base │ ├── files │ │ ├── startup.sh │ │ ├── supervisord.conf │ │ ├── supervisord.d │ │ │ └── bootstrap.conf │ │ └── change_timezone.sh │ ├── README.md │ └── Dockerfile ├── iop-base │ ├── files │ │ ├── startup.sh │ │ ├── supervisord.conf │ │ └── supervisord.d │ │ │ └── bootstrap.conf │ ├── README.md │ └── Dockerfile ├── iop-hive │ ├── files │ │ ├── socks-proxy.sh │ │ ├── conf │ │ │ ├── hive-env.sh │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hive-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── yarn-site.xml │ │ │ └── capacity-scheduler.xml │ │ ├── conf-zookeeper │ │ │ ├── zoo.cfg │ │ │ ├── log4j.properties │ │ │ └── zookeeper-env.sh │ │ ├── conf-setup │ │ │ ├── core-site.xml │ │ │ └── hdfs-site.xml │ │ ├── supervisord.d │ │ │ ├── socks-proxy.conf │ │ │ ├── zookeeper.conf │ │ │ ├── hdfs-datanode.conf │ │ │ ├── hdfs-namenode.conf │ │ │ ├── hive-server2.conf │ │ │ ├── hive-metastore.conf │ │ │ ├── mysql-metastore.conf │ │ │ ├── yarn-nodemanager.conf │ │ │ ├── yarn-resourcemanager.conf │ │ │ └── mapreduce-historyserver.conf │ │ ├── bootstrap.sh │ │ └── setup.sh │ ├── capabilities.txt │ ├── README.md │ └── Dockerfile ├── cdh5-hive │ ├── files │ │ ├── socks-proxy.sh │ │ ├── conf │ │ │ ├── hive-env.sh │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hive-site.xml │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ │ ├── supervisord.d │ │ │ ├── zookeeper.conf │ │ │ ├── hdfs-datanode.conf │ │ │ ├── hdfs-namenode.conf │ │ │ ├── hive-server2.conf │ │ │ ├── socks-proxy.conf │ │ │ ├── hive-metastore.conf │ │ │ ├── mysql-metastore.conf │ │ │ ├── yarn-nodemanager.conf │ │ │ ├── yarn-resourcemanager.conf │ │ │ └── mapreduce-historyserver.conf │ │ ├── bootstrap.sh │ │ └── setup.sh │ ├── capabilities.txt │ ├── README.md │ └── Dockerfile ├── hdp2.5-hive │ ├── files │ │ ├── socks-proxy.sh │ │ ├── conf │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hive-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── yarn-site.xml │ │ │ └── capacity-scheduler.xml │ │ ├── conf-zookeeper │ │ │ ├── zoo.cfg │ │ │ ├── log4j.properties │ │ │ └── zookeeper-env.sh │ │ ├── supervisord.d │ │ │ ├── sshd.conf │ │ │ ├── zookeeper.conf │ │ │ ├── hive-server2.conf │ │ │ ├── socks-proxy.conf │ │ │ ├── hdfs-datanode.conf │ │ │ ├── hdfs-namenode.conf │ │ │ ├── hive-metastore.conf │ │ │ ├── mysql-metastore.conf │ │ │ ├── yarn-nodemanager.conf │ │ │ ├── yarn-resourcemanager.conf │ │ │ └── mapreduce-historyserver.conf │ │ ├── conf-setup │ │ │ ├── core-site.xml │ │ │ └── hdfs-site.xml │ │ ├── bootstrap.sh │ │ └── setup.sh │ ├── capabilities.txt │ ├── README.md │ └── Dockerfile ├── presto-yarn-cdh5-hive-slave │ ├── files │ │ ├── keys │ │ │ ├── key │ │ │ └── key.pub │ │ └── supervisord.d │ │ │ └── sshd.conf │ ├── README.md │ └── Dockerfile ├── presto-yarn-hdp2.5-master │ ├── files │ │ ├── keys │ │ │ ├── key │ │ │ └── key.pub │ │ └── supervisord.conf │ ├── README.md │ └── Dockerfile ├── presto-yarn-hdp2.5-slave │ ├── files │ │ ├── keys │ │ │ ├── key │ │ │ └── key.pub │ │ └── supervisord.conf │ ├── README.md │ └── Dockerfile ├── cdh5-hive-kerberized │ ├── files │ │ ├── kerberos │ │ │ ├── kadm5.acl │ │ │ ├── kdc.conf │ │ │ └── krb5.conf │ │ ├── bootstrap-kerberos.sh │ │ ├── conf │ │ │ ├── taskcontroller.cfg │ │ │ ├── container-executor.cfg │ │ │ ├── hiveserver2-site.xml │ │ │ ├── core-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── hive-site.xml │ │ │ ├── yarn-site.xml │ │ │ └── hdfs-site.xml │ │ └── supervisord.d │ │ │ └── bootstrap-kerberos.conf │ ├── README.md │ └── Dockerfile ├── cdh5-hive-master │ ├── files │ │ ├── conf │ │ │ ├── hive-env.sh │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hive-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── yarn-site.xml │ │ │ └── capacity-scheduler.xml │ │ ├── conf-setup │ │ │ └── core-site.xml │ │ ├── supervisord.d │ │ │ ├── zookeeper.conf │ │ │ ├── hive-server2.conf │ │ │ ├── hdfs-namenode.conf │ │ │ ├── hive-metastore.conf │ │ │ ├── mysql-metastore.conf │ │ │ ├── yarn-nodemanager.conf │ │ │ ├── yarn-resourcemanager.conf │ │ │ └── mapreduce-historyserver.conf │ │ ├── bootstrap.sh │ │ └── setup.sh │ ├── Dockerfile │ └── README.md ├── hdp2.5-slave │ ├── files │ │ ├── conf │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ │ ├── supervisord.d │ │ │ ├── hdfs-datanode.conf │ │ │ └── yarn-nodemanager.conf │ │ └── bootstrap.sh │ ├── README.md │ └── Dockerfile ├── presto-yarn-cdh5-hive-master │ ├── files │ │ ├── keys │ │ │ ├── key │ │ │ └── key.pub │ │ └── supervisord.d │ │ │ └── sshd.conf │ ├── README.md │ └── Dockerfile ├── cdh5-hive-slave │ ├── files │ │ ├── conf │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── mapred-site.xml │ │ │ └── yarn-site.xml │ │ ├── supervisord.d │ │ │ ├── hdfs-datanode.conf │ │ │ └── yarn-nodemanager.conf │ │ └── bootstrap.sh │ ├── Dockerfile │ └── README.md ├── hdp2.5-master │ ├── files │ │ ├── conf │ │ │ ├── core-site.xml │ │ │ ├── hadoop-env.sh │ │ │ ├── hive-site.xml │ │ │ ├── mapred-site.xml │ │ │ ├── yarn-site.xml │ │ │ └── capacity-scheduler.xml │ │ ├── conf-zookeeper │ │ │ ├── zoo.cfg │ │ │ ├── log4j.properties │ │ │ └── zookeeper-env.sh │ │ ├── conf-setup │ │ │ └── core-site.xml │ │ ├── supervisord.d │ │ │ ├── zookeeper.conf │ │ │ ├── hdfs-namenode.conf │ │ │ ├── hive-server2.conf │ │ │ ├── hive-metastore.conf │ │ │ ├── mysql-metastore.conf │ │ │ ├── yarn-nodemanager.conf │ │ │ ├── yarn-resourcemanager.conf │ │ │ └── mapreduce-historyserver.conf │ │ ├── bootstrap.sh │ │ └── setup.sh │ ├── README.md │ └── Dockerfile ├── centos6-java8-oracle-openldap │ ├── files │ │ ├── openldap-certificate.pem │ │ ├── start_slapd.sh │ │ ├── setup │ │ │ ├── createOU.ldif │ │ │ └── modify_server.ldif │ │ ├── overlay │ │ │ ├── refint.ldif │ │ │ └── memberof.ldif │ │ └── certs │ │ │ └── private.pem │ ├── README.md │ └── Dockerfile ├── ubuntu-trusty-python2.6 │ ├── README.md │ └── Dockerfile ├── presto-admin-devenv │ ├── README.md │ └── Dockerfile ├── dns │ ├── Dockerfile │ └── README.md ├── centos6-java8-oracle │ ├── README.md │ └── Dockerfile └── centos6-ssh-oj8 │ ├── README.md │ ├── vagrant_insecure_rsa │ └── Dockerfile ├── commons ├── conf │ ├── hive-env.sh │ ├── hadoop-env.sh │ ├── mapred-site.xml │ ├── hive-site.xml │ ├── yarn-site.xml │ │ ├── cdh5.xml │ │ ├── iop.xml │ │ └── hdp2.5.xml │ ├── core-site.xml │ └── capacity-scheduler.xml ├── startup.sh ├── socks-proxy.sh ├── supervisord.d │ ├── bootstrap.conf │ ├── sshd.conf │ ├── mysql-metastore.conf │ ├── hive-server2.conf │ ├── hdfs-datanode.conf │ ├── hdfs-namenode.conf │ ├── hive-metastore.conf │ ├── socks-proxy.conf │ ├── yarn-nodemanager.conf │ ├── zookeeper.conf │ ├── yarn-resourcemanager.conf │ └── mapreduce-historyserver.conf ├── keys │ ├── key.pub │ └── key ├── supervisord.conf ├── conf-zookeeper │ ├── log4j.properties │ ├── zookeeper-env.sh │ └── zoo.cfg ├── openldap-certificate.pem └── conf-setup │ └── core-site.xml ├── push.sh ├── .gitignore ├── .travis └── github_tags_push_deploy_key.enc ├── find_broken_symlinks.sh ├── test ├── docker-compose.yml ├── image_tests │ └── image_tests.bats.sh └── Dockerfile ├── label-parent.sh ├── flag.sh ├── .travis.yml ├── depend.sh └── README.md /teradatalabs/centos6-java8-oracle-ldap/files/serial.txt: -------------------------------------------------------------------------------- 1 | 1234 -------------------------------------------------------------------------------- /teradatalabs/cdh5-base/files/startup.sh: -------------------------------------------------------------------------------- 1 | ../../../commons/startup.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-base/files/startup.sh: -------------------------------------------------------------------------------- 1 | ../../../commons/startup.sh -------------------------------------------------------------------------------- /teradatalabs/iop-base/files/startup.sh: -------------------------------------------------------------------------------- 1 | ../../../commons/startup.sh -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/socks-proxy.sh: -------------------------------------------------------------------------------- 1 | ../../../commons/socks-proxy.sh -------------------------------------------------------------------------------- /teradatalabs/cdh5-base/files/supervisord.conf: -------------------------------------------------------------------------------- 1 | ../../../commons/supervisord.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/socks-proxy.sh: -------------------------------------------------------------------------------- 1 | ../../../commons/socks-proxy.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/socks-proxy.sh: -------------------------------------------------------------------------------- 1 | ../../../commons/socks-proxy.sh -------------------------------------------------------------------------------- /teradatalabs/iop-base/files/supervisord.conf: -------------------------------------------------------------------------------- 1 | ../../../commons/supervisord.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-env.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-base/files/supervisord.conf: -------------------------------------------------------------------------------- 1 | ../../../commons/supervisord.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-env.sh -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-slave/files/keys/key: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-master/files/keys/key: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-slave/files/keys/key: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/kerberos/kadm5.acl: -------------------------------------------------------------------------------- 1 | */admin@LABS.TERADATA.COM * -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-env.sh -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-site.xml -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-master/files/keys/key: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-slave/files/keys/key.pub: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key.pub -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hadoop-env.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/hive-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf-zookeeper/zoo.cfg: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/zoo.cfg -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/iop.xml -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-master/files/keys/key.pub: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key.pub -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-slave/files/keys/key.pub: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key.pub -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-master/files/keys/key.pub: -------------------------------------------------------------------------------- 1 | ../../../../commons/keys/key.pub -------------------------------------------------------------------------------- /commons/conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | export HADOOP_OPTS="$HADOOP_OPTS -Dhive.root.logger=INFO,console" 2 | -------------------------------------------------------------------------------- /push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | while [ -n "$1" ]; do 4 | docker push "$1" 5 | shift 6 | done 7 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/mapred-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/cdh5.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf-zookeeper/zoo.cfg: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/zoo.cfg -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/hdp2.5.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/sshd.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf-zookeeper/zoo.cfg: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/zoo.cfg -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf-setup/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-setup/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/cdh5.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/cdh5.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf-setup/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-setup/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf-setup/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-setup/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/hdp2.5.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/yarn-site.xml/hdp2.5.xml -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf/capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/capacity-scheduler.xml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | depends 4 | flags 5 | graphviz 6 | dependency_graph.svg 7 | *.swp 8 | -------------------------------------------------------------------------------- /commons/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # run supervisord 4 | exec supervisord -c /etc/supervisord.conf 5 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-base/files/supervisord.d/bootstrap.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/bootstrap.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf-setup/core-site.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-setup/core-site.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/zookeeper.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-base/files/supervisord.d/bootstrap.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/bootstrap.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf/capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/capacity-scheduler.xml -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/zookeeper.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf/capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/capacity-scheduler.xml -------------------------------------------------------------------------------- /teradatalabs/iop-base/files/supervisord.d/bootstrap.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/bootstrap.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/socks-proxy.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/zookeeper.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/conf/capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf/capacity-scheduler.xml -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/zookeeper.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-datanode.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-namenode.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-server2.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/socks-proxy.conf -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-ldap/files/openldap-certificate.pem: -------------------------------------------------------------------------------- 1 | ../../../commons/openldap-certificate.pem -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf-zookeeper/log4j.properties: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/log4j.properties -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf-zookeeper/zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/zookeeper-env.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-server2.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/socks-proxy.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/zookeeper.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf-zookeeper/log4j.properties: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/log4j.properties -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf-zookeeper/zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/zookeeper-env.sh -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-datanode.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-namenode.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-server2.conf -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-slave/files/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/sshd.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-server2.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mysql-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/openldap-certificate.pem: -------------------------------------------------------------------------------- 1 | ../../../commons/openldap-certificate.pem -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-datanode.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-namenode.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf-zookeeper/log4j.properties: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/log4j.properties -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/conf-zookeeper/zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | ../../../../commons/conf-zookeeper/zookeeper-env.sh -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-namenode.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-server2.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-datanode.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mysql-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-master/files/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/sshd.conf -------------------------------------------------------------------------------- /commons/socks-proxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /usr/bin/ssh -o StrictHostKeyChecking=no -v -N -D 0.0.0.0:1180 localhost 4 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-namenode.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hdfs-datanode.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mysql-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/hive-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mysql-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mysql-metastore.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-nodemanager.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-resourcemanager.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-resourcemanager.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mapreduce-historyserver.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-resourcemanager.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-resourcemanager.conf -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mapreduce-historyserver.conf -------------------------------------------------------------------------------- /.travis/github_tags_push_deploy_key.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teradata/docker-images/HEAD/.travis/github_tags_push_deploy_key.enc -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/yarn-resourcemanager.conf -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/start_slapd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | service slapd restart 3 | tail -f /dev/null 4 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mapreduce-historyserver.conf -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mapreduce-historyserver.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | ../../../../commons/supervisord.d/mapreduce-historyserver.conf -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/capabilities.txt: -------------------------------------------------------------------------------- 1 | exposes_hive 2 | allows_creating_a_table_in_hive 3 | allows_selecting_from_the_table 4 | exposes_socks_proxy 5 | -------------------------------------------------------------------------------- /teradatalabs/iop-hive/capabilities.txt: -------------------------------------------------------------------------------- 1 | exposes_hive 2 | allows_creating_a_table_in_hive 3 | allows_selecting_from_the_table 4 | exposes_socks_proxy 5 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/capabilities.txt: -------------------------------------------------------------------------------- 1 | exposes_hive 2 | allows_creating_a_table_in_hive 3 | allows_selecting_from_the_table 4 | exposes_socks_proxy 5 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-base/files/change_timezone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export TZ=$1 4 | ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 5 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/bootstrap-kerberos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start Kerberos 4 | /sbin/service krb5kdc start 5 | /sbin/service kadmin start 6 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-base/files/change_timezone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export TZ=$1 4 | ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 5 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/taskcontroller.cfg: -------------------------------------------------------------------------------- 1 | hadoop.log.dir=/var/log/hadoop/mapreduce 2 | mapreduce.tasktracker.group=mapred 3 | banned.users=mapred,bin 4 | min.user.id=400 5 | -------------------------------------------------------------------------------- /commons/supervisord.d/bootstrap.conf: -------------------------------------------------------------------------------- 1 | [program:bootstrap] 2 | command=/root/bootstrap.sh 3 | redirect_stderr=true 4 | stdout_logfile=/var/log/bootstrap.log 5 | autostart=true 6 | autorestart=false 7 | -------------------------------------------------------------------------------- /commons/supervisord.d/sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/sshd 9 | autostart=false 10 | -------------------------------------------------------------------------------- /commons/supervisord.d/mysql-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:mysql-metastore] 2 | command=/usr/bin/mysqld_safe 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mysql 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/mysql/mysql.log 8 | autostart=true 9 | -------------------------------------------------------------------------------- /find_broken_symlinks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BROKEN_LINKS="$(find -L . -type l)" 4 | 5 | if [ ! "${BROKEN_LINKS}" = '' ]; then 6 | echo "The following symlinks are broken:" 7 | printf '%s\n' "${BROKEN_LINKS}" 8 | exit 1 9 | fi 10 | -------------------------------------------------------------------------------- /commons/supervisord.d/hive-server2.conf: -------------------------------------------------------------------------------- 1 | [program:hive-server2] 2 | command=hive --service hiveserver2 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-server2.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /commons/supervisord.d/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-datanode] 2 | command=hdfs datanode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-datanode.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /commons/supervisord.d/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:hdfs-namenode] 2 | command=hdfs namenode 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hdfs 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/hdfs/hadoop-hdfs-namenode.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /commons/supervisord.d/hive-metastore.conf: -------------------------------------------------------------------------------- 1 | [program:hive-metastore] 2 | command=hive --service metastore 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=hive 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hive/hive-metastore.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /commons/supervisord.d/socks-proxy.conf: -------------------------------------------------------------------------------- 1 | [program:socks-proxy] 2 | command=/root/socks-proxy.sh 3 | startsecs=2 4 | stopwaitsecs=10 5 | startretries=30 6 | user=root 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/socks-proxy 9 | autostart=false 10 | -------------------------------------------------------------------------------- /commons/supervisord.d/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-nodemanager] 2 | command=yarn nodemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-nodemanager.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /commons/supervisord.d/zookeeper.conf: -------------------------------------------------------------------------------- 1 | [program:zookeeper] 2 | command=/usr/bin/zookeeper-server start-foreground 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=zookeeper 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/zookeeper/zookeeper.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /commons/supervisord.d/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:yarn-resourcemanager] 2 | command=yarn resourcemanager 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=yarn 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/yarn/hadoop-yarn-resourcemanager.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/supervisord.d/bootstrap-kerberos.conf: -------------------------------------------------------------------------------- 1 | [program:bootstrap-kerberos] 2 | command=/root/bootstrap-kerberos.sh 3 | priority=0 4 | redirect_stderr=true 5 | stdout_logfile=/var/log/bootstrap-kerberos.log 6 | autostart=true 7 | autorestart=false 8 | -------------------------------------------------------------------------------- /test/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | 4 | hadoop-master: 5 | image: ${TESTED_IMAGE} 6 | hostname: hadoop-master 7 | ports: 8 | - '1180' 9 | 10 | test-runner: 11 | build: . 12 | volumes: 13 | - './image_tests:/image_tests:ro' 14 | -------------------------------------------------------------------------------- /commons/supervisord.d/mapreduce-historyserver.conf: -------------------------------------------------------------------------------- 1 | [program:mapreduce-historyserver] 2 | command=mapred historyserver 3 | startsecs=2 4 | stopwaitsecs=10 5 | user=mapred 6 | redirect_stderr=true 7 | stdout_logfile=/var/log/hadoop/mapreduce/hadoop-mapreduce-historyserver.log 8 | autostart=false 9 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/container-executor.cfg: -------------------------------------------------------------------------------- 1 | yarn.nodemanager.local-dirs=/var/lib/hadoop-yarn/cache/yarn/nm-local-dir 2 | yarn.nodemanager.linux-container-executor.group=yarn 3 | yarn.nodemanager.log-dirs=/var/log/hadoop/yarn/containers 4 | banned.users=yarn,mapred,bin 5 | min.user.id=400 6 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start hdfs-datanode 11 | 12 | supervisorctl start yarn-nodemanager 13 | 14 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start hdfs-datanode 11 | 12 | supervisorctl start yarn-nodemanager 13 | 14 | -------------------------------------------------------------------------------- /commons/keys/key.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdDZRQJgCrlyqY76F0q/ZSg7oJIFAaGn5unkHw/dkCuHP82RBkxanmFpbzaYGfl3uKhMZQdBa22qzlZBtSw4VulDJ1gjl3ZhQEdzqD1zpAUEOPHwKwnRyl+RzN5GwKhe7pr5VuheXYE8PuMZdJIYlDEWsobOHLCDU04po6SWaWNgrPeBLcEsaIF4nTihx5QxMospaztQSc2oFpX/vLQapieFay/E0pi/OQK5kRq9jItT4fokCX/TcdsqZR6w9RJq6t+Trv9UcQfW9cgIWc29D8sH2a9kmlya44VN0bSa6R21rutp2RU0IenrGurmHpEIcDci/HvI/ccgOO9E91Xr3j kogut@haxu 2 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/setup/createOU.ldif: -------------------------------------------------------------------------------- 1 | dn: dc=presto,dc=testldap,dc=com 2 | dc: presto 3 | o: teradata 4 | objectClass: organization 5 | objectClass: dcObject 6 | 7 | dn: ou=America,dc=presto,dc=testldap,dc=com 8 | objectClass: organizationalUnit 9 | ou: America 10 | 11 | dn: ou=Asia,dc=presto,dc=testldap,dc=com 12 | objectClass: organizationalUnit 13 | ou: Asia 14 | 15 | -------------------------------------------------------------------------------- /label-parent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Requires that the top layer is a LABEL layer. If it is, return the checksum 5 | # of its parent layer and exit 0. If the top layer isn't a LABEL layer, exit 1. 6 | # 7 | docker history --no-trunc --format "{{.ID}}\t{{.CreatedBy}}" "$1" | awk ' 8 | BEGIN { 9 | ec = 1; 10 | } 11 | $5 == "LABEL" && NR == 1 { 12 | getline; 13 | ec = 0; 14 | exit; 15 | } 16 | END { 17 | if (ec == 0) { 18 | print $1 19 | } 20 | exit ec 21 | }' 22 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/kerberos/kdc.conf: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | LABS.TERADATA.COM = { 7 | #master_key_type = aes256-cts 8 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 9 | dict_file = /usr/share/dict/words 10 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 11 | supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal 12 | } 13 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start zookeeper 11 | 12 | supervisorctl start hdfs-namenode 13 | 14 | supervisorctl start yarn-resourcemanager mapreduce-historyserver 15 | 16 | supervisorctl start hive-metastore hive-server2 17 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start zookeeper 11 | 12 | supervisorctl start hdfs-namenode 13 | 14 | supervisorctl start yarn-resourcemanager mapreduce-historyserver 15 | 16 | supervisorctl start hive-metastore hive-server2 17 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/kerberos/krb5.conf: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = LABS.TERADATA.COM 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | 14 | [realms] 15 | LABS.TERADATA.COM = { 16 | kdc = hadoop-master 17 | admin_server = hadoop-master 18 | } 19 | -------------------------------------------------------------------------------- /teradatalabs/ubuntu-trusty-python2.6/README.md: -------------------------------------------------------------------------------- 1 | # ubuntu-trusty-python2.6 [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/ubuntu-trusty-python2.6.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/ubuntu-trusty-python2.6 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/ubuntu-trusty-python2.6.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/ubuntu-trusty-python2.6 7 | 8 | Ubuntu Trusty + Python 2.6 -------------------------------------------------------------------------------- /teradatalabs/presto-admin-devenv/README.md: -------------------------------------------------------------------------------- 1 | # presto-admin-devenv [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/presto-admin-devenv.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/presto-admin-devenv 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/presto-admin-devenv.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/presto-admin-devenv 7 | 8 | A development environment image for Presto Admin. 9 | -------------------------------------------------------------------------------- /commons/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups=10 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | directory = /tmp 9 | strip_ansi = false 10 | 11 | [unix_http_server] 12 | file = /tmp/supervisor.sock 13 | 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl = unix:///tmp/supervisor.sock 19 | prompt = cdh-pseudo-distributed 20 | 21 | [include] 22 | files = /etc/supervisord.d/*.conf 23 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start zookeeper 11 | 12 | supervisorctl start hdfs-namenode hdfs-datanode 13 | 14 | supervisorctl start yarn-resourcemanager yarn-nodemanager mapreduce-historyserver 15 | 16 | supervisorctl start hive-metastore hive-server2 17 | 18 | /sbin/service sshd start 19 | 20 | supervisorctl start socks-proxy 21 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start zookeeper 11 | 12 | supervisorctl start hdfs-namenode hdfs-datanode 13 | 14 | supervisorctl start yarn-resourcemanager yarn-nodemanager mapreduce-historyserver 15 | 16 | supervisorctl start hive-metastore hive-server2 17 | 18 | /sbin/service sshd start 19 | 20 | supervisorctl start socks-proxy 21 | -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script executed by supervisord as bootstrap 4 | #### 5 | 6 | # It manager the hadoop processes startup sequence via 7 | # the systemctl command. This is the only script launched 8 | # at startup by supervisord. 9 | 10 | supervisorctl start zookeeper 11 | 12 | supervisorctl start hdfs-namenode hdfs-datanode 13 | 14 | supervisorctl start yarn-resourcemanager yarn-nodemanager mapreduce-historyserver 15 | 16 | supervisorctl start hive-metastore hive-server2 17 | 18 | /sbin/service sshd start 19 | 20 | supervisorctl start socks-proxy 21 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/overlay/refint.ldif: -------------------------------------------------------------------------------- 1 | # This overlay ensures that when user accounts are modified or deleted, 2 | # the corresponding member attribute in groups is also handled accordingly. 3 | dn: cn=module{1},cn=config 4 | cn: module 5 | objectclass: olcModuleList 6 | objectclass: top 7 | olcmoduleload: refint.la 8 | 9 | dn: olcOverlay={1}refint,olcDatabase={2}bdb,cn=config 10 | objectClass: olcConfig 11 | objectClass: olcOverlayConfig 12 | objectClass: olcRefintConfig 13 | objectClass: top 14 | olcOverlay: {1}refint 15 | olcRefintAttribute: memberof member manager owner 16 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/overlay/memberof.ldif: -------------------------------------------------------------------------------- 1 | # This overlay helps in determing which group a given user belongs to 2 | dn: cn=module,cn=config 3 | cn: module 4 | objectClass: olcModuleList 5 | olcModuleLoad: memberof.la 6 | olcModulePath: /usr/lib64/openldap 7 | 8 | dn: olcOverlay={0}memberof,olcDatabase={2}bdb,cn=config 9 | objectClass: olcConfig 10 | objectClass: olcMemberOf 11 | objectClass: olcOverlayConfig 12 | objectClass: top 13 | olcOverlay: memberof 14 | olcMemberOfDangling: ignore 15 | olcMemberOfRefInt: TRUE 16 | olcMemberOfGroupOC: groupOfNames 17 | olcMemberOfMemberAD: member 18 | olcMemberOfMemberOfAD: memberOf 19 | -------------------------------------------------------------------------------- /flag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | usage() { 4 | echo "$0 {target image}" >&2 5 | } 6 | 7 | # 8 | # This doesn't handle ARGs with a default supplied in the Dockerfile. 9 | # Feel free to add that functionality if needed. For now, YAGNI. 10 | # 11 | find_args() { 12 | local target_image=$(dirname "$target_dockerfile") 13 | awk -v image="$target_image" ' 14 | BEGIN { 15 | ARG_PATTERN = "^\\s*ARG"; 16 | print "DBFLAGS_" image " :="; 17 | } 18 | 19 | $1 == "ARG" { 20 | key = $2; 21 | print "DBFLAGS_" image " += --build-arg " key "=$(" key ")"; 22 | }' "$1" 23 | } 24 | 25 | if [ $# -lt 1 ]; then 26 | usage 27 | exit 1 28 | fi 29 | 30 | target_dockerfile=$1 31 | 32 | find_args "$target_dockerfile" 33 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-base/README.md: -------------------------------------------------------------------------------- 1 | # cdh5-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/cdh5-base.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/cdh5-base 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/cdh5-base.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/cdh5-base 7 | 8 | Docker image with Hive installed from Cloudera repositories. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/dns/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM alpine:3.3 16 | MAINTAINER Teradata Docker Team 17 | RUN apk --no-cache add dnsmasq 18 | EXPOSE 53 53/udp 19 | ENTRYPOINT ["dnsmasq", "-k"] 20 | -------------------------------------------------------------------------------- /teradatalabs/iop-base/README.md: -------------------------------------------------------------------------------- 1 | # iop-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/iop-base.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/iop-base 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/iop-base.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/iop-base 7 | 8 | A Hadoop base image for creating images testing Presto, based on the IOP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/hdp2.5-hive.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/hdp2.5-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/hdp2.5-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/hdp2.5-hive 7 | 8 | A Hadoop master + slave image for testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-slave [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/hdp2.5-slave.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/hdp2.5-slave 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/hdp2.5-slave.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/hdp2.5-slave 7 | 8 | A Hadoop slave image for testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /commons/conf-zookeeper/log4j.properties: -------------------------------------------------------------------------------- 1 | # Define some default values that can be overridden by system properties 2 | zookeeper.root.logger=INFO, CONSOLE 3 | zookeeper.console.threshold=INFO 4 | zookeeper.log.threshold=DEBUG 5 | log4j.rootLogger=INFO,CONSOLE 6 | 7 | # Example with rolling log file 8 | #log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE 9 | 10 | # Example with rolling log file and tracing 11 | #log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE 12 | 13 | # 14 | # Log INFO level and above messages to the console 15 | # 16 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 17 | log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} 18 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n 20 | 21 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-base/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/hdp2.5-base.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/hdp2.5-base 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/hdp2.5-base.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/hdp2.5-base 7 | 8 | A Hadoop base image for creating images testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/README.md: -------------------------------------------------------------------------------- 1 | # hdp2.5-master [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/hdp2.5-master.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/hdp2.5-master 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/hdp2.5-master.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/hdp2.5-master 7 | 8 | A Hadoop master image for testing Presto, based on the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this container, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle/README.md: -------------------------------------------------------------------------------- 1 | # centos6-java8-oracle [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/centos6-java8-oracle.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/centos6-java8-oracle 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/centos6-java8-oracle.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/centos6-java8-oracle 7 | 8 | Docker image of CentOS 6 with Oracle JDK 8 installed. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-slave/README.md: -------------------------------------------------------------------------------- 1 | # presto-yarn-hdp2.5-slave [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/presto-yarn-hdp2.5-slave.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/presto-yarn-hdp2.5-slave 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/presto-yarn-hdp2.5-slave.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/presto-yarn-hdp2.5-slave 7 | 8 | A Hadoop slave image for testing presto-yarn, using the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-master/README.md: -------------------------------------------------------------------------------- 1 | # presto-yarn-hdp2.5-master [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/presto-yarn-hdp2.5-master.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/presto-yarn-hdp2.5-master 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/presto-yarn-hdp2.5-master.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/presto-yarn-hdp2.5-master 7 | 8 | A Hadoop master image for testing presto-yarn, using the HDP distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-slave/README.md: -------------------------------------------------------------------------------- 1 | # presto-yarn-cdh5-hive-slave [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/presto-yarn-cdh5-hive-slave.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/presto-yarn-cdh5-hive-slave 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/presto-yarn-cdh5-hive-slave.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/presto-yarn-cdh5-hive-slave 7 | 8 | A Hadoop slave image for testing presto-yarn, using the CDH distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-master/README.md: -------------------------------------------------------------------------------- 1 | # presto-yarn-cdh5-hive-master [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/presto-yarn-cdh5-hive-master.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/presto-yarn-cdh5-hive-master 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/presto-yarn-cdh5-hive-master.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/presto-yarn-cdh5-hive-master 7 | 8 | A Hadoop master image for testing presto-yarn, using the CDH distro. 9 | 10 | ## Oracle license 11 | 12 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 13 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 14 | -------------------------------------------------------------------------------- /commons/conf-zookeeper/zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | export JAVA_HOME=${JAVA_HOME} 16 | export ZOO_LOG_DIR=${LOG_DIR} 17 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/README.md: -------------------------------------------------------------------------------- 1 | # centos6-java8-oracle-openldap [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/centos6-java8-oracle-openldap.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/centos6-java8-oracle-openldap 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/centos6-java8-oracle-openldap.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/centos6-java8-oracle-openldap 7 | 8 | Docker image of CentOS 6 with Oracle JDK 8 installed. This has OpenLdap needed for 9 | Presto when running product-tests with front-end LDAP authentication. 10 | 11 | ## Oracle license 12 | 13 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 14 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 15 | -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/conf-setup/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | dfs.namenode.name.dir 21 | /var/lib/hadoop-hdfs/cache/name/ 22 | 23 | 24 | 25 | dfs.datanode.data.dir 26 | /var/lib/hadoop-hdfs/cache/data/ 27 | 28 | 29 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/conf-setup/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | dfs.namenode.name.dir 21 | /var/lib/hadoop-hdfs/cache/name/ 22 | 23 | 24 | 25 | dfs.datanode.data.dir 26 | /var/lib/hadoop-hdfs/cache/data/ 27 | 28 | 29 | -------------------------------------------------------------------------------- /teradatalabs/centos6-ssh-oj8/README.md: -------------------------------------------------------------------------------- 1 | # centos6-ssh-oj8 [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/centos6-ssh-oj8.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/centos6-ssh-oj8 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/centos6-ssh-oj8.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/centos6-ssh-oj8 7 | 8 | Docker image of CentOS 6 with Oracle JDK 8 installed, and with sshd 9 | running. Passwordless ssh for the user app-admin is also set up. 10 | 11 | ## Further documentation 12 | For more documentation on this image, see the documentation for the 13 | base centos6-ssh image it uses: 14 | https://registry.hub.docker.com/u/jdeathe/centos-ssh/ 15 | 16 | ## Oracle license 17 | 18 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 19 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 20 | -------------------------------------------------------------------------------- /teradatalabs/ubuntu-trusty-python2.6/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM ubuntu:14.04 16 | MAINTAINER Teradata Docker Team 17 | 18 | RUN echo "deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main" \ 19 | > /etc/apt/sources.list.d/fkrull-deadsnakes-trusty.list \ 20 | && sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys DB82666C \ 21 | && sudo apt-get update \ 22 | && sudo apt-get -y install python2.6 wget \ 23 | && ln -s /usr/bin/python2.6 /usr/bin/python 24 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-ldap/README.md: -------------------------------------------------------------------------------- 1 | # centos6-java8-oracle-ldap [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/centos6-java8-oracle-ldap.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/centos6-java8-oracle-ldap 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/centos6-java8-oracle-ldap.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/centos6-java8-oracle-ldap 7 | 8 | Docker image of CentOS 6 with Oracle JDK 8 installed. This will act 9 | as the base image for presto when running product-tests with front-end 10 | LDAP authentication. This has the certificates for AD and OpenLDAP. 11 | The AD certificates are valid for an year and needs to be regenerated 12 | every year. 13 | 14 | ## Oracle license 15 | 16 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 17 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 18 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/cdh5-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL HDFS DATANODE and YARN NODEMANAGER 19 | RUN \ 20 | yum install -y \ 21 | hadoop-hdfs-datanode \ 22 | hadoop-yarn-nodemanager \ 23 | python-setuptools \ 24 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 25 | 26 | # ADD BOOTSTRAP SCRIPT 27 | COPY files/bootstrap.sh /root/ 28 | 29 | # CREATE CONF DIRECTORIES 30 | COPY files/conf/ /etc/hadoop/conf 31 | 32 | CMD /root/startup.sh 33 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-slave/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/cdh5-hive-slave:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | RUN yum install -y openssh-server openssh-clients passwd libcgroup 19 | 20 | RUN mkdir /var/run/sshd && mkdir /root/.ssh 21 | 22 | RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N '' 23 | 24 | COPY files/keys/key.pub /root/.ssh/authorized_keys 25 | 26 | COPY files/supervisord.d/* /etc/supervisord.d/ 27 | 28 | RUN chmod 400 /root/.ssh/authorized_keys 29 | 30 | EXPOSE 22 31 | 32 | CMD /root/startup.sh 33 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-slave/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/hdp2.5-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # Good grief. Get rid of the default hdp config. 19 | RUN rm -r /etc/hadoop/conf/* 20 | 21 | # ADD BOOTSTRAP SCRIPT 22 | COPY files/bootstrap.sh /root/ 23 | 24 | # Copy the real config in now that setup has run 25 | COPY files/conf/* /etc/hadoop/conf/ 26 | 27 | # HDFS PORTS 28 | EXPOSE 50010 1004 50075 1006 50020 8020 50070 50470 29 | 30 | # YARN PORTS 31 | EXPOSE 8032 8030 8031 8033 8088 8040 8042 8041 10020 19888 32 | 33 | CMD /root/startup.sh 34 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-cdh5-hive-master/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/cdh5-hive-master:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | RUN yum install -y openssh-server openssh-clients passwd libcgroup unzip && yum -y clean all 19 | 20 | RUN mkdir /var/run/sshd && mkdir /root/.ssh 21 | 22 | RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N '' 23 | 24 | COPY files/keys/key.pub /root/.ssh/authorized_keys 25 | COPY files/supervisord.d/* /etc/supervisord.d/ 26 | 27 | RUN chmod 400 /root/.ssh/authorized_keys 28 | 29 | EXPOSE 22 30 | 31 | CMD /root/startup.sh 32 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-slave/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/hdp2.5-slave:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | ARG JDK_PATH 19 | 20 | RUN rpm -e cracklib-dicts --nodeps && \ 21 | yum install -y cracklib-dicts passwd libcgroup && \ 22 | yum -y clean all 23 | 24 | RUN mkdir /root/.ssh 25 | 26 | COPY files/keys/key.pub /root/.ssh/authorized_keys 27 | 28 | RUN chmod 400 /root/.ssh/authorized_keys 29 | 30 | RUN passwd --unlock root 31 | 32 | RUN echo export JAVA_HOME=$JDK_PATH/ > /etc/profile.d/java.sh 33 | 34 | COPY files/supervisord.conf /etc/ 35 | -------------------------------------------------------------------------------- /teradatalabs/iop-hive/README.md: -------------------------------------------------------------------------------- 1 | # iop-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/iop-hive.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/iop-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/iop-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/iop-hive 7 | 8 | Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. 9 | For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | ## Run 14 | 15 | ``` 16 | $ docker run -d --name hadoop-master -h hadoop-master teradatalabs/iop-hive 17 | ``` 18 | 19 | ## Oracle license 20 | 21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 23 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/README.md: -------------------------------------------------------------------------------- 1 | # cdh5-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/cdh5-hive.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/cdh5-hive 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/cdh5-hive.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/cdh5-hive 7 | 8 | Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. 9 | For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | ## Run 14 | 15 | ``` 16 | $ docker run -d --name hadoop-master -h hadoop-master teradatalabs/cdh5-hive 17 | ``` 18 | 19 | ## Oracle license 20 | 21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 23 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/setup/modify_server.ldif: -------------------------------------------------------------------------------- 1 | dn: olcDatabase={2}bdb,cn=config 2 | changetype: modify 3 | replace: olcRootDN 4 | olcRootDN: cn=admin,dc=presto,dc=testldap,dc=com 5 | - 6 | replace: olcSuffix 7 | olcSuffix: dc=presto,dc=testldap,dc=com 8 | - 9 | replace: olcRootPW 10 | olcRootPW: admin 11 | - 12 | replace: olcAccess 13 | olcAccess: {0}to attrs=userPassword by self write by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" write by anonymous auth by * none 14 | olcAccess: {1}to * by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" write by self write by * read 15 | 16 | dn: olcDatabase={1}monitor,cn=config 17 | changetype: modify 18 | replace: olcAccess 19 | olcAccess: {0}to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=exter 20 | nal,cn=auth" read by dn.base="cn=admin,dc=presto,dc=testldap,dc=com" read by * n 21 | one 22 | 23 | dn: cn=config 24 | changetype: modify 25 | replace: olcTLSCACertificateFile 26 | olcTLSCACertificateFile: /etc/openldap/openldap-certificate.pem 27 | - 28 | replace: olcTLSCertificateFile 29 | olcTLSCertificateFile: /etc/openldap/openldap-certificate.pem 30 | - 31 | replace: olcTLSCertificateKeyFile 32 | olcTLSCertificateKeyFile: /etc/openldap/certs/private.pem 33 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-master/files/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisor/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups = 2 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | minfds = 1024 8 | minprocs = 200 9 | nodaemon = true 10 | directory = /tmp 11 | strip_ansi = false 12 | 13 | [program:sshd_bootstrap] 14 | priority = 5 15 | command = /etc/ssh-bootstrap 16 | startsecs = 0 17 | startretries = 0 18 | autorestart = false 19 | stdout_logfile = /var/log/secure 20 | stdout_events_enabled = true 21 | 22 | [program:sshd] 23 | priority = 10 24 | command = /usr/sbin/sshd -D -e 25 | redirect_stderr = true 26 | stdout_logfile = /var/log/secure 27 | stdout_events_enabled = true 28 | 29 | [eventlistener:supervisor_stdout] 30 | command = /usr/bin/supervisor_stdout 31 | buffer_size = 100 32 | events = PROCESS_LOG 33 | result_handler = supervisor_stdout:event_handler 34 | 35 | [include] 36 | files = /etc/supervisord.d/*.conf 37 | 38 | [unix_http_server] 39 | file = /tmp/supervisor.sock 40 | 41 | [rpcinterface:supervisor] 42 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 43 | 44 | [supervisorctl] 45 | serverurl = unix:///tmp/supervisor.sock 46 | prompt = cdh-pseudo-distributed 47 | 48 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-slave/files/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisor/supervisord.log 3 | logfile_maxbytes = 50MB 4 | logfile_backups = 2 5 | loglevel = info 6 | pidfile = /var/run/supervisord.pid 7 | minfds = 1024 8 | minprocs = 200 9 | nodaemon = true 10 | directory = /tmp 11 | strip_ansi = false 12 | 13 | [program:sshd_bootstrap] 14 | priority = 5 15 | command = /etc/ssh-bootstrap 16 | startsecs = 0 17 | startretries = 0 18 | autorestart = false 19 | stdout_logfile = /var/log/secure 20 | stdout_events_enabled = true 21 | 22 | [program:sshd] 23 | priority = 10 24 | command = /usr/sbin/sshd -D -e 25 | redirect_stderr = true 26 | stdout_logfile = /var/log/secure 27 | stdout_events_enabled = true 28 | 29 | [eventlistener:supervisor_stdout] 30 | command = /usr/bin/supervisor_stdout 31 | buffer_size = 100 32 | events = PROCESS_LOG 33 | result_handler = supervisor_stdout:event_handler 34 | 35 | [include] 36 | files = /etc/supervisord.d/*.conf 37 | 38 | [unix_http_server] 39 | file = /tmp/supervisor.sock 40 | 41 | [rpcinterface:supervisor] 42 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 43 | 44 | [supervisorctl] 45 | serverurl = unix:///tmp/supervisor.sock 46 | prompt = cdh-pseudo-distributed 47 | 48 | -------------------------------------------------------------------------------- /teradatalabs/presto-yarn-hdp2.5-master/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/hdp2.5-master:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | ARG JDK_PATH 19 | 20 | RUN rpm -e cracklib-dicts --nodeps && \ 21 | yum install -y cracklib-dicts passwd libcgroup unzip && \ 22 | yum -y clean all 23 | 24 | RUN mkdir /root/.ssh 25 | 26 | RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N '' 27 | 28 | COPY files/keys/key.pub /root/.ssh/authorized_keys 29 | 30 | RUN chmod 400 /root/.ssh/authorized_keys 31 | 32 | RUN passwd --unlock root 33 | 34 | RUN echo export JAVA_HOME=$JDK_PATH/ > /etc/profile.d/java.sh 35 | 36 | COPY files/supervisord.conf /etc/ 37 | -------------------------------------------------------------------------------- /commons/conf-zookeeper/zoo.cfg: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | maxClientCnxns=50 17 | # The number of milliseconds of each tick 18 | tickTime=2000 19 | # The number of ticks that the initial 20 | # synchronization phase can take 21 | initLimit=10 22 | # The number of ticks that can pass between 23 | # sending a request and getting an acknowledgement 24 | syncLimit=5 25 | # the directory where the snapshot is stored. 26 | dataDir=/var/lib/zookeeper 27 | # the port at which the clients will connect 28 | clientPort=2181 29 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/README.md: -------------------------------------------------------------------------------- 1 | # cdh5-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/cdh5-hive-kerberized.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/cdh5-hive-kerberized 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/cdh5-hive-kerberized.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/cdh5-hive-kerberized 7 | 8 | Docker image with HDFS, YARN and HIVE installed in a kerberized environment. Please note that running services have lower memory heap size set. 9 | For more details please check [blob/master/images/cdh5-hive/files/conf/hadoop-env.sh](configuration) file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | ## Run 14 | 15 | ``` 16 | $ docker run -d --name hadoop-master -h hadoop-master teradatalabs/cdh5-hive-kerberized 17 | ``` 18 | 19 | ## Oracle license 20 | 21 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 22 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 23 | -------------------------------------------------------------------------------- /commons/openldap-certificate.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDgTCCAmmgAwIBAgIJAKvPRrEwaoF0MA0GCSqGSIb3DQEBBQUAMFcxCzAJBgNV 3 | BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg 4 | Q29tcGFueSBMdGQxEzARBgNVBAMMCmxkYXBzZXJ2ZXIwHhcNMTYwODEwMTczOTIx 5 | WhcNMTkwODEwMTczOTIxWjBXMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVs 6 | dCBDaXR5MRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRkMRMwEQYDVQQDDAps 7 | ZGFwc2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA29sHE6G4 8 | 8F0MmHEuir/ZWMpIMjabWtMCp6q2Vbp52tqxSGVWqeVAKW+1qXmvNNMMlHzngmjT 9 | 0EIBoWk9LCKFeXkDCjwzASvjmGU1p9L0tFo9yEclDNlHo1B/Dq81ZnuaXZyBA7vd 10 | sbkcXqsES0m1736j9TP4Yfl915jrzma8JF92b0p479fH3xf/ZVTO3VNjtFYp186a 11 | ImIzeWDeNZk5ujPB6sJcKLm0l4q+wD4qpUbfcOqaT2YuauCsQGtqYAIf5F2uxyDA 12 | FdQt18zddlRocvuU/uPuF3bBNYxNICPrirqxsIJYudjdzmPjG+V1rDdQsl02c27e 13 | brbfq9d2togWDQIDAQABo1AwTjAdBgNVHQ4EFgQUfSaAXBYaUUlZ8myIwFJxmZI8 14 | EqIwHwYDVR0jBBgwFoAUfSaAXBYaUUlZ8myIwFJxmZI8EqIwDAYDVR0TBAUwAwEB 15 | /zANBgkqhkiG9w0BAQUFAAOCAQEAbTtHWLNJHhJUiZznPUTcAcX0YN+xcimcSquD 16 | JBKT6jqRgj+gOFXzG0gXVtATvXsvtXk/3wkbXEnnE6mBG/RMaP3pmsQ8BW1pUx1g 17 | bZofvPeEv8vRxd2Cs/dKhYv3MbTSAUPhuYub8oFB44TKS8uPmykIrTa9xG2uOYj/ 18 | LdRPATE6W8KcCw55OUMfUdFuJB9fkiDqTYNT/hscBeuBrRJeDOuKSvhPXDYb1+ie 19 | 8rL6EM5YOlT9MNAYohMb9re1ry4u4jwrWuO8X461NdFhCHxwIXYH/p2JW4cHl0yt 20 | JvmE/yFsEsUu21RtL798p2uV8ADwpX7VwcjxkTSjyS6ktTyI/A== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /commons/conf/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Set Hadoop-specific environment variables here. 17 | # Forcing YARN-based mapreduce implementaion. 18 | # Make sure to comment out if you want to go back to the default or 19 | # if you want this to be tweakable on a per-user basis 20 | # export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce 21 | 22 | # The maximum amount of heap to use, in MB. Default is 1000. 23 | export HADOOP_HEAPSIZE=256 24 | 25 | # Extra Java runtime options. Empty by default. 26 | export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -Xmx512m" 27 | export YARN_OPTS="$YARN_OPTS -Xmx256m" 28 | -------------------------------------------------------------------------------- /test/image_tests/image_tests.bats.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | skip_if_needed() { 4 | SHOULD_RUN=true 5 | # Can't rely on exit codes here, as BATS will fail the test if any of the statements 6 | # returns non-zero exit code. `set +e` does not solve the problem. 7 | echo "${EXPECTED_CAPABILITIES}" | grep -q "$BATS_TEST_DESCRIPTION" || SHOULD_RUN=false 8 | if [[ "$SHOULD_RUN" == false ]]; then 9 | skip 10 | fi 11 | } 12 | 13 | assert_run() { 14 | run "$@" 15 | echo "Output of [$*]:" 16 | echo 17 | printf '%s\n' "${lines[@]}" | nl -v 0 18 | return ${status} 19 | } 20 | 21 | assert_output_contains() { 22 | printf '%s\n' "${lines[@]}" | grep -q $1 23 | } 24 | 25 | function exposes_hive { 26 | skip_if_needed 27 | assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s 28 | } 29 | 30 | function allows_creating_a_table_in_hive { 31 | skip_if_needed 32 | assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' 33 | } 34 | 35 | function allows_selecting_from_the_table { 36 | skip_if_needed 37 | assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' 38 | assert_output_contains 'test.id' 39 | assert_output_contains '42' 40 | } 41 | 42 | function exposes_socks_proxy { 43 | skip_if_needed 44 | PROXY_ADDRESS='hadoop-master:1180' 45 | assert_run dockerize -wait tcp://${PROXY_ADDRESS} -timeout 10s 46 | assert_run curl --socks5 ${PROXY_ADDRESS} google.com 47 | } 48 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-slave/README.md: -------------------------------------------------------------------------------- 1 | # cdh5-hive-slave [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/cdh5-hive-slave.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/cdh5-hive-slave 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/cdh5-hive-slave.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/cdh5-hive-slave 7 | 8 | Docker image for slave node with CDH5 hadoop distribution. Please note that running services have lower memory heap size set. 9 | For more details please check [blob/master/images/cdh5-hive/files/conf/hadoop-env.sh](configuration) file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | Image is to be used together with one ore more containers running `cdh5-hive-master` image. 14 | 15 | ## Run 16 | 17 | ### Directly 18 | 19 | ``` 20 | docker run -d --name hadoop-slave -h hadoop-slave teradatalabs/cdh5-hive-slave 21 | ``` 22 | 23 | ### Using docker-compose 24 | 25 | See [example for cdh5-hive-master](../cdh5-hive-master/README.md#using-docker-compose). 26 | 27 | ## Oracle license 28 | 29 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 30 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 31 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/centos6-java8-oracle:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | RUN yum -y install openldap openldap-clients openldap-servers 19 | COPY files /tmp/files 20 | RUN cp -r /tmp/files/openldap-certificate.pem /etc/openldap/ 21 | RUN cp -r /tmp/files/certs/* /etc/openldap/certs/ 22 | RUN chmod a+x /tmp/files/start_slapd.sh 23 | RUN service slapd restart && \ 24 | ldapmodify -Y EXTERNAL -H ldapi:/// -f /tmp/files/setup/modify_server.ldif && \ 25 | ldapadd -Y EXTERNAL -H ldapi:/// -f /tmp/files/overlay/memberof.ldif && \ 26 | ldapadd -Y EXTERNAL -H ldapi:/// -f /tmp/files/overlay/refint.ldif && \ 27 | ldapadd -f /tmp/files/setup/createOU.ldif -D cn=admin,dc=presto,dc=testldap,dc=com -w admin && \ 28 | sed -i 's/SLAPD_LDAPS=no/SLAPD_LDAPS=yes/g' /etc/sysconfig/ldap && \ 29 | service slapd restart 30 | CMD /tmp/files/start_slapd.sh 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: generic 2 | dist: trusty 3 | 4 | env: 5 | matrix: 6 | - IMAGES='teradatalabs/centos6-java8-oracle.dependants' 7 | - IMAGES='teradatalabs/centos6-ssh-oj8.dependants' 8 | - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv} ) 9 | 10 | # Prevent duplicate builds on tag pushes. 11 | # See https://github.com/mockito/mockito/commit/500519aac4cc13770aa47c1eb7d28e905c27e7aa 12 | branches: 13 | except: 14 | - /^release.*/ 15 | - /^latest-snapshot.*/ 16 | 17 | services: 18 | - docker 19 | 20 | before_install: 21 | - sudo apt-get install dc 22 | - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 23 | - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 24 | - sudo apt-get update 25 | - sudo apt-get -y install docker-ce 26 | - docker --version 27 | 28 | install: make ${IMAGES[@]} 29 | 30 | before_deploy: 31 | # decrypt the github deploy key 32 | - openssl aes-256-cbc -K $encrypted_cd2beb64619c_key -iv $encrypted_cd2beb64619c_iv 33 | -in .travis/github_tags_push_deploy_key.enc -out .travis/github_tags_push_deploy_key -d 34 | - chmod 600 .travis/github_tags_push_deploy_key 35 | - eval "$(ssh-agent -s)" 36 | - ssh-add .travis/github_tags_push_deploy_key 37 | - docker login --username=${DOCKERHUB_LOGIN} --password=${DOCKERHUB_PASSWORD} 38 | 39 | # Publish snapshots to Dockerhub for commits tagged `latest-snapshot-*` 40 | deploy: 41 | on: 42 | all_branches: true 43 | condition: $( git tag --points-at HEAD | grep '^latest-snapshot' ) != '' 44 | skip_cleanup: true 45 | provider: script 46 | script: make snapshot 47 | -------------------------------------------------------------------------------- /teradatalabs/dns/README.md: -------------------------------------------------------------------------------- 1 | # docker-dnsmasq [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/dns.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/dns 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/dns.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/dns 7 | 8 | Based on https://github.com/andyshinn/docker-dnsmasq 9 | 10 | It's a [dnsmasq][dnsmasq] Docker image. It is only 6 MB in size. It is just an `ENTRYPOINT` to the `dnsmasq` binary. Can you smell what the rock is cookin'? 11 | 12 | ## Usage 13 | 14 | It is usually a good idea to use a tag other than `latest` if you are using this image in a production setting. 15 | 16 | [dnsmasq][dnsmasq] requires `NET_ADMIN` capabilities to run correctly. Start it with something like `docker run -p 53:53/tcp -p 53:53/udp --cap-add=NET_ADMIN teradatalabs/dns:2.75`. 17 | 18 | The configuration is all handled on the command line (no wrapper scripts here). The `ENTRYPOINT` is `dnsmasq -k` to keep it running in the foreground. If you wanted to send requests for an internal domain (such as Consul) you can forward the requests upstream using something like `docker run -p 53:53/tcp -p 53:53/udp --cap-add=NET_ADMIN teradatalabs/dns:2.75 -S /consul/10.17.0.2`. This will send a request for `redis.service.consul` to `10.17.0.2` 19 | 20 | As this is a very barebones entrypoint with just enough to run in the foreground, there is no logging enabled by default. To send logging to stdout you can add `--log-facility=-` as an option. 21 | 22 | [dnsmasq]: http://www.thekelleys.org.uk/dnsmasq/doc.html 23 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/files/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 1 format namenode 4 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 5 | 6 | # workaround for 'could not open session' bug as suggested here: 7 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 8 | rm -f /etc/security/limits.d/hdfs.conf 9 | su -c "echo 'N' | hdfs namenode -format" hdfs 10 | 11 | # 2 start hdfs 12 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 13 | 14 | # 3 wait for process starting 15 | sleep 10 16 | 17 | # 4 init basic hdfs directories 18 | /usr/hdp/2.5.0.0-1245/hadoop/libexec/init-hdfs.sh 19 | 20 | # 4.1 Create an hdfs home directory for the yarn user. For some reason, init-hdfs doesn't do so. 21 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/yarn && /usr/bin/hadoop fs -chown yarn:yarn /user/yarn' 22 | 23 | # 5 init hive directories 24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 27 | 28 | # 6 stop hdfs 29 | killall java 30 | 31 | # 7 setup metastore 32 | mysql_install_db 33 | 34 | /usr/bin/mysqld_safe & 35 | sleep 10s 36 | 37 | cd /usr/hdp/2.5.*/hive/scripts/metastore/upgrade/mysql/ 38 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 39 | echo "CREATE DATABASE metastore; USE metastore; SOURCE hive-schema-1.2.1000.mysql.sql;" | mysql 40 | /usr/bin/mysqladmin -u root password 'root' 41 | 42 | killall mysqld 43 | sleep 10s 44 | mkdir /var/log/mysql/ 45 | chown mysql:mysql /var/log/mysql/ 46 | 47 | exit 0 48 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/hiveserver2-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | hive.security.authorization.manager 22 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory 23 | SQL standards based Hive authorization 24 | 25 | 26 | 27 | hive.security.authorization.enabled 28 | true 29 | 30 | 31 | 32 | hive.security.authenticator.manager 33 | org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator 34 | 35 | 36 | 37 | hive.metastore.uris 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-base/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/centos6-ssh-oj8:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | RUN \ 19 | wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo -P /etc/yum.repos.d \ 20 | 21 | && yum install -y \ 22 | hadoop-hdfs-namenode \ 23 | hadoop-hdfs-secondarynamenode \ 24 | hadoop-hdfs-datanode \ 25 | 26 | hadoop-mapreduce-historyserver \ 27 | 28 | hadoop-yarn-resourcemanager \ 29 | hadoop-yarn-nodemanager \ 30 | 31 | zookeeper \ 32 | zookeeper-server \ 33 | 34 | hive \ 35 | hive-metastore \ 36 | hive-server2 \ 37 | 38 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 39 | 40 | # Copy supervisord startup script and base configs 41 | COPY files/startup.sh /root/startup.sh 42 | COPY files/supervisord.conf /etc/supervisord.conf 43 | COPY files/supervisord.d/bootstrap.conf /etc/supervisord.d/bootstrap.conf 44 | COPY files/change_timezone.sh /root/change_timezone.sh 45 | 46 | # Add supervisord configs in child images 47 | ONBUILD COPY files/supervisord.d/* /etc/supervisord.d/ 48 | -------------------------------------------------------------------------------- /commons/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | mapred.job.tracker 21 | hadoop-master:8021 22 | 23 | 24 | 25 | mapreduce.framework.name 26 | yarn 27 | 28 | 29 | 30 | mapreduce.jobhistory.address 31 | hadoop-master:10020 32 | 33 | 34 | mapreduce.jobhistory.webapp.address 35 | hadoop-master:19888 36 | 37 | 38 | 39 | To set the value of tmp directory for map and reduce tasks. 40 | mapreduce.task.tmp.dir 41 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /commons/keys/key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAnQ2UUCYAq5cqmO+hdKv2UoO6CSBQGhp+bp5B8P3ZArhz/NkQ 3 | ZMWp5haW82mBn5d7ioTGUHQWttqs5WQbUsOFbpQydYI5d2YUBHc6g9c6QFBDjx8C 4 | sJ0cpfkczeRsCoXu6a+VboXl2BPD7jGXSSGJQxFrKGzhywg1NOKaOklmljYKz3gS 5 | 3BLGiBeJ04oceUMTKLKWs7UEnNqBaV/7y0GqYnhWsvxNKYvzkCuZEavYyLU+H6JA 6 | l/03HbKmUesPUSaurfk67/VHEH1vXICFnNvQ/LB9mvZJpcmuOFTdG0mukdta7rad 7 | kVNCHp6xrq5h6RCHA3Ivx7yP3HIDjvRPdV694wIDAQABAoIBAFFTo9axJy2z9OIH 8 | TPNOzRpDjkWFjxVFXO1JVFpICWVIQP/RI5UbanR0hCx+GRqcZeQAB3XH968uo9OR 9 | uDhueR0e7tY83ic/G+SIHSwCeu6s4Z8ubsUoRpqH2muf+nakjmhCfT4jZjczXQVu 10 | dqnvDNZHIddoMrBhirPdGAJQ2iez+sm+R57rYxPYuf5HCUISxtdBGLBHuffz6Hok 11 | ydSqmt3gkS3aYvqtA+yKcDxDJsGCx/VX2dXZANOeBm0gcgTl92IW8jOfldWJl4ZU 12 | 7GW655Yc6pnMJNr6FsIKzUUOAnAGeN8xcLMyx3f8zKIJSgmOsvgMvOeMRfnX18TP 13 | O5mjXVkCgYEAy7pT9/1VaDpDsqlUl2d8YYVXykA2LLhgH/ozBjJBSDlQO1Njb7Bf 14 | SYb50hrVUAB1un73QUMZgw1UKdwzlQWdLSbrQdnAu27oTW++XScaB3qJJ4F1++hm 15 | kERGL3CUIcModd7pr6oVcljxsqvSM/P6FTAI7fz/bjVkVWLz8sZSt+UCgYEAxVl2 16 | BBCd1kekteng+hyhfZS4kikZdVNQYiC8yhOQLdYQHlekYpk90DdjLjhq1lLBecZm 17 | xB5Gy7kOM5ja0y8LvY6zevT8W9ukC5rCJw1PFmfUuxe5DiIN2yfYlRdv/wtTYZNP 18 | Qds8fnil8Do03dkFTVUyh7HC5AHRHg7cfXnSMicCgYA2vtRPoKDxyC5m9T/JC8MN 19 | xbJIpCaqr8UM8sQBV0HZsaUQvCDNY5zHemDph9JolCcOvY2d2PUjFVLXiNfHpOGO 20 | v5WadXRoKa59GJkUGSzSc6glmJFm5xWgkOg3WU5FIFdgDU3IqbDnsGUKuUdcNKSw 21 | St8rbMALEofqthON2qNhzQKBgQCmQhP2oLkYSQsuZnauv2gdkClemVllSmcyBDqI 22 | U6rnsquppFRM7KTywXbA/a6kGNv+Z3M0Tf1+q8yXE3Nm7v+JFquGLtZb3NJ7JEnk 23 | 9IVobtj9NaMlkPLzasI/+JrV1wjL73qaFMUGF0ZU9/SJ4cdhuyDyyINT62PtWr/O 24 | xc6biQKBgGT3rab4YTD/L3LKJfCYnojRLAfqJKYeQSjrlSHgoj61OGuH05tJ2fZG 25 | OOyScySWsqUUEqkd0IJ/3uVHPjGrff/CraerTW8xHSrQcImdm2d7p2IP2CMUUC0p 26 | H/jEYfrjCTz9TShBjtxbv1O/85tJRKl6Irtu9J+tLGkByeK/qQ/8 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /teradatalabs/presto-admin-devenv/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM centos:6 16 | MAINTAINER Teradata Docker Team 17 | 18 | # Install presto-admin dependences (already has python 2.6) 19 | RUN \ 20 | yum install -y epel-release && \ 21 | yum install -y wget bzip2 gcc python-devel python-pip tar expec libffi-devel openssl-devel && \ 22 | yum -y clean all && \ 23 | rm -rf \tmp\* \var\tmp\* 24 | 25 | RUN \ 26 | pip install --upgrade setuptools==20.1.1 && \ 27 | pip install --upgrade pip==7.1.2 && \ 28 | pip install --upgrade wheel==0.23.0 && \ 29 | pip install --upgrade argparse==1.4 && \ 30 | pip install --upgrade paramiko==1.15.3 && \ 31 | pip install --upgrade flake8==2.5.4 && \ 32 | pip install --upgrade py==1.4.26 && \ 33 | pip install --upgrade Sphinx==1.3.1 && \ 34 | pip install --upgrade fabric==1.10.1 && \ 35 | pip install --upgrade requests==2.7.0 && \ 36 | pip install --upgrade certifi==2015.4.28 && \ 37 | pip install --upgrade fudge==1.1.0 && \ 38 | pip install --upgrade PyYAML==3.11 && \ 39 | pip install --upgrade overrides==0.5 && \ 40 | pip install --upgrade retrying==1.3.3 41 | -------------------------------------------------------------------------------- /teradatalabs/centos6-ssh-oj8/vagrant_insecure_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI 3 | w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP 4 | kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 5 | hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO 6 | Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW 7 | yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd 8 | ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 9 | Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf 10 | TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK 11 | iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A 12 | sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf 13 | 4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP 14 | cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk 15 | EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN 16 | CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX 17 | 3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG 18 | YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj 19 | 3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ 20 | dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz 21 | 6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC 22 | P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF 23 | llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ 24 | kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH 25 | +vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ 26 | NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-openldap/files/certs/private.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEA29sHE6G48F0MmHEuir/ZWMpIMjabWtMCp6q2Vbp52tqxSGVW 3 | qeVAKW+1qXmvNNMMlHzngmjT0EIBoWk9LCKFeXkDCjwzASvjmGU1p9L0tFo9yEcl 4 | DNlHo1B/Dq81ZnuaXZyBA7vdsbkcXqsES0m1736j9TP4Yfl915jrzma8JF92b0p4 5 | 79fH3xf/ZVTO3VNjtFYp186aImIzeWDeNZk5ujPB6sJcKLm0l4q+wD4qpUbfcOqa 6 | T2YuauCsQGtqYAIf5F2uxyDAFdQt18zddlRocvuU/uPuF3bBNYxNICPrirqxsIJY 7 | udjdzmPjG+V1rDdQsl02c27ebrbfq9d2togWDQIDAQABAoIBAQDBip4/NQYWawN0 8 | i2Z6dItEc8Z8lb/Lg6h5q3maHG+7LvwUDEnvhEYodBPK3rlTWltfuFkhX2Dfm8vN 9 | qVRVrworNlFryDTqWhWgibBjTf1na22qCp9eIUTES0+ewLTUxAjPUb/tZouyMY/8 10 | 8rX0BS6pZ6W0bQzCSxzyF8e2SpBjvxhA+MYleMtxqc4rsn2lASu+ITNvgRIHXlQT 11 | ApD7IW3oG8DsSXE9UTAum/UoFKiv4VVqtKi5dAeSPRAabV/fo/Qtf8suQAvf4JxV 12 | tyUJtyMUvpRHV6KP6B1F0S+UYaXCWrD1r9xm8uA2WFtA4XvFFVVikreoos1soU7+ 13 | VJuOQlxNAoGBAPHHiYOWKl83+UrR94Mmjb9NJ7s4CKau9Y1Iu18ct+tt2rKihktE 14 | OkZk/duOKmW2oXxTuruz16d3+WGDErqbCM+hJS7rUs7Go9qPq+umjSbQHtOWUDwg 15 | 5osPorAR+9h/+xu9CPwaibT/9C/THa/gh0FsBEFw6ViHK5045NRDs7QLAoGBAOjJ 16 | Yj5/bhWyIdWawimZEvsNjG7zCDlM85yM/gd0xjvdBuPXTL9xY23TXHbodqw+x174 17 | K72w/6/B34eV/FLCcOK+esataod8J215O+OKvgul+E1SG1/Sfn4e/0rTVuHzzRvz 18 | aIbALMdU6PbPTVc73t7y5ALzQVpVOK723GwLLNVHAoGAIvGz7lDFzv1ICohOPaZU 19 | 1lFQbL6sgWrIEqNwcU61RlGaSjczMa2qvoHaLuFdP1YmANIZKwfTI2EzqeRH0LCX 20 | lWq09RMN5NTQ2cXqEyT/QZzS18Hh1s7qgy6/VLNEswXMRTYNFmK/hGl54W+h2smY 21 | wtvZwZuQHjtrEk4mxIcEcgECgYAoA12ownyorArWzZgsUEyL7MND2vCB8PE66p1L 22 | qM3eQlrJ1RszGQlvOlJ9ee3UI6Jfd6cFHySEEH4crzJFA3kl8HnoJqzWBwg78AfU 23 | T+fk1q18lO1xphvqD+fv863Ofvq1oDxKVUQdK0I0EmQH55milvuMakXEgjuRD4vJ 24 | 4o3dRwKBgQC5JH3iqCNhrS5jOlXsLnXiZQVvO4viWMFgyJi56dMkDM/dhhYhrr7r 25 | i5h/6LJOufhUfmvvoAaOG4mBkNwWfZImCGrj7VxQWMlwgIUFb5atgnx0f/SkOrqn 26 | +V7Dmj05PwtnYTpltKgCLjsoQGyzLeXV3bFXfSCt9Xmcyp1qRKrxEg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/hdp2.5-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL MYSQL (Hive Metastore) 19 | RUN yum install -y mysql-server mysql-connector-java \ 20 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 21 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/hdp/current/hive-metastore/lib/mysql-connector-java.jar 22 | 23 | # ADD SETUP AND BOOTSTRAP SCRIPTS 24 | COPY files/bootstrap.sh files/setup.sh /root/ 25 | 26 | # Create needed dirs 27 | RUN mkdir /var/lib/zookeeper 28 | RUN chown zookeeper:hadoop /var/lib/zookeeper 29 | 30 | # DELETE ORIGINAL CONFIGURATION 31 | RUN rm -rf /etc/hadoop/conf/* 32 | RUN rm -r /etc/zookeeper/conf/* 33 | RUN rm -r /etc/hive/conf/hive-site.xml 34 | 35 | # COPY SETUP CONFIGURATION 36 | COPY files/conf-setup/* /etc/hadoop/conf/ 37 | 38 | # RUN SETUP script 39 | RUN /root/setup.sh && rm -rf /tmp/* /var/tmp/* 40 | 41 | # COPY TARGET CONFIGURATION 42 | COPY files/conf/* /etc/hadoop/conf/ 43 | COPY files/conf-zookeeper/* /etc/zookeeper/conf/ 44 | COPY files/conf/hive-site.xml /etc/hive/conf/ 45 | 46 | # HDFS PORTS 47 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 48 | 49 | # YARN PORTS 50 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 51 | 52 | CMD /root/startup.sh 53 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/cdh5-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL HADOOP AND HIVE 19 | RUN \ 20 | yum install -y \ 21 | hive \ 22 | hive-server \ 23 | hadoop-hdfs-namenode \ 24 | hadoop-yarn-resourcemanager \ 25 | hadoop-yarn-nodemanager \ 26 | zookeeper-server \ 27 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 28 | 29 | # INSTALL MYSQL (Hive Metastore) 30 | RUN yum install -y mysql-server mysql-connector-java \ 31 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 32 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/mysql-connector-java.jar 33 | 34 | # ADD SETUP AND BOOTSTRAP SCRIPTS 35 | COPY files/bootstrap.sh files/setup.sh /root/ 36 | 37 | # DELETE ORIGINAL CONFIGURATION 38 | RUN rm -rf /etc/hadoop/conf/* 39 | 40 | # COPY SETUP CONFIGURATION 41 | COPY files/conf-setup/ /etc/hadoop/conf 42 | 43 | # RUN SETUP script 44 | RUN /root/setup.sh && rm -rf /tmp/* /var/tmp/* 45 | 46 | # COPY TARGET CONFIGURATION 47 | COPY files/conf/ /etc/hadoop/conf 48 | 49 | # HDFS PORTS 50 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 51 | 52 | # YARN PORTS 53 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 54 | 55 | # HIVE PORT 56 | EXPOSE 9083 10000 57 | 58 | CMD /root/startup.sh 59 | -------------------------------------------------------------------------------- /teradatalabs/iop-hive/files/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 1 format namenode 4 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 5 | 6 | # workaround for 'could not open session' bug as suggested here: 7 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 8 | rm -f /etc/security/limits.d/hdfs.conf 9 | su -c "echo 'N' | hdfs namenode -format" hdfs 10 | 11 | # 2 start hdfs 12 | su -c "hdfs datanode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-datanode.log" hdfs& 13 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 14 | 15 | # 3 wait for process starting 16 | sleep 10 17 | 18 | # 4 exec hdfs init script 19 | /usr/iop/4.2.0.0/hadoop/libexec/init-hdfs.sh 20 | 21 | # 5 init hive directories 22 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 23 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 25 | 26 | # 6 stop hdfs 27 | killall java 28 | 29 | # 7 setup metastore 30 | mysql_install_db 31 | 32 | /usr/bin/mysqld_safe & 33 | sleep 10s 34 | 35 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 36 | echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/iop/4.2.0.0/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql 37 | /usr/bin/mysqladmin -u root password 'root' 38 | 39 | killall mysqld 40 | sleep 10s 41 | mkdir /var/log/mysql/ 42 | chown mysql:mysql /var/log/mysql/ 43 | 44 | # 8 copy configuration 45 | cp /tmp/hadoop_conf/hive-site.xml /etc/hive/conf/ 46 | cp /tmp/hadoop_conf/capacity-scheduler.xml /etc/hadoop/conf 47 | cp /tmp/hadoop_conf/core-site.xml /etc/hadoop/conf 48 | cp /tmp/hadoop_conf/mapred-site.xml /etc/hadoop/conf 49 | cp /tmp/hadoop_conf/yarn-site.xml /etc/hadoop/conf 50 | cp /tmp/hadoop_conf/hadoop-env.sh /etc/hadoop/conf 51 | cp /tmp/hadoop_conf/hive-env.sh /etc/hive/conf 52 | rm -r /tmp/hadoop_conf 53 | 54 | # 9 Init zookeeper 55 | /usr/iop/4.2.0.0/zookeeper/bin/zookeeper-server-initialize 56 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/cdh5-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL HADOOP AND HIVE 19 | RUN \ 20 | yum install -y \ 21 | hive \ 22 | hive-server2 \ 23 | zookeeper-server \ 24 | hadoop-conf-pseudo \ 25 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 26 | 27 | # INSTALL MYSQL (Hive Metastore) 28 | RUN yum install -y mysql-server mysql-connector-java \ 29 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 30 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/lib/hive/lib/mysql-connector-java.jar 31 | 32 | # ADD SETUP AND BOOTSTRAP SCRIPTS 33 | COPY files/bootstrap.sh files/setup.sh /root/ 34 | 35 | # CREATE CONF DIRECTORIES 36 | RUN mkdir /tmp/hadoop_conf 37 | COPY files/conf/ /tmp/hadoop_conf 38 | 39 | # RUN SETUP script 40 | RUN /root/setup.sh && rm -rf /tmp/* /var/tmp/* 41 | 42 | # SETUP SOCKS PROXY 43 | RUN yum install -y openssh openssh-clients openssh-server 44 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa 45 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 46 | ADD files/socks-proxy.sh /root/socks-proxy.sh 47 | 48 | # HDFS PORTS 49 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 50 | 51 | # YARN PORTS 52 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 53 | 54 | # HIVE PORT 55 | EXPOSE 9083 10000 56 | 57 | # SOCKS PORT 58 | EXPOSE 1180 59 | 60 | CMD /root/startup.sh 61 | -------------------------------------------------------------------------------- /teradatalabs/iop-base/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/centos6-java8-oracle:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | RUN \ 19 | # Add IBM IOP repo 20 | wget -nv https://ibm-open-platform.ibm.com/repos/IOP/rhel/6/x86_64/4.2.x/GA/IOP.repo -P /etc/yum.repos.d \ 21 | 22 | # Install supervisord 23 | && yum install -y python-setuptools \ 24 | && easy_install pip \ 25 | && pip install supervisor \ 26 | && mkdir /etc/supervisord.d/ \ 27 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 28 | # ... and its missing dependency 29 | && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ 30 | && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ 31 | && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ 32 | 33 | # Install hive 34 | && yum install -y hive_4_2_0_0 \ 35 | 36 | # Alias log directories so that files in `/common` can use `/var/log/hadoop/*` format 37 | && mkdir -p /var/log/hadoop \ 38 | && ln -s /var/log/hadoop-hdfs /var/log/hadoop/hdfs \ 39 | && ln -s /var/log/hadoop-mapreduce /var/log/hadoop/mapreduce \ 40 | && ln -s /var/log/hadoop-yarn /var/log/hadoop/yarn 41 | 42 | # Copy supervisord startup script and base configs 43 | COPY files/startup.sh /root/startup.sh 44 | COPY files/supervisord.conf /etc/supervisord.conf 45 | COPY files/supervisord.d/bootstrap.conf /etc/supervisord.d/bootstrap.conf 46 | 47 | # Add supervisord configs in child images 48 | ONBUILD COPY files/supervisord.d/* /etc/supervisord.d/ 49 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive/files/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 1 format namenode 4 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 5 | 6 | # workaround for 'could not open session' bug as suggested here: 7 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 8 | rm -f /etc/security/limits.d/hdfs.conf 9 | su -c "echo 'N' | hdfs namenode -format" hdfs 10 | 11 | # 2 start hdfs 12 | su -c "hdfs datanode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-datanode.log" hdfs& 13 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 14 | 15 | # 3 wait for process starting 16 | sleep 10 17 | 18 | # remove a broken symlink created by cdh installer so that init-hdfs.sh does no blow up on it 19 | # (hbase-annotations.jar seems not needed in our case) 20 | rm /usr/lib/hive/lib/hbase-annotations.jar 21 | 22 | # 4 exec cloudera hdfs init script 23 | /usr/lib/hadoop/libexec/init-hdfs.sh 24 | 25 | # 5 init hive directories 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 28 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 29 | 30 | # 6 stop hdfs 31 | killall java 32 | 33 | # 7 setup metastore 34 | mysql_install_db 35 | 36 | /usr/bin/mysqld_safe & 37 | sleep 10s 38 | 39 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 40 | echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql 41 | /usr/bin/mysqladmin -u root password 'root' 42 | 43 | killall mysqld 44 | sleep 10s 45 | mkdir /var/log/mysql/ 46 | chown mysql:mysql /var/log/mysql/ 47 | 48 | # 8 copy configuration 49 | cp /tmp/hadoop_conf/hive-site.xml /etc/hive/conf/ 50 | cp /tmp/hadoop_conf/core-site.xml /etc/hadoop/conf 51 | cp /tmp/hadoop_conf/mapred-site.xml /etc/hadoop/conf 52 | cp /tmp/hadoop_conf/yarn-site.xml /etc/hadoop/conf 53 | cp /tmp/hadoop_conf/hadoop-env.sh /etc/hadoop/conf 54 | cp /tmp/hadoop_conf/hive-env.sh /etc/hive/conf 55 | rm -r /tmp/hadoop_conf 56 | 57 | # 9 Init zookeeper 58 | /etc/init.d/zookeeper-server init 59 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/centos6-java8-oracle 16 | 17 | ENV DOCKERIZE_VERSION v0.3.0 18 | 19 | RUN \ 20 | yum install -y \ 21 | wget \ 22 | unzip \ 23 | 24 | # setup CDH repo, pin the CDH distribution to a concrete version 25 | && wget -nv http://archive.cloudera.com/cdh5/one-click-install/redhat/6/x86_64/cloudera-cdh-5-0.x86_64.rpm \ 26 | && yum --nogpgcheck localinstall -y cloudera-cdh-5-0.x86_64.rpm \ 27 | && rm cloudera-cdh-5-0.x86_64.rpm \ 28 | && rpm --import http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera \ 29 | && sed -i '/^baseurl=/c\baseurl=https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.8.2/' /etc/yum.repos.d/cloudera-cdh5.repo \ 30 | 31 | # install BATS bash test lib 32 | && wget -nv https://github.com/ArturGajowy/bats/archive/6959d91.zip -O /tmp/bats.zip \ 33 | && unzip -q /tmp/bats.zip -d /tmp \ 34 | && /tmp/bats-*/install.sh /usr/local \ 35 | 36 | # install dockerize 37 | && wget -nv https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 38 | && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 39 | && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 40 | 41 | # install hive so that we can use beeline 42 | && yum install -y hive \ 43 | 44 | # cleanup 45 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 46 | 47 | # create 'image_tests' volume mount path 48 | && mkdir /image_tests 49 | 50 | VOLUME /image_tests 51 | 52 | ENTRYPOINT bats -t /image_tests 53 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/README.md: -------------------------------------------------------------------------------- 1 | # cdh5-hive-master [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] 2 | 3 | [layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/cdh5-hive-master.svg 4 | [layers-link]: https://microbadger.com/images/teradatalabs/cdh5-hive-master 5 | [version-badge]: https://images.microbadger.com/badges/version/teradatalabs/cdh5-hive-master.svg 6 | [dockerhub-link]: https://hub.docker.com/r/teradatalabs/cdh5-hive-master 7 | 8 | Docker image for master node with CDH5 hadoop distribution. Please note that running services have lower memory heap size set. 9 | For more details please check [blob/master/images/cdh5-hive/files/conf/hadoop-env.sh](configuration) file. 10 | If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal 11 | for general correctness testing. 12 | 13 | Image is to be used together with one ore more containers running `cdh5-hive-slave` image. 14 | 15 | ## Run 16 | 17 | ### Directly 18 | 19 | ``` 20 | docker run -d --name hadoop-master -h hadoop-master teradatalabs/cdh5-hive-master 21 | ``` 22 | 23 | ### Using docker-compose 24 | 25 | Following example shows how to build 4 node hadoop cluster using docker-compose 26 | 27 | ```yaml 28 | version: '2' 29 | services: 30 | hadoop-master: 31 | hostname: hadoop-master 32 | image: 'teradatalabs/cdh5-hive-master' 33 | ports: 34 | - '8020:8020' 35 | - '8088:8088' 36 | - '9083:9083' 37 | - '10000:10000' 38 | - '50070:50070' 39 | - '50075:50075' 40 | 41 | hadoop-slave1: 42 | hostname: 'hadoop-slave1' 43 | image: 'teradatalabs/cdh5-hive-slave' 44 | 45 | hadoop-slave2: 46 | hostname: 'hadoop-slave2' 47 | image: 'teradatalabs/cdh5-hive-slave' 48 | 49 | hadoop-slave3: 50 | hostname: 'hadoop-slave3' 51 | image: 'teradatalabs/cdh5-hive-slave' 52 | ``` 53 | 54 | ## Oracle license 55 | 56 | By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: 57 | [http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) 58 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/hdp2.5-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL MYSQL (Hive Metastore) 19 | RUN yum install -y mysql-server mysql-connector-java \ 20 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 21 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/hdp/current/hive-metastore/lib/mysql-connector-java.jar 22 | 23 | # ADD SETUP AND BOOTSTRAP SCRIPTS 24 | COPY files/bootstrap.sh files/setup.sh /root/ 25 | 26 | # Create needed dirs 27 | RUN mkdir /var/lib/zookeeper 28 | RUN chown zookeeper:hadoop /var/lib/zookeeper 29 | 30 | # DELETE ORIGINAL CONFIGURATION 31 | RUN rm -rf /etc/hadoop/conf/* 32 | RUN rm -r /etc/zookeeper/conf/* 33 | RUN rm -r /etc/hive/conf/hive-site.xml 34 | 35 | # COPY SETUP CONFIGURATION 36 | COPY files/conf-setup/* /etc/hadoop/conf/ 37 | 38 | # RUN SETUP script 39 | RUN /root/setup.sh && rm -rf /tmp/* /var/tmp/* 40 | 41 | # COPY TARGET CONFIGURATION 42 | COPY files/conf/* /etc/hadoop/conf/ 43 | COPY files/conf-zookeeper/* /etc/zookeeper/conf/ 44 | COPY files/conf/hive-site.xml /etc/hive/conf/ 45 | 46 | # SETUP SOCKS PROXY 47 | RUN yum install -y openssh openssh-clients openssh-server 48 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa 49 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 50 | RUN passwd --unlock root 51 | ADD files/socks-proxy.sh /root/socks-proxy.sh 52 | 53 | # HDFS PORTS 54 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 55 | 56 | # YARN PORTS 57 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 58 | 59 | # HIVE PORT 60 | EXPOSE 9083 10000 61 | 62 | # SOCKS PORT 63 | EXPOSE 1180 64 | 65 | CMD /root/startup.sh 66 | -------------------------------------------------------------------------------- /teradatalabs/iop-hive/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/iop-base:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL HADOOP AND HIVE 19 | RUN \ 20 | yum install -y \ 21 | zookeeper_4_2_0_0-server \ 22 | && ln -s /usr/iop/4.2.0.0/zookeeper/bin/zookeeper-server /usr/bin/zookeeper-server \ 23 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* 24 | 25 | # INSTALL MYSQL (Hive Metastore) 26 | RUN yum install -y mysql-server mysql-connector-java \ 27 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 28 | && ln -s /usr/share/java/mysql-connector-java.jar /usr/iop/4.2.0.0/hive/lib/mysql-connector-java.jar 29 | 30 | # ADD SETUP AND BOOTSTRAP SCRIPTS 31 | COPY files/bootstrap.sh files/setup.sh /root/ 32 | 33 | # CREATE CONF DIRECTORIES 34 | RUN mkdir /tmp/hadoop_conf 35 | COPY files/conf/ /tmp/hadoop_conf 36 | 37 | # COPY SETUP CONFIGURATION 38 | COPY files/conf-setup/ /etc/hadoop/conf 39 | 40 | COPY files/conf-zookeeper/* /etc/zookeeper/conf/ 41 | 42 | # Create needed dirs 43 | RUN mkdir /var/lib/zookeeper 44 | RUN chown zookeeper:hadoop /var/lib/zookeeper 45 | 46 | # RUN SETUP script 47 | RUN /root/setup.sh && rm -rf /tmp/* /var/tmp/* 48 | 49 | # SETUP SOCKS PROXY 50 | RUN yum install -y openssh openssh-clients openssh-server 51 | RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa 52 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 53 | ADD files/socks-proxy.sh /root/socks-proxy.sh 54 | 55 | # HDFS PORTS 56 | EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470 57 | 58 | # YARN PORTS 59 | EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888 60 | 61 | # HIVE PORT 62 | EXPOSE 9083 10000 63 | 64 | # SOCKS PORT 65 | EXPOSE 1180 66 | 67 | CMD /root/startup.sh 68 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-ldap/files/active-directory-certificate.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIGJTCCBQ2gAwIBAgITHQAAAAUM8gMo/ohRwwAAAAAABTANBgkqhkiG9w0BAQUF 3 | ADBlMRMwEQYKCZImiZPyLGQBGRYDY29tMRgwFgYKCZImiZPyLGQBGRYIdGVzdGxk 4 | YXAxFjAUBgoJkiaJk/IsZAEZFgZwcmVzdG8xHDAaBgNVBAMTE3ByZXN0by1BRC1U 5 | RVNULUNBLTEwHhcNMTcwNjI3MDA0NTM1WhcNMTgwNjI3MDA0NTM1WjAAMIIBIjAN 6 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoFyafCrwVdNqQ1Y1D1HMQVScNAc9 7 | m5l0nBbezmtbWExSgKXKE0DOIHLLZB38F8TKJZoa3m3J7Oin+GISrzWgyhoW6+jg 8 | KKGD9PZR5y4Xhu23Fv8IVlN9wf08vDploq34uzRFoBAwvjUcW42FPe+6MoKeMkat 9 | KgrfhCkcXAGhg0SOYmdCAHcPA63HBCNmpgYAZ38iqwiZugnraIXgn0VGhcmLlj7c 10 | S5Sv1GYOUtEd0MXELepAZPUXx8HivPvnz09tz++W6aKAkLsYvQShijPVnFinOAa9 11 | 8ZNlHyz8v5UiUbS/9kyQOY86mM+1U+qK9BQVIbRVvBSkLXPY2ZAvyZ9pCwIDAQAB 12 | o4IDMTCCAy0wPQYJKwYBBAGCNxUHBDAwLgYmKwYBBAGCNxUIhdXlWYazpGSH5Z0V 13 | hczUNYOhlVGBRKj2FYbb3DICAWQCAQEwMgYDVR0lBCswKQYHKwYBBQIDBQYKKwYB 14 | BAGCNxQCAgYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB/wQEAwIFoDBABgkr 15 | BgEEAYI3FQoEMzAxMAkGBysGAQUCAwUwDAYKKwYBBAGCNxQCAjAKBggrBgEFBQcD 16 | ATAKBggrBgEFBQcDAjAdBgNVHQ4EFgQU74Oc6DMfI847F2OplR+db0pIQJEwHwYD 17 | VR0jBBgwFoAUYVLOSj8PMaLbsTkN+P/R5RwEvMIwgdwGA1UdHwSB1DCB0TCBzqCB 18 | y6CByIaBxWxkYXA6Ly8vQ049cHJlc3RvLUFELVRFU1QtQ0EtMSxDTj1hZC10ZXN0 19 | LENOPUNEUCxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD 20 | Tj1Db25maWd1cmF0aW9uLERDPXByZXN0byxEQz10ZXN0bGRhcCxEQz1jb20/Y2Vy 21 | dGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENsYXNzPWNSTERpc3Ry 22 | aWJ1dGlvblBvaW50MIHQBggrBgEFBQcBAQSBwzCBwDCBvQYIKwYBBQUHMAKGgbBs 23 | ZGFwOi8vL0NOPXByZXN0by1BRC1URVNULUNBLTEsQ049QUlBLENOPVB1YmxpYyUy 24 | MEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9 25 | cHJlc3RvLERDPXRlc3RsZGFwLERDPWNvbT9jQUNlcnRpZmljYXRlP2Jhc2U/b2Jq 26 | ZWN0Q2xhc3M9Y2VydGlmaWNhdGlvbkF1dGhvcml0eTB0BgNVHREBAf8EajBooCwG 27 | CisGAQQBgjcUAgOgHgwcQUQtVEVTVCRAcHJlc3RvLnRlc3RsZGFwLmNvbYIbYWQt 28 | dGVzdC5wcmVzdG8udGVzdGxkYXAuY29tghNwcmVzdG8udGVzdGxkYXAuY29tggZQ 29 | UkVTVE8wDQYJKoZIhvcNAQEFBQADggEBAAtuksGPz4cwkIB7g8nqI1ysi0uaPxBO 30 | DSdq3wXPA9jS+W6bR39iiflugaeOW1pfQsZhMJadVCMkvQ/sYC4y1WmD+j6VSVs2 31 | 1znc4Gh+41teF3x1XRHgvQ1Vum+7GJIeKhZEUuB1F9SpuYxQT3wOIX2VnKq+1Z3/ 32 | 7TlRyZ9bAG8tmSeYXK4/dL5q+L7o1UgRU47V7RzhOJp/bmQFQd0gtXRpSH5LJZhk 33 | +lIlmCxgsjkawBBvXHWd9K6R/5SIr/S/RfOIVdU/WbIcOzzAzZr9YqhmB3POc4aL 34 | f4IY0N7Rxc/dBW1bO833+lNOI4dJJgtdZoOxCe3BiHsM2EbZzUiu1HA= 35 | -----END CERTIFICATE----- 36 | -------------------------------------------------------------------------------- /commons/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.security.authorization.createtable.owner.grants 58 | ALL 59 | The set of privileges automatically granted to the owner whenever a table gets created. 60 | 61 | 62 | 63 | hive.users.in.admin.role 64 | hdfs,hive 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM centos:6.8 16 | MAINTAINER Teradata Docker Team 17 | 18 | ENV DOCKERIZE_VERSION v0.3.0 19 | 20 | ARG JDK_URL 21 | ARG JDK_RPM 22 | ARG JDK_PATH 23 | 24 | RUN \ 25 | # disable the notoriuosly unstable EPEL repo... 26 | rm -rf /etc/yum.repos.d/epel* && \ 27 | 28 | yum install -y wget && \ 29 | 30 | # install and trim Oracle JDK 31 | wget -nv --header "Cookie: oraclelicense=accept-securebackup-cookie" $JDK_URL && \ 32 | rpm -ivh $JDK_RPM && rm $JDK_RPM && \ 33 | rm -rf $JDK_PATH/*src.zip \ 34 | $JDK_PATH/lib/missioncontrol \ 35 | $JDK_PATH/lib/visualvm \ 36 | $JDK_PATH/lib/*javafx* \ 37 | $JDK_PATH/jre/lib/plugin.jar \ 38 | $JDK_PATH/jre/lib/ext/jfxrt.jar \ 39 | $JDK_PATH/jre/bin/javaws \ 40 | $JDK_PATH/jre/lib/javaws.jar \ 41 | $JDK_PATH/jre/lib/desktop \ 42 | $JDK_PATH/jre/plugin \ 43 | $JDK_PATH/jre/lib/deploy* \ 44 | $JDK_PATH/jre/lib/*javafx* \ 45 | $JDK_PATH/jre/lib/*jfx* \ 46 | $JDK_PATH/jre/lib/amd64/libdecora_sse.so \ 47 | $JDK_PATH/jre/lib/amd64/libprism_*.so \ 48 | $JDK_PATH/jre/lib/amd64/libfxplugins.so \ 49 | $JDK_PATH/jre/lib/amd64/libglass.so \ 50 | $JDK_PATH/jre/lib/amd64/libgstreamer-lite.so \ 51 | $JDK_PATH/jre/lib/amd64/libjavafx*.so \ 52 | $JDK_PATH/jre/lib/amd64/libjfx*.so && \ 53 | # install dockerize 54 | wget -nv https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 55 | && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 56 | && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ 57 | # cleanup 58 | yum -y clean all && rm -rf /tmp/* /var/tmp/* 59 | 60 | ENV JAVA_HOME $JDK_PATH/jre/ 61 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-base/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/centos6-java8-oracle:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # Setup CDH repo, pin the CDH distribution to a concrete version 19 | RUN wget -nv http://archive.cloudera.com/cdh5/one-click-install/redhat/6/x86_64/cloudera-cdh-5-0.x86_64.rpm \ 20 | && yum --nogpgcheck localinstall -y cloudera-cdh-5-0.x86_64.rpm \ 21 | && rm cloudera-cdh-5-0.x86_64.rpm \ 22 | && rpm --import http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera \ 23 | && sed -i '/^baseurl=/c\baseurl=https://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.9.1/' /etc/yum.repos.d/cloudera-cdh5.repo \ 24 | 25 | # Install hive, as it's needed by all child images 26 | && yum install -y hive \ 27 | 28 | # Install supervisord 29 | && yum install -y python-setuptools \ 30 | && easy_install pip \ 31 | && pip install supervisor \ 32 | && mkdir /etc/supervisord.d/ \ 33 | # ... and its missing dependency 34 | && wget -nv http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ 35 | && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ 36 | && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ 37 | 38 | # Cleanup 39 | && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 40 | 41 | # Alias log directories so that files in `/common` can use `/var/log/hadoop/*` format 42 | && mkdir -p /var/log/hadoop \ 43 | && ln -s /var/log/hadoop-hdfs /var/log/hadoop/hdfs \ 44 | && ln -s /var/log/hadoop-mapreduce /var/log/hadoop/mapreduce \ 45 | && ln -s /var/log/hadoop-yarn /var/log/hadoop/yarn 46 | 47 | # Copy supervisord startup script and base configs 48 | COPY files/startup.sh /root/startup.sh 49 | COPY files/supervisord.conf /etc/supervisord.conf 50 | COPY files/supervisord.d/bootstrap.conf /etc/supervisord.d/bootstrap.conf 51 | COPY files/change_timezone.sh /root/change_timezone.sh 52 | 53 | # Add supervisord configs in child images 54 | ONBUILD COPY files/supervisord.d/* /etc/supervisord.d/ 55 | -------------------------------------------------------------------------------- /commons/conf-setup/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | fs.defaultFS 21 | hdfs://127.0.0.1:8020 22 | 23 | 24 | 25 | fs.default.name 26 | hdfs://localhost:9000 27 | 28 | 29 | 30 | 31 | hadoop.proxyuser.oozie.hosts 32 | * 33 | 34 | 35 | hadoop.proxyuser.oozie.groups 36 | * 37 | 38 | 39 | 40 | 41 | hadoop.proxyuser.httpfs.hosts 42 | * 43 | 44 | 45 | hadoop.proxyuser.httpfs.groups 46 | * 47 | 48 | 49 | 50 | 51 | hadoop.proxyuser.llama.hosts 52 | * 53 | 54 | 55 | hadoop.proxyuser.llama.groups 56 | * 57 | 58 | 59 | 60 | 61 | hadoop.proxyuser.hue.hosts 62 | * 63 | 64 | 65 | hadoop.proxyuser.hue.groups 66 | * 67 | 68 | 69 | 70 | 71 | hadoop.proxyuser.mapred.hosts 72 | * 73 | 74 | 75 | hadoop.proxyuser.mapred.groups 76 | * 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /commons/conf/yarn-site.xml/cdh5.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | yarn.nodemanager.aux-services 22 | mapreduce_shuffle 23 | 24 | 25 | 26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 27 | org.apache.hadoop.mapred.ShuffleHandler 28 | 29 | 30 | 31 | yarn.log-aggregation-enable 32 | true 33 | 34 | 35 | 36 | yarn.dispatcher.exit-on-error 37 | true 38 | 39 | 40 | 41 | List of directories to store localized files in. 42 | yarn.nodemanager.local-dirs 43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 44 | 45 | 46 | 47 | Where to store container logs. 48 | yarn.nodemanager.log-dirs 49 | /var/log/hadoop/yarn/containers 50 | 51 | 52 | 53 | Where to aggregate logs to. 54 | yarn.nodemanager.remote-app-log-dir 55 | /var/log/hadoop/yarn/apps 56 | 57 | 58 | 59 | Classpath for typical applications. 60 | yarn.application.classpath 61 | 62 | /etc/hadoop/conf, 63 | /usr/lib/hadoop/*, 64 | /usr/lib/hadoop/lib/*, 65 | /usr/lib/hadoop-hdfs/*, 66 | /usr/lib/hadoop-hdfs/lib/*, 67 | /usr/lib/hadoop-yarn/*, 68 | /usr/lib/hadoop-yarn/lib/*, 69 | /usr/lib/hadoop-mapreduce/*, 70 | /usr/lib/hadoop-mapreduce/lib/* 71 | 72 | 73 | 74 | 75 | yarn.resourcemanager.hostname 76 | hadoop-master 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /commons/conf/yarn-site.xml/iop.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | yarn.nodemanager.aux-services 22 | mapreduce_shuffle 23 | 24 | 25 | 26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 27 | org.apache.hadoop.mapred.ShuffleHandler 28 | 29 | 30 | 31 | yarn.log-aggregation-enable 32 | true 33 | 34 | 35 | 36 | yarn.dispatcher.exit-on-error 37 | true 38 | 39 | 40 | 41 | List of directories to store localized files in. 42 | yarn.nodemanager.local-dirs 43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 44 | 45 | 46 | 47 | Where to store container logs. 48 | yarn.nodemanager.log-dirs 49 | /var/log/hadoop/yarn/containers 50 | 51 | 52 | 53 | Where to aggregate logs to. 54 | yarn.nodemanager.remote-app-log-dir 55 | /var/log/hadoop/yarn/apps 56 | 57 | 58 | 59 | Classpath for typical applications. 60 | yarn.application.classpath 61 | 62 | /etc/hadoop/conf, 63 | /usr/iop/4.2.0.0/hadoop/*, 64 | /usr/iop/4.2.0.0/hadoop/lib/*, 65 | /usr/iop/4.2.0.0/hadoop-hdfs/*, 66 | /usr/iop/4.2.0.0/hadoop-hdfs/lib/*, 67 | /usr/iop/4.2.0.0/hadoop-yarn/*, 68 | /usr/iop/4.2.0.0/hadoop-yarn/lib/*, 69 | /usr/iop/4.2.0.0/hadoop-mapreduce/*, 70 | /usr/iop/4.2.0.0/hadoop-mapreduce/lib/* 71 | 72 | 73 | 74 | 75 | yarn.resourcemanager.hostname 76 | hadoop-master 77 | 78 | 79 | 80 | yarn.log.server.url 81 | http://hadoop-master:19888/jobhistory/logs 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /commons/conf/yarn-site.xml/hdp2.5.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | yarn.nodemanager.aux-services 22 | mapreduce_shuffle 23 | 24 | 25 | 26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 27 | org.apache.hadoop.mapred.ShuffleHandler 28 | 29 | 30 | 31 | yarn.log-aggregation-enable 32 | true 33 | 34 | 35 | 36 | yarn.dispatcher.exit-on-error 37 | true 38 | 39 | 40 | 41 | List of directories to store localized files in. 42 | yarn.nodemanager.local-dirs 43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 44 | 45 | 46 | 47 | Where to store container logs. 48 | yarn.nodemanager.log-dirs 49 | /var/log/hadoop/yarn/containers 50 | 51 | 52 | 53 | Where to aggregate logs to. 54 | yarn.nodemanager.remote-app-log-dir 55 | /var/log/hadoop/yarn/apps 56 | 57 | 58 | 59 | Classpath for typical applications. 60 | yarn.application.classpath 61 | 62 | /etc/hadoop/conf, 63 | /usr/hdp/current/hadoop-client/*, 64 | /usr/hdp/current/hadoop-client/lib/*, 65 | /usr/hdp/current/hadoop-hdfs-client/*, 66 | /usr/hdp/current/hadoop-hdfs-client/lib/*, 67 | /usr/hdp/current/hadoop-yarn-client/*, 68 | /usr/hdp/current/hadoop-yarn-client/lib/*, 69 | /usr/hdp/current/hadoop-mapreduce-client/*, 70 | /usr/hdp/current/hadoop-mapreduce-client/lib/* 71 | 72 | 73 | 74 | 75 | yarn.resourcemanager.hostname 76 | hadoop-master 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | fs.defaultFS 21 | hdfs://hadoop-master:8020 22 | 23 | 24 | 25 | 26 | hadoop.proxyuser.oozie.hosts 27 | * 28 | 29 | 30 | hadoop.proxyuser.oozie.groups 31 | * 32 | 33 | 34 | 35 | 36 | hadoop.proxyuser.httpfs.hosts 37 | * 38 | 39 | 40 | hadoop.proxyuser.httpfs.groups 41 | * 42 | 43 | 44 | 45 | 46 | hadoop.proxyuser.llama.hosts 47 | * 48 | 49 | 50 | hadoop.proxyuser.llama.groups 51 | * 52 | 53 | 54 | 55 | 56 | hadoop.proxyuser.hue.hosts 57 | * 58 | 59 | 60 | hadoop.proxyuser.hue.groups 61 | * 62 | 63 | 64 | 65 | 66 | hadoop.proxyuser.mapred.hosts 67 | * 68 | 69 | 70 | hadoop.proxyuser.mapred.groups 71 | * 72 | 73 | 74 | 75 | 76 | hadoop.security.authentication 77 | kerberos 78 | 79 | 80 | 81 | hadoop.security.authorization 82 | true 83 | 84 | 85 | 86 | 87 | hadoop.proxyuser.hive.hosts 88 | * 89 | 90 | 91 | 92 | hadoop.proxyuser.hive.groups 93 | * 94 | 95 | 96 | -------------------------------------------------------------------------------- /commons/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | fs.defaultFS 21 | hdfs://hadoop-master:8020 22 | 23 | 24 | 25 | fs.default.name 26 | hdfs://hadoop-master:9000 27 | 28 | 29 | 30 | 31 | hadoop.proxyuser.oozie.hosts 32 | * 33 | 34 | 35 | hadoop.proxyuser.oozie.groups 36 | * 37 | 38 | 39 | 40 | 41 | hadoop.proxyuser.httpfs.hosts 42 | * 43 | 44 | 45 | hadoop.proxyuser.httpfs.groups 46 | * 47 | 48 | 49 | 50 | 51 | hadoop.proxyuser.llama.hosts 52 | * 53 | 54 | 55 | hadoop.proxyuser.llama.groups 56 | * 57 | 58 | 59 | 60 | 61 | hadoop.proxyuser.hue.hosts 62 | * 63 | 64 | 65 | hadoop.proxyuser.hue.groups 66 | * 67 | 68 | 69 | 70 | 71 | hadoop.proxyuser.mapred.hosts 72 | * 73 | 74 | 75 | hadoop.proxyuser.mapred.groups 76 | * 77 | 78 | 79 | 80 | 81 | hadoop.proxyuser.hive.hosts 82 | * 83 | 84 | 85 | 86 | hadoop.proxyuser.hive.groups 87 | * 88 | 89 | 90 | 91 | 92 | hadoop.proxyuser.hdfs.groups 93 | * 94 | 95 | 96 | 97 | hadoop.proxyuser.hdfs.hosts 98 | * 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | mapred.job.tracker 21 | hadoop-master:8021 22 | 23 | 24 | 25 | mapreduce.framework.name 26 | yarn 27 | 28 | 29 | 30 | mapreduce.jobhistory.address 31 | hadoop-master:10020 32 | 33 | 34 | mapreduce.jobhistory.webapp.address 35 | hadoop-master:19888 36 | 37 | 38 | 39 | To set the value of tmp directory for map and reduce tasks. 40 | mapreduce.task.tmp.dir 41 | /var/lib/hadoop-mapreduce/cache/${user.name}/tasks 42 | 43 | 44 | 45 | 46 | mapreduce.jobhistory.keytab 47 | /etc/hadoop/conf/mapred.keytab 48 | 49 | 50 | mapreduce.jobhistory.principal 51 | mapred/hadoop-master@LABS.TERADATA.COM 52 | 53 | 54 | 55 | 56 | mapreduce.jobtracker.kerberos.principal 57 | mapred/hadoop-master@LABS.TERADATA.COM 58 | 59 | 60 | 61 | mapreduce.jobtracker.keytab.file 62 | /etc/hadoop/conf/mapred.keytab 63 | 64 | 65 | 66 | 67 | mapreduce.tasktracker.kerberos.principal 68 | mapred/hadoop-master@LABS.TERADATA.COM 69 | 70 | 71 | mapreduce.tasktracker.keytab.file 72 | /etc/hadoop/conf/mapred.keytab 73 | 74 | 75 | 76 | 77 | mapred.task.tracker.task-controller 78 | org.apache.hadoop.mapred.LinuxTaskController 79 | 80 | 81 | 82 | mapreduce.tasktracker.group 83 | mapred 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-master/files/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 1 format namenode 4 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 5 | 6 | # workaround for 'could not open session' bug as suggested here: 7 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 8 | rm -f /etc/security/limits.d/hdfs.conf 9 | su -c "echo 'N' | hdfs namenode -format" hdfs 10 | 11 | # 2 start hdfs 12 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 13 | 14 | # 3 wait for process starting 15 | sleep 10 16 | 17 | # 4 init hdfs directories (subset of stuff from /usr/lib/hadoop/libexec/init-hdfs.sh) 18 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /tmp' 19 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp' 20 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /var' 21 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /var/log' 22 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1775 /var/log' 23 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown yarn:mapred /var/log' 24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /tmp/hadoop-yarn' 25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn' 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /tmp/hadoop-yarn/staging/history/done_intermediate' 27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn/staging' 28 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp' 29 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /var/log/hadoop/yarn/apps' 30 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /var/log/hadoop/yarn/apps' 31 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown yarn:mapred /var/log/hadoop/yarn/apps' 32 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /user' 33 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /user/history' 34 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown mapred /user/history' 35 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /user/hive' 36 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 777 /user/hive' 37 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive' 38 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /user/root' 39 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 777 /user/root' 40 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown root /user/root' 41 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /var/lib/hadoop-hdfs/cache/mapred/mapred/staging' 42 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /var/lib/hadoop-hdfs/cache/mapred/mapred/staging' 43 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown -R mapred /var/lib/hadoop-hdfs/cache/mapred' 44 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 45 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 1777 /user/hive/warehouse' 46 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 47 | 48 | # 6 stop hdfs 49 | killall java 50 | 51 | # 7 setup metastore 52 | mysql_install_db 53 | 54 | /usr/bin/mysqld_safe & 55 | sleep 10s 56 | 57 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 58 | echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql 59 | /usr/bin/mysqladmin -u root password 'root' 60 | 61 | killall mysqld 62 | sleep 10s 63 | mkdir /var/log/mysql/ 64 | chown mysql:mysql /var/log/mysql 65 | 66 | # 8 Init zookeeper 67 | /etc/init.d/zookeeper-server init 68 | -------------------------------------------------------------------------------- /teradatalabs/hdp2.5-master/files/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | # 1 format namenode 6 | chown hdfs:hdfs /var/lib/hadoop-hdfs/cache/ 7 | 8 | # workaround for 'could not open session' bug as suggested here: 9 | # https://github.com/docker/docker/issues/7056#issuecomment-49371610 10 | rm -f /etc/security/limits.d/hdfs.conf 11 | su -c "echo 'N' | hdfs namenode -format" hdfs 12 | 13 | # 2 start hdfs 14 | su -c "hdfs namenode 2>&1 > /var/log/hadoop/hdfs/hadoop-hdfs-namenode.log" hdfs& 15 | 16 | # 3 wait for process starting 17 | sleep 10 18 | 19 | # 4 init basic hdfs directories 20 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp' 21 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /tmp' 22 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /var' 23 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /var/log' 24 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1775 /var/log' 25 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown yarn:mapred /var/log' 26 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /tmp/hadoop-yarn' 27 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn' 28 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 777 /tmp/hadoop-yarn' 29 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir -p /var/log/hadoop/yarn/apps' 30 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 1777 /var/log/hadoop/yarn/apps' 31 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown yarn:mapred /var/log/hadoop/yarn/apps' 32 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user' 33 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 755 /user' 34 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hdfs /user' 35 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/history' 36 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown mapred:mapred /user/history' 37 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod 755 /user/history' 38 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive' 39 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 777 /user/hive' 40 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive' 41 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/root' 42 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod -R 777 /user/root' 43 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown root /user/root' 44 | 45 | # 4.1 Create an hdfs home directory for the yarn user. For some reason, init-hdfs doesn't do so. 46 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/yarn && /usr/bin/hadoop fs -chown yarn:yarn /user/yarn' 47 | 48 | # 5 init hive directories 49 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -mkdir /user/hive/warehouse' 50 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chmod g+w /user/hive/warehouse' 51 | su -s /bin/bash hdfs -c '/usr/bin/hadoop fs -chown hive /user/hive/warehouse' 52 | 53 | # 6 stop hdfs 54 | killall java 55 | 56 | # 7 setup metastore 57 | mysql_install_db 58 | 59 | /usr/bin/mysqld_safe & 60 | sleep 10s 61 | 62 | cp /usr/hdp/2.5.*/hive/scripts/metastore/upgrade/mysql/hive-schema-0.14.0.mysql.sql /tmp/hive-schema-0.14.0.mysql.sql 63 | echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql 64 | echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/hdp/2.5.0.0-1245/hive/scripts/metastore/upgrade/mysql/hive-schema-0.14.0.mysql.sql;" | mysql 65 | /usr/bin/mysqladmin -u root password 'root' 66 | rm -f /tmp/hive-schema-0.14.0.mysql.sql 67 | 68 | killall mysqld 69 | sleep 10s 70 | mkdir /var/log/mysql/ 71 | chown mysql:mysql /var/log/mysql/ 72 | 73 | exit 0 74 | -------------------------------------------------------------------------------- /commons/conf/capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | yarn.scheduler.capacity.maximum-applications 19 | 10000 20 | 21 | Maximum number of applications that can be pending and running. 22 | 23 | 24 | 25 | 26 | yarn.scheduler.capacity.maximum-am-resource-percent 27 | 0.5 28 | 29 | Maximum percent of resources in the cluster which can be used to run 30 | application masters i.e. controls number of concurrent running 31 | applications. 32 | 33 | 34 | 35 | 36 | yarn.scheduler.capacity.root.queues 37 | default 38 | 39 | The queues at the this level (root is the root queue). 40 | 41 | 42 | 43 | 44 | yarn.scheduler.capacity.root.default.capacity 45 | 100 46 | Default queue target capacity. 47 | 48 | 49 | 50 | yarn.scheduler.capacity.root.default.maximum-capacity 51 | 100 52 | 53 | The maximum capacity of the default queue. 54 | 55 | 56 | 57 | 58 | yarn.scheduler.capacity.root.default.state 59 | RUNNING 60 | 61 | The state of the default queue. State can be one of RUNNING or STOPPED. 62 | 63 | 64 | 65 | 66 | yarn.scheduler.capacity.root.default.acl_submit_applications 67 | * 68 | 69 | The ACL of who can submit jobs to the default queue. 70 | 71 | 72 | 73 | 74 | yarn.scheduler.capacity.root.default.user-limit-factor 75 | 1 76 | 77 | Default queue user limit a percentage from 0.0 to 1.0. 78 | 79 | 80 | 81 | 82 | yarn.scheduler.capacity.root.default.acl_administer_queue 83 | * 84 | 85 | The ACL of who can administer jobs on the default queue. 86 | 87 | 88 | 89 | 90 | yarn.scheduler.capacity.node-locality-delay 91 | -1 92 | 93 | Number of missed scheduling opportunities after which the CapacityScheduler 94 | attempts to schedule rack-local containers. 95 | Typically this should be set to number of racks in the cluster, this 96 | feature is disabled by default, set to -1. 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /teradatalabs/centos6-java8-oracle-ldap/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/centos6-java8-oracle:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | ARG JDK_PATH 19 | 20 | ENV LDAP_PATH=/etc/ldap JAVA_HOME=$JDK_PATH 21 | 22 | # Import the Active Directory certificate into presto-coordinator 23 | COPY files $LDAP_PATH/files 24 | RUN keytool -import -alias presto -storepass changeit -keystore $JAVA_HOME/jre/lib/security/cacerts \ 25 | -noprompt -trustcacerts -file $LDAP_PATH/files/active-directory-certificate.crt && \ 26 | rm -rf $LDAP_PATH/files/active-directory-certificate.crt 27 | 28 | # Import the Open LDAP certificate into presto-coordinator 29 | RUN keytool -import -alias prestowithopenldap -storepass changeit -keystore $JAVA_HOME/jre/lib/security/cacerts \ 30 | -noprompt -trustcacerts -file $LDAP_PATH/files/openldap-certificate.pem && \ 31 | rm -rf $LDAP_PATH/files/openldap-certificate.pem 32 | 33 | # Generate a keystore. 34 | RUN keytool -genkey -alias coordinator -storepass testldap -keystore $LDAP_PATH/coordinator.jks \ 35 | -keypass testldap -keyalg RSA -sigalg SHA1withRSA -dname "CN=presto-master, OU=, O=, L=, S=, C=" 36 | # Generate a certificate and CSR for the keystore. CN should match the hostname of presto-coordinator 37 | RUN keytool -export -alias coordinator -storepass testldap -keystore $LDAP_PATH/coordinator.jks \ 38 | -file $LDAP_PATH/coordinator.cer && \ 39 | keytool -certreq -alias coordinator -storepass testldap -keystore $LDAP_PATH/coordinator.jks \ 40 | -file $LDAP_PATH/coordinator.csr 41 | 42 | # create a test CA and generate caroot.cer( root certificate of the CA ). 43 | RUN openssl req -new -keyout $LDAP_PATH/cakey.pem -out $LDAP_PATH/careq.pem -nodes \ 44 | -subj "/C=US/ST=Massachusetts/L=Boston/O=Teradata/OU=Finance/CN=teradata" && \ 45 | openssl x509 -req -in $LDAP_PATH/careq.pem -out $LDAP_PATH/caroot.cer -days 3650 \ 46 | -signkey $LDAP_PATH/cakey.pem 47 | 48 | # sign the server certificate using the testCA 49 | RUN openssl x509 -req -in $LDAP_PATH/coordinator.csr -out $LDAP_PATH/TestCA.cer -days 365 \ 50 | -CA $LDAP_PATH/caroot.cer -CAkey $LDAP_PATH/cakey.pem -CAserial $LDAP_PATH/files/serial.txt 51 | 52 | # Import the caroot.cer to the keystore and replace the previously generated self-signed cert with the new CA signed one in the keystore 53 | # Note: The -alias name (coordinator) should be the same as the alias used when coordinator.cer was generated 54 | RUN keytool -import -alias TestCA -storepass testldap -keystore $LDAP_PATH/coordinator.jks \ 55 | -noprompt -file $LDAP_PATH/caroot.cer && \ 56 | keytool -import -alias coordinator -storepass testldap -keystore $LDAP_PATH/coordinator.jks \ 57 | -file $LDAP_PATH/TestCA.cer 58 | 59 | # Import the root certificate of the issuer of the presto-coordinator certificate. 60 | # This would be used by clients ( cli, jdbc ) to run tests 61 | RUN keytool -import -alias caroot -storepass testldap -keystore $LDAP_PATH/cacerts.jks -noprompt \ 62 | -file $LDAP_PATH/caroot.cer 63 | -------------------------------------------------------------------------------- /depend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "$0 {-d|-g|-p {tag}|-x} {target image Dockerfile} [known image tags]" >&2 5 | } 6 | 7 | find_parent() { 8 | awk ' 9 | BEGIN { 10 | ec = 1; 11 | } 12 | 13 | $1 == "FROM" && parent { 14 | ec = 2; 15 | exit; 16 | } 17 | 18 | $1 == "FROM" { 19 | split($0, a); 20 | parent = $2; 21 | ec = 0 22 | print parent; 23 | } 24 | 25 | END { 26 | exit ec 27 | }' "$1" 28 | } 29 | 30 | contains() { 31 | needle=$1 32 | shift 33 | echo "$@" | grep -q -E "\<$needle\>" 34 | } 35 | 36 | # 37 | # You cannot use a colon (:) in a target or prerequisite name in a 38 | # Makefile. Change colons to @-signs, which docker does not allow as part 39 | # of an name or tag. The Makefile will have to translate them back before 40 | # invoking `docker pull' to fetch the base images. 41 | # 42 | make_friendly_name() { 43 | echo "$1" | sed 's/:/@/g' 44 | } 45 | 46 | untag() { 47 | echo "${1%:*}" 48 | } 49 | 50 | noop() { 51 | : 52 | } 53 | 54 | depfiles_own_image() { 55 | local target_image=$1 56 | local make_friendly_parent=$(make_friendly_name "$2") 57 | local untagged_parent=$(untag "$2") 58 | 59 | echo "$target_image@latest: $make_friendly_parent" 60 | echo ".PHONY: $target_image.dependants $untagged_parent.dependants" 61 | echo "$untagged_parent.dependants: $target_image" 62 | echo "$untagged_parent.dependants: $target_image.dependants" 63 | } 64 | 65 | depfiles_ext_image() { 66 | local target_image="$1" 67 | local make_friendly_parent=$(make_friendly_name "$2") 68 | 69 | echo "$target_image@latest: $make_friendly_parent" 70 | } 71 | 72 | list_ext_image() { 73 | local make_friendly_parent=$(make_friendly_name "$2") 74 | echo "$make_friendly_parent" 75 | } 76 | 77 | graph_own_image() { 78 | local untagged_parent=$(untag "$2") 79 | cat <<-EOF 80 | "$1" [shape=box] 81 | "$untagged_parent" [shape=box] 82 | "$1" -> "$untagged_parent" 83 | 84 | EOF 85 | } 86 | 87 | graph_ext_image() { 88 | cat <<-EOF 89 | "$1" [shape=box] 90 | "$2" [shape=house; style=filled; fillcolor="#a0a0a0"] 91 | "$1" -> "$2" 92 | EOF 93 | } 94 | 95 | require_parent_tag() { 96 | local target_image=$1 97 | local parent_image=$2 98 | 99 | if ! echo "$parent_image" | grep ":${required_parent_tag}\$"; then 100 | echo "FROM in Dockerfile for $target_image must specify a parent with the tag '$required_parent_tag'" >&2 101 | exit 1 102 | fi 103 | } 104 | 105 | while getopts ":dgp:x" c; do 106 | case $c in 107 | d) 108 | own_image_function=depfiles_own_image 109 | ext_image_function=depfiles_ext_image 110 | ;; 111 | x) 112 | own_image_function=noop 113 | ext_image_function=list_ext_image 114 | ;; 115 | g) 116 | own_image_function=graph_own_image 117 | ext_image_function=graph_ext_image 118 | ;; 119 | p) 120 | own_image_function=require_parent_tag 121 | required_parent_tag=$OPTARG 122 | ext_image_function=noop 123 | ;; 124 | \?) 125 | echo "Unrecognized option -$OPTARG" >&2 126 | exit 1 127 | ;; 128 | :) 129 | echo "Option -$OPTARG requires an argument" >&2 130 | exit 1 131 | ;; 132 | esac 133 | done 134 | 135 | shift "$(dc -e"$OPTIND 1 - p")" 136 | 137 | if [ -z "$own_image_function" ] || [ $# -lt 2 ]; then 138 | usage 139 | exit 1 140 | fi 141 | 142 | target_dockerfile=$1 143 | target_image=$(dirname "$target_dockerfile") 144 | shift 145 | known_images="$*" 146 | 147 | parent_image_tag=$(find_parent "$target_dockerfile") 148 | ec=$? 149 | case $ec in 150 | 0) ;; 151 | 1) 152 | echo "Failed to find a parent docker image in $target_dockerfile" >&2 153 | exit $ec 154 | ;; 155 | 2) 156 | echo "Found multiple parent docker images in $target_dockerfile" >&2 157 | exit $ec 158 | ;; 159 | esac 160 | 161 | if contains "$parent_image_tag" "$known_images"; then 162 | $own_image_function "$target_image" "$parent_image_tag" 163 | else 164 | $ext_image_function "$target_image" "$parent_image_tag" 165 | fi 166 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | javax.jdo.option.ConnectionURL 22 | jdbc:mysql://localhost/metastore 23 | the URL of the MySQL database 24 | 25 | 26 | 27 | javax.jdo.option.ConnectionDriverName 28 | com.mysql.jdbc.Driver 29 | 30 | 31 | 32 | javax.jdo.option.ConnectionUserName 33 | root 34 | 35 | 36 | 37 | javax.jdo.option.ConnectionPassword 38 | root 39 | 40 | 41 | 42 | datanucleus.autoCreateSchema 43 | false 44 | 45 | 46 | 47 | datanucleus.fixedDatastore 48 | true 49 | 50 | 51 | 52 | datanucleus.autoStartMechanism 53 | SchemaTable 54 | 55 | 56 | 57 | hive.server2.authentication 58 | KERBEROS 59 | 60 | 61 | 62 | hive.server2.enable.impersonation 63 | true 64 | 65 | 66 | 67 | hive.server2.authentication.kerberos.principal 68 | hive/hadoop-master@LABS.TERADATA.COM 69 | 70 | 71 | 72 | hive.server2.authentication.kerberos.keytab 73 | /etc/hive/conf/hive.keytab 74 | 75 | 76 | 77 | hive.metastore.sasl.enabled 78 | true 79 | 80 | 81 | 82 | hive.metastore.kerberos.keytab.file 83 | /etc/hive/conf/hive.keytab 84 | 85 | 86 | 87 | hive.metastore.kerberos.principal 88 | hive/hadoop-master@LABS.TERADATA.COM 89 | 90 | 91 | 92 | hive.security.authorization.createtable.owner.grants 93 | ALL 94 | The set of privileges automatically granted to the owner whenever a table gets created. 95 | 96 | 97 | 98 | hive.users.in.admin.role 99 | hdfs,hive 100 | 101 | 102 | 103 | hive.security.authorization.manager 104 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory 105 | 106 | 107 | 108 | hive.security.authorization.task.factory 109 | org.apache.hadoop.hive.ql.parse.authorization.HiveAuthorizationTaskFactoryImpl 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | yarn.nodemanager.aux-services 22 | mapreduce_shuffle 23 | 24 | 25 | 26 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 27 | org.apache.hadoop.mapred.ShuffleHandler 28 | 29 | 30 | 31 | yarn.log-aggregation-enable 32 | true 33 | 34 | 35 | 36 | yarn.dispatcher.exit-on-error 37 | true 38 | 39 | 40 | 41 | List of directories to store localized files in. 42 | yarn.nodemanager.local-dirs 43 | /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir 44 | 45 | 46 | 47 | Where to store container logs. 48 | yarn.nodemanager.log-dirs 49 | /var/log/hadoop/yarn/containers 50 | 51 | 52 | 53 | Where to aggregate logs to. 54 | yarn.nodemanager.remote-app-log-dir 55 | /var/log/hadoop/yarn/apps 56 | 57 | 58 | 59 | Classpath for typical applications. 60 | yarn.application.classpath 61 | 62 | /etc/hadoop/conf, 63 | /usr/lib/hadoop/*, 64 | /usr/lib/hadoop/lib/*, 65 | /usr/lib/hadoop-hdfs/*, 66 | /usr/lib/hadoop-hdfs/lib/*, 67 | /usr/lib/hadoop-yarn/*, 68 | /usr/lib/hadoop-yarn/lib/*, 69 | /usr/lib/hadoop-mapreduce/*, 70 | /usr/lib/hadoop-mapreduce/lib/* 71 | 72 | 73 | 74 | 75 | 76 | yarn.resourcemanager.keytab 77 | /etc/hadoop/conf/yarn.keytab 78 | 79 | 80 | yarn.resourcemanager.principal 81 | yarn/hadoop-master@LABS.TERADATA.COM 82 | 83 | 84 | 85 | 86 | yarn.nodemanager.keytab 87 | /etc/hadoop/conf/yarn.keytab 88 | 89 | 90 | 91 | yarn.nodemanager.principal 92 | yarn/hadoop-master@LABS.TERADATA.COM 93 | 94 | 95 | 96 | yarn.nodemanager.container-executor.class 97 | org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor 98 | 99 | 100 | 101 | yarn.nodemanager.linux-container-executor.path 102 | /usr/lib/hadoop-yarn/bin/container-executor 103 | 104 | 105 | 106 | yarn.nodemanager.linux-container-executor.group 107 | yarn 108 | 109 | 110 | 111 | yarn.resourcemanager.hostname 112 | hadoop-master 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /teradatalabs/centos6-ssh-oj8/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM jdeathe/centos-ssh:centos-6-1.8.1 16 | MAINTAINER Teradata Docker Team 17 | 18 | ENV DOCKERIZE_VERSION v0.3.0 19 | 20 | ARG JDK_URL 21 | ARG JDK_RPM 22 | ARG JDK_PATH 23 | 24 | # Install Oracle Java and presto-admin dependences (already has python 2.6) 25 | RUN \ 26 | # disable the notoriuosly unstable EPEL repo... 27 | rm -rf /etc/yum.repos.d/epel* && \ 28 | 29 | yum install -y \ 30 | wget \ 31 | bzip2 \ 32 | gcc \ 33 | python-devel \ 34 | tar \ 35 | expect \ 36 | python-devel \ 37 | openssl-devel \ 38 | libffi-devel && \ 39 | # install and trim Oracle JDK 40 | wget -nv --header "Cookie: oraclelicense=accept-securebackup-cookie" $JDK_URL && \ 41 | rpm -ivh $JDK_RPM && rm $JDK_RPM && \ 42 | rm -rf $JDK_PATH/*src.zip \ 43 | $JDK_PATH/lib/missioncontrol \ 44 | $JDK_PATH/lib/visualvm \ 45 | $JDK_PATH/lib/*javafx* \ 46 | $JDK_PATH/jre/lib/plugin.jar \ 47 | $JDK_PATH/jre/lib/ext/jfxrt.jar \ 48 | $JDK_PATH/jre/bin/javaws \ 49 | $JDK_PATH/jre/lib/javaws.jar \ 50 | $JDK_PATH/jre/lib/desktop \ 51 | $JDK_PATH/jre/plugin \ 52 | $JDK_PATH/jre/lib/deploy* \ 53 | $JDK_PATH/jre/lib/*javafx* \ 54 | $JDK_PATH/jre/lib/*jfx* \ 55 | $JDK_PATH/jre/lib/amd64/libdecora_sse.so \ 56 | $JDK_PATH/jre/lib/amd64/libprism_*.so \ 57 | $JDK_PATH/jre/lib/amd64/libfxplugins.so \ 58 | $JDK_PATH/jre/lib/amd64/libglass.so \ 59 | $JDK_PATH/jre/lib/amd64/libgstreamer-lite.so \ 60 | $JDK_PATH/jre/lib/amd64/libjavafx*.so \ 61 | $JDK_PATH/jre/lib/amd64/libjfx*.so && \ 62 | # install dockerize 63 | wget -nv https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 64 | && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 65 | && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ 66 | # install pip 67 | easy_install pip && \ 68 | # cleanup 69 | yum -y clean all && rm -rf /tmp/* /var/tmp/* 70 | 71 | # Copy Vagrant insecure SSH keys for passwordless SSH between 72 | # containers and to localhost 73 | COPY vagrant_insecure_rsa /etc/services-config/ssh/id_rsa 74 | 75 | # Modify the ssh-bootstrap script to copy id_rsa to ~/.ssh (the script already makes ~/.ssh 76 | # and copies authorized_keys) 77 | RUN printf "cp -f /etc/services-config/ssh/id_rsa \${OPTS_SSH_USER_HOME}/.ssh/id_rsa\n \ 78 | chown -R \${OPTS_SSH_USER}:${OPTS_SSH_USER} \${OPTS_SSH_USER_HOME}/.ssh/id_rsa\n \ 79 | chmod 600 \${OPTS_SSH_USER_HOME}/.ssh/id_rsa\n \ 80 | cp -r \${OPTS_SSH_USER_HOME}/.ssh /root/.ssh\n\n \ 81 | useradd -ms /bin/bash testuser\n \ 82 | echo "testuser:testpass" | chpasswd" \ 83 | > /usr/sbin/bootstrap-extra 84 | 85 | RUN awk -v file="/usr/sbin/bootstrap-extra" -v lineno=$(($(wc -l < /usr/sbin/sshd-bootstrap) - 4)) ' \ 86 | NR==lineno {system("echo ""; cat " file); print; next} \ 87 | 1' /usr/sbin/sshd-bootstrap > /usr/sbin/sshd-bootstrap2 && \ 88 | mv /usr/sbin/sshd-bootstrap2 /usr/sbin/sshd-bootstrap && \ 89 | chmod +x /usr/sbin/sshd-bootstrap 90 | 91 | # 92 | # The sshd-bootstrap process engages in some tomfoolery with the 93 | # /etc/ssh/sshd_config file during the bootstrap process. Namely, it forcibly 94 | # symlinks /etc/services-config/ssh/sshd_config over it under conditions that 95 | # prevail in our images. This means the canonical source of the sshd 96 | # configuration is the file in /etc/services-config, and we need to modify it, 97 | # rather than the customary file in /etc/ssh. 98 | # 99 | RUN sed -i \ 100 | -e 's/^PermitRootLogin no/PermitRootLogin without-password/g' \ 101 | -e 's/^PasswordAuthentication no/PasswordAuthentication yes/g' \ 102 | /etc/services-config/ssh/sshd_config 103 | 104 | # Set default password for testing 105 | RUN printf "\nSSH_USER_PASSWORD=password" >> /etc/services-config/ssh/sshd-bootstrap.conf 106 | 107 | EXPOSE 22 8080 108 | 109 | ENV JAVA_HOME $JDK_PATH/jre/ 110 | 111 | # Clean up 112 | RUN yum -y clean all && rm -rf /tmp/* /var/tmp/* 113 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Teradata 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM teradatalabs/cdh5-hive:unlabelled 16 | MAINTAINER Teradata Docker Team 17 | 18 | # INSTALL KERBEROS 19 | RUN yum install -y krb5-libs krb5-server krb5-workstation 20 | 21 | # ADD KERBEROS CONFIGURATION 22 | ADD files/kerberos/krb5.conf /etc/krb5.conf 23 | ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf 24 | ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl 25 | 26 | # CREATE KERBEROS DATABASE 27 | RUN /usr/sbin/kdb5_util create -s -P password 28 | 29 | # ADD SUPERVISORD BOOTSTRAP KERBEROS SCRIPT 30 | ADD files/bootstrap-kerberos.sh /root/bootstrap-kerberos.sh 31 | ADD files/supervisord.d/bootstrap-kerberos.conf /etc/supervisord.d/bootstrap-kerberos.conf 32 | 33 | # ADD HADOOP PRINCIPALS 34 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey hdfs/hadoop-master@LABS.TERADATA.COM" 35 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey mapred/hadoop-master@LABS.TERADATA.COM" 36 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey yarn/hadoop-master@LABS.TERADATA.COM" 37 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey HTTP/hadoop-master@LABS.TERADATA.COM" 38 | 39 | # CREATE HADOOP KEYTAB FILES 40 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/hadoop/conf/hdfs.keytab hdfs/hadoop-master HTTP/hadoop-master" 41 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/hadoop/conf/mapred.keytab mapred/hadoop-master HTTP/hadoop-master" 42 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/hadoop/conf/yarn.keytab yarn/hadoop-master HTTP/hadoop-master" 43 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/hadoop/conf/HTTP.keytab HTTP/hadoop-master" 44 | RUN chown hdfs:hadoop /etc/hadoop/conf/hdfs.keytab 45 | RUN chown mapred:hadoop /etc/hadoop/conf/mapred.keytab 46 | RUN chown yarn:hadoop /etc/hadoop/conf/yarn.keytab 47 | RUN chown hdfs:hadoop /etc/hadoop/conf/HTTP.keytab 48 | RUN chmod 644 /etc/hadoop/conf/*.keytab 49 | 50 | # ENABLE HADOOP SECURITY 51 | ADD files/conf/core-site.xml /etc/hadoop/conf/core-site.xml 52 | ADD files/conf/hdfs-site.xml /etc/hadoop/conf/hdfs-site.xml 53 | 54 | # ENABLE MAPRED SECURITY 55 | ADD files/conf/mapred-site.xml /etc/hadoop/conf/mapred-site.xml 56 | ADD files/conf/taskcontroller.cfg /etc/hadoop/conf/taskcontroller.cfg 57 | 58 | # ENABLE YARN SECURITY 59 | ADD files/conf/yarn-site.xml /etc/hadoop/conf/yarn-site.xml 60 | ADD files/conf/container-executor.cfg /etc/hadoop/conf/container-executor.cfg 61 | RUN chmod 6050 /etc/hadoop/conf/container-executor.cfg 62 | 63 | # CREATE HIVE PRINCIPAL AND KEYTAB 64 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey hive/hadoop-master@LABS.TERADATA.COM" 65 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/hive/conf/hive.keytab hive/hadoop-master" 66 | 67 | # ENABLE HIVE SECURITY 68 | ADD files/conf/hive-site.xml /etc/hive/conf/hive-site.xml 69 | RUN chown hive:hadoop /etc/hive/conf/hive.keytab 70 | RUN chmod 644 /etc/hive/conf/hive.keytab 71 | 72 | # ENABLE AUTHORIZATION IN HIVE SERVER 73 | ADD files/conf/hiveserver2-site.xml /etc/hive/conf/hiveserver2-site.xml 74 | 75 | # CREATE PRESTO PRINCIPAL AND KEYTAB 76 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey presto-server/presto-master.docker.cluster@LABS.TERADATA.COM" 77 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey presto-server/presto-worker.docker.cluster@LABS.TERADATA.COM" 78 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey presto-server/presto-worker-1.docker.cluster@LABS.TERADATA.COM" 79 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey presto-server/presto-worker-2.docker.cluster@LABS.TERADATA.COM" 80 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey HTTP/presto-master.docker.cluster@LABS.TERADATA.COM" 81 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey presto-client/presto-master.docker.cluster@LABS.TERADATA.COM" 82 | RUN /usr/sbin/kadmin.local -q "addprinc -randkey hive/presto-master.docker.cluster@LABS.TERADATA.COM" 83 | RUN mkdir -p /etc/presto/conf 84 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-server.keytab presto-server/presto-master.docker.cluster presto-server/presto-worker.docker.cluster presto-server/presto-worker-1.docker.cluster presto-server/presto-worker-2.docker.cluster" 85 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-server-HTTP.keytab HTTP/presto-master.docker.cluster" 86 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-client.keytab presto-client/presto-master.docker.cluster" 87 | RUN /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/hive-presto-master.keytab hive/presto-master.docker.cluster" 88 | RUN chmod 644 /etc/presto/conf/*.keytab 89 | 90 | # CREATE SSL KEYSTORE 91 | RUN keytool -genkeypair \ 92 | -alias presto \ 93 | -keyalg RSA \ 94 | -keystore /etc/presto/conf/keystore.jks \ 95 | -keypass password \ 96 | -storepass password \ 97 | -dname "CN=presto-master, OU=, O=, L=, S=, C=" 98 | RUN chmod 644 /etc/presto/conf/keystore.jks 99 | 100 | # EXPOSE KERBEROS PORTS 101 | EXPOSE 88 102 | EXPOSE 749 103 | 104 | CMD /root/startup.sh 105 | -------------------------------------------------------------------------------- /teradatalabs/cdh5-hive-kerberized/files/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | dfs.replication 22 | 1 23 | 24 | 25 | 27 | 28 | dfs.safemode.extension 29 | 0 30 | 31 | 32 | 33 | dfs.safemode.min.datanodes 34 | 1 35 | 36 | 37 | 38 | hadoop.tmp.dir 39 | /var/lib/hadoop-hdfs/cache/${user.name} 40 | 41 | 42 | 43 | dfs.namenode.name.dir 44 | file:///var/lib/hadoop-hdfs/cache/${user.name}/dfs/name 45 | 46 | 47 | 48 | dfs.namenode.checkpoint.dir 49 | file:///var/lib/hadoop-hdfs/cache/${user.name}/dfs/namesecondary 50 | 51 | 52 | 53 | dfs.datanode.data.dir 54 | file:///var/lib/hadoop-hdfs/cache/${user.name}/dfs/data 55 | 56 | 57 | 58 | dfs.client.read.shortcircuit 59 | true 60 | 61 | 62 | 63 | dfs.client.file-block-storage-locations.timeout.millis 64 | 10000 65 | 66 | 67 | 68 | dfs.domain.socket.path 69 | /var/run/hadoop-hdfs/dn._PORT 70 | 71 | 72 | 73 | dfs.datanode.hdfs-blocks-metadata.enabled 74 | true 75 | 76 | 77 | 78 | 79 | dfs.block.access.token.enable 80 | true 81 | 82 | 83 | 84 | 85 | dfs.namenode.keytab.file 86 | /etc/hadoop/conf/hdfs.keytab 87 | 88 | 89 | dfs.namenode.kerberos.principal 90 | hdfs/hadoop-master@LABS.TERADATA.COM 91 | 92 | 93 | dfs.namenode.kerberos.internal.spnego.principal 94 | HTTP/hadoop-master@LABS.TERADATA.COM 95 | 96 | 97 | 98 | 99 | dfs.secondary.namenode.keytab.file 100 | /etc/hadoop/conf/hdfs.keytab 101 | 102 | 103 | dfs.secondary.namenode.kerberos.principal 104 | hdfs/hadoop-master@LABS.TERADATA.COM 105 | 106 | 107 | dfs.secondary.namenode.kerberos.internal.spnego.principal 108 | HTTP/hadoop-master@LABS.TERADATA.COM 109 | 110 | 111 | 112 | 113 | dfs.datanode.keytab.file 114 | /etc/hadoop/conf/hdfs.keytab 115 | 116 | 117 | dfs.datanode.kerberos.principal 118 | hdfs/hadoop-master@LABS.TERADATA.COM 119 | 120 | 121 | 122 | 123 | dfs.webhdfs.enabled 124 | true 125 | 126 | 127 | 128 | 129 | dfs.web.authentication.kerberos.principal 130 | HTTP/hadoop-master@LABS.TERADATA.COM 131 | 132 | 133 | 134 | dfs.web.authentication.kerberos.keytab 135 | /etc/hadoop/conf/HTTP.keytab 136 | 137 | 138 | 139 | ignore.secure.ports.for.testing 140 | true 141 | 142 | 143 | 144 | dfs.http.policy 145 | HTTP_ONLY 146 | 147 | 148 | 149 | 150 | hadoop.proxyuser.hdfs.groups 151 | * 152 | 153 | 154 | 155 | hadoop.proxyuser.hdfs.hosts 156 | * 157 | 158 | 159 | 160 | hadoop.proxyuser.presto-server.groups 161 | * 162 | 163 | 164 | 165 | hadoop.proxyuser.presto-server.hosts 166 | * 167 | 168 | 169 | 170 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Teradata docker images 2 | 3 | ## Docker Image Names 4 | 5 | The docker images in this repository are expected to be given names of the form 6 | teradatalabs/cdh5-hive. The Dockerfile and other files needed to build the 7 | teradatalabs/cdh5-hive image are located in the directory 8 | teradatalabs/cdh5-hive. 9 | 10 | Generally speaking, the images should *not* be built manually with docker 11 | build. 12 | 13 | ## Building docker images 14 | 15 | The docker images should be built using `make`. To build the docker image named 16 | `teradatalabs/cdh5-hive`, run `make teradatalabs/cdh5-hive`. Make will build 17 | the image and its dependencies in the correct order. 18 | 19 | If you are going to release an image, you should release it and all of its 20 | dependencies. Master and slave images should be built from the same chain of 21 | parent images. You can ensure that both are built from the same set of parent 22 | images by running e.g. `make teradatalabs/cdh5-hive-master 23 | terdatalabs/cdh5-hive-slave` 24 | 25 | If you want to build a base image and all the images depending on it, 26 | you can use the `*.dependants` targets. E.g. 27 | 28 | ``` 29 | make teradatalabs/cdh5-base.dependants 30 | ``` 31 | 32 | will build the `cdh5-base` and all the images depending on it (transitively). 33 | 34 | ## Releasing (pushing) docker image 35 | 36 | All of the docker images in the repository share the same version number. This 37 | is because most of the images depend on a parent image that is also in the 38 | repository (e.g. teradatalabs/hdp2.5-master is FROM teradatalabs/hdp2.5-base), 39 | or are meant to be used together in testing (teradatalabs/cdh5-hive-master and 40 | teradatalabs/cdh5-hive-slave). 41 | 42 | Having all of the images on the same version number make troubleshooting easy: 43 | Iff all of the docker images you are using have the same version number then 44 | they are in a consistent state. 45 | 46 | This means that we treat the repository as a single codebase that creates 47 | multiple artifacts (Docker images) that all need to be released together. The 48 | Makefile uses [docker-release](https://github.com/kokosing/docker-release) to 49 | automate this process and ensure that the images on dockerhub are in a 50 | consistent state provided all of the push operations run to completion. 51 | 52 | docker-release also handles tagging the images and repository appropriately so 53 | that you can easily find the Dockerfile used to create an image starting from 54 | just the tags on a Docker image. 55 | 56 | Best practice for publishing a snapshot or release version is to use the Jenkins job. Login to Jenkins and search for `docker-images`. If you must publish a new version manually, follow these steps: 57 | 58 | To release a snapshot version of the repository do the following 59 | 60 | 1. `docker login` 61 | 2. Verify in the Makefile that `VERSION` is set to something ending in -SNAPSHOT. 62 | 3. `make snapshot` 63 | 64 | To release a release (final) version of the repository do the following 65 | 66 | 1. `docker login` 67 | 2. Verify in the Makefile that `VERSION` is set to something *not* ending in -SNAPSHOT. 68 | 3. `make release` 69 | 70 | To release a snapshot or final version, you must log in to docker using the 71 | docker `login` command. 72 | 73 | ### Typical workflow 74 | 75 | Normally developers are working on a snapshot version of the next release, and 76 | the `VERSION` macro in the Makefile should be set to a snapshot version such as 77 | 35-SNAPSHOT. A typical workflow is as follows: 78 | 79 | 1. Develop changes 80 | 2. Commit changes 81 | 3. `make snapshot` to push snapshot releases to dockerhub as needed 82 | 4. Repeat as needed 83 | 84 | Eventually, version 35-SNAPSHOT is ready for release. To release version 35, do 85 | the following: 86 | 87 | 1. Change `VERSION` to the release version: 35-SNAPSHOT -> 35 88 | 2. Commit the repository 89 | 3. `make release` to push the images to dockerhub and tag the repository 90 | 4. Change `VERSION` to the next snapshot version: 35 -> 36-SNAPSHOT 91 | 5. Commit the repository 92 | 6. Continue developing as described above 93 | 94 | `make snapshot` does the following: 95 | 96 | * Updates the 'latest' tag for the image on dockerhub 97 | * Creates a tag for the image with the git hash of the git repository on dockerhub 98 | 99 | `make release` does the following: 100 | 101 | * Updates the 'latest' tag for the image on dockerhub 102 | * Creates a tag for the image on dockerhub with the git hash of the git repository 103 | * Creates a tag for the image on dockerhub with the $(VERSION) specified in the Makefile 104 | * Creates a tag in the git repository with the name release-$(VERSION) 105 | 106 | `docker-release` enforces several rules about the state of the repository when pushing to dockerhub: 107 | 108 | * For a snapshot or a release, the repository must be in a clean state (no uncommitted files) 109 | * For a release, the branch must be master 110 | 111 | ## Upgrading Docker images for consumers 112 | 113 | For a project that uses Travis for continuous integration, you can upgrade the 114 | docker images used by the project using the following process. 115 | 116 | 1. Develop locally, testing your changes 117 | 2. When you are satisfied with your changes, run `make snapshot` to release a snapshot build to dockerhub. 118 | 3. Create a branch of the dependent project 119 | 4. Set the tag for the images on the project to the tag of the snapshot build 120 | 5. Push the branch to github. 121 | 6. Merge your changes into docker-images/master 122 | 7. Update the release version as described above, and run `make release` 123 | 8. Create a PR against upstream/master that uses the new release of the docker images. 124 | 9. After Travis passes, merge the PR 125 | 126 | ## Docker build arguments (ARG, --build-arg) 127 | 128 | Docker build arguments are documented in the [Dockerfile 129 | reference](https://docs.docker.com/engine/reference/builder/#/arg) 130 | 131 | Args are used by specifying the ARG directive in a Dockerfile: 132 | 133 | ``` 134 | ARG FOO 135 | RUN echo $FOO >/etc/foo 136 | ``` 137 | 138 | The value of FOO then needs to be set in the Makefile: 139 | 140 | ``` 141 | FOO := Docker images build on $(shell uname -s) are superior to all others. 142 | ``` 143 | 144 | Note that `docker build` does *not* allow the variable reference `$FOO` to be 145 | written `${FOO}` or `$(FOO)`. Further note that it won't warn you about this; 146 | instead, you'll likely end up with an error later in the build or a broken 147 | image. 148 | 149 | `docker build` won't let you pass `--build-arg`s that don't have a 150 | corresponding key in the Dockerfile. This means that the build system can't 151 | just pass the union of all of the `--build-arg`s needed by every Dockerfile in 152 | the repository. The build system handles this largely the same way it handles 153 | figuring out what the correct dependency order is for building the images, 154 | described below. 155 | 156 | Build args with a default value are not handled at present. Feel free to add 157 | that functionality in `flag.sh` if needed. 158 | 159 | ## Java 160 | 161 | Individual Dockerfiles shouldn't contain the URL for downloading Java, the name 162 | of the RPM, or the path that java gets installed in. Doing this makes upgrading 163 | Java across the repo a pain with a bunch of touch points. 164 | 165 | Instead, the build system exposes the [Docker build 166 | arguments](https://docs.docker.com/engine/reference/builder/#/arg) `JDK_URL`, 167 | `JDK_RPM`, and `JDK_PATH`. These can be used in your Dockerfile as follows: 168 | 169 | ``` 170 | ARG JDK_URL 171 | RUN wget $JDK_URL 172 | ``` 173 | 174 | ## How the build system works. 175 | 176 | At a high level, a docker image depends on two things: 177 | 178 | 1. Its Dockerfile 179 | 2. Its parent image, specified by the from FROM line in the Dockerfile. 180 | 181 | Using the relative directory from the root of the repo as the image name, we 182 | could, in principle, write a rule of the form 183 | 184 | ``` 185 | teradatalabs/foo: teradatalabs/foo/Dockerfile $(extract_parent teradatalabs/foo/Dockerfile) 186 | cd teradatalabs/foo && docker build -t teradatalabs/foo . 187 | ``` 188 | 189 | Using automatic variables we could shorten that to the following: 190 | 191 | ``` 192 | teradatalabs/foo: $@/Dockerfile $(extract_parent $@/Dockerfile) 193 | cd $@ && docker build -t $@ . 194 | ``` 195 | 196 | This is conceptually valid, but it doesn't work: Automatic variables aren't 197 | available in the prerequisites. The solution to solve that is to use a 198 | pattern rule: 199 | 200 | ``` 201 | $(images): %: %/Dockerfile $(extract_parent %/Dockerfile) 202 | ... 203 | ``` 204 | 205 | That almost works. Almost because you can't use the stem (%) in a [function 206 | call](https://www.gnu.org/savannah-checkouts/gnu/make/manual/html_node/Pattern-Rules.html). 207 | 208 | Instead, we can use three features of make together to accomplish the same thing. 209 | 210 | 1. You can specify the same target multiple times with different 211 | dependencies. Make will build all of the dependencies before running the 212 | commands to build the target. 213 | 2. you can use the include directive to tell make to include another file. 214 | 3. If a file specified by an include directive doesn't exist, make will 215 | look for a rule to create that file. 216 | 217 | ``` 218 | teradatalabs/foo: teradatalabs/foo_parent 219 | teradatalabs/foo: teradatalabs/foo/Dockerfile 220 | ... 221 | ``` 222 | 223 | The strategy is to include a separate file that specifies the dependency on 224 | the parent image. This file isn't in the repo, so the Makefile has a rule to 225 | make it from the image's Dockerfile. The second rule specifies the dependency 226 | on the Dockerfile and builds the image using docker build. 227 | 228 | [Recursive Make Considered Harmful](http://lcgapp.cern.ch/project/architecture/recursive_make.pdf) 229 | explains this technique in section 5.4 and applies it to C source files and the 230 | .h files they include. I've adapted it here. 231 | 232 | The depend.sh script generates a .d file in $(DEPDIR) from the Dockerfile for 233 | the image: 234 | 235 | ``` 236 | $(DEPDIR)/teradatalabs/foo.d: teradatalabs/foo/Dockerfile 237 | ... 238 | ``` 239 | 240 | The corresponding .d file will take one of two forms: 241 | 242 | 1. if foo's parent is built from this repository 243 | 244 | ``` 245 | teradatalabs/foo: teradatalabs/foo_parent 246 | ``` 247 | 248 | 2. if foo's parent should be pulled from dockerhub 249 | 250 | ``` 251 | teradatalabs/foo: 252 | ``` 253 | 254 | In the first case, make now knows that foo_parent is a dependency of foo, and 255 | builds it first. 256 | 257 | In the second case, we don't add a dependency for make, and docker itself is 258 | responsible for pulling foo's parent from dockerhub as part of the docker build 259 | process. 260 | 261 | A major difference between the approach explained in Recursive Make 262 | Considered Harmful is that depend.sh needs to know what images the repo knows 263 | how to build so it can output the second form for parent images we *don't* 264 | know how to build. We do this by passing in the names of all of the images we 265 | know how to build. 266 | --------------------------------------------------------------------------------