├── centos-base-ssh ├── .gitignore ├── supervisord-sshd.conf ├── Makefile ├── supervisord.conf └── Dockerfile ├── watchmen ├── .gitignore ├── Makefile ├── watchmen-config │ ├── storage.js │ ├── general.js │ └── hosts.js.template ├── supervisord-redis.conf ├── Dockerfile ├── supervisord-watchmen.conf └── redis.conf ├── accumulo ├── accumulo.conf │ ├── gc │ ├── masters │ ├── monitor │ ├── tracers │ ├── slaves │ ├── proxy.properties │ ├── auditLog.xml │ ├── log4j.properties │ ├── accumulo-metrics.xml │ ├── monitor_logger.properties │ ├── generic_logger.properties │ ├── monitor_logger.xml │ ├── accumulo-env.sh │ ├── generic_logger.xml │ ├── accumulo-site.xml │ └── accumulo.policy.example ├── accumulo_profile.sh ├── add_user.sh ├── dotssh │ ├── accumulo.pub │ ├── authorized_keys │ └── accumulo ├── init_accumulo.sh ├── Dockerfile ├── supervisor │ └── accumulo.conf ├── start.sh └── Makefile ├── dnsmasq ├── resolv.dnsmasq.conf ├── dnsmasq.hosts │ └── README ├── dnsmasq.conf ├── Makefile └── Dockerfile ├── consul-cluster ├── checkscripts │ └── sshd ├── services.json ├── consul.json ├── Dockerfile ├── README.md └── Makefile ├── doop ├── doop-cluster.png ├── dnsmasq.conf ├── README.md ├── check_hadoop.sh ├── bootstrap-node.conf ├── hadoop │ ├── dfs_data_dir_mount.hist │ ├── mapred-env.sh │ ├── hadoop-metrics2.properties │ ├── taskcontroller.cfg │ ├── commons-logging.properties │ ├── hadoop-policy.xml │ ├── health_check │ ├── core-site.xml │ ├── capacity-scheduler.xml │ ├── yarn-env.sh │ ├── task-log4j.properties │ ├── hdfs-site.xml │ ├── mapred-site.xml │ ├── hadoop-env.sh │ ├── log4j.properties │ └── yarn-site.xml ├── bootstrap-node.sh ├── supervisord.conf ├── Dockerfile ├── Makefile └── hadoop-group.conf ├── .gitignore ├── shared ├── config.zk │ ├── zookeeper-env.sh │ └── zoo.cfg ├── config.supervisor │ ├── dnsmasq.conf │ ├── zookeeper.conf │ ├── hdfs-datanode.conf │ ├── yarn-nodemanager.conf │ ├── yarn-resourcemanager.conf │ ├── yarn-jobhistory.conf │ └── hdfs-namenode.conf ├── Makefile.vars ├── insecure.pub ├── insecure.pem └── Makefile ├── apache-hadoop-build ├── Makefile ├── build_hadoop.sh └── Dockerfile ├── firefox ├── Makefile └── Dockerfile ├── nexus ├── nexus.properties ├── Makefile └── Dockerfile ├── zookeeper ├── Makefile └── Dockerfile ├── xmind ├── Makefile └── Dockerfile ├── elasticsearch ├── Makefile └── Dockerfile ├── minidoop ├── Dockerfile └── Makefile └── README.md /centos-base-ssh/.gitignore: -------------------------------------------------------------------------------- 1 | insecure.pub 2 | -------------------------------------------------------------------------------- /watchmen/.gitignore: -------------------------------------------------------------------------------- 1 | supervisord.conf 2 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/gc: -------------------------------------------------------------------------------- 1 | master.docker.local 2 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/masters: -------------------------------------------------------------------------------- 1 | master.docker.local 2 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/monitor: -------------------------------------------------------------------------------- 1 | master.docker.local 2 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/tracers: -------------------------------------------------------------------------------- 1 | master.docker.local 2 | -------------------------------------------------------------------------------- /dnsmasq/resolv.dnsmasq.conf: -------------------------------------------------------------------------------- 1 | nameserver 8.8.8.8 2 | 3 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/slaves: -------------------------------------------------------------------------------- 1 | datanode0.docker.local 2 | datanode1.docker.local 3 | -------------------------------------------------------------------------------- /consul-cluster/checkscripts/sshd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /sbin/pidof sshd 4 | exit $? 5 | -------------------------------------------------------------------------------- /doop/doop-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sroegner/docker-builds/HEAD/doop/doop-cluster.png -------------------------------------------------------------------------------- /doop/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | port=53 2 | bind-interfaces 3 | resolv-file=/etc/resolv.dnsmasq.conf 4 | expand-hosts 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | watchmen/watchmen-config/hosts.js 2 | saltfiles 3 | *.container.id 4 | conf.zk 5 | conf.supervisor 6 | 7 | -------------------------------------------------------------------------------- /centos-base-ssh/supervisord-sshd.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D -e 3 | autorestart=true 4 | startretries=3 5 | 6 | -------------------------------------------------------------------------------- /dnsmasq/dnsmasq.hosts/README: -------------------------------------------------------------------------------- 1 | # place hosts files in this directory and dnsmasq will pick them up at startup (no automatic re-load) 2 | -------------------------------------------------------------------------------- /shared/config.zk/zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | #export JAVA_HOME=/usr/java/latest 2 | export ZOO_LOG_DIR=/var/log/zookeeper 3 | export ZOO_LOG4J_PROP=DEBUG,ROLLINGFILE 4 | -------------------------------------------------------------------------------- /apache-hadoop-build/Makefile: -------------------------------------------------------------------------------- 1 | include ../shared/Makefile.vars 2 | include ../shared/Makefile 3 | 4 | hadoop: 5 | $(DOCKER_CMD) run ${tag} /build_hadoop.sh 2.7.1 6 | -------------------------------------------------------------------------------- /accumulo/accumulo_profile.sh: -------------------------------------------------------------------------------- 1 | if [ -x /usr/lib/accumulo/bin/accumulo ] 2 | then 3 | export ACCUMULO_HOME=/usr/lib/accumulo 4 | export PATH=$PATH:$ACCUMULO_HOME/bin 5 | fi 6 | -------------------------------------------------------------------------------- /shared/config.zk/zoo.cfg: -------------------------------------------------------------------------------- 1 | maxClientCnxns=150 2 | tickTime=2000 3 | initLimit=10 4 | syncLimit=5 5 | dataDir=/var/lib/zookeeper 6 | clientPort=2181 7 | autopurge.snapRetainCount=3 8 | 9 | -------------------------------------------------------------------------------- /dnsmasq/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | port=5353 2 | bind-interfaces 3 | resolv-file=/etc/resolv.dnsmasq.conf 4 | addn-hosts=/dnsmasq.hosts 5 | expand-hosts 6 | domain=testdomain.local 7 | log-queries 8 | 9 | -------------------------------------------------------------------------------- /watchmen/Makefile: -------------------------------------------------------------------------------- 1 | hostname := centos-base 2 | 3 | volumes := --volume=$(CURDIR)/watchmen-config:/srv/watchmen/config 4 | include ../shared/Makefile.vars 5 | include ../shared/Makefile 6 | 7 | -------------------------------------------------------------------------------- /firefox/Makefile: -------------------------------------------------------------------------------- 1 | hostname := firefox-vnc 2 | 3 | include ../shared/Makefile.vars 4 | include ../shared/Makefile 5 | 6 | 7 | firefox: clean container 8 | ssh -X $(sshopts) root@$(IP) firefox 9 | 10 | -------------------------------------------------------------------------------- /firefox/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sroegner/centos-base-ssh:6 2 | MAINTAINER Steffen Roegner 'steffen.roegner@gmail.com' 3 | 4 | RUN yum -y install firefox icedtea-web.x86_64 java-1.6.0-openjdk libXtst xorg-x11-xauth.x86_64 5 | 6 | -------------------------------------------------------------------------------- /watchmen/watchmen-config/storage.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | provider : 'redis', 3 | options : { 4 | 'redis' : { 5 | port: 6379, 6 | host: '127.0.0.1', 7 | db: 1 8 | } 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /watchmen/supervisord-redis.conf: -------------------------------------------------------------------------------- 1 | [program:redis] 2 | command=/usr/sbin/redis-server /etc/redis.conf 3 | autostart=true 4 | autorestart=true 5 | user=root 6 | stdout_logfile=/var/log/redis/stdout.log 7 | stderr_logfile=/var/log/redis/stderr.log 8 | -------------------------------------------------------------------------------- /dnsmasq/Makefile: -------------------------------------------------------------------------------- 1 | hostname := dnsmasq 2 | 3 | include ../shared/Makefile.vars 4 | include ../shared/Makefile 5 | 6 | container: 7 | $(DOCKER_CMD) run -d -v=$(CURDIR)/dnsmasq.hosts:/dnsmasq.hosts --name=$(container_name) --publish=$(IP):5353:5353 $(tag) 8 | 9 | -------------------------------------------------------------------------------- /consul-cluster/services.json: -------------------------------------------------------------------------------- 1 | { 2 | "service": { 3 | "name": "ssh", 4 | "tags": ["sshd", "system"], 5 | "port": 22, 6 | "check": { 7 | "interval": "10s", 8 | "script": "/var/lib/consul/scripts/sshd" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /doop/README.md: -------------------------------------------------------------------------------- 1 | DOOP 2 | 3 | Hortonworks HDP 2.2 namenode and datanode in one container. 4 | Also runs a zookeeper and sshd process on their default ports. 5 | 6 | Use the Makefile (`make image`) to build, (`make container`) to run, stop with `make clean`. 7 | 8 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/proxy.properties: -------------------------------------------------------------------------------- 1 | protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory 2 | tokenClass=org.apache.accumulo.core.client.security.tokens.PasswordToken 3 | port=42424 4 | instance=accumulo 5 | zookeepers=zookeeper.docker.local:2181 6 | 7 | -------------------------------------------------------------------------------- /centos-base-ssh/Makefile: -------------------------------------------------------------------------------- 1 | hostname := centos-base 2 | 3 | include ../shared/Makefile.vars 4 | tag := "$(reg)/$(box_name):7" 5 | include ../shared/Makefile 6 | 7 | image: 8 | cp ../shared/insecure.pub $(CURDIR)/ 9 | $(DOCKER_CMD) build -t=$(tag) . 10 | 11 | -------------------------------------------------------------------------------- /nexus/nexus.properties: -------------------------------------------------------------------------------- 1 | # nexus properties inside docker 2 | application-port=8081 3 | application-host=0.0.0.0 4 | nexus-webapp=${bundleBasedir}/nexus 5 | nexus-webapp-context-path=/ 6 | nexus-work=${bundleBasedir}/../sonatype-work/nexus 7 | runtime=${bundleBasedir}/nexus/WEB-INF 8 | -------------------------------------------------------------------------------- /consul-cluster/consul.json: -------------------------------------------------------------------------------- 1 | { 2 | "datacenter": "docker", 3 | "data_dir": "/var/lib/consul/data", 4 | "ui_dir": "/var/lib/consul/ui/dist", 5 | "log_level": "DEBUG", 6 | "client_addr": "0.0.0.0", 7 | "ports": { 8 | "dns": 53 9 | }, 10 | "recursor": "8.8.8.8" 11 | } 12 | -------------------------------------------------------------------------------- /doop/check_hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PORT=${1} 4 | HOST=${2:-"127.0.0.1"} 5 | URL="http://${HOST}:${PORT}" 6 | ERROR=1 7 | RC=0 8 | 9 | if [ "${PORT}" = "" ] 10 | then 11 | exit $ERROR 12 | fi 13 | 14 | curl -L --connect-timeout 1 $URL -o /dev/null &>/dev/null 15 | exit $? 16 | -------------------------------------------------------------------------------- /dnsmasq/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sroegner/centos-base-ssh:6 2 | MAINTAINER Steffen Roegner 'steffen.roegner@gmail.com' 3 | 4 | 5 | RUN yum -y install dnsmasq 6 | 7 | COPY dnsmasq.conf /etc/ 8 | COPY resolv.dnsmasq.conf /etc/ 9 | 10 | VOLUME /dnsmasq.hosts 11 | 12 | EXPOSE 5353 13 | 14 | ENTRYPOINT ["/usr/sbin/dnsmasq", "-d"] 15 | 16 | -------------------------------------------------------------------------------- /nexus/Makefile: -------------------------------------------------------------------------------- 1 | hostname := nexus 2 | 3 | include ../shared/Makefile.vars 4 | include ../shared/Makefile 5 | 6 | container: 7 | sudo docker run -d --name=$(container_name) -p=$(IP):8081:8081 $(tag) 8 | sleep 10 9 | sudo docker logs $(container_name) 10 | echo 11 | echo "Nexus will be available at http://$(IP):8081" 12 | 13 | -------------------------------------------------------------------------------- /shared/config.supervisor/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | [program:dnsmasq] 2 | user=root 3 | command=/usr/sbin/dnsmasq -d 4 | autorestart=true 5 | startsecs=2 6 | startretries=3 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/supervisor/dnsmasq.out 9 | stdout_logfile_maxbytes=1MB 10 | stdout_logfile_backups=10 11 | stdout_capture_maxbytes=1MB 12 | -------------------------------------------------------------------------------- /shared/Makefile.vars: -------------------------------------------------------------------------------- 1 | EXECUTOR_NUMBER?=0 2 | DOCKER_CMD := docker 3 | box_name = $(notdir $(CURDIR)) 4 | USER := $(shell id -un) 5 | container_name = $(box_name)-$(USER)-$(EXECUTOR_NUMBER) 6 | reg := docker.io/sroegner 7 | tag := $(reg)/$(box_name) 8 | sshopts := -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i $(CURDIR)/../shared/insecure.pem 9 | 10 | -------------------------------------------------------------------------------- /doop/bootstrap-node.conf: -------------------------------------------------------------------------------- 1 | [program:bootstrap] 2 | user=root 3 | command=/usr/local/sbin/bootstrap-node.sh 4 | priority=0 5 | autostart=true 6 | autorestart=false 7 | startsecs=1 8 | startretries=0 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/supervisor/bootstrap 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | -------------------------------------------------------------------------------- /shared/config.supervisor/zookeeper.conf: -------------------------------------------------------------------------------- 1 | [program:zookeeper] 2 | user=zookeeper 3 | command=/usr/lib/zookeeper/bin/zkServer.sh start-foreground 4 | autorestart=true 5 | startsecs=10 6 | startretries=3 7 | redirect_stderr=true 8 | stdout_logfile=/var/log/zookeeper/zookeeper.out 9 | stdout_logfile_maxbytes=1MB 10 | stdout_logfile_backups=10 11 | stdout_capture_maxbytes=1MB 12 | -------------------------------------------------------------------------------- /zookeeper/Makefile: -------------------------------------------------------------------------------- 1 | hostname := centos-base 2 | 3 | include ../shared/Makefile.vars 4 | include ../shared/Makefile 5 | 6 | image: 7 | cp -r ../shared/config.supervisor conf.supervisor 8 | cp -r ../shared/config.zk conf.zk 9 | $(DOCKER_CMD) build -t=$(tag) . 10 | 11 | container: 12 | $(DOCKER_CMD) run -d --name=$(container_name) --publish=$(IP):2181:2181 $(tag) 13 | 14 | -------------------------------------------------------------------------------- /watchmen/watchmen-config/general.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 'notifications' : { 3 | enabled: false, //if disabled, no notifications will be sent 4 | to: ['ivan@iloire.com'], //default notification list if no alert_to is specified for host or url 5 | postmark : { 6 | from: 'ivan@iloire.com', 7 | api_key : 'your-postmark-key-here' 8 | } 9 | } 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /centos-base-ssh/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [include] 5 | files = /etc/supervisor/conf.d/*.conf 6 | 7 | [rpcinterface:supervisor] 8 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 9 | 10 | [supervisorctl] 11 | serverurl=unix:///tmp/supervisor.sock 12 | 13 | [unix_http_server] 14 | file=/tmp/supervisor.sock 15 | chmod = 0777 16 | 17 | -------------------------------------------------------------------------------- /apache-hadoop-build/build_hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERSION=${1:-2.7.1} 4 | BUILD_DIR=/hadoop_build 5 | HADOOP_URL=http://apache.osuosl.org/hadoop/common/hadoop-${VERSION}/hadoop-${VERSION}-src.tar.gz 6 | 7 | rm -rf $BUILD_DIR 8 | mkdir -p $BUILD_DIR 9 | cd $BUILD_DIR 10 | curl -L $HADOOP_URL | tar xz --strip-components=1 11 | ls -l 12 | /usr/local/bin/mvn package -Pdist,native -DskipTests -Dmaven.test.skip -Dtar 13 | 14 | -------------------------------------------------------------------------------- /doop/hadoop/dfs_data_dir_mount.hist: -------------------------------------------------------------------------------- 1 | # This file keeps track of the last known mount-point for each DFS data dir. 2 | # It is safe to delete, since it will get regenerated the next time that the DataNode starts. 3 | # However, it is not advised to delete this file since Ambari may 4 | # re-create a DFS data dir that used to be mounted on a drive but is now mounted on the root. 5 | # Comments begin with a hash (#) symbol 6 | # data_dir,mount_point 7 | /hadoop/hdfs/data,/ 8 | -------------------------------------------------------------------------------- /shared/config.supervisor/hdfs-datanode.conf: -------------------------------------------------------------------------------- 1 | [program:datanode] 2 | user=hdfs 3 | command=/usr/lib/hadoop-hdfs/bin/hdfs datanode 4 | autorestart=true 5 | startsecs=10 6 | startretries=3 7 | priority=10 8 | environment=HADOOP_LIBEXEC_DIR="/usr/lib/hadoop/libexec",HADOOP_LOGFILE=hdfs-datanode.log,HADOOP_ROOT_LOGGER="INFO,RFA" 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/hadoop/hdfs-datanode.out 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | 15 | -------------------------------------------------------------------------------- /doop/bootstrap-node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # a generic bootstrapper to start services via supervisor 4 | # The supervisor config has to be part of the image, the list of service 5 | # is declared as comma-separated list 6 | # docker run ... -e SVCLIST=aservice,bservice ... 7 | 8 | if [ "$SVCLIST" != "" ] 9 | then 10 | services=(${SVCLIST//,/ }) 11 | for svc in "${services[@]}" 12 | do 13 | echo "[$(date)] starting $svc" 14 | supervisorctl start $svc 15 | sleep 1 16 | supervisorctl status $svc 17 | done 18 | fi 19 | 20 | -------------------------------------------------------------------------------- /shared/config.supervisor/yarn-nodemanager.conf: -------------------------------------------------------------------------------- 1 | [program:nodemanager] 2 | user=yarn 3 | command=/usr/lib/hadoop-yarn/bin/yarn nodemanager 4 | autorestart=true 5 | startsecs=10 6 | startretries=3 7 | priority=50 8 | environment=HADOOP_LIBEXEC_DIR="/usr/lib/hadoop/libexec",YARN_LOGFILE="yarn-nodemanager.log",YARN_ROOT_LOGGER="INFO,RFA",YARN_LOG_DIR="/var/log/hadoop-yarn" 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/hadoop/nodemanager.out 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | 15 | 16 | -------------------------------------------------------------------------------- /shared/config.supervisor/yarn-resourcemanager.conf: -------------------------------------------------------------------------------- 1 | [program:resourcemanager] 2 | user=yarn 3 | command=/usr/lib/hadoop-yarn/bin/yarn resourcemanager 4 | autorestart=true 5 | startsecs=10 6 | startretries=3 7 | priority=50 8 | environment=HADOOP_LIBEXEC_DIR="/usr/lib/hadoop/libexec",YARN_LOGFILE="yarn-resourcemanager.log",YARN_ROOT_LOGGER="INFO,RFA",YARN_LOG_DIR="/var/log/hadoop-yarn" 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/hadoop/resourcemanager.out 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | 15 | -------------------------------------------------------------------------------- /accumulo/add_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export USERNAME=${1:-bob} 4 | export PASSWORD=${2:-robert} 5 | 6 | su - accumulo << ENDE 7 | date 8 | echo -e "${PASSWORD}\n${PASSWORD}\n" | /usr/lib/accumulo/bin/accumulo shell -u root -p DOCKERDEFAULT -e "createuser ${USERNAME}" 2>/dev/null 9 | echo -e "${PASSWORD}\n" | /usr/lib/accumulo/bin/accumulo shell -u root -p DOCKERDEFAULT -e "authenticate ${USERNAME}" 2>/dev/null 10 | echo "Allow ${USERNAME} to create tables" 11 | /usr/lib/accumulo/bin/accumulo shell -u root -p DOCKERDEFAULT -e "grant System.CREATE_TABLE -s -u ${USERNAME}" 2>/dev/null 12 | ENDE 13 | -------------------------------------------------------------------------------- /watchmen/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sroegner/centos-base-ssh 2 | 3 | RUN yum -y install nodejs npm git redis; \ 4 | git clone https://github.com/iloire/WatchMen.git /srv/watchmen && \ 5 | cd /srv/watchmen && npm install && mv config config.dist; \ 6 | mkdir -p /var/log/watchmen 7 | 8 | ADD . /config 9 | RUN cp -v /config/supervisord-watchmen.conf /etc/supervisor/conf.d/; \ 10 | cp -v /config/supervisord-redis.conf /etc/supervisor/conf.d/; \ 11 | cp -v /config/redis.conf /etc/redis.conf 12 | 13 | VOLUME /srv/watchmen/config 14 | EXPOSE 22 80 3000 15 | 16 | CMD ["/usr/bin/supervisord", "-n"] 17 | -------------------------------------------------------------------------------- /watchmen/supervisord-watchmen.conf: -------------------------------------------------------------------------------- 1 | [program:watchmen-monitor] 2 | user=root 3 | command=/usr/bin/node server.js 4 | directory=/srv/watchmen 5 | autorestart=true 6 | startsecs=2 7 | startretries=3 8 | stdout_logfile=/var/log/watchmen/monitor-stdout.log 9 | stderr_logfile=/var/log/watchmen/monitor-stderr.log 10 | 11 | [program:watchmen-webapp] 12 | user=root 13 | command=/usr/bin/node webserver/app.js 3000 14 | directory=/srv/watchmen 15 | autorestart=true 16 | startsecs=2 17 | startretries=3 18 | stdout_logfile=/var/log/watchmen/webapp-stdout.log 19 | stderr_logfile=/var/log/watchmen/webapp-stderr.log 20 | 21 | -------------------------------------------------------------------------------- /accumulo/dotssh/accumulo.pub: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBAKEI5G/uFvgFtLCDwyNELn78y1arv3aBWPKcqs7Ee43q4e2Pd+kf7WXp4065CVJcrDcqUkEjSxPsDZc+qCj5T3zdV6LGnPXfw4MehTy4UE96J/u7ZBXblPCExvUW45NiawbPOrthWP5G6160rd8FB+OzjBCoUMpPPRd9IochFK4zAAAAFQCgHiWD1X/1iKJ6LCkKSMmgidcWMQAAAIAMQ1GQehTituftMCudgzV51xAvbvzo/F1wU/VbdAjX2zV97NjfLQWMgFMLdWowXq4L6mJqijMQ6gCdNDTOY6MHXCLexFyMR15wmuu1/5i+/LdizyS3jpp92B5Jden1aPjgaQ1CcVKhryZ/xxVmZ0lejW1WdIc59s4HHpL+kt1IfgAAAIBNRASsGyq3L2a5Nx2Yd7plq2PKdzeG+rH4es57Pj5TaPvU/U+9Xq8CbOoTt3Q3bXu9y5gLcahE0lhIQMbUPtSMGy7Vr+kMVxjc9Ov5kd3StZbUOybNdEn/K3jFpRT9gxxQ+px5Q4aMgPsicZEPff00DKiLqV0CyD1LeBBsua/UKg== accumulo 2 | -------------------------------------------------------------------------------- /accumulo/dotssh/authorized_keys: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBAKEI5G/uFvgFtLCDwyNELn78y1arv3aBWPKcqs7Ee43q4e2Pd+kf7WXp4065CVJcrDcqUkEjSxPsDZc+qCj5T3zdV6LGnPXfw4MehTy4UE96J/u7ZBXblPCExvUW45NiawbPOrthWP5G6160rd8FB+OzjBCoUMpPPRd9IochFK4zAAAAFQCgHiWD1X/1iKJ6LCkKSMmgidcWMQAAAIAMQ1GQehTituftMCudgzV51xAvbvzo/F1wU/VbdAjX2zV97NjfLQWMgFMLdWowXq4L6mJqijMQ6gCdNDTOY6MHXCLexFyMR15wmuu1/5i+/LdizyS3jpp92B5Jden1aPjgaQ1CcVKhryZ/xxVmZ0lejW1WdIc59s4HHpL+kt1IfgAAAIBNRASsGyq3L2a5Nx2Yd7plq2PKdzeG+rH4es57Pj5TaPvU/U+9Xq8CbOoTt3Q3bXu9y5gLcahE0lhIQMbUPtSMGy7Vr+kMVxjc9Ov5kd3StZbUOybNdEn/K3jFpRT9gxxQ+px5Q4aMgPsicZEPff00DKiLqV0CyD1LeBBsua/UKg== accumulo 2 | -------------------------------------------------------------------------------- /shared/config.supervisor/yarn-jobhistory.conf: -------------------------------------------------------------------------------- 1 | [program:jobhistory] 2 | user=yarn 3 | command=/usr/lib/hadoop-mapreduce/bin/mapred --config /etc/hadoop/conf historyserver start 4 | autorestart=true 5 | startsecs=10 6 | startretries=3 7 | priority=999 8 | environment=HADOOP_LIBEXEC_DIR="/usr/lib/hadoop/libexec",HADOOP_MAPRED_LOGFILE="yarn-jobhistory.log",HADOOP_MAPRED_ROOT_LOGGER="INFO,RFA",HADOOP_MAPRED_LOG_DIR="/var/log/hadoop-yarn" 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/supervisor/jobhistory.log 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | 15 | -------------------------------------------------------------------------------- /shared/insecure.pub: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBANObAV3eyF0qmCkAjRVLZS/GcY+qcm4wvhT5IhIt6O9c5Jj9ji6qv86A4aGbThaU0OIYw5UW0opW2EuF4dCa11c2GJ+SmQ1oKf2YMh5+Ng/7a/2JH+e1wF5ygTg896f8gpkoYwdTz6TrdGV/B8/gO/xKS9rAOhX59nREpGH5hLbZAAAAFQD479dOCcq/aaSl7PVQVKqn/HxwKQAAAIEAi9TeGHXOSMi20bQFPYHoPcIclZzd6WHIdDwnC5sOfVNlMa1VzK3LnKbZldCT1XREObMm1gL+LRg2FyL6F552ONCVJi52C3927wkyOmkKpM+56yJhm2/cd0T7zCtHQcwI0V8Tr8h1nfrkm04DV7t5FHd3nSA1bdeejx5hZOnmAQgAAACBAJ3ISNgRO4bNmU0bOoBTWMXLT0DTQw+immvRk1Ze+pbWmy/VFP+QFrmV/7y1kUmXDw46b2y8lX4/mA/dje8+f3u0gvF+ySpVMecki6zUQVtgyGh84iCQg0wkFd0bRJEM/lmcmX9gwi4/DwO/XealAFSohO+qCLDNthnF8l6v5yBA insecure docker key 2 | -------------------------------------------------------------------------------- /xmind/Makefile: -------------------------------------------------------------------------------- 1 | include ../shared/Makefile.vars 2 | tag := "$(reg)/$(box_name)" 3 | include ../shared/Makefile 4 | 5 | xmind: 6 | docker run -it --net host --cpuset-cpus 0 --memory 2048mb \ 7 | -v /tmp/.X11-unix:/tmp/.X11-unix \ 8 | -v $(HOME)/Downloads:/root/Downloads -v $(HOME)/Documents:/root/Documents \ 9 | --device /dev/snd --name $(container_name) $(tag) 10 | 11 | xshell: 12 | docker run -it --net host --cpuset-cpus 0 --memory 2048mb -v /tmp/.X11-unix:/tmp/.X11-unix -v $(HOME)/Downloads:/root/Downloads -v $(HOME)/Documents:/root/Documents --device /dev/snd --name $(container_name) $(tag) /usr/bin/xterm 13 | -------------------------------------------------------------------------------- /xmind/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Steffen Roegner "steffen.roegner@gmail.com" 3 | USER root 4 | 5 | ENV REFRESHED_AT 2015-08-14 6 | 7 | ENV DEBIAN_FRONTEND noninteractive 8 | RUN apt-get update 9 | RUN locale-gen en_US en_US.UTF-8 10 | ENV LANG en_US.UTF-8 11 | RUN echo "export PS1='\e[1;31m\]\u@\h:\w\\$\[\e[0m\] '" >> /root/.bashrc 12 | 13 | RUN apt-get install -y curl default-jre libwebkitgtk-1.0-0 lame xterm 14 | 15 | RUN curl --fail --silent -L http://www.xmind.net/xmind/downloads/xmind-7-update1-linux_amd64.deb -o /tmp/xmind.deb && \ 16 | dpkg -i /tmp/xmind.deb 17 | 18 | ENV DISPLAY :0 19 | CMD ["/usr/bin/XMind"] 20 | -------------------------------------------------------------------------------- /doop/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | logfile = /var/log/supervisord.log 3 | logfile_maxbytes = 5MB 4 | logfile_backups=10 5 | loglevel = warn 6 | pidfile = /var/run/supervisord.pid 7 | nodaemon = true 8 | minfds = 1024 9 | minprocs = 200 10 | umask = 022 11 | identifier = supervisor 12 | directory = /tmp 13 | nocleanup = true 14 | childlogdir = /tmp 15 | strip_ansi = false 16 | 17 | [include] 18 | files = /etc/supervisor/conf.d/*.conf 19 | 20 | [rpcinterface:supervisor] 21 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 22 | 23 | [inet_http_server] 24 | port = 0.0.0.0:59001 25 | 26 | [supervisorctl] 27 | serverurl=http://127.0.0.1:59001 28 | 29 | -------------------------------------------------------------------------------- /accumulo/dotssh/accumulo: -------------------------------------------------------------------------------- 1 | -----BEGIN DSA PRIVATE KEY----- 2 | MIIBugIBAAKBgQChCORv7hb4BbSwg8MjRC5+/MtWq792gVjynKrOxHuN6uHtj3fp 3 | H+1l6eNOuQlSXKw3KlJBI0sT7A2XPqgo+U983Veixpz138ODHoU8uFBPeif7u2QV 4 | 25TwhMb1FuOTYmsGzzq7YVj+RutetK3fBQfjs4wQqFDKTz0XfSKHIRSuMwIVAKAe 5 | JYPVf/WIonosKQpIyaCJ1xYxAoGADENRkHoU4rbn7TArnYM1edcQL2786PxdcFP1 6 | W3QI19s1fezY3y0FjIBTC3VqMF6uC+piaoozEOoAnTQ0zmOjB1wi3sRcjEdecJrr 7 | tf+Yvvy3Ys8kt46afdgeSXXp9Wj44GkNQnFSoa8mf8cVZmdJXo1tVnSHOfbOBx6S 8 | /pLdSH4CgYBNRASsGyq3L2a5Nx2Yd7plq2PKdzeG+rH4es57Pj5TaPvU/U+9Xq8C 9 | bOoTt3Q3bXu9y5gLcahE0lhIQMbUPtSMGy7Vr+kMVxjc9Ov5kd3StZbUOybNdEn/ 10 | K3jFpRT9gxxQ+px5Q4aMgPsicZEPff00DKiLqV0CyD1LeBBsua/UKgIUBVuFCh9+ 11 | YQ0au1Qd0FVogFaC78Q= 12 | -----END DSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /shared/insecure.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN DSA PRIVATE KEY----- 2 | MIIBvQIBAAKBgQDTmwFd3shdKpgpAI0VS2UvxnGPqnJuML4U+SISLejvXOSY/Y4u 3 | qr/OgOGhm04WlNDiGMOVFtKKVthLheHQmtdXNhifkpkNaCn9mDIefjYP+2v9iR/n 4 | tcBecoE4PPen/IKZKGMHU8+k63RlfwfP4Dv8SkvawDoV+fZ0RKRh+YS22QIVAPjv 5 | 104Jyr9ppKXs9VBUqqf8fHApAoGBAIvU3hh1zkjIttG0BT2B6D3CHJWc3elhyHQ8 6 | JwubDn1TZTGtVcyty5ym2ZXQk9V0RDmzJtYC/i0YNhci+heedjjQlSYudgt/du8J 7 | MjppCqTPuesiYZtv3HdE+8wrR0HMCNFfE6/IdZ365JtOA1e7eRR3d50gNW3Xno8e 8 | YWTp5gEIAoGBAJ3ISNgRO4bNmU0bOoBTWMXLT0DTQw+immvRk1Ze+pbWmy/VFP+Q 9 | FrmV/7y1kUmXDw46b2y8lX4/mA/dje8+f3u0gvF+ySpVMecki6zUQVtgyGh84iCQ 10 | g0wkFd0bRJEM/lmcmX9gwi4/DwO/XealAFSohO+qCLDNthnF8l6v5yBAAhUArqBO 11 | 9aMvz5pWMDFTGnsEmoJM3Us= 12 | -----END DSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /doop/hadoop/mapred-env.sh: -------------------------------------------------------------------------------- 1 | 2 | # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ 3 | 4 | export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=900 5 | 6 | export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA 7 | 8 | #export HADOOP_JOB_HISTORYSERVER_OPTS= 9 | #export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. 10 | #export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. 11 | #export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. 12 | #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default 13 | #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. 14 | export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" 15 | -------------------------------------------------------------------------------- /zookeeper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sroegner/centos-base-ssh:7 2 | MAINTAINER Steffen Roegner 'steffen.roegner@gmail.com' 3 | 4 | ENV ZOO_LOG4J_PROP WARN,CONSOLE 5 | 6 | RUN curl -L http://public-repo-1.hortonworks.com/HDP/centos6/2.x/GA/2.2.0.0/hdp.repo -o /etc/yum.repos.d/hdp.repo; \ 7 | yum -y install zookeeper-server; \ 8 | mkdir -p /var/lib/zookeeper; chown zookeeper:hadoop /var/lib/zookeeper 9 | 10 | COPY conf.supervisor/zookeeper.conf /etc/supervisor/conf.d/ 11 | COPY conf.zk/zoo.cfg /etc/zookeeper/conf/ 12 | RUN echo "export ZOO_LOG4J_PROP=WARN,CONSOLE" > /etc/zookeeper/conf/zooekeeper-env.sh 13 | 14 | EXPOSE 2181 15 | 16 | USER zookeeper 17 | ENTRYPOINT ["/usr/hdp/current/zookeeper-client/bin/zkServer.sh", "start-foreground"] 18 | 19 | -------------------------------------------------------------------------------- /accumulo/init_accumulo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | su - hdfs << EOF 4 | echo 5 | echo "Preparing for Accumulo initialization" 6 | JAVA_HOME=/usr HADOOP_ROOT_LOGGER="ERROR,console" /usr/bin/hdfs dfs -mkdir -p /user /tmp 7 | JAVA_HOME=/usr HADOOP_ROOT_LOGGER="ERROR,console" /usr/bin/hdfs dfs -chmod -R 1777 /tmp 8 | JAVA_HOME=/usr HADOOP_ROOT_LOGGER="ERROR,console" /usr/bin/hdfs dfs -mkdir /accumulo /user/accumulo 9 | JAVA_HOME=/usr HADOOP_ROOT_LOGGER="ERROR,console" /usr/bin/hdfs dfs -chown accumulo:accumulo /accumulo /user/accumulo 10 | JAVA_HOME=/usr HADOOP_ROOT_LOGGER="ERROR,console" /usr/bin/hdfs dfs -ls / 11 | EOF 12 | 13 | echo "Initializing Accumulo" 14 | su - accumulo -c '/usr/lib/accumulo/bin/accumulo init --instance-name accumulo --password DOCKERDEFAULT' 15 | 16 | -------------------------------------------------------------------------------- /consul-cluster/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | MAINTAINER Steffen Roegner 'steffen.roegner@gmail.com' 3 | 4 | ENV CONSUL_VERSION 0.6.3 5 | 6 | RUN yum -y install curl unzip 7 | RUN mkdir -p /var/log/consul /etc/consul /var/lib/consul/data /var/lib/consul/ui /var/lib/consul/scripts 8 | RUN curl --fail -q -L https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip -o /tmp/c.zip && \ 9 | curl --fail -q -L https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_web_ui.zip -o /tmp/ui.zip && \ 10 | unzip /tmp/c.zip -d /usr/sbin && \ 11 | unzip /tmp/ui.zip -d /var/lib/consul/ui 12 | 13 | COPY *.json /etc/consul/ 14 | COPY checkscripts /var/lib/consul/scripts/ 15 | 16 | EXPOSE 8400 8500 8600 17 | 18 | CMD ["/usr/bin/supervisord", "-n"] 19 | -------------------------------------------------------------------------------- /watchmen/watchmen-config/hosts.js.template: -------------------------------------------------------------------------------- 1 | var one_tick = 20; //seconds 2 | 3 | //service name must be unique for a certain host. 4 | //host name must be unique 5 | 6 | module.exports = 7 | [ 8 | { 9 | name:'Apple HTTPS', 10 | host: 'www.apple.com', 11 | port:443, 12 | protocol: 'https', 13 | ping_service_name: 'http', 14 | timeout:10000, 15 | ping_interval: one_tick, //seconds 16 | failed_ping_interval: one_tick / 3, //minutes 17 | enabled: true, 18 | alert_to: ['ivan@iloire.com'], 19 | warning_if_takes_more_than: 1500, //miliseconds 20 | services : [ 21 | { 22 | name: 'home', 23 | method: 'get', 24 | url : '/', 25 | expected: {statuscode: 200, contains: 'Apple Inc'} 26 | } 27 | ] 28 | } 29 | ] 30 | -------------------------------------------------------------------------------- /nexus/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sroegner/centos-base-ssh:6 2 | MAINTAINER Steffen Roegner 'steffen.roegner@gmail.com' 3 | 4 | RUN yum -y install createrepo; \ 5 | mkdir /tmp/nexus && \ 6 | cd /tmp/nexus && \ 7 | curl -L http://download.sonatype.com/nexus/oss/nexus-latest-bundle.tar.gz | tar xz --no-same-owner 8 | 9 | RUN useradd -m -d /srv/nexus nexus 10 | 11 | RUN mv -v $(ls -1d /tmp/nexus/nex*|head -1) /srv/nexus/; \ 12 | ln -s $(ls -1d /srv/nexus/nexus-*|head -1) /srv/nexus/nexus-server; \ 13 | mv -v /tmp/nexus/sonatype-work /srv/nexus && \ 14 | chown -R nexus:nexus /srv/nexus/* 15 | 16 | COPY nexus.properties /srv/nexus/nexus-server/conf/nexus.properties 17 | 18 | EXPOSE 8081 19 | USER nexus 20 | WORKDIR /srv/nexus/nexus-server 21 | 22 | ENTRYPOINT ["/srv/nexus/nexus-server/bin/jsw/linux-x86-64/wrapper", "-c", "/srv/nexus/nexus-server/bin/jsw/conf/wrapper.conf"] 23 | 24 | -------------------------------------------------------------------------------- /elasticsearch/Makefile: -------------------------------------------------------------------------------- 1 | include ../shared/Makefile.vars 2 | include ../shared/Makefile 3 | 4 | container: 5 | $(DOCKER_CMD) run -d --name=$(container_name) $(tag) 6 | echo For the UI go to http://$$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)):9200/_plugin/head 7 | echo iMarvel is at http://$$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)):9200/_plugin/marvel 8 | 9 | node1: 10 | $(DOCKER_CMD) run -d --name=$(container_name)-1 $(tag) 11 | 12 | node2: 13 | $(DOCKER_CMD) run -d --name=$(container_name)-2 $(tag) 14 | 15 | cluster: container node1 node2 16 | 17 | clean: 18 | $(DOCKER_CMD) rm -f $(container_name) 2>/dev/null || : 19 | $(DOCKER_CMD) rm -f $(container_name)-1 2>/dev/null || : 20 | $(DOCKER_CMD) rm -f $(container_name)-2 2>/dev/null || : 21 | 22 | shell: 23 | $(DOCKER_CMD) run -t -i --rm --name=$(container_name)-shell $(tag) /usr/bin/bash 24 | -------------------------------------------------------------------------------- /consul-cluster/README.md: -------------------------------------------------------------------------------- 1 | consul cluster in docker containers 2 | 3 | See http://www.consul.io/ for what consul is. 4 | This is just to learn about what can and what cannot be achieved 5 | with consul (inside docker) - the idea is to use the service registry 6 | and dns to orchestrate services in multiple docker containers from within, without the need for external (and static) name serice. 7 | 8 | You can point a browser to port 8500 of any of the nodes for the consul web ui. Also runs an sshd process on port 2211 of each container. 9 | 10 | Containers: 11 | 12 | - server1: 127.0.0.20 13 | - server2: 127.0.0.21 14 | - server3: 127.0.0.22 15 | - agent1: 127.0.0.23 16 | - agent2: 127.0.0.24 17 | - agent3: 127.0.0.25 18 | 19 | Use the Makefile (`make image`) to build, (`make cluster`) to run, stop with `make clean`. There is also `make server1` and `make agent1` which will login to the respective container with ssh. Just take a look at a "cold" container with `make shell`. 20 | 21 | -------------------------------------------------------------------------------- /doop/hadoop/hadoop-metrics2.properties: -------------------------------------------------------------------------------- 1 | 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # syntax: [prefix].[source|sink|jmx].[instance].[options] 18 | # See package.html for org.apache.hadoop.metrics2 for details 19 | 20 | -------------------------------------------------------------------------------- /shared/config.supervisor/hdfs-namenode.conf: -------------------------------------------------------------------------------- 1 | [program:namenode] 2 | user=hdfs 3 | command=/usr/lib/hadoop-hdfs/bin/hdfs namenode 4 | autorestart=true 5 | startsecs=10 6 | startretries=3 7 | priority=1 8 | environment=HADOOP_LIBEXEC_DIR="/usr/lib/hadoop/libexec",HADOOP_LOGFILE=hdfs-namenode.log,HADOOP_ROOT_LOGGER="INFO,RFA" 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/hadoop/hdfs-namenode.out 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | 15 | [program:secondarynamenode] 16 | user=hdfs 17 | command=/usr/lib/hadoop-hdfs/bin/hdfs secondarynamenode 18 | autorestart=true 19 | startsecs=10 20 | startretries=3 21 | priority=10 22 | environment=HADOOP_LIBEXEC_DIR="/usr/lib/hadoop/libexec",HADOOP_LOGFILE=hdfs-secondarynamenode.log,HADOOP_ROOT_LOGGER="INFO,RFA" 23 | redirect_stderr=true 24 | stdout_logfile=/var/log/hadoop/hdfs-secondarynamenode.out 25 | stdout_logfile_maxbytes=1MB 26 | stdout_logfile_backups=10 27 | stdout_capture_maxbytes=1MB 28 | 29 | -------------------------------------------------------------------------------- /doop/hadoop/taskcontroller.cfg: -------------------------------------------------------------------------------- 1 | 2 | #/* 3 | # * Licensed to the Apache Software Foundation (ASF) under one 4 | # * or more contributor license agreements. See the NOTICE file 5 | # * distributed with this work for additional information 6 | # * regarding copyright ownership. The ASF licenses this file 7 | # * to you under the Apache License, Version 2.0 (the 8 | # * "License"); you may not use this file except in compliance 9 | # * with the License. You may obtain a copy of the License at 10 | # * 11 | # * http://www.apache.org/licenses/LICENSE-2.0 12 | # * 13 | # * Unless required by applicable law or agreed to in writing, software 14 | # * distributed under the License is distributed on an "AS IS" BASIS, 15 | # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # * See the License for the specific language governing permissions and 17 | # * limitations under the License. 18 | # */ 19 | mapred.local.dir=/tmp/hadoop-mapred/mapred/local 20 | mapreduce.tasktracker.group=hadoop 21 | hadoop.log.dir=/var/log/hadoop/mapred 22 | -------------------------------------------------------------------------------- /apache-hadoop-build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | MAINTAINER Steffen Roegner "steffen.roegner@gmail.com" 3 | USER root 4 | 5 | ENV JAVA_HOME=/usr 6 | ENV REFRESHED_AT 2015-09-03 7 | ENV M2_HOME=/usr/local 8 | 9 | RUN rpm -ivh http://epel.mirror.constant.com/7/x86_64/e/epel-release-7-5.noarch.rpm; \ 10 | yum -y -q upgrade; \ 11 | yum -y install python-pip snappy lzo rsync which tar bind-utils java-1.7.0-openjdk-devel unzip bzip2; \ 12 | yum clean all 13 | 14 | RUN yum -y install gcc automake zlib-devel openssl-devel autoreconf 15 | RUN cd /usr/local && curl -L http://www.us.apache.org/dist/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz | tar xz --strip-components=1 16 | RUN mkdir /protoc && cd /protoc && curl -L https://github.com/google/protobuf/archive/v2.5.0.tar.gz | tar xz --strip-components=1 && \ 17 | ./autogen.sh; ./configure --prefix=/usr; make; make install; 18 | protoc --version 19 | RUN cd /protoc/java && /usr/local/bin/mvn install 20 | 21 | 22 | ADD build_hadoop.sh / 23 | 24 | RUN chmod 755 /build_hadoop.sh 25 | -------------------------------------------------------------------------------- /watchmen/redis.conf: -------------------------------------------------------------------------------- 1 | daemonize no 2 | pidfile /var/run/redis.pid 3 | port 6379 4 | timeout 0 5 | loglevel notice 6 | logfile stdout 7 | databases 32 8 | save 900 1 9 | save 300 10 10 | save 60 10000 11 | # stop-writes-on-bgsave-error yes 12 | rdbcompression yes 13 | # rdbchecksum yes 14 | dbfilename dump.rdb 15 | dir /var/lib/redis 16 | slave-serve-stale-data yes 17 | # slave-read-only yes 18 | # slave-priority 100 19 | appendonly yes 20 | appendfsync everysec 21 | no-appendfsync-on-rewrite no 22 | auto-aof-rewrite-percentage 100 23 | auto-aof-rewrite-min-size 64mb 24 | # lua-time-limit 5000 25 | slowlog-log-slower-than 10000 26 | slowlog-max-len 128 27 | # hash-max-ziplist-entries 512 28 | # hash-max-ziplist-value 64 29 | list-max-ziplist-entries 512 30 | list-max-ziplist-value 64 31 | set-max-intset-entries 512 32 | zset-max-ziplist-entries 128 33 | zset-max-ziplist-value 64 34 | activerehashing yes 35 | # client-output-buffer-limit normal 0 0 0 36 | # client-output-buffer-limit slave 256mb 64mb 60 37 | # client-output-buffer-limit pubsub 32mb 8mb 60 38 | 39 | -------------------------------------------------------------------------------- /shared/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | echo "Can do 'clean', 'erase', 'image', 'push', 'shell' or 'container' (and then 'ssh', and 'logs')" 3 | 4 | clean: 5 | rm -rvf conf.* 6 | $(DOCKER_CMD) stop $(container_name) || : 7 | $(DOCKER_CMD) rm $(container_name) || : 8 | 9 | pre-image-hook: 10 | echo nop 11 | 12 | image: pre-image-hook 13 | $(DOCKER_CMD) build -t=$(tag) . 14 | 15 | push: 16 | $(DOCKER_CMD) push $(tag) 17 | 18 | container: 19 | $(DOCKER_CMD) run -d --name=$(container_name) $(volumes) \ 20 | --hostname=$(hostname) $(tag) /usr/bin/supervisord -n 21 | ssh: 22 | ssh $(sshopts) root@$$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)) 23 | 24 | exec: 25 | $(DOCKER_CMD) exec -i -t $(container_name) /bin/bash 26 | 27 | shell: 28 | $(DOCKER_CMD) run -i -t $(tag) /bin/bash 29 | 30 | erase: clean 31 | $(DOCKER_CMD) rmi -f $(tag) || : 32 | 33 | logs: 34 | $(DOCKER_CMD) logs $(container_name) 35 | 36 | info: 37 | echo "Buildinfo for $(tag)" 38 | 39 | check-network: 40 | $(DOCKER_CMD) network inspect $(NET) &>/dev/null || $(DOCKER_CMD) network create -d bridge --subnet=$(CIDR) $(NET) 41 | 42 | -------------------------------------------------------------------------------- /doop/hadoop/commons-logging.properties: -------------------------------------------------------------------------------- 1 | 2 | #/* 3 | # * Licensed to the Apache Software Foundation (ASF) under one 4 | # * or more contributor license agreements. See the NOTICE file 5 | # * distributed with this work for additional information 6 | # * regarding copyright ownership. The ASF licenses this file 7 | # * to you under the Apache License, Version 2.0 (the 8 | # * "License"); you may not use this file except in compliance 9 | # * with the License. You may obtain a copy of the License at 10 | # * 11 | # * http://www.apache.org/licenses/LICENSE-2.0 12 | # * 13 | # * Unless required by applicable law or agreed to in writing, software 14 | # * distributed under the License is distributed on an "AS IS" BASIS, 15 | # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # * See the License for the specific language governing permissions and 17 | # * limitations under the License. 18 | # */ 19 | 20 | #Logging Implementation 21 | 22 | #Log4J 23 | org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger 24 | 25 | #JDK Logger 26 | #org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger 27 | -------------------------------------------------------------------------------- /elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:centos7 2 | MAINTAINER Steffen Roegner 'steffen.roegner@gmail.com' 3 | 4 | RUN rpm -ivh http://epel.mirror.constant.com/7/x86_64/e/epel-release-7-5.noarch.rpm; \ 5 | yum -y install java-1.8.0-openjdk-devel 6 | RUN yum -y install https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.5.2.noarch.rpm; \ 7 | /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head; \ 8 | /usr/share/elasticsearch/bin/plugin -install elasticsearch/marvel/latest; \ 9 | usermod -s /usr/bin/bash elasticsearch 10 | 11 | USER elasticsearch 12 | ENV CONF_DIR /etc/elasticsearch 13 | ENV CONF_FILE /etc/elasticsearch/elasticsearch.yml 14 | ENV ES_HOME /usr/share/elasticsearch 15 | ENV LOG_DIR /var/log/elasticsearch 16 | ENV DATA_DIR /var/lib/elasticsearch 17 | ENV WORK_DIR /tmp/elasticsearch 18 | 19 | CMD ["/usr/share/elasticsearch/bin/elasticsearch", "-p", "/var/run/elasticsearch/elasticsearch.pid", \ 20 | "-Des.default.config=$CONF_FILE", \ 21 | "-Des.default.path.home=$ES_HOME", \ 22 | "-Des.default.path.logs=$LOG_DIR", \ 23 | "-Des.default.path.data=$DATA_DIR", \ 24 | "-Des.default.path.work=$WORK_DIR", \ 25 | "-Des.default.path.conf=$CONF_DIR"] 26 | -------------------------------------------------------------------------------- /accumulo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sroegner/doop 2 | MAINTAINER Steffen Roegner "steffen.roegner@gmail.com" 3 | USER root 4 | 5 | RUN rpmdb --rebuilddb; yum -y install gcc-c++ 6 | ENV ACCUMULO_VERSION 1.7.1 7 | 8 | RUN curl -L http://apache.osuosl.org/accumulo/${ACCUMULO_VERSION}/accumulo-${ACCUMULO_VERSION}-bin.tar.gz | tar xz --no-same-owner -C /usr/lib 9 | 10 | RUN ln -s /usr/lib/accumulo-${ACCUMULO_VERSION} /usr/lib/accumulo; \ 11 | useradd -u 6040 -G hadoop -d /var/lib/accumulo accumulo; \ 12 | mkdir -p /etc/accumulo /var/lib/accumulo/conf /var/log/accumulo; \ 13 | chown accumulo.accumulo /var/lib/accumulo /var/log/accumulo; \ 14 | mv /usr/lib/accumulo/conf /usr/lib/accumulo/conf.dist; \ 15 | rm -rf /usr/lib/accumulo/logs; \ 16 | ln -s /var/lib/accumulo/conf /usr/lib/accumulo/conf; \ 17 | ln -s /var/lib/accumulo/conf /etc/accumulo/conf; \ 18 | ln -s /var/log/accumulo /usr/lib/accumulo/logs; \ 19 | JAVA_HOME=/usr/lib/jvm/java /usr/lib/accumulo/bin/build_native_library.sh 20 | 21 | COPY accumulo_profile.sh /etc/profile.d/accumulo_profile.sh 22 | COPY accumulo.conf /var/lib/accumulo/conf 23 | COPY init_accumulo.sh /usr/lib/accumulo/bin/ 24 | COPY add_user.sh /tmp/ 25 | COPY supervisor/ /etc/supervisor/conf.d/ 26 | 27 | RUN chown -R accumulo.accumulo /var/lib/accumulo/conf; \ 28 | chmod 700 /var/lib/accumulo/conf 29 | 30 | CMD ["/usr/bin/supervisord", "-n"] 31 | 32 | -------------------------------------------------------------------------------- /centos-base-ssh/Dockerfile: -------------------------------------------------------------------------------- 1 | # centos:centos6 2 | # adding some packages and a jdk all needed in hadoop 3 | # + openssh server run by supervisor 4 | 5 | #FROM sroegner/centos-base:6 6 | FROM centos:centos7 7 | MAINTAINER Steffen Roegner "steffen.roegner@gmail.com" 8 | USER root 9 | 10 | ENV REFRESHED_AT 2015-01-31 11 | ENV JAVA_HOME /usr 12 | ENV JAVA /usr 13 | 14 | RUN rpm -ivh http://epel.mirror.constant.com/7/x86_64/e/epel-release-7-5.noarch.rpm 15 | 16 | RUN yum -y -q upgrade; \ 17 | yum -y install passwd python-pip openssl snappy lzo sudo openssh-server openssh-clients rsync which tar bind-utils java-1.7.0-openjdk-devel; \ 18 | yum clean all; \ 19 | pip install supervisor 20 | 21 | RUN mkdir -p /root/.ssh; \ 22 | chmod 700 /root/.ssh; \ 23 | mkdir -p /var/run/sshd; \ 24 | chmod 700 /var/run/sshd; \ 25 | sed -i "s/GSSAPIAuthentication yes/GSSAPIAuthentication no/" /etc/ssh/sshd_config; \ 26 | /usr/sbin/sshd-keygen; \ 27 | ssh-keygen -q -t dsa -f /root/.ssh/id_dsa -N '' -C 'keypair generated during docker build' && cat /root/.ssh/id_dsa.pub > /root/.ssh/authorized_keys; \ 28 | chmod 600 /root/.ssh/authorized_keys; \ 29 | echo changeme | passwd --stdin root; \ 30 | mkdir -p /etc/supervisor/conf.d /var/log/supervisor 31 | 32 | COPY supervisord-sshd.conf /etc/supervisor/conf.d/sshd.conf 33 | COPY supervisord.conf /etc/ 34 | COPY insecure.pub /tmp/ 35 | 36 | RUN cat /tmp/insecure.pub >> /root/.ssh/authorized_keys 37 | 38 | EXPOSE 22 39 | 40 | CMD ["/usr/bin/supervisord", "-n"] 41 | -------------------------------------------------------------------------------- /doop/hadoop/hadoop-policy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | security.admin.operations.protocol.acl 6 | hadoop 7 | 8 | 9 | 10 | security.client.datanode.protocol.acl 11 | * 12 | 13 | 14 | 15 | security.client.protocol.acl 16 | * 17 | 18 | 19 | 20 | security.datanode.protocol.acl 21 | * 22 | 23 | 24 | 25 | security.inter.datanode.protocol.acl 26 | * 27 | 28 | 29 | 30 | security.inter.tracker.protocol.acl 31 | * 32 | 33 | 34 | 35 | security.job.client.protocol.acl 36 | * 37 | 38 | 39 | 40 | security.job.task.protocol.acl 41 | * 42 | 43 | 44 | 45 | security.namenode.protocol.acl 46 | * 47 | 48 | 49 | 50 | security.refresh.policy.protocol.acl 51 | hadoop 52 | 53 | 54 | 55 | security.refresh.usertogroups.mappings.protocol.acl 56 | hadoop 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/auditLog.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /doop/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | MAINTAINER Steffen Roegner "steffen.roegner@gmail.com" 3 | USER root 4 | 5 | ENV JAVA_HOME=/usr 6 | ENV HADOOP_HOME=/usr/hdp/current/hadoop-client 7 | ENV HADOOP_HDFS_HOME=/usr/hdp/current/hadoop-hdfs-client 8 | ENV HADOOP_MAPRED_HOME=/usr/hdp/current/hadoop-mapreduce-client 9 | ENV HADOOP_YARN_HOME=/usr/hdp/current/hadoop-yarn-client 10 | ENV HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec 11 | 12 | ENV REFRESHED_AT 2016-04-21 13 | 14 | RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm; \ 15 | yum -y -q upgrade; \ 16 | yum -y install python-pip snappy lzo rsync which tar bind-utils java-1.7.0-openjdk-devel unzip; \ 17 | yum clean all; \ 18 | pip install supervisor 19 | 20 | RUN curl -L http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.4.0.0/hdp.repo -o /etc/yum.repos.d/hdp.repo; \ 21 | yum -y install hadoop hadoop-hdfs hadoop-libhdfs hadoop-yarn hadoop-mapreduce hadoop-client zookeeper 22 | 23 | RUN mkdir -p /data1/hdfs /data1/mapred /data1/yarn /var/log/hadoop /var/log/hadoop-yarn /var/log/supervisor /var/lib/zookeeper; \ 24 | chown hdfs.hadoop /data1/hdfs && \ 25 | chown mapred.hadoop /data1/mapred && \ 26 | chown yarn.hadoop /data1/yarn; \ 27 | chown zookeeper.hadoop /var/lib/zookeeper; \ 28 | chmod 775 /var/log/hadoop; chgrp hadoop /var/log/hadoop 29 | 30 | COPY supervisord.conf /etc/ 31 | COPY hadoop /etc/hadoop/conf 32 | COPY conf.zk/zookeeper-env.sh /etc/zookeeper/conf/ 33 | COPY conf.zk/zoo.cfg /etc/zookeeper/conf/ 34 | COPY check_hadoop.sh /usr/local/sbin/ 35 | COPY bootstrap-node.conf /etc/supervisor/conf.d/ 36 | COPY bootstrap-node.sh /usr/local/sbin/ 37 | COPY hadoop-group.conf /etc/supervisor/conf.d/ 38 | 39 | USER hdfs 40 | RUN HADOOP_ROOT_LOGGER="WARN,console" /usr/bin/hdfs namenode -format 41 | 42 | USER root 43 | VOLUME /etc/hadoop/conf 44 | -------------------------------------------------------------------------------- /doop/hadoop/health_check: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | # 4 | #/* 5 | # * Licensed to the Apache Software Foundation (ASF) under one 6 | # * or more contributor license agreements. See the NOTICE file 7 | # * distributed with this work for additional information 8 | # * regarding copyright ownership. The ASF licenses this file 9 | # * to you under the Apache License, Version 2.0 (the 10 | # * "License"); you may not use this file except in compliance 11 | # * with the License. You may obtain a copy of the License at 12 | # * 13 | # * http://www.apache.org/licenses/LICENSE-2.0 14 | # * 15 | # * Unless required by applicable law or agreed to in writing, software 16 | # * distributed under the License is distributed on an "AS IS" BASIS, 17 | # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # * See the License for the specific language governing permissions and 19 | # * limitations under the License. 20 | # */ 21 | 22 | err=0; 23 | 24 | function check_disks { 25 | 26 | for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do 27 | fsdev="" 28 | fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`; 29 | if [ -z "$fsdev" -a "$m" != "/mnt" ] ; then 30 | msg_="$msg_ $m(u)" 31 | else 32 | msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`" 33 | fi 34 | done 35 | 36 | if [ -z "$msg_" ] ; then 37 | echo "disks ok" ; exit 0 38 | else 39 | echo "$msg_" ; exit 2 40 | fi 41 | 42 | } 43 | 44 | # Run all checks 45 | for check in disks ; do 46 | msg=`check_${check}` ; 47 | if [ $? -eq 0 ] ; then 48 | ok_msg="$ok_msg$msg," 49 | else 50 | err_msg="$err_msg$msg," 51 | fi 52 | done 53 | 54 | if [ ! -z "$err_msg" ] ; then 55 | echo -n "ERROR $err_msg " 56 | fi 57 | if [ ! -z "$ok_msg" ] ; then 58 | echo -n "OK: $ok_msg" 59 | fi 60 | 61 | echo 62 | 63 | # Success! 64 | exit 0 65 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # default logging properties: 17 | # by default, log everything at INFO or higher to the console 18 | log4j.rootLogger=INFO,A1 19 | 20 | # hide Jetty junk 21 | log4j.logger.org.mortbay.log=WARN,A1 22 | 23 | # hide "Got brand-new compressor" messages 24 | log4j.logger.org.apache.hadoop.io.compress=WARN,A1 25 | log4j.logger.org.apache.accumulo.core.file.rfile.bcfile.Compression=WARN,A1 26 | 27 | # hide junk from TestRandomDeletes 28 | log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1 29 | 30 | # hide junk from VFS 31 | log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1 32 | 33 | # hide almost everything from zookeeper 34 | log4j.logger.org.apache.zookeeper=ERROR,A1 35 | 36 | # hide AUDIT messages in the shell, alternatively you could send them to a different logger 37 | log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1 38 | 39 | # Send most things to the console 40 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 41 | log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n 42 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 43 | -------------------------------------------------------------------------------- /doop/hadoop/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | io.serializations 5 | org.apache.hadoop.io.serializer.WritableSerialization 6 | 7 | 8 | proxyuser_group 9 | users 10 | 11 | 12 | fs.trash.interval 13 | 0 14 | 15 | 16 | hadoop.security.authentication 17 | simple 18 | 19 | 20 | io.compression.codecs 21 | org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec 22 | 23 | 24 | mapreduce.jobtracker.webinterface.trusted 25 | false 26 | 27 | 28 | hadoop.security.authorization 29 | false 30 | 31 | 32 | ipc.client.connection.maxidletime 33 | 30000 34 | 35 | 36 | hadoop.http.authentication.simple.anonymous.allowed 37 | true 38 | 39 | 40 | ipc.client.connect.max.retries 41 | 50 42 | 43 | 44 | hadoop.security.auth_to_local 45 | RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/ 46 | RULE:[2:$1@$0](jhs@.*)s/.*/mapred/ 47 | RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/ 48 | RULE:[2:$1@$0](hm@.*)s/.*/hbase/ 49 | RULE:[2:$1@$0](rs@.*)s/.*/hbase/ 50 | DEFAULT 51 | 52 | 53 | io.file.buffer.size 54 | 131072 55 | 56 | 57 | ipc.server.tcpnodelay 58 | true 59 | 60 | 61 | ipc.client.idlethreshold 62 | 8000 63 | 64 | 65 | fs.defaultFS 66 | hdfs://namenode.docker.local:8020 67 | 68 | 69 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/accumulo-metrics.xml: -------------------------------------------------------------------------------- 1 | 17 | 21 | 22 | 25 | 26 | ${ACCUMULO_HOME}/metrics 27 | 28 | 32 | 33 | false 34 | false 35 | 36 | 37 | false 38 | false 39 | 40 | false 41 | false 42 | 43 | 44 | false 45 | false 46 | 47 | 48 | false 49 | false 50 | 51 | 52 | 53 | false 54 | false 55 | 56 | 57 | -------------------------------------------------------------------------------- /minidoop/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.1 2 | MAINTAINER Steffen Roegner "steffen.roegner@gmail.com" 3 | USER root 4 | 5 | ENV JAVA_HOME=/usr 6 | ENV HADOOP_HOME=/usr/hdp/current/hadoop-client 7 | ENV HADOOP_HDFS_HOME=/usr/hdp/current/hadoop-hdfs-client 8 | ENV HADOOP_MAPRED_HOME=/usr/hdp/current/hadoop-mapreduce-client 9 | ENV HADOOP_YARN_HOME=/usr/hdp/current/hadoop-yarn-client 10 | ENV HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec 11 | 12 | ENV REFRESHED_AT 2016-03-29 13 | 14 | RUN apk add --update curl py-pip lzo rsync zip openjdk7-jre-base; \ 15 | rm -rf /var/cache/apk/*; \ 16 | pip install supervisor 17 | 18 | RUN curl -L http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0/tars/hadoop-2.7.1.2.3.4.0-3485.tar.gz | tar xz -C /usr/lib 19 | RUN curl -L http://apache.osuosl.org/zookeeper/stable/zookeeper-3.4.6.tar.gz | tar xz -C /usr/lib 20 | 21 | RUN cd /usr/lib/hadoop-2.7.1.2.3.4.0-3485 && \ 22 | rm -rf share/doc share/hadoop/kms share/hadoop/httpfs; \ 23 | find . -name '*sources*jar' -exec rm -f {} \; 24 | 25 | RUN addgroup hadoop; \ 26 | for n in hdfs mapred yarn zookeeper; do \ 27 | echo -e "hadoop\nhadoop" adduser -S -G hadoop $n; \ 28 | done 29 | 30 | RUN getent passwd hdfs 31 | 32 | RUN mkdir -p /data1/hdfs /data1/mapred /data1/yarn /var/log/hadoop /var/log/hadoop-yarn /var/log/supervisor /var/log/consul /var/lib/consul/data /var/lib/consul/ui /etc/consul /etc/consul-leader /var/lib/zookeeper; \ 33 | chown hdfs.hadoop /data1/hdfs && \ 34 | chown mapred.hadoop /data1/mapred && \ 35 | chown yarn.hadoop /data1/yarn; \ 36 | chown zookeeper.hadoop /var/lib/zookeeper; \ 37 | chmod 775 /var/log/hadoop; chgrp hadoop /var/log/hadoop 38 | 39 | COPY supervisord.conf /etc/ 40 | COPY hadoop /etc/hadoop/conf 41 | COPY conf.zk/zookeeper-env.sh /etc/zookeeper/conf/ 42 | COPY conf.zk/zoo.cfg /etc/zookeeper/conf/ 43 | COPY check_hadoop.sh /usr/local/sbin/ 44 | COPY bootstrap-node.conf /etc/supervisor/conf.d/ 45 | COPY bootstrap-node.sh /usr/local/sbin/ 46 | COPY hadoop-group.conf /etc/supervisor/conf.d/ 47 | COPY consul/consul.conf /etc/supervisor/conf.d/ 48 | COPY consul /etc/consul/ 49 | COPY consul/consul.json /etc/consul-leader/ 50 | 51 | USER hdfs 52 | RUN HADOOP_ROOT_LOGGER="WARN,console" /usr/bin/hdfs namenode -format 53 | USER root 54 | VOLUME /etc/hadoop/conf 55 | -------------------------------------------------------------------------------- /doop/hadoop/capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.scheduler.capacity.default.minimum-user-limit-percent 5 | 100 6 | 7 | 8 | 9 | yarn.scheduler.capacity.maximum-am-resource-percent 10 | 0.2 11 | 12 | 13 | 14 | yarn.scheduler.capacity.maximum-applications 15 | 10000 16 | 17 | 18 | 19 | yarn.scheduler.capacity.node-locality-delay 20 | 40 21 | 22 | 23 | 24 | yarn.scheduler.capacity.resource-calculator 25 | org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator 26 | 27 | 28 | 29 | yarn.scheduler.capacity.root.accessible-node-labels 30 | * 31 | 32 | 33 | 34 | yarn.scheduler.capacity.root.acl_administer_queue 35 | * 36 | 37 | 38 | 39 | yarn.scheduler.capacity.root.capacity 40 | 100 41 | 42 | 43 | 44 | yarn.scheduler.capacity.root.default.acl_administer_jobs 45 | * 46 | 47 | 48 | 49 | yarn.scheduler.capacity.root.default.acl_submit_applications 50 | * 51 | 52 | 53 | 54 | yarn.scheduler.capacity.root.default.capacity 55 | 100 56 | 57 | 58 | 59 | yarn.scheduler.capacity.root.default.maximum-capacity 60 | 100 61 | 62 | 63 | 64 | yarn.scheduler.capacity.root.default.state 65 | RUNNING 66 | 67 | 68 | 69 | yarn.scheduler.capacity.root.default.user-limit-factor 70 | 1 71 | 72 | 73 | 74 | yarn.scheduler.capacity.root.queues 75 | default 76 | 77 | 78 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/monitor_logger.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Write out everything at the DEBUG level to the debug log 17 | log4j.appender.A2=org.apache.log4j.RollingFileAppender 18 | log4j.appender.A2.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.debug.log 19 | log4j.appender.A2.MaxFileSize=100MB 20 | log4j.appender.A2.MaxBackupIndex=10 21 | log4j.appender.A2.Threshold=DEBUG 22 | log4j.appender.A2.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.A2.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n 24 | 25 | # Write out INFO and higher to the regular log 26 | log4j.appender.A3=org.apache.log4j.RollingFileAppender 27 | log4j.appender.A3.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.log 28 | log4j.appender.A3.MaxFileSize=100MB 29 | log4j.appender.A3.MaxBackupIndex=10 30 | log4j.appender.A3.Threshold=INFO 31 | log4j.appender.A3.layout=org.apache.log4j.PatternLayout 32 | log4j.appender.A3.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n 33 | 34 | # Keep the last few log messages for display to the user 35 | log4j.appender.GUI=org.apache.accumulo.server.monitor.LogService 36 | log4j.appender.GUI.Keep=50 37 | log4j.appender.GUI.Threshold=WARN 38 | 39 | # Log accumulo messages to debug, normal and GUI 40 | log4j.logger.org.apache.accumulo=DEBUG, A2, A3, GUI 41 | log4j.additivity.org.apache.accumulo=false 42 | 43 | # Log non-accumulo messages to debug, normal logs. 44 | log4j.rootLogger=INFO, A2, A3 45 | -------------------------------------------------------------------------------- /accumulo/supervisor/accumulo.conf: -------------------------------------------------------------------------------- 1 | [program:accumulo-tserver] 2 | user=accumulo 3 | command=/usr/lib/accumulo/bin/accumulo tserver --address %(ENV_HOSTNAME)s 4 | autostart=false 5 | autorestart=true 6 | startsecs=2 7 | startretries=3 8 | priority=1000 9 | redirect_stderr=true 10 | stdout_logfile=/var/log/accumulo/tserver_%(ENV_HOSTNAME)s.out 11 | stdout_logfile_maxbytes=1MB 12 | stdout_logfile_backups=10 13 | stdout_capture_maxbytes=1MB 14 | 15 | [program:accumulo-master] 16 | user=accumulo 17 | command=/usr/lib/accumulo/bin/accumulo master --address %(ENV_HOSTNAME)s 18 | autostart=false 19 | autorestart=true 20 | startsecs=2 21 | startretries=3 22 | priority=1000 23 | redirect_stderr=true 24 | stdout_logfile=/var/log/accumulo/master_%(ENV_HOSTNAME)s.out 25 | stdout_logfile_maxbytes=1MB 26 | stdout_logfile_backups=10 27 | stdout_capture_maxbytes=1MB 28 | 29 | [program:accumulo-monitor] 30 | user=accumulo 31 | command=/usr/lib/accumulo/bin/accumulo monitor --address 0.0.0.0 32 | autostart=false 33 | autorestart=true 34 | startsecs=2 35 | startretries=3 36 | priority=1000 37 | redirect_stderr=true 38 | stdout_logfile=/var/log/accumulo/monitor_%(ENV_HOSTNAME)s.out 39 | stdout_logfile_maxbytes=1MB 40 | stdout_logfile_backups=10 41 | stdout_capture_maxbytes=1MB 42 | 43 | [program:accumulo-gc] 44 | user=accumulo 45 | command=/usr/lib/accumulo/bin/accumulo gc --address %(ENV_HOSTNAME)s 46 | autostart=false 47 | autorestart=true 48 | startsecs=2 49 | startretries=3 50 | priority=1000 51 | redirect_stderr=true 52 | stdout_logfile=/var/log/accumulo/gc_%(ENV_HOSTNAME)s.out 53 | stdout_logfile_maxbytes=1MB 54 | stdout_logfile_backups=10 55 | stdout_capture_maxbytes=1MB 56 | 57 | [program:accumulo-tracer] 58 | user=accumulo 59 | command=/usr/lib/accumulo/bin/accumulo tracer --address %(ENV_HOSTNAME)s 60 | autostart=false 61 | autorestart=true 62 | startsecs=2 63 | startretries=3 64 | priority=1000 65 | redirect_stderr=true 66 | stdout_logfile=/var/log/accumulo/tracer_%(ENV_HOSTNAME)s.out 67 | stdout_logfile_maxbytes=1MB 68 | stdout_logfile_backups=10 69 | stdout_capture_maxbytes=1MB 70 | 71 | [program:accumulo-proxy] 72 | user=accumulo 73 | command=/usr/lib/accumulo/bin/accumulo proxy -p /var/lib/accumulo/conf/proxy.properties 74 | autostart=false 75 | autorestart=true 76 | startsecs=2 77 | startretries=3 78 | priority=1000 79 | redirect_stderr=true 80 | stdout_logfile=/var/log/accumulo/proxy_%(ENV_HOSTNAME)s.out 81 | stdout_logfile_maxbytes=1MB 82 | stdout_logfile_backups=10 83 | stdout_capture_maxbytes=1MB 84 | 85 | -------------------------------------------------------------------------------- /accumulo/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tag=docker.io/sroegner/accumulo 4 | container_name=accumulo-cluster 5 | accumulo_net=accumulo-docker-local 6 | 7 | if [ "$1" = "-clean" -o "$1" = "shutdown" ] 8 | then 9 | echo "Shutting down all Accumulo containers ($1)" 10 | for c in namenode zookeeper tserver0 tserver1 tserver2 master proxy 11 | do 12 | docker rm -f ${container_name}-${c} 13 | done 14 | fi 15 | 16 | if [ "$1" = "shutdown" ] 17 | then 18 | exit 0 19 | fi 20 | 21 | run_container() { 22 | fqdn="${1}.docker.local" 23 | /usr/bin/docker run -d --net=${accumulo_net} \ 24 | --hostname=${fqdn} --net-alias=${fqdn} \ 25 | -e="SVCLIST=${2}" \ 26 | --name=${container_name}-${1} ${tag} \ 27 | /usr/bin/supervisord -n 28 | } 29 | 30 | # the cluster requires DNS to work which we use a custom docker network 31 | # with its embedded DNS server for 32 | $(/usr/bin/docker network inspect ${accumulo_net} &>/dev/null) || { 33 | # TODO: would be nice to check that the local docker client is new enough for this 34 | /usr/bin/docker network create -d bridge --subnet=172.25.10.0/24 $accumulo_net 35 | } 36 | 37 | echo "Welcome to Accumulo on Docker!" 38 | echo 39 | echo "starting hdfs" 40 | run_container namenode 'namenode,datanode' 41 | echo "start zookeeper" 42 | run_container zookeeper 'zookeeper' 43 | echo "hdfs needs a moment to become available" 44 | sleep 10 45 | echo "initializing Accumulo instance" 46 | docker exec ${container_name}-namenode /usr/lib/accumulo/bin/init_accumulo.sh 47 | echo "start tservers" 48 | run_container tserver0 'datanode,accumulo-tserver' 49 | run_container tserver1 'datanode,accumulo-tserver' 50 | run_container tserver2 'datanode,accumulo-tserver' 51 | echo "start accumulo master" 52 | run_container master 'accumulo-master,accumulo-monitor' 53 | echo "start accumulo proxy, gc and tracer" 54 | run_container proxy 'accumulo-tracer,accumulo-gc,accumulo-proxy' 55 | echo "wait for accumulo services to start" 56 | sleep 10 57 | echo create user alfred with password batman 58 | docker exec ${container_name}-tserver0 /tmp/add_user.sh alfred batman 59 | echo 60 | 61 | master_ip=$(docker inspect --format '{{(index .NetworkSettings.Networks "accumulo-docker-local").IPAddress}}' ${container_name}-master) 62 | echo "Accumulo Monitor is at http://${master_ip}:50095" 63 | echo -e "Login to accumulo with \n\t docker exec -u accumulo -it ${container_name}-tserver0 /usr/lib/accumulo/bin/accumulo shell -u alfred -p batman" 64 | echo 65 | -------------------------------------------------------------------------------- /doop/Makefile: -------------------------------------------------------------------------------- 1 | dnsdomain := docker.local 2 | NET := doop-local 3 | CIDR := 172.26.1.0/24 4 | 5 | include ../shared/Makefile.vars 6 | hostname_template := $(container_name).$(dnsdomain) 7 | include ../shared/Makefile 8 | INSPECT_ALIAS := $(DOCKER_CMD) inspect --format '{{(index .NetworkSettings.Networks "$(NET)").Aliases}}' 9 | INSPECT_ADDRESS := $(DOCKER_CMD) inspect --format '{{(index .NetworkSettings.Networks "$(NET)").IPAddress}}' 10 | 11 | image: 12 | cp -rv ../shared/config.zk $(CURDIR)/conf.zk 13 | $(DOCKER_CMD) build -t=$(tag) . 14 | 15 | cluster: check-network 16 | $(DOCKER_CMD) run -d --net=$(NET) --hostname=namenode.$(dnsdomain) --net-alias=namenode.$(dnsdomain) -e="SVCLIST=namenode,secondarynamenode" --name=$(container_name) $(tag) /usr/bin/supervisord -n 17 | $(DOCKER_CMD) run -d --net=$(NET) --hostname=resourcemanager.$(dnsdomain) --net-alias=resourcemanager.$(dnsdomain) -e="SVCLIST=resourcemanager" --name=$(container_name)-rm $(tag) /usr/bin/supervisord -n 18 | $(DOCKER_CMD) run -d --net=$(NET) --hostname=datanode0.$(dnsdomain) --net-alias=datanode0.$(dnsdomain) -e="SVCLIST=datanode,nodemanager" --name=$(container_name)-dn0 $(tag) /usr/bin/supervisord -n 19 | $(DOCKER_CMD) run -d --net=$(NET) --hostname=datanode1.$(dnsdomain) --net-alias=datanode1.$(dnsdomain) -e="SVCLIST=datanode,nodemanager" --name=$(container_name)-dn1 $(tag) /usr/bin/supervisord -n 20 | $(DOCKER_CMD) run -d --net=$(NET) --hostname=zookeeper.$(dnsdomain) --net-alias=zookeeper.$(dnsdomain) -e="SVCLIST=zookeeper" --name=$(container_name)-zk0 $(tag) /usr/bin/supervisord -n 21 | 22 | clean: 23 | @$(DOCKER_CMD) rm -f $(container_name)-zk0 || : 24 | @$(DOCKER_CMD) rm -f $(container_name)-rm || : 25 | @$(DOCKER_CMD) rm -f $(container_name)-dn0 || : 26 | @$(DOCKER_CMD) rm -f $(container_name)-dn1 || : 27 | @$(DOCKER_CMD) rm -f $(container_name) || : 28 | 29 | exec-nn: 30 | $(DOCKER_CMD) exec -i -t $(container_name) /bin/bash 31 | 32 | exec-rm: 33 | $(DOCKER_CMD) exec -i -t $(container_name)-rm /bin/bash 34 | 35 | exec-dn0: 36 | $(DOCKER_CMD) exec -i -t $(container_name)-dn0 /bin/bash 37 | 38 | exec-dn1: 39 | $(DOCKER_CMD) exec -i -t $(container_name)-dn1 /bin/bash 40 | 41 | info: 42 | @$(INSPECT_ALIAS) $(container_name) 43 | @$(INSPECT_ADDRESS) $(container_name) 44 | @$(INSPECT_ALIAS) $(container_name)-dn0 45 | @$(INSPECT_ADDRESS) $(container_name)-dn0 46 | @$(INSPECT_ALIAS) $(container_name)-dn1 47 | @$(INSPECT_ADDRESS) $(container_name)-dn1 48 | @$(INSPECT_ALIAS) $(container_name)-rm 49 | @$(INSPECT_ADDRESS) $(container_name)-rm 50 | @$(INSPECT_ALIAS) $(container_name)-zk0 51 | @$(INSPECT_ADDRESS) $(container_name)-zk0 52 | 53 | nn-logs: 54 | $(DOCKER_CMD) exec -i -t $(container_name) tail -40f /var/log/hadoop/hdfs-namenode.log 55 | 56 | -------------------------------------------------------------------------------- /accumulo/Makefile: -------------------------------------------------------------------------------- 1 | dnsdomain := docker.local 2 | -include ../shared/Makefile.vars 3 | -include ../shared/Makefile 4 | NET := accumulo-local 5 | CIDR := 172.26.2.0/24 6 | INSPECT_ADDRESS := $(DOCKER_CMD) inspect --format '{{(index .NetworkSettings.Networks "$(NET)").IPAddress}}' 7 | 8 | all: 9 | echo "Can do 'clean', 'image' or 'container'" 10 | 11 | image: 12 | $(DOCKER_CMD) build -t=$(tag) . 13 | 14 | cluster: check-network 15 | @echo Starting hdfs and zookeeper 16 | @$(DOCKER_CMD) run -d --net=$(NET) --hostname=namenode.$(dnsdomain) --net-alias=namenode.$(dnsdomain) -e="SVCLIST=namenode,datanode" --name=$(container_name) $(tag) /usr/bin/supervisord -n 17 | @$(DOCKER_CMD) run -d --net=$(NET) --hostname=zookeeper.$(dnsdomain) --net-alias=zookeeper.$(dnsdomain) -e="SVCLIST=zookeeper" --name=$(container_name)-zk0 $(tag) /usr/bin/supervisord -n 18 | @echo "Wait for hadoop to start" 19 | @sleep 15 20 | @echo "Initialize accumulo" 21 | @$(DOCKER_CMD) exec $(container_name) /usr/lib/accumulo/bin/init_accumulo.sh 22 | @$(DOCKER_CMD) run -d --net=$(NET) --hostname=master.$(dnsdomain) --net-alias=master.$(dnsdomain) -e="SVCLIST=accumulo-master,accumulo-monitor" --name=$(container_name)-master $(tag) /usr/bin/supervisord -n 23 | @$(DOCKER_CMD) run -d --net=$(NET) --hostname=tserver0.$(dnsdomain) --net-alias=tserver0.$(dnsdomain) -e="SVCLIST=datanode,accumulo-tserver" --name=$(container_name)-tserver0 $(tag) /usr/bin/supervisord -n 24 | @$(DOCKER_CMD) run -d --net=$(NET) --hostname=tserver1.$(dnsdomain) --net-alias=tserver1.$(dnsdomain) -e="SVCLIST=datanode,accumulo-tserver" --name=$(container_name)-tserver1 $(tag) /usr/bin/supervisord -n 25 | @$(DOCKER_CMD) run -d --net=$(NET) --hostname=proxy.$(dnsdomain) --net-alias=proxy.$(dnsdomain) -e="SVCLIST=accumulo-proxy,accumulo-tracer,accumulo--gc" --name=$(container_name)-proxy $(tag) /usr/bin/supervisord -n 26 | @echo "Wait for accumulo to start" 27 | @sleep 10 28 | @echo "Add a user" 29 | @$(DOCKER_CMD) exec $(container_name)-tserver0 /tmp/add_user.sh 30 | @echo "The Accumulo monitor page is here: http://$$($(INSPECT_ADDRESS) $(container_name)-master):50095" 31 | 32 | clean: 33 | @$(DOCKER_CMD) rm -f $(container_name)-proxy || : 34 | @$(DOCKER_CMD) rm -f $(container_name)-zk0 || : 35 | @$(DOCKER_CMD) rm -f $(container_name)-tserver0 || : 36 | @$(DOCKER_CMD) rm -f $(container_name)-tserver1 || : 37 | @$(DOCKER_CMD) rm -f $(container_name)-master || : 38 | @$(DOCKER_CMD) rm -f $(container_name) || : 39 | 40 | accumulo-rootshell: 41 | $(DOCKER_CMD) exec -it $(container_name)-tserver0 su - accumulo -c "/usr/lib/accumulo/bin/accumulo shell -u root -p DOCKERDEFAULT" 42 | 43 | accumulo-shell: 44 | $(DOCKER_CMD) exec -it $(container_name)-tserver0 su - accumulo -c "/usr/lib/accumulo/bin/accumulo shell -u bob -p robert" 45 | 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker Builds 2 | 3 | A collection of docker projects for daily use - generally testing. The general idea includes the use of a Makefile for interaction with and automated configuration of images and containers: 4 | 5 | * directory name is the box name (from which the image and container names are derived) 6 | * general make commands include 7 | * _image_: yes - make an image 8 | * _container_: create a new container 9 | * _ssh_: log into a running container (if supervisor/sshd are running) 10 | * _shell_: run a shell container using the current image 11 | * _logs_ 12 | * _clean_: stop and remove container(s), remove temporary files 13 | * _erase_: clean + remove the image 14 | 15 | Some of the projects have more make commands available. To avoid collisions as good as possible any container will only bind to one specific loopback IP. Worked into IP determination is - among other things - the environment variable EXECUTOR_NUMBER (defaulting to 0) which if set inside a jenkins job allows for parallel execution of the same container without bind conflicts. 16 | 17 | ## centos-base-ssh 18 | 19 | All other projects build on top of this. OpenJDK7, ssh (running on port 2212) on top of CentOS 6. 20 | 21 | ## zookeeper 22 | 23 | Just runs a single zookeeper process, no ssh. 24 | 25 | ## doop 26 | 27 | Fully configured Hortonworks HDP-2.2 (Hadoop 2.6) cluster on CentOS 7. Nodes (containers) are started from one and the same image, with the SVCLIST environment variable controlling which services should run in each container. 28 | Uses a dedicated consul.io container (started first) node discovery, service checks and dns. All (except the consul master) containers use supervisor to startup the consul agent and the listed services. 29 | There is no sshd - all ops interaction uses _docker exec_. 30 | 31 | ## accumulo 32 | 33 | Built on top of doop - provides an easy way to run an Accumulo cluster (currently 1.6.2) in docker. 34 | The choice of node distribution is relatively free as long as there are semantically named hosts (containers) for namenode and zookeeper. 35 | The Makefile is the easiest way to understand the idea of how this works - specifically the _cluster_ target in both - doop and accumulo. 36 | 37 | In addition to initialization the `make cluster` command will also create the example user bob with rights to create tables as described in the Accumulo manual. 38 | You can use `make accumulo-shell` and `make accumulo-rootshell` to login. 39 | 40 | ## nexus 41 | 42 | A single Sonatype Nexus java process, no ssh. Good for local maven testing - exposes port 8081. 43 | 44 | ## zookeeper 45 | 46 | A single zookeeper java process, no ssh. Exposes port 2181. 47 | 48 | ## consul-cluster 49 | 50 | Consul by HashiCorp! Try it in single-container or cluster mode. 51 | 52 | ## firefox 53 | 54 | You can run the firefox in this container via X11 forwarding - it runs on top of an 1.6 OpenJDK to be used with older Applets (like in Supermicro server IPMI web console) 55 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/generic_logger.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Write out everything at the DEBUG level to the debug log 17 | log4j.appender.A2=org.apache.log4j.RollingFileAppender 18 | log4j.appender.A2.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.debug.log 19 | log4j.appender.A2.MaxFileSize=1000MB 20 | log4j.appender.A2.MaxBackupIndex=10 21 | log4j.appender.A2.Threshold=DEBUG 22 | log4j.appender.A2.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.A2.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n 24 | 25 | # Write out INFO and higher to the regular log 26 | log4j.appender.A3=org.apache.log4j.RollingFileAppender 27 | log4j.appender.A3.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.log 28 | log4j.appender.A3.MaxFileSize=1000MB 29 | log4j.appender.A3.MaxBackupIndex=10 30 | log4j.appender.A3.Threshold=INFO 31 | log4j.appender.A3.layout=org.apache.log4j.PatternLayout 32 | log4j.appender.A3.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n 33 | 34 | # Send all logging data to a centralized logger 35 | # If the centralized logger is down, buffer the log events, but drop them if it stays down 36 | log4j.appender.ASYNC=org.apache.accumulo.core.util.AsyncSocketAppender 37 | log4j.appender.ASYNC.RemoteHost=${org.apache.accumulo.core.host.log} 38 | log4j.appender.ASYNC.Port=${org.apache.accumulo.core.host.log.port} 39 | log4j.appender.ASYNC.Application=${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname} 40 | log4j.appender.ASYNC.Threshold=WARN 41 | 42 | # Log accumulo events to the debug, normal and remote logs. 43 | log4j.logger.org.apache.accumulo=DEBUG, A2, A3, ASYNC 44 | log4j.additivity.org.apache.accumulo=false 45 | 46 | # change to INFO for authorization events 47 | log4j.logger.org.apache.accumulo.server.security.Auditor=WARN 48 | 49 | log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO 50 | 51 | log4j.logger.org.apache.accumulo.examples.wikisearch=INFO 52 | 53 | log4j.logger.org.mortbay.log=WARN 54 | 55 | log4j.logger.com.yahoo.zookeeper=ERROR 56 | 57 | # Log non-accumulo events to the debug and normal logs 58 | log4j.rootLogger=INFO, A2, A3 59 | -------------------------------------------------------------------------------- /minidoop/Makefile: -------------------------------------------------------------------------------- 1 | dnsdomain := docker.local 2 | 3 | include ../shared/Makefile.vars 4 | hostname_template := $(container_name).$(dnsdomain) 5 | include ../shared/Makefile 6 | 7 | image: 8 | cp -rv ../shared/config.zk $(CURDIR)/conf.zk 9 | $(DOCKER_CMD) build -t=$(tag) . 10 | 11 | cluster: 12 | $(DOCKER_CMD) run -d --dns-search=node.doop.local --name=consul-leader $(tag) /usr/sbin/consul agent -server -bootstrap-expect=1 -data-dir=/var/lib/consul/data -config-dir=/etc/consul-leader 13 | $(DOCKER_CMD) run -d --dns-search=node.doop.local --hostname=namenode --dns="$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' consul-leader)" -e="SVCLIST=namenode,secondarynamenode" --name=$(container_name) --link=consul-leader:consul-leader $(tag) /usr/bin/supervisord -n 14 | sleep 1 15 | $(DOCKER_CMD) run -d --dns-search=node.doop.local --hostname=resourcemanager --dns="$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' consul-leader)" -e="SVCLIST=resourcemanager" --name=$(container_name)-rm --link=consul-leader:consul-leader $(tag) /usr/bin/supervisord -n 16 | sleep 1 17 | $(DOCKER_CMD) run -d --dns-search=node.doop.local --hostname=datanode0 --dns="$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' consul-leader)" -e="SVCLIST=datanode,nodemanager" --name=$(container_name)-dn0 --link=consul-leader:consul-leader $(tag) /usr/bin/supervisord -n 18 | $(DOCKER_CMD) run -d --dns-search=node.doop.local --hostname=datanode1 --dns="$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' consul-leader)" -e="SVCLIST=datanode,nodemanager" --name=$(container_name)-dn1 --link=consul-leader:consul-leader $(tag) /usr/bin/supervisord -n 19 | $(DOCKER_CMD) run -d --dns-search=node.doop.local --hostname=zookeeper --dns="$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' consul-leader)" -e="SVCLIST=zookeeper" --name=$(container_name)-zk0 --link=consul-leader:consul-leader $(tag) /usr/bin/supervisord -n 20 | 21 | clean: 22 | $(DOCKER_CMD) rm -f consul-leader || : 23 | $(DOCKER_CMD) rm -f $(container_name)-zk0 || : 24 | $(DOCKER_CMD) rm -f $(container_name)-rm || : 25 | $(DOCKER_CMD) rm -f $(container_name)-dn0 || : 26 | $(DOCKER_CMD) rm -f $(container_name)-dn1 || : 27 | $(DOCKER_CMD) rm -f $(container_name) || : 28 | 29 | exec-consul: 30 | $(DOCKER_CMD) exec -i -t consul-leader /bin/bash 31 | 32 | exec-dn0: 33 | $(DOCKER_CMD) exec -i -t $(container_name)-dn0 /bin/bash 34 | 35 | exec-dn1: 36 | $(DOCKER_CMD) exec -i -t $(container_name)-dn1 /bin/bash 37 | 38 | info: 39 | $(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' consul-leader 40 | $(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name) 41 | $(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)-dn0 42 | $(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)-dn1 43 | $(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)-rm 44 | $(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' $(container_name)-zk0 45 | 46 | consul-logs: 47 | $(DOCKER_CMD) logs -f consul-leader 48 | 49 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/monitor_logger.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/accumulo-env.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | ### 19 | ### Configure these environment variables to point to your local installations. 20 | ### 21 | ### The functional tests require conditional values, so keep this style: 22 | ### 23 | ### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/lib/jvm/java 24 | ### 25 | ### 26 | ### Note that the -Xmx -Xms settings below require substantial free memory: 27 | ### you may want to use smaller values, especially when running everything 28 | ### on a single machine. 29 | ### 30 | if [ -z "$HADOOP_HOME" ] 31 | then 32 | test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/usr/hdp/current/hadoop-client 33 | else 34 | HADOOP_PREFIX="$HADOOP_HOME" 35 | unset HADOOP_HOME 36 | fi 37 | 38 | # hadoop-1.2: 39 | # test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf" 40 | # hadoop-2.0: 41 | test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="/etc/hadoop/conf" 42 | 43 | test -z "$JAVA_HOME" && export JAVA_HOME=/usr 44 | test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/usr/hdp/current/zookeeper-client 45 | test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs 46 | if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ] 47 | then 48 | POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy" 49 | fi 50 | test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m " 51 | test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m" 52 | test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m" 53 | test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m" 54 | test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true" 55 | test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m" 56 | # what do when the JVM runs out of heap memory 57 | export ACCUMULO_KILL_CMD='kill -9 %p' 58 | 59 | ### Optionally look for hadoop and accumulo native libraries for your 60 | ### platform in additional directories. (Use DYLD_LIBRARY_PATH on Mac OS X.) 61 | ### May not be necessary for Hadoop 2.x or using an RPM that installs to 62 | ### the correct system library directory. 63 | export LD_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${PLATFORM}:${LD_LIBRARY_PATH} 64 | 65 | # Should the monitor bind to all network interfaces -- default: false 66 | export ACCUMULO_MONITOR_BIND_ALL="true" 67 | -------------------------------------------------------------------------------- /doop/hadoop-group.conf: -------------------------------------------------------------------------------- 1 | [program:namenode] 2 | user=hdfs 3 | command=/usr/bin/hdfs namenode 4 | autostart=false 5 | autorestart=true 6 | startsecs=10 7 | startretries=3 8 | priority=1 9 | environment=HADOOP_LIBEXEC_DIR="/usr/hdp/current/hadoop-client/libexec",HADOOP_LOGFILE=hdfs-namenode.log,HADOOP_ROOT_LOGGER="INFO,RFA" 10 | redirect_stderr=true 11 | stdout_logfile=/var/log/hadoop/hdfs-namenode.out 12 | stdout_logfile_maxbytes=1MB 13 | stdout_logfile_backups=10 14 | stdout_capture_maxbytes=1MB 15 | 16 | [program:secondarynamenode] 17 | user=hdfs 18 | command=/usr/bin/hdfs secondarynamenode 19 | autostart=false 20 | autorestart=true 21 | startsecs=10 22 | startretries=3 23 | priority=2 24 | environment=HADOOP_LIBEXEC_DIR="/usr/hdp/current/hadoop-client/libexec",HADOOP_LOGFILE=hdfs-secondarynamenode.log,HADOOP_ROOT_LOGGER="INFO,RFA" 25 | redirect_stderr=true 26 | stdout_logfile=/var/log/hadoop/hdfs-secondarynamenode.out 27 | stdout_logfile_maxbytes=1MB 28 | stdout_logfile_backups=10 29 | stdout_capture_maxbytes=1MB 30 | 31 | [program:jobhistory] 32 | user=yarn 33 | command=/usr/bin/mapred --config /etc/hadoop/conf historyserver start 34 | autostart=false 35 | autorestart=true 36 | startsecs=10 37 | startretries=3 38 | priority=999 39 | environment=HADOOP_LIBEXEC_DIR="/usr/hdp/current/hadoop-client/libexec",HADOOP_MAPRED_LOGFILE="yarn-jobhistory.log",HADOOP_MAPRED_ROOT_LOGGER="INFO,RFA",HADOOP_MAPRED_LOG_DIR="/var/log/hadoop-yarn" 40 | redirect_stderr=true 41 | stdout_logfile=/var/log/supervisor/jobhistory.log 42 | stdout_logfile_maxbytes=1MB 43 | stdout_logfile_backups=10 44 | stdout_capture_maxbytes=1MB 45 | 46 | [program:datanode] 47 | user=hdfs 48 | command=/usr/bin/hdfs datanode 49 | autostart=false 50 | autorestart=true 51 | startsecs=10 52 | startretries=3 53 | priority=3 54 | environment=HADOOP_LIBEXEC_DIR="/usr/hdp/current/hadoop-client/libexec",HADOOP_LOGFILE=hdfs-datanode.log,HADOOP_ROOT_LOGGER="INFO,RFA" 55 | redirect_stderr=true 56 | stdout_logfile=/var/log/hadoop/hdfs-datanode.out 57 | stdout_logfile_maxbytes=1MB 58 | stdout_logfile_backups=10 59 | stdout_capture_maxbytes=1MB 60 | 61 | [program:nodemanager] 62 | user=yarn 63 | command=/usr/bin/yarn nodemanager 64 | autostart=false 65 | autorestart=true 66 | startsecs=10 67 | startretries=3 68 | priority=50 69 | environment=HADOOP_LIBEXEC_DIR="/usr/hdp/current/hadoop-client/libexec",YARN_LOGFILE="yarn-nodemanager.log",YARN_ROOT_LOGGER="INFO,RFA" 70 | redirect_stderr=true 71 | stdout_logfile=/var/log/hadoop/nodemanager.out 72 | stdout_logfile_maxbytes=1MB 73 | stdout_logfile_backups=10 74 | stdout_capture_maxbytes=1MB 75 | 76 | [program:resourcemanager] 77 | user=yarn 78 | command=/usr/bin/yarn resourcemanager 79 | autostart=false 80 | autorestart=true 81 | startsecs=10 82 | startretries=3 83 | priority=50 84 | environment=HADOOP_LIBEXEC_DIR="/usr/hdp/current/hadoop-client/libexec",YARN_LOGFILE="yarn-resourcemanager.log",YARN_ROOT_LOGGER="INFO,RFA",YARN_LOG_DIR="/var/log/hadoop-yarn" 85 | redirect_stderr=true 86 | stdout_logfile=/var/log/hadoop/resourcemanager.out 87 | stdout_logfile_maxbytes=1MB 88 | stdout_logfile_backups=10 89 | stdout_capture_maxbytes=1MB 90 | 91 | [program:zookeeper] 92 | user=zookeeper 93 | command=/usr/hdp/current/zookeeper-client/bin/zkServer.sh start-foreground 94 | autostart=false 95 | autorestart=true 96 | priority=20 97 | startsecs=10 98 | startretries=3 99 | redirect_stderr=true 100 | stdout_logfile=/var/log/zookeeper/zookeeper.out 101 | stdout_logfile_maxbytes=1MB 102 | stdout_logfile_backups=10 103 | stdout_capture_maxbytes=1MB 104 | 105 | -------------------------------------------------------------------------------- /consul-cluster/Makefile: -------------------------------------------------------------------------------- 1 | hostname := namenode 2 | 3 | include ../shared/Makefile.vars 4 | include ../shared/Makefile 5 | 6 | data_dir := /var/lib/consul/data 7 | ui_dir := /var/lib/consul/ui/dist 8 | 9 | clean: 10 | sudo docker rm -f $(container_name) || : 11 | sudo docker rm -f server1 || : 12 | sudo docker rm -f server2 || : 13 | sudo docker rm -f server3 || : 14 | sudo docker rm -f server4 || : 15 | sudo docker rm -f agent1 || : 16 | sudo docker rm -f agent2 || : 17 | sudo docker rm -f agent3 || : 18 | 19 | erase: clean 20 | sudo docker rmi $(tag) 21 | 22 | cluster: 23 | sudo docker run -d --env='SRV_ARGS=-server -bootstrap' --name=server1 --hostname=server1 -p=127.0.0.20:2212:2212 -p=127.0.0.20:8600:53 -p=127.0.0.20:8500:8500 $(tag) /usr/bin/supervisord -n 24 | sudo docker run -d --link=server1:peer --env='SRV_ARGS=-server -join=peer' --name=server2 --hostname=server2 -p=127.0.0.21:2212:2212 -p=127.0.0.21:8500:8500 $(tag) /usr/bin/supervisord -n 25 | sudo docker run -d --link=server2:peer --env='SRV_ARGS=-server -join=peer' --name=server3 --hostname=server3 -p=127.0.0.22:2212:2212 -p=127.0.0.22:8500:8500 $(tag) /usr/bin/supervisord -n 26 | sudo docker run -d --link=server3:peer --env='SRV_ARGS=-join=peer' --name=agent1 --hostname=agent1 -p=127.0.0.23:2212:2212 -p=127.0.0.23:8500:8500 $(tag) /usr/bin/supervisord -n 27 | sudo docker run -d --link=agent1:peer --env='SRV_ARGS=-join=peer' --name=agent2 --hostname=agent2 -p=127.0.0.24:2212:2212 -p=127.0.0.24:8500:8500 $(tag) /usr/bin/supervisord -n 28 | sudo docker run -d --link=agent2:peer --env='SRV_ARGS=-join=peer' --name=agent3 --hostname=agent3 -p=127.0.0.25:2212:2212 -p=127.0.0.25:8500:8500 $(tag) /usr/bin/supervisord -n 29 | 30 | cluster-noquorum2: 31 | sudo docker run -d --dns=localhost --env='SRV_ARGS=-server -bootstrap' --name=server1 --hostname=server1 -p=127.0.0.20:2212:2212 -p=127.0.0.20:8600:53 -p=127.0.0.20:8500:8500 $(tag) /usr/bin/supervisord -n 32 | sudo docker run -d --link=server1:peer --env='SRV_ARGS=-server -join=peer' --name=server2 --hostname=server2 -p=127.0.0.21:2212:2212 -p=127.0.0.21:8500:8500 $(tag) /usr/bin/supervisord -n 33 | sudo docker run -d --link=server1:peer --env='SRV_ARGS=-join=peer' --name=agent2 --hostname=agent2 -p=127.0.0.24:2212:2212 -p=127.0.0.24:8500:8500 $(tag) /usr/bin/supervisord -n 34 | sudo docker run -d --link=agent2:peer --env='SRV_ARGS=-join=peer' --name=agent3 --hostname=agent3 -p=127.0.0.25:2212:2212 -p=127.0.0.25:8500:8500 $(tag) /usr/bin/supervisord -n 35 | 36 | cluster-noquorum4: 37 | sudo docker run -d --dns=localhost --env='SRV_ARGS=-server -bootstrap' --name=server1 --hostname=server1 -p=127.0.0.20:2212:2212 -p=127.0.0.20:8600:53 -p=127.0.0.20:8500:8500 $(tag) /usr/bin/supervisord -n 38 | sudo docker run -d --link=server1:peer --env='SRV_ARGS=-server -join=peer' --name=server2 --hostname=server2 -p=127.0.0.21:2212:2212 -p=127.0.0.21:8500:8500 $(tag) /usr/bin/supervisord -n 39 | sudo docker run -d --link=server2:peer --env='SRV_ARGS=-server -join=peer' --name=server3 --hostname=server3 -p=127.0.0.22:2212:2212 -p=127.0.0.22:8500:8500 $(tag) /usr/bin/supervisord -n 40 | sudo docker run -d --link=server3:peer --env='SRV_ARGS=-server -join=peer' --name=server4 --hostname=server4 -p=127.0.0.26:2212:2212 -p=127.0.0.26:8500:8500 $(tag) /usr/bin/supervisord -n 41 | sudo docker run -d --link=server3:peer --env='SRV_ARGS=-join=peer' --name=agent1 --hostname=agent1 -p=127.0.0.23:2212:2212 -p=127.0.0.23:8500:8500 $(tag) /usr/bin/supervisord -n 42 | sudo docker run -d --link=agent1:peer --env='SRV_ARGS=-join=peer' --name=agent2 --hostname=agent2 -p=127.0.0.24:2212:2212 -p=127.0.0.24:8500:8500 $(tag) /usr/bin/supervisord -n 43 | 44 | server1: 45 | ssh $(sshopts) root@$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' server1) 46 | 47 | agent1: 48 | ssh $(sshopts) root@$$($(DOCKER_CMD) inspect --format '{{ .NetworkSettings.IPAddress }}' agent1) 49 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/generic_logger.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /doop/hadoop/yarn-env.sh: -------------------------------------------------------------------------------- 1 | 2 | export HADOOP_YARN_HOME=/usr/hdp/current/hadoop-yarn-client 3 | export YARN_LOG_DIR=/var/log/hadoop/$USER 4 | export YARN_PID_DIR=/var/run/hadoop/$USER 5 | export HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec 6 | export JAVA_HOME=/usr 7 | 8 | # User for YARN daemons 9 | export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} 10 | 11 | # resolve links - $0 may be a softlink 12 | export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" 13 | 14 | # some Java parameters 15 | # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ 16 | if [ "$JAVA_HOME" != "" ]; then 17 | #echo "run java in $JAVA_HOME" 18 | JAVA_HOME=$JAVA_HOME 19 | fi 20 | 21 | if [ "$JAVA_HOME" = "" ]; then 22 | echo "Error: JAVA_HOME is not set." 23 | exit 1 24 | fi 25 | 26 | JAVA=$JAVA_HOME/bin/java 27 | JAVA_HEAP_MAX=-Xmx1000m 28 | 29 | # For setting YARN specific HEAP sizes please use this 30 | # Parameter and set appropriately 31 | YARN_HEAPSIZE=1024 32 | 33 | # check envvars which might override default args 34 | if [ "$YARN_HEAPSIZE" != "" ]; then 35 | JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" 36 | fi 37 | 38 | # Resource Manager specific parameters 39 | 40 | # Specify the max Heapsize for the ResourceManager using a numerical value 41 | # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set 42 | # the value to 1000. 43 | # This value will be overridden by an Xmx setting specified in either YARN_OPTS 44 | # and/or YARN_RESOURCEMANAGER_OPTS. 45 | # If not specified, the default value will be picked from either YARN_HEAPMAX 46 | # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. 47 | export YARN_RESOURCEMANAGER_HEAPSIZE=1024 48 | 49 | # Specify the JVM options to be used when starting the ResourceManager. 50 | # These options will be appended to the options specified as YARN_OPTS 51 | # and therefore may override any similar flags set in YARN_OPTS 52 | #export YARN_RESOURCEMANAGER_OPTS= 53 | 54 | # Node Manager specific parameters 55 | 56 | # Specify the max Heapsize for the NodeManager using a numerical value 57 | # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set 58 | # the value to 1000. 59 | # This value will be overridden by an Xmx setting specified in either YARN_OPTS 60 | # and/or YARN_NODEMANAGER_OPTS. 61 | # If not specified, the default value will be picked from either YARN_HEAPMAX 62 | # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. 63 | export YARN_NODEMANAGER_HEAPSIZE=1024 64 | 65 | # Specify the max Heapsize for the HistoryManager using a numerical value 66 | # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set 67 | # the value to 1024. 68 | # This value will be overridden by an Xmx setting specified in either YARN_OPTS 69 | # and/or YARN_HISTORYSERVER_OPTS. 70 | # If not specified, the default value will be picked from either YARN_HEAPMAX 71 | # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. 72 | export YARN_HISTORYSERVER_HEAPSIZE=1024 73 | 74 | # Specify the JVM options to be used when starting the NodeManager. 75 | # These options will be appended to the options specified as YARN_OPTS 76 | # and therefore may override any similar flags set in YARN_OPTS 77 | #export YARN_NODEMANAGER_OPTS= 78 | 79 | # so that filenames w/ spaces are handled correctly in loops below 80 | IFS= 81 | 82 | 83 | # default log directory and file 84 | if [ "$YARN_LOG_DIR" = "" ]; then 85 | YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" 86 | fi 87 | if [ "$YARN_LOGFILE" = "" ]; then 88 | YARN_LOGFILE='yarn.log' 89 | fi 90 | 91 | # default policy file for service-level authorization 92 | if [ "$YARN_POLICYFILE" = "" ]; then 93 | YARN_POLICYFILE="hadoop-policy.xml" 94 | fi 95 | 96 | # restore ordinary behaviour 97 | unset IFS 98 | 99 | 100 | YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" 101 | YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" 102 | YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" 103 | YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" 104 | YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" 105 | YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" 106 | YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" 107 | YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" 108 | if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then 109 | YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" 110 | fi 111 | YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" 112 | -------------------------------------------------------------------------------- /doop/hadoop/task-log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | # 21 | 22 | 23 | # Define some default values that can be overridden by system properties 24 | hadoop.root.logger=INFO,console 25 | hadoop.log.dir=. 26 | hadoop.log.file=hadoop.log 27 | 28 | # 29 | # Job Summary Appender 30 | # 31 | # Use following logger to send summary to separate file defined by 32 | # hadoop.mapreduce.jobsummary.log.file rolled daily: 33 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 34 | # 35 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 36 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 37 | 38 | # Define the root logger to the system property "hadoop.root.logger". 39 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 40 | 41 | # Logging Threshold 42 | log4j.threshhold=ALL 43 | 44 | # 45 | # Daily Rolling File Appender 46 | # 47 | 48 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 49 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 50 | 51 | # Rollver at midnight 52 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 53 | 54 | # 30-day backup 55 | #log4j.appender.DRFA.MaxBackupIndex=30 56 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 57 | 58 | # Pattern format: Date LogLevel LoggerName LogMessage 59 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 60 | # Debugging Pattern format 61 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 62 | 63 | 64 | # 65 | # console 66 | # Add "console" to rootlogger above if you want to use this 67 | # 68 | 69 | log4j.appender.console=org.apache.log4j.ConsoleAppender 70 | log4j.appender.console.target=System.err 71 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 72 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 73 | 74 | # 75 | # TaskLog Appender 76 | # 77 | 78 | #Default values 79 | hadoop.tasklog.taskid=null 80 | hadoop.tasklog.iscleanup=false 81 | hadoop.tasklog.noKeepSplits=4 82 | hadoop.tasklog.totalLogFileSize=100 83 | hadoop.tasklog.purgeLogSplits=true 84 | hadoop.tasklog.logsRetainHours=12 85 | 86 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 87 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 88 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 89 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 90 | 91 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 92 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 93 | 94 | # 95 | # Rolling File Appender 96 | # 97 | 98 | #log4j.appender.RFA=org.apache.log4j.RollingFileAppender 99 | #log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 100 | 101 | # Logfile size and and 30-day backups 102 | #log4j.appender.RFA.MaxFileSize=1MB 103 | #log4j.appender.RFA.MaxBackupIndex=30 104 | 105 | #log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 106 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n 107 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 108 | 109 | 110 | # Custom Logging levels 111 | 112 | hadoop.metrics.log.level=INFO 113 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 114 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 115 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 116 | log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} 117 | 118 | # Jets3t library 119 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 120 | 121 | # 122 | # Null Appender 123 | # Trap security logger on the hadoop client side 124 | # 125 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 126 | 127 | # 128 | # Event Counter Appender 129 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 130 | # 131 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 132 | 133 | # Removes "deprecated" messages 134 | log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 135 | -------------------------------------------------------------------------------- /doop/hadoop/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dfs.namenode.avoid.write.stale.datanode 5 | true 6 | 7 | 8 | dfs.namenode.startup.delay.block.deletion.sec 9 | 3600 10 | 11 | 12 | dfs.datanode.handler.count 13 | 128 14 | 15 | 16 | dfs.namenode.checkpoint.txns 17 | 1000000 18 | 19 | 20 | dfs.webhdfs.enabled 21 | false 22 | 23 | 24 | dfs.block.access.token.enable 25 | true 26 | 27 | 28 | dfs.support.append 29 | true 30 | 31 | 32 | dfs.cluster.administrators 33 | hdfs 34 | 35 | 36 | dfs.datanode.balance.bandwidthPerSec 37 | 6250000 38 | 39 | 40 | dfs.namenode.safemode.threshold-pct 41 | 1.0f 42 | 43 | 44 | dfs.namenode.checkpoint.edits.dir 45 | ${dfs.namenode.checkpoint.dir} 46 | 47 | 48 | dfs.permissions.enabled 49 | true 50 | 51 | 52 | dfs.client.read.shortcircuit 53 | true 54 | 55 | 56 | dfs.datanode.synconclose 57 | true 58 | 59 | 60 | dfs.journalnode.edits.dir 61 | /hadoop/hdfs/journalnode 62 | 63 | 64 | dfs.blocksize 65 | 134217728 66 | 67 | 68 | dfs.datanode.max.transfer.threads 69 | 1024 70 | 71 | 72 | dfs.heartbeat.interval 73 | 3 74 | 75 | 76 | dfs.replication 77 | 1 78 | 79 | 80 | dfs.namenode.handler.count 81 | 128 82 | 83 | 84 | dfs.namenode.checkpoint.dir 85 | file:///data1/hdfs/namesecondary 86 | 87 | 88 | fs.permissions.umask-mode 89 | 022 90 | 91 | 92 | dfs.namenode.name.dir 93 | file:///data1/hdfs/namenode 94 | 95 | 96 | dfs.durable.sync 97 | true 98 | 99 | 100 | dfs.datanode.data.dir 101 | file:///data1/hdfs/data 102 | 103 | 104 | dfs.blockreport.initialDelay 105 | 120 106 | 107 | 108 | dfs.datanode.failed.volumes.tolerated 109 | 0 110 | 111 | 112 | dfs.namenode.accesstime.precision 113 | 0 114 | 115 | 116 | dfs.namenode.write.stale.datanode.ratio 117 | 1.0f 118 | 119 | 120 | dfs.namenode.rpc-bind-host 121 | 0.0.0.0 122 | 123 | 124 | dfs.namenode.stale.datanode.interval 125 | 30000 126 | 127 | 128 | dfs.datanode.du.reserved 129 | 1073741824 130 | 131 | 132 | dfs.client.read.shortcircuit.streams.cache.size 133 | 4096 134 | 135 | 136 | dfs.datanode.sync.behind.writes 137 | true 138 | 139 | 140 | dfs.http.policy 141 | HTTP_ONLY 142 | 143 | 144 | dfs.permissions.superusergroup 145 | hdfs-super 146 | 147 | 148 | dfs.https.port 149 | 50470 150 | 151 | 152 | dfs.domain.socket.path 153 | /var/lib/hadoop-hdfs/dn_socket 154 | 155 | 156 | dfs.namenode.avoid.read.stale.datanode 157 | true 158 | 159 | 160 | dfs.datanode.data.dir.perm 161 | 750 162 | 163 | 164 | dfs.namenode.name.dir.restore 165 | true 166 | 167 | 168 | dfs.replication.max 169 | 50 170 | 171 | 172 | dfs.namenode.checkpoint.period 173 | 21600 174 | 175 | 176 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/accumulo-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 23 | 24 | 25 | instance.zookeeper.host 26 | zookeeper.docker.local:2181 27 | comma separated list of zookeeper servers 28 | 29 | 30 | 31 | logger.dir.walog 32 | walogs 33 | The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local 34 | filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4 35 | write ahead logs into DFS. It is possible to specify a comma-separated list of directories. 36 | 37 | 38 | 39 | 40 | instance.secret 41 | DEFAULT 42 | A secret unique to a given instance that all servers must know in order to communicate with one another. 43 | Change it before initialization. To 44 | change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd], 45 | and then update this file. 46 | 47 | 48 | 49 | 50 | tserver.memory.maps.max 51 | 256M 52 | 53 | 54 | 55 | tserver.memory.maps.native.enabled 56 | true 57 | 58 | 59 | 60 | tserver.cache.data.size 61 | 15M 62 | 63 | 64 | 65 | tserver.cache.index.size 66 | 40M 67 | 68 | 69 | 70 | trace.token.property.password 71 | DOCKERDEFAULT 72 | 73 | 74 | 75 | trace.user 76 | root 77 | 78 | 79 | 80 | tserver.sort.buffer.size 81 | 50M 82 | 83 | 84 | 85 | tserver.walog.max.size 86 | 256M 87 | 88 | 89 | 90 | general.classpaths 91 | 92 | 93 | $HADOOP_CONF_DIR, 94 | $ACCUMULO_HOME/lib/[^.].*.jar, 95 | 96 | $HADOOP_PREFIX/share/hadoop/common/.*.jar, 97 | $HADOOP_PREFIX/share/hadoop/common/lib/(?!\.)(?!slf4j-log4j).*\.jar, 98 | $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar, 99 | $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar, 100 | $HADOOP_PREFIX/share/hadoop/yarn/.*.jar, 101 | $HADOOP_PREFIX/share/hadoop/hdfs/lib/.*.jar, 102 | $HADOOP_PREFIX/share/hadoop/mapreduce/lib/.*.jar, 103 | $HADOOP_PREFIX/share/hadoop/yarn/lib/.*.jar, 104 | 105 | $HADOOP_PREFIX/../hadoop/.*.jar, 106 | $HADOOP_PREFIX/../hadoop/lib/(?!\.)(?!slf4j-log4j).*\.jar, 107 | $HADOOP_PREFIX/../hadoop-hdfs/.*.jar, 108 | $HADOOP_PREFIX/../hadoop-hdfs/lib/.*.jar, 109 | $HADOOP_PREFIX/../hadoop-mapreduce/.*.jar, 110 | $HADOOP_PREFIX/../hadoop-mapreduce/lib/*.*.jar, 111 | $HADOOP_PREFIX/../hadoop-yarn/.*.jar, 112 | $HADOOP_PREFIX/../hadoop-yarn/lib/.*.jar 113 | 114 | /usr/hdp/current/hadoop-client/.*.jar, 115 | /usr/hdp/current/hadoop-client/lib/.*.jar, 116 | /usr/hdp/current/hadoop-hdfs-client/.*.jar, 117 | /usr/hdp/current/hadoop-hdfs-client/lib/.*.jar 118 | /usr/hdp/current/hadoop-mapreduce-client/.*.jar, 119 | /usr/hdp/current/hadoop-mapreduce-client/lib/.*.jar 120 | /usr/hdp/current/hadoop-yarn-client/.*.jar, 121 | /usr/hdp/current/hadoop-yarn-client/lib/.*.jar 122 | 123 | Classpaths that accumulo checks for updates and class files. 124 | 125 | 126 | -------------------------------------------------------------------------------- /doop/hadoop/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mapreduce.cluster.administrators 5 | hadoop 6 | 7 | 8 | mapreduce.reduce.input.buffer.percent 9 | 0.0 10 | 11 | 12 | mapreduce.output.fileoutputformat.compress 13 | false 14 | 15 | 16 | mapreduce.framework.name 17 | yarn 18 | 19 | 20 | mapreduce.map.speculative 21 | false 22 | 23 | 24 | mapreduce.reduce.shuffle.merge.percent 25 | 0.66 26 | 27 | 28 | yarn.app.mapreduce.am.resource.mb 29 | 1024 30 | 31 | 32 | mapreduce.map.java.opts 33 | -Xmx546m 34 | 35 | 36 | mapreduce.application.classpath 37 | $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*,$PWD/mr-framework/hadoop/share/hadoop/common/*,$PWD/mr-framework/hadoop/share/hadoop/common/lib/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*,/etc/hadoop/conf,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/* 38 | 39 | 40 | mapreduce.job.reduce.slowstart.completedmaps 41 | 0.05 42 | 43 | 44 | mapreduce.output.fileoutputformat.compress.type 45 | BLOCK 46 | 47 | 48 | mapreduce.reduce.speculative 49 | false 50 | 51 | 52 | mapreduce.reduce.java.opts 53 | -Xmx546m 54 | 55 | 56 | mapreduce.am.max-attempts 57 | 2 58 | 59 | 60 | yarn.app.mapreduce.am.admin-command-opts 61 | -Dhdp.version=${hdp.version} 62 | 63 | 64 | mapreduce.reduce.log.level 65 | INFO 66 | 67 | 68 | mapreduce.map.sort.spill.percent 69 | 0.7 70 | 71 | 72 | mapreduce.job.emit-timeline-data 73 | false 74 | 75 | 76 | mapreduce.task.io.sort.mb 77 | 273 78 | 79 | 80 | mapreduce.task.timeout 81 | 300000 82 | 83 | 84 | mapreduce.map.memory.mb 85 | 682 86 | 87 | 88 | mapreduce.task.io.sort.factor 89 | 100 90 | 91 | 92 | mapreduce.jobhistory.intermediate-done-dir 93 | /mr-history/tmp 94 | 95 | 96 | mapreduce.reduce.memory.mb 97 | 682 98 | 99 | 100 | yarn.app.mapreduce.am.log.level 101 | INFO 102 | 103 | 104 | mapreduce.map.log.level 105 | INFO 106 | 107 | 108 | mapreduce.shuffle.port 109 | 13562 110 | 111 | 112 | mapreduce.reduce.shuffle.fetch.retry.timeout-ms 113 | 30000 114 | 115 | 116 | mapreduce.admin.user.env 117 | LD_LIBRARY_PATH=$HADOOP_PREFIX/lib/native 118 | 119 | 120 | mapreduce.map.output.compress 121 | false 122 | 123 | 124 | mapreduce.reduce.shuffle.parallelcopies 125 | 30 126 | 127 | 128 | mapreduce.reduce.shuffle.input.buffer.percent 129 | 0.7 130 | 131 | 132 | yarn.app.mapreduce.am.staging-dir 133 | /user 134 | 135 | 136 | mapreduce.jobhistory.done-dir 137 | /mr-history/done 138 | 139 | 140 | mapreduce.admin.reduce.child.java.opts 141 | -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} 142 | 143 | 144 | mapreduce.reduce.shuffle.fetch.retry.enabled 145 | 1 146 | 147 | 148 | mapreduce.application.framework.path 149 | /hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework 150 | 151 | 152 | yarn.app.mapreduce.am.command-opts 153 | -Xmx546m -Dhdp.version=${hdp.version} 154 | 155 | 156 | mapreduce.reduce.shuffle.fetch.retry.interval-ms 157 | 1000 158 | 159 | 160 | mapreduce.jobhistory.bind-host 161 | 0.0.0.0 162 | 163 | 164 | mapreduce.admin.map.child.java.opts 165 | -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} 166 | 167 | 168 | -------------------------------------------------------------------------------- /doop/hadoop/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | 2 | # Set Hadoop-specific environment variables here. 3 | 4 | # The only required environment variable is JAVA_HOME. All others are 5 | # optional. When running a distributed configuration it is best to 6 | # set JAVA_HOME in this file, so that it is correctly defined on 7 | # remote nodes. 8 | 9 | # The java implementation to use. Required. 10 | export JAVA_HOME=/usr 11 | export HADOOP_HOME_WARN_SUPPRESS=1 12 | 13 | # Hadoop home directory 14 | export HADOOP_HOME=${HADOOP_HOME:-/usr/hdp/current/hadoop-client} 15 | 16 | # Hadoop Configuration Directory 17 | 18 | 19 | # Path to jsvc required by secure HDP 2.0 datanode 20 | export JSVC_HOME=/usr/lib/bigtop-utils 21 | 22 | 23 | # The maximum amount of heap to use, in MB. Default is 1000. 24 | export HADOOP_HEAPSIZE="1024" 25 | 26 | export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m" 27 | 28 | # Extra Java runtime options. Empty by default. 29 | export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" 30 | 31 | # Command specific options appended to HADOOP_OPTS when specified 32 | export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}" 33 | HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" 34 | 35 | HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" 36 | export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}" 37 | HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}" 38 | 39 | export HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS 40 | 41 | # The following applies to multiple commands (fs, dfs, fsck, distcp etc) 42 | export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS" 43 | 44 | # On secure datanodes, user to run the datanode as after dropping privileges 45 | export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-""} 46 | 47 | # Extra ssh options. Empty by default. 48 | export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" 49 | 50 | # Where log files are stored. $HADOOP_HOME/logs by default. 51 | export HADOOP_LOG_DIR=/var/log/hadoop/$USER 52 | 53 | # History server logs 54 | export HADOOP_MAPRED_LOG_DIR=/var/log/hadoop-mapreduce/$USER 55 | 56 | # Where log files are stored in the secure data environment. 57 | export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop/$HADOOP_SECURE_DN_USER 58 | 59 | # File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. 60 | # export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves 61 | 62 | # host:path where hadoop code should be rsync'd from. Unset by default. 63 | # export HADOOP_MASTER=master:/home/$USER/src/hadoop 64 | 65 | # Seconds to sleep between slave commands. Unset by default. This 66 | # can be useful in large clusters, where, e.g., slave rsyncs can 67 | # otherwise arrive faster than the master can service them. 68 | # export HADOOP_SLAVE_SLEEP=0.1 69 | 70 | # The directory where pid files are stored. /tmp by default. 71 | export HADOOP_PID_DIR=/var/run/hadoop/$USER 72 | export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER 73 | 74 | # History server pid 75 | export HADOOP_MAPRED_PID_DIR=/var/run/hadoop-mapreduce/$USER 76 | 77 | YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" 78 | 79 | # A string representing this instance of hadoop. $USER by default. 80 | export HADOOP_IDENT_STRING=$USER 81 | 82 | # The scheduling priority for daemon processes. See 'man nice'. 83 | 84 | # export HADOOP_NICENESS=10 85 | 86 | # Use libraries from standard classpath 87 | JAVA_JDBC_LIBS="" 88 | #Add libraries required by mysql connector 89 | for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null` 90 | do 91 | JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile 92 | done 93 | # Add libraries required by oracle connector 94 | for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null` 95 | do 96 | JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile 97 | done 98 | # Add libraries required by nodemanager 99 | MAPREDUCE_LIBS=/usr/hdp/current/hadoop-mapreduce-client/* 100 | export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} 101 | 102 | # added to the HADOOP_CLASSPATH 103 | if [ -d "/usr/hdp/current/tez-client" ]; then 104 | if [ -d "/etc/tez/conf/" ]; then 105 | # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP. 106 | export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/ 107 | fi 108 | fi 109 | 110 | # Setting path to hdfs command line 111 | export HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec 112 | 113 | # Mostly required for hadoop 2.0 114 | export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH} 115 | 116 | export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" 117 | -------------------------------------------------------------------------------- /accumulo/accumulo.conf/accumulo.policy.example: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | grant codeBase "file:${java.home}/lib/ext/*" { 19 | permission java.security.AllPermission; 20 | }; 21 | 22 | // These should all be empty in a fielded system 23 | grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/server/target/classes/" { 24 | permission java.security.AllPermission; 25 | }; 26 | grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/core/target/classes/" { 27 | permission java.security.AllPermission; 28 | }; 29 | grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/start/target/classes/" { 30 | permission java.security.AllPermission; 31 | }; 32 | grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/examples/target/classes/" { 33 | permission java.security.AllPermission; 34 | }; 35 | 36 | grant codebase "file:${hadoop.home.dir}/*" { 37 | permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; 38 | permission java.lang.RuntimePermission "shutdownHooks"; // hadoop libs use executables to discover usernames, groups, etc. 39 | permission java.lang.RuntimePermission "loadLibrary.*"; 40 | permission java.io.FilePermission "<>", "read, execute"; 41 | permission java.io.FilePermission "/tmp", "write, delete"; 42 | permission java.io.FilePermission "/tmp/-", "write, delete"; 43 | permission java.io.FilePermission "/", "write"; 44 | permission java.net.SocketPermission "*", "connect, resolve"; 45 | permission java.util.PropertyPermission "java.library.path", "read"; 46 | permission java.util.PropertyPermission "user.dir", "read"; 47 | permission java.util.PropertyPermission "org.apache.commons.logging.*", "read"; 48 | permission java.util.PropertyPermission "entityExpansionLimit", "read"; 49 | permission java.util.PropertyPermission "maxOccurLimit", "read"; 50 | permission java.util.PropertyPermission "os.name", "read"; 51 | }; 52 | 53 | grant codebase "file:${hadoop.home.dir}/lib/*" { 54 | // monitor's jetty web service 55 | permission java.security.SecurityPermission "configurationPermission"; 56 | permission java.security.SecurityPermission "tablesPermission"; 57 | permission java.security.SecurityPermission "zookeeperWriterPermission"; 58 | permission java.security.SecurityPermission "tableManagerPermission"; 59 | permission java.security.SecurityPermission "transportPoolPermission"; 60 | permission java.security.SecurityPermission "systemCredentialsPermission"; 61 | permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; 62 | // need to accept web requests, and talk to job tracker, name node, etc. 63 | permission java.net.SocketPermission "*", "accept, listen, resolve, connect, resolve"; 64 | permission java.lang.RuntimePermission "getenv.*"; 65 | permission java.lang.RuntimePermission "loadLibrary.*"; 66 | permission java.util.PropertyPermission "org.mortbay.*", "read"; 67 | permission java.util.PropertyPermission "VERBOSE", "read"; 68 | permission java.util.PropertyPermission "IGNORED", "read"; 69 | permission java.util.PropertyPermission "ISO_8859_1", "read"; 70 | permission java.util.PropertyPermission "org.apache.commons.logging.*", "read"; 71 | permission java.util.PropertyPermission "accumulo.*", "read"; 72 | permission java.util.PropertyPermission "org.jfree.*", "read"; 73 | permission java.util.PropertyPermission "elementAttributeLimit", "read"; 74 | permission java.util.PropertyPermission "entityExpansionLimit", "read"; 75 | permission java.util.PropertyPermission "maxOccurLimit", "read"; 76 | // some resources come out of accumulo jars 77 | permission java.lang.RuntimePermission "getClassLoader"; 78 | permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/lib/*", "read"; 79 | permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/src/-", "read"; 80 | permission java.io.FilePermission "${hadoop.home.dir}/lib/*", "read"; 81 | // images are cached in /tmp 82 | permission java.io.FilePermission "/tmp/*", "read, write"; 83 | permission java.io.FilePermission "/", "write"; 84 | }; 85 | 86 | grant codebase "file:${zookeeper.home.dir}/*" { 87 | permission java.net.SocketPermission "*", "connect, resolve"; 88 | permission java.util.PropertyPermission "user.*", "read"; 89 | permission java.util.PropertyPermission "java.*", "read"; 90 | permission java.util.PropertyPermission "zookeeper.*", "read"; 91 | permission java.util.PropertyPermission "jute.*", "read"; 92 | permission java.util.PropertyPermission "os.*", "read"; 93 | // accumulo properties read in callbacks 94 | permission java.util.PropertyPermission "accumulo.*", "read"; 95 | permission java.security.SecurityPermission "configurationPermission"; 96 | permission java.security.SecurityPermission "tablesPermission"; 97 | permission java.security.SecurityPermission "zookeeperWriterPermission"; 98 | permission java.security.SecurityPermission "tableManagerPermission"; 99 | permission java.security.SecurityPermission "transportPoolPermission"; 100 | permission java.security.SecurityPermission "systemCredentialsPermission"; 101 | permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; 102 | permission java.lang.RuntimePermission "exitVM"; 103 | }; 104 | 105 | grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/ext/*" { 106 | }; 107 | 108 | grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/*" { 109 | permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; 110 | // logging, configuration and getting user id 111 | permission java.io.FilePermission "<>", "read, write, execute, delete"; 112 | permission java.util.PropertyPermission "*", "read, write"; 113 | permission java.lang.RuntimePermission "getenv.*"; 114 | permission java.lang.RuntimePermission "getClassLoader"; 115 | permission java.lang.RuntimePermission "loadLibrary.*"; 116 | permission java.lang.RuntimePermission "accessDeclaredMembers"; 117 | permission java.lang.RuntimePermission "selectorProvider"; 118 | permission java.lang.RuntimePermission "accessClassInPackage.*"; 119 | permission java.lang.RuntimePermission "readFileDescriptor"; 120 | permission java.lang.RuntimePermission "writeFileDescriptor"; 121 | permission java.lang.RuntimePermission "modifyThread"; 122 | permission java.lang.RuntimePermission "modifyThreadGroup"; 123 | permission java.lang.RuntimePermission "createClassLoader"; 124 | permission java.lang.RuntimePermission "setContextClassLoader"; 125 | permission java.lang.RuntimePermission "exitVM"; 126 | permission java.lang.RuntimePermission "shutdownHooks"; 127 | permission java.security.SecurityPermission "getPolicy"; 128 | permission java.security.SecurityPermission "getProperty.*"; 129 | permission java.security.SecurityPermission "putProviderProperty.*"; 130 | permission java.security.SecurityPermission "setSystemScope"; 131 | permission java.security.SecurityPermission "configurationPermission"; 132 | permission java.security.SecurityPermission "tablesPermission"; 133 | permission java.security.SecurityPermission "zookeeperWriterPermission"; 134 | permission java.security.SecurityPermission "tableManagerPermission"; 135 | permission java.security.SecurityPermission "transportPoolPermission"; 136 | permission java.security.SecurityPermission "systemCredentialsPermission"; 137 | permission java.util.logging.LoggingPermission "control"; 138 | permission java.net.NetPermission "getProxySelector"; 139 | permission javax.management.MBeanServerPermission "createMBeanServer"; 140 | permission javax.management.MBeanTrustPermission "register"; 141 | permission javax.management.MBeanPermission "*", "registerMBean"; 142 | permission java.net.SocketPermission "*", "accept, connect, listen, resolve"; 143 | }; 144 | -------------------------------------------------------------------------------- /doop/hadoop/log4j.properties: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | 22 | # Define some default values that can be overridden by system properties 23 | # To change daemon root logger use hadoop_root_logger in hadoop-env 24 | hadoop.root.logger=INFO,console 25 | hadoop.log.dir=. 26 | hadoop.log.file=hadoop.log 27 | 28 | 29 | # Define the root logger to the system property "hadoop.root.logger". 30 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 31 | 32 | # Logging Threshold 33 | log4j.threshhold=ALL 34 | 35 | # 36 | # Daily Rolling File Appender 37 | # 38 | 39 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 40 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 41 | 42 | # Rollver at midnight 43 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 44 | 45 | # 30-day backup 46 | #log4j.appender.DRFA.MaxBackupIndex=30 47 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 48 | 49 | # Pattern format: Date LogLevel LoggerName LogMessage 50 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 51 | # Debugging Pattern format 52 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 53 | 54 | 55 | # 56 | # console 57 | # Add "console" to rootlogger above if you want to use this 58 | # 59 | 60 | log4j.appender.console=org.apache.log4j.ConsoleAppender 61 | log4j.appender.console.target=System.err 62 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 63 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 64 | 65 | # 66 | # TaskLog Appender 67 | # 68 | 69 | #Default values 70 | hadoop.tasklog.taskid=null 71 | hadoop.tasklog.iscleanup=false 72 | hadoop.tasklog.noKeepSplits=4 73 | hadoop.tasklog.totalLogFileSize=100 74 | hadoop.tasklog.purgeLogSplits=true 75 | hadoop.tasklog.logsRetainHours=12 76 | 77 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 78 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 79 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 80 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 81 | 82 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 83 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 84 | 85 | # 86 | #Security audit appender 87 | # 88 | hadoop.security.logger=INFO,console 89 | hadoop.security.log.maxfilesize=256MB 90 | hadoop.security.log.maxbackupindex=20 91 | log4j.category.SecurityLogger=${hadoop.security.logger} 92 | hadoop.security.log.file=SecurityAuth.audit 93 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 94 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 95 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 96 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 97 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 98 | 99 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 100 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 101 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 102 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 103 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 104 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 105 | 106 | # 107 | # hdfs audit logging 108 | # 109 | hdfs.audit.logger=INFO,console 110 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 111 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 112 | log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender 113 | log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 114 | log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout 115 | log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 116 | log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd 117 | 118 | # 119 | # mapred audit logging 120 | # 121 | mapred.audit.logger=INFO,console 122 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 123 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 124 | log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender 125 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 126 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 127 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 128 | log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd 129 | 130 | # 131 | # Rolling File Appender 132 | # 133 | 134 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 135 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 136 | 137 | # Logfile size and and 30-day backups 138 | log4j.appender.RFA.MaxFileSize=256MB 139 | log4j.appender.RFA.MaxBackupIndex=10 140 | 141 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 142 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n 143 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 144 | 145 | 146 | # Custom Logging levels 147 | 148 | hadoop.metrics.log.level=INFO 149 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 150 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 151 | #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG 152 | log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} 153 | 154 | # Jets3t library 155 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 156 | 157 | # 158 | # Null Appender 159 | # Trap security logger on the hadoop client side 160 | # 161 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 162 | 163 | # 164 | # Event Counter Appender 165 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 166 | # 167 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 168 | 169 | # Removes "deprecated" messages 170 | log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 171 | 172 | # 173 | # HDFS block state change log from block manager 174 | # 175 | # Uncomment the following to suppress normal block state change 176 | # messages from BlockManager in NameNode. 177 | #log4j.logger.BlockStateChange=WARN 178 | #Relative to Yarn Log Dir Prefix 179 | yarn.log.dir=. 180 | # 181 | # Job Summary Appender 182 | # 183 | # Use following logger to send summary to separate file defined by 184 | # hadoop.mapreduce.jobsummary.log.file rolled daily: 185 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 186 | # 187 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 188 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 189 | log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender 190 | # Set the ResourceManager summary log filename 191 | yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log 192 | # Set the ResourceManager summary log level and appender 193 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 194 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 195 | 196 | # To enable AppSummaryLogging for the RM, 197 | # set yarn.server.resourcemanager.appsummary.logger to 198 | # LEVEL,RMSUMMARY in hadoop-env.sh 199 | 200 | # Appender for ResourceManager Application Summary Log 201 | # Requires the following properties to be set 202 | # - hadoop.log.dir (Hadoop Log directory) 203 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 204 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 205 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 206 | log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 207 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 208 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 209 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 210 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 211 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 212 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 213 | log4j.appender.JSA.DatePattern=.yyyy-MM-dd 214 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 215 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 216 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false -------------------------------------------------------------------------------- /doop/hadoop/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.client.nodemanager-connect.max-wait-ms 5 | 900000 6 | 7 | 8 | yarn.resourcemanager.hostname 9 | resourcemanager.docker.local 10 | 11 | 12 | yarn.resourcemanager.zk-num-retries 13 | 1000 14 | 15 | 16 | yarn.timeline-service.bind-host 17 | 0.0.0.0 18 | 19 | 20 | yarn.resourcemanager.ha.enabled 21 | false 22 | 23 | 24 | yarn.nodemanager.linux-container-executor.cgroups.hierarchy 25 | hadoop-yarn 26 | 27 | 28 | yarn.resourcemanager.state-store.max-completed-applications 29 | ${yarn.resourcemanager.max-completed-applications} 30 | 31 | 32 | yarn.timeline-service.enabled 33 | false 34 | 35 | 36 | yarn.nodemanager.recovery.enabled 37 | false 38 | 39 | 40 | yarn.timeline-service.http-authentication.type 41 | simple 42 | 43 | 44 | yarn.resourcemanager.am.max-attempts 45 | 2 46 | 47 | 48 | yarn.nodemanager.log-aggregation.debug-enabled 49 | false 50 | 51 | 52 | yarn.resourcemanager.system-metrics-publisher.enabled 53 | true 54 | 55 | 56 | yarn.nodemanager.vmem-pmem-ratio 57 | 2.1 58 | 59 | 60 | yarn.resourcemanager.nodes.exclude-path 61 | /etc/hadoop/conf/yarn.exclude 62 | 63 | 64 | yarn.nodemanager.linux-container-executor.cgroups.mount 65 | false 66 | 67 | 68 | yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size 69 | 10 70 | 71 | 72 | yarn.application.classpath 73 | /etc/hadoop/conf,/usr/lib/hadoop/client/*,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-mapreduce-client/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-mapreduce-client/lib/*,/usr/hdp/current/hadoop-yarn-client/lib/* 74 | 75 | 76 | yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled 77 | false 78 | 79 | 80 | yarn.nodemanager.local-dirs 81 | /data1/yarn/local 82 | 83 | 84 | yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage 85 | false 86 | 87 | 88 | yarn.nodemanager.remote-app-log-dir-suffix 89 | logs 90 | 91 | 92 | yarn.resourcemanager.bind-host 93 | 0.0.0.0 94 | 95 | 96 | yarn.timeline-service.ttl-ms 97 | 2678400000 98 | 99 | 100 | yarn.nodemanager.container-monitor.interval-ms 101 | 3000 102 | 103 | 104 | yarn.node-labels.fs-store.retry-policy-spec 105 | 2000, 500 106 | 107 | 108 | yarn.resourcemanager.zk-acl 109 | world:anyone:rwcda 110 | 111 | 112 | yarn.log-aggregation-enable 113 | true 114 | 115 | 116 | yarn.nodemanager.delete.debug-delay-sec 117 | 3600 118 | 119 | 120 | yarn.timeline-service.store-class 121 | org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore 122 | 123 | 124 | yarn.timeline-service.client.retry-interval-ms 125 | 1000 126 | 127 | 128 | yarn.app.mapreduce.am.env 129 | LD_LIBRARY_PATH=$HADOOP_PREFIX/lib/native 130 | 131 | 132 | yarn.nodemanager.aux-services 133 | mapreduce_shuffle 134 | 135 | 136 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 137 | org.apache.hadoop.mapred.ShuffleHandler 138 | 139 | 140 | yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage 141 | 90 142 | 143 | 144 | yarn.resourcemanager.zk-timeout-ms 145 | 10000 146 | 147 | 148 | yarn.nodemanager.linux-container-executor.group 149 | hadoop 150 | 151 | 152 | yarn.nodemanager.remote-app-log-dir 153 | /app-logs 154 | 155 | 156 | yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms 157 | 10000 158 | 159 | 160 | yarn.timeline-service.generic-application-history.store-class 161 | org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore 162 | 163 | 164 | yarn.nodemanager.disk-health-checker.min-healthy-disks 165 | 0.25 166 | 167 | 168 | yarn.resourcemanager.work-preserving-recovery.enabled 169 | false 170 | 171 | 172 | yarn.nodemanager.health-checker.script.timeout-ms 173 | 60000 174 | 175 | 176 | yarn.resourcemanager.scheduler.class 177 | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler 178 | 179 | 180 | yarn.nodemanager.resource.memory-mb 181 | 7168 182 | 183 | 184 | yarn.nodemanager.resource.cpu-vcores 185 | 1 186 | 187 | 188 | yarn.scheduler.maximum-allocation-mb 189 | 2048 190 | 191 | 192 | yarn.nodemanager.resource.percentage-physical-cpu-limit 193 | 100 194 | 195 | 196 | yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb 197 | 1000 198 | 199 | 200 | yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds 201 | -1 202 | 203 | 204 | yarn.nodemanager.log.retain-second 205 | 604800 206 | 207 | 208 | yarn.node-labels.fs-store.root-dir 209 | /system/yarn/node-labels 210 | 211 | 212 | yarn.timeline-service.client.max-retries 213 | 30 214 | 215 | 216 | yarn.nodemanager.health-checker.interval-ms 217 | 135000 218 | 219 | 220 | yarn.nodemanager.admin-env 221 | MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX 222 | 223 | 224 | yarn.nodemanager.vmem-check-enabled 225 | true 226 | 227 | 228 | yarn.acl.enable 229 | false 230 | 231 | 232 | yarn.node-labels.manager-class 233 | org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager 234 | 235 | 236 | yarn.timeline-service.leveldb-timeline-store.read-cache-size 237 | 104857600 238 | 239 | 240 | yarn.nodemanager.linux-container-executor.resources-handler.class 241 | org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler 242 | 243 | 244 | yarn.timeline-service.http-authentication.simple.anonymous.allowed 245 | true 246 | 247 | 248 | yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size 249 | 10000 250 | 251 | 252 | yarn.resourcemanager.connect.max-wait.ms 253 | 900000 254 | 255 | 256 | yarn.resourcemanager.zk-address 257 | localhost:2181 258 | 259 | 260 | yarn.nodemanager.recovery.dir 261 | /var/log/hadoop-yarn/nodemanager/recovery-state 262 | 263 | 264 | yarn.nodemanager.container-executor.class 265 | org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor 266 | 267 | 268 | yarn.resourcemanager.store.class 269 | org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore 270 | 271 | 272 | yarn.timeline-service.leveldb-timeline-store.path 273 | /hadoop/yarn/timeline 274 | 275 | 276 | yarn.scheduler.minimum-allocation-mb 277 | 682 278 | 279 | 280 | yarn.timeline-service.ttl-enable 281 | true 282 | 283 | 284 | yarn.log-aggregation.retain-seconds 285 | 2592000 286 | 287 | 288 | hadoop.registry.rm.enabled 289 | false 290 | 291 | 292 | yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms 293 | 300000 294 | 295 | 296 | yarn.resourcemanager.fs.state-store.retry-policy-spec 297 | 2000, 500 298 | 299 | 300 | yarn.resourcemanager.zk-state-store.parent-path 301 | /rmstore 302 | 303 | 304 | yarn.nodemanager.log-aggregation.compression-type 305 | gz 306 | 307 | 308 | yarn.nodemanager.log-aggregation.num-log-files-per-app 309 | 30 310 | 311 | 312 | yarn.resourcemanager.recovery.enabled 313 | false 314 | 315 | 316 | yarn.nodemanager.bind-host 317 | 0.0.0.0 318 | 319 | 320 | yarn.resourcemanager.zk-retry-interval-ms 321 | 1000 322 | 323 | 324 | yarn.nodemanager.localizer.cache.target-size-mb 325 | 1024 326 | 327 | 328 | yarn.nodemanager.log-dirs 329 | /data1/yarn/log 330 | 331 | 332 | yarn.client.nodemanager-connect.retry-interval-ms 333 | 10000 334 | 335 | 336 | yarn.timelina-servica.enabled 337 | false 338 | 339 | 340 | yarn.resourcemanager.connect.retry-interval.ms 341 | 30000 342 | 343 | 344 | yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size 345 | 10000 346 | 347 | 348 | --------------------------------------------------------------------------------