├── .gitignore ├── MyDocker ├── Dockerfile ├── build_docker_image.sh ├── clean_container.sh ├── configs │ ├── core-site.xml │ ├── hadoop-env.sh │ ├── hdfs-site.xml │ ├── mapred-site.xml │ ├── master │ ├── slaves │ ├── ssh_config │ └── yarn-site.xml ├── create_network.sh ├── restart_container.sh ├── script │ ├── restart-hadoop.sh │ └── start-hadoop.sh ├── start_container.sh └── stop_container.sh ├── MyDocker_Hadoop ├── Dockerfile ├── build_docker_image.sh ├── clean_container.sh ├── configs │ ├── core-site.xml │ ├── hadoop-env.sh │ ├── hdfs-site.xml │ ├── mapred-site.xml │ ├── master │ ├── slaves │ ├── ssh_config │ └── yarn-site.xml ├── create_network.sh ├── restart_container.sh ├── script │ ├── restart-hadoop.sh │ └── start-hadoop.sh ├── start_container.sh └── stop_container.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Built application files 2 | *.apk 3 | *.aar 4 | *.ap_ 5 | *.aab 6 | 7 | # Files for the ART/Dalvik VM 8 | *.dex 9 | 10 | # Java class files 11 | *.class 12 | 13 | # Generated files 14 | bin/ 15 | gen/ 16 | out/ 17 | # Uncomment the following line in case you need and you don't have the release build type files in your app 18 | # release/ 19 | 20 | # Gradle files 21 | .gradle/ 22 | build/ 23 | 24 | # Local configuration file (sdk path, etc) 25 | local.properties 26 | 27 | # Proguard folder generated by Eclipse 28 | proguard/ 29 | 30 | # Log Files 31 | *.log 32 | 33 | # Android Studio Navigation editor temp files 34 | .navigation/ 35 | 36 | # Android Studio captures folder 37 | captures/ 38 | 39 | # IntelliJ 40 | *.iml 41 | .idea/workspace.xml 42 | .idea/tasks.xml 43 | .idea/gradle.xml 44 | .idea/assetWizardSettings.xml 45 | .idea/dictionaries 46 | .idea/libraries 47 | # Android Studio 3 in .gitignore file. 48 | .idea/caches 49 | .idea/modules.xml 50 | # Comment next line if keeping position of elements in Navigation Editor is relevant for you 51 | .idea/navEditor.xml 52 | 53 | # Keystore files 54 | # Uncomment the following lines if you do not want to check your keystore files in. 55 | #*.jks 56 | #*.keystore 57 | 58 | # External native build folder generated in Android Studio 2.2 and later 59 | .externalNativeBuild 60 | .cxx/ 61 | 62 | # Google Services (e.g. APIs or Firebase) 63 | # google-services.json 64 | 65 | # Freeline 66 | freeline.py 67 | freeline/ 68 | freeline_project_description.json 69 | 70 | # fastlane 71 | fastlane/report.xml 72 | fastlane/Preview.html 73 | fastlane/screenshots 74 | fastlane/test_output 75 | fastlane/readme.md 76 | 77 | # Version control 78 | vcs.xml 79 | 80 | # lint 81 | lint/intermediates/ 82 | lint/generated/ 83 | lint/outputs/ 84 | lint/tmp/ 85 | # lint/reports/ 86 | -------------------------------------------------------------------------------- /MyDocker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | MAINTAINER Zhangqx zhangqx@ss.pku.edu.cn 3 | 4 | LABEL Discription="hadoop base of centos7" version="1.0" 5 | 6 | #安装必备的软件包 7 | RUN yum -y install net-tools 8 | RUN yum -y install which 9 | RUN yum -y install openssh-server openssh-clients 10 | RUN yum clean all 11 | 12 | 13 | #配置SSH免密登录 14 | RUN ssh-keygen -q -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -N '' 15 | RUN ssh-keygen -q -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N '' 16 | RUN ssh-keygen -q -t dsa -f /etc/ssh/ssh_host_ed25519_key -N '' 17 | RUN ssh-keygen -f /root/.ssh/id_rsa -N '' 18 | RUN touch /root/.ssh/authorized_keys 19 | RUN cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys 20 | RUN echo "root:ss123456" | chpasswd 21 | COPY ./configs/ssh_config /etc/ssh/ssh_config 22 | 23 | #添加JDK 增加JAVA_HOME环境变量 24 | ADD ./tools/jdk-8u212-linux-x64.tar.gz /usr/local/ 25 | ENV JAVA_HOME /usr/local/jdk1.8.0_212/ 26 | ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar 27 | 28 | #添加Hadoop并设置环境变量 29 | ADD ./tools/hadoop-2.8.5.tar.gz /usr/local 30 | ENV HADOOP_HOME /usr/local/hadoop-2.8.5 31 | 32 | #将环境变量添加到系统变量中 33 | ENV PATH $HADOOP_HOME/bin:$JAVA_HOME/bin:$PATH 34 | 35 | #拷贝Hadoop相关的配置文件到镜像中 36 | COPY ./configs/hadoop-env.sh $HADOOP_HOME/etc/hadoop/hadoop-env.sh 37 | COPY ./configs/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml 38 | COPY ./configs/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml 39 | COPY ./configs/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml 40 | COPY ./configs/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml 41 | COPY ./configs/master $HADOOP_HOME/etc/hadoop/master 42 | COPY ./configs/slaves $HADOOP_HOME/etc/hadoop/slaves 43 | COPY ./script/start-hadoop.sh $HADOOP_HOME/start-hadoop.sh 44 | COPY ./script/restart-hadoop.sh $HADOOP_HOME/restart-hadoop.sh 45 | 46 | #增加执行权限 47 | RUN chmod 700 $HADOOP_HOME/start-hadoop.sh 48 | RUN chmod 700 $HADOOP_HOME/restart-hadoop.sh 49 | 50 | #创建数据目录 51 | RUN mkdir -p /data/hadoop/dfs/data && \ 52 | mkdir -p /data/hadoop/dfs/name && \ 53 | mkdir -p /data/hadoop/tmp 54 | 55 | #开启SSH 22 端口 56 | EXPOSE 22 57 | 58 | #启动容器时执行的脚本文件 59 | CMD ["/usr/sbin/sshd","-D"] 60 | 61 | -------------------------------------------------------------------------------- /MyDocker/build_docker_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo build centos-hadoop images 4 | 5 | docker build -t="centos-hadoop" . 6 | -------------------------------------------------------------------------------- /MyDocker/clean_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo clean containers 4 | 5 | docker ps -a 6 | 7 | docker kill $(docker ps -a -q) 8 | 9 | docker rm $(docker ps -q -f status=exited) 10 | 11 | docker ps -a 12 | -------------------------------------------------------------------------------- /MyDocker/configs/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | fs.defaultFS 21 | hdfs://hadoop-node1:9000/ 22 | 23 | 24 | hadoop.tmp.dir 25 | file:/data/hadoop/tmp 26 | 27 | 28 | -------------------------------------------------------------------------------- /MyDocker/configs/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Set Hadoop-specific environment variables here. 18 | 19 | # The only required environment variable is JAVA_HOME. All others are 20 | # optional. When running a distributed configuration it is best to 21 | # set JAVA_HOME in this file, so that it is correctly defined on 22 | # remote nodes. 23 | 24 | # The java implementation to use. 25 | export JAVA_HOME=/usr/local/jdk1.8.0_212 26 | 27 | # The jsvc implementation to use. Jsvc is required to run secure datanodes 28 | # that bind to privileged ports to provide authentication of data transfer 29 | # protocol. Jsvc is not required if SASL is configured for authentication of 30 | # data transfer protocol using non-privileged ports. 31 | #export JSVC_HOME=${JSVC_HOME} 32 | 33 | export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} 34 | 35 | # Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. 36 | for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do 37 | if [ "$HADOOP_CLASSPATH" ]; then 38 | export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f 39 | else 40 | export HADOOP_CLASSPATH=$f 41 | fi 42 | done 43 | 44 | # The maximum amount of heap to use, in MB. Default is 1000. 45 | #export HADOOP_HEAPSIZE= 46 | #export HADOOP_NAMENODE_INIT_HEAPSIZE="" 47 | 48 | # Extra Java runtime options. Empty by default. 49 | export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" 50 | 51 | # Command specific options appended to HADOOP_OPTS when specified 52 | export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS" 53 | export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" 54 | 55 | export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" 56 | 57 | export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS" 58 | export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS" 59 | 60 | # The following applies to multiple commands (fs, dfs, fsck, distcp etc) 61 | export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" 62 | #HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" 63 | 64 | # On secure datanodes, user to run the datanode as after dropping privileges. 65 | # This **MUST** be uncommented to enable secure HDFS if using privileged ports 66 | # to provide authentication of data transfer protocol. This **MUST NOT** be 67 | # defined if SASL is configured for authentication of data transfer protocol 68 | # using non-privileged ports. 69 | export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} 70 | 71 | # Where log files are stored. $HADOOP_HOME/logs by default. 72 | #export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER 73 | 74 | # Where log files are stored in the secure data environment. 75 | export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} 76 | 77 | ### 78 | # HDFS Mover specific parameters 79 | ### 80 | # Specify the JVM options to be used when starting the HDFS Mover. 81 | # These options will be appended to the options specified as HADOOP_OPTS 82 | # and therefore may override any similar flags set in HADOOP_OPTS 83 | # 84 | # export HADOOP_MOVER_OPTS="" 85 | 86 | ### 87 | # Advanced Users Only! 88 | ### 89 | 90 | # The directory where pid files are stored. /tmp by default. 91 | # NOTE: this should be set to a directory that can only be written to by 92 | # the user that will run the hadoop daemons. Otherwise there is the 93 | # potential for a symlink attack. 94 | export HADOOP_PID_DIR=${HADOOP_PID_DIR} 95 | export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} 96 | 97 | # A string representing this instance of hadoop. $USER by default. 98 | export HADOOP_IDENT_STRING=$USER 99 | -------------------------------------------------------------------------------- /MyDocker/configs/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | dfs.replication 22 | 2 23 | 24 | 25 | dfs.permissions.enabled 26 | false 27 | 28 | 29 | dfs.namenode.name.dir 30 | file:/data/hadoop/dfs/name 31 | 32 | 33 | dfs.datanode.data.dir 34 | file:/data/hadoop/dfs/data 35 | 36 | 37 | dfs.webhdfs.enabled 38 | true 39 | 40 | 41 | -------------------------------------------------------------------------------- /MyDocker/configs/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | mapreduce.framework.name 22 | yarn 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /MyDocker/configs/master: -------------------------------------------------------------------------------- 1 | hadoop-node1 2 | -------------------------------------------------------------------------------- /MyDocker/configs/slaves: -------------------------------------------------------------------------------- 1 | hadoop-node1 2 | hadoop-node2 3 | hadoop-node3 4 | -------------------------------------------------------------------------------- /MyDocker/configs/ssh_config: -------------------------------------------------------------------------------- 1 | Host localhost 2 | StrictHostKeyChecking no 3 | 4 | Host 0.0.0.0 5 | StrictHostKeyChecking no 6 | 7 | Host hadoop-* 8 | StrictHostKeyChecking no 9 | 10 | -------------------------------------------------------------------------------- /MyDocker/configs/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.nodemanager.aux-services 5 | mapreduce_shuffle 6 | 7 | 8 | -------------------------------------------------------------------------------- /MyDocker/create_network.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo create network 4 | docker network create --subnet=172.18.0.0/16 hadoop 5 | echo create success 6 | docker network ls 7 | -------------------------------------------------------------------------------- /MyDocker/restart_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo stop containers 4 | docker stop hadoop-node1 5 | docker stop hadoop-node2 6 | docker stop hadoop-node3 7 | 8 | echo start containers 9 | docker start hadoop-node1 10 | docker start hadoop-node2 11 | docker start hadoop-node3 12 | 13 | docker exec -it hadoop-node1 /usr/sbin/sshd 14 | docker exec -it hadoop-node2 /usr/sbin/sshd 15 | docker exec -it hadoop-node3 /usr/sbin/sshd 16 | docker exec -it hadoop-node1 /usr/local/hadoop-2.8.5/restart-hadoop.sh 17 | echo containers started 18 | 19 | docker ps -a 20 | 21 | 22 | -------------------------------------------------------------------------------- /MyDocker/script/restart-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $HADOOP_HOME/sbin/stop-all.sh 4 | $HADOOP_HOME/sbin/start-dfs.sh 5 | $HADOOP_HOME/sbin/start-yarn.sh 6 | 7 | 8 | -------------------------------------------------------------------------------- /MyDocker/script/start-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | hdfs namenode -format 4 | 5 | $HADOOP_HOME/sbin/start-dfs.sh 6 | $HADOOP_HOME/sbin/start-yarn.sh 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /MyDocker/start_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo start containers 4 | 5 | echo start hadoop-node1 container ... 6 | docker run -itd --restart=always --net hadoop --ip 172.18.0.2 --privileged -p 18032:8032 -p 28080:18080 -p 29888:19888 -p 17077:7077 -p 51070:50070 -p 18888:8888 -p 19000:9000 -p 11100:11000 -p 51030:50030 -p 18050:8050 -p 18081:8081 -p 18900:8900 --name hadoop-node1 --hostname hadoop-node1 --add-host hadoop-node2:172.18.0.3 --add-host hadoop-node3:172.18.0.4 centos-hadoop /bin/bash 7 | echo "start hadoop-node2 container..." 8 | docker run -itd --restart=always --net hadoop --ip 172.18.0.3 --privileged -p 18042:8042 -p 51010:50010 -p 51020:50020 --name hadoop-node2 --hostname hadoop-node2 --add-host hadoop-node1:172.18.0.2 --add-host hadoop-node3:172.18.0.4 centos-hadoop /bin/bash 9 | echo "start hadoop-node3 container..." 10 | docker run -itd --restart=always --net hadoop --ip 172.18.0.4 --privileged -p 18043:8042 -p 51011:50011 -p 51021:50021 --name hadoop-node3 --hostname hadoop-node3 --add-host hadoop-node1:172.18.0.2 --add-host hadoop-node2:172.18.0.3 centos-hadoop /bin/bash 11 | 12 | docker exec -it hadoop-node1 /usr/sbin/sshd 13 | docker exec -it hadoop-node2 /usr/sbin/sshd 14 | docker exec -it hadoop-node3 /usr/sbin/sshd 15 | docker exec -it hadoop-node1 /usr/local/hadoop-2.8.5/start-hadoop.sh 16 | 17 | echo finished 18 | docker ps 19 | -------------------------------------------------------------------------------- /MyDocker/stop_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo stop containers 4 | docker stop hadoop-node1 5 | docker stop hadoop-node2 6 | docker stop hadoop-node3 7 | 8 | docker ps -a 9 | 10 | 11 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | MAINTAINER Zhangqx zhangqx@ss.pku.edu.cn 3 | 4 | LABEL Discription="hadoop base of centos7" version="1.0" 5 | 6 | #安装必备的软件包 7 | RUN yum -y install net-tools 8 | RUN yum -y install which 9 | RUN yum -y install openssh-server openssh-clients 10 | RUN yum clean all 11 | 12 | 13 | #配置SSH免密登录 14 | RUN ssh-keygen -q -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -N '' 15 | RUN ssh-keygen -q -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N '' 16 | RUN ssh-keygen -q -t dsa -f /etc/ssh/ssh_host_ed25519_key -N '' 17 | RUN ssh-keygen -f /root/.ssh/id_rsa -N '' 18 | RUN touch /root/.ssh/authorized_keys 19 | RUN cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys 20 | RUN echo "root:ss123456" | chpasswd 21 | COPY ./configs/ssh_config /etc/ssh/ssh_config 22 | 23 | #添加JDK 增加JAVA_HOME环境变量 24 | ADD ./tools/jdk-8u212-linux-x64.tar.gz /usr/local/ 25 | ENV JAVA_HOME /usr/local/jdk1.8.0_212/ 26 | ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar 27 | 28 | #添加Hadoop并设置环境变量 29 | ADD ./tools/hadoop-2.8.5.tar.gz /usr/local 30 | ENV HADOOP_HOME /usr/local/hadoop-2.8.5 31 | 32 | #将环境变量添加到系统变量中 33 | ENV PATH $HADOOP_HOME/bin:$JAVA_HOME/bin:$PATH 34 | 35 | #拷贝Hadoop相关的配置文件到镜像中 36 | COPY ./configs/hadoop-env.sh $HADOOP_HOME/etc/hadoop/hadoop-env.sh 37 | COPY ./configs/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml 38 | COPY ./configs/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml 39 | COPY ./configs/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml 40 | COPY ./configs/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml 41 | COPY ./configs/master $HADOOP_HOME/etc/hadoop/master 42 | COPY ./configs/slaves $HADOOP_HOME/etc/hadoop/slaves 43 | COPY ./script/start-hadoop.sh $HADOOP_HOME/start-hadoop.sh 44 | COPY ./script/restart-hadoop.sh $HADOOP_HOME/restart-hadoop.sh 45 | 46 | #增加执行权限 47 | RUN chmod 700 $HADOOP_HOME/start-hadoop.sh 48 | RUN chmod 700 $HADOOP_HOME/restart-hadoop.sh 49 | 50 | #创建数据目录 51 | RUN mkdir -p /data/hadoop/dfs/data && \ 52 | mkdir -p /data/hadoop/dfs/name && \ 53 | mkdir -p /data/hadoop/tmp 54 | 55 | #开启SSH 22 端口 56 | EXPOSE 22 57 | 58 | #启动容器时执行的脚本文件 59 | CMD ["/usr/sbin/sshd","-D"] 60 | 61 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/build_docker_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo build centos-hadoop images 4 | 5 | docker build -t="centos-hadoop" . 6 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/clean_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo clean containers 4 | 5 | docker ps -a 6 | 7 | docker kill $(docker ps -a -q) 8 | 9 | docker rm $(docker ps -q -f status=exited) 10 | 11 | docker ps -a 12 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | fs.defaultFS 21 | hdfs://hadoop-node1:9000/ 22 | 23 | 24 | hadoop.tmp.dir 25 | file:/data/hadoop/tmp 26 | 27 | 28 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Set Hadoop-specific environment variables here. 18 | 19 | # The only required environment variable is JAVA_HOME. All others are 20 | # optional. When running a distributed configuration it is best to 21 | # set JAVA_HOME in this file, so that it is correctly defined on 22 | # remote nodes. 23 | 24 | # The java implementation to use. 25 | export JAVA_HOME=/usr/local/jdk1.8.0_212 26 | 27 | # The jsvc implementation to use. Jsvc is required to run secure datanodes 28 | # that bind to privileged ports to provide authentication of data transfer 29 | # protocol. Jsvc is not required if SASL is configured for authentication of 30 | # data transfer protocol using non-privileged ports. 31 | #export JSVC_HOME=${JSVC_HOME} 32 | 33 | export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} 34 | 35 | # Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. 36 | for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do 37 | if [ "$HADOOP_CLASSPATH" ]; then 38 | export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f 39 | else 40 | export HADOOP_CLASSPATH=$f 41 | fi 42 | done 43 | 44 | # The maximum amount of heap to use, in MB. Default is 1000. 45 | #export HADOOP_HEAPSIZE= 46 | #export HADOOP_NAMENODE_INIT_HEAPSIZE="" 47 | 48 | # Extra Java runtime options. Empty by default. 49 | export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" 50 | 51 | # Command specific options appended to HADOOP_OPTS when specified 52 | export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS" 53 | export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" 54 | 55 | export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" 56 | 57 | export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS" 58 | export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS" 59 | 60 | # The following applies to multiple commands (fs, dfs, fsck, distcp etc) 61 | export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" 62 | #HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" 63 | 64 | # On secure datanodes, user to run the datanode as after dropping privileges. 65 | # This **MUST** be uncommented to enable secure HDFS if using privileged ports 66 | # to provide authentication of data transfer protocol. This **MUST NOT** be 67 | # defined if SASL is configured for authentication of data transfer protocol 68 | # using non-privileged ports. 69 | export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} 70 | 71 | # Where log files are stored. $HADOOP_HOME/logs by default. 72 | #export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER 73 | 74 | # Where log files are stored in the secure data environment. 75 | export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} 76 | 77 | ### 78 | # HDFS Mover specific parameters 79 | ### 80 | # Specify the JVM options to be used when starting the HDFS Mover. 81 | # These options will be appended to the options specified as HADOOP_OPTS 82 | # and therefore may override any similar flags set in HADOOP_OPTS 83 | # 84 | # export HADOOP_MOVER_OPTS="" 85 | 86 | ### 87 | # Advanced Users Only! 88 | ### 89 | 90 | # The directory where pid files are stored. /tmp by default. 91 | # NOTE: this should be set to a directory that can only be written to by 92 | # the user that will run the hadoop daemons. Otherwise there is the 93 | # potential for a symlink attack. 94 | export HADOOP_PID_DIR=${HADOOP_PID_DIR} 95 | export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} 96 | 97 | # A string representing this instance of hadoop. $USER by default. 98 | export HADOOP_IDENT_STRING=$USER 99 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | dfs.replication 22 | 2 23 | 24 | 25 | dfs.permissions.enabled 26 | false 27 | 28 | 29 | dfs.namenode.name.dir 30 | file:/data/hadoop/dfs/name 31 | 32 | 33 | dfs.datanode.data.dir 34 | file:/data/hadoop/dfs/data 35 | 36 | 37 | dfs.webhdfs.enabled 38 | true 39 | 40 | 41 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | mapreduce.framework.name 22 | yarn 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/master: -------------------------------------------------------------------------------- 1 | hadoop-node1 2 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/slaves: -------------------------------------------------------------------------------- 1 | hadoop-node1 2 | hadoop-node2 3 | hadoop-node3 4 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/ssh_config: -------------------------------------------------------------------------------- 1 | Host localhost 2 | StrictHostKeyChecking no 3 | 4 | Host 0.0.0.0 5 | StrictHostKeyChecking no 6 | 7 | Host hadoop-* 8 | StrictHostKeyChecking no 9 | 10 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/configs/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.nodemanager.aux-services 5 | mapreduce_shuffle 6 | 7 | 8 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/create_network.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo create network 4 | docker network create --subnet=172.18.0.0/16 hadoop 5 | echo create success 6 | docker network ls 7 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/restart_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo stop containers 4 | docker stop hadoop-node1 5 | docker stop hadoop-node2 6 | docker stop hadoop-node3 7 | 8 | echo start containers 9 | docker start hadoop-node1 10 | docker start hadoop-node2 11 | docker start hadoop-node3 12 | 13 | sleep 5 14 | 15 | docker exec -it hadoop-node1 /usr/sbin/sshd 16 | docker exec -it hadoop-node2 /usr/sbin/sshd 17 | docker exec -it hadoop-node3 /usr/sbin/sshd 18 | 19 | sleep 5 20 | 21 | docker exec -it hadoop-node1 /usr/local/hadoop-2.8.5/restart-hadoop.sh 22 | echo containers started 23 | 24 | docker ps -a 25 | 26 | 27 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/script/restart-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $HADOOP_HOME/sbin/stop-all.sh 4 | $HADOOP_HOME/sbin/start-dfs.sh 5 | $HADOOP_HOME/sbin/start-yarn.sh 6 | 7 | 8 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/script/start-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | hdfs namenode -format 4 | 5 | $HADOOP_HOME/sbin/start-dfs.sh 6 | $HADOOP_HOME/sbin/start-yarn.sh 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/start_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo start containers 4 | 5 | echo start hadoop-node1 container ... 6 | docker run -itd --restart=always --net hadoop --ip 172.18.0.2 --privileged -p 18032:8032 -p 28080:18080 -p 29888:19888 -p 17077:7077 -p 51070:50070 -p 18888:8888 -p 19000:9000 -p 11100:11000 -p 51030:50030 -p 18050:8050 -p 18081:8081 -p 18900:8900 --name hadoop-node1 --hostname hadoop-node1 --add-host hadoop-node2:172.18.0.3 --add-host hadoop-node3:172.18.0.4 centos-hadoop /bin/bash 7 | echo "start hadoop-node2 container..." 8 | docker run -itd --restart=always --net hadoop --ip 172.18.0.3 --privileged -p 18042:8042 -p 51010:50010 -p 51020:50020 --name hadoop-node2 --hostname hadoop-node2 --add-host hadoop-node1:172.18.0.2 --add-host hadoop-node3:172.18.0.4 centos-hadoop /bin/bash 9 | echo "start hadoop-node3 container..." 10 | docker run -itd --restart=always --net hadoop --ip 172.18.0.4 --privileged -p 18043:8042 -p 51011:50011 -p 51021:50021 --name hadoop-node3 --hostname hadoop-node3 --add-host hadoop-node1:172.18.0.2 --add-host hadoop-node2:172.18.0.3 centos-hadoop /bin/bash 11 | 12 | sleep 5 13 | docker exec -it hadoop-node1 /usr/sbin/sshd 14 | docker exec -it hadoop-node2 /usr/sbin/sshd 15 | docker exec -it hadoop-node3 /usr/sbin/sshd 16 | sleep 5 17 | docker exec -it hadoop-node1 /usr/local/hadoop-2.8.5/start-hadoop.sh 18 | 19 | echo finished 20 | docker ps 21 | -------------------------------------------------------------------------------- /MyDocker_Hadoop/stop_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo stop containers 4 | docker stop hadoop-node1 5 | docker stop hadoop-node2 6 | docker stop hadoop-node3 7 | 8 | docker ps -a 9 | 10 | 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BigData 2 | 大数据基础与实践课程 3 | 4 | 相关软件包 5 | * jdk-8u212-linux-x64.tar.gz 6 | * hadoop-2.8.5.tar.gz 7 | 8 | 下载地址：百度网盘链接: https://pan.baidu.com/s/1nOvFQGj12N3ODNilOYMYjg 密码: r8qo 9 | --------------------------------------------------------------------------------