├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md └── client-conf ├── core-site.xml ├── hbase-site.xml ├── hdfs-site.xml ├── hive-site.xml ├── mapred-site.xml └── yarn-site.xml /.gitignore: -------------------------------------------------------------------------------- 1 | ### Eclipse 2 | *.pydevproject 3 | .metadata 4 | .gradle 5 | tmp/ 6 | *.tmp 7 | *.bak 8 | *.swp 9 | *~.nib 10 | *.versionsBackup 11 | local.properties 12 | release.properties 13 | .loadpath 14 | 15 | #IDEA 16 | *.iml 17 | #.settings 18 | 19 | # External tool builders 20 | .externalToolBuilders/ 21 | 22 | # Locally stored "Eclipse launch configurations" 23 | *.launch 24 | 25 | # CDT-specific 26 | .cproject 27 | 28 | # PDT-specific 29 | .buildpath 30 | 31 | # sbteclipse plugin 32 | .target 33 | 34 | # TeXlipse plugin 35 | .texlipse 36 | 37 | ### Java 38 | *.class 39 | 40 | # Mobile Tools for Java (J2ME) 41 | .mtj.tmp/ 42 | 43 | # Package Files # 44 | *.jar 45 | *.war 46 | *.ear 47 | 48 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 49 | hs_err_pid* 50 | 51 | 52 | ### Origin 53 | # / 54 | .classpath 55 | .project 56 | .idea 57 | .DS_Store 58 | target 59 | logs 60 | *.releaseBackup 61 | *.log 62 | *.DS_Store 63 | 64 | # .settings 65 | org.eclipse.m2e.core.prefs 66 | org.eclipse.wst.common.component 67 | org.eclipse.wst.common.project.facet.core.xml 68 | org.eclipse.wst.validation.prefs 69 | server/.settings/.jsdtscope 70 | server/.settings/org.eclipse.wst.common.project.facet.core.prefs.xml 71 | server/.settings/org.eclipse.wst.jsdt.ui.superType.container 72 | server/.settings/org.eclipse.wst.jsdt.ui.superType.name 73 | #web 74 | webapp/app/WEB-INF/* 75 | webapp/dist 76 | webapp/node_modules: 77 | webapp/CHANGELOG.md 78 | 79 | # package file 80 | build/tomcat 81 | build/lib 82 | *.tgz 83 | *.tar.gz 84 | webapp/app/components/* 85 | 86 | ebay* 87 | build/commit_SHA1 88 | dist/ 89 | tomcat/ 90 | webapp/app/kylin-servlet.xml 91 | webapp/app/web.xml -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:8-jre 2 | 3 | MAINTAINER Kyligence Inc 4 | 5 | WORKDIR /tmp 6 | 7 | RUN set -x \ 8 | && apt-get update && apt-get install -y wget vim telnet ntp \ 9 | && update-rc.d ntp defaults 10 | 11 | ARG MIRROR=mirror.bit.edu.cn 12 | 13 | # Installing Hadoop 14 | ARG HADOOP_VERSION=2.7.4 15 | # COPY hadoop-${HADOOP_VERSION}.tar.gz . 16 | RUN set -x \ 17 | && wget -q http://${MIRROR}/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz \ 18 | && tar -xzvf hadoop-${HADOOP_VERSION}.tar.gz -C /usr/local/ \ 19 | && mv /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop 20 | ENV HADOOP_HOME=/usr/local/hadoop 21 | ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop 22 | ENV YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop 23 | 24 | # Installing Spark 25 | ARG SPARK_VERSION=2.2.1 26 | # COPY spark-${SPARK_VERSION}-bin-without-hadoop.tgz . 27 | RUN set -x \ 28 | && wget -q http://${MIRROR}/apache/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-without-hadoop.tgz \ 29 | && tar -xzvf spark-${SPARK_VERSION}-bin-without-hadoop.tgz -C /usr/local/ \ 30 | && mv /usr/local/spark-${SPARK_VERSION}-bin-without-hadoop /usr/local/spark 31 | ENV SPARK_HOME=/usr/local/spark 32 | ENV LD_LIBRARY_PATH=$HADOOP_HOME/lib/native/:$LD_LIBRARY_PATH 33 | 34 | # Installing Hive 35 | ARG HIVE_VERSION=1.2.2 36 | # COPY apache-hive-${HIVE_VERSION}-bin.tar.gz . 37 | RUN set -x \ 38 | && wget -q http://${MIRROR}/apache/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz \ 39 | && tar -xzvf apache-hive-${HIVE_VERSION}-bin.tar.gz -C /usr/local/ \ 40 | && mv /usr/local/apache-hive-${HIVE_VERSION}-bin /usr/local/hive 41 | ENV HIVE_HOME=/usr/local/hive 42 | ENV HCAT_HOME=$HIVE_HOME/hcatalog 43 | ENV HIVE_CONF=$HIVE_HOME/conf 44 | 45 | # Installing HBase 46 | ARG HBASE_VERSION=1.3.1 47 | # COPY hbase-${HBASE_VERSION}-bin.tar.gz . 48 | RUN set -x \ 49 | && wget -q http://${MIRROR}/apache/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz \ 50 | && tar -xzvf hbase-${HBASE_VERSION}-bin.tar.gz -C /usr/local/ \ 51 | && mv /usr/local/hbase-${HBASE_VERSION} /usr/local/hbase 52 | ENV HBASE_HOME=/usr/local/hbase 53 | 54 | # Installing Kylin 55 | ARG KYLIN_VERSION=2.2.0 56 | # COPY apache-kylin-${KYLIN_VERSION}-bin-hbase1x.tar.gz . 57 | RUN set -x \ 58 | && wget -q http://${MIRROR}/apache/kylin/apache-kylin-${KYLIN_VERSION}/apache-kylin-${KYLIN_VERSION}-bin-hbase1x.tar.gz \ 59 | && tar -xzvf apache-kylin-${KYLIN_VERSION}-bin-hbase1x.tar.gz -C /usr/local/ \ 60 | && mv /usr/local/apache-kylin-${KYLIN_VERSION}-bin /usr/local/kylin 61 | ENV KYLIN_HOME=/usr/local/kylin 62 | 63 | # Setting the PATH environment variable 64 | ENV PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$SPARK_HOME/bin:$HIVE_HOME/bin:$HBASE_HOME/bin:$KYLIN_HOME/bin 65 | 66 | COPY client-conf/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml 67 | COPY client-conf/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml 68 | COPY client-conf/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml 69 | COPY client-conf/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml 70 | COPY client-conf/hbase-site.xml $HBASE_HOME/conf/hbase-site.xml 71 | COPY client-conf/hdfs-site.xml $HBASE_HOME/conf/hdfs-site.xml 72 | COPY client-conf/hive-site.xml $HIVE_HOME/conf/hive-site.xml 73 | COPY client-conf/mapred-site.xml $HIVE_HOME/conf/mapred-site.xml 74 | 75 | # Cleanup 76 | RUN rm -rf /tmp/* 77 | 78 | WORKDIR /root 79 | EXPOSE 7070 80 | 81 | VOLUME /usr/local/kylin/logs 82 | 83 | ENTRYPOINT ["sh", "-c", "/usr/local/kylin/bin/kylin.sh start; bash"] 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | exec: 2 | docker exec -it kylin bash 3 | 4 | run: 5 | docker run -itd --name kylin -p 7070:7070 -v `pwd`/logs:/usr/local/kylin/logs --add-host="MASTER-NODE-HOSTNAME:10.1.2.48" --add-host="sandbox.hortonworks.com:10.1.2.48" kylin:2.2.0 6 | 7 | build: 8 | docker build -t kylin:2.2.0 . 9 | 10 | clean: 11 | docker stop kylin && docker rm kylin 12 | docker images | grep '' | awk '{print $3}' | xargs docker rmi >/dev/null 2>&1 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kylin on Docker 2 | This repository trackes the code and files for building docker images with [Apache Kylin](http://kylin.apache.org). 3 | 4 | Please note: this is the master branch, which doesn't have the scripts and Dockerfile; You need checkout the specific branch which named with Kylin version and Hadoop release name. 5 | 6 | ## Background 7 | 8 | Apache Kylin is an open source Distributed Analytics Engine designed to provide SQL interface and multi-dimensional analysis (OLAP) on Hadoop supporting extremely large datasets. For more information you can visit Kylin home page at http://kylin.apache.org 9 | 10 | Usually Kylin is deployed in a dedicated Hadoop client node, on which the Hadoop, HBase, Hive and other clients have been properly configured to communicate with the cluster; Kylin will use the client Jars and configuration files to work with other components; Besides, all Kylin metadata and cube data are persistended in HBase and HDFS, not in local, so all these make it very reasonable to build Kylin as a docker image for quick deploy. 11 | 12 | ## How to make it 13 | 14 | The main idea is building Hadoop/HBase/Hive clients and Kylin binary package into one image; User can pull this image, and then just add client configuration files like core-site.xml, hdfs-site.xml, yarn-site.xml, hbase-site.xml, hive-site.xml and kylin.properties to the effective paths to make a new image (has verified), or upload these files during starting up (not verified yet); 15 | 16 | Before start, you need do some preparations: 17 | * check the Hadoop versions, and make sure the client libs in the image are compitable with the cluster; 18 | * prepare a kylin.properties file for this deployment; 19 | * ensure the Hadoop security constraint will not block Docker's adoption; you may need run additional component in the container if kerberos is enabled. 20 | 21 | ## Steps 22 | 23 | Below is a sample of building and running a docker image for Hortonworks HDP 2.2 cluster. 24 | 25 | 1. Collect the client configuration files 26 | Get the *-site.xml files from a working Hadoop client node, to a local folder say "~/hadoop-conf/"; 27 | 28 | 2. Prepare kylin.properties 29 | 30 | The kylin.properties file is the main configuration file for Kylin; you need prepare such a file and put it to the "~/hadoop-conf/" folder, together with other conf files; suggest to double check the parameters in it; e.g, the "kylin.metadata.url" points to the right metadata table, "kylin.hdfs.working.dir" is an existing HDFS folder and you have permission to write, etc. 31 | 32 | 3. Clone this repository, checkout the correct branch; 33 | ``` 34 | git clone https://github.com/Kyligence/kylin-docker 35 | cd kylin-docker 36 | git checkout kylin152-hdp22 37 | ``` 38 | 39 | 4. Copy the client configuration files to "kylin-docker/conf" folder, overwriting those template files; 40 | ``` 41 | cp -rf ~/hadoop-conf/* conf/ 42 | ``` 43 | 44 | 5. Build docker image, which may take a while, just take a cup of tea; 45 | ``` 46 | docker build -t kyligence/kylin:152 . 47 | ``` 48 | 49 | After the build finished, should be able to see the image with "docker images" commmand; 50 | ``` 51 | [root@ip-10-0-0-38 ~]# docker images 52 | REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE 53 | kyligence/kylin 152 7ece32097fa3 About an hour ago 3.043 GB 54 | ``` 55 | 56 | 6. Now you can run a contianer with the bootstrap command (in which will start Kylin server). The "-bash" argument is telling to keep in bash so you can continue to run bash commands; If don't need, you can use the "-d" argument: 57 | ``` 58 | [root@ip-10-0-0-38 ~]# docker run -i -t -p 7070:7070 kyligence/kylin:152 /etc/bootstrap.sh -bash 59 | Generating SSH1 RSA host key: [ OK ] 60 | Starting sshd: [ OK ] 61 | KYLIN_HOME is set to /usr/local/kylin/bin/../ 62 | kylin.security.profile is set to ldap 63 | 16/06/30 04:50:31 WARN conf.HiveConf: HiveConf of name hive.optimize.mapjoin.mapreduce does not exist 64 | 16/06/30 04:50:31 WARN conf.HiveConf: HiveConf of name hive.heapsize does not exist 65 | 16/06/30 04:50:31 WARN conf.HiveConf: HiveConf of name hive.server2.enable.impersonation does not exist 66 | 16/06/30 04:50:31 WARN conf.HiveConf: HiveConf of name hive.auto.convert.sortmerge.join.noconditionaltask does not exist 67 | Logging initialized using configuration in file:/etc/hive/conf/hive-log4j.properties 68 | HCAT_HOME not found, try to find hcatalog path from hadoop home 69 | A new Kylin instance is started by , stop it using "kylin.sh stop" 70 | Please visit http://:7070/kylin 71 | You can check the log at /usr/local/kylin/bin/..//logs/kylin.log 72 | ``` 73 | 74 | 7. After a minute, you can open web browser with address http://host:7070/kylin , here the "host" is the hostname or IP address of the hosting machine which runs Docker; Its 7070 port will redirect to the contianer's 7070 port as we specified in the "docker run" command; You can change to other port as you like. 75 | 76 | 8. Now you can use Kylin as usually: import Hive tables, design cubes, build, query, etc. 77 | 78 | ## Thanks 79 | 80 | Thanks to SequenceIQ's [hadoop-docker](https://github.com/sequenceiq/hadoop-docker/) and other projects, which inspires us on developing this. 81 | 82 | 83 | -------------------------------------------------------------------------------- /client-conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://MASTER-NODE-HOSTNAME:8020/ 6 | 7 | 8 | -------------------------------------------------------------------------------- /client-conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | dfs.domain.socket.path 4 | /var/lib/hadoop-hdfs/dn_socket 5 | 6 | 7 | 8 | hbase.bucketcache.ioengine 9 | 10 | 11 | 12 | 13 | hbase.bucketcache.percentage.in.combinedcache 14 | 15 | 16 | 17 | 18 | hbase.bucketcache.size 19 | 20 | 21 | 22 | 23 | hbase.bulkload.staging.dir 24 | /apps/hbase/staging 25 | 26 | 27 | 28 | hbase.client.keyvalue.maxsize 29 | 1048576 30 | 31 | 32 | 33 | hbase.client.retries.number 34 | 35 35 | 36 | 37 | 38 | hbase.client.scanner.caching 39 | 100 40 | 41 | 42 | 43 | hbase.cluster.distributed 44 | true 45 | 46 | 47 | 48 | hbase.coprocessor.master.classes 49 | org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor 50 | 51 | 52 | 53 | hbase.coprocessor.region.classes 54 | 55 | org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor 56 | 57 | 58 | 59 | 60 | hbase.coprocessor.regionserver.classes 61 | 62 | 63 | 64 | 65 | hbase.defaults.for.version.skip 66 | true 67 | 68 | 69 | 70 | hbase.hregion.majorcompaction 71 | 604800000 72 | 73 | 74 | 75 | hbase.hregion.majorcompaction.jitter 76 | 0.50 77 | 78 | 79 | 80 | hbase.hregion.max.filesize 81 | 10737418240 82 | 83 | 84 | 85 | hbase.hregion.memstore.block.multiplier 86 | 4 87 | 88 | 89 | 90 | hbase.hregion.memstore.flush.size 91 | 134217728 92 | 93 | 94 | 95 | hbase.hregion.memstore.mslab.enabled 96 | true 97 | 98 | 99 | 100 | hbase.hstore.blockingStoreFiles 101 | 10 102 | 103 | 104 | 105 | hbase.hstore.compaction.max 106 | 10 107 | 108 | 109 | 110 | hbase.hstore.compactionThreshold 111 | 3 112 | 113 | 114 | 115 | hbase.local.dir 116 | ${hbase.tmp.dir}/local 117 | 118 | 119 | 120 | hbase.master.info.bindAddress 121 | 0.0.0.0 122 | 123 | 124 | 125 | hbase.master.info.port 126 | 16010 127 | 128 | 129 | 130 | hbase.master.port 131 | 16000 132 | 133 | 134 | 135 | hbase.region.server.rpc.scheduler.factory.class 136 | 137 | 138 | 139 | 140 | hbase.regionserver.global.memstore.size 141 | 0.4 142 | 143 | 144 | 145 | hbase.regionserver.handler.count 146 | 30 147 | 148 | 149 | 150 | hbase.regionserver.info.port 151 | 16030 152 | 153 | 154 | 155 | hbase.regionserver.port 156 | 16020 157 | 158 | 159 | 160 | hbase.regionserver.wal.codec 161 | org.apache.hadoop.hbase.regionserver.wal.WALCellCodec 162 | 163 | 164 | 165 | hbase.rootdir 166 | hdfs://MASTER-NODE-HOSTNAME:8020/apps/hbase/data 167 | 168 | 169 | 170 | hbase.rpc.controllerfactory.class 171 | 172 | 173 | 174 | 175 | hbase.rpc.engine 176 | org.apache.hadoop.hbase.ipc.SecureRpcEngine 177 | 178 | 179 | 180 | hbase.rpc.protection 181 | PRIVACY 182 | 183 | 184 | 185 | hbase.rpc.timeout 186 | 90000 187 | 188 | 189 | 190 | hbase.security.authentication 191 | simple 192 | 193 | 194 | 195 | hbase.security.authorization 196 | true 197 | 198 | 199 | 200 | hbase.superuser 201 | hbase 202 | 203 | 204 | 205 | hbase.tmp.dir 206 | /tmp/hbase-${user.name} 207 | 208 | 209 | 210 | hbase.zookeeper.property.clientPort 211 | 2181 212 | 213 | 214 | 215 | hbase.zookeeper.quorum 216 | MASTER-NODE-HOSTNAME 217 | 218 | 219 | 220 | hbase.zookeeper.useMulti 221 | true 222 | 223 | 224 | 225 | hfile.block.cache.size 226 | 0.4 227 | 228 | 229 | 230 | phoenix.functions.allowUserDefinedFunctions 231 | 232 | 233 | 234 | 235 | phoenix.query.timeoutMs 236 | 60000 237 | 238 | 239 | 240 | zookeeper.session.timeout 241 | 60000 242 | 243 | 244 | 245 | zookeeper.znode.parent 246 | /hbase-unsecure 247 | 248 | 249 | -------------------------------------------------------------------------------- /client-conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dfs.namenode.name.dir 5 | file:///data/hdfs/namenode 6 | NameNode directory for namespace and transaction logs storage. 7 | 8 | 9 | dfs.datanode.data.dir 10 | file:///data/hdfs/datanode 11 | DataNode directory 12 | 13 | 14 | -------------------------------------------------------------------------------- /client-conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ambari.hive.db.schema.name 5 | hive 6 | 7 | 8 | 9 | atlas.cluster.name 10 | Sandbox 11 | 12 | 13 | 14 | atlas.hook.hive.maxThreads 15 | 1 16 | 17 | 18 | 19 | atlas.hook.hive.minThreads 20 | 1 21 | 22 | 23 | 24 | atlas.hook.hive.synchronous 25 | true 26 | 27 | 28 | 29 | atlas.rest.address 30 | http://MASTER-NODE-HOSTNAME:21000 31 | 32 | 33 | 34 | datanucleus.autoCreateSchema 35 | false 36 | 37 | 38 | 39 | datanucleus.cache.level2.type 40 | none 41 | 42 | 43 | 44 | datanucleus.fixedDatastore 45 | true 46 | 47 | 48 | 49 | hive.auto.convert.join 50 | true 51 | 52 | 53 | 54 | hive.auto.convert.join.noconditionaltask 55 | true 56 | 57 | 58 | 59 | hive.auto.convert.join.noconditionaltask.size 60 | 357913941 61 | 62 | 63 | 64 | hive.auto.convert.sortmerge.join 65 | true 66 | 67 | 68 | 69 | hive.auto.convert.sortmerge.join.to.mapjoin 70 | false 71 | 72 | 73 | 74 | hive.cbo.enable 75 | true 76 | 77 | 78 | 79 | hive.cli.print.header 80 | false 81 | 82 | 83 | 84 | hive.cluster.delegation.token.store.class 85 | org.apache.hadoop.hive.thrift.ZooKeeperTokenStore 86 | 87 | 88 | 89 | hive.cluster.delegation.token.store.zookeeper.connectString 90 | MASTER-NODE-HOSTNAME:2181 91 | 92 | 93 | 94 | hive.cluster.delegation.token.store.zookeeper.znode 95 | /hive/cluster/delegation 96 | 97 | 98 | 99 | hive.compactor.abortedtxn.threshold 100 | 1000 101 | 102 | 103 | 104 | hive.compactor.check.interval 105 | 300s 106 | 107 | 108 | 109 | hive.compactor.delta.num.threshold 110 | 10 111 | 112 | 113 | 114 | hive.compactor.delta.pct.threshold 115 | 0.1f 116 | 117 | 118 | 119 | hive.compactor.initiator.on 120 | true 121 | 122 | 123 | 124 | hive.compactor.worker.threads 125 | 0 126 | 127 | 128 | 129 | hive.compactor.worker.timeout 130 | 86400s 131 | 132 | 133 | 134 | hive.compute.query.using.stats 135 | true 136 | 137 | 138 | 139 | hive.conf.restricted.list 140 | hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role 141 | 142 | 143 | 144 | hive.convert.join.bucket.mapjoin.tez 145 | false 146 | 147 | 148 | 149 | hive.default.fileformat 150 | TextFile 151 | 152 | 153 | 154 | hive.default.fileformat.managed 155 | TextFile 156 | 157 | 158 | 159 | hive.enforce.bucketing 160 | true 161 | 162 | 163 | 164 | hive.enforce.sorting 165 | true 166 | 167 | 168 | 169 | hive.enforce.sortmergebucketmapjoin 170 | true 171 | 172 | 173 | 174 | hive.exec.compress.intermediate 175 | false 176 | 177 | 178 | 179 | hive.exec.compress.output 180 | false 181 | 182 | 183 | 184 | hive.exec.dynamic.partition 185 | true 186 | 187 | 188 | 189 | hive.exec.dynamic.partition.mode 190 | nonstrict 191 | 192 | 193 | 194 | hive.exec.failure.hooks 195 | org.apache.hadoop.hive.ql.hooks.ATSHook 196 | 197 | 198 | 199 | hive.exec.max.created.files 200 | 100000 201 | 202 | 203 | 204 | hive.exec.max.dynamic.partitions 205 | 5000 206 | 207 | 208 | 209 | hive.exec.max.dynamic.partitions.pernode 210 | 2000 211 | 212 | 213 | 214 | hive.exec.orc.compression.strategy 215 | SPEED 216 | 217 | 218 | 219 | hive.exec.orc.default.compress 220 | ZLIB 221 | 222 | 223 | 224 | hive.exec.orc.default.stripe.size 225 | 67108864 226 | 227 | 228 | 229 | hive.exec.orc.encoding.strategy 230 | SPEED 231 | 232 | 233 | 234 | hive.exec.parallel 235 | false 236 | 237 | 238 | 239 | hive.exec.parallel.thread.number 240 | 8 241 | 242 | 243 | 244 | hive.exec.post.hooks 245 | org.apache.hadoop.hive.ql.hooks.ATSHook 246 | 247 | 248 | 249 | hive.exec.pre.hooks 250 | org.apache.hadoop.hive.ql.hooks.ATSHook 251 | 252 | 253 | 254 | hive.exec.reducers.bytes.per.reducer 255 | 67108864 256 | 257 | 258 | 259 | hive.exec.reducers.max 260 | 1009 261 | 262 | 263 | 264 | hive.exec.scratchdir 265 | /tmp/hive 266 | 267 | 268 | 269 | hive.exec.submit.local.task.via.child 270 | true 271 | 272 | 273 | 274 | hive.exec.submitviachild 275 | false 276 | 277 | 278 | 279 | hive.execution.engine 280 | tez 281 | 282 | 283 | 284 | hive.fetch.task.aggr 285 | false 286 | 287 | 288 | 289 | hive.fetch.task.conversion 290 | more 291 | 292 | 293 | 294 | hive.fetch.task.conversion.threshold 295 | 1073741824 296 | 297 | 298 | 299 | hive.limit.optimize.enable 300 | true 301 | 302 | 303 | 304 | hive.limit.pushdown.memory.usage 305 | 0.04 306 | 307 | 308 | 309 | hive.map.aggr 310 | true 311 | 312 | 313 | 314 | hive.map.aggr.hash.force.flush.memory.threshold 315 | 0.9 316 | 317 | 318 | 319 | hive.map.aggr.hash.min.reduction 320 | 0.5 321 | 322 | 323 | 324 | hive.map.aggr.hash.percentmemory 325 | 0.5 326 | 327 | 328 | 329 | hive.mapjoin.bucket.cache.size 330 | 10000 331 | 332 | 333 | 334 | hive.mapjoin.optimized.hashtable 335 | true 336 | 337 | 338 | 339 | hive.mapred.reduce.tasks.speculative.execution 340 | false 341 | 342 | 343 | 344 | hive.merge.mapfiles 345 | true 346 | 347 | 348 | 349 | hive.merge.mapredfiles 350 | false 351 | 352 | 353 | 354 | hive.merge.orcfile.stripe.level 355 | true 356 | 357 | 358 | 359 | hive.merge.rcfile.block.level 360 | true 361 | 362 | 363 | 364 | hive.merge.size.per.task 365 | 256000000 366 | 367 | 368 | 369 | hive.merge.smallfiles.avgsize 370 | 16000000 371 | 372 | 373 | 374 | hive.merge.tezfiles 375 | false 376 | 377 | 378 | 379 | hive.metastore.authorization.storage.checks 380 | false 381 | 382 | 383 | 384 | hive.metastore.cache.pinobjtypes 385 | Table,Database,Type,FieldSchema,Order 386 | 387 | 388 | 389 | hive.metastore.client.connect.retry.delay 390 | 5s 391 | 392 | 393 | 394 | hive.metastore.client.socket.timeout 395 | 1800s 396 | 397 | 398 | 399 | hive.metastore.connect.retries 400 | 24 401 | 402 | 403 | 404 | hive.metastore.execute.setugi 405 | true 406 | 407 | 408 | 409 | hive.metastore.failure.retries 410 | 24 411 | 412 | 413 | 414 | hive.metastore.kerberos.keytab.file 415 | /etc/security/keytabs/hive.service.keytab 416 | 417 | 418 | 419 | hive.metastore.kerberos.principal 420 | hive/_HOST@EXAMPLE.COM 421 | 422 | 423 | 424 | hive.metastore.pre.event.listeners 425 | org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener 426 | 427 | 428 | 429 | hive.metastore.sasl.enabled 430 | false 431 | 432 | 433 | 434 | hive.metastore.server.max.threads 435 | 100000 436 | 437 | 438 | 439 | hive.metastore.uris 440 | thrift://MASTER-NODE-HOSTNAME:9083 441 | 442 | 443 | 444 | hive.metastore.warehouse.dir 445 | /apps/hive/warehouse 446 | 447 | 448 | 449 | hive.optimize.bucketmapjoin 450 | true 451 | 452 | 453 | 454 | hive.optimize.bucketmapjoin.sortedmerge 455 | false 456 | 457 | 458 | 459 | hive.optimize.constant.propagation 460 | true 461 | 462 | 463 | 464 | hive.optimize.index.filter 465 | true 466 | 467 | 468 | 469 | hive.optimize.metadataonly 470 | true 471 | 472 | 473 | 474 | hive.optimize.null.scan 475 | true 476 | 477 | 478 | 479 | hive.optimize.reducededuplication 480 | true 481 | 482 | 483 | 484 | hive.optimize.reducededuplication.min.reducer 485 | 4 486 | 487 | 488 | 489 | hive.optimize.sort.dynamic.partition 490 | false 491 | 492 | 493 | 494 | hive.orc.compute.splits.num.threads 495 | 10 496 | 497 | 498 | 499 | hive.orc.splits.include.file.footer 500 | false 501 | 502 | 503 | 504 | hive.prewarm.enabled 505 | false 506 | 507 | 508 | 509 | hive.prewarm.numcontainers 510 | 3 511 | 512 | 513 | 514 | hive.security.authenticator.manager 515 | org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator 516 | 517 | 518 | 519 | hive.security.authorization.enabled 520 | false 521 | 522 | 523 | 524 | hive.security.authorization.manager 525 | org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory 526 | 527 | 528 | 529 | hive.security.metastore.authenticator.manager 530 | org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator 531 | 532 | 533 | 534 | hive.security.metastore.authorization.auth.reads 535 | true 536 | 537 | 538 | 539 | hive.security.metastore.authorization.manager 540 | org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider 541 | 542 | 543 | 544 | hive.server2.allow.user.substitution 545 | true 546 | 547 | 548 | 549 | hive.server2.authentication 550 | NONE 551 | 552 | 553 | 554 | hive.server2.authentication.spnego.keytab 555 | HTTP/_HOST@EXAMPLE.COM 556 | 557 | 558 | 559 | hive.server2.authentication.spnego.principal 560 | /etc/security/keytabs/spnego.service.keytab 561 | 562 | 563 | 564 | hive.server2.enable.doAs 565 | true 566 | 567 | 568 | 569 | hive.server2.logging.operation.enabled 570 | true 571 | 572 | 573 | 574 | hive.server2.logging.operation.log.location 575 | /tmp/hive/operation_logs 576 | 577 | 578 | 579 | hive.server2.support.dynamic.service.discovery 580 | true 581 | 582 | 583 | 584 | hive.server2.table.type.mapping 585 | CLASSIC 586 | 587 | 588 | 589 | hive.server2.tez.default.queues 590 | default 591 | 592 | 593 | 594 | hive.server2.tez.initialize.default.sessions 595 | false 596 | 597 | 598 | 599 | hive.server2.tez.sessions.per.default.queue 600 | 1 601 | 602 | 603 | 604 | hive.server2.thrift.http.path 605 | cliservice 606 | 607 | 608 | 609 | hive.server2.thrift.http.port 610 | 10001 611 | 612 | 613 | 614 | hive.server2.thrift.max.worker.threads 615 | 500 616 | 617 | 618 | 619 | hive.server2.thrift.port 620 | 10000 621 | 622 | 623 | 624 | hive.server2.thrift.sasl.qop 625 | auth 626 | 627 | 628 | 629 | hive.server2.transport.mode 630 | binary 631 | 632 | 633 | 634 | hive.server2.use.SSL 635 | false 636 | 637 | 638 | 639 | hive.server2.zookeeper.namespace 640 | hiveserver2 641 | 642 | 643 | 644 | hive.smbjoin.cache.rows 645 | 10000 646 | 647 | 648 | 649 | hive.stats.autogather 650 | true 651 | 652 | 653 | 654 | hive.stats.dbclass 655 | fs 656 | 657 | 658 | 659 | hive.stats.fetch.column.stats 660 | false 661 | 662 | 663 | 664 | hive.stats.fetch.partition.stats 665 | true 666 | 667 | 668 | 669 | hive.support.concurrency 670 | true 671 | 672 | 673 | 674 | hive.tez.auto.reducer.parallelism 675 | false 676 | 677 | 678 | 679 | hive.tez.container.size 680 | 1024 681 | 682 | 683 | 684 | hive.tez.cpu.vcores 685 | -1 686 | 687 | 688 | 689 | hive.tez.dynamic.partition.pruning 690 | true 691 | 692 | 693 | 694 | hive.tez.dynamic.partition.pruning.max.data.size 695 | 104857600 696 | 697 | 698 | 699 | hive.tez.dynamic.partition.pruning.max.event.size 700 | 1048576 701 | 702 | 703 | 704 | hive.tez.input.format 705 | org.apache.hadoop.hive.ql.io.HiveInputFormat 706 | 707 | 708 | 709 | hive.tez.java.opts 710 | -server -Xmx200m -Djava.net.preferIPv4Stack=true 711 | 712 | 713 | 714 | hive.tez.log.level 715 | INFO 716 | 717 | 718 | 719 | hive.tez.max.partition.factor 720 | 2.0 721 | 722 | 723 | 724 | hive.tez.min.partition.factor 725 | 0.25 726 | 727 | 728 | 729 | hive.tez.smb.number.waves 730 | 0.5 731 | 732 | 733 | 734 | hive.txn.manager 735 | org.apache.hadoop.hive.ql.lockmgr.DbTxnManager 736 | 737 | 738 | 739 | hive.txn.max.open.batch 740 | 1000 741 | 742 | 743 | 744 | hive.txn.timeout 745 | 300 746 | 747 | 748 | 749 | hive.user.install.directory 750 | /user/ 751 | 752 | 753 | 754 | hive.users.in.admin.role 755 | hue,hive 756 | 757 | 758 | 759 | hive.vectorized.execution.enabled 760 | true 761 | 762 | 763 | 764 | hive.vectorized.execution.reduce.enabled 765 | false 766 | 767 | 768 | 769 | hive.vectorized.groupby.checkinterval 770 | 4096 771 | 772 | 773 | 774 | hive.vectorized.groupby.flush.percent 775 | 0.1 776 | 777 | 778 | 779 | hive.vectorized.groupby.maxentries 780 | 100000 781 | 782 | 783 | 784 | hive.zookeeper.client.port 785 | 2181 786 | 787 | 788 | 789 | hive.zookeeper.namespace 790 | hive_zookeeper_namespace 791 | 792 | 793 | 794 | hive.zookeeper.quorum 795 | MASTER-NODE-HOSTNAME:2181 796 | 797 | 798 | 799 | hive_metastore_user_passwd 800 | hive 801 | 802 | 803 | 804 | javax.jdo.option.ConnectionDriverName 805 | com.mysql.jdbc.Driver 806 | 807 | 808 | 809 | javax.jdo.option.ConnectionURL 810 | jdbc:mysql://MASTER-NODE-HOSTNAME/hive?createDatabaseIfNotExist=true 811 | 812 | 813 | 814 | javax.jdo.option.ConnectionUserName 815 | hive 816 | 817 | 818 | -------------------------------------------------------------------------------- /client-conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mapreduce.framework.name 5 | yarn 6 | 7 | 8 | -------------------------------------------------------------------------------- /client-conf/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.nodemanager.aux-services 5 | mapreduce_shuffle 6 | 7 | 8 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 9 | org.apache.hadoop.mapred.ShuffleHandler 10 | 11 | 12 | yarn.resourcemanager.hostname 13 | HADOOP_NAMENODE_HOST 14 | 15 | 16 | yarn.nodemanager.hostname 17 | HADOOP_CURRENT_HOST 18 | 19 | 20 | yarn.nodemanager.webapp.address 21 | ${yarn.nodemanager.hostname}:8042 22 | 23 | 24 | yarn.log-aggregation-enable 25 | true 26 | 27 | 28 | 29 | Where to aggregate logs to. 30 | yarn.nodemanager.remote-app-log-dir 31 | /logs/hadoop 32 | 33 | 34 | 35 | yarn.log-aggregation.retain-seconds 36 | 259200 37 | 38 | 39 | 40 | yarn.log-aggregation.retain-check-interval-seconds 41 | 3600 42 | 43 | 44 | --------------------------------------------------------------------------------