├── .DS_Store ├── .github └── workflows │ └── docker-image.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── conf ├── metastore-log4j2.properties └── metastore-site.xml ├── scripts ├── entrypoint.sh └── setup-container-structure-test.sh └── test_config.yaml /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/docker-hive/edbc96b4d35b2b802f859d1b5d10f3eb2046649d/.DS_Store -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: docker-hive CI 2 | on: 3 | push: 4 | paths-ignore: 5 | - '**.md' 6 | pull_request: 7 | paths-ignore: 8 | - '**.md' 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Build the image 16 | uses: docker/build-push-action@v2 17 | with: 18 | context: . 19 | push: false 20 | tags: docker-hive:latest 21 | 22 | - name: run structure tests 23 | uses: plexsystems/container-structure-test-action@v0.1.0 24 | with: 25 | image: docker-hive:latest 26 | config: test_config.yaml 27 | 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM azul/zulu-openjdk-debian:17 2 | 3 | WORKDIR /opt 4 | 5 | ENV HADOOP_HOME=/opt/hadoop 6 | ENV HADOOP_VERSION=3.3.5 7 | ENV HIVE_HOME=/opt/hive 8 | ENV HIVE_VERSION=3.1.3 9 | ENV LOG4J_VERSION=2.20.0 10 | ENV LOG4J_LOCATION="https://repo1.maven.org/maven2/org/apache/logging/log4j" 11 | 12 | RUN mkdir ${HIVE_HOME} 13 | RUN mkdir ${HADOOP_HOME} 14 | RUN apt-get clean && \ 15 | apt-get update && \ 16 | apt-get upgrade -y && \ 17 | apt-get -qqy install curl && \ 18 | curl -L https://dlcdn.apache.org/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz | tar zxf - && \ 19 | curl -L https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | tar zxf - && \ 20 | mv apache-hive-${HIVE_VERSION}-bin/* ${HIVE_HOME} && \ 21 | mv hadoop-${HADOOP_VERSION}/* ${HADOOP_HOME} && \ 22 | apt-get install --only-upgrade openssl libssl1.1 libexpat1 && \ 23 | apt-get install -y libk5crypto3 libkrb5-3 libsqlite3-0 24 | 25 | RUN rm ${HIVE_HOME}/lib/postgresql-9.4.1208.jre7.jar 26 | 27 | RUN curl -o ${HIVE_HOME}/lib/postgresql-42.2.25.jre7.jar -L https://jdbc.postgresql.org/download/postgresql-42.2.25.jre7.jar 28 | 29 | # Configure Hadoop AWS Jars to be available to hive 30 | RUN ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/*aws* ${HIVE_HOME}/lib 31 | 32 | COPY conf ${HIVE_HOME}/conf 33 | COPY scripts/entrypoint.sh ${HIVE_HOME}/entrypoint.sh 34 | 35 | # Remove vulnerable Log4j version and install latest 36 | RUN \ 37 | rm -f ${HADOOP_HOME}/share/hadoop/common/lib/slf4j-log4j12* && \ 38 | rm -f ${HADOOP_HOME}/share/hadoop/common/lib/log4j* && \ 39 | rm -f ${HADOOP_HOME}/share/hadoop/hdfs/lib/log4j* && \ 40 | rm -f ${HADOOP_HOME}/share/hadoop/yarn/hadoop-yarn-applications-catalog-webapp-3.3.1.war && \ 41 | rm -f ${HIVE_HOME}/lib/log4j-* && \ 42 | curl -o ${HIVE_HOME}/lib/log4j-1.2-api-${LOG4J_VERSION}.jar ${LOG4J_LOCATION}/log4j-1.2-api/${LOG4J_VERSION}/log4j-1.2-api-${LOG4J_VERSION}.jar && \ 43 | curl -o ${HIVE_HOME}/lib/log4j-api-${LOG4J_VERSION}.jar ${LOG4J_LOCATION}/log4j-api/${LOG4J_VERSION}/log4j-api-${LOG4J_VERSION}.jar && \ 44 | curl -o ${HIVE_HOME}/lib/log4j-core-${LOG4J_VERSION}.jar ${LOG4J_LOCATION}/log4j-core/${LOG4J_VERSION}/log4j-core-${LOG4J_VERSION}.jar && \ 45 | curl -o ${HIVE_HOME}/lib/log4j-slf4j-impl-${LOG4J_VERSION}.jar ${LOG4J_LOCATION}/log4j-slf4j-impl/${LOG4J_VERSION}/log4j-slf4j-impl-${LOG4J_VERSION}.jar 46 | 47 | # https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html 48 | # Java caches dns results forever, don't cache dns results forever: 49 | RUN touch ${JAVA_HOME}/lib/security/java.security 50 | RUN sed -i '/networkaddress.cache.ttl/d' ${JAVA_HOME}/lib/security/java.security 51 | RUN sed -i '/networkaddress.cache.negative.ttl/d' ${JAVA_HOME}/lib/security/java.security 52 | RUN echo 'networkaddress.cache.ttl=0' >> ${JAVA_HOME}/lib/security/java.security 53 | RUN echo 'networkaddress.cache.negative.ttl=0' >> ${JAVA_HOME}/lib/security/java.security 54 | 55 | # imagebuilder expects the directory to be created before VOLUME 56 | RUN mkdir -p /var/lib/hive /.beeline ${HOME}/.beeline 57 | # to allow running as non-root 58 | RUN chown -R 1002:0 ${HIVE_HOME} ${HADOOP_HOME} /var/lib/hive /.beeline ${HOME}/.beeline /etc/passwd $(readlink -f ${JAVA_HOME}/lib/security/cacerts) && \ 59 | chmod -R u+rwx,g+rwx ${HIVE_HOME} ${HADOOP_HOME} /var/lib/hive /.beeline ${HOME}/.beeline /etc/passwd $(readlink -f ${JAVA_HOME}/lib/security/cacerts) && \ 60 | chown 1002:0 ${HIVE_HOME}/entrypoint.sh && chmod +x ${HIVE_HOME}/entrypoint.sh 61 | 62 | USER 1002 63 | WORKDIR $HIVE_HOME 64 | EXPOSE 9083 65 | 66 | ENTRYPOINT ["sh", "-c", "/opt/hive/entrypoint.sh"] 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CONTAINER_IMAGE_NAME := test 2 | 3 | setup-container-structure-test: 4 | scripts/setup-container-structure-test.sh 5 | 6 | build-docker-image: 7 | docker build -t "$(CONTAINER_IMAGE_NAME)" . 8 | 9 | test-docker-image: 10 | container-structure-test test --image "$(CONTAINER_IMAGE_NAME)" --config ./test_config.yaml 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hive Meatastore 2 | 3 | ![Hive Metastore Build](https://github.com/IBM/docker-hive/actions/workflows/docker-image.yml/badge.svg) 4 | 5 | ## Build 6 | 7 | CD is run through DockerHub in [this repo](https://cloud.docker.com/repository/docker/meneal/docker-hive/general). 8 | 9 | ```SHELL 10 | docker build -t "IBM/hive-metastore:master" . 11 | ``` 12 | -------------------------------------------------------------------------------- /conf/metastore-log4j2.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | status = INFO 18 | name = HiveLog4j2 19 | packages = org.apache.hadoop.hive.ql.log 20 | 21 | # list of properties 22 | property.hive.log.level = INFO 23 | property.hive.root.logger = console 24 | property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} 25 | property.hive.log.file = hive.log 26 | property.hive.perflogger.log.level = INFO 27 | 28 | # list of all appenders 29 | appenders = console 30 | 31 | # console appender 32 | appender.console.type = Console 33 | appender.console.name = console 34 | appender.console.target = SYSTEM_ERR 35 | appender.console.layout.type = PatternLayout 36 | appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n 37 | 38 | # list of all loggers 39 | loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger 40 | 41 | logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn 42 | logger.NIOServerCnxn.level = WARN 43 | 44 | logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO 45 | logger.ClientCnxnSocketNIO.level = WARN 46 | 47 | logger.DataNucleus.name = DataNucleus 48 | logger.DataNucleus.level = ERROR 49 | 50 | logger.Datastore.name = Datastore 51 | logger.Datastore.level = ERROR 52 | 53 | logger.JPOX.name = JPOX 54 | logger.JPOX.level = ERROR 55 | 56 | logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger 57 | logger.PerfLogger.level = ${sys:hive.perflogger.log.level} 58 | 59 | # root logger 60 | rootLogger.level = ${sys:hive.log.level} 61 | rootLogger.appenderRefs = root 62 | rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} 63 | -------------------------------------------------------------------------------- /conf/metastore-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar:${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar 4 | export HIVE_OPTS="${HIVE_OPTS} --hiveconf metastore.root.logger=${HIVE_LOGLEVEL},console " 5 | export PATH=${HIVE_HOME}/bin:${HADOOP_HOME}/bin:$PATH 6 | 7 | set +e 8 | if schematool -dbType postgres -info -verbose; then 9 | echo "Hive metastore schema verified." 10 | else 11 | if schematool -dbType postgres -initSchema -verbose; then 12 | echo "Hive metastore schema created." 13 | else 14 | echo "Error creating hive metastore: $?" 15 | fi 16 | fi 17 | set -e 18 | 19 | start-metastore 20 | -------------------------------------------------------------------------------- /scripts/setup-container-structure-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # install tool container-structure-test 4 | 5 | install-container-structure-test() { 6 | os_name=$(uname 2> /dev/null | tr '[:upper:]' '[:lower:]') 7 | 8 | file_name="container-structure-test-${os_name}-amd64" 9 | release="v1.7.0" 10 | 11 | curl -LO https://storage.googleapis.com/container-structure-test/${release}/${file_name} && 12 | mkdir -p ${HOME}/bin && 13 | mv ${file_name} ${HOME}/bin/container-structure-test && 14 | chmod +x $HOME/bin/container-structure-test 15 | } 16 | 17 | if [ -z "$(which container-structure-test)" ]; then 18 | install-container-structure-test 19 | else 20 | echo "container-structure-test is installed already. skipping." 21 | fi 22 | -------------------------------------------------------------------------------- /test_config.yaml: -------------------------------------------------------------------------------- 1 | schemaVersion: "2.0.0" 2 | commandTests: 3 | - name: jre 4 | command: java 5 | args: ["-version"] 6 | expectedError: ['openjdk version "17.0.6"*'] 7 | - name: openssl 8 | command: apt 9 | args: ["list", "openssl"] 10 | expectedOutput: ['.*1.1.1n-0\+deb11u3*.'] 11 | fileExistenceTests: 12 | - name: hive-home 13 | path: /opt/hive 14 | shouldExist: true 15 | uid: 1002 16 | gid: 0 17 | - name: metastore-site.xml 18 | path: /opt/hive/conf/metastore-site.xml 19 | shouldExist: true 20 | - name: log4j2.properties 21 | path: /opt/hive/conf/metastore-log4j2.properties 22 | shouldExist: true 23 | - name: aws-java-sdk-bundle-jar 24 | path: /opt/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-1.12.262.jar 25 | shouldExist: true 26 | - name: hadoop-aws-jar 27 | path: /opt/hadoop/share/hadoop/tools/lib/hadoop-aws-3.3.4.jar 28 | shouldExist: true 29 | metadataTest: 30 | env: 31 | - key: JAVA_HOME 32 | value: /usr/lib/jvm/zulu17 33 | - key: HADOOP_HOME 34 | value: /opt/hadoop 35 | - key: HIVE_HOME 36 | value: /opt/hive 37 | exposedPorts: ["9083"] 38 | workdir: "/opt/hive" 39 | entrypoint: ["sh", "-c", "/opt/hive/entrypoint.sh"] 40 | cmd: [] 41 | --------------------------------------------------------------------------------