├── .gitignore ├── ACCUMULO ├── pom.xml └── src │ ├── aux │ ├── accumulo-metrics.xml │ └── client │ │ ├── accumulo-env.sh │ │ └── log4j.properties │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ └── accumulo.sh ├── ACCUMULO16 ├── pom.xml └── src │ ├── aux │ ├── accumulo-metrics.xml │ └── client │ │ ├── accumulo-env.sh │ │ └── log4j.properties │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ ├── accumulo.sh │ ├── generic_extra_log4j.properties │ └── monitor_extra_log4j.properties ├── ECHO ├── pom.xml └── src │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ ├── cc.sh │ └── control.sh ├── KAFKA ├── assembly.xml ├── pom.xml ├── src │ ├── descriptor │ │ └── service.sdl │ ├── images │ │ └── kafka.png │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── kafka │ │ │ │ └── metrics │ │ │ │ ├── KafkaServiceMetricsSchemaGenerator.java │ │ │ │ ├── UnitConstants.java │ │ │ │ ├── broker │ │ │ │ ├── BrokerMetrics.java │ │ │ │ ├── BrokerTopicMetrics.java │ │ │ │ ├── ControllerMetrics.java │ │ │ │ ├── NetworkMetrics.java │ │ │ │ ├── PurgatoryMetrics.java │ │ │ │ ├── ReplicaManagerMetrics.java │ │ │ │ └── RequestMetrics.java │ │ │ │ ├── replica │ │ │ │ └── ReplicaMetrics.java │ │ │ │ └── topic │ │ │ │ └── TopicMetrics.java │ │ └── resources │ │ │ └── kafka_base.mdl │ └── scripts │ │ ├── control.sh │ │ ├── deploy_client_config.sh │ │ └── mirrormaker_control.sh └── test │ └── kafka_test.sh ├── KEYTRUSTEE ├── pom.xml └── src │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ └── control.sh ├── KMS ├── pom.xml └── src │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ └── control.sh ├── LICENSE.txt ├── README.md ├── SPARK ├── pom.xml └── src │ ├── aux │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ ├── common.sh │ └── control.sh ├── SPARK_ON_YARN ├── pom.xml └── src │ ├── aux │ └── client │ │ └── spark-env.sh │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.png │ └── scripts │ ├── common.sh │ └── control.sh ├── SPARK_ON_YARN53 ├── pom.xml └── src │ ├── aux │ ├── descriptor │ └── service.sdl │ ├── images │ └── scripts ├── SPARK_ON_YARN54 ├── pom.xml └── src │ ├── aux │ ├── descriptor │ └── service.sdl │ ├── images │ └── scripts ├── SPARK_ON_YARN5_10 ├── pom.xml └── src │ ├── aux │ ├── descriptor │ └── service.sdl │ ├── images │ └── scripts ├── SQOOP_CLIENT ├── pom.xml └── src │ ├── aux │ └── sqoop-env.sh │ ├── descriptor │ └── service.sdl │ ├── images │ └── sqoop.png │ └── scripts │ └── sqoop.sh ├── ZEPPELIN ├── assembly.xml ├── pom.xml └── src │ ├── aux │ ├── interpreter.json │ └── zeppelin-env.sh │ ├── descriptor │ └── service.sdl │ ├── images │ └── icon.svg │ └── scripts │ ├── common.sh │ ├── control.sh │ └── update_interpreter.py ├── assembly.xml └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | eclipse-classes 3 | .classpath 4 | .settings 5 | .project 6 | 7 | # Gradle 8 | build 9 | 10 | # Python compiled files 11 | *.pyc 12 | *.pyo 13 | 14 | # vim artifacts 15 | .*.swp 16 | .*.swo 17 | 18 | # OS X Artifacts 19 | .DS_Store 20 | 21 | # IntelliJ artifacts 22 | *.iml 23 | *.ipr 24 | *.iws 25 | .idea 26 | 27 | # emacs backup files 28 | *~ 29 | 30 | # NetBeans artifacts 31 | nb-configuration.xml 32 | nbactions.xml 33 | -------------------------------------------------------------------------------- /ACCUMULO/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | ACCUMULO 30 | 5.14.0 31 | The Accumulo CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /ACCUMULO/src/aux/accumulo-metrics.xml: -------------------------------------------------------------------------------- 1 | 17 | 21 | 22 | 25 | 26 | ${ACCUMULO_LOG_DIR}/metrics 27 | 28 | 32 | 33 | false 34 | false 35 | 36 | 37 | false 38 | false 39 | 40 | 41 | false 42 | false 43 | 44 | false 45 | false 46 | 47 | 48 | false 49 | false 50 | 51 | 52 | false 53 | false 54 | 55 | 56 | 57 | false 58 | false 59 | 60 | 61 | -------------------------------------------------------------------------------- /ACCUMULO/src/aux/client/accumulo-env.sh: -------------------------------------------------------------------------------- 1 | ## 2 | # 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="{{accumulo_general_opts}}" 21 | test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="{{accumulo_other_opts}}" 22 | export MONITOR='unused' 23 | export ACCUMULO_VERIFY_ONLY='true' 24 | -------------------------------------------------------------------------------- /ACCUMULO/src/aux/client/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # default logging properties: 17 | # by default, log everything at INFO or higher to the console 18 | log4j.rootLogger=INFO,A1 19 | 20 | 21 | # hide Jetty junk 22 | log4j.logger.org.mortbay.log=WARN,A1 23 | 24 | # hide "Got brand-new compresssor" messages 25 | log4j.logger.org.apache.hadoop.io.compress=WARN,A1 26 | 27 | # hide junk from TestRandomDeletes 28 | log4j.logger.org.apache.accumulo.server.test.TestRandomDeletes=WARN,A1 29 | 30 | # hide almost everything from zookeeper 31 | log4j.logger.org.apache.zookeeper=ERROR,A1 32 | 33 | # hide AUDIT messages in the shell, alternatively you could send them to a different logger 34 | log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1 35 | 36 | # Send most things to the console 37 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 38 | log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n 39 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 40 | -------------------------------------------------------------------------------- /ACCUMULO/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/ACCUMULO/src/images/icon.png -------------------------------------------------------------------------------- /ACCUMULO/src/scripts/accumulo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # 4 | # Licensed to Cloudera, Inc. under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. Cloudera, Inc. licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | ## 20 | 21 | add_to_accumulo_site() { 22 | FILE=`find $CONF_DIR -name accumulo-site.xml` 23 | CONF_END="" 24 | NEW_PROPERTY="$1$2" 25 | TMP_FILE=$CONF_DIR/tmp-accumulo-site 26 | cat $FILE | sed "s#$CONF_END#$NEW_PROPERTY#g" > $TMP_FILE 27 | cp $TMP_FILE $FILE 28 | rm -f $TMP_FILE 29 | echo $CONF_END >> $FILE 30 | } 31 | 32 | set -x 33 | export CDH_VERSION=4 34 | 35 | # Source the common script to use acquire_kerberos_tgt 36 | . $COMMON_SCRIPT 37 | 38 | # Set env vars needed for Accumulo 39 | export ACCUMULO_HOME=${CDH_ACCUMULO_HOME:-/usr/lib/accumulo} 40 | export HADOOP_HOME_WARN_SUPPRESS=true 41 | export HADOOP_HOME=$CDH_HADOOP_HOME 42 | export HADOOP_PREFIX=$HADOOP_HOME 43 | export HADOOP_CLIENT_HOME=$HADOOP_HOME/client-0.20 44 | export HADOOP_MAPRED_HOME=$HADOOP_HOME/../hadoop-0.20-mapreduce 45 | if [ "$1" = "client" ]; then 46 | export HADOOP_CONF_DIR=/etc/hadoop/conf 47 | else 48 | export HADOOP_CONF_DIR=$CONF_DIR/hadoop-conf 49 | fi 50 | export ACCUMULO_CONF_DIR=$CONF_DIR 51 | export ZOOKEEPER_HOME=$CDH_ZOOKEEPER_HOME 52 | 53 | # Set this because we don't want accumulo's config.sh script to create directories 54 | export ACCUMULO_VERIFY_ONLY=true 55 | 56 | # Set GC and MONITOR because otherwise accumulo's config.sh will exit 57 | export GC=unused 58 | export MONITOR=unused 59 | 60 | cp $CONF_DIR/aux/accumulo-metrics.xml $CONF_DIR/ 61 | 62 | # Add zk quorum to accumulo-site.xml 63 | add_to_accumulo_site instance.zookeeper.host $ZK_QUORUM 64 | 65 | # Add classpath to accumulo-site.xml 66 | FULL_CLASSPATH="\$ACCUMULO_HOME/src/server/target/classes/,\$ACCUMULO_HOME/src/core/target/classes/,\$ACCUMULO_HOME/src/start/target/classes/,\$ACCUMULO_HOME/src/examples/target/classes/,\$ACCUMULO_HOME/lib/[^.].\$ACCUMULO_VERSION.jar,\$ACCUMULO_HOME/lib/[^.].*.jar,\$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\$HADOOP_HOME/[^.].*.jar,\$HADOOP_HOME/lib/[^.].*.jar,\$HADOOP_CLIENT_HOME/[^.].*.jar,\$HADOOP_MAPRED_HOME/[^.].*.jar,\$HADOOP_MAPRED_HOME/lib/[^.].*.jar,\$HADOOP_CONF_DIR" 67 | 68 | if [ "$ACCUMULO_CLASSPATH" != "" ]; then 69 | # Pre-pend any user specified directories 70 | FULL_CLASSPATH="$ACCUMULO_CLASSPATH,$FULL_CLASSPATH" 71 | fi 72 | add_to_accumulo_site general.classpaths $FULL_CLASSPATH 73 | 74 | if [ "$accumulo_principal" != "" ]; then 75 | add_to_accumulo_site general.kerberos.keytab $CONF_DIR/accumulo.keytab 76 | fi 77 | 78 | if [ -z $ACCUMULO_OTHER_OPTS ]; then 79 | export ACCUMULO_OTHER_OPTS=" -Xmx1g " 80 | fi 81 | 82 | if [ "$1" = "master" ]; then 83 | $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.master.state.SetGoalState NORMAL 84 | elif [ "$1" = "client" ]; then 85 | CLIENT_CONF_DIR=$CONF_DIR/accumulo-conf 86 | perl -pi -e "s#{{accumulo_general_opts}}#$ACCUMULO_GENERAL_OPTS#g" $CLIENT_CONF_DIR/accumulo-env.sh 87 | perl -pi -e "s#{{accumulo_other_opts}}#$ACCUMULO_OTHER_OPTS#g" $CLIENT_CONF_DIR/accumulo-env.sh 88 | chmod 777 $CLIENT_CONF_DIR/* 89 | exit 0 90 | elif [ "$1" = "init" ]; then 91 | echo $INSTANCE_NAME > script 92 | echo $INSTANCE_PASSWORD >> script 93 | echo $INSTANCE_PASSWORD >> script 94 | if [ "$accumulo_principal" != "" ]; then 95 | export SCM_KERBEROS_PRINCIPAL=$accumulo_principal 96 | acquire_kerberos_tgt accumulo.keytab 97 | fi 98 | cat script | $ACCUMULO_HOME/bin/accumulo init 99 | exit $? 100 | fi 101 | 102 | if [ "$1" = "admin" ]; then 103 | exec $ACCUMULO_HOME/bin/accumulo "$@" 104 | else 105 | HOST=`hostname` 106 | exec $ACCUMULO_HOME/bin/accumulo $1 --address $HOST 107 | fi 108 | -------------------------------------------------------------------------------- /ACCUMULO16/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | ACCUMULO16 30 | 5.14.0 31 | The Accumulo 1.6 CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /ACCUMULO16/src/aux/accumulo-metrics.xml: -------------------------------------------------------------------------------- 1 | 17 | 21 | 22 | 25 | 26 | ${ACCUMULO_LOG_DIR}/metrics 27 | 28 | 32 | 33 | false 34 | false 35 | 36 | 37 | false 38 | false 39 | 40 | 41 | false 42 | false 43 | 44 | false 45 | false 46 | 47 | 48 | false 49 | false 50 | 51 | 52 | false 53 | false 54 | 55 | 56 | 57 | false 58 | false 59 | 60 | 61 | -------------------------------------------------------------------------------- /ACCUMULO16/src/aux/client/accumulo-env.sh: -------------------------------------------------------------------------------- 1 | ## 2 | # 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="{{accumulo_general_opts}}" 21 | test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="{{accumulo_other_opts}}" 22 | export MONITOR='unused' 23 | export ACCUMULO_VERIFY_ONLY='true' 24 | -------------------------------------------------------------------------------- /ACCUMULO16/src/aux/client/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # default logging properties: 17 | # by default, log everything at INFO or higher to the console 18 | log4j.rootLogger=INFO,A1 19 | 20 | # hide Jetty junk 21 | log4j.logger.org.mortbay.log=WARN,A1 22 | 23 | # hide "Got brand-new compressor" messages 24 | log4j.logger.org.apache.hadoop.io.compress=WARN,A1 25 | log4j.logger.org.apache.accumulo.core.file.rfile.bcfile.Compression=WARN,A1 26 | 27 | # hide junk from TestRandomDeletes 28 | log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1 29 | 30 | # hide junk from VFS 31 | log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1 32 | 33 | # hide almost everything from zookeeper 34 | log4j.logger.org.apache.zookeeper=ERROR,A1 35 | 36 | # hide AUDIT messages in the shell, alternatively you could send them to a different logger 37 | log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1 38 | 39 | # Send most things to the console 40 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 41 | log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n 42 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 43 | -------------------------------------------------------------------------------- /ACCUMULO16/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/ACCUMULO16/src/images/icon.png -------------------------------------------------------------------------------- /ACCUMULO16/src/scripts/accumulo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # 4 | # Licensed to Cloudera, Inc. under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. Cloudera, Inc. licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | ## 20 | 21 | add_to_accumulo_site() { 22 | FILE=`find $CONF_DIR -name accumulo-site.xml` 23 | CONF_END="" 24 | NEW_PROPERTY="$1$2" 25 | TMP_FILE=$CONF_DIR/tmp-accumulo-site 26 | cat $FILE | sed "s#$CONF_END#$NEW_PROPERTY#g" > $TMP_FILE 27 | cp $TMP_FILE $FILE 28 | rm -f $TMP_FILE 29 | echo $CONF_END >> $FILE 30 | } 31 | 32 | set -x 33 | 34 | # Source the common script to use acquire_kerberos_tgt 35 | . $COMMON_SCRIPT 36 | 37 | # Set env vars needed for Accumulo 38 | export ACCUMULO_HOME=${CDH_ACCUMULO_HOME:-/usr/lib/accumulo} 39 | export HADOOP_HOME_WARN_SUPPRESS=true 40 | export HADOOP_HOME=$CDH_HADOOP_HOME 41 | export HADOOP_PREFIX=$HADOOP_HOME 42 | export HADOOP_CLIENT_HOME=$HADOOP_HOME/client-0.20 43 | if [ $CDH_VERSION -gt 4 ]; then 44 | export HADOOP_CLIENT_HOME=$HADOOP_HOME/client 45 | fi 46 | if [ "$1" = "client" ]; then 47 | export HADOOP_CONF_DIR=/etc/hadoop/conf 48 | else 49 | export HADOOP_CONF_DIR=$CONF_DIR/hadoop-conf 50 | fi 51 | export ACCUMULO_CONF_DIR=$CONF_DIR 52 | export ZOOKEEPER_HOME=$CDH_ZOOKEEPER_HOME 53 | 54 | # Set this because we don't want accumulo's config.sh script to create directories 55 | export ACCUMULO_VERIFY_ONLY=true 56 | export ACCUMULO_TEST=true 57 | 58 | # Set GC and MONITOR because otherwise accumulo's config.sh will exit 59 | export GC=unused 60 | export MONITOR=unused 61 | 62 | cp $CONF_DIR/aux/accumulo-metrics.xml $CONF_DIR/ 63 | 64 | # Add zk quorum to accumulo-site.xml 65 | add_to_accumulo_site instance.zookeeper.host $ZK_QUORUM 66 | 67 | # Add classpath to accumulo-site.xml 68 | FULL_CLASSPATH="\$ACCUMULO_HOME/lib/[^.].*.jar" 69 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CONF_DIR" 70 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CLIENT_HOME/[^.](?!lf4j-log4j|uava|vro).*-[0-9a.]*.jar" 71 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CLIENT_HOME/slf4j-log4j12.jar" 72 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CLIENT_HOME/avro.jar" 73 | # Needed for C5.5+ until HTrace has a non-incubating release. 74 | # on C5.4 will add an unneeded extra jar to the classpath. 75 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CLIENT_HOME/htrace-core.jar" 76 | # and on later versions with HTrace 4. 77 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CLIENT_HOME/htrace-core4.jar" 78 | FULL_CLASSPATH="$FULL_CLASSPATH,\$HADOOP_CLIENT_HOME/[^.](?!ookeeper).*-[0-9.]*(?:-[^-]*)?-cdh.*.jar" 79 | FULL_CLASSPATH="$FULL_CLASSPATH,\$ZOOKEEPER_HOME/zookeeper.*-[0-9].*.jar" 80 | 81 | if [ "$ACCUMULO_CLASSPATH" != "" ]; then 82 | # Pre-pend any user specified directories 83 | FULL_CLASSPATH="$ACCUMULO_CLASSPATH,$FULL_CLASSPATH" 84 | fi 85 | add_to_accumulo_site general.classpaths $FULL_CLASSPATH 86 | 87 | if [ "$accumulo_principal" != "" ]; then 88 | add_to_accumulo_site general.kerberos.keytab $CONF_DIR/accumulo16.keytab 89 | fi 90 | 91 | if [ -z $ACCUMULO_OTHER_OPTS ]; then 92 | export ACCUMULO_OTHER_OPTS=" -Xmx1g " 93 | fi 94 | 95 | 96 | # New for 1.6 - Add extra properties to logging configuration 97 | if [ "$1" = "monitor" ]; then 98 | sed -i '/log4j\.appender\.RFA\.layout\.ConversionPattern/d' $CONF_DIR/log4j.properties 99 | cat $CONF_DIR/scripts/monitor_extra_log4j.properties >> $CONF_DIR/log4j.properties 100 | mv $CONF_DIR/log4j.properties $CONF_DIR/monitor_logger.properties 101 | elif [ "$1" != "client" ]; then 102 | sed -i '/log4j\.appender\.RFA\.layout\.ConversionPattern/d' $CONF_DIR/log4j.properties 103 | cat $CONF_DIR/scripts/generic_extra_log4j.properties >> $CONF_DIR/log4j.properties 104 | mv $CONF_DIR/log4j.properties $CONF_DIR/generic_logger.properties 105 | fi 106 | # End of new work for 1.6 107 | 108 | 109 | if [ "$1" = "master" ]; then 110 | $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.master.state.SetGoalState NORMAL 111 | elif [ "$1" = "client" ]; then 112 | CLIENT_CONF_DIR=$CONF_DIR/accumulo-conf 113 | perl -pi -e "s#{{accumulo_general_opts}}#$ACCUMULO_GENERAL_OPTS#g" $CLIENT_CONF_DIR/accumulo-env.sh 114 | perl -pi -e "s#{{accumulo_other_opts}}#$ACCUMULO_OTHER_OPTS#g" $CLIENT_CONF_DIR/accumulo-env.sh 115 | chmod 777 $CLIENT_CONF_DIR/* 116 | exit 0 117 | elif [ "$1" = "init" ]; then 118 | echo $INSTANCE_NAME > script 119 | echo $INSTANCE_PASSWORD >> script 120 | echo $INSTANCE_PASSWORD >> script 121 | if [ "$accumulo_principal" != "" ]; then 122 | export SCM_KERBEROS_PRINCIPAL=$accumulo_principal 123 | acquire_kerberos_tgt accumulo16.keytab 124 | fi 125 | cat script | $ACCUMULO_HOME/bin/accumulo init 126 | exit $? 127 | elif [ "$1" = "upgrade" ]; then 128 | echo "Upgrading Accumulo WALs from 1.4 to 1.6" 129 | exec $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.tserver.log.LocalWALRecovery 130 | fi 131 | 132 | if [ "$1" = "admin" ]; then 133 | exec $ACCUMULO_HOME/bin/accumulo "$@" 134 | else 135 | HOST=`hostname` 136 | if [ "$1" = "monitor" -a "$ACCUMULO_MONITOR_BIND_ALL" = "true" ]; then 137 | HOST="0.0.0.0" 138 | fi 139 | exec $ACCUMULO_HOME/bin/accumulo $1 --address $HOST 140 | fi 141 | -------------------------------------------------------------------------------- /ACCUMULO16/src/scripts/generic_extra_log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n 2 | 3 | # Send all logging data to a centralized logger 4 | # If the centralized logger is down, buffer the log events, but drop them if it stays down 5 | log4j.appender.ASYNC=org.apache.accumulo.core.util.AsyncSocketAppender 6 | log4j.appender.ASYNC.RemoteHost=${org.apache.accumulo.core.host.log} 7 | log4j.appender.ASYNC.Port=${org.apache.accumulo.core.host.log.port} 8 | log4j.appender.ASYNC.Application=${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname} 9 | log4j.appender.ASYNC.Threshold=WARN 10 | 11 | # Log accumulo events to the debug, normal and remote logs. 12 | log4j.logger.org.apache.accumulo=${log.threshold}, RFA, ASYNC 13 | log4j.additivity.org.apache.accumulo=false 14 | 15 | # change to INFO for authorization events 16 | log4j.logger.org.apache.accumulo.server.security.Auditor=WARN 17 | 18 | log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO 19 | 20 | log4j.logger.org.apache.accumulo.examples.wikisearch=INFO 21 | 22 | log4j.logger.org.mortbay.log=WARN 23 | 24 | log4j.logger.com.yahoo.zookeeper=ERROR 25 | -------------------------------------------------------------------------------- /ACCUMULO16/src/scripts/monitor_extra_log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n 2 | 3 | # Keep the last few log messages for display to the user 4 | log4j.appender.GUI=org.apache.accumulo.server.monitor.LogService 5 | log4j.appender.GUI.Keep=50 6 | log4j.appender.GUI.Threshold=WARN 7 | 8 | # Log accumulo messages to debug, normal and GUI 9 | log4j.logger.org.apache.accumulo=${log.threshold}, RFA, GUI 10 | log4j.additivity.org.apache.accumulo=false 11 | -------------------------------------------------------------------------------- /ECHO/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | com.cloudera 8 | csd 9 | 5.14.0 10 | 11 | 12 | com.cloudera.csd 13 | ECHO 14 | 5.14.0 15 | The Echo Service CSD 16 | pom 17 | 18 | 19 | 20 | 21 | org.apache.maven.plugins 22 | maven-assembly-plugin 23 | 24 | false 25 | 26 | ../assembly.xml 27 | 28 | 29 | 30 | 31 | make-assembly 32 | package 33 | 34 | single 35 | 36 | 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | validate-schema 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /ECHO/src/descriptor/service.sdl: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "ECHO", 3 | "label" : "Echo", 4 | "description" : "The echo service", 5 | "version" : "5.8.0", 6 | "runAs" : { 7 | "user" : "root", 8 | "group" : "root" 9 | }, 10 | "compatibility" : { 11 | "generation" : 1 12 | }, 13 | "icon" : "images/icon.png", 14 | "parameters" : [ 15 | { 16 | "name" : "service_var1", 17 | "label" : "Service Var1 Label", 18 | "description" : "Service Var1 Description", 19 | "configName" : "service.var1.config", 20 | "type" : "string", 21 | "default" : "this is a default" 22 | }, 23 | { 24 | "name" : "service_var2", 25 | "label" : "Service Var2 Label", 26 | "description" : "Service Var2 Description", 27 | "required" : "true", 28 | "type" : "long", 29 | "default" : 20 , 30 | "min" : 1 31 | } 32 | ], 33 | "hdfsDirs" : [ 34 | { 35 | "name" : "CreateHomeDirCommand", 36 | "label" : "Create Echo Home Dir", 37 | "description" : "Create the home directory in HDFS for echo service", 38 | "directoryDescription" : "Home directory used by Echo service", 39 | "path" : "/echo", 40 | "permissions" : "0700" 41 | } 42 | ], 43 | "commands" : [ 44 | { 45 | "name" : "EchoServiceCommandSingle", 46 | "label" : "Service Command Single Label", 47 | "description" : "Service Command Single Help", 48 | "roleName" : "ECHO_WEBSERVER", 49 | "roleCommand" : "role_cmd1", 50 | "runMode" : "single" 51 | }, 52 | { 53 | "name" : "EchoServiceCommandAll", 54 | "label" : "Service Command All Label", 55 | "description" : "Service Command All Help", 56 | "roleName" : "ECHO_WEBSERVER", 57 | "roleCommand" : "role_cmd1", 58 | "runMode" : "all" 59 | } 60 | ], 61 | "roles" : [ 62 | { 63 | "name" : "ECHO_WEBSERVER", 64 | "label" : "Web Server", 65 | "pluralLabel" : "Web Servers", 66 | "startRunner" : { 67 | "program" : "scripts/control.sh", 68 | "args" : [ 69 | "start", 70 | "./params.properties", 71 | "./workers.properties" 72 | ], 73 | "environmentVariables" : { 74 | "TEST_MESSAGE" : "Hello there people", 75 | "BACKGROUND_COLOR" : "#d2d2d2" 76 | } 77 | }, 78 | "commands" : [ 79 | { 80 | "name" : "role_cmd1", 81 | "label" : "Role Cmd1 Label", 82 | "description" : "Role Cmd1 Help", 83 | "expectedExitCodes" : [0, 1, 2], 84 | "requiredRoleState" : "running", 85 | "commandRunner" : { 86 | "program" : "scripts/control.sh", 87 | "args" : ["cmd1"] 88 | } 89 | } 90 | ], 91 | "logging" : { 92 | "dir" : "/var/log/echo", 93 | "filename" : "echo-webserver-${host}.log", 94 | "configName" : "log.dir", 95 | "loggingType" : "other" 96 | }, 97 | "parameters" : [ 98 | { 99 | "name" : "role_var1", 100 | "label" : "Role Var1 Label", 101 | "description" : "Role Var1 Description", 102 | "configName" : "role.var1.config", 103 | "type" : "string", 104 | "default" : "defaultVal" 105 | }, 106 | { 107 | "name" : "portNum", 108 | "label" : "Webserver port", 109 | "description" : "The webserver port number", 110 | "configName" : "portNum", 111 | "required" : "true", 112 | "type" : "port", 113 | "default" : 8080 114 | } 115 | ], 116 | "configWriter" : { 117 | "generators" : [ 118 | { 119 | "filename" : "params.properties", 120 | "configFormat" : "properties" 121 | } 122 | ], 123 | "peerConfigGenerators" : [ 124 | { 125 | "filename" : "workers.properties", 126 | "params" : ["portNum"] 127 | } 128 | ] 129 | } 130 | } 131 | ], 132 | "gateway" : { 133 | "alternatives" : { 134 | "name" : "echo-conf", 135 | "priority" : 50, 136 | "linkRoot" : "/etc/echo" 137 | }, 138 | "scriptRunner" : { 139 | "program" : "scripts/cc.sh", 140 | "args" : [ 141 | "arg1", 142 | "arg2" 143 | ], 144 | "environmentVariables" : { 145 | "ENV1" : "Hello there people", 146 | "ENV2" : "Hi" 147 | } 148 | }, 149 | "parameters" : [ 150 | { 151 | "name" : "client_var1", 152 | "label" : "Client Var1 Label", 153 | "description" : "Client Var1 Description", 154 | "configName" : "client.var1.config", 155 | "type" : "string" 156 | }, 157 | { 158 | "name" : "client_var2", 159 | "label" : "Client Var2 Label", 160 | "description" : "Client Var2 Description", 161 | "required" : "true", 162 | "type" : "long", 163 | "configName" : "client.var2.config", 164 | "default" : 1 165 | } 166 | ], 167 | "configWriter" : { 168 | "generators" : [ 169 | { 170 | "filename" : "echo-conf/echo_client.xml", 171 | "configFormat" : "hadoop_xml", 172 | "excludedParams" : ["client_var2"] 173 | }, 174 | { 175 | "filename" : "echo-conf/echo_client.properties", 176 | "configFormat" : "properties", 177 | "includedParams" : ["service_var2", "client_var1", "client_var2"] 178 | }, 179 | { 180 | "filename" : "echo-conf/all_params.properties", 181 | "configFormat" : "properties" 182 | } 183 | ] 184 | } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /ECHO/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/ECHO/src/images/icon.png -------------------------------------------------------------------------------- /ECHO/src/scripts/cc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Time marker for both stderr and stdout 4 | date; date 1>&2 5 | 6 | timestamp=$(date) 7 | pwd=$(pwd) 8 | 9 | echo "$timestamp Running echo deploy client config script" 10 | echo "using $CONF_DIR as CONF_DIR" 11 | echo "pwd is $pwd" 12 | echo "using $@ as args" 13 | echo "ENV1 is $ENV1" 14 | echo "ENV2 is $ENV2" 15 | -------------------------------------------------------------------------------- /ECHO/src/scripts/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Time marker for both stderr and stdout 4 | date; date 1>&2 5 | 6 | cmd=$1 7 | paramFile=$2 8 | workerFile=$3 9 | timestamp=$(date) 10 | 11 | portNum="" 12 | for line in $(cat $paramFile) 13 | do 14 | IFS='=' read -a tokens <<< "$line" 15 | key=${tokens[0]} 16 | value=${tokens[1]} 17 | if [ $key = "portNum" ] 18 | then 19 | portNum=$value 20 | fi 21 | done 22 | 23 | hostfile="hostlist" 24 | for line in $(cat $workerFile) 25 | do 26 | IFS=':' read -a tokens <<< "$line" 27 | host=${tokens[0]} 28 | echo $host >> $hostfile 29 | done 30 | 31 | if [ "start" = "$cmd" ]; then 32 | echo "$timestamp Starting Server on port $portNum" 33 | exec python -m SimpleHTTPServer $portNum 34 | elif [ "stop" = "$cmd" ]; then 35 | echo "$timestamp Stopping Server" 36 | else 37 | echo "$timestamp Don't understand [$cmd]" 38 | fi 39 | -------------------------------------------------------------------------------- /KAFKA/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | assemble 3 | 4 | jar 5 | 6 | false 7 | 8 | 9 | ${project.basedir}/src 10 | / 11 | 12 | main/** 13 | 14 | 15 | 16 | ${project.basedir}/target/classes 17 | /descriptor/ 18 | 19 | service.mdl 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /KAFKA/src/images/kafka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/KAFKA/src/images/kafka.png -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/KafkaServiceMetricsSchemaGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics; 19 | 20 | import com.cloudera.csd.descriptors.MetricDescriptor; 21 | import com.cloudera.csd.tools.JsonUtil; 22 | import com.cloudera.csd.tools.codahale.AbstractCodahaleFixtureGenerator; 23 | import com.cloudera.csd.tools.codahale.CodahaleCommonMetricSets; 24 | import com.cloudera.csd.tools.codahale.CodahaleCommonMetricSets.MetricServlet2XAdapter; 25 | import com.cloudera.csd.tools.codahale.CodahaleCommonMetricSets.MetricSet; 26 | import com.cloudera.csd.tools.codahale.CodahaleCommonMetricSets.Version; 27 | import com.cloudera.csd.tools.codahale.CodahaleMetricDefinitionFixture; 28 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 29 | import com.google.common.collect.Lists; 30 | import com.google.common.collect.Maps; 31 | import org.apache.commons.io.FileUtils; 32 | import org.apache.commons.io.FilenameUtils; 33 | import org.apache.kafka.metrics.broker.BrokerMetrics; 34 | import org.apache.kafka.metrics.replica.ReplicaMetrics; 35 | import org.apache.kafka.metrics.topic.TopicMetrics; 36 | 37 | import java.io.File; 38 | import java.util.List; 39 | import java.util.Map; 40 | 41 | public class KafkaServiceMetricsSchemaGenerator extends AbstractCodahaleFixtureGenerator { 42 | 43 | private static final String SERVICE_NAME = "KAFKA"; 44 | 45 | private static final String KAFKA_BROKER = "KAFKA_BROKER"; 46 | private static final String KAFKA_BROKER_TOPIC = "KAFKA_BROKER_TOPIC"; 47 | private static final String KAFKA_REPLICA = "KAFKA_REPLICA"; 48 | 49 | private static final String COMMON_METRICS_FILE_NAME = "common_metrics_fixture.json"; 50 | 51 | public KafkaServiceMetricsSchemaGenerator(String[] args) throws Exception { 52 | super(args); 53 | } 54 | 55 | /** 56 | * Generate the fixture. 57 | */ 58 | @Override 59 | public CodahaleMetricDefinitionFixture generateFixture() throws Exception { 60 | final CodahaleMetricDefinitionFixture ret = new CodahaleMetricDefinitionFixture(); 61 | 62 | ret.setServiceName(SERVICE_NAME); 63 | 64 | for (CodahaleMetric metric : BrokerMetrics.getMetrics()) { 65 | ret.addRoleMetric(KAFKA_BROKER, metric); 66 | } 67 | 68 | for (CodahaleMetric metric: TopicMetrics.getMetrics()) { 69 | ret.addEntityMetric(KAFKA_BROKER_TOPIC, metric); 70 | } 71 | 72 | for (CodahaleMetric metric: ReplicaMetrics.getMetrics()) { 73 | ret.addEntityMetric(KAFKA_REPLICA, metric); 74 | } 75 | 76 | FileUtils.write( 77 | new File(config.getString(OPT_GENERATED_OUPTUT.getLongOpt(), 78 | CODAHALE_OUT_DEFAULT_FILE_NAME)), 79 | JsonUtil.valueAsString(ret)); 80 | return ret; 81 | } 82 | 83 | private void generateCommonMetricsFixture() throws Exception { 84 | final List memoryMetricDescriptors = 85 | CodahaleCommonMetricSets.generateMetricDescritptorsForMetricSet( 86 | MetricSet.MEMORY, 87 | Version.CODAHALE_2_X_VIRTUAL_MACHINE_METRICS, 88 | new MetricServlet2XAdapter("::", MetricSet.MEMORY), SERVICE_NAME); 89 | 90 | final List threadStateMetricDescriptors = 91 | CodahaleCommonMetricSets.generateMetricDescritptorsForMetricSet( 92 | MetricSet.THREAD_STATE, 93 | Version.CODAHALE_2_X_VIRTUAL_MACHINE_METRICS, 94 | new MetricServlet2XAdapter("::", MetricSet.THREAD_STATE), SERVICE_NAME); 95 | 96 | final List commonMetrics = Lists.newArrayList(); 97 | commonMetrics.addAll(memoryMetricDescriptors); 98 | commonMetrics.addAll(threadStateMetricDescriptors); 99 | 100 | final Map> fixture = Maps.newTreeMap(); 101 | fixture.put(KAFKA_BROKER, commonMetrics); 102 | 103 | final String path = FilenameUtils.getFullPath(config.getString( 104 | AbstractCodahaleFixtureGenerator.OPT_GENERATED_OUPTUT.getLongOpt(), 105 | CODAHALE_OUT_DEFAULT_FILE_NAME)); 106 | 107 | FileUtils.write(new File(path, COMMON_METRICS_FILE_NAME), JsonUtil.valueAsString(fixture, true)); 108 | } 109 | 110 | public static void main(String[] args) throws Exception { 111 | KafkaServiceMetricsSchemaGenerator generator = new KafkaServiceMetricsSchemaGenerator(args); 112 | generator.generateFixture(); 113 | generator.generateCommonMetricsFixture(); 114 | } 115 | } 116 | 117 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/UnitConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics; 19 | 20 | public class UnitConstants { 21 | private UnitConstants() {} 22 | 23 | public static final String partitions = "partitions"; 24 | public static final String replicas = "replicas"; 25 | public static final String controller = "controller"; 26 | public static final String requests = "requests"; 27 | public static final String responses = "responses"; 28 | public static final String messages = "messages"; 29 | public static final String message_batches = "message_batches"; 30 | public static final String bytes = "bytes"; 31 | public static final String segments = "segments"; 32 | public static final String groups = "groups"; 33 | 34 | public static final String elections = "elections"; 35 | public static final String expansions = "expansions"; 36 | public static final String shrinks = "shrinks"; 37 | public static final String flushes = "flushes"; 38 | 39 | public static final String second = "second"; 40 | public static final String nanoseconds = "nanoseconds"; 41 | public static final String ms = "ms"; 42 | public static final String offset = "offset"; 43 | public static final String percent_idle = "percent_idle"; 44 | public static final String state = "state"; 45 | 46 | public static final String fetch_requests = "fetch_requests"; 47 | public static final String fetchRequests = "fetch requests"; 48 | } 49 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/broker/BrokerMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.broker; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import com.google.common.collect.Lists; 23 | import org.apache.kafka.metrics.UnitConstants; 24 | 25 | import java.util.List; 26 | 27 | public class BrokerMetrics { 28 | 29 | private BrokerMetrics() {} 30 | 31 | // KafkaServer 32 | private static final String KAFKA_SERVER_CONTEXT_FORMAT = "kafka.server.KafkaServer::%s"; 33 | 34 | private static final CodahaleMetric BROKER_STATE_METRIC = 35 | new CodahaleMetric.Builder() 36 | .setName("broker_state") 37 | .setLabel("Broker State") 38 | .setDescription("The state the broker is in. 0 = NotRunning, 1 = Starting, " + 39 | "2 = RecoveringFromUncleanShutdown, 3 = RunningAsBroker, 4 = RunningAsController, " + 40 | "6 = PendingControlledShutdown, 7 = BrokerShuttingDown") 41 | .setNumerator(UnitConstants.state) 42 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 43 | .setContext(String.format(KAFKA_SERVER_CONTEXT_FORMAT, "BrokerState")) 44 | .build(); 45 | 46 | // LogFlushStats 47 | private static final String LOG_FLUSH_STATS_CONTEXT_FORMAT = "kafka.log.LogFlushStats::%s"; 48 | 49 | private static final CodahaleMetric LOG_FLUSH_METRIC = 50 | new CodahaleMetric.Builder() 51 | .setName("log_flush") 52 | .setLabel("Log Flush") 53 | .setDescription("Rate of flushing Kafka logs to disk") 54 | .setNumerator(UnitConstants.ms) 55 | .setNumeratorForCounterMetric(UnitConstants.flushes) 56 | .setDenominatorForRateMetrics(UnitConstants.second) 57 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.TIMER) 58 | .setContext(String.format(LOG_FLUSH_STATS_CONTEXT_FORMAT, "LogFlushRateAndTimeMs")) 59 | .build(); 60 | 61 | // OffsetManager 62 | private static final String OFFSET_MANAGER_CONTEXT_FORMAT = "kafka.server.OffsetManager::%s"; 63 | 64 | private static final CodahaleMetric NUM_GROUPS_METRIC = 65 | new CodahaleMetric.Builder() 66 | .setName("offsets_groups") 67 | .setLabel("Offsets Groups") 68 | .setDescription("The number of consumer groups in the offsets cache") 69 | .setNumerator(UnitConstants.groups) 70 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 71 | .setContext(String.format(OFFSET_MANAGER_CONTEXT_FORMAT, "NumGroups")) 72 | .build(); 73 | 74 | private static final CodahaleMetric NUM_OFFSETS_METRIC = 75 | new CodahaleMetric.Builder() 76 | .setName("offsets") 77 | .setLabel("Offsets") 78 | .setDescription("The size of the offsets cache") 79 | .setNumerator(UnitConstants.groups) 80 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 81 | .setContext(String.format(OFFSET_MANAGER_CONTEXT_FORMAT, "NumOffsets")) 82 | .build(); 83 | 84 | public static List getMetrics() { 85 | List metrics = Lists.newArrayList(); 86 | metrics.add(BROKER_STATE_METRIC); 87 | metrics.add(LOG_FLUSH_METRIC); 88 | metrics.add(NUM_GROUPS_METRIC); 89 | metrics.add(NUM_OFFSETS_METRIC); 90 | metrics.addAll(BrokerTopicMetrics.getMetrics()); 91 | metrics.addAll(ControllerMetrics.getMetrics()); 92 | metrics.addAll(NetworkMetrics.getMetrics()); 93 | metrics.addAll(PurgatoryMetrics.getMetrics()); 94 | metrics.addAll(ReplicaManagerMetrics.getMetrics()); 95 | metrics.addAll(RequestMetrics.getMetrics()); 96 | return metrics; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/broker/BrokerTopicMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.broker; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | class BrokerTopicMetrics { 28 | 29 | private BrokerTopicMetrics() {} 30 | 31 | private static final String BROKER_TOPICS_METRICS_CONTEXT_FORMAT = "kafka.server.BrokerTopicMetrics::%s"; 32 | 33 | private static final CodahaleMetric MESSAGES_RECEIVED_METRIC = 34 | new CodahaleMetric.Builder() 35 | .setName("messages_received") 36 | .setLabel("Messages Received") 37 | .setDescription("Number of messages written to topic on this broker") 38 | .setNumerator(UnitConstants.messages) 39 | .setDenominator(UnitConstants.second) 40 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 41 | .setContext(String.format(BROKER_TOPICS_METRICS_CONTEXT_FORMAT, "MessagesInPerSec")) 42 | .build(); 43 | 44 | private static final CodahaleMetric BYTES_RECEIVED_METRIC = 45 | new CodahaleMetric.Builder() 46 | .setName("bytes_received") 47 | .setLabel("Bytes Received") 48 | .setDescription("Amount of data written to topic on this broker") 49 | .setNumerator(UnitConstants.bytes) 50 | .setDenominator(UnitConstants.second) 51 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 52 | .setContext(String.format(BROKER_TOPICS_METRICS_CONTEXT_FORMAT, "BytesInPerSec")) 53 | .build(); 54 | 55 | private static final CodahaleMetric BYTES_FETCHED_METRIC = 56 | new CodahaleMetric.Builder() 57 | .setName("bytes_fetched") 58 | .setLabel("Bytes Fetched") 59 | .setDescription("Amount of data consumers fetched from this topic on this broker") 60 | .setNumerator(UnitConstants.bytes) 61 | .setDenominator(UnitConstants.second) 62 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 63 | .setContext(String.format(BROKER_TOPICS_METRICS_CONTEXT_FORMAT, "BytesOutPerSec")) 64 | .build(); 65 | 66 | private static final CodahaleMetric BYTES_REJECTED_METRIC = 67 | new CodahaleMetric.Builder() 68 | .setName("bytes_rejected") 69 | .setLabel("Bytes Rejected") 70 | .setDescription("Amount of data in messages rejected by broker for this topic") 71 | .setNumerator(UnitConstants.bytes) 72 | .setDenominator(UnitConstants.second) 73 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 74 | .setContext(String.format(BROKER_TOPICS_METRICS_CONTEXT_FORMAT, "BytesRejectedPerSec")) 75 | .build(); 76 | 77 | private static final CodahaleMetric REJECTED_MESSAGE_BATCHES_METRIC = 78 | new CodahaleMetric.Builder() 79 | .setName("rejected_message_batches") 80 | .setLabel("Rejected Message Batches") 81 | .setDescription("Number of message batches sent by producers that the broker rejected for this topic") 82 | .setNumerator(UnitConstants.message_batches) 83 | .setDenominator(UnitConstants.second) 84 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 85 | .setContext(String.format(BROKER_TOPICS_METRICS_CONTEXT_FORMAT, "FailedProduceRequestsPerSec")) 86 | .build(); 87 | 88 | private static final CodahaleMetric FETCH_REQUEST_FAILURES_METRIC = 89 | new CodahaleMetric.Builder() 90 | .setName("fetch_request_failures") 91 | .setLabel("Fetch Request Failures") 92 | .setDescription("Number of data read requests from consumers that brokers failed to process for this topic") 93 | .setNumerator(UnitConstants.fetch_requests) 94 | .setDenominator(UnitConstants.second) 95 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 96 | .setContext(String.format(BROKER_TOPICS_METRICS_CONTEXT_FORMAT, "FailedFetchRequestsPerSec")) 97 | .build(); 98 | 99 | public static List getMetrics() { 100 | return Arrays.asList( 101 | MESSAGES_RECEIVED_METRIC, 102 | BYTES_RECEIVED_METRIC, 103 | BYTES_FETCHED_METRIC, 104 | BYTES_REJECTED_METRIC, 105 | REJECTED_MESSAGE_BATCHES_METRIC, 106 | FETCH_REQUEST_FAILURES_METRIC 107 | ); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/broker/ControllerMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.broker; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | class ControllerMetrics { 28 | 29 | private ControllerMetrics() {} 30 | 31 | // ControllerStats 32 | private static final String CONTROLLER_STATS_CONTEXT_FORMAT = "kafka.controller.ControllerStats::%s"; 33 | 34 | private static final CodahaleMetric LEADER_ELECTIONS_METRIC = 35 | new CodahaleMetric.Builder() 36 | .setName("leader_election") 37 | .setLabel("Leader Elections") 38 | .setDescription("Leader elections") 39 | .setNumerator(UnitConstants.ms) 40 | .setNumeratorForCounterMetric(UnitConstants.elections) 41 | .setDenominatorForRateMetrics(UnitConstants.second) 42 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.TIMER) 43 | .setContext(String.format(CONTROLLER_STATS_CONTEXT_FORMAT, "LeaderElectionRateAndTimeMs")) 44 | .build(); 45 | 46 | private static final CodahaleMetric UNCLEAN_LEADER_ELECTIONS_METRIC = 47 | new CodahaleMetric.Builder() 48 | .setName("unclean_leader_elections") 49 | .setLabel("Unclean Leader Elections") 50 | .setDescription("Unclean leader elections. Cloudera recommends disabling unclean leader elections, " + 51 | "to avoid potential data loss, so this should be 0") 52 | .setNumerator(UnitConstants.elections) 53 | .setDenominator(UnitConstants.second) 54 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 55 | .setContext(String.format(CONTROLLER_STATS_CONTEXT_FORMAT, "UncleanLeaderElectionsPerSec")) 56 | .build(); 57 | 58 | 59 | // KafkaController 60 | private static final String KAFKA_CONTROLLER_CONTEXT_FORMAT = "kafka.controller.KafkaController::%s"; 61 | 62 | private static final CodahaleMetric ACTIVE_CONTROLLER_METRIC = 63 | new CodahaleMetric.Builder() 64 | .setName("active_controller") 65 | .setLabel("Active Controller") 66 | .setDescription("Will be 1 if this broker is the active controller, 0 otherwise") 67 | .setNumerator(UnitConstants.controller) 68 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 69 | .setContext(String.format(KAFKA_CONTROLLER_CONTEXT_FORMAT, "ActiveControllerCount")) 70 | .build(); 71 | 72 | private static final CodahaleMetric PREFERRED_REPLICA_IMBALANCE_METRIC = 73 | new CodahaleMetric.Builder() 74 | .setName("preferred_replica_imbalance") 75 | .setLabel("Preferred Replica Imbalance") 76 | .setDescription("Number of partitions where the lead replica is not the preferred replica") 77 | .setNumerator(UnitConstants.partitions) 78 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 79 | .setContext(String.format(KAFKA_CONTROLLER_CONTEXT_FORMAT, "PreferredReplicaImbalanceCount")) 80 | .build(); 81 | 82 | private static final CodahaleMetric OFFLINE_PARTITIONS_METRIC = 83 | new CodahaleMetric.Builder() 84 | .setName("offline_partitions") 85 | .setLabel("Offline Partitions") 86 | .setDescription("Number of unavailable partitions") 87 | .setNumerator(UnitConstants.partitions) 88 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 89 | .setContext(String.format(KAFKA_CONTROLLER_CONTEXT_FORMAT, "OfflinePartitionsCount")) 90 | .build(); 91 | 92 | public static List getMetrics() { 93 | return Arrays.asList( 94 | LEADER_ELECTIONS_METRIC, 95 | UNCLEAN_LEADER_ELECTIONS_METRIC, 96 | ACTIVE_CONTROLLER_METRIC, 97 | PREFERRED_REPLICA_IMBALANCE_METRIC, 98 | OFFLINE_PARTITIONS_METRIC 99 | ); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/broker/NetworkMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.broker; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | class NetworkMetrics { 28 | 29 | private NetworkMetrics() {} 30 | 31 | // RequestChannel 32 | private static final String REQUEST_CHANNEL_CONTEXT_FORMAT = "kafka.network.RequestChannel::%s"; 33 | 34 | private static final CodahaleMetric REQUEST_QUEUE_SIZE_METRIC = 35 | new CodahaleMetric.Builder() 36 | .setName("request_queue_size") 37 | .setLabel("Request Queue Size") 38 | .setDescription("Request Queue Size") 39 | .setNumerator(UnitConstants.requests) 40 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 41 | .setContext(String.format(REQUEST_CHANNEL_CONTEXT_FORMAT, "RequestQueueSize")) 42 | .build(); 43 | 44 | private static final CodahaleMetric RESPONSE_QUEUE_SIZE_METRIC = 45 | new CodahaleMetric.Builder() 46 | .setName("response_queue_size") 47 | .setLabel("Response Queue Size") 48 | .setDescription("Response Queue Size") 49 | .setNumerator(UnitConstants.responses) 50 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 51 | .setContext(String.format(REQUEST_CHANNEL_CONTEXT_FORMAT, "ResponseQueueSize")) 52 | .build(); 53 | 54 | // SocketServer 55 | private static final String SOCKET_SERVER_CONTEXT_FORMAT = "kafka.network.SocketServer::%s"; 56 | 57 | private static final CodahaleMetric NETWORK_PROCESSOR_AVG_IDLE_METRIC = 58 | new CodahaleMetric.Builder() 59 | .setName("network_processor_avg_idle") 60 | .setLabel("Network Processor Average Idle") 61 | .setDescription("The average free capacity of the network processors") 62 | .setNumerator(UnitConstants.percent_idle) 63 | .setDenominator(UnitConstants.nanoseconds) 64 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 65 | .setContext(String.format(SOCKET_SERVER_CONTEXT_FORMAT, "NetworkProcessorAvgIdle")) 66 | .build(); 67 | 68 | private static final CodahaleMetric RESPONSES_BEING_SENT_METRIC = 69 | new CodahaleMetric.Builder() 70 | .setName("responses_being_sent") 71 | .setLabel("Responses Being Sent") 72 | .setDescription("The number of responses being sent by the network processors") 73 | .setNumerator(UnitConstants.responses) 74 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 75 | .setContext(String.format(SOCKET_SERVER_CONTEXT_FORMAT, "ResponsesBeingSent")) 76 | .build(); 77 | 78 | public static List getMetrics() { 79 | return Arrays.asList( 80 | REQUEST_QUEUE_SIZE_METRIC, 81 | RESPONSE_QUEUE_SIZE_METRIC, 82 | NETWORK_PROCESSOR_AVG_IDLE_METRIC, 83 | RESPONSES_BEING_SENT_METRIC 84 | ); 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/broker/PurgatoryMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.broker; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | class PurgatoryMetrics { 28 | 29 | private PurgatoryMetrics() {} 30 | 31 | // ProducerRequestPurgatory 32 | private static final String PRODUCER_REQUEST_PURGATORY_CONTEXT_FORMAT = "kafka.server.ProducerRequestPurgatory::%s"; 33 | 34 | private static final CodahaleMetric PRODUCER_PURGATORY_SIZE_METRIC = 35 | new CodahaleMetric.Builder() 36 | .setName("producer_purgatory_size") 37 | .setLabel("Requests waiting in the producer purgatory") 38 | .setDescription("Requests waiting in the producer purgatory. This should be non-zero when acks = -1 is used in producers") 39 | .setNumerator(UnitConstants.requests) 40 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 41 | .setContext(String.format(PRODUCER_REQUEST_PURGATORY_CONTEXT_FORMAT, "PurgatorySize")) 42 | .build(); 43 | 44 | private static final CodahaleMetric PRODUCER_PURGATORY_DELAYED_REQUESTS_METRIC = 45 | new CodahaleMetric.Builder() 46 | .setName("producer_purgatory_delayed_requests") 47 | .setLabel("Number of requests delayed in the producer purgatory") 48 | .setDescription("Number of requests delayed in the producer purgatory") 49 | .setNumerator(UnitConstants.requests) 50 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 51 | .setContext(String.format(PRODUCER_REQUEST_PURGATORY_CONTEXT_FORMAT, "NumDelayedRequests")) 52 | .build(); 53 | 54 | // FetchRequestPurgatory 55 | private static final String FETCH_REQUEST_PURGATORY_CONTEXT_FORMAT = "kafka.server.FetchRequestPurgatory::%s"; 56 | 57 | private static final CodahaleMetric FETCH_PURGATORY_SIZE_METRIC = 58 | new CodahaleMetric.Builder() 59 | .setName("fetch_purgatory_size") 60 | .setLabel("Requests waiting in the fetch purgatory") 61 | .setDescription("Requests waiting in the fetch purgatory. This depends on value of fetch.wait.max.ms in the consumer") 62 | .setNumerator(UnitConstants.requests) 63 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 64 | .setContext(String.format(FETCH_REQUEST_PURGATORY_CONTEXT_FORMAT, "PurgatorySize")) 65 | .build(); 66 | 67 | private static final CodahaleMetric FETCH_PURGATORY_DELAYED_REQUESTS_METRIC = 68 | new CodahaleMetric.Builder() 69 | .setName("fetch_purgatory_delayed_requests") 70 | .setLabel("Number of requests delayed in the fetch purgatory") 71 | .setDescription("Number of requests delayed in the fetch purgatory") 72 | .setNumerator(UnitConstants.requests) 73 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 74 | .setContext(String.format(FETCH_REQUEST_PURGATORY_CONTEXT_FORMAT, "NumDelayedRequests")) 75 | .build(); 76 | 77 | 78 | public static List getMetrics() { 79 | return Arrays.asList( 80 | PRODUCER_PURGATORY_SIZE_METRIC, 81 | PRODUCER_PURGATORY_DELAYED_REQUESTS_METRIC, 82 | FETCH_PURGATORY_SIZE_METRIC, 83 | FETCH_PURGATORY_DELAYED_REQUESTS_METRIC 84 | ); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/broker/ReplicaManagerMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.broker; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | class ReplicaManagerMetrics { 28 | 29 | private ReplicaManagerMetrics() {} 30 | 31 | // ReplicaManager 32 | private static final String REPLICA_MANAGER_CONTEXT_FORMAT = "kafka.server.ReplicaManager::%s"; 33 | 34 | private static final CodahaleMetric PARTITIONS_METRIC = 35 | new CodahaleMetric.Builder() 36 | .setName("partitions") 37 | .setLabel("Partitions") 38 | .setDescription("Number of partitions (lead or follower replicas) on broker") 39 | .setNumerator(UnitConstants.partitions) 40 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 41 | .setContext(String.format(REPLICA_MANAGER_CONTEXT_FORMAT, "PartitionCount")) 42 | .build(); 43 | 44 | private static final CodahaleMetric LEADER_REPLICAS_METRIC = 45 | new CodahaleMetric.Builder() 46 | .setName("leader_replicas") 47 | .setLabel("Leader Replicas") 48 | .setDescription("Number of leader replicas on broker") 49 | .setNumerator(UnitConstants.replicas) 50 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 51 | .setContext(String.format(REPLICA_MANAGER_CONTEXT_FORMAT, "LeaderCount")) 52 | .build(); 53 | 54 | private static final CodahaleMetric UNDER_REPLICATED_PARTITIONS_METRIC = 55 | new CodahaleMetric.Builder() 56 | .setName("under_replicated_partitions") 57 | .setLabel("Under Replicated Partitions") 58 | .setDescription("Number of partitions with unavailable replicas") 59 | .setNumerator(UnitConstants.partitions) 60 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 61 | .setContext(String.format(REPLICA_MANAGER_CONTEXT_FORMAT, "UnderReplicatedPartitions")) 62 | .build(); 63 | 64 | private static final CodahaleMetric ISR_EXPANDS_METRIC = 65 | new CodahaleMetric.Builder() 66 | .setName("isr_expands") 67 | .setLabel("ISR Expansions") 68 | .setDescription("Number of times ISR for a partition expanded") 69 | .setNumerator(UnitConstants.expansions) 70 | .setDenominator(UnitConstants.second) 71 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 72 | .setContext(String.format(REPLICA_MANAGER_CONTEXT_FORMAT, "IsrExpandsPerSec")) 73 | .build(); 74 | 75 | private static final CodahaleMetric ISR_SHRINKS_METRIC = 76 | new CodahaleMetric.Builder() 77 | .setName("isr_shrinks") 78 | .setLabel("ISR Shrinks") 79 | .setDescription("Number of times ISR for a partition shrank") 80 | .setNumerator(UnitConstants.shrinks) 81 | .setDenominator(UnitConstants.second) 82 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 83 | .setContext(String.format(REPLICA_MANAGER_CONTEXT_FORMAT, "IsrShrinksPerSec")) 84 | .build(); 85 | 86 | // ReplicaFetcherManager 87 | private static final String REPLICA_FETCHER_MANAGER_CONTEXT_FORMAT = "kafka.server.ReplicaFetcherManager.clientId.Replica::%s"; 88 | 89 | private static final CodahaleMetric MAX_REPLICATION_LAG_METRIC = 90 | new CodahaleMetric.Builder() 91 | .setName("max_replication_lag") 92 | .setLabel("Maximum Replication Lag on Broker") 93 | .setDescription("Maximum replication lag on broker, across all fetchers, topics and partitions") 94 | .setNumerator(UnitConstants.messages) 95 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 96 | .setContext(String.format(REPLICA_FETCHER_MANAGER_CONTEXT_FORMAT, "MaxLag")) 97 | .build(); 98 | 99 | private static final CodahaleMetric MIN_REPLICATION_RATE = 100 | new CodahaleMetric.Builder() 101 | .setName("min_replication_rate") 102 | .setLabel("Minimum Replication Rate") 103 | .setDescription("Minimum replication rate, across all fetchers, topics and partitions. Measured in average fetch requests per sec in the last minute") 104 | .setNumerator(UnitConstants.fetchRequests) 105 | .setDenominator(UnitConstants.second) 106 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 107 | .setContext(String.format(REPLICA_FETCHER_MANAGER_CONTEXT_FORMAT, "MinFetchRate")) 108 | .build(); 109 | 110 | public static List getMetrics() { 111 | return Arrays.asList( 112 | PARTITIONS_METRIC, 113 | LEADER_REPLICAS_METRIC, 114 | UNDER_REPLICATED_PARTITIONS_METRIC, 115 | ISR_EXPANDS_METRIC, 116 | ISR_SHRINKS_METRIC, 117 | MAX_REPLICATION_LAG_METRIC, 118 | MIN_REPLICATION_RATE 119 | ); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/replica/ReplicaMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.replica; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | /** 28 | * Note: The context is missing its "root" because that is generated per replica in CM 29 | */ 30 | public class ReplicaMetrics { 31 | 32 | private ReplicaMetrics() {} 33 | 34 | private static final CodahaleMetric LOG_END_OFFSET_METRIC = 35 | new CodahaleMetric.Builder() 36 | .setName("log_end_offset") 37 | .setLabel("Log End Offset") 38 | .setDescription("The offset of the next message that will be appended to the log") 39 | .setNumerator(UnitConstants.offset) 40 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 41 | .setContext("LogEndOffset") 42 | .build(); 43 | 44 | private static final CodahaleMetric LOG_START_OFFSET_METRIC = 45 | new CodahaleMetric.Builder() 46 | .setName("log_start_offset") 47 | .setLabel("Log Start Offset") 48 | .setDescription("The earliest message offset in the log") 49 | .setNumerator(UnitConstants.offset) 50 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 51 | .setContext("LogStartOffset") 52 | .build(); 53 | 54 | private static final CodahaleMetric NUM_LOG_SEGMENTS_METRIC = 55 | new CodahaleMetric.Builder() 56 | .setName("num_log_segments") 57 | .setLabel("Number of log segments") 58 | .setDescription("The number of segments in the log") 59 | .setNumerator(UnitConstants.segments) 60 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 61 | .setContext("NumLogSegments") 62 | .build(); 63 | 64 | private static final CodahaleMetric SIZE_METRIC = 65 | new CodahaleMetric.Builder() 66 | .setName("size") 67 | .setLabel("Log size") 68 | .setDescription("The size of the log") 69 | .setNumerator(UnitConstants.bytes) 70 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.GAUGE) 71 | .setContext("Size") 72 | .build(); 73 | 74 | public static List getMetrics() { 75 | return Arrays.asList( 76 | LOG_END_OFFSET_METRIC, 77 | LOG_START_OFFSET_METRIC, 78 | NUM_LOG_SEGMENTS_METRIC, 79 | SIZE_METRIC 80 | ); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /KAFKA/src/main/java/org/apache/kafka/metrics/topic/TopicMetrics.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.kafka.metrics.topic; 19 | 20 | import com.cloudera.csd.tools.codahale.CodahaleMetric; 21 | import com.cloudera.csd.tools.codahale.CodahaleMetricTypes; 22 | import org.apache.kafka.metrics.UnitConstants; 23 | 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | /** 28 | * Note: The context is missing its "root" because that is generated per topic in CM 29 | */ 30 | public class TopicMetrics { 31 | 32 | private static final CodahaleMetric MESSAGES_RECEIVED_METRIC = 33 | new CodahaleMetric.Builder() 34 | .setName("messages_received") 35 | .setLabel("Messages Received") 36 | .setDescription("Number of messages written to topic on this broker") 37 | .setNumerator(UnitConstants.messages) 38 | .setDenominator(UnitConstants.second) 39 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 40 | .setContext("MessagesInPerSec") 41 | .build(); 42 | 43 | private static final CodahaleMetric BYTES_RECEIVED_METRIC = 44 | new CodahaleMetric.Builder() 45 | .setName("bytes_received") 46 | .setLabel("Bytes Received") 47 | .setDescription("Amount of data written to topic on this broker") 48 | .setNumerator(UnitConstants.bytes) 49 | .setDenominator(UnitConstants.second) 50 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 51 | .setContext("BytesInPerSec") 52 | .build(); 53 | 54 | private static final CodahaleMetric BYTES_FETCHED_METRIC = 55 | new CodahaleMetric.Builder() 56 | .setName("bytes_fetched") 57 | .setLabel("Bytes Fetched") 58 | .setDescription("Amount of data consumers fetched from this topic on this broker") 59 | .setNumerator(UnitConstants.bytes) 60 | .setDenominator(UnitConstants.second) 61 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 62 | .setContext("BytesOutPerSec") 63 | .build(); 64 | 65 | private static final CodahaleMetric BYTES_REJECTED_METRIC = 66 | new CodahaleMetric.Builder() 67 | .setName("bytes_rejected") 68 | .setLabel("Bytes Rejected") 69 | .setDescription("Amount of data in messages rejected by broker for this topic") 70 | .setNumerator(UnitConstants.bytes) 71 | .setDenominator(UnitConstants.second) 72 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 73 | .setContext("BytesRejectedPerSec") 74 | .build(); 75 | 76 | private static final CodahaleMetric REJECTED_MESSAGE_BATCHES_METRIC = 77 | new CodahaleMetric.Builder() 78 | .setName("rejected_message_batches") 79 | .setLabel("Rejected Message Batches") 80 | .setDescription("Number of message batches sent by producers that the broker rejected for this topic") 81 | .setNumerator(UnitConstants.message_batches) 82 | .setDenominator(UnitConstants.second) 83 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 84 | .setContext("FailedProduceRequestsPerSec") 85 | .build(); 86 | 87 | private static final CodahaleMetric FETCH_REQUEST_FAILURES_METRIC = 88 | new CodahaleMetric.Builder() 89 | .setName("fetch_request_failures") 90 | .setLabel("Fetch Request Failures") 91 | .setDescription("Number of data read requests from consumers that brokers failed to process for this topic") 92 | .setNumerator(UnitConstants.fetch_requests) 93 | .setDenominator(UnitConstants.second) 94 | .setCodahaleMetricType(CodahaleMetricTypes.CodahaleMetricType.METER) 95 | .setContext("FailedFetchRequestsPerSec") 96 | .build(); 97 | 98 | public static List getMetrics() { 99 | return Arrays.asList( 100 | MESSAGES_RECEIVED_METRIC, 101 | BYTES_RECEIVED_METRIC, 102 | BYTES_FETCHED_METRIC, 103 | BYTES_REJECTED_METRIC, 104 | REJECTED_MESSAGE_BATCHES_METRIC, 105 | FETCH_REQUEST_FAILURES_METRIC 106 | ); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /KAFKA/src/main/resources/kafka_base.mdl: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "KAFKA", 3 | "version" : "1.4.0", 4 | "metricDefinitions" : [], 5 | "nameForCrossEntityAggregateMetrics" : "kafkas", 6 | "roles" : [ 7 | { 8 | "name" : "KAFKA_BROKER", 9 | "nameForCrossEntityAggregateMetrics" : "kafka_brokers", 10 | "additionalMutableAttributeNames" : [ 11 | "kafkaBrokerId" 12 | ], 13 | "metricDefinitions" : [] 14 | } 15 | ], 16 | "metricEntityAttributeDefinitions" : [ 17 | { 18 | "name" : "kafkaTopicName", 19 | "label" : "Topic Name", 20 | "description" : "Name for Kafka Topic.", 21 | "valueCaseSensitive": true 22 | }, 23 | { 24 | "name" : "kafkaInternalTopic", 25 | "label" : "Is Internal Topic", 26 | "description" : "Topic is an internal Kafka topic.", 27 | "valueCaseSensitive": false 28 | }, 29 | { 30 | "name" : "kafkaPartitionId", 31 | "label" : "Partition Id", 32 | "description" : "Id of a Kafka topic partition.", 33 | "valueCaseSensitive": false 34 | }, 35 | { 36 | "name" : "kafkaBrokerId", 37 | "label" : "Broker Id", 38 | "description" : "Id of a Kafka broker.", 39 | "valueCaseSensitive": false 40 | } 41 | ], 42 | "metricEntityTypeDefinitions" : [ 43 | { 44 | "name" : "KAFKA_TOPIC", 45 | "nameForCrossEntityAggregateMetrics" : "kafka_topics", 46 | "entityNameFormat" : [ 47 | "serviceName", 48 | "kafkaTopicName" 49 | ], 50 | "label" : "Kafka Topic", 51 | "labelPlural" : "Kafka Topics", 52 | "description" : "A Kafka topic.", 53 | "immutableAttributeNames" : [ 54 | "kafkaTopicName", 55 | "kafkaInternalTopic", 56 | "serviceName" 57 | ] 58 | }, 59 | { 60 | "name" : "KAFKA_BROKER_TOPIC", 61 | "nameForCrossEntityAggregateMetrics" : "kafka_broker_topics", 62 | "entityNameFormat" : [ 63 | "roleName", 64 | "kafkaTopicName" 65 | ], 66 | "label" : "Kafka Broker Topic", 67 | "labelPlural" : "Kafka Broker Topics", 68 | "description" : "Broker view of a specific topic.", 69 | "immutableAttributeNames" : [ 70 | "kafkaTopicName", 71 | "roleName", 72 | "serviceName" 73 | ], 74 | "parentMetricEntityTypeNames" : [ 75 | "KAFKA_TOPIC", 76 | "KAFKA-KAFKA_BROKER" 77 | ] 78 | }, 79 | { 80 | "name" : "KAFKA_REPLICA", 81 | "nameForCrossEntityAggregateMetrics" : "kafka_replicas", 82 | "entityNameFormat" : [ 83 | "roleName", 84 | "kafkaTopicName", 85 | "kafkaPartitionId" 86 | ], 87 | "label" : "Kafka Replica", 88 | "labelPlural" : "Kafka Replicas", 89 | "description" : "A replica for a given partition of a Kafka topic.", 90 | "immutableAttributeNames" : [ 91 | "kafkaPartitionId", 92 | "kafkaTopicName", 93 | "roleName", 94 | "serviceName" 95 | ], 96 | "parentMetricEntityTypeNames" : [ 97 | "KAFKA_BROKER_TOPIC", 98 | "KAFKA-KAFKA_BROKER" 99 | ] 100 | } 101 | ] 102 | } 103 | -------------------------------------------------------------------------------- /KAFKA/src/scripts/deploy_client_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # for debugging 21 | set -x 22 | 23 | DEFAULT_KAFKA_HOME=/usr/lib/kafka 24 | KAFKA_HOME=${KAFKA_HOME:-$DEFAULT_KAFKA_HOME} 25 | 26 | # For better debugging 27 | echo "" 28 | echo "Date: `date`" 29 | echo "Host: `hostname`" 30 | echo "Pwd: `pwd`" 31 | echo "KAFKA_HOME: $KAFKA_HOME" 32 | echo "CONF_DIR: $CONF_DIR" 33 | echo "Zookeeper Quorum: $ZK_QUORUM" 34 | echo "Zookeeper Chroot: $CHROOT" 35 | 36 | echo "Deploying client configuration" 37 | 38 | KAFKA_CONF_DIR="$CONF_DIR/kafka-conf" 39 | KAFKA_CLIENT_CONF="$KAFKA_CONF_DIR/kafka-client.conf" 40 | SENTRY_CONF_DIR="$CONF_DIR/sentry-conf" 41 | SENTRY_CLIENT_CONF_DIR="$KAFKA_CONF_DIR/sentry-conf" 42 | SENTRY_SITE_XML="sentry-site.xml" 43 | 44 | # Generating Zookeeper quorum 45 | QUORUM=$ZK_QUORUM 46 | if [[ -n $CHROOT ]]; then 47 | QUORUM="${QUORUM}${CHROOT}" 48 | fi 49 | echo "Final Zookeeper Quorum is $QUORUM" 50 | # Replace zookeeper.connect placeholder 51 | perl -pi -e "s#\#zookeeper.connect={{QUORUM}}#zookeeper.connect=${QUORUM}#" $KAFKA_CLIENT_CONF 52 | 53 | # If Sentry is configured, move Sentry configuration under Kafka config 54 | if [[ -f $SENTRY_CONF_DIR/$SENTRY_SITE_XML ]]; then 55 | mkdir "$SENTRY_CLIENT_CONF_DIR" 56 | for i in "$SENTRY_CONF_DIR"/*; do 57 | mv $i "$SENTRY_CLIENT_CONF_DIR" 58 | done 59 | rm -rf "$SENTRY_CONF_DIR" 60 | fi -------------------------------------------------------------------------------- /KAFKA/src/scripts/mirrormaker_control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # for debugging 21 | set -x 22 | 23 | DEFAULT_KAFKA_HOME=/usr/lib/kafka 24 | KAFKA_HOME=${KAFKA_HOME:-$DEFAULT_KAFKA_HOME} 25 | MIN_REFACTORED_MIRROR_MAKER_VERSION=2 26 | 27 | # For better debugging 28 | echo "" 29 | echo "Date: `date`" 30 | echo "Host: `hostname -f`" 31 | echo "Pwd: `pwd`" 32 | echo "CONF_DIR: $CONF_DIR" 33 | echo "KAFKA_HOME: $KAFKA_HOME" 34 | echo "Zookeeper Quorum: $ZK_QUORUM" 35 | echo "Zookeeper Chroot: $CHROOT" 36 | echo "no.data.loss: ${NO_DATA_LOSS}" 37 | echo "whitelist: ${WHITELIST}" 38 | echo "blacklist: ${BLACKLIST}" 39 | echo "num.producers: ${NUM_PRODUCERS}" 40 | echo "num.streams: ${NUM_STREAMS}" 41 | echo "queue.size: ${QUEUE_SIZE}" 42 | echo "queue.byte.size: ${QUEUE_BYTE_SIZE}" 43 | echo "JMX_PORT: $JMX_PORT" 44 | echo "MM_HEAP_SIZE: ${MM_HEAP_SIZE}" 45 | echo "MM_JAVA_OPTS: ${MM_JAVA_OPTS}" 46 | echo "abort.on.send.failure: ${ABORT_ON_SEND_FAILURE}" 47 | echo "offset.commit.interval.ms: ${OFFSET_COMMIT_INTERVAL_MS}" 48 | echo "consumer.rebalance.listener: ${CONSUMER_REBALANCE_LISTENER}" 49 | echo "consumer.rebalance.listener.args: ${CONSUMER_REBALANCE_LISTENER_ARGS}" 50 | echo "message.handler: ${MESSAGE_HANDLER}" 51 | echo "message.handler.args: ${MESSAGE_HANDLER_ARGS}" 52 | echo "SOURCE_SECURITY_PROTOCOL: ${SOURCE_SECURITY_PROTOCOL}" 53 | echo "DESTINATION_SECURITY_PROTOCOL: ${DESTINATION_SECURITY_PROTOCOL}" 54 | echo "KAFKA_MIRROR_MAKER_PRINCIPAL: ${KAFKA_MIRROR_MAKER_PRINCIPAL}" 55 | echo "SOURCE_SSL_CLIENT_AUTH: ${SOURCE_SSL_CLIENT_AUTH}" 56 | echo "DESTINATION_SSL_CLIENT_AUTH: ${DESTINATION_SSL_CLIENT_AUTH}" 57 | 58 | KAFKA_VERSION=$(grep "^version=" $KAFKA_HOME/cloudera/cdh_version.properties | cut -d '=' -f 2) 59 | KAFKA_MAJOR_VERSION=$(echo $KAFKA_VERSION | cut -d '-' -f 2 | sed 's/kafka//g' | cut -d '.' -f 1) 60 | echo "Kafka version found: ${KAFKA_VERSION}" 61 | 62 | if [[ -n ${WHITELIST} ]]; then 63 | WL_TOPICS="--whitelist ${WHITELIST}" 64 | echo "Using topic whitelist: ${WL_TOPICS}" 65 | fi 66 | 67 | if [[ -n ${NUM_STREAMS} ]]; then 68 | STREAM_PARAM="--num.streams ${NUM_STREAMS}" 69 | fi 70 | 71 | if [[ $KAFKA_MAJOR_VERSION < $MIN_REFACTORED_MIRROR_MAKER_VERSION ]]; then 72 | # Generating Zookeeper quorum 73 | QUORUM=$ZK_QUORUM 74 | if [[ -n $CHROOT ]]; then 75 | QUORUM="${QUORUM}${CHROOT}" 76 | fi 77 | echo "Final Zookeeper Quorum is $QUORUM" 78 | 79 | if ! grep zookeeper.connect= ${CONF_DIR}/mirror_maker_consumers.properties; then 80 | echo "zookeeper.connect=$QUORUM" >> ${CONF_DIR}/mirror_maker_consumers.properties 81 | fi 82 | 83 | if [[ ${NO_DATA_LOSS} == "true" ]]; then 84 | DATA_LOSS_PARAM="--no.data.loss" 85 | fi 86 | 87 | echo "data loss param: ${DATA_LOSS_PARAM}" 88 | 89 | if [[ -n ${BLACKLIST} ]]; then 90 | BL_TOPICS="--blacklist ${BLACKLIST}" 91 | echo "Using topic blacklist ${BL_TOPICS}" 92 | fi 93 | 94 | if [[ -n ${NUM_PRODUCERS} ]]; then 95 | PRODUCER_PARAM="--num.producers ${NUM_PRODUCERS}" 96 | fi 97 | 98 | if [[ -n ${QUEUE_SIZE} ]]; then 99 | QUEUE_SIZE_PARAM="--queue.size ${QUEUE_SIZE}" 100 | fi 101 | 102 | if [[ -n ${QUEUE_BYTE_SIZE} ]]; then 103 | QUEUE_BYTE_SIZE_PARAM="--queue.byte.size ${QUEUE_BYTE_SIZE}" 104 | fi 105 | else 106 | if [[ -n ${ABORT_ON_SEND_FAILURE} ]]; then 107 | ABORT_ON_SEND_FAILURE_FLAG="--abort.on.send.failure ${ABORT_ON_SEND_FAILURE}" 108 | fi 109 | 110 | if [[ -n ${OFFSET_COMMIT_INTERVAL_MS} ]]; then 111 | OFFSET_COMMIT_INTERVAL_MS_PARAM="--offset.commit.interval.ms ${OFFSET_COMMIT_INTERVAL_MS}" 112 | fi 113 | 114 | if [[ -n ${CONSUMER_REBALANCE_LISTENER} ]]; then 115 | CONSUMER_REBALANCE_LISTENER_PARAM="--consumer.rebalance.listener ${CONSUMER_REBALANCE_LISTENER}" 116 | if [[ -n ${CONSUMER_REBALANCE_LISTENER_ARGS} ]]; then 117 | CONSUMER_REBALANCE_LISTENER_ARGS_PARAM="--consumer.rebalance.listener.args ${CONSUMER_REBALANCE_LISTENER_ARGS}" 118 | fi 119 | fi 120 | 121 | if [[ -n ${MESSAGE_HANDLER} ]]; then 122 | MESSAGE_HANDLER_PARAM="--message.handler ${MESSAGE_HANDLER}" 123 | if [[ -n ${MESSAGE_HANDLER_ARGS} ]]; then 124 | MESSAGE_HANDLER_ARGS_PARAM="--message.handler.args ${MESSAGE_HANDLER_ARGS}" 125 | fi 126 | fi 127 | 128 | if [[ ${SOURCE_SECURITY_PROTOCOL} == *"SSL"* ]]; then 129 | set +x 130 | # Append other ssl params from ssl.properties 131 | SSL_CONFIGS=$(cat ssl_client.properties) 132 | if [[ ${SOURCE_SSL_CLIENT_AUTH} == "true" ]]; then 133 | SSL_SERVER_CONFIGS=$(cat ssl_server.properties) 134 | SSL_CONFIGS="${SSL_CONFIGS} 135 | ${SSL_SERVER_CONFIGS}" 136 | fi 137 | 138 | # Replace SSL_CONFIGS's placeholder 139 | perl -pi -e "s#\#ssl.configs={{SSL_CONFIGS}}#${SSL_CONFIGS}#" $CONF_DIR/mirror_maker_consumers.properties 140 | set -x 141 | else 142 | # Remove SSL_CONFIGS's placeholder 143 | perl -pi -e "s#\#ssl.configs={{SSL_CONFIGS}}##" $CONF_DIR/mirror_maker_consumers.properties 144 | fi 145 | 146 | if [[ ${DESTINATION_SECURITY_PROTOCOL} == *"SSL"* ]]; then 147 | set +x 148 | # Append other ssl params from ssl.properties 149 | SSL_CONFIGS=$(cat ssl_client.properties) 150 | if [[ ${DESTINATION_SSL_CLIENT_AUTH} == "true" ]]; then 151 | SSL_SERVER_CONFIGS=$(cat ssl_server.properties) 152 | SSL_CONFIGS="${SSL_CONFIGS} 153 | ${SSL_SERVER_CONFIGS}" 154 | fi 155 | 156 | # Replace SSL_CONFIGS's placeholder 157 | perl -pi -e "s#\#ssl.configs={{SSL_CONFIGS}}#${SSL_CONFIGS}#" $CONF_DIR/mirror_maker_producers.properties 158 | set -x 159 | else 160 | # Remove SSL_CONFIGS's placeholder 161 | perl -pi -e "s#\#ssl.configs={{SSL_CONFIGS}}##" $CONF_DIR/mirror_maker_producers.properties 162 | fi 163 | 164 | if [[ ${SOURCE_SECURITY_PROTOCOL} == *"SASL"* || ${DESTINATION_SECURITY_PROTOCOL} == *"SASL"* ]]; then 165 | if [[ -z "${JAAS_CONFIGS}" ]]; then 166 | KEYTAB_FILE="${CONF_DIR}/kafka.keytab" 167 | JAAS_CONFIGS="KafkaClient { 168 | com.sun.security.auth.module.Krb5LoginModule required 169 | useKeyTab=true 170 | storeKey=true 171 | keyTab=\"$KEYTAB_FILE\" 172 | principal=\"$KAFKA_MIRROR_MAKER_PRINCIPAL\"; 173 | };" 174 | fi 175 | echo "${JAAS_CONFIGS}" > $CONF_DIR/jaas.conf 176 | 177 | export KAFKA_OPTS="${KAFKA_OPTS} -Djava.security.auth.login.config=${CONF_DIR}/jaas.conf" 178 | fi 179 | fi 180 | 181 | # Propagating logger information to Kafka 182 | export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$CONF_DIR/log4j.properties" 183 | 184 | # Set LOG_DIR to pwd as this directory exists and hence the underlaying run-kafka-class.sh won't try to create a new directory inside the parcel 185 | export LOG_DIR=`pwd` 186 | 187 | # Set heap size 188 | if [ -z "$KAFKA_HEAP_OPTS" ]; then 189 | export KAFKA_HEAP_OPTS="-Xmx${MM_HEAP_SIZE}M" 190 | else 191 | echo "KAFKA_HEAP_OPTS is already set." 192 | fi 193 | 194 | # Set java opts 195 | if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then 196 | export KAFKA_JVM_PERFORMANCE_OPTS="${CSD_JAVA_OPTS} ${MM_JAVA_OPTS}" 197 | else 198 | echo "KAFKA_JVM_PERFORMANCE_OPTS is already set." 199 | fi 200 | 201 | # And finally run Kafka MirrorMaker itself 202 | if [[ $KAFKA_MAJOR_VERSION < $MIN_REFACTORED_MIRROR_MAKER_VERSION ]]; then 203 | exec $KAFKA_HOME/bin/kafka-mirror-maker.sh --new.producer ${WL_TOPICS} ${BL_TOPICS} ${DATA_LOSS_PARAM} ${PRODUCER_PARAM} ${STREAM_PARAM} ${QUEUE_SIZE_PARAM} ${QUEUE_BYTE_SIZE_PARAM} --consumer.config $CONF_DIR/mirror_maker_consumers.properties --producer.config $CONF_DIR/mirror_maker_producers.properties 204 | else 205 | exec $KAFKA_HOME/bin/kafka-mirror-maker.sh ${ABORT_ON_SEND_FAILURE_FLAG} ${WL_TOPICS} --new.consumer ${STREAM_PARAM} ${OFFSET_COMMIT_INTERVAL_MS_PARAM} ${CONSUMER_REBALANCE_LISTENER_PARAM} ${CONSUMER_REBALANCE_LISTENER_ARGS_PARAM} ${MESSAGE_HANDLER_PARAM} ${MESSAGE_HANDLER_ARGS_PARAM} --consumer.config $CONF_DIR/mirror_maker_consumers.properties --producer.config $CONF_DIR/mirror_maker_producers.properties 206 | fi 207 | -------------------------------------------------------------------------------- /KAFKA/test/kafka_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2017 Cloudera, Inc. All rights reserved. 4 | 5 | # This is a bash test script for kafka sentry version check changes to fix 6 | # OPSAPS-37907 and OPSAPS-38677 7 | 8 | # for debugging 9 | set -x 10 | 11 | # Resets the given test directory and populates the following variables: 12 | # * TEST_DIR 13 | # * CONF_DIR 14 | # * KAFKA_HOME 15 | # * CDH_SENTRY_HOME 16 | # 17 | # Creates Kafka and Sentry version files as appropriate. 18 | # Creates sentry-conf/sentry_site.xml as appropriate. 19 | # Creates a fake kafka-server-start script. 20 | # 21 | # Takes arguments: 22 | # 1: KAFKA_VER - full version string for Kafka, like 0.10.0-kafka2.1.1. 23 | # Can also be "skip" to avoid creating the Kafka version file 24 | # altogether. 25 | # 2: SENTRY_VER - full version string for Sentry, like 1.5.1-cdh5.11.0. Can 26 | # also be "skip" to avoid creating the Sentry version file 27 | # altogether. 28 | # 3: CONFIGURE_SENTRY_SITE - "true" or "false" to indicate whether to create the sentry-site.xml file 29 | setup_test_directory() { 30 | if [ $# -ne 3 ]; then 31 | echo expected 3 arguments: KAFKA_VER, SENTRY_VER, CONFIGURE_SENTRY_SITE 32 | exit 1; 33 | fi 34 | 35 | TEST_DIR="target/test/kafka_sentry_${NUM}" 36 | if [[ ${1} == "skip" ]]; then 37 | KAFKA_VER= 38 | else 39 | KAFKA_VER=$1 40 | fi 41 | if [[ ${2} == "skip" ]]; then 42 | SENTRY_VER= 43 | else 44 | SENTRY_VER=$2 45 | fi 46 | CONFIGURE_SENTRY_SITE=$3 47 | 48 | if [ -d $TEST_DIR ]; then 49 | rm -r $TEST_DIR 50 | fi 51 | mkdir -p $TEST_DIR 52 | 53 | export CONF_DIR=$TEST_DIR/conf_dir 54 | mkdir $CONF_DIR 55 | 56 | export KAFKA_HOME=$TEST_DIR/kafka_home 57 | mkdir $KAFKA_HOME 58 | mkdir $KAFKA_HOME/bin 59 | echo "#!/bin/bash" >> $KAFKA_HOME/bin/kafka-server-start.sh 60 | echo "echo fake start Kafka" >> $KAFKA_HOME/bin/kafka-server-start.sh 61 | chmod +x $KAFKA_HOME/bin/kafka-server-start.sh 62 | if [[ -n ${KAFKA_VER} ]]; then 63 | mkdir $KAFKA_HOME/cloudera 64 | echo "version=${KAFKA_VER}" >> $KAFKA_HOME/cloudera/cdh_version.properties 65 | fi 66 | 67 | export CDH_SENTRY_HOME=$TEST_DIR/sentry_home 68 | if [[ -n ${SENTRY_VER} ]]; then 69 | mkdir $CDH_SENTRY_HOME 70 | mkdir $CDH_SENTRY_HOME/cloudera 71 | echo "version=${SENTRY_VER}" >> $CDH_SENTRY_HOME/cloudera/cdh_version.properties 72 | fi 73 | 74 | if [[ ${CONFIGURE_SENTRY_SITE} == "true" ]]; then 75 | mkdir $CONF_DIR/sentry-conf 76 | touch $CONF_DIR/sentry-conf/sentry-site.xml 77 | fi 78 | } 79 | 80 | # return 0 (true) if kafka.properties has sentry configs 81 | verify_kafka_properties_has_sentry() { 82 | if [[ ! -e $CONF_DIR/kafka.properties ]]; then 83 | echo "no" 84 | else 85 | grep -q sentry $CONF_DIR/kafka.properties 86 | if [[ $? -eq 0 ]]; then 87 | echo "yes" 88 | else 89 | echo "no" 90 | fi 91 | fi 92 | } 93 | 94 | add_error() { 95 | echo $1 >> $TEST_DIR/errors.log 96 | >&2 echo $1 97 | if [[ -z ${ERRORS} ]]; then 98 | ERRORS=$1 99 | else 100 | ERRORS="${ERRORS}; $1" 101 | fi 102 | } 103 | 104 | KAFKA_CONTROL=../src/scripts/control.sh 105 | echo "control script missing the execute bit, but we'll want that to test" 106 | chmod +x $KAFKA_CONTROL 107 | 108 | # versions taken from nightly5{8,9,10}-1.gce.cloudera.com 109 | KAFKA_58=0.9.0-kafka2.0.1 110 | KAFKA_59=0.10.0-kafka2.1.0 111 | SENTRY_58=1.5.1-cdh5.8.5-SNAPSHOT 112 | SENTRY_59=1.5.1-cdh5.9.2-SNAPSHOT 113 | SENTRY_510=1.5.1-cdh5.10.1-SNAPSHOT 114 | # Future major version bumps made up, they don't exist yet 115 | # Minor is intentionally less than the minor of first version supporting feature 116 | KAFKA_FUTURE=20.0.0-kafka11.0.0-SNAPSHOT 117 | SENTRY_FUTURE=20.0.0-cdh13.0.0 118 | 119 | NUM=1 120 | echo "Test $NUM - Happy path - Sentry, Kafka 2.1, CDH 5.10" 121 | setup_test_directory $KAFKA_59 $SENTRY_59 true 122 | $KAFKA_CONTROL 123 | if [[ $? -ne 0 ]]; then 124 | add_error "Test $NUM control script hit an error" 125 | else 126 | if [[ $(verify_kafka_properties_has_sentry) == "no" ]]; then 127 | add_error "Test $NUM expected sentry properties" 128 | else 129 | echo "Test $NUM Success!" 130 | fi 131 | fi 132 | 133 | NUM=$(($NUM + 1)) 134 | echo "Test $NUM - Sentry ignored because Kafka too old" 135 | setup_test_directory $KAFKA_58 $SENTRY_59 true 136 | $KAFKA_CONTROL 137 | if [[ $? -ne 0 ]]; then 138 | add_error "Test $NUM control script hit an error" 139 | else 140 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 141 | add_error "Test $NUM expected no sentry properties" 142 | else 143 | echo "Test $NUM Success!" 144 | fi 145 | fi 146 | 147 | NUM=$(($NUM + 1)) 148 | echo "Test $NUM - Sentry ignored because CDH too old" 149 | setup_test_directory $KAFKA_59 $SENTRY_58 true 150 | $KAFKA_CONTROL 151 | if [[ $? -ne 0 ]]; then 152 | add_error "Test $NUM control script hit an error" 153 | else 154 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 155 | add_error "Test $NUM expected no sentry properties" 156 | else 157 | echo "Test $NUM Success!" 158 | fi 159 | fi 160 | 161 | NUM=$(($NUM + 1)) 162 | echo "Test $NUM - Sentry ignored because not configured" 163 | setup_test_directory $KAFKA_59 $SENTRY_59 false 164 | $KAFKA_CONTROL 165 | if [[ $? -ne 0 ]]; then 166 | add_error "Test $NUM control script hit an error" 167 | else 168 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 169 | add_error "Test $NUM expected no sentry properties" 170 | else 171 | echo "Test $NUM Success!" 172 | fi 173 | fi 174 | 175 | NUM=$(($NUM + 1)) 176 | echo "Test $NUM - Sentry ignored because both Kafka and Sentry too old" 177 | setup_test_directory $KAFKA_58 $SENTRY_58 true 178 | $KAFKA_CONTROL 179 | if [[ $? -ne 0 ]]; then 180 | add_error "Test $NUM control script hit an error" 181 | else 182 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 183 | add_error "Test $NUM expected no sentry properties" 184 | else 185 | echo "Test $NUM Success!" 186 | fi 187 | fi 188 | 189 | NUM=$(($NUM + 1)) 190 | echo "Test $NUM - Sentry ignored because Kafka version unspecified" 191 | setup_test_directory "skip" $SENTRY_59 true 192 | $KAFKA_CONTROL 193 | if [[ $? -ne 0 ]]; then 194 | add_error "Test $NUM control script hit an error" 195 | else 196 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 197 | add_error "Test $NUM expected no sentry properties" 198 | else 199 | echo "Test $NUM Success!" 200 | fi 201 | fi 202 | 203 | NUM=$(($NUM + 1)) 204 | echo "Test $NUM - Sentry ignored because Sentry version unspecified" 205 | setup_test_directory $KAFKA_59 "skip" true 206 | $KAFKA_CONTROL 207 | if [[ $? -ne 0 ]]; then 208 | add_error "Test $NUM control script hit an error" 209 | else 210 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 211 | add_error "Test $NUM expected no sentry properties" 212 | else 213 | echo "Test $NUM Success!" 214 | fi 215 | fi 216 | 217 | NUM=$(($NUM + 1)) 218 | echo "Test $NUM - Sentry ignored because both versions unspecified" 219 | setup_test_directory "skip" "skip" true 220 | $KAFKA_CONTROL 221 | if [[ $? -ne 0 ]]; then 222 | add_error "Test $NUM control script hit an error" 223 | else 224 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 225 | add_error "Test $NUM expected no sentry properties" 226 | else 227 | echo "Test $NUM Success!" 228 | fi 229 | fi 230 | 231 | 232 | NUM=$(($NUM + 1)) 233 | echo "Test $NUM - Kafka59 Sentry510, exercising 2 digit minor version" 234 | setup_test_directory $KAFKA_59 $SENTRY_510 true 235 | $KAFKA_CONTROL 236 | if [[ $? -ne 0 ]]; then 237 | add_error "Test $NUM control script hit an error" 238 | else 239 | if [[ $(verify_kafka_properties_has_sentry) == "no" ]]; then 240 | add_error "Test $NUM expected sentry properties" 241 | else 242 | echo "Test $NUM Success!" 243 | fi 244 | fi 245 | 246 | NUM=$(($NUM + 1)) 247 | echo "Test $NUM - newer major version of Kafka and Sentry, lower minor version" 248 | setup_test_directory $KAFKA_FUTURE $SENTRY_FUTURE true 249 | $KAFKA_CONTROL 250 | if [[ $? -ne 0 ]]; then 251 | add_error "Test $NUM control script hit an error" 252 | else 253 | if [[ $(verify_kafka_properties_has_sentry) == "no" ]]; then 254 | add_error "Test $NUM expected sentry properties" 255 | else 256 | echo "Test $NUM Success!" 257 | fi 258 | fi 259 | 260 | NUM=$(($NUM + 1)) 261 | echo "Test $NUM - Versions proper, but CDH_SENTRY_HOME not set" 262 | setup_test_directory $KAFKA_59 $SENTRY_59 true 263 | export CDH_SENTRY_HOME= 264 | $KAFKA_CONTROL 265 | if [[ $? -ne 0 ]]; then 266 | add_error "Test $NUM control script hit an error" 267 | else 268 | if [[ $(verify_kafka_properties_has_sentry) == "yes" ]]; then 269 | add_error "Test $NUM expected no sentry properties" 270 | else 271 | echo "Test $NUM Success!" 272 | fi 273 | fi 274 | 275 | echo "cleanup: reset execute bit on control script" 276 | chmod -x $KAFKA_CONTROL 277 | 278 | if [[ -n ${ERRORS} ]]; then 279 | echo "FAILED due to: ${ERRORS}" 280 | exit 1; 281 | fi 282 | echo "Successfully ran all $NUM tests!" 283 | 284 | -------------------------------------------------------------------------------- /KEYTRUSTEE/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | KEYTRUSTEE 30 | 5.14.0 31 | The Cloudera Navigator Key Trustee CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /KEYTRUSTEE/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/KEYTRUSTEE/src/images/icon.png -------------------------------------------------------------------------------- /KEYTRUSTEE/src/scripts/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | add_to_kms_site() { 21 | FILE=`find $CONF_DIR -name kms-site.xml` 22 | CONF_END="" 23 | NEW_PROPERTY="$1$2" 24 | TMP_FILE=$CONF_DIR/tmp-kms-site 25 | cat $FILE | sed "s#$CONF_END#$NEW_PROPERTY#g" > $TMP_FILE 26 | cp $TMP_FILE $FILE 27 | rm -f $TMP_FILE 28 | echo $CONF_END >> $FILE 29 | } 30 | 31 | set -x 32 | 33 | # Time marker for both stderr and stdout 34 | date; date 1>&2 35 | 36 | CMD=$1 37 | shift 38 | 39 | DEFAULT_KMS_HOME=/usr/lib/hadoop-kms 40 | 41 | # Use CDH_KMS_HOME if available 42 | export KMS_HOME=${KMS_HOME:-$CDH_KMS_HOME} 43 | # If KMS_HOME is still not set, use the default value 44 | export KMS_HOME=${KMS_HOME:-$DEFAULT_KMS_HOME} 45 | 46 | # Set KMS config dir to conf dir 47 | export KMS_CONFIG=${CONF_DIR} 48 | 49 | # We want verbose startup logs 50 | export KMS_SILENT=false 51 | 52 | KMS_RUN=$CONF_DIR/run/ 53 | export KMS_TEMP=$KMS_RUN 54 | 55 | # Need to set the libexec dir to find kms-config.sh 56 | export HADOOP_HOME=${CDH_HADOOP_HOME} 57 | export HADOOP_LIBEXEC_DIR=${HADOOP_HOME}/libexec 58 | 59 | # Needed to find catalina.sh 60 | export KMS_CATALINA_HOME=$TOMCAT_HOME 61 | 62 | export CATALINA_TMPDIR=$PWD/temp 63 | # Create temp directory for Catalina 64 | mkdir -p $CATALINA_TMPDIR 65 | 66 | # Choose between the non-SSL and SSL tomcat configs 67 | TOMCAT_CONFIG_FOLDER=tomcat-conf.http 68 | if [ "x$SSL_ENABLED" == "xtrue" ]; then 69 | TOMCAT_CONFIG_FOLDER=tomcat-conf.https 70 | else 71 | SSL_ENABLED=false 72 | fi 73 | 74 | # Package settings for tomcat deployment 75 | DEPLOY_SCRIPT_BASE=/usr/lib/hadoop-kms/ 76 | TOMCAT_CONF_BASE=/etc/hadoop-kms/ 77 | 78 | # Rejigger the above if we're using parcels 79 | if [ "$CDH_KMS_HOME" != "$DEFAULT_KMS_HOME" ]; then 80 | TOMCAT_CONF_BASE=$CDH_KMS_HOME/../../etc/hadoop-kms/ 81 | DEPLOY_SCRIPT_BASE=$CDH_KMS_HOME 82 | fi 83 | 84 | # Construct the actual TOMCAT_CONF from the base and folder 85 | TOMCAT_CONF=$TOMCAT_CONF_BASE/$TOMCAT_CONFIG_FOLDER 86 | 87 | export CATALINA_BASE="$KMS_STAGING_DIR/tomcat-deployment" 88 | 89 | # Set up the number of threads and heap size 90 | export KMS_MAX_THREADS 91 | export KMS_PROTOCOL 92 | export KMS_ACCEPT_COUNT 93 | export KMS_ACCEPTOR_THREAD_COUNT 94 | export CATALINA_OPTS="-Xmx${KMS_HEAP_SIZE} ${CSD_JAVA_OPTS} ${KMS_JAVA_OPTS}" 95 | 96 | # do some ssl password stuff in private 97 | set +x 98 | 99 | # Make sure settings are coherent 100 | if [ "$SSL_ENABLED" = "true" -a \( -z "$KMS_SSL_KEYSTORE_FILE" -o -z "$KMS_SSL_KEYSTORE_PASS" \) ]; then 101 | echo "When SSL is enabled, the keystore location and password must be configured." 102 | exit 1 103 | fi 104 | 105 | #turn back on the logging 106 | set -x 107 | 108 | # Get Parcel Root to fix Key Trustee configuration directory location 109 | PARCEL_ROOT=${KEYTRUSTEE_KP_HOME%%KEYTRUSTEE*} 110 | echo "PARCEL_ROOT is ${PARCEL_ROOT}" 111 | 112 | MIN_CDH_MAJOR_VERSION_WITH_BLANK_TRUSTSTORE_PWD=5 113 | MIN_CDH_MINOR_VERSION_WITH_BLANK_TRUSTSTORE_PWD=10 114 | UNKNOWN_VERSION="unknown version" 115 | 116 | if [[ ! -f $KMS_HOME/cloudera/cdh_version.properties ]]; then 117 | CDH_VERSION=$UNKNOWN_VERSION 118 | CDH_MAJOR_VERSION=5 119 | CDH_MINOR_VERSION=4 120 | echo "$KMS_HOME/cloudera/cdh_version.properties not found. Assuming older version of CDH is being used." 121 | else 122 | # example first line of version file: version=2.6.0-cdh5.9.3 123 | CDH_VERSION=$(grep "^version=" $KMS_HOME/cloudera/cdh_version.properties | cut -d '=' -f 2) 124 | CDH_MAJOR_VERSION=$(echo $CDH_VERSION | cut -d '-' -f 2 | sed 's/cdh//g' | cut -d '.' -f 1) 125 | CDH_MINOR_VERSION=$(echo $CDH_VERSION | cut -d '-' -f 2 | sed 's/cdh//g' | cut -d '.' -f 2) 126 | echo "CDH version found: ${CDH_VERSION}" 127 | fi 128 | 129 | # Setup Tomcat Truststore options 130 | if [[ ${CDH_VERSION} == ${UNKNOWN_VERSION} || \ 131 | ${CDH_MAJOR_VERSION} -lt ${MIN_CDH_MAJOR_VERSION_WITH_BLANK_TRUSTSTORE_PWD} || \ 132 | ${CDH_MAJOR_VERSION} -eq ${MIN_CDH_MAJOR_VERSION_WITH_BLANK_TRUSTSTORE_PWD} && \ 133 | ${CDH_MINOR_VERSION} -lt ${MIN_CDH_MINOR_VERSION_WITH_BLANK_TRUSTSTORE_PWD} ]]; then 134 | set +x 135 | export CATALINA_OPTS="$CATALINA_OPTS -Djavax.net.ssl.trustStore=${KMS_SSL_TRUSTSTORE_FILE} -Djavax.net.ssl.trustStorePassword=${KMS_SSL_TRUSTSTORE_PASS} -Dcdh.parcel.root=${PARCEL_ROOT}" 136 | CATALINA_OPTS_DISP=`echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/'` 137 | #turn back on the logging 138 | set -x 139 | print "Using CATALINA_OPTS: ${CATALINA_OPTS_DISP}" 140 | else 141 | export CATALINA_OPTS="$CATALINA_OPTS -Djavax.net.ssl.trustStore=${KMS_SSL_TRUSTSTORE_FILE} -Dcdh.parcel.root=${PARCEL_ROOT}" 142 | print "Using CATALINA_OPTS: ${CATALINA_OPTS}" 143 | fi 144 | 145 | KMS_PLUGIN_DIR=${KEYTRUSTEE_KP_HOME:-/usr/share/keytrustee-keyprovider}/lib 146 | 147 | # Deploy KMS tomcat app. 148 | env TOMCAT_CONF=${TOMCAT_CONF} TOMCAT_DEPLOYMENT=${CATALINA_BASE} KMS_HOME=${KMS_HOME} \ 149 | KMS_PLUGIN_DIR=${KMS_PLUGIN_DIR} \ 150 | bash ${DEPLOY_SCRIPT_BASE}/tomcat-deployment.sh 151 | 152 | # Print out all the env vars we've set 153 | echo "KMS_HOME is ${KMS_HOME}" 154 | echo "KMS_LOG is ${KMS_LOG}" 155 | echo "KMS_CONFIG is ${KMS_CONFIG}" 156 | echo "KMS_MAX_THREADS is ${KMS_MAX_THREADS}" 157 | echo "KMS_PROTOCOL is ${KMS_PROTOCOL}" 158 | echo "KMS_ACCEPT_COUNT is ${KMS_ACCEPT_COUNT}" 159 | echo "KMS_ACCEPTOR_THREAD_COUNT is ${KMS_ACCEPTOR_THREAD_COUNT}" 160 | echo "KMS_HEAP_SIZE is ${KMS_HEAP_SIZE}" 161 | echo "TOMCAT_CONF is ${TOMCAT_CONF}" 162 | echo "CATALINA_BASE is ${CATALINA_BASE}" 163 | echo "SSL_ENABLED is ${SSL_ENABLED}" 164 | echo "KMS_SSL_KEYSTORE_FILE is ${KMS_SSL_KEYSTORE_FILE}" 165 | 166 | echo "KMS_PLUGIN_DIR is ${KMS_PLUGIN_DIR}" 167 | echo "KMS_SSL_TRUSTSTORE_FILE is ${KMS_SSL_TRUSTSTORE_FILE}" 168 | echo "CSD_JAVA_OPTS is ${CSD_JAVA_OPTS}" 169 | echo "KMS_JAVA_OPTS is ${KMS_JAVA_OPTS}" 170 | 171 | # Add zk quorum to kms-site.xml 172 | add_to_kms_site hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string $ZK_QUORUM 173 | 174 | # replace {{CONF_DIR}} template in kms-site.xml 175 | perl -pi -e "s#{{CONF_DIR}}#${CONF_DIR}#" ${CONF_DIR}/kms-site.xml 176 | 177 | # replace {{TKP_INSECURE}} template in kts-site.xml 178 | # "insecure" needs to be set to the opposite of "secure", aka SSL 179 | TKP_INSECURE="false" 180 | if [ "$SSL_ENABLED" = "false" ]; then 181 | TKP_INSECURE="true" 182 | unset KMS_SSL_KEYSTORE_PASS 183 | unset KMS_SSL_TRUSTSTORE_PASS 184 | fi 185 | if [ -n "$ZK_QUORUM" ]; then 186 | add_to_kms_site hadoop.kms.authentication.signer.secret.provider zookeeper 187 | add_to_kms_site hadoop.kms.authentication.zk-dt-secret-manager.enable true 188 | fi 189 | perl -pi -e "s#{{TKP_INSECURE}}#${TKP_INSECURE}#" ${CONF_DIR}/kts-site.xml 190 | 191 | case $CMD in 192 | (start) 193 | cmd="${KMS_HOME}/sbin/kms.sh run" 194 | exec ${cmd} 195 | ;; 196 | (backup) 197 | if [ -f $KMS_PLUGIN_DIR/../../bin/ktbackup.sh ]; then 198 | cmd="${KMS_PLUGIN_DIR}/../../bin/ktbackup.sh --cleartext --confdir=${KMS_CONFDIR}/.keytrustee --output=${KMS_CONFDIR}/.." 199 | exec ${cmd} 200 | else 201 | echo " The backup script does not exist. Will not be taking the backup." 202 | fi 203 | ;; 204 | (*) 205 | echo "Unknown command ${CMD}" 206 | exit 1 207 | ;; 208 | esac 209 | -------------------------------------------------------------------------------- /KMS/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | KMS 30 | 5.14.0 31 | The Hadoop Key Management Server CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /KMS/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/KMS/src/images/icon.png -------------------------------------------------------------------------------- /KMS/src/scripts/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | set -x 21 | 22 | # Time marker for both stderr and stdout 23 | date; date 1>&2 24 | 25 | CMD=$1 26 | shift 27 | 28 | DEFAULT_KMS_HOME=/usr/lib/hadoop-kms 29 | 30 | # Use CDH_KMS_HOME if available 31 | export KMS_HOME=${KMS_HOME:-$CDH_KMS_HOME} 32 | # If KMS_HOME is still not set, use the default value 33 | export KMS_HOME=${KMS_HOME:-$DEFAULT_KMS_HOME} 34 | 35 | # Set KMS config dir to conf dir 36 | export KMS_CONFIG=${CONF_DIR} 37 | 38 | # Disabling logging to protect the password 39 | set +x 40 | 41 | # Make sure settings are coherent 42 | if [ "$SSL_ENABLED" = "true" -a \( -z "$KMS_SSL_KEYSTORE_FILE" -o -z "$KMS_SSL_KEYSTORE_PASS" \) ]; then 43 | echo "When SSL is enabled, the keystore location and password must be configured." 44 | exit 1 45 | fi 46 | 47 | # Make sure the password gets set 48 | export HADOOP_KEYSTORE_PASSWORD=${KMS_JKS_PASSWORD:-none} 49 | 50 | # Re-enabling logging 51 | set -x 52 | 53 | # We want verbose startup logs 54 | export KMS_SILENT=false 55 | 56 | KMS_RUN=$CONF_DIR/run/ 57 | export KMS_TEMP=$KMS_RUN 58 | 59 | # Need to set the libexec dir to find kms-config.sh 60 | export HADOOP_HOME=${CDH_HADOOP_HOME} 61 | export HADOOP_LIBEXEC_DIR=${HADOOP_HOME}/libexec 62 | 63 | # Needed to find catalina.sh 64 | export KMS_CATALINA_HOME=$TOMCAT_HOME 65 | 66 | export CATALINA_TMPDIR=$PWD/temp 67 | # Create temp directory for Catalina 68 | mkdir -p $CATALINA_TMPDIR 69 | 70 | # Choose between the non-SSL and SSL tomcat configs 71 | TOMCAT_CONFIG_FOLDER=tomcat-conf.http 72 | if [ "x$SSL_ENABLED" == "xtrue" ]; then 73 | TOMCAT_CONFIG_FOLDER=tomcat-conf.https 74 | else 75 | SSL_ENABLED=false 76 | fi 77 | 78 | # Package settings for tomcat deployment 79 | DEPLOY_SCRIPT_BASE=/usr/lib/hadoop-kms/ 80 | TOMCAT_CONF_BASE=/etc/hadoop-kms/ 81 | 82 | # Rejigger the above if we're using parcels 83 | if [ "$CDH_KMS_HOME" != "$DEFAULT_KMS_HOME" ]; then 84 | TOMCAT_CONF_BASE=$CDH_KMS_HOME/../../etc/hadoop-kms/ 85 | DEPLOY_SCRIPT_BASE=$CDH_KMS_HOME 86 | fi 87 | 88 | # Construct the actual TOMCAT_CONF from the base and folder 89 | TOMCAT_CONF=$TOMCAT_CONF_BASE/$TOMCAT_CONFIG_FOLDER 90 | 91 | export CATALINA_BASE="$KMS_STAGING_DIR/tomcat-deployment" 92 | 93 | # Set up the number of threads and heap size 94 | export KMS_MAX_THREADS 95 | export KMS_PROTOCOL 96 | export KMS_ACCEPT_COUNT 97 | export KMS_ACCEPTOR_THREAD_COUNT 98 | export CATALINA_OPTS="-Xmx${KMS_HEAP_SIZE} ${CSD_JAVA_OPTS} ${KMS_JAVA_OPTS}" 99 | 100 | # Deploy KMS tomcat app. 101 | env TOMCAT_CONF=${TOMCAT_CONF} TOMCAT_DEPLOYMENT=${CATALINA_BASE} KMS_HOME=${KMS_HOME} \ 102 | bash ${DEPLOY_SCRIPT_BASE}/tomcat-deployment.sh 103 | 104 | # Print out all the env vars we've set 105 | echo "KMS_HOME is ${KMS_HOME}" 106 | echo "KMS_LOG is ${KMS_LOG}" 107 | echo "KMS_CONFIG is ${KMS_CONFIG}" 108 | echo "KMS_MAX_THREADS is ${KMS_MAX_THREADS}" 109 | echo "KMS_PROTOCOL is ${KMS_PROTOCOL}" 110 | echo "KMS_ACCEPT_COUNT is ${KMS_ACCEPT_COUNT}" 111 | echo "KMS_ACCEPTOR_THREAD_COUNT is ${KMS_ACCEPTOR_THREAD_COUNT}" 112 | echo "KMS_HEAP_SIZE is ${KMS_HEAP_SIZE}" 113 | echo "TOMCAT_CONF is ${TOMCAT_CONF}" 114 | echo "CATALINA_BASE is ${CATALINA_BASE}" 115 | echo "SSL_ENABLED is ${SSL_ENABLED}" 116 | echo "KMS_SSL_KEYSTORE_FILE is ${KMS_SSL_KEYSTORE_FILE}" 117 | echo "CSD_JAVA_OPTS is ${CSD_JAVA_OPTS}" 118 | echo "KMS_JAVA_OPTS is ${KMS_JAVA_OPTS}" 119 | 120 | 121 | if [ "$SSL_ENABLED" = "false" ]; then 122 | unset KMS_SSL_KEYSTORE_PASS 123 | unset KMS_SSL_TRUSTSTORE_PASS 124 | fi 125 | 126 | # replace {{CONF_DIR}} template in kms-site.xml 127 | perl -pi -e "s#{{CONF_DIR}}#${CONF_DIR}#" ${CONF_DIR}/kms-site.xml 128 | 129 | case $CMD in 130 | (start) 131 | cmd="${KMS_HOME}/sbin/kms.sh run" 132 | exec ${cmd} 133 | ;; 134 | (*) 135 | echo "Unknown command ${CMD}" 136 | exit 1 137 | ;; 138 | esac 139 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Cloudera Manager CSDs 2 | ======= 3 | 4 | A collection of Custom Service Descriptors. 5 | 6 | Requirements 7 | ------------ 8 | 9 | * Maven 3 (to build) 10 | 11 | ## Building the CSDs 12 | 13 | The CSDs can be build by running: 14 | 15 | ```bash 16 | $ mvn install 17 | ``` 18 | 19 | The CSD itself is a jar file located under the target 20 | directory of each CSD. For Spark, the CSD is located: 21 | 22 | ```bash 23 | $ ls SPARK/target/SPARK-1.0-SNAPSHOT.jar 24 | ``` 25 | 26 | All source in this repository is [Apache-Licensed](LICENSE.txt). 27 | 28 | -------------------------------------------------------------------------------- /SPARK/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | SPARK 30 | 5.14.0 31 | The Spark CSD 32 | pom 33 | 34 | 35 | 36 | 37 | clover 38 | 39 | 40 | 41 | com.cloudera.enterprise 42 | schema-validator-maven-plugin 43 | 44 | 45 | ${clover.version} 46 | com.atlassian.clover 47 | clover 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-assembly-plugin 61 | 62 | false 63 | 64 | ../assembly.xml 65 | 66 | 67 | 68 | 69 | make-assembly 70 | package 71 | 72 | single 73 | 74 | 75 | 76 | 77 | 78 | com.cloudera.enterprise 79 | schema-validator-maven-plugin 80 | 81 | 82 | validate-schema 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /SPARK/src/aux: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/aux -------------------------------------------------------------------------------- /SPARK/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/SPARK/src/images/icon.png -------------------------------------------------------------------------------- /SPARK/src/scripts/common.sh: -------------------------------------------------------------------------------- 1 | ../../../SPARK_ON_YARN/src/scripts/common.sh -------------------------------------------------------------------------------- /SPARK/src/scripts/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | . $(cd $(dirname $0) && pwd)/common.sh 21 | 22 | MASTER_FILE="$CONF_DIR/spark-conf/master.properties" 23 | 24 | ### Let's run everything with JVM runtime, instead of Scala 25 | export SPARK_LAUNCH_WITH_SCALA=0 26 | export SPARK_LIBRARY_PATH=${SPARK_HOME}/lib 27 | export SCALA_LIBRARY_PATH=${SPARK_HOME}/lib 28 | 29 | if [ -n "$HADOOP_HOME" ]; then 30 | SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native 31 | fi 32 | export SPARK_LIBRARY_PATH 33 | 34 | if [ -f $MASTER_FILE ]; then 35 | MASTER_IP= 36 | MASTER_PORT= 37 | for line in $(cat $MASTER_FILE) 38 | do 39 | readconf "$line" 40 | case $key in 41 | server.address) 42 | if [ -n "$value" ]; then 43 | MASTER_IP=$value 44 | fi 45 | ;; 46 | server.port) 47 | if [ -z "$MASTER_IP" ]; then 48 | MASTER_IP=$host 49 | fi 50 | MASTER_PORT=$value 51 | ;; 52 | esac 53 | done 54 | log "Found a master on $MASTER_IP listening on port $MASTER_PORT" 55 | rm "$MASTER_FILE" 56 | fi 57 | 58 | case $1 in 59 | 60 | (start_master) 61 | log "Starting Spark master on $MASTER_IP and port $MASTER_PORT" 62 | ARGS=( 63 | "org.apache.spark.deploy.master.Master" 64 | "--ip" 65 | $MASTER_IP 66 | ) 67 | 68 | prepare_spark_env $SPARK_CONF_DIR/$ENV_FILENAME 69 | run_spark_class "${ARGS[@]}" 70 | ;; 71 | 72 | (start_worker) 73 | log "Starting Spark worker using $MASTER_URL" 74 | MASTER_URL="spark://$MASTER_IP:$MASTER_PORT" 75 | ARGS=( 76 | "org.apache.spark.deploy.worker.Worker" 77 | $MASTER_URL 78 | ) 79 | run_spark_class "${ARGS[@]}" 80 | ;; 81 | 82 | (start_history_server) 83 | start_history_server 84 | ;; 85 | 86 | (client) 87 | log "Deploying client configuration" 88 | deploy_client_config 89 | echo "spark.master=spark://$MASTER_IP:$MASTER_PORT" >> $SPARK_DEFAULTS 90 | ;; 91 | 92 | (upload_jar) 93 | upload_jar 94 | ;; 95 | 96 | (*) 97 | log "Don't understand [$1]" 98 | exit 1 99 | ;; 100 | 101 | esac 102 | -------------------------------------------------------------------------------- /SPARK_ON_YARN/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | SPARK_ON_YARN 30 | 5.14.0 31 | The Spark on YARN CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /SPARK_ON_YARN/src/aux/client/spark-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ## 3 | # Generated by Cloudera Manager and should not be modified directly 4 | ## 5 | 6 | SELF="$(cd $(dirname $BASH_SOURCE) && pwd)" 7 | if [ -z "$SPARK_CONF_DIR" ]; then 8 | export SPARK_CONF_DIR="$SELF" 9 | fi 10 | 11 | export SPARK_HOME={{SPARK_HOME}} 12 | export DEFAULT_HADOOP_HOME={{HADOOP_HOME}} 13 | 14 | ### Path of Spark assembly jar in HDFS 15 | export SPARK_JAR_HDFS_PATH=${SPARK_JAR_HDFS_PATH:-'{{SPARK_JAR_HDFS_PATH}}'} 16 | 17 | ### Some definitions needed by older versions of CDH. 18 | export SPARK_LAUNCH_WITH_SCALA=0 19 | export SPARK_LIBRARY_PATH=${SPARK_HOME}/lib 20 | export SCALA_LIBRARY_PATH=${SPARK_HOME}/lib 21 | 22 | SPARK_PYTHON_PATH="{{PYTHON_PATH}}" 23 | if [ -n "$SPARK_PYTHON_PATH" ]; then 24 | export PYTHONPATH="$PYTHONPATH:$SPARK_PYTHON_PATH" 25 | fi 26 | 27 | export HADOOP_HOME=${HADOOP_HOME:-$DEFAULT_HADOOP_HOME} 28 | 29 | if [ -n "$HADOOP_HOME" ]; then 30 | LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${HADOOP_HOME}/lib/native 31 | fi 32 | 33 | SPARK_EXTRA_LIB_PATH="{{SPARK_EXTRA_LIB_PATH}}" 34 | if [ -n "$SPARK_EXTRA_LIB_PATH" ]; then 35 | LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPARK_EXTRA_LIB_PATH 36 | fi 37 | 38 | export LD_LIBRARY_PATH 39 | 40 | HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-$SPARK_CONF_DIR/{{HADOOP_CONF_DIR_NAME}}} 41 | HIVE_CONF_DIR=${HIVE_CONF_DIR:-/etc/hive/conf} 42 | if [ -d "$HIVE_CONF_DIR" ]; then 43 | HADOOP_CONF_DIR="$HADOOP_CONF_DIR:$HIVE_CONF_DIR" 44 | fi 45 | export HADOOP_CONF_DIR 46 | 47 | PYLIB="$SPARK_HOME/python/lib" 48 | if [ -f "$PYLIB/pyspark.zip" ]; then 49 | PYSPARK_ARCHIVES_PATH= 50 | for lib in "$PYLIB"/*.zip; do 51 | if [ -n "$PYSPARK_ARCHIVES_PATH" ]; then 52 | PYSPARK_ARCHIVES_PATH="$PYSPARK_ARCHIVES_PATH,local:$lib" 53 | else 54 | PYSPARK_ARCHIVES_PATH="local:$lib" 55 | fi 56 | done 57 | export PYSPARK_ARCHIVES_PATH 58 | fi 59 | 60 | # Set distribution classpath. This is only used in CDH 5.3 and later. 61 | export SPARK_DIST_CLASSPATH=$(paste -sd: "$SELF/classpath.txt") 62 | 63 | # Spark uses `set -a` to export all variables created or modified in this 64 | # script as env vars. We use a temporary variables to avoid env var name 65 | # collisions. 66 | # If PYSPARK_PYTHON is unset, set to CDH_PYTHON 67 | TMP_PYSPARK_PYTHON=${PYSPARK_PYTHON:-'{{CDH_PYTHON}}'} 68 | # If PYSPARK_DRIVER_PYTHON is unset, set to CDH_PYTHON 69 | TMP_PYSPARK_DRIVER_PYTHON=${PYSPARK_DRIVER_PYTHON:-{{CDH_PYTHON}}} 70 | 71 | if [ -n "$TMP_PYSPARK_PYTHON" ] && [ -n "$TMP_PYSPARK_DRIVER_PYTHON" ]; then 72 | export PYSPARK_PYTHON="$TMP_PYSPARK_PYTHON" 73 | export PYSPARK_DRIVER_PYTHON="$TMP_PYSPARK_DRIVER_PYTHON" 74 | fi 75 | -------------------------------------------------------------------------------- /SPARK_ON_YARN/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/SPARK_ON_YARN/src/images/icon.png -------------------------------------------------------------------------------- /SPARK_ON_YARN/src/scripts/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | . $(cd $(dirname $0) && pwd)/common.sh 21 | 22 | case $1 in 23 | (start_history_server) 24 | start_history_server 25 | ;; 26 | 27 | (client) 28 | deploy_client_config 29 | # The default deploy mode. Passed as an argument to the script. Prepend it with "yarn-" 30 | # to build the default master configuration, but try not to overwrite user configs. 31 | if ! grep -q 'spark.master' $SPARK_DEFAULTS; then 32 | echo "spark.master=yarn-$DEPLOY_MODE" >> $SPARK_DEFAULTS 33 | fi 34 | ;; 35 | 36 | (upload_jar) 37 | upload_jar 38 | ;; 39 | 40 | (*) 41 | log "Don't understand [$1]" 42 | exit 1 43 | ;; 44 | esac 45 | -------------------------------------------------------------------------------- /SPARK_ON_YARN53/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | SPARK_ON_YARN53 30 | 5.14.0 31 | The Spark on YARN 53 CSD 32 | pom 33 | 34 | 35 | 36 | 37 | clover 38 | 39 | 40 | 41 | com.cloudera.enterprise 42 | schema-validator-maven-plugin 43 | 44 | 45 | ${clover.version} 46 | com.atlassian.clover 47 | clover 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-assembly-plugin 61 | 62 | false 63 | 64 | ../assembly.xml 65 | 66 | 67 | 68 | 69 | make-assembly 70 | package 71 | 72 | single 73 | 74 | 75 | 76 | 77 | 78 | com.cloudera.enterprise 79 | schema-validator-maven-plugin 80 | 81 | 82 | validate-schema 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /SPARK_ON_YARN53/src/aux: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/aux -------------------------------------------------------------------------------- /SPARK_ON_YARN53/src/descriptor/service.sdl: -------------------------------------------------------------------------------- 1 | // Licensed to Cloudera, Inc. under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. Cloudera, Inc. licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | { 17 | "name" : "SPARK_ON_YARN", 18 | "label" : "Spark", 19 | "description" : "Apache Spark is an open source cluster computing system. This service runs Spark as an application on YARN.", 20 | "version" : "5.14.0", 21 | "compatibility" : { "cdhVersion" : { "min" : "5", "max" : "5.4" } }, 22 | "runAs" : { 23 | "user" : "spark", 24 | "group" : "spark", 25 | "principal" : "spark" 26 | }, 27 | "inExpressWizard" : true, 28 | "icon" : "images/icon.png", 29 | "parcel" : { 30 | "requiredTags" : ["spark", "cdh"], 31 | "optionalTags" : ["spark-plugin"] 32 | }, 33 | "serviceDependencies" : [ 34 | { 35 | "name" : "YARN", 36 | "required" : "true" 37 | } 38 | ], 39 | "commands" : [ 40 | { 41 | "name" : "SparkUploadJarServiceCommand", 42 | "label" : "Install Spark JAR", 43 | "description" : "Install Spark assembly JAR on HDFS.", 44 | "roleName" : "SPARK_YARN_HISTORY_SERVER", 45 | "roleCommand" : "SparkUploadJarCommand", 46 | "runMode" : "single" 47 | } 48 | ], 49 | "hdfsDirs" : [ 50 | { 51 | "name" : "CreateSparkUserDirCommand", 52 | "label" : "Create Spark User Dir", 53 | "description" : "Creates the Spark user directory in HDFS.", 54 | "directoryDescription" : "Spark HDFS user directory", 55 | "path" : "/user/${principal}", 56 | "permissions" : "0751" 57 | }, 58 | { 59 | "name" : "CreateSparkHistoryDirCommand", 60 | "label" : "Create Spark History Log Dir", 61 | "description" : "Creates the directory in HDFS where application history will be stored.", 62 | "directoryDescription" : "Spark Application History directory", 63 | "path" : "${spark_history_log_dir}", 64 | "permissions" : "1777" 65 | } 66 | ], 67 | "serviceInit" : { 68 | "preStartSteps" : [ 69 | { 70 | "commandName" : "CreateSparkUserDirCommand" 71 | }, 72 | { 73 | "commandName" : "CreateSparkHistoryDirCommand" 74 | }, 75 | { 76 | "commandName" : "SparkUploadJarServiceCommand" 77 | } 78 | ] 79 | }, 80 | "parameters" : [ 81 | { 82 | "name" : "spark_jar_hdfs_path", 83 | "label" : "Spark JAR Location (HDFS)", 84 | "description" : "The location of the Spark JAR in HDFS. If left blank, Cloudera Manager will use the Spark JAR installed on the cluster nodes.", 85 | "type" : "path", 86 | "pathType" : "serviceSpecific", 87 | "default" : "" 88 | }, 89 | { 90 | "name" : "spark_history_log_dir", 91 | "label" : "Spark History Location (HDFS)", 92 | "description" : "The location of Spark application history logs in HDFS. Changing this value will not move existing logs to the new location.", 93 | "configName" : "spark.eventLog.dir", 94 | "default" : "/user/spark/applicationHistory", 95 | "type" : "path", 96 | "pathType" : "serviceSpecific", 97 | "required" : "true" 98 | } 99 | ], 100 | "rolesWithExternalLinks" : ["SPARK_YARN_HISTORY_SERVER"], 101 | "roles" : [ 102 | { 103 | "name" : "SPARK_YARN_HISTORY_SERVER", 104 | "label" : "History Server", 105 | "pluralLabel" : "History Servers", 106 | "jvmBased": true, 107 | "startRunner" : { 108 | "program" : "scripts/control.sh", 109 | "args" : [ "start_history_server" ], 110 | "environmentVariables" : { 111 | "HISTORY_LOG_DIR" : "${spark_history_log_dir}", 112 | "SPARK_DAEMON_MEMORY" : "${history_server_max_heapsize}", 113 | // TODO: move to command line args History Server enhancements are backported. 114 | "SPARK_DAEMON_JAVA_OPTS" : "-Dspark.history.ui.port=${history_server_web_port}" 115 | } 116 | }, 117 | "kerberosPrincipals" : [ 118 | { 119 | "name" : "SPARK_PRINCIPAL", 120 | "primary" : "${principal}", 121 | "instance" : "${host}" 122 | } 123 | ], 124 | "commands" : [ 125 | { 126 | "name" : "SparkUploadJarCommand", 127 | "label" : "Install Spark JAR", 128 | "description" : "Install Spark assembly JAR on HDFS.", 129 | "expectedExitCodes" : [0], 130 | "requiredRoleState" : "stopped", 131 | "commandRunner" : { 132 | "program" : "scripts/control.sh", 133 | "args" : [ "upload_jar" ], 134 | "environmentVariables" : { 135 | "SPARK_JAR" : "${spark_jar_hdfs_path}" 136 | } 137 | } 138 | } 139 | ], 140 | "externalLink" : { 141 | "name" : "history_server_web_ui", 142 | "label" : "History Server Web UI", 143 | "url" : "http://${host}:${history_server_web_port}" 144 | }, 145 | "topology" : { "minInstances" : 1, "maxInstances" : 1 }, 146 | "logging" : { 147 | "configFilename" : "spark-conf/log4j.properties", 148 | "dir" : "/var/log/spark", 149 | "filename" : "spark-history-server-${host}.log", 150 | "modifiable" : true, 151 | "loggingType" : "log4j" 152 | }, 153 | "parameters" : [ 154 | { 155 | "name" : "history_server_web_port", 156 | "label" : "History Server WebUI Port", 157 | "configName" : "history.port", 158 | "description" : "The port of the history server WebUI", 159 | "required" : "true", 160 | "type" : "port", 161 | "default" : 18088 162 | }, 163 | { 164 | "name" : "history_server_max_heapsize", 165 | "label" : "Java Heap Size of History Server in Bytes", 166 | "description" : "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", 167 | "required" : "true", 168 | "type" : "memory", 169 | "unit" : "bytes", 170 | "min" : 67108864, 171 | "default" : 536870912 172 | } 173 | ], 174 | "configWriter" : { 175 | "auxConfigGenerators" : [ 176 | { 177 | "filename" : "spark-conf/spark-env.sh", 178 | "sourceFilename" : "aux/client/spark-env.sh" 179 | } 180 | ] 181 | }, 182 | "healthAggregation" : { 183 | "type" : "singleton" 184 | } 185 | } 186 | ], 187 | "gateway" : { 188 | "alternatives" : { 189 | "name" : "spark-conf", 190 | // The priority is set to be higher than Spark standalone by default 191 | "priority" : 51, 192 | "linkRoot" : "/etc/spark" 193 | }, 194 | "parameters" : [ 195 | { 196 | "name" : "spark_history_enabled", 197 | "label" : "Enable History", 198 | "description" : "Write Spark application history logs to HDFS.", 199 | "configName" : "spark.eventLog.enabled", 200 | "required" : "false", 201 | "type" : "boolean", 202 | "default" : true 203 | }, 204 | { 205 | "name" : "spark_deploy_mode", 206 | "label" : "Default Application Deploy Mode", 207 | "description" : "Which deploy mode to use by default. Can be overridden by users when launching applications.", 208 | "required" : "false", 209 | "type" : "string_enum", 210 | "validValues" : [ "client", "cluster" ], 211 | "default" : "client" 212 | }, 213 | { 214 | "name" : "spark_data_serializer", 215 | "label" : "Spark Data Serializer", 216 | "description" : "Name of class implementing org.apache.spark.serializer.Serializer to use in Spark applications.", 217 | "configName" : "spark.serializer", 218 | "default" : "org.apache.spark.serializer.KryoSerializer", 219 | "type" : "string", 220 | "required" : "true" 221 | }, 222 | { 223 | "name" : "spark_python_path", 224 | "label" : "Extra Python Path", 225 | "description" : "Python library paths to add to PySpark applications.", 226 | "required" : "false", 227 | "type" : "path_array", 228 | "pathType" : "serviceSpecific", 229 | "separator" : ":", 230 | "default" : [ ] 231 | }, 232 | { 233 | "name" : "spark_gateway_ui_kill_enabled", 234 | "label" : "Enable Kill From UI", 235 | "description" : "Whether to allow users to kill running stages from the Spark Web UI.", 236 | "configName" : "spark.ui.killEnabled", 237 | "required" : "true", 238 | "type" : "boolean", 239 | "default" : true 240 | } 241 | ], 242 | "scriptRunner" : { 243 | "program" : "scripts/control.sh", 244 | "args" : [ "client" ], 245 | "environmentVariables" : { 246 | "DEPLOY_MODE" : "${spark_deploy_mode}", 247 | "SPARK_JAR" : "${spark_jar_hdfs_path}", 248 | "PYTHON_PATH" : "${spark_python_path}" 249 | } 250 | }, 251 | "logging" : { 252 | "configFilename" : "spark-conf/log4j.properties", 253 | "loggingType" : "log4j", 254 | "additionalConfigs" : [ 255 | { "key" : "log4j.logger.org.eclipse.jetty", "value" : "WARN" }, 256 | { "key" : "log4j.logger.org.spark-project.jetty", "value" : "WARN" }, 257 | { "key" : "log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle", "value" : "ERROR" }, 258 | { "key" : "log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper", "value" : "INFO" }, 259 | { "key" : "log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter", "value" : "INFO" }, 260 | { "key" : "log4j.logger.org.apache.parquet", "value" : "ERROR" }, 261 | { "key" : "log4j.logger.parquet", "value" : "ERROR" }, 262 | { "key" : "log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler", "value" : "FATAL" }, 263 | { "key" : "log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry", "value" : "ERROR" } 264 | ] 265 | }, 266 | "configWriter" : { 267 | "generators" : [ 268 | { 269 | "filename" : "spark-conf/spark-defaults.conf", 270 | "configFormat" : "properties", 271 | "includedParams" : [ 272 | "spark_history_enabled", 273 | "spark_history_log_dir", 274 | "spark_data_serializer", 275 | "spark_gateway_ui_kill_enabled" 276 | ] 277 | } 278 | ], 279 | "auxConfigGenerators" : [ 280 | { 281 | "filename" : "spark-conf/spark-env.sh", 282 | "sourceFilename" : "aux/client/spark-env.sh" 283 | } 284 | ], 285 | "peerConfigGenerators" : [ 286 | { 287 | "filename" : "spark-conf/history.properties", 288 | "params" : ["history_server_web_port"], 289 | "roleName" : "SPARK_YARN_HISTORY_SERVER" 290 | } 291 | ] 292 | } 293 | } 294 | } 295 | 296 | 297 | -------------------------------------------------------------------------------- /SPARK_ON_YARN53/src/images: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/images -------------------------------------------------------------------------------- /SPARK_ON_YARN53/src/scripts: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/scripts -------------------------------------------------------------------------------- /SPARK_ON_YARN54/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | SPARK_ON_YARN54 30 | 5.14.0 31 | The Spark on YARN 54 CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /SPARK_ON_YARN54/src/aux: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/aux -------------------------------------------------------------------------------- /SPARK_ON_YARN54/src/images: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/images -------------------------------------------------------------------------------- /SPARK_ON_YARN54/src/scripts: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/scripts -------------------------------------------------------------------------------- /SPARK_ON_YARN5_10/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | SPARK_ON_YARN5_10 30 | 5.14.0 31 | The Spark on YARN 5.10 CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /SPARK_ON_YARN5_10/src/aux: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/aux -------------------------------------------------------------------------------- /SPARK_ON_YARN5_10/src/images: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/images -------------------------------------------------------------------------------- /SPARK_ON_YARN5_10/src/scripts: -------------------------------------------------------------------------------- 1 | ../../SPARK_ON_YARN/src/scripts -------------------------------------------------------------------------------- /SQOOP_CLIENT/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 20 | 4.0.0 21 | 22 | 23 | com.cloudera 24 | csd 25 | 5.14.0 26 | 27 | 28 | com.cloudera.csd 29 | SQOOP_CLIENT 30 | 5.14.0 31 | The Sqoop Client CSD 32 | pom 33 | 34 | 35 | 36 | clover 37 | 38 | 39 | 40 | com.cloudera.enterprise 41 | schema-validator-maven-plugin 42 | 43 | 44 | ${clover.version} 45 | com.atlassian.clover 46 | clover 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-assembly-plugin 60 | 61 | false 62 | 63 | ../assembly.xml 64 | 65 | 66 | 67 | 68 | make-assembly 69 | package 70 | 71 | single 72 | 73 | 74 | 75 | 76 | 77 | com.cloudera.enterprise 78 | schema-validator-maven-plugin 79 | 80 | 81 | validate-schema 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /SQOOP_CLIENT/src/aux/sqoop-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ## 3 | # Generated by Cloudera Manager and should not be modified directly 4 | ## 5 | -------------------------------------------------------------------------------- /SQOOP_CLIENT/src/descriptor/service.sdl: -------------------------------------------------------------------------------- 1 | // Licensed to Cloudera, Inc. under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. Cloudera, Inc. licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | { 17 | "name" : "SQOOP_CLIENT", 18 | "label" : "Sqoop 1 Client", 19 | "description" : "Configuration and connector management for Sqoop 1.", 20 | "version" : "5.14.0", 21 | "compatibility" : { "cdhVersion" : { "min" : "5", "max" : "5" } }, 22 | "runAs" : { 23 | "user" : "root", 24 | "group" : "root" 25 | }, 26 | "maxInstances" : 1, 27 | "icon" : "images/sqoop.png", 28 | "parcel" : { 29 | "repoUrl" : "https://archive.cloudera.com/sqoop-connectors/parcels/latest/", 30 | "requiredTags" : [ "cdh" ], 31 | "optionalTags" : [ "sqoop-plugin" ] 32 | }, 33 | "gateway" : { 34 | "alternatives" : { 35 | "name" : "sqoop-conf", 36 | "priority" : 50, 37 | "linkRoot" : "/etc/sqoop" 38 | }, 39 | "parameters" : [ 40 | { 41 | "name" : "sqoop_connection_factories", 42 | "label" : "Sqoop connection factories", 43 | "description" : "A list of ManagerFactory implementations which are consulted, in order, to instantiate ConnManager instances used to drive connections to databases.", 44 | "required" : "false", 45 | "configName" : "sqoop.connection.factories", 46 | "type" : "string_array", 47 | "separator" : ",", 48 | "default" : [] 49 | }, 50 | { 51 | "name" : "sqoop_tool_plugins", 52 | "label" : "Sqoop Tool Plugins", 53 | "description" : "A list of ToolPlugin implementations which are consulted, in order, to register SqoopTool instances which allow third-party tools to be used.", 54 | "required" : "false", 55 | "configName" : "sqoop.tool.plugins", 56 | "type" : "string_array", 57 | "separator" : ",", 58 | "default" : [] 59 | } 60 | ], 61 | "scriptRunner" : { 62 | "program" : "scripts/sqoop.sh", 63 | "args" : [ "client" ] 64 | }, 65 | "configWriter" : { 66 | "auxConfigGenerators" : [ 67 | { 68 | "filename" : "sqoop-conf/sqoop-env.sh", 69 | "sourceFilename" : "aux/sqoop-env.sh" 70 | }, 71 | { 72 | "filename" : "sqoop-conf/managers.d/cm_manager_overrides" 73 | } 74 | ], 75 | "generators" : [ 76 | { 77 | "filename" : "sqoop-conf/sqoop-site.xml", 78 | "configFormat" : "hadoop_xml" 79 | } 80 | ] 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /SQOOP_CLIENT/src/images/sqoop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cm_csds/2facc25af147e3791d8f9a5e819b392d4a9147d1/SQOOP_CLIENT/src/images/sqoop.png -------------------------------------------------------------------------------- /SQOOP_CLIENT/src/scripts/sqoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | # Configuration variables that can be exported by connector parcels 21 | # 22 | # CSD_SQOOP_CONNECTOR_PATH 23 | # List of paths to all connectors. Separated by ":". 24 | # 25 | # CSD_SQOOP_EXTRA_CLASSPATH 26 | # Extra CLASSPATH entry that should be available to Sqoop. 27 | # 28 | 29 | # Time marker for both stderr and stdout 30 | date 1>&2 31 | 32 | # Running command 33 | CMD=$1 34 | 35 | # Printout with timestamp 36 | function log { 37 | timestamp=$(date +"%Y-%m-%d %H:%M:%S") 38 | echo "$timestamp: $1" 39 | } 40 | 41 | CLIENT_CONF_DIR=$CONF_DIR/sqoop-conf 42 | MANAGERS_D_DIR=$CLIENT_CONF_DIR/managers.d 43 | 44 | log "Loaded CSD_SQOOP_CONNECTOR_PATH: $CSD_SQOOP_CONNECTOR_PATH" 45 | 46 | case $CMD in 47 | 48 | (client) 49 | log "Creating managers.d directory at $MANAGERS_D_DIR" 50 | if [ ! -d $MANAGERS_D_DIR ]; then 51 | mkdir $MANAGERS_D_DIR 52 | fi 53 | 54 | # Extra generated classpath 55 | EXTRA_CLASSPATH="" 56 | 57 | HOMES=(${CSD_SQOOP_CONNECTOR_PATH//:/ }) 58 | for HOME_DIR in ${HOMES[@]}; do 59 | log "Found connector in $HOME_DIR" 60 | 61 | # Configuration file managers.d 62 | for FILE_PATH in $HOME_DIR/managers.d/*; do 63 | if [ -f $FILE_PATH ]; then 64 | FILENAME=$(basename "$FILE_PATH") 65 | cp $FILE_PATH $MANAGERS_D_DIR/. 66 | perl -pi -e "s#{{ROOT}}#$HOME_DIR#g" $MANAGERS_D_DIR/$FILENAME 67 | fi 68 | done 69 | 70 | # Extra libraries 71 | if [ -d $HOME_DIR/lib/ ]; then 72 | for file in `ls $HOME_DIR/lib/*`; do 73 | log "Found library: $file" 74 | EXTRA_CLASSPATH=$EXTRA_CLASSPATH:$file 75 | done 76 | fi 77 | done 78 | 79 | # The parcels can also export CSD_SQOOP_EXTRA_CLASSPATH to put arbitrary items 80 | # to final classpath. 81 | EXTRA_CLASSPATH="$EXTRA_CLASSPATH:$CSD_SQOOP_EXTRA_CLASSPATH" 82 | 83 | # Append our generated CLASSPATH at the end to ensure that it's there 84 | #echo -e "\n" >> $CLIENT_CONF_DIR/sqoop-env.sh 85 | echo -e "\nexport HADOOP_CLASSPATH=\$HADOOP_CLASSPATH:$EXTRA_CLASSPATH" >> $CLIENT_CONF_DIR/sqoop-env.sh 86 | 87 | log "Processing has finished successfully" 88 | exit 0 89 | ;; 90 | 91 | (*) 92 | log "Don't understand [$CMD]" 93 | ;; 94 | 95 | esac 96 | 97 | -------------------------------------------------------------------------------- /ZEPPELIN/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | assemble 4 | 5 | jar 6 | 7 | false 8 | 9 | 10 | ${project.basedir}/src 11 | / 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /ZEPPELIN/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | com.cloudera 8 | csd 9 | 5.14.0 10 | 11 | 12 | com.cloudera.csd 13 | ZEPPELIN 14 | 7.3.1 15 | Zeppelin CSD for CDH 7.3.1+ 16 | pom 17 | 18 | 19 | true 20 | 21 | 22 | 23 | 24 | 25 | org.apache.maven.plugins 26 | maven-assembly-plugin 27 | 28 | false 29 | 30 | assembly.xml 31 | 32 | 33 | 34 | 35 | make-assembly 36 | package 37 | 38 | single 39 | 40 | 41 | 42 | 43 | 44 | com.cloudera.enterprise 45 | schema-validator-maven-plugin 46 | 7.6.0 47 | 48 | 49 | validate-schema 50 | 51 | src 52 | true 53 | 54 | LIVY_FOR_SPARK3 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /ZEPPELIN/src/aux/interpreter.json: -------------------------------------------------------------------------------- 1 | { 2 | "interpreterSettings": { 3 | "angular": { 4 | "id": "angular", 5 | "name": "angular", 6 | "group": "angular", 7 | "properties": {}, 8 | "status": "READY", 9 | "interpreterGroup": [ 10 | { 11 | "name": "angular", 12 | "class": "org.apache.zeppelin.angular.AngularInterpreter", 13 | "defaultInterpreter": false, 14 | "editor": { 15 | "editOnDblClick": true, 16 | "completionSupport": false 17 | } 18 | } 19 | ], 20 | "dependencies": [], 21 | "option": { 22 | "remote": true, 23 | "port": -1, 24 | "isExistingProcess": false, 25 | "setPermission": false, 26 | "owners": [], 27 | "isUserImpersonate": false 28 | } 29 | }, 30 | "livy": { 31 | "id": "livy", 32 | "name": "livy", 33 | "group": "livy", 34 | "properties": { 35 | "livy.spark.executor.instances": { 36 | "name": "livy.spark.executor.instances", 37 | "value": "", 38 | "type": "number" 39 | }, 40 | "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": { 41 | "name": "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout", 42 | "value": "", 43 | "type": "string" 44 | }, 45 | "zeppelin.livy.concurrentSQL": { 46 | "name": "zeppelin.livy.concurrentSQL", 47 | "value": false, 48 | "type": "checkbox" 49 | }, 50 | "zeppelin.livy.url": { 51 | "name": "zeppelin.livy.url", 52 | "value": "http://localhost:8998", 53 | "type": "url" 54 | }, 55 | "zeppelin.livy.pull_status.interval.millis": { 56 | "name": "zeppelin.livy.pull_status.interval.millis", 57 | "value": "1000", 58 | "type": "number" 59 | }, 60 | "livy.spark.executor.memory": { 61 | "name": "livy.spark.executor.memory", 62 | "value": "", 63 | "type": "string" 64 | }, 65 | "zeppelin.livy.restart_dead_session": { 66 | "name": "zeppelin.livy.restart_dead_session", 67 | "value": false, 68 | "type": "checkbox" 69 | }, 70 | "livy.spark.dynamicAllocation.enabled": { 71 | "name": "livy.spark.dynamicAllocation.enabled", 72 | "value": false, 73 | "type": "checkbox" 74 | }, 75 | "zeppelin.livy.maxLogLines": { 76 | "name": "zeppelin.livy.maxLogLines", 77 | "value": "1000", 78 | "type": "number" 79 | }, 80 | "livy.spark.dynamicAllocation.minExecutors": { 81 | "name": "livy.spark.dynamicAllocation.minExecutors", 82 | "value": "", 83 | "type": "number" 84 | }, 85 | "livy.spark.executor.cores": { 86 | "name": "livy.spark.executor.cores", 87 | "value": "", 88 | "type": "number" 89 | }, 90 | "zeppelin.livy.session.create_timeout": { 91 | "name": "zeppelin.livy.session.create_timeout", 92 | "value": "120", 93 | "type": "number" 94 | }, 95 | "zeppelin.livy.spark.sql.maxResult": { 96 | "name": "zeppelin.livy.spark.sql.maxResult", 97 | "value": "1000", 98 | "type": "number" 99 | }, 100 | "livy.spark.driver.cores": { 101 | "name": "livy.spark.driver.cores", 102 | "value": "", 103 | "type": "number" 104 | }, 105 | "livy.spark.jars.packages": { 106 | "name": "livy.spark.jars.packages", 107 | "value": "", 108 | "type": "textarea" 109 | }, 110 | "zeppelin.livy.spark.sql.field.truncate": { 111 | "name": "zeppelin.livy.spark.sql.field.truncate", 112 | "value": true, 113 | "type": "checkbox" 114 | }, 115 | "livy.spark.driver.memory": { 116 | "name": "livy.spark.driver.memory", 117 | "value": "", 118 | "type": "string" 119 | }, 120 | "zeppelin.livy.displayAppInfo": { 121 | "name": "zeppelin.livy.displayAppInfo", 122 | "value": false, 123 | "type": "checkbox" 124 | }, 125 | "zeppelin.livy.principal": { 126 | "name": "zeppelin.livy.principal", 127 | "value": "", 128 | "type": "string" 129 | }, 130 | "zeppelin.livy.keytab": { 131 | "name": "zeppelin.livy.keytab", 132 | "value": "", 133 | "type": "textarea" 134 | }, 135 | "livy.spark.dynamicAllocation.maxExecutors": { 136 | "name": "livy.spark.dynamicAllocation.maxExecutors", 137 | "value": "", 138 | "type": "number" 139 | }, 140 | "livy.spark.dynamicAllocation.initialExecutors": { 141 | "name": "livy.spark.dynamicAllocation.initialExecutors", 142 | "value": "", 143 | "type": "number" 144 | } 145 | }, 146 | "status": "READY", 147 | "interpreterGroup": [ 148 | { 149 | "name": "spark", 150 | "class": "org.apache.zeppelin.livy.LivySparkInterpreter", 151 | "defaultInterpreter": true, 152 | "editor": { 153 | "language": "scala", 154 | "editOnDblClick": false, 155 | "completionKey": "TAB", 156 | "completionSupport": true 157 | } 158 | }, 159 | { 160 | "name": "sql", 161 | "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", 162 | "defaultInterpreter": false, 163 | "editor": { 164 | "language": "sql", 165 | "editOnDblClick": false, 166 | "completionKey": "TAB", 167 | "completionSupport": true 168 | } 169 | }, 170 | { 171 | "name": "pyspark", 172 | "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", 173 | "defaultInterpreter": false, 174 | "editor": { 175 | "language": "python", 176 | "editOnDblClick": false, 177 | "completionKey": "TAB", 178 | "completionSupport": true 179 | } 180 | }, 181 | { 182 | "name": "sparkr", 183 | "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", 184 | "defaultInterpreter": false, 185 | "editor": { 186 | "language": "r", 187 | "editOnDblClick": false, 188 | "completionKey": "TAB", 189 | "completionSupport": true 190 | } 191 | }, 192 | { 193 | "name": "shared", 194 | "class": "org.apache.zeppelin.livy.LivySharedInterpreter", 195 | "defaultInterpreter": false 196 | } 197 | ], 198 | "dependencies": [], 199 | "option": { 200 | "remote": true, 201 | "port": -1, 202 | "perNote": "shared", 203 | "perUser": "scoped", 204 | "isExistingProcess": false, 205 | "setPermission": false, 206 | "owners": [], 207 | "isUserImpersonate": false 208 | } 209 | }, 210 | "md": { 211 | "id": "md", 212 | "name": "md", 213 | "group": "md", 214 | "properties": { 215 | "markdown.parser.type": { 216 | "name": "markdown.parser.type", 217 | "value": "flexmark", 218 | "type": "string" 219 | } 220 | }, 221 | "status": "READY", 222 | "interpreterGroup": [ 223 | { 224 | "name": "md", 225 | "class": "org.apache.zeppelin.markdown.Markdown", 226 | "defaultInterpreter": false, 227 | "editor": { 228 | "language": "markdown", 229 | "editOnDblClick": true, 230 | "completionSupport": false 231 | } 232 | } 233 | ], 234 | "dependencies": [], 235 | "option": { 236 | "remote": true, 237 | "port": -1, 238 | "isExistingProcess": false, 239 | "setPermission": false, 240 | "owners": [], 241 | "isUserImpersonate": false 242 | } 243 | } 244 | }, 245 | "interpreterBindings": {}, 246 | "interpreterRepositories": [ 247 | { 248 | "id": "central", 249 | "type": "default", 250 | "url": "http://repo1.maven.org/maven2/", 251 | "releasePolicy": { 252 | "enabled": true, 253 | "updatePolicy": "daily", 254 | "checksumPolicy": "warn" 255 | }, 256 | "snapshotPolicy": { 257 | "enabled": true, 258 | "updatePolicy": "daily", 259 | "checksumPolicy": "warn" 260 | }, 261 | "mirroredRepositories": [], 262 | "repositoryManager": false 263 | }, 264 | { 265 | "id": "local", 266 | "type": "default", 267 | "url": "file:///var/lib/zeppelin/.m2/repository", 268 | "releasePolicy": { 269 | "enabled": true, 270 | "updatePolicy": "daily", 271 | "checksumPolicy": "warn" 272 | }, 273 | "snapshotPolicy": { 274 | "enabled": true, 275 | "updatePolicy": "daily", 276 | "checksumPolicy": "warn" 277 | }, 278 | "mirroredRepositories": [], 279 | "repositoryManager": false 280 | } 281 | ] 282 | } 283 | -------------------------------------------------------------------------------- /ZEPPELIN/src/aux/zeppelin-env.sh: -------------------------------------------------------------------------------- 1 | 2 | # export JAVA_HOME= 3 | # export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode. 4 | export MASTER=yarn-client 5 | 6 | # export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" 7 | # export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m 8 | # export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m 9 | # export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. 10 | # export ZEPPELIN_SSL_PORT # ssl port (used when ssl environment variable is set to true) 11 | 12 | # export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default. 13 | # export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default. 14 | 15 | #Provided by default through service.sdl. This is to escape the setting of log directory to interpreters in interpreters.sh 16 | export ZEPPELIN_LOG_DIR="/" 17 | export ZEPPELIN_PID_DIR="/var/run/zeppelin" 18 | 19 | # export ZEPPELIN_WAR_TEMPDIR # The location of jetty temporary directory. 20 | # export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved 21 | # export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z 22 | # export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false" 23 | # export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved 24 | # export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket 25 | # export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json 26 | # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. 27 | # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. 28 | # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading 29 | # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). 30 | # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? 31 | # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise 32 | export ZEPPELIN_INTP_CLASSPATH_OVERRIDES="/var/lib/zeppelin/conf/external-dependency-conf" 33 | #### Spark interpreter configuration #### 34 | 35 | ## Kerberos ticket refresh setting 36 | ## 37 | export KINIT_FAIL_THRESHOLD=5 38 | export KERBEROS_REFRESH_INTERVAL=1d 39 | export SPARK_HOME="${PARCELS_ROOT}/CDH/lib/spark" 40 | 41 | ## Use provided spark installation ## 42 | ## defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit 43 | ## 44 | # export SPARK_HOME # (required) When it is defined, load it instead of Zeppelin embedded Spark libraries 45 | # export SPARK_HOME= 46 | # export SPARK_SUBMIT_OPTIONS # (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G". 47 | # export SPARK_APP_NAME # (optional) The name of spark application. 48 | 49 | ## Use embedded spark binaries ## 50 | ## without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries. 51 | ## however, it is not encouraged when you can define SPARK_HOME 52 | ## 53 | # Options read in YARN client mode 54 | # export HADOOP_CONF_DIR # yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. 55 | export HADOOP_CONF_DIR={{HADOOP_CONF_DIR}} 56 | # Pyspark (supported with Spark 1.2.1 and above) 57 | # To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI 58 | # export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers. 59 | # export PYTHONPATH 60 | 61 | ## Spark interpreter options ## 62 | ## 63 | # export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default. 64 | # export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default. 65 | # export ZEPPELIN_SPARK_IMPORTIMPLICIT # Import implicits, UDF collection, and sql if set true. true by default. 66 | # export ZEPPELIN_SPARK_MAXRESULT # Max number of Spark SQL result to display. 1000 by default. 67 | # export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000 68 | 69 | 70 | #### HBase interpreter configuration #### 71 | 72 | ## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set 73 | 74 | # export HBASE_HOME= # (require) Under which HBase scripts and configuration should be 75 | # export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml 76 | 77 | # export ZEPPELIN_IMPERSONATE_CMD # Optional, when user want to run interpreter as end web user. eg) 'sudo -H -u ${ZEPPELIN_IMPERSONATE_USER} bash -c ' 78 | -------------------------------------------------------------------------------- /ZEPPELIN/src/images/icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ZEPPELIN/src/scripts/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # Licensed to Cloudera, Inc. under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. Cloudera, Inc. licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | . $(cd $(dirname $0) && pwd)/common.sh 21 | . ${COMMON_SCRIPT} 22 | 23 | set -ex 24 | export CDP_ROOT=$(cd $(m_readlink "$CDH_HADOOP_HOME/../..") && pwd) 25 | CDH_ENV_KINIT=${CDH_ENV_KINIT:-/usr/bin/kinit} 26 | 27 | 28 | start_zeppelin_server() { 29 | export ZEPPELIN_INTERPRETER_CONFIG_DIR=${ZEPPELIN_INTERPRETER_CONFIG_DIR:7} 30 | if [ ! -f "${ZEPPELIN_INTERPRETER_CONFIG_DIR}/interpreter.json" ]; then 31 | log "interpreter.json not found. Copying default interpreter.json" 32 | mkdir -p ${ZEPPELIN_INTERPRETER_CONFIG_DIR} 33 | cp aux/interpreter.json ${ZEPPELIN_INTERPRETER_CONFIG_DIR} 34 | fi 35 | 36 | configure_livy_interpreter 37 | configure_livy_interpreter 3 38 | 39 | log "Starting Zeppelin server (CDP $CDH_VERSION)" 40 | 41 | export USER=$(whoami) 42 | export ZEPPELIN_IDENT_STRING=$(whoami) 43 | export SPARK_CONF_DIR= 44 | export HIVE_CONF_DIR= 45 | export ZEPPELIN_HOME="$CDP_ROOT/lib/zeppelin" 46 | export ZEPPELIN_CONF_DIR="$CONF_DIR/zeppelin-conf" 47 | 48 | export HADOOP_CONF_DIR="$CONF_DIR/hadoop-conf" 49 | perl -pi -e "s#\{\{HADOOP_CONF_DIR}}#${HADOOP_CONF_DIR}#g" ${ZEPPELIN_CONF_DIR}/zeppelin-env.sh 50 | 51 | 52 | # required to start zeppelin 53 | cd ${ZEPPELIN_HOME} 54 | . "bin/common.sh" 55 | . "bin/functions.sh" 56 | 57 | HOSTNAME=$(hostname) 58 | ZEPPELIN_NAME="Zeppelin" 59 | ZEPPELIN_MAIN=org.apache.zeppelin.server.ZeppelinServer 60 | JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" 61 | 62 | # Uncomment below to override additional jvm arguments provided by cloudera-config.sh: 63 | ## JAVA8_ADDITIONAL_JVM_ARGS="my jvm argument for JAVA8" 64 | JAVA11_ADDITIONAL_JVM_ARGS="--add-modules jdk.unsupported \ 65 | --add-opens java.base/java.nio=ALL-UNNAMED \ 66 | --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ 67 | --add-opens java.base/java.lang=ALL-UNNAMED \ 68 | --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ 69 | --add-opens java.base/java.lang.reflect=ALL-UNNAMED \ 70 | --add-opens java.base/java.util=ALL-UNNAMED \ 71 | --add-opens java.base/java.util.concurrent=ALL-UNNAMED \ 72 | --add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ 73 | --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED \ 74 | --add-exports java.base/sun.net.dns=ALL-UNNAMED \ 75 | --add-exports java.base/sun.net.util=ALL-UNNAMED" 76 | JAVA17_ADDITIONAL_JVM_ARGS="${JAVA11_ADDITIONAL_JVM_ARGS} \ 77 | --add-opens java.base/jdk.internal.util.random=ALL-UNNAMED" 78 | 79 | # function from cloudera-config.sh provides ADDITIONAL_JVM_ARGS based on java version 80 | set_additional_jvm_args_based_on_java_version 81 | 82 | get_generic_java_opts 83 | JAVA_OPTS="${JAVA_OPTS} ${GENERIC_JAVA_OPTS} ${ADDITIONAL_JVM_ARGS}" 84 | 85 | # construct classpath 86 | if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then 87 | ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" 88 | fi 89 | 90 | addJarInDir "${ZEPPELIN_HOME}" 91 | addJarInDir "${ZEPPELIN_HOME}/lib" 92 | addJarInDir "${ZEPPELIN_HOME}/lib/interpreter" 93 | addJarInDir "${ZEPPELIN_HOME}/interpreter" 94 | 95 | CLASSPATH+=":${ZEPPELIN_CLASSPATH}" 96 | 97 | SHIRO_CONTENT="" 98 | IS_KNOX_CONFIGURED="false" 99 | 100 | # build shiro.ini start 101 | if [[ -n "$ZEPPELIN_PRINCIPAL" ]]; then 102 | 103 | KEYTAB_FILE="${CONF_DIR}/zeppelin.keytab" 104 | perl -pi -e "s#\{\{KEYTAB_FILE}}#${KEYTAB_FILE}#g" ${ZEPPELIN_CONF_DIR}/zeppelin-site.xml 105 | 106 | if [[ -n "$SPNEGO_PRINCIPAL" ]]; then 107 | if [[ -n "$KNOX_SERVICE" && "$KNOX_SERVICE" != "none" ]];then 108 | 109 | dd if=/dev/urandom of=${CONF_DIR}/http_secret bs=1024 count=1 110 | chmod 444 ${CONF_DIR}/http_secret 111 | 112 | export DOMAIN=".$(echo ${SPNEGO_PRINCIPAL} | cut -d'/' -f2 | cut -d'@' -f1 | cut -d'.' -f2-)" 113 | IS_KNOX_CONFIGURED="true" 114 | 115 | shiro_knox_main_block=$(echo "${shiro_knox_main_block}" | sed "s#{{KEYTAB_FILE}}#${KEYTAB_FILE}#g") 116 | shiro_knox_main_block=$(echo "${shiro_knox_main_block}" | sed "s#{{SPNEGO_PRINCIPAL}}#${SPNEGO_PRINCIPAL}#g") 117 | shiro_knox_main_block=$(echo "${shiro_knox_main_block}" | sed "s#{{DOMAIN}}#${DOMAIN}#g") 118 | shiro_knox_main_block=$(echo "${shiro_knox_main_block}" | sed "s#{{CONF_DIR}}#${CONF_DIR}#g") 119 | 120 | SHIRO_CONTENT+="[main]\n" 121 | SHIRO_CONTENT+="${shiro_knox_main_block}\n" 122 | 123 | fi 124 | fi 125 | fi 126 | 127 | if [[ "$IS_KNOX_CONFIGURED" == "false" ]]; then 128 | SHIRO_CONTENT="[users]\n" 129 | SHIRO_CONTENT+="${shiro_user_block}\n" 130 | SHIRO_CONTENT+="[main]\n" 131 | SHIRO_CONTENT+="${shiro_main_block}\n" 132 | fi 133 | 134 | SHIRO_CONTENT+="${shiro_main_session_block}\n" 135 | SHIRO_CONTENT+="[roles]\n" 136 | SHIRO_CONTENT+="${shiro_roles_block}\n" 137 | 138 | shiro_urls_block=$(echo "${shiro_urls_block}" | sed "s#{{zeppelin_admin_group}}#${zeppelin_admin_group}#g") 139 | SHIRO_CONTENT+="[urls]\n" 140 | SHIRO_CONTENT+="${shiro_urls_block}\n" 141 | 142 | echo -e "$SHIRO_CONTENT" > "$ZEPPELIN_CONF_DIR/shiro.ini" 143 | # build shiro.ini end 144 | 145 | exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_MAIN 146 | } 147 | 148 | configure_livy_interpreter() { 149 | export LIVY_URI= 150 | export LIVY3_URI= 151 | local SERVER_LIST="$CONF_DIR/livy$1-conf/livy-server.properties" 152 | local SERVER_HOST= 153 | local SERVER_PORT= 154 | local SCHEME="http" 155 | for line in $(cat "$SERVER_LIST") 156 | do 157 | readconf "$line" 158 | case $key in 159 | (livy.server.port) 160 | SERVER_HOST="$host" 161 | SERVER_PORT="$value" 162 | ;; 163 | (livy.tls.enabled) 164 | if [ "$value" = "true" ]; then 165 | SCHEME="https" 166 | fi 167 | ;; 168 | esac 169 | done 170 | if [ -n "$SERVER_HOST" ]; then 171 | if [ -n "$1" ]; then 172 | LIVY3_URI="$SCHEME://$SERVER_HOST:$SERVER_PORT" 173 | else 174 | LIVY_URI="$SCHEME://$SERVER_HOST:$SERVER_PORT" 175 | fi 176 | fi 177 | 178 | if [ "$LIVY_URI" ]; then 179 | log "Found Livy URI $LIVY_URI. Configuring interpreter.json" 180 | PYTHON_COMMAND_INVOKER=${PYTHON_COMMAND_INVOKER:-python} 181 | $(${PYTHON_COMMAND_INVOKER} ${CONF_DIR}/scripts/update_interpreter.py) 182 | fi 183 | 184 | if [ "$LIVY3_URI" ]; then 185 | log "Found Livy3 URI $LIVY3_URI. Configuring interpreter.json" 186 | PYTHON_COMMAND_INVOKER=${PYTHON_COMMAND_INVOKER:-python} 187 | $(${PYTHON_COMMAND_INVOKER} ${CONF_DIR}/scripts/update_interpreter.py) 188 | fi 189 | } 190 | 191 | gen_client_conf() { 192 | log "Configuring Zeppelin server (CDP $CDH_VERSION)" 193 | 194 | if [[ "${zeppelin_notebook_storage}" == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo' ]] 195 | then 196 | if [[ -n "$ZEPPELIN_PRINCIPAL" ]]; then 197 | $CDH_ENV_KINIT -kt "${CONF_DIR}/zeppelin.keytab" "${ZEPPELIN_PRINCIPAL}" 198 | fi 199 | 200 | log "Copying default notebook shipped with Zeppelin to HDFS/S3 file system" 201 | "$HDFS_BIN" dfs -mkdir -p "${zeppelin_notebook_dir}" 202 | "$HDFS_BIN" dfs -put $CDP_ROOT/lib/zeppelin/notebook/* "${zeppelin_notebook_dir}" && echo "All notebooks copied." || echo "Notebook(s) already exists, did not attempt to overwrite." 203 | 204 | else 205 | log "Copying default notebook shipped with Zeppelin to local file system" 206 | cp -r $CDP_ROOT/lib/zeppelin/notebook ${zeppelin_notebook_dir} 207 | fi 208 | } 209 | 210 | case $1 in 211 | (start_zeppelin_server) 212 | start_zeppelin_server 213 | ;; 214 | 215 | (gen_client_conf) 216 | gen_client_conf 217 | ;; 218 | 219 | (*) 220 | log "Unknown command [$1]" 221 | exit 1 222 | ;; 223 | esac 224 | -------------------------------------------------------------------------------- /ZEPPELIN/src/scripts/update_interpreter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import json 4 | from xml.etree import ElementTree as ET 5 | 6 | def update_interpreter_properties(interpreter, name, value, type): 7 | interpreter['properties'][name] = {'name': name, 'value': value, 'type': type} 8 | 9 | def update_zeppelin_interpreter(): 10 | livy_url = "http://localhost:8998" #default livy host:port 11 | livy_url = os.environ.get("LIVY_URI") 12 | livy3_url = os.environ.get("LIVY3_URI") 13 | zeppelin_principal = os.environ.get("ZEPPELIN_PRINCIPAL") 14 | keytab_file = os.path.join(os.environ.get("CONF_DIR"), "zeppelin.keytab") 15 | zeppelin_site_dict = get_zeppelin_site_xml_dict() 16 | 17 | interpreter_json_file = os.path.join(os.environ.get("ZEPPELIN_INTERPRETER_CONFIG_DIR"), "interpreter.json") 18 | config_data = read_interpreter_config(interpreter_json_file) 19 | interpreter_settings = config_data["interpreterSettings"] 20 | 21 | if livy_url: 22 | livy2_found = False 23 | #Check whether livy2 is present in existing settings of interpreter.json 24 | for setting_key in interpreter_settings.keys(): 25 | interpreter = interpreter_settings[setting_key] 26 | if interpreter['name'] == 'livy': 27 | livy2_found = True 28 | update_interpreter_properties(interpreter, "zeppelin.livy.url", livy_url, "url") 29 | break 30 | #The livy2 configuration is not found in the existing settings. We create new interpreter settings for livy2 31 | if livy2_found == False: 32 | interpreter_json_file_temp = os.path.join("aux", "interpreter.json") 33 | config_data_temp = read_interpreter_config(interpreter_json_file_temp) 34 | interpreter_settings_temp = config_data_temp["interpreterSettings"] 35 | config_data["interpreterSettings"]["livy"] = interpreter_settings_temp["livy"] 36 | config_data["interpreterSettings"]["livy"]["id"] = "livy" 37 | config_data["interpreterSettings"]["livy"]["name"] = "livy" 38 | 39 | interpreter = config_data["interpreterSettings"]["livy"] 40 | update_interpreter_properties(interpreter, "zeppelin.livy.url", livy_url, "url") 41 | 42 | if livy3_url: 43 | livy3_found = False 44 | for setting_key in interpreter_settings.keys(): 45 | interpreter = interpreter_settings[setting_key] 46 | if interpreter['name'] == 'livy3': 47 | livy3_found = True 48 | update_interpreter_properties(interpreter, "zeppelin.livy.url", livy3_url, "url") 49 | break 50 | 51 | if livy3_found == False: 52 | interpreter_json_file_temp = os.path.join("aux", "interpreter.json") 53 | config_data_temp = read_interpreter_config(interpreter_json_file_temp) 54 | interpreter_settings_temp = config_data_temp["interpreterSettings"] 55 | config_data["interpreterSettings"]["livy3"] = interpreter_settings_temp["livy"] 56 | config_data["interpreterSettings"]["livy3"]["id"] = "livy3" 57 | config_data["interpreterSettings"]["livy3"]["name"] = "livy3" 58 | 59 | interpreter = config_data["interpreterSettings"]["livy3"] 60 | update_interpreter_properties(interpreter, "zeppelin.livy.url", livy3_url, "url") 61 | 62 | livy_conf_path = os.path.join(os.environ.get("CONF_DIR"), "livy-conf") 63 | livy3_conf_path = os.path.join(os.environ.get("CONF_DIR"), "livy3-conf") 64 | if not os.path.exists(livy_conf_path): 65 | config_data["interpreterSettings"].pop('livy', None) 66 | if not os.path.exists(livy3_conf_path): 67 | config_data["interpreterSettings"].pop('livy3', None) 68 | 69 | update_kerberos_properties(interpreter_settings, zeppelin_principal, keytab_file, zeppelin_site_dict) 70 | write_interpreter_config(interpreter_json_file, config_data) 71 | 72 | def update_kerberos_properties(interpreter_settings, zeppelin_principal, keytab_file, zeppelin_site_dict): 73 | for setting_key in interpreter_settings.keys(): 74 | interpreter = interpreter_settings[setting_key] 75 | if interpreter['group'] == 'livy': 76 | if zeppelin_principal and keytab_file: 77 | interpreter['properties']['zeppelin.livy.principal']['value'] = zeppelin_principal 78 | interpreter['properties']['zeppelin.livy.keytab']['value'] = keytab_file 79 | else: 80 | interpreter['properties']['zeppelin.livy.principal']['value'] = "" 81 | interpreter['properties']['zeppelin.livy.keytab']['value'] = "" 82 | 83 | if zeppelin_site_dict['zeppelin.ssl.truststore.password']: 84 | update_interpreter_properties(interpreter, "zeppelin.livy.ssl.trustStorePassword", 85 | zeppelin_site_dict['zeppelin.ssl.truststore.password'], "password") 86 | if zeppelin_site_dict['zeppelin.ssl.truststore.path']: 87 | update_interpreter_properties(interpreter, "zeppelin.livy.ssl.trustStore", 88 | zeppelin_site_dict['zeppelin.ssl.truststore.path'], "textarea") 89 | if zeppelin_site_dict['zeppelin.ssl.truststore.type']: 90 | update_interpreter_properties(interpreter, "zeppelin.livy.ssl.trustStoreType", 91 | zeppelin_site_dict['zeppelin.ssl.truststore.type'], "string") 92 | elif interpreter['group'] == 'spark': 93 | if zeppelin_principal and keytab_file: 94 | update_interpreter_properties(interpreter, "spark.yarn.principal", zeppelin_principal, "textarea") 95 | update_interpreter_properties(interpreter, "spark.yarn.keytab", keytab_file, "textarea") 96 | else: 97 | update_interpreter_properties(interpreter, "spark.yarn.principal", "", "textarea") 98 | update_interpreter_properties(interpreter, "spark.yarn.keytab", "", "textarea") 99 | elif interpreter['group'] == 'jdbc': 100 | if zeppelin_principal and keytab_file: 101 | update_interpreter_properties(interpreter, "zeppelin.jdbc.auth.type", "KERBEROS", "textarea") 102 | update_interpreter_properties(interpreter, "zeppelin.jdbc.principal", zeppelin_principal, "textarea") 103 | update_interpreter_properties(interpreter, "zeppelin.jdbc.keytab.location", keytab_file, "textarea") 104 | else: 105 | update_interpreter_properties(interpreter, "zeppelin.jdbc.auth.type", "SIMPLE", "textarea") 106 | update_interpreter_properties(interpreter, "zeppelin.jdbc.principal", "", "textarea") 107 | update_interpreter_properties(interpreter, "zeppelin.jdbc.keytab.location", "", "textarea") 108 | elif interpreter['group'] == 'sh': 109 | if zeppelin_principal and keytab_file: 110 | update_interpreter_properties(interpreter, "zeppelin.shell.auth.type", "KERBEROS", "string") 111 | update_interpreter_properties(interpreter, "zeppelin.shell.principal", zeppelin_principal, "textarea") 112 | update_interpreter_properties(interpreter, "zeppelin.shell.keytab.location", keytab_file, "textarea") 113 | else: 114 | update_interpreter_properties(interpreter, "zeppelin.shell.auth.type", "", "string") 115 | update_interpreter_properties(interpreter, "zeppelin.shell.principal", "", "textarea") 116 | update_interpreter_properties(interpreter, "zeppelin.shell.keytab.location", "", "textarea") 117 | 118 | def write_interpreter_config(file, config_data): 119 | try: 120 | with open(file, 'w') as outfile: 121 | json.dump(config_data, outfile, indent=2) 122 | except: 123 | print("failed to write " + file) 124 | 125 | def read_interpreter_config(file): 126 | try: 127 | with open(file) as f: 128 | config_data = json.load(f) 129 | return config_data 130 | except: 131 | print("unable to read or the file is corrupted " + file) 132 | 133 | def get_zeppelin_site_xml_dict(): 134 | zeppelin_site_xml = os.path.join(os.environ.get("CONF_DIR"), "zeppelin-conf", "zeppelin-site.xml") 135 | xml = ET.parse(zeppelin_site_xml) 136 | root = xml.getroot() 137 | dic = {} 138 | for childnode in root.iter('property'): 139 | if childnode.find('value').text is not None: 140 | dic[childnode.find('name').text] = childnode.find('value').text 141 | else: 142 | dic[childnode.find('name').text] = None 143 | return dic 144 | 145 | 146 | if __name__ == '__main__': 147 | update_zeppelin_interpreter() 148 | -------------------------------------------------------------------------------- /assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | assemble 4 | 5 | jar 6 | 7 | false 8 | 9 | 10 | ${project.basedir}/src 11 | / 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | com.cloudera 6 | csd 7 | 5.14.0 8 | The Enterprise CSDs 9 | pom 10 | 11 | 12 | ACCUMULO 13 | ACCUMULO16 14 | ECHO 15 | KAFKA 16 | KEYTRUSTEE 17 | KMS 18 | SPARK 19 | SPARK_ON_YARN 20 | SPARK_ON_YARN53 21 | SPARK_ON_YARN54 22 | SPARK_ON_YARN5_10 23 | SQOOP_CLIENT 24 | ZEPPELIN 25 | 26 | 27 | 28 | 29 | 30 | org.apache.maven.plugins 31 | maven-assembly-plugin 32 | 33 | false 34 | 35 | assembly.xml 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 45 | com.cloudera.enterprise 46 | schema-validator-maven-plugin 47 | 5.14.0 48 | 49 | 50 | validate-schema 51 | test 52 | 53 | validate 54 | 55 | 56 | src 57 | true 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | cdh.releases.repo 69 | http://maven.jenkins.cloudera.com:8081/artifactory/libs-release-local 70 | Releases Repository 71 | 72 | 73 | cdh.snapshots.repo 74 | http://maven.jenkins.cloudera.com:8081/artifactory/libs-snapshot-local 75 | Snapshots Repository 76 | 77 | 78 | 79 | 80 | 81 | cloudera-snapshot-internal 82 | http://maven.jenkins.cloudera.com:8081/artifactory/libs-snapshot-local 83 | 84 | false 85 | 86 | 87 | 88 | cloudera-release-internal 89 | http://maven.jenkins.cloudera.com:8081/artifactory/libs-release-local 90 | 91 | false 92 | 93 | 94 | 95 | cloudera-staging 96 | https://repository.cloudera.com/artifactory/libs-staging-local 97 | 98 | false 99 | 100 | 101 | 102 | 103 | --------------------------------------------------------------------------------