├── .gitignore ├── MANIFEST ├── README.md ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src └── main ├── mpack ├── common-services │ └── SOLR │ │ ├── 5.5.5 │ │ ├── alerts.json │ │ ├── configuration │ │ │ ├── example-collection.xml │ │ │ ├── solr-cloud.xml │ │ │ ├── solr-config-env.xml │ │ │ ├── solr-hdfs.xml │ │ │ ├── solr-log4j.xml │ │ │ ├── solr-metrics.xml │ │ │ ├── solr-security.xml │ │ │ └── solr-ssl.xml │ │ ├── kerberos.json │ │ ├── metainfo.xml │ │ ├── metrics.json │ │ ├── package │ │ │ ├── alerts │ │ │ │ ├── alert_solr_cpu_metrics.py │ │ │ │ └── alert_solr_memory_metrics.py │ │ │ ├── scripts │ │ │ │ ├── params.py │ │ │ │ ├── service_check.py │ │ │ │ ├── setup_solr.py │ │ │ │ ├── setup_solr_cloud.py │ │ │ │ ├── setup_solr_hdfs_support.py │ │ │ │ ├── setup_solr_kerberos_auth.py │ │ │ │ ├── setup_solr_metrics.py │ │ │ │ ├── setup_solr_ssl_support.py │ │ │ │ ├── solr.py │ │ │ │ ├── solr_utils.py │ │ │ │ └── status_params.py │ │ │ └── templates │ │ │ │ ├── log4j2.xml │ │ │ │ ├── solr.xml.j2 │ │ │ │ ├── solr_metrics_jaas.conf.j2 │ │ │ │ └── solr_server_jaas.conf.j2 │ │ ├── quicklinks │ │ │ └── quicklinks.json │ │ ├── role_command_order.json │ │ └── service_advisor.py │ │ ├── 6.6.2 │ │ ├── alerts.json │ │ ├── configuration │ │ │ ├── solr-config-env.xml │ │ │ ├── solr-metrics.xml │ │ │ └── solr-ssl.xml │ │ ├── dashboards │ │ │ └── grafana-dashboards │ │ │ │ ├── grafana-solr-662-cores.json │ │ │ │ └── grafana-solr-662-hosts.json │ │ ├── metainfo.xml │ │ ├── metrics.json │ │ ├── package │ │ │ ├── alerts │ │ │ │ ├── alert_solr_cpu_metrics.py │ │ │ │ └── alert_solr_memory_metrics.py │ │ │ ├── scripts │ │ │ │ ├── params.py │ │ │ │ ├── service_check.py │ │ │ │ ├── setup_solr.py │ │ │ │ ├── setup_solr_cloud.py │ │ │ │ ├── setup_solr_hdfs_support.py │ │ │ │ ├── setup_solr_kerberos_auth.py │ │ │ │ ├── setup_solr_metrics.py │ │ │ │ ├── setup_solr_ssl_support.py │ │ │ │ ├── solr.py │ │ │ │ ├── solr_utils.py │ │ │ │ └── status_params.py │ │ │ └── templates │ │ │ │ ├── log4j2.xml │ │ │ │ ├── solr_metrics_jaas.conf.j2 │ │ │ │ └── solr_server_jaas.conf.j2 │ │ └── properties │ │ │ └── solr.xml.j2 │ │ └── 7.4.0 │ │ ├── alerts.json │ │ ├── configuration │ │ ├── example-collection.xml │ │ ├── solr-config-env.xml │ │ ├── solr-log4j.xml │ │ ├── solr-metrics.xml │ │ └── solr-ssl.xml │ │ ├── dashboards │ │ └── grafana-dashboards │ │ │ ├── grafana-solr-740-cores.json │ │ │ └── grafana-solr-740-hosts.json │ │ ├── metainfo.xml │ │ ├── package │ │ ├── alerts │ │ │ ├── alert_solr_cpu_metrics.py │ │ │ └── alert_solr_memory_metrics.py │ │ ├── scripts │ │ │ ├── params.py │ │ │ ├── service_check.py │ │ │ ├── setup_solr.py │ │ │ ├── setup_solr_cloud.py │ │ │ ├── setup_solr_hdfs_support.py │ │ │ ├── setup_solr_kerberos_auth.py │ │ │ ├── setup_solr_metrics.py │ │ │ ├── setup_solr_ssl_support.py │ │ │ ├── solr.py │ │ │ ├── solr_utils.py │ │ │ └── status_params.py │ │ └── templates │ │ │ ├── log4j2.xml │ │ │ ├── solr_metrics_jaas.conf.j2 │ │ │ └── solr_server_jaas.conf.j2 │ │ └── properties │ │ └── solr.xml.j2 ├── custom-services │ └── SOLR │ │ ├── 5.5.5 │ │ └── metainfo.xml │ │ ├── 6.6.2 │ │ └── metainfo.xml │ │ └── 7.4.0 │ │ └── metainfo.xml ├── extensions │ └── SOLR │ │ ├── 5.5.5 │ │ └── metainfo.xml │ │ ├── 6.6.2 │ │ └── metainfo.xml │ │ └── 7.3.1 │ │ └── metainfo.xml └── mpack.json └── template └── repos └── repoinfo.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .class 3 | .swp 4 | .idea 5 | *.iml 6 | *.ipr 7 | *.iws 8 | .gradle/ 9 | build/ 10 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | solr-stack | 4.0.0 | git@github.com:lucidworks/solr-stack.git | master | 29fd1e262d819763991ca2b4d8f4fca9e3100ca1 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Solr service for Ambari 2 | 3 | Stop Ambari server: 4 | 5 | ambari-server stop 6 | 7 | Download solr-stack repository: 8 | 9 | git clone https://github.com/lucidworks/solr-stack 10 | 11 | Create Solr service pack: 12 | 13 | ./gradlew clean makePackage 14 | 15 | Deploy the Solr service on Ambari server: 16 | 17 | ambari-server install-mpack --mpack=build/solr-service-mpack-${version}.tar.gz -v 18 | 19 | Start Ambari server: 20 | 21 | ambari-server start -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | mavenCentral() 4 | } 5 | } 6 | 7 | apply plugin: 'idea' 8 | apply plugin: 'distribution' 9 | 10 | task wrapper(type: Wrapper) { 11 | gradleVersion = '3.4.1' 12 | } 13 | 14 | ext.src = 'src/main/mpack/' 15 | ext.mpack = 'mpack.json' 16 | ext.repostemplate = 'src/main/template/repos/repoinfo.xml' 17 | ext.reposinfo555 = 'custom-services/SOLR/5.5.5/repos/' 18 | ext.reposinfo662 = 'custom-services/SOLR/6.6.2/repos/' 19 | ext.reposinfo740 = 'custom-services/SOLR/7.4.0/repos/' 20 | 21 | distributions { 22 | main { 23 | contents { 24 | baseName = packageName 25 | from { src } 26 | 27 | from { src + mpack } 28 | filter { line -> 29 | line.replace("{VERSION}", "${version}") 30 | } 31 | 32 | into(reposinfo555) { 33 | from { repostemplate } 34 | filter { line -> 35 | line.replace("{REPOID}", "${repoid555}") 36 | } 37 | } 38 | 39 | into(reposinfo662) { 40 | from { repostemplate } 41 | filter { line -> 42 | line.replace("{REPOID}", "${repoid662}") 43 | } 44 | } 45 | into(reposinfo740) { 46 | from { repostemplate } 47 | filter { line -> 48 | line.replace("{REPOID}", "${repoid740}") 49 | } 50 | } 51 | } 52 | } 53 | } 54 | 55 | distTar { 56 | destinationDir = file(buildDir) 57 | appendix = 'mpack' 58 | compression = Compression.GZIP 59 | extension = 'tar.gz' 60 | } 61 | 62 | 63 | task makePackage(dependsOn: [clean, distTar]) { 64 | } 65 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=4.0.0 2 | packageName=solr-service 3 | # repos ids 4 | # TODO a Solr repo is needed 5 | repoid555=HDP-SOLR-2.6-100 6 | 7 | # Solr 6.6.2 repo 8 | # TODO a Solr repo is needed 9 | repoid662=HDP-SOLR-3.0.0-100 10 | 11 | # Solr 7.4.0 repo 12 | # TODO a Solr repo is needed 13 | repoid740=HDP-SOLR-3.0.0-100 14 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lucidworks/solr-stack/6768524fc029aa36ada937d98d9bbd227c691817/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Mar 13 10:11:22 PET 2017 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-3.4.1-all.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn ( ) { 37 | echo "$*" 38 | } 39 | 40 | die ( ) { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save ( ) { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/alerts.json: -------------------------------------------------------------------------------- 1 | { 2 | "SOLR": { 3 | "SOLR_SERVER": [ 4 | { 5 | "name": "solr", 6 | "label": "Solr Web UI", 7 | "description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.", 8 | "interval": 1, 9 | "scope": "ANY", 10 | "source": { 11 | "type": "WEB", 12 | "uri": { 13 | "http": "{{solr-config-env/solr_config_port}}", 14 | "https": "{{solr-config-env/solr_config_port}}", 15 | "https_property": "{{solr-ssl/solr_ssl_enable}}", 16 | "https_property_value": "true", 17 | "kerberos_keytab": "{{solr-config-env/solr_spnego_keytab_path}}", 18 | "kerberos_principal": "{{solr-config-env/solr_spnego_principal_name}}", 19 | "connection_timeout": 5.0, 20 | "default_port": 8983 21 | }, 22 | "reporting": { 23 | "ok": { 24 | "text": "HTTP {0} response in {2:.3f}s" 25 | }, 26 | "warning": { 27 | "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" 28 | }, 29 | "critical": { 30 | "text": "Connection failed to {1} ({3})" 31 | } 32 | } 33 | } 34 | }, 35 | { 36 | "name": "solr_cpu_usage", 37 | "label": "Solr CPU Utilization", 38 | "description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", 39 | "interval": 5, 40 | "scope": "ANY", 41 | "source": { 42 | "type": "SCRIPT", 43 | "path": "SOLR/5.5.5/package/alerts/alert_solr_cpu_metrics.py", 44 | "parameters": [ 45 | { 46 | "name": "connection.timeout", 47 | "value": 5.0, 48 | "type": "NUMERIC", 49 | "units": "seconds", 50 | "visibility": "HIDDEN" 51 | }, 52 | { 53 | "name": "app.id", 54 | "value": "solr-host-app", 55 | "type": "STRING", 56 | "visibility": "HIDDEN" 57 | }, 58 | { 59 | "name": "metric.names", 60 | "value": "solr.admin.info.system.processCpuLoad", 61 | "type": "STRING", 62 | "visibility": "HIDDEN" 63 | }, 64 | { 65 | "name": "metric.solr.cpu.warning.threshold", 66 | "display_name": "Warning threshold", 67 | "type": "PERCENT", 68 | "units": "%", 69 | "value": 75, 70 | "description": "CPU usage is greater than 50%", 71 | "threshold": "WARNING" 72 | }, 73 | { 74 | "name": "metric.solr.cpu.critical.threshold", 75 | "display_name": "Critical threshold", 76 | "type": "PERCENT", 77 | "units": "%", 78 | "value": 90, 79 | "description": "CPU usage is greater than 75%", 80 | "threshold": "CRITICAL" 81 | } 82 | ] 83 | } 84 | }, 85 | { 86 | "name": "solr_memory", 87 | "label": "Solr Memory Utilization", 88 | "description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", 89 | "interval": 5, 90 | "scope": "ANY", 91 | "source": { 92 | "type": "SCRIPT", 93 | "path": "SOLR/5.5.5/package/alerts/alert_solr_memory_metrics.py", 94 | "parameters": [ 95 | { 96 | "name": "connection.timeout", 97 | "value": 5.0, 98 | "type": "NUMERIC", 99 | "units": "seconds", 100 | "visibility": "HIDDEN" 101 | }, 102 | { 103 | "name": "app.id", 104 | "value": "solr-host-app", 105 | "type": "STRING", 106 | "visibility": "HIDDEN" 107 | }, 108 | { 109 | "name": "metric.names", 110 | "value": "solr.admin.info.jvm.memory.used,solr.admin.info.jvm.memory.max", 111 | "type": "STRING", 112 | "visibility": "HIDDEN" 113 | }, 114 | { 115 | "name": "metric.solr.memory.warning.threshold", 116 | "display_name": "Warning threshold", 117 | "type": "PERCENT", 118 | "units": "%", 119 | "value": 75, 120 | "description": "Memory usage is greater than 50%", 121 | "threshold": "WARNING" 122 | }, 123 | { 124 | "name": "metric.solr.memory.critical.threshold", 125 | "display_name": "Critical threshold", 126 | "type": "PERCENT", 127 | "units": "%", 128 | "value": 90, 129 | "description": "Memory usage is greater than 75%", 130 | "threshold": "CRITICAL" 131 | } 132 | ] 133 | } 134 | } 135 | ] 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/example-collection.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_collection_sample_create 7 | true 8 | True to create a sample collection when Solr is deployed 9 | Create sample collection 10 | 11 | boolean 12 | false 13 | 14 | 15 | 16 | 17 | solr_collection_sample_name 18 | collection1 19 | Solr sample collection name. Mandatory 20 | Sample collection name 21 | 22 | 23 | 24 | solr_collection_sample_config_directory 25 | data_driven_schema_configs 26 | Solr sample collection configurations directory. Mandatory 27 | This directory path is relative to /opt/lucidworks-hadoop/solr/server/solr/configset. 28 | It must contain at least solrconfig.xml and schema.xml 29 | 30 | Solr configuration directory 31 | 32 | 33 | 34 | solr_collection_sample_shards 35 | 2 36 | Number of Solr shards, for details refer to 37 | (https://cwiki.apache.org/confluence/display/solr/Shards+and+Indexing+Data+in+SolrCloud) 38 | 39 | 40 | int 41 | 42 | Number of shards 43 | 44 | 45 | 46 | solr_collection_sample_replicas 47 | 1 48 | Number of Solr replicas, for details refer to 49 | (https://cwiki.apache.org/confluence/display/solr/NRT%2C+Replication%2C+and+Disaster+Recovery+with+SolrCloud) 50 | 51 | 52 | int 53 | 54 | Number of replicas 55 | 56 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/solr-cloud.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_cloud_enable 7 | true 8 | Whether Solr should be started in SolrCloud mode. This option cannot be changed. 9 | Enable SolrCloud mode 10 | 11 | boolean 12 | false 13 | 14 | 15 | 16 | 17 | solr_cloud_zk_directory 18 | /solr 19 | ZooKeeper directory (chroot) for shared Solr files. 20 | This allows Solr's files to be isolated from files for other applications sharing 21 | the same ZooKeeper instance. 22 | ZooKeeper root directory for Solr files 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/solr-hdfs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_hdfs_enable 7 | true 8 | Enables storing Solr's indexes in HDFS. 9 | Store Solr indexes in HDFS 10 | 11 | boolean 12 | false 13 | 14 | 15 | 16 | 17 | solr_hdfs_directory 18 | /solr 19 | The HDFS path where Solr indexes will be stored. 20 | HDFS directory for Solr indexes 21 | 22 | 23 | 24 | solr_hdfs_delete_write_lock_files 25 | false 26 | Solr locks indexes against multiple writes. In some cases, restarting Solr will leave lock 27 | files behind, which prevent successful startup. This option automatically checks for and removes any 28 | lock files found in any collection on any node every time Solr starts. 29 | 30 | Delete write.lock files on HDFS 31 | 32 | boolean 33 | false 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/solr-log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | content 8 | Custom log4j.properties 9 | 10 | # Licensed to the Apache Software Foundation (ASF) under one or more 11 | # contributor license agreements. See the NOTICE file distributed with 12 | # this work for additional information regarding copyright ownership. 13 | # The ASF licenses this file to You under the Apache License, Version 2.0 14 | # (the "License"); you may not use this file except in compliance with 15 | # the License. You may obtain a copy of the License at 16 | # 17 | # http://www.apache.org/licenses/LICENSE-2.0 18 | # 19 | # Unless required by applicable law or agreed to in writing, software 20 | # distributed under the License is distributed on an "AS IS" BASIS, 21 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 22 | # See the License for the specific language governing permissions and 23 | # limitations under the License. 24 | 25 | # Logging level 26 | solr.log={{solr_config_log_dir}} 27 | #log4j.rootLogger=INFO, file, CONSOLE 28 | log4j.rootLogger=INFO, file 29 | 30 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 31 | 32 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] \u2013 %m%n 34 | 35 | #- size rotation with log cleanup. 36 | log4j.appender.file=org.apache.log4j.RollingFileAppender 37 | log4j.appender.file.MaxFileSize=10MB 38 | log4j.appender.file.MaxBackupIndex=9 39 | 40 | #- File to log to and log format 41 | log4j.appender.file.File=${solr.log}/solr.log 42 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 43 | log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n 44 | 45 | log4j.logger.org.apache.zookeeper=WARN 46 | log4j.logger.org.apache.hadoop=WARN 47 | 48 | # set to INFO to enable infostream log messages 49 | log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF 50 | 51 | 52 | content 53 | false 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/solr-metrics.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | solr_enable_metrics 8 | false 9 | Enable Solr Metrics (Requires Ambari Metrics) 10 | Enable Solr Metrics 11 | 12 | boolean 13 | 14 | 15 | 16 | 17 | solr_core_stats 18 | true 19 | Enable Solr Core metrics 20 | Solr Core stats 21 | 22 | boolean 23 | 24 | 25 | 26 | 27 | solr_cache_stats 28 | true 29 | Enable Solr Cache metrics 30 | Solr Cache stats 31 | 32 | boolean 33 | 34 | 35 | 36 | 37 | solr_updatehandler_stats 38 | true 39 | Enable Solr UpdateHandler metrics 40 | Solr UpdateHandler stats 41 | 42 | boolean 43 | 44 | 45 | 46 | 47 | solr_queryhandler_stats 48 | true 49 | Enable Solr QueryHandler metrics 50 | Solr QueryHandler stats 51 | 52 | boolean 53 | 54 | 55 | 56 | 57 | solr_system_stats 58 | true 59 | Enable Solr System metrics 60 | Solr System stats 61 | 62 | boolean 63 | 64 | 65 | 66 | 75 | 76 | 77 | solr_metrics_config_conf_dir 78 | /opt/lucidworks-hdpsearch/metrics/conf 79 | Solr Metrics conf directory 80 | 81 | directory 82 | 83 | Solr Metrics configuration directory 84 | 85 | 86 | 87 | solr_metrics_config_pid_dir 88 | /opt/lucidworks-hdpsearch/metrics/var 89 | Solr Metrics PID directory 90 | 91 | directory 92 | 93 | Solr Metrics PID directory 94 | 95 | 96 | 97 | solr_metrics_config_log_dir 98 | /opt/lucidworks-hdpsearch/metrics/log 99 | Solr Metrics log directory 100 | 101 | directory 102 | 103 | Solr Metrics log directory 104 | 105 | 106 | 107 | solr_metrics_delay 108 | 500 109 | 110 | Time in milliseconds before task is to be executed. 111 | Default value is 500 milliseconds. 112 | 113 | 114 | int 115 | 116 | Solr Metrics delay 117 | 118 | 119 | 120 | solr_metrics_period 121 | 5000 122 | 123 | Time in milliseconds between successive task executions. 124 | Default value is 5000 milliseconds / 5 seconds. 125 | 126 | 127 | int 128 | 129 | Solr Metrics period 130 | 131 | 132 | 133 | solr_metrics_properties 134 | This is the jinja template for solr.metrics.properties file 135 | 136 | # Security 137 | truststore.path={{metric_truststore_path}} 138 | truststore.type={{metric_truststore_type}} 139 | truststore.password={{metric_truststore_password}} 140 | 141 | solr.truststore.path={{solr_ssl_trust_store}} 142 | solr.truststore.password={{solr_ssl_trust_store_password}} 143 | 144 | jaas.file={{solr_metrics_kerberos_jaas_config}} 145 | 146 | # Metrics 147 | metrics.delay={{solr_metrics_delay}} 148 | metrics.period={{solr_metrics_period}} 149 | 150 | metrics.core.stats={{solr_core_stats}} 151 | metrics.cache.stats={{solr_cache_stats}} 152 | metrics.updatehandler.stats={{solr_updatehandler_stats}} 153 | metrics.queryhandler.stats={{solr_queryhandler_stats}} 154 | metrics.system.stats={{solr_system_stats}} 155 | metrics.status.stats={{solr_status_stats}} 156 | 157 | # Collector 158 | collector.hosts={{ams_collector_hosts}} 159 | collector.port={{metric_collector_port}} 160 | collector.protocol={{metric_collector_protocol}} 161 | 162 | # Solr 163 | solr.host={{solr_protocol}}://{{hostname}}:{{solr_config_port}}/solr 164 | solr.retry.times=3 165 | solr.retry.interval=10 166 | 167 | 168 | content 169 | 170 | 171 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/solr-security.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | solr_security_json 8 | Solr security.json 9 | The security.json will be uploaded to Zookeeper. 10 | Note: if a security.json is already stored in Zookeeper, this security.json will not overwrite it. 11 | 12 | 13 | { 14 | "authentication": { 15 | "class": "org.apache.solr.security.KerberosPlugin" 16 | } 17 | } 18 | 19 | 20 | content 21 | false 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/configuration/solr-ssl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_ssl_enable 7 | false 8 | Enable SSL support 9 | Enable SSL support 10 | 11 | boolean 12 | false 13 | 14 | 15 | 16 | 17 | solr_ssl_key_store 18 | etc/solr-ssl.keystore.jks 19 | SSL keystore file 20 | SSL keystore file 21 | 22 | 23 | 24 | solr_ssl_key_store_password 25 | secret 26 | PASSWORD 27 | SSL keystore password 28 | 29 | password 30 | false 31 | 32 | SSL keystore password 33 | 34 | 35 | 36 | solr_ssl_trust_store 37 | etc/solr-ssl.keystore.jks 38 | SSL truststore file 39 | SSL truststore file 40 | 41 | 42 | 43 | solr_ssl_trust_store_password 44 | secret 45 | PASSWORD 46 | SSL truststore password 47 | 48 | password 49 | false 50 | 51 | SSL truststore password 52 | 53 | 54 | 55 | solr_ssl_need_client_auth 56 | false 57 | Need client authentication 58 | Need client authentication 59 | 60 | boolean 61 | false 62 | 63 | 64 | 65 | 66 | solr_ssl_want_client_auth 67 | false 68 | Want client authentication 69 | Want client authentication 70 | 71 | boolean 72 | false 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/kerberos.json: -------------------------------------------------------------------------------- 1 | { 2 | "services": [ 3 | { 4 | "name": "SOLR", 5 | "identities": [ 6 | { 7 | "name": "/spnego", 8 | "principal": { 9 | "configuration": "solr-config-env/solr_spnego_principal_name" 10 | }, 11 | "keytab": { 12 | "configuration": "solr-config-env/solr_spnego_keytab_path" 13 | } 14 | } 15 | ], 16 | "components": [ 17 | { 18 | "name": "SOLR_SERVER", 19 | "identities": [ 20 | { 21 | "name": "solr_kerberos", 22 | "principal": { 23 | "value": "solr/_HOST@${realm}", 24 | "type": "service", 25 | "configuration": "solr-config-env/solr_principal_name" 26 | }, 27 | "keytab": { 28 | "file": "${keytab_dir}/solr.service.keytab", 29 | "owner": { 30 | "name": "${solr-config-env/solr_config_user}", 31 | "access": "r" 32 | }, 33 | "group": { 34 | "name": "${cluster-env/user_group}", 35 | "access": "" 36 | }, 37 | "configuration": "solr-config-env/solr_keytab_path" 38 | } 39 | },{ 40 | "name": "solr_metrics_kerberos", 41 | "principal": { 42 | "value": "solr_metrics/_HOST@${realm}", 43 | "type": "service", 44 | "configuration": "solr-config-env/solr_metrics_principal_name" 45 | }, 46 | "keytab": { 47 | "file": "${keytab_dir}/solr.metrics.service.keytab", 48 | "owner": { 49 | "name": "${solr-config-env/solr_config_user}", 50 | "access": "r" 51 | }, 52 | "group": { 53 | "name": "${cluster-env/user_group}", 54 | "access": "" 55 | }, 56 | "configuration": "solr-config-env/solr_metrics_keytab_path" 57 | } 58 | } 59 | ] 60 | } 61 | ] 62 | } 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 2.0 4 | 5 | 6 | SOLR 7 | Solr 8 | Solr is a search platform from the Apache Lucene project. 9 | Its major features include full-text search, hit highlighting, 10 | faceted search, dynamic clustering, database integration, and 11 | rich document (e.g., Word, PDF) handling. 12 | 13 | 5.5.5 14 | 15 | 16 | 17 | SOLR_SERVER 18 | Solr 19 | MASTER 20 | 1+ 21 | false 22 | 23 | 24 | PYTHON 25 | 5000 26 | 27 | 28 | 29 | 30 | 31 | 32 | any 33 | 34 | 35 | lsof 36 | 37 | 38 | lucidworks-hdpsearch 39 | 40 | 41 | 42 | 43 | 44 | 45 | ZOOKEEPER 46 | HDFS 47 | 48 | 49 | 50 | 51 | PYTHON 52 | 300 53 | 54 | 55 | 56 | solr-config-env 57 | solr-cloud 58 | solr-hdfs 59 | solr-log4j 60 | solr-ssl 61 | example-collection 62 | solr-security 63 | solr-metrics 64 | ams-env 65 | 66 | true 67 | 68 | 69 | 70 | quicklinks.json 71 | true 72 | 73 | 74 | 75 | metrics.json 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/alerts/alert_solr_cpu_metrics.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import httplib 3 | import json 4 | import time 5 | 6 | from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames 7 | from ambari_commons.ambari_metrics_helper import load_properties_from_file 8 | 9 | RESULT_STATE_OK = 'OK' 10 | RESULT_STATE_CRITICAL = 'CRITICAL' 11 | RESULT_STATE_WARNING = 'WARNING' 12 | RESULT_STATE_UNKNOWN = 'UNKNOWN' 13 | RESULT_STATE_SKIPPED = 'SKIPPED' 14 | 15 | KERBEROS_KEYTAB = '{{hdfs-site/dfs.web.authentication.kerberos.keytab}}' 16 | KERBEROS_PRINCIPAL = '{{hdfs-site/dfs.web.authentication.kerberos.principal}}' 17 | SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' 18 | SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' 19 | 20 | METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' 21 | METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' 22 | METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' 23 | 24 | SOLR_METRICS_CONF_DIR = '{{solr-metrics/solr_metrics_config_conf_dir}}' 25 | SOLR_METRICS_PERIOD = '{{solr-metrics/solr_metrics_period}}' 26 | 27 | AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s" 28 | 29 | CONNECTION_TIMEOUT_DEFAULT = 5.0 30 | CONNECTION_TIMEOUT_KEY = "connection.timeout" 31 | 32 | METRIC_NAMES_DEFAULT = "solr.admin.info.system.processCpuLoad" 33 | METRIC_NAMES_KEY = "metric.names" 34 | 35 | APP_ID_DEFAULT = "solr-host-app" 36 | APP_ID_KEY = "app.id" 37 | 38 | WARNING_THRESHOLD_DEFAULT = 50 39 | WARNING_THRESHOLD_KEY = "metric.solr.cpu.warning.threshold" 40 | 41 | CRITICAL_THRESHOLD_DEFAULT = 75 42 | CRITICAL_THRESHOLD_KEY = "metric.solr.cpu.critical.threshold" 43 | 44 | 45 | def get_tokens(): 46 | """ 47 | Returns a tuple of tokens in the format {{site/property}} that will be used 48 | to build the dictionary passed into execute 49 | """ 50 | return (SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, METRICS_COLLECTOR_VIP_HOST_KEY, 51 | METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, SOLR_METRICS_CONF_DIR, 52 | SOLR_METRICS_PERIOD) 53 | 54 | 55 | def execute(configurations={}, parameters={}, host_name=None): 56 | collector_host, collector_port = get_collector_config(configurations) 57 | 58 | if collector_host is None or collector_port is None: 59 | return RESULT_STATE_UNKNOWN, ['Undefined collector host: {0} or collector port {1}'.format(collector_host, 60 | collector_port)] 61 | 62 | metric_name = METRIC_NAMES_DEFAULT 63 | if METRIC_NAMES_KEY in parameters: 64 | metric_name = parameters[METRIC_NAMES_KEY] 65 | 66 | app_id = APP_ID_DEFAULT 67 | if APP_ID_KEY in parameters: 68 | app_id = parameters[APP_ID_KEY] 69 | 70 | connection_timeout = CONNECTION_TIMEOUT_DEFAULT 71 | if CONNECTION_TIMEOUT_KEY in parameters: 72 | connection_timeout = parameters[CONNECTION_TIMEOUT_KEY] 73 | 74 | warning_threshold = WARNING_THRESHOLD_DEFAULT 75 | if WARNING_THRESHOLD_KEY in parameters: 76 | warning_threshold = parameters[WARNING_THRESHOLD_KEY] 77 | 78 | critical_threshold = CRITICAL_THRESHOLD_DEFAULT 79 | if CRITICAL_THRESHOLD_KEY in parameters: 80 | critical_threshold = parameters[CRITICAL_THRESHOLD_KEY] 81 | 82 | current_time = int(time.time()) * 1000 83 | metric_period = int(configurations[SOLR_METRICS_PERIOD]) 84 | start_time = current_time - (metric_period * 2) 85 | 86 | get_metrics_parameters = { 87 | "metricNames": metric_name, 88 | "appId": app_id, 89 | "hostname": host_name, 90 | "startTime": start_time, 91 | "endTime": current_time, 92 | "grouped": "true", 93 | } 94 | 95 | encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters) 96 | 97 | try: 98 | conn = httplib.HTTPConnection(collector_host, int(collector_port), timeout=connection_timeout) 99 | conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters) 100 | response = conn.getresponse() 101 | data = response.read() 102 | conn.close() 103 | except Exception: 104 | return RESULT_STATE_UNKNOWN, ["Unable to retrieve metrics from the Ambari Metrics service."] 105 | 106 | data_json = json.loads(data) 107 | cpu_load_value = -1 108 | 109 | for metrics_data in data_json["metrics"]: 110 | if "solr.admin.info.system.processCpuLoad" in metrics_data["metricname"]: 111 | metrics = metrics_data["metrics"].values() 112 | if len(metrics) > 0: 113 | cpu_load_value = metrics[0] * 100 114 | break 115 | 116 | if int(cpu_load_value) == -1: 117 | return RESULT_STATE_UNKNOWN, ["There is not enough data"] 118 | 119 | response = 'CPU load {0:.2f} %'.format(cpu_load_value) 120 | 121 | if int(cpu_load_value) >= warning_threshold and int(cpu_load_value) < critical_threshold: 122 | return RESULT_STATE_WARNING, [response] 123 | 124 | if cpu_load_value >= critical_threshold: 125 | return RESULT_STATE_CRITICAL, [response] 126 | 127 | return RESULT_STATE_OK, [response] 128 | 129 | 130 | def get_collector_config(configurations): 131 | solr_metrics_props = "{0}/{1}".format(configurations[SOLR_METRICS_CONF_DIR], "solr.metrics.properties") 132 | props = load_properties_from_file(solr_metrics_props) 133 | collector_hosts = props.get("collector.hosts") 134 | 135 | if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: 136 | collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] 137 | collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) 138 | else: 139 | collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") 140 | if valid_collector_webapp_address(collector_webapp_address): 141 | collector_host = select_metric_collector_hosts_from_hostnames(collector_hosts) 142 | collector_port = int(collector_webapp_address[1]) 143 | else: 144 | return None, None 145 | return collector_host, collector_port 146 | 147 | 148 | def valid_collector_webapp_address(webapp_address): 149 | if len(webapp_address) == 2 and webapp_address[0] != '127.0.0.1' and webapp_address[1].isdigit(): 150 | return True 151 | return False 152 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/alerts/alert_solr_memory_metrics.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import httplib 3 | import json 4 | import time 5 | 6 | from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames 7 | from ambari_commons.ambari_metrics_helper import load_properties_from_file 8 | 9 | RESULT_STATE_OK = 'OK' 10 | RESULT_STATE_CRITICAL = 'CRITICAL' 11 | RESULT_STATE_WARNING = 'WARNING' 12 | RESULT_STATE_UNKNOWN = 'UNKNOWN' 13 | RESULT_STATE_SKIPPED = 'SKIPPED' 14 | 15 | KERBEROS_KEYTAB = '{{hdfs-site/dfs.web.authentication.kerberos.keytab}}' 16 | KERBEROS_PRINCIPAL = '{{hdfs-site/dfs.web.authentication.kerberos.principal}}' 17 | SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' 18 | SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' 19 | 20 | METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' 21 | METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' 22 | METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' 23 | 24 | SOLR_METRICS_CONF_DIR = '{{solr-metrics/solr_metrics_config_conf_dir}}' 25 | SOLR_METRICS_PERIOD = "{{solr-metrics/solr_metrics_period}}" 26 | 27 | AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s" 28 | 29 | CONNECTION_TIMEOUT_DEFAULT = 5.0 30 | CONNECTION_TIMEOUT_KEY = "connection.timeout" 31 | 32 | METRIC_NAMES_DEFAULT = "solr.admin.info.jvm.memory.used,solr.admin.info.jvm.memory.max" 33 | METRIC_NAMES_KEY = "metric.names" 34 | 35 | APP_ID_DEFAULT = "solr-host-app" 36 | APP_ID_KEY = "app.id" 37 | 38 | WARNING_THRESHOLD_DEFAULT = 50 39 | WARNING_THRESHOLD_KEY = "metric.solr.memory.warning.threshold" 40 | 41 | CRITICAL_THRESHOLD_DEFAULT = 75 42 | CRITICAL_THRESHOLD_KEY = "metric.solr.memory.critical.threshold" 43 | 44 | def get_tokens(): 45 | """ 46 | Returns a tuple of tokens in the format {{site/property}} that will be used 47 | to build the dictionary passed into execute 48 | """ 49 | return (SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, METRICS_COLLECTOR_VIP_HOST_KEY, 50 | METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, SOLR_METRICS_CONF_DIR, 51 | SOLR_METRICS_PERIOD) 52 | 53 | 54 | def execute(configurations={}, parameters={}, host_name=None): 55 | collector_host, collector_port = get_collector_config(configurations) 56 | 57 | if collector_host is None or collector_port is None: 58 | return RESULT_STATE_UNKNOWN, ['Undefined collector host: {0} or collector port {1}'.format(collector_host, 59 | collector_port)] 60 | 61 | metric_name = METRIC_NAMES_DEFAULT 62 | if METRIC_NAMES_KEY in parameters: 63 | metric_name = parameters[METRIC_NAMES_KEY] 64 | 65 | app_id = APP_ID_DEFAULT 66 | if APP_ID_KEY in parameters: 67 | app_id = parameters[APP_ID_KEY] 68 | 69 | connection_timeout = CONNECTION_TIMEOUT_DEFAULT 70 | if CONNECTION_TIMEOUT_KEY in parameters: 71 | connection_timeout = parameters[CONNECTION_TIMEOUT_KEY] 72 | 73 | warning_threshold = WARNING_THRESHOLD_DEFAULT 74 | if WARNING_THRESHOLD_KEY in parameters: 75 | warning_threshold = parameters[WARNING_THRESHOLD_KEY] 76 | 77 | critical_threshold = CRITICAL_THRESHOLD_DEFAULT 78 | if CRITICAL_THRESHOLD_KEY in parameters: 79 | critical_threshold = parameters[CRITICAL_THRESHOLD_KEY] 80 | 81 | current_time = int(time.time()) * 1000 82 | metric_period = int(configurations[SOLR_METRICS_PERIOD]) 83 | start_time = current_time - (metric_period * 2) 84 | 85 | get_metrics_parameters = { 86 | "metricNames": metric_name, 87 | "appId": app_id, 88 | "hostname": host_name, 89 | "startTime": start_time, 90 | "endTime": current_time, 91 | "grouped": "true", 92 | } 93 | 94 | encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters) 95 | 96 | try: 97 | conn = httplib.HTTPConnection(collector_host, int(collector_port), timeout=connection_timeout) 98 | conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters) 99 | response = conn.getresponse() 100 | data = response.read() 101 | conn.close() 102 | except Exception: 103 | return RESULT_STATE_UNKNOWN, ["Unable to retrieve metrics from the Ambari Metrics service."] 104 | 105 | data_json = json.loads(data) 106 | 107 | memory_max = -1 108 | memory_used = -1 109 | 110 | for metrics_data in data_json["metrics"]: 111 | if "solr.admin.info.jvm.memory.max" in metrics_data["metricname"]: 112 | metrics = metrics_data["metrics"].values() 113 | if len(metrics) > 0: 114 | memory_max = metrics[0] / (1024 * 1024) 115 | continue 116 | if "solr.admin.info.jvm.memory.used" in metrics_data["metricname"]: 117 | metrics = metrics_data["metrics"].values() 118 | if len(metrics) > 0: 119 | memory_used = metrics[0] / (1024 * 1024) 120 | continue 121 | 122 | if int(memory_max) == -1 or int(memory_used) == -1: 123 | return RESULT_STATE_UNKNOWN, ["There is not enough data to compare"] 124 | 125 | memory_value = (memory_used / memory_max) * 100 126 | response = 'Memory usage is {0:.2f} %'.format(memory_value) 127 | 128 | if int(memory_value) >= warning_threshold and int(memory_value) < critical_threshold: 129 | return RESULT_STATE_WARNING, [response] 130 | 131 | if memory_value >= critical_threshold: 132 | return RESULT_STATE_CRITICAL, [response] 133 | 134 | return RESULT_STATE_OK, [response] 135 | 136 | 137 | def get_collector_config(configurations): 138 | solr_metrics_props = "{0}/{1}".format(configurations[SOLR_METRICS_CONF_DIR], "solr.metrics.properties") 139 | props = load_properties_from_file(solr_metrics_props) 140 | collector_hosts = props.get("collector.hosts") 141 | 142 | if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: 143 | collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] 144 | collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) 145 | else: 146 | collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") 147 | if valid_collector_webapp_address(collector_webapp_address): 148 | collector_host = select_metric_collector_hosts_from_hostnames(collector_hosts) 149 | collector_port = int(collector_webapp_address[1]) 150 | else: 151 | return None, None 152 | return collector_host, collector_port 153 | 154 | 155 | def valid_collector_webapp_address(webapp_address): 156 | if len(webapp_address) == 2 and webapp_address[0] != '127.0.0.1' and webapp_address[1].isdigit(): 157 | return True 158 | return False 159 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/service_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from resource_management.core.logger import Logger 6 | from resource_management.core.resources.system import Execute 7 | from resource_management.libraries.functions.format import format 8 | from resource_management.libraries.script import Script 9 | 10 | from solr_utils import exists_collection 11 | 12 | 13 | class ServiceCheck(Script): 14 | def service_check(self, env): 15 | import params 16 | env.set_params(params) 17 | 18 | if not os.path.isfile(params.solr_config_pid_file): 19 | Logger.error(format("PID file {solr_config_pid_file} does not exist")) 20 | exit(1) 21 | 22 | if not params.solr_collection_sample_create: 23 | Logger.info("Create sample collection unchecked, skipping ...") 24 | return 25 | 26 | if exists_collection(params.solr_collection_name): 27 | Logger.warning(format("Collection {solr_collection_name} already exists, skipping ...")) 28 | return 29 | 30 | if not params.solr_cloud_mode: 31 | Execute( 32 | format( 33 | '{solr_config_bin_dir}/solr create_core -c {solr_collection_name}' + 34 | ' -d {solr_collection_config_dir} -p {solr_config_port} >> {solr_config_service_log_file} 2>&1' 35 | ), 36 | environment={'JAVA_HOME': params.java64_home}, 37 | user=params.solr_config_user 38 | ) 39 | else: 40 | Execute(format( 41 | '{solr_config_bin_dir}/solr create_collection -c {solr_collection_name}' + 42 | ' -d {solr_collection_config_dir} -p {solr_config_port}' + 43 | ' -s {solr_collection_shards} -rf {solr_collection_replicas}' + 44 | ' >> {solr_config_service_log_file} 2>&1'), 45 | environment={'JAVA_HOME': params.java64_home}, 46 | user=params.solr_config_user 47 | ) 48 | 49 | 50 | if __name__ == "__main__": 51 | ServiceCheck().execute() 52 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/setup_solr.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Directory, File, Execute 2 | from resource_management.core.source import Template, InlineTemplate 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr(): 7 | import params 8 | 9 | Directory( 10 | [ 11 | params.solr_config_dir, 12 | params.solr_config_log_dir, 13 | params.solr_config_service_log_dir, 14 | params.solr_config_pid_dir, 15 | params.solr_config_conf_dir, 16 | params.solr_config_home_dir 17 | ], 18 | mode=0755, 19 | cd_access='a', 20 | owner=params.solr_config_user, 21 | group=params.solr_config_group, 22 | create_parents=True 23 | ) 24 | 25 | Execute( 26 | ('chmod', '-R', '777', params.solr_webapp_dir), 27 | sudo=True 28 | ) 29 | 30 | File( 31 | format("{solr_config_bin_dir}/solr.in.sh"), 32 | content=InlineTemplate(params.solr_config_in_sh), 33 | owner=params.solr_config_user 34 | ) 35 | 36 | File( 37 | format("{solr_config_conf_dir}/log4j.properties"), 38 | content=InlineTemplate(params.log4j_properties), 39 | owner=params.solr_config_user 40 | ) 41 | 42 | File( 43 | format("{solr_config_home_dir}/solr.xml"), 44 | content=Template("solr.xml.j2"), 45 | owner=params.solr_config_user 46 | ) 47 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/setup_solr_cloud.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute 3 | from resource_management.core.shell import call 4 | from resource_management.libraries.functions.format import format 5 | 6 | 7 | def setup_solr_cloud(): 8 | import params 9 | 10 | code, output = call( 11 | format( 12 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{clusterstate_json}' 13 | ), 14 | env={'JAVA_HOME': params.java64_home}, 15 | timeout=60 16 | ) 17 | 18 | if not ("NoNodeException" in output): 19 | Logger.info( 20 | format( 21 | "ZK node {solr_cloud_zk_directory}{clusterstate_json} already exists, skipping ..." 22 | ) 23 | ) 24 | return 25 | 26 | Execute( 27 | format( 28 | '{zk_client_prefix} -cmd makepath {solr_cloud_zk_directory}' 29 | ), 30 | environment={'JAVA_HOME': params.java64_home}, 31 | ignore_failures=True, 32 | user=params.solr_config_user 33 | ) 34 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/setup_solr_hdfs_support.py: -------------------------------------------------------------------------------- 1 | def setup_solr_hdfs_support(): 2 | import params 3 | 4 | # Create a home directory for solr user 5 | params.HdfsResource( 6 | params.solr_hdfs_home_directory, 7 | type="directory", 8 | action="create_on_execute", 9 | owner=params.solr_config_user 10 | ) 11 | 12 | params.HdfsResource(params.solr_hdfs_directory, 13 | type="directory", 14 | action="create_on_execute", 15 | owner=params.solr_config_user 16 | ) 17 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/setup_solr_kerberos_auth.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute, File 3 | from resource_management.core.shell import call 4 | from resource_management.core.source import Template 5 | from resource_management.libraries.functions.format import format 6 | 7 | 8 | def setup_solr_kerberos_auth(): 9 | import params 10 | 11 | File(format("{solr_kerberos_jaas_config}"), 12 | content=Template("solr_server_jaas.conf.j2"), 13 | owner=params.solr_config_user 14 | ) 15 | 16 | if _has_security_json(): 17 | Logger.info("Solr Security Json was found, it will not be overridden") 18 | return 19 | 20 | command = format('{zk_client_prefix} -cmd put {solr_cloud_zk_directory}{security_json} ') 21 | command += format('\'{solr_security_json}\'') 22 | Execute(command, 23 | environment={'JAVA_HOME': params.java64_home}, 24 | ignore_failures=True, 25 | user=params.solr_config_user 26 | ) 27 | 28 | 29 | def remove_solr_kerberos_auth(): 30 | import params 31 | 32 | if not _has_security_json(): 33 | Logger.debug(format("Solr Security Json not found {solr_cloud_zk_directory}{security_json}")) 34 | return 35 | 36 | Execute(format('{zk_client_prefix} -cmd clear {solr_cloud_zk_directory}{security_json}'), 37 | environment={'JAVA_HOME': params.java64_home}, 38 | timeout=60, 39 | ignore_failures=True, 40 | user=params.solr_config_user 41 | ) 42 | 43 | 44 | def _has_security_json(): 45 | import params 46 | 47 | if not params.solr_cloud_mode: 48 | return False 49 | 50 | code, output = call( 51 | format('{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{security_json}'), 52 | env={'JAVA_HOME': params.java64_home}, 53 | timeout=60 54 | ) 55 | 56 | if "{}" == output: 57 | return False 58 | return "NoNodeException" not in output 59 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/setup_solr_metrics.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Directory, File 2 | from resource_management.core.source import InlineTemplate, Template 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr_metrics_support(): 7 | import params 8 | 9 | Directory( 10 | [ 11 | params.solr_metrics_config_conf_dir, 12 | params.solr_metrics_config_pid_dir, 13 | params.solr_metrics_config_log_dir 14 | ], 15 | mode=0755, 16 | cd_access='a', 17 | owner=params.solr_config_user, 18 | group=params.solr_config_group, 19 | create_parents=True 20 | ) 21 | 22 | File( 23 | format("{solr_metrics_config_conf_dir}/solr.metrics.properties"), 24 | content=InlineTemplate(params.solr_metrics_properties), 25 | owner=params.solr_config_user 26 | ) 27 | 28 | File( 29 | format("{solr_metrics_config_conf_dir}/log4j2.xml"), 30 | content=Template("log4j2.xml"), 31 | owner=params.solr_config_user 32 | ) 33 | 34 | if params.security_enabled: 35 | File( 36 | format("{solr_metrics_kerberos_jaas_config}"), 37 | content=Template("solr_metrics_jaas.conf.j2"), 38 | owner=params.solr_config_user 39 | ) 40 | 41 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/setup_solr_ssl_support.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Execute 2 | from resource_management.core.shell import call 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr_ssl_support(): 7 | import params 8 | 9 | if not params.solr_cloud_mode: 10 | return 11 | 12 | Execute( 13 | format( 14 | '{zk_client_prefix}{solr_cloud_zk_directory} -cmd clusterprop -name urlScheme -val https' 15 | ), 16 | environment={'JAVA_HOME': params.java64_home}, 17 | ignore_failures=True, 18 | user=params.solr_config_user 19 | ) 20 | 21 | 22 | def remove_solr_ssl_support(): 23 | import params 24 | 25 | if not params.solr_cloud_mode: 26 | return 27 | 28 | code, output = call( 29 | format( 30 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{clusterprops_json}' 31 | ), 32 | env={'JAVA_HOME': params.java64_home}, 33 | timeout=60 34 | ) 35 | 36 | if "NoNodeException" in output: 37 | return 38 | 39 | Execute( 40 | format( 41 | '{zk_client_prefix} -cmd clear {solr_cloud_zk_directory}{clusterprops_json}' 42 | ), 43 | environment={'JAVA_HOME': params.java64_home}, 44 | ignore_failures=True, 45 | user=params.solr_config_user 46 | ) 47 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/solr.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute, File 3 | from resource_management.libraries.functions.check_process_status import check_process_status 4 | from resource_management.libraries.functions.format import format 5 | from resource_management.libraries.script.script import Script 6 | 7 | from setup_solr import setup_solr 8 | from setup_solr_cloud import setup_solr_cloud 9 | from setup_solr_hdfs_support import setup_solr_hdfs_support 10 | from setup_solr_kerberos_auth import setup_solr_kerberos_auth, remove_solr_kerberos_auth 11 | from setup_solr_metrics import setup_solr_metrics_support 12 | from setup_solr_ssl_support import setup_solr_ssl_support, remove_solr_ssl_support 13 | from solr_utils import is_solr_running, solr_port_validation, delete_write_lock_files 14 | 15 | 16 | class Solr(Script): 17 | def install(self, env): 18 | import params 19 | env.set_params(params) 20 | self.install_packages(env) 21 | 22 | def configure(self, env): 23 | import params 24 | env.set_params(params) 25 | setup_solr() 26 | 27 | if params.solr_cloud_mode: 28 | setup_solr_cloud() 29 | 30 | if params.solr_hdfs_enable: 31 | setup_solr_hdfs_support() 32 | 33 | if params.solr_ssl_enable: 34 | setup_solr_ssl_support() 35 | else: 36 | remove_solr_ssl_support() 37 | 38 | if params.security_enabled: 39 | setup_solr_kerberos_auth() 40 | else: 41 | remove_solr_kerberos_auth() 42 | 43 | if params.solr_hdfs_enable and params.solr_hdfs_delete_write_lock_files: 44 | delete_write_lock_files() 45 | 46 | if params.has_metric_collector and params.solr_enable_metrics: 47 | setup_solr_metrics_support() 48 | 49 | def start(self, env): 50 | import params 51 | env.set_params(params) 52 | self.configure(env) 53 | 54 | if not solr_port_validation(): 55 | exit(1) 56 | 57 | if is_solr_running(): 58 | Logger.info("Solr is running, it can not be started it again") 59 | exit(1) 60 | 61 | Logger.info("Starting Solr ... ") 62 | start_command = format('{solr_config_bin_dir}/solr start >> {solr_config_service_log_file} 2>&1') 63 | 64 | Execute( 65 | start_command, 66 | environment={'JAVA_HOME': params.java64_home}, 67 | user=params.solr_config_user 68 | ) 69 | 70 | if params.has_metric_collector and params.solr_enable_metrics: 71 | Logger.info('Starting Solr Metrics Sink.') 72 | 73 | if params.security_enabled: 74 | Execute( 75 | format('{kinit_path_local} -kt {solr_metrics_kerberos_keytab} {solr_metrics_kerberos_principal}'), 76 | user=params.solr_config_user 77 | ) 78 | 79 | Execute( 80 | format('{solr_metrics_sink_bin}/solr.metrics.sh start'), 81 | environment={ 82 | 'JAVA_HOME': params.java64_home, 83 | 'SOLR_METRICS_PID_DIR': params.solr_metrics_config_pid_dir, 84 | 'SOLR_METRICS_LOGS': params.solr_metrics_config_log_dir, 85 | 'SOLR_METRIC_CONF': params.solr_metrics_config_conf_dir 86 | }, 87 | user=params.solr_config_user 88 | ) 89 | 90 | def stop(self, env): 91 | import params 92 | env.set_params(params) 93 | 94 | if params.has_metric_collector and params.solr_enable_metrics: 95 | Logger.info('Stopping Solr Metrics Sink.') 96 | 97 | if params.security_enabled: 98 | Execute( 99 | format('{kinit_path_local} -kt {solr_metrics_kerberos_keytab} {solr_metrics_kerberos_principal}'), 100 | user=params.solr_config_user 101 | ) 102 | 103 | Execute( 104 | format('{solr_metrics_sink_bin}/solr.metrics.sh stop'), 105 | environment={ 106 | 'JAVA_HOME': params.java64_home, 107 | 'SOLR_METRICS_PID_DIR': params.solr_metrics_config_pid_dir, 108 | 'SOLR_METRICS_LOGS': params.solr_metrics_config_log_dir, 109 | 'SOLR_METRIC_CONF': params.solr_metrics_config_conf_dir 110 | }, 111 | ignore_failures=True, 112 | user=params.solr_config_user 113 | ) 114 | 115 | if not is_solr_running(): 116 | Logger.info("Solr is not running, it can not be stopped it again") 117 | return 118 | 119 | Execute( 120 | format( 121 | '{solr_config_bin_dir}/solr stop -all >> {solr_config_service_log_file} 2>&1'), 122 | environment={'JAVA_HOME': params.java64_home}, 123 | user=params.solr_config_user 124 | ) 125 | 126 | File( 127 | params.solr_config_pid_file, 128 | action="delete" 129 | ) 130 | 131 | def status(self, env): 132 | import status_params 133 | env.set_params(status_params) 134 | 135 | check_process_status(status_params.solr_config_pid_file) 136 | 137 | 138 | if __name__ == "__main__": 139 | Solr().execute() 140 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/solr_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from resource_management.core.logger import Logger 5 | from resource_management.core.resources.system import Execute 6 | from resource_management.core.shell import call 7 | from resource_management.libraries.functions.format import format 8 | 9 | COLLECTION_PATTERN = "{solr_hdfs_directory}/[a-zA-Z0-9\._-]+" 10 | CORE_PATTERN = "{collection_path}\/core_node[0-9]+" 11 | WRITE_LOCK_PATTERN = "{0}/data/index/write.lock " 12 | HOSTNAME_VERIFIER_PATTERN = "{core_node_name}\":{{((?![^_]shard|core_node).)*\"node_name\":\"{solr_hostname}" 13 | 14 | 15 | def solr_port_validation(): 16 | code, output = call( 17 | format('netstat -lnt | awk -v v1={solr_config_port} \'$6 == "LISTEN" && $4 ~ ":"v1\''), 18 | timeout=60) 19 | Logger.info(format("Solr port validation output: {output}")) 20 | 21 | if "LISTEN" in output: 22 | Logger.error(format("The port {solr_config_port} is not available")) 23 | return False 24 | 25 | return True 26 | 27 | 28 | def is_solr_running(): 29 | import params 30 | 31 | code, output = call( 32 | format('{solr_config_bin_dir}/solr status'), 33 | env={'JAVA_HOME': params.java64_home}, 34 | timeout=60 35 | ) 36 | Logger.info(format("Solr status output: {output}")) 37 | pattern_port = re.compile(format('running on port {solr_config_port}')) 38 | 39 | if pattern_port.search(output): 40 | Logger.error(format("A Solr instance is running on port {solr_config_port}")) 41 | return True 42 | 43 | return False 44 | 45 | 46 | def exists_collection(collection_name): 47 | import params 48 | 49 | if not params.solr_cloud_mode: 50 | if os.path.isdir(format("{solr_config_home_dir}/{collection_name}")): 51 | return True 52 | return False 53 | 54 | code, output = call(format( 55 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}'), 56 | env={'JAVA_HOME': params.java64_home}, 57 | timeout=60 58 | ) 59 | 60 | if "NoNodeException" in output: 61 | return False 62 | return True 63 | 64 | 65 | def get_collection_paths(hadoop_output): 66 | import params 67 | pattern = re.compile(format(COLLECTION_PATTERN)) 68 | 69 | collection_paths = re.findall(pattern, hadoop_output) 70 | return collection_paths 71 | 72 | 73 | def get_core_paths(hadoop_output, collection_path): 74 | pattern = re.compile(format(CORE_PATTERN)) 75 | core_paths = re.findall(pattern, hadoop_output) 76 | return core_paths 77 | 78 | 79 | def get_write_lock_files_solr_cloud(hadoop_prefix, collections): 80 | import params 81 | 82 | write_locks_to_delete = '' 83 | 84 | for collection_path in collections: 85 | code, output = call(format('{hadoop_prefix} -ls {collection_path}')) 86 | core_paths = get_core_paths(output, collection_path) 87 | 88 | collection_name = collection_path.replace(format('{solr_hdfs_directory}/'), '') 89 | zk_code, zk_output = call(format( 90 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}/state.json'), 91 | env={'JAVA_HOME': params.java64_home}, 92 | timeout=60 93 | ) 94 | if zk_code != 0: 95 | Logger.error(format('Cannot determine cores owned by [{solr_hostname}] in collection [{collection_name}] due to ZK error.')) 96 | continue 97 | 98 | for core_path in core_paths: 99 | core_node_name = core_path.replace(format('{collection_path}/'), '') 100 | pattern = re.compile(format(HOSTNAME_VERIFIER_PATTERN), re.MULTILINE|re.DOTALL) 101 | core_on_hostname = re.search(pattern, zk_output) 102 | if core_on_hostname is not None: 103 | write_locks_to_delete += WRITE_LOCK_PATTERN.format(core_path) 104 | 105 | return write_locks_to_delete 106 | 107 | 108 | def get_write_lock_files_solr_standalone(collections): 109 | write_locks_to_delete = '' 110 | 111 | for collection_path in collections: 112 | write_locks_to_delete += WRITE_LOCK_PATTERN.format(collection_path) 113 | 114 | return write_locks_to_delete 115 | 116 | 117 | def delete_write_lock_files(): 118 | import params 119 | 120 | if params.security_enabled: 121 | kinit_if_needed = format( 122 | '{kinit_path_local} {hdfs_principal_name} -kt {hdfs_user_keytab}; ') 123 | else: 124 | kinit_if_needed = '' 125 | 126 | hadoop_prefix = format('{kinit_if_needed}hadoop --config {hadoop_conf_dir} dfs') 127 | code, output = call(format('{hadoop_prefix} -ls {solr_hdfs_directory}')) 128 | collections = get_collection_paths(output) 129 | 130 | if params.solr_cloud_mode: 131 | write_locks_to_delete = get_write_lock_files_solr_cloud(hadoop_prefix, collections) 132 | else: 133 | write_locks_to_delete = get_write_lock_files_solr_standalone(collections) 134 | 135 | if len(write_locks_to_delete) > 1: 136 | Logger.info(format('For hostname: \'{solr_hostname}\' lock files \'{write_locks_to_delete}\' will be deleted.')) 137 | Execute( 138 | format('{hadoop_prefix} -rm -f {write_locks_to_delete}'), 139 | user=params.hdfs_user 140 | ) 141 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/scripts/status_params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from resource_management.libraries.functions.format import format 4 | from resource_management.libraries.script.script import Script 5 | 6 | config = Script.get_config() 7 | 8 | map_solr_config = config['configurations']['solr-config-env'] 9 | 10 | solr_config_port = map_solr_config['solr_config_port'] 11 | solr_config_pid_dir = map_solr_config['solr_config_pid_dir'] 12 | solr_config_pid_file = format('{solr_config_pid_dir}/solr-{solr_config_port}.pid') 13 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/templates/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | 9 | 10 | 12 | 13 | %d{ISO8601} - %-5p [%t:%C{1}@%L] - %X - %m%n 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/templates/solr.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 28 | 29 | 30 | 31 | 32 | 33 | ${host:} 34 | ${jetty.port:8983} 35 | ${hostContext:solr} 36 | 37 | ${genericCoreNodeNames:true} 38 | 39 | ${zkClientTimeout:30000} 40 | ${distribUpdateSoTimeout:600000} 41 | ${distribUpdateConnTimeout:60000} 42 | 43 | 44 | 45 | 47 | ${socketTimeout:600000} 48 | ${connTimeout:60000} 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/templates/solr_metrics_jaas.conf.j2: -------------------------------------------------------------------------------- 1 | SolrJClient { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{solr_metrics_kerberos_keytab}}" 5 | storeKey=true 6 | useTicketCache=false 7 | debug=false 8 | principal="{{solr_metrics_kerberos_principal}}"; 9 | }; -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/package/templates/solr_server_jaas.conf.j2: -------------------------------------------------------------------------------- 1 | Client { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{solr_kerberos_keytab}}" 5 | storeKey=true 6 | useTicketCache=true 7 | debug=false 8 | principal="{{solr_kerberos_principal}}"; 9 | }; 10 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/quicklinks/quicklinks.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default", 3 | "description": "default quick links configuration", 4 | "configuration": { 5 | "protocol": { 6 | "type": "https", 7 | "checks": [ 8 | { 9 | "property": "solr_ssl_enable", 10 | "desired": "true", 11 | "site": "solr-ssl" 12 | } 13 | ] 14 | }, 15 | "links": [ 16 | { 17 | "name": "solr_ui", 18 | "label": "Solr UI", 19 | "component_name": "SOLR_SERVER", 20 | "url": "%@://%@:%@/solr", 21 | "requires_user_name": "false", 22 | "port": { 23 | "http_property": "solr_config_port", 24 | "http_default_port": "8983", 25 | "https_property": "solr_config_port", 26 | "https_default_port": "8983", 27 | "regex": "^(\\d+)$", 28 | "site": "solr-config-env" 29 | } 30 | }, 31 | { 32 | "name": "solr_banana_ui", 33 | "label": "Banana UI", 34 | "component_name": "SOLR_SERVER", 35 | "url": "%@://%@:%@/solr/banana/index.html", 36 | "requires_user_name": "false", 37 | "port": { 38 | "http_property": "solr_config_port", 39 | "http_default_port": "8983", 40 | "https_property": "solr_config_port", 41 | "https_default_port": "8983", 42 | "regex": "^(\\d+)$", 43 | "site": "solr-config-env" 44 | } 45 | } 46 | ] 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/5.5.5/role_command_order.json: -------------------------------------------------------------------------------- 1 | { 2 | "general_deps": { 3 | "_comment": "Dependencies for Solr service", 4 | "SOLR_SERVER-START": [ 5 | "ZOOKEEPER_SERVER-START", 6 | "NAMENODE-START", 7 | "DATANODE-START", 8 | "SECONDARY_NAMENODE-START", 9 | "HDFS_SERVICE_CHECK-SERVICE_CHECK", 10 | "METRICS_COLLECTOR-START" 11 | ], 12 | "SOLR_SERVICE_CHECK-SERVICE_CHECK": [ 13 | "SOLR_SERVER-START" 14 | ], 15 | "DATANODE-STOP": [ 16 | "SOLR_SERVER-STOP" 17 | ], 18 | "NAMENODE-STOP": [ 19 | "SOLR_SERVER-STOP" 20 | ], 21 | "ZOOKEEPER_SERVER-STOP": [ 22 | "SOLR_SERVER-STOP" 23 | ] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/alerts.json: -------------------------------------------------------------------------------- 1 | { 2 | "SOLR": { 3 | "SOLR_SERVER": [ 4 | { 5 | "name": "solr", 6 | "label": "Solr Web UI", 7 | "description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.", 8 | "interval": 1, 9 | "scope": "ANY", 10 | "source": { 11 | "type": "WEB", 12 | "uri": { 13 | "http": "{{solr-config-env/solr_config_port}}", 14 | "https": "{{solr-config-env/solr_config_port}}", 15 | "https_property": "{{solr-ssl/solr_ssl_enable}}", 16 | "https_property_value": "true", 17 | "kerberos_keytab": "{{solr-config-env/solr_spnego_keytab_path}}", 18 | "kerberos_principal": "{{solr-config-env/solr_spnego_principal_name}}", 19 | "connection_timeout": 5.0, 20 | "default_port": 8983 21 | }, 22 | "reporting": { 23 | "ok": { 24 | "text": "HTTP {0} response in {2:.3f}s" 25 | }, 26 | "warning": { 27 | "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" 28 | }, 29 | "critical": { 30 | "text": "Connection failed to {1} ({3})" 31 | } 32 | } 33 | } 34 | }, 35 | { 36 | "name": "solr_cpu_usage", 37 | "label": "Solr CPU Utilization", 38 | "description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", 39 | "interval": 5, 40 | "scope": "ANY", 41 | "source": { 42 | "type": "SCRIPT", 43 | "path": "SOLR/6.6.2/package/alerts/alert_solr_cpu_metrics.py", 44 | "parameters": [ 45 | { 46 | "name": "connection.timeout", 47 | "value": 5.0, 48 | "type": "NUMERIC", 49 | "units": "seconds", 50 | "visibility": "HIDDEN" 51 | }, 52 | { 53 | "name": "app.id", 54 | "value": "solr-host-app", 55 | "type": "STRING", 56 | "visibility": "HIDDEN" 57 | }, 58 | { 59 | "name": "metric.names", 60 | "value": "solr.jvm.gauge.os.processCpuLoad", 61 | "type": "STRING", 62 | "visibility": "HIDDEN" 63 | }, 64 | { 65 | "name": "metric.solr.cpu.warning.threshold", 66 | "display_name": "Warning threshold", 67 | "type": "PERCENT", 68 | "units": "%", 69 | "value": 75, 70 | "description": "CPU usage is greater than 50%", 71 | "threshold": "WARNING" 72 | }, 73 | { 74 | "name": "metric.solr.cpu.critical.threshold", 75 | "display_name": "Critical threshold", 76 | "type": "PERCENT", 77 | "units": "%", 78 | "value": 90, 79 | "description": "CPU usage is greater than 75%", 80 | "threshold": "CRITICAL" 81 | } 82 | ] 83 | } 84 | }, 85 | { 86 | "name": "solr_memory", 87 | "label": "Solr Memory Utilization", 88 | "description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", 89 | "interval": 5, 90 | "scope": "ANY", 91 | "source": { 92 | "type": "SCRIPT", 93 | "path": "SOLR/6.6.2/package/alerts/alert_solr_memory_metrics.py", 94 | "parameters": [ 95 | { 96 | "name": "connection.timeout", 97 | "value": 5.0, 98 | "type": "NUMERIC", 99 | "units": "seconds", 100 | "visibility": "HIDDEN" 101 | }, 102 | { 103 | "name": "app.id", 104 | "value": "solr-host-app", 105 | "type": "STRING", 106 | "visibility": "HIDDEN" 107 | }, 108 | { 109 | "name": "metric.names", 110 | "value": "solr.jvm.gauge.memory.total.used,solr.jvm.gauge.memory.total.max", 111 | "type": "STRING", 112 | "visibility": "HIDDEN" 113 | }, 114 | { 115 | "name": "metric.solr.memory.warning.threshold", 116 | "display_name": "Warning threshold", 117 | "type": "PERCENT", 118 | "units": "%", 119 | "value": 75, 120 | "description": "Memory usage is greater than 50%", 121 | "threshold": "WARNING" 122 | }, 123 | { 124 | "name": "metric.solr.memory.critical.threshold", 125 | "display_name": "Critical threshold", 126 | "type": "PERCENT", 127 | "units": "%", 128 | "value": 90, 129 | "description": "Memory usage is greater than 75%", 130 | "threshold": "CRITICAL" 131 | } 132 | ] 133 | } 134 | } 135 | ] 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/configuration/solr-metrics.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | solr_enable_metrics 8 | false 9 | Enable Solr Metrics (Requires Ambari Metrics) 10 | Enable Solr Metrics 11 | 12 | boolean 13 | 14 | 15 | 16 | 17 | solr_core_metrics 18 | true 19 | Enable Solr Core metrics 20 | Solr Core Metrics 21 | 22 | boolean 23 | 24 | 25 | 26 | 27 | solr_jetty_metrics 28 | true 29 | Enable Solr Jetty metrics 30 | Solr Jetty metrics 31 | 32 | boolean 33 | 34 | 35 | 36 | 37 | solr_jvm_metrics 38 | true 39 | Enable Solr JVM metrics 40 | Solr JVM stats 41 | 42 | boolean 43 | 44 | 45 | 46 | 47 | solr_node_metrics 48 | true 49 | Enable Solr Node metrics 50 | Solr Node stats 51 | 52 | boolean 53 | 54 | 55 | 56 | 57 | solr_metrics_config_conf_dir 58 | {solr_package_dir}/metrics/conf 59 | Solr Metrics conf directory 60 | Solr Metrics configuration directory 61 | 62 | 63 | 64 | solr_metrics_config_pid_dir 65 | {solr_package_dir}/metrics/var 66 | Solr Metrics PID directory 67 | Solr Metrics PID directory 68 | 69 | 70 | 71 | solr_metrics_config_log_dir 72 | {solr_package_dir}/metrics/log 73 | Solr Metrics log directory 74 | Solr Metrics log directory 75 | 76 | 77 | 78 | solr_metrics_delay 79 | 500 80 | 81 | Time in milliseconds before task is to be executed. 82 | Default value is 500 milliseconds. 83 | 84 | 85 | int 86 | 87 | Solr Metrics delay 88 | 89 | 90 | 91 | solr_metrics_period 92 | 5000 93 | 94 | Time in milliseconds between successive task executions. 95 | Default value is 5000 milliseconds / 5 seconds. 96 | 97 | 98 | int 99 | 100 | Solr Metrics period 101 | 102 | 103 | 104 | solr_metrics_properties 105 | This is the jinja template for solr.metrics.properties file 106 | 107 | # Security 108 | truststore.path={{metric_truststore_path}} 109 | truststore.type={{metric_truststore_type}} 110 | truststore.password={{metric_truststore_password}} 111 | 112 | solr.truststore.path={{solr_ssl_trust_store}} 113 | solr.truststore.password={{solr_ssl_trust_store_password}} 114 | 115 | jaas.file={{solr_metrics_kerberos_jaas_config}} 116 | 117 | # MetricsLoader 118 | metrics.delay={{solr_metrics_delay}} 119 | metrics.period={{solr_metrics_period}} 120 | 121 | metrics.json= 122 | metrics.core.metrics={{solr_core_metrics}} 123 | metrics.jetty.metrics={{solr_jetty_metrics}} 124 | metrics.jvm.metrics={{solr_jvm_metrics}} 125 | metrics.node.metrics={{solr_node_metrics}} 126 | 127 | # Collector 128 | collector.hosts={{ams_collector_hosts}} 129 | collector.port={{metric_collector_port}} 130 | collector.protocol={{metric_collector_protocol}} 131 | 132 | # Solr 133 | solr.host={{solr_protocol}}://{{hostname}}:{{solr_config_port}}/solr 134 | solr.retry.exponential.wait=3 135 | solr.retry.stop.after.delay=2 136 | 137 | 138 | content 139 | 140 | 141 | 142 | 143 | solr_metrics_log4j2 144 | This is the jinja template for log4j2.json file 145 | 146 | { 147 | "configuration": { 148 | "name": "Default", 149 | "properties": { 150 | "property": [ 151 | { 152 | "name": "directory", 153 | "value": "{{solr_metrics_config_log_dir}}" 154 | }, 155 | { 156 | "name": "log_name", 157 | "value": "solr-metrics" 158 | } 159 | ] 160 | }, 161 | "appenders": { 162 | "RollingFile": { 163 | "name": "file", 164 | "fileName": "${directory}/${log_name}.log", 165 | "filePattern": "${directory}/${log_name}.%d{yyyy-MM-dd}-%i.log.gz", 166 | "PatternLayout": { 167 | "pattern": "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %X - %m%n" 168 | }, 169 | "Policies": { 170 | "TimeBasedTriggeringPolicy": { 171 | }, 172 | "SizeBasedTriggeringPolicy": { 173 | "size": "100MB" 174 | } 175 | }, 176 | "DefaultRolloverStrategy": { 177 | "delete": { 178 | "basePath": "${directory}", 179 | "maxDepth": 1, 180 | "ifFileName": { 181 | "glob": "${log_name}*.log.gz", 182 | "ifAny": { 183 | "ifLastModified": { 184 | "age": "30d" 185 | } 186 | } 187 | } 188 | } 189 | } 190 | } 191 | }, 192 | "loggers": { 193 | "logger": [ 194 | { 195 | "name": "com.lucidworks.ambari", 196 | "level": "INFO" 197 | }, 198 | { 199 | "name": "org.apache.zookeeper", 200 | "level": "WARN" 201 | } 202 | ], 203 | "root": { 204 | "level": "INFO", 205 | "appender-ref": [ 206 | { 207 | "ref": "file" 208 | } 209 | ] 210 | } 211 | } 212 | } 213 | } 214 | 215 | 216 | content 217 | 218 | 219 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/configuration/solr-ssl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_ssl_key_store_type 7 | JKS 8 | SSL keystore type 9 | SSL keystore type 10 | 11 | 12 | 13 | solr_ssl_trust_store_type 14 | JKS 15 | SSL Trust store type 16 | SSL Trust store type 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/dashboards/grafana-dashboards/grafana-solr-662-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": null, 3 | "title": "Solr 6.6.2 - Hosts", 4 | "originalTitle": "Solr 6.6.2 - Hosts", 5 | "tags": [ 6 | "solr", 7 | "solr-6.6.2" 8 | ], 9 | "style": "dark", 10 | "timezone": "browser", 11 | "editable": true, 12 | "hideControls": false, 13 | "sharedCrosshair": false, 14 | "rows": [ 15 | { 16 | "collapse": false, 17 | "editable": true, 18 | "height": "25px", 19 | "panels": [ 20 | { 21 | "content": "

Solr Hosts Dashboard

", 22 | "editable": true, 23 | "error": false, 24 | "id": 3, 25 | "isNew": true, 26 | "links": [], 27 | "mode": "html", 28 | "span": 12, 29 | "style": {}, 30 | "title": "", 31 | "type": "text" 32 | } 33 | ], 34 | "title": "New row" 35 | }, 36 | { 37 | "collapse": false, 38 | "editable": true, 39 | "height": "250px", 40 | "panels": [ 41 | { 42 | "aliasColors": {}, 43 | "bars": false, 44 | "datasource": null, 45 | "editable": true, 46 | "error": false, 47 | "fill": 1, 48 | "grid": { 49 | "leftLogBase": 1, 50 | "leftMax": null, 51 | "leftMin": null, 52 | "rightLogBase": 1, 53 | "rightMax": null, 54 | "rightMin": null, 55 | "threshold1": null, 56 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 57 | "threshold2": null, 58 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 59 | }, 60 | "id": 1, 61 | "isNew": true, 62 | "legend": { 63 | "avg": false, 64 | "current": false, 65 | "max": false, 66 | "min": false, 67 | "show": true, 68 | "total": false, 69 | "values": false 70 | }, 71 | "lines": true, 72 | "linewidth": 2, 73 | "links": [], 74 | "nullPointMode": "connected", 75 | "percentage": false, 76 | "pointradius": 5, 77 | "points": false, 78 | "renderer": "flot", 79 | "seriesOverrides": [], 80 | "span": 6, 81 | "stack": false, 82 | "steppedLine": false, 83 | "targets": [ 84 | { 85 | "aggregator": "none", 86 | "app": "solr-host-app", 87 | "downsampleAggregator": "avg", 88 | "errors": {}, 89 | "metric": "solr.jvm.gauge.os.processCpuLoad", 90 | "precision": "default", 91 | "refId": "A", 92 | "seriesAggregator": "none", 93 | "templatedHost": "", 94 | "transform": "none" 95 | } 96 | ], 97 | "timeFrom": null, 98 | "timeShift": null, 99 | "title": "Panel Title", 100 | "tooltip": { 101 | "shared": true, 102 | "value_type": "cumulative" 103 | }, 104 | "type": "graph", 105 | "x-axis": true, 106 | "y-axis": true, 107 | "y_formats": [ 108 | "percent", 109 | "none" 110 | ] 111 | }, 112 | { 113 | "aliasColors": {}, 114 | "bars": false, 115 | "datasource": null, 116 | "editable": true, 117 | "error": false, 118 | "fill": 1, 119 | "grid": { 120 | "leftLogBase": 1, 121 | "leftMax": null, 122 | "leftMin": null, 123 | "rightLogBase": 1, 124 | "rightMax": null, 125 | "rightMin": null, 126 | "threshold1": null, 127 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 128 | "threshold2": null, 129 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 130 | }, 131 | "id": 2, 132 | "isNew": true, 133 | "legend": { 134 | "avg": false, 135 | "current": false, 136 | "max": false, 137 | "min": false, 138 | "show": true, 139 | "total": false, 140 | "values": false 141 | }, 142 | "lines": true, 143 | "linewidth": 2, 144 | "links": [], 145 | "nullPointMode": "connected", 146 | "percentage": false, 147 | "pointradius": 5, 148 | "points": false, 149 | "renderer": "flot", 150 | "seriesOverrides": [], 151 | "span": 6, 152 | "stack": false, 153 | "steppedLine": false, 154 | "targets": [ 155 | { 156 | "aggregator": "none", 157 | "app": "solr-host-app", 158 | "downsampleAggregator": "avg", 159 | "errors": {}, 160 | "metric": "solr.jvm.gauge.memory.total.used", 161 | "precision": "default", 162 | "refId": "A", 163 | "seriesAggregator": "none", 164 | "templatedHost": "", 165 | "transform": "none" 166 | } 167 | ], 168 | "timeFrom": null, 169 | "timeShift": null, 170 | "title": "Panel Title", 171 | "tooltip": { 172 | "shared": true, 173 | "value_type": "cumulative" 174 | }, 175 | "type": "graph", 176 | "x-axis": true, 177 | "y-axis": true, 178 | "y_formats": [ 179 | "bytes", 180 | "bytes" 181 | ] 182 | } 183 | ], 184 | "title": "Row" 185 | } 186 | ], 187 | "time": { 188 | "from": "now-1h", 189 | "to": "now" 190 | }, 191 | "timepicker": { 192 | "now": true, 193 | "refresh_intervals": [ 194 | "5s", 195 | "10s", 196 | "30s", 197 | "1m", 198 | "5m", 199 | "15m", 200 | "30m", 201 | "1h", 202 | "2h", 203 | "1d" 204 | ], 205 | "time_options": [ 206 | "5m", 207 | "15m", 208 | "1h", 209 | "6h", 210 | "12h", 211 | "24h", 212 | "2d", 213 | "7d", 214 | "30d" 215 | ] 216 | }, 217 | "templating": { 218 | "list": [ 219 | { 220 | "allFormat": "glob", 221 | "current": { 222 | "text": "solr-host-app", 223 | "value": "solr-host-app" 224 | }, 225 | "datasource": null, 226 | "includeAll": false, 227 | "multi": false, 228 | "multiFormat": "glob", 229 | "name": "components", 230 | "options": [ 231 | { 232 | "selected": true, 233 | "text": "solr-host-app", 234 | "value": "solr-host-app" 235 | } 236 | ], 237 | "query": "solr-host-app", 238 | "refresh": false, 239 | "regex": "", 240 | "type": "custom" 241 | }, 242 | { 243 | "allFormat": "glob", 244 | "current": { 245 | }, 246 | "datasource": null, 247 | "includeAll": false, 248 | "multi": false, 249 | "multiFormat": "glob", 250 | "name": "hosts", 251 | "options": [ 252 | 253 | ], 254 | "query": "hosts", 255 | "refresh": true, 256 | "regex": "", 257 | "type": "query" 258 | } 259 | ] 260 | }, 261 | "annotations": { 262 | "list": [] 263 | }, 264 | "schemaVersion": 8, 265 | "version": 2, 266 | "links": [] 267 | } -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 2.0 18 | 19 | 20 | SOLR 21 | 6.6.2 22 | common-services/SOLR/5.5.5 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/alerts/alert_solr_cpu_metrics.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import httplib 3 | import json 4 | import time 5 | 6 | from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames 7 | from ambari_commons.ambari_metrics_helper import load_properties_from_file 8 | from resource_management.libraries.functions.format import format 9 | 10 | RESULT_STATE_OK = 'OK' 11 | RESULT_STATE_CRITICAL = 'CRITICAL' 12 | RESULT_STATE_WARNING = 'WARNING' 13 | RESULT_STATE_UNKNOWN = 'UNKNOWN' 14 | RESULT_STATE_SKIPPED = 'SKIPPED' 15 | 16 | KERBEROS_KEYTAB = '{{hdfs-site/dfs.web.authentication.kerberos.keytab}}' 17 | KERBEROS_PRINCIPAL = '{{hdfs-site/dfs.web.authentication.kerberos.principal}}' 18 | SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' 19 | SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' 20 | 21 | METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' 22 | METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' 23 | METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' 24 | 25 | SOLR_METRICS_CONF_DIR = '{{solr-metrics/solr_metrics_config_conf_dir}}' 26 | SOLR_METRICS_PERIOD = '{{solr-metrics/solr_metrics_period}}' 27 | 28 | AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s" 29 | 30 | CONNECTION_TIMEOUT_DEFAULT = 5.0 31 | CONNECTION_TIMEOUT_KEY = "connection.timeout" 32 | 33 | METRIC_NAMES_DEFAULT = "solr.jvm.gauge.os.processCpuLoad" 34 | METRIC_NAMES_KEY = "metric.names" 35 | 36 | APP_ID_DEFAULT = "solr-host-app" 37 | APP_ID_KEY = "app.id" 38 | 39 | WARNING_THRESHOLD_DEFAULT = 50 40 | WARNING_THRESHOLD_KEY = "metric.solr.cpu.warning.threshold" 41 | 42 | CRITICAL_THRESHOLD_DEFAULT = 75 43 | CRITICAL_THRESHOLD_KEY = "metric.solr.cpu.critical.threshold" 44 | 45 | 46 | def get_tokens(): 47 | """ 48 | Returns a tuple of tokens in the format {{site/property}} that will be used 49 | to build the dictionary passed into execute 50 | """ 51 | return (SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, METRICS_COLLECTOR_VIP_HOST_KEY, 52 | METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, SOLR_METRICS_CONF_DIR, 53 | SOLR_METRICS_PERIOD) 54 | 55 | 56 | def execute(configurations={}, parameters={}, host_name=None): 57 | collector_host, collector_port = get_collector_config(configurations) 58 | 59 | if collector_host is None or collector_port is None: 60 | return RESULT_STATE_UNKNOWN, ['Undefined collector host: {0} or collector port {1}'.format(collector_host, 61 | collector_port)] 62 | 63 | metric_name = METRIC_NAMES_DEFAULT 64 | if METRIC_NAMES_KEY in parameters: 65 | metric_name = parameters[METRIC_NAMES_KEY] 66 | 67 | app_id = APP_ID_DEFAULT 68 | if APP_ID_KEY in parameters: 69 | app_id = parameters[APP_ID_KEY] 70 | 71 | connection_timeout = CONNECTION_TIMEOUT_DEFAULT 72 | if CONNECTION_TIMEOUT_KEY in parameters: 73 | connection_timeout = parameters[CONNECTION_TIMEOUT_KEY] 74 | 75 | warning_threshold = WARNING_THRESHOLD_DEFAULT 76 | if WARNING_THRESHOLD_KEY in parameters: 77 | warning_threshold = parameters[WARNING_THRESHOLD_KEY] 78 | 79 | critical_threshold = CRITICAL_THRESHOLD_DEFAULT 80 | if CRITICAL_THRESHOLD_KEY in parameters: 81 | critical_threshold = parameters[CRITICAL_THRESHOLD_KEY] 82 | 83 | get_metrics_parameters = { 84 | "metricNames": metric_name, 85 | "appId": app_id, 86 | "hostname": host_name, 87 | "grouped": "true", 88 | } 89 | 90 | encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters) 91 | 92 | try: 93 | conn = httplib.HTTPConnection(collector_host, int(collector_port), timeout=float(connection_timeout)) 94 | conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters) 95 | response = conn.getresponse() 96 | data = response.read() 97 | conn.close() 98 | except Exception as e: 99 | message = "Unable to retrieve metrics from the Ambari Metrics service. Error: {}".format(str(e)) 100 | return RESULT_STATE_UNKNOWN, [message] 101 | 102 | data_json = json.loads(data) 103 | timestamp = -1 104 | 105 | for metrics_data in data_json["metrics"]: 106 | if metric_name in metrics_data["metricname"]: 107 | metrics = metrics_data["metrics"].values() 108 | timestamp = metrics_data["timestamp"] 109 | 110 | if int(timestamp) == -1: 111 | return RESULT_STATE_UNKNOWN, ["There is not enough data"] 112 | 113 | current_time = int(time.time()) * 1000 114 | metric_period_ms = 10 * 1000 * 60 115 | difference = current_time - timestamp 116 | 117 | if difference > metric_period_ms: 118 | message = "Data retrieved is older than 10 minutes, please check the Solr node, and confirm that the Solr " \ 119 | "ambari metrics is running" 120 | return RESULT_STATE_WARNING, [message] 121 | 122 | cpu_load_value = metrics[0] * 100 123 | response = 'CPU load {0:.2f} %'.format(cpu_load_value) 124 | 125 | if int(cpu_load_value) >= warning_threshold and int(cpu_load_value) < critical_threshold: 126 | return RESULT_STATE_WARNING, [response] 127 | 128 | if cpu_load_value >= critical_threshold: 129 | return RESULT_STATE_CRITICAL, [response] 130 | 131 | return RESULT_STATE_OK, [response] 132 | 133 | 134 | def get_collector_config(configurations): 135 | solr_package_dir = '/opt/lucidworks-hdpsearch' 136 | solr_metrics_config_conf_dir = format(configurations[SOLR_METRICS_CONF_DIR]) 137 | solr_metrics_props = "{0}/{1}".format(solr_metrics_config_conf_dir, "solr.metrics.properties") 138 | props = load_properties_from_file(solr_metrics_props) 139 | collector_hosts = props.get("collector.hosts") 140 | 141 | if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: 142 | collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] 143 | collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) 144 | else: 145 | collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") 146 | if valid_collector_webapp_address(collector_webapp_address): 147 | collector_host = select_metric_collector_hosts_from_hostnames(collector_hosts) 148 | collector_port = int(collector_webapp_address[1]) 149 | else: 150 | return None, None 151 | return collector_host, collector_port 152 | 153 | 154 | def valid_collector_webapp_address(webapp_address): 155 | if len(webapp_address) == 2 and webapp_address[0] != '127.0.0.1' and webapp_address[1].isdigit(): 156 | return True 157 | return False 158 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/alerts/alert_solr_memory_metrics.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import httplib 3 | import json 4 | import time 5 | 6 | from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames 7 | from ambari_commons.ambari_metrics_helper import load_properties_from_file 8 | from resource_management.libraries.functions.format import format 9 | 10 | RESULT_STATE_OK = 'OK' 11 | RESULT_STATE_CRITICAL = 'CRITICAL' 12 | RESULT_STATE_WARNING = 'WARNING' 13 | RESULT_STATE_UNKNOWN = 'UNKNOWN' 14 | RESULT_STATE_SKIPPED = 'SKIPPED' 15 | 16 | KERBEROS_KEYTAB = '{{hdfs-site/dfs.web.authentication.kerberos.keytab}}' 17 | KERBEROS_PRINCIPAL = '{{hdfs-site/dfs.web.authentication.kerberos.principal}}' 18 | SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' 19 | SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' 20 | 21 | METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' 22 | METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' 23 | METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' 24 | 25 | SOLR_METRICS_CONF_DIR = '{{solr-metrics/solr_metrics_config_conf_dir}}' 26 | SOLR_METRICS_PERIOD = "{{solr-metrics/solr_metrics_period}}" 27 | 28 | AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s" 29 | 30 | CONNECTION_TIMEOUT_DEFAULT = 5.0 31 | CONNECTION_TIMEOUT_KEY = "connection.timeout" 32 | 33 | METRIC_NAMES_DEFAULT = "solr.jvm.gauge.memory.total.used,solr.jvm.gauge.memory.total.max" 34 | METRIC_NAMES_KEY = "metric.names" 35 | 36 | APP_ID_DEFAULT = "solr-host-app" 37 | APP_ID_KEY = "app.id" 38 | 39 | WARNING_THRESHOLD_DEFAULT = 50 40 | WARNING_THRESHOLD_KEY = "metric.solr.memory.warning.threshold" 41 | 42 | CRITICAL_THRESHOLD_DEFAULT = 75 43 | CRITICAL_THRESHOLD_KEY = "metric.solr.memory.critical.threshold" 44 | 45 | def get_tokens(): 46 | """ 47 | Returns a tuple of tokens in the format {{site/property}} that will be used 48 | to build the dictionary passed into execute 49 | """ 50 | return (SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, METRICS_COLLECTOR_VIP_HOST_KEY, 51 | METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, SOLR_METRICS_CONF_DIR, 52 | SOLR_METRICS_PERIOD) 53 | 54 | 55 | def execute(configurations={}, parameters={}, host_name=None): 56 | collector_host, collector_port = get_collector_config(configurations) 57 | 58 | if collector_host is None or collector_port is None: 59 | return RESULT_STATE_UNKNOWN, ['Undefined collector host: {0} or collector port {1}'.format(collector_host, 60 | collector_port)] 61 | 62 | metric_name = METRIC_NAMES_DEFAULT 63 | if METRIC_NAMES_KEY in parameters: 64 | metric_name = parameters[METRIC_NAMES_KEY] 65 | 66 | app_id = APP_ID_DEFAULT 67 | if APP_ID_KEY in parameters: 68 | app_id = parameters[APP_ID_KEY] 69 | 70 | connection_timeout = CONNECTION_TIMEOUT_DEFAULT 71 | if CONNECTION_TIMEOUT_KEY in parameters: 72 | connection_timeout = parameters[CONNECTION_TIMEOUT_KEY] 73 | 74 | warning_threshold = WARNING_THRESHOLD_DEFAULT 75 | if WARNING_THRESHOLD_KEY in parameters: 76 | warning_threshold = parameters[WARNING_THRESHOLD_KEY] 77 | 78 | critical_threshold = CRITICAL_THRESHOLD_DEFAULT 79 | if CRITICAL_THRESHOLD_KEY in parameters: 80 | critical_threshold = parameters[CRITICAL_THRESHOLD_KEY] 81 | 82 | get_metrics_parameters = { 83 | "metricNames": metric_name, 84 | "appId": app_id, 85 | "hostname": host_name, 86 | "grouped": "true", 87 | } 88 | 89 | encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters) 90 | 91 | try: 92 | conn = httplib.HTTPConnection(collector_host, int(collector_port), timeout=float(connection_timeout)) 93 | conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters) 94 | response = conn.getresponse() 95 | data = response.read() 96 | conn.close() 97 | except Exception as e: 98 | message = "Unable to retrieve metrics from the Ambari Metrics service. Error: {}".format(str(e)) 99 | return RESULT_STATE_UNKNOWN, [message] 100 | 101 | data_json = json.loads(data) 102 | 103 | split = metric_name.split(",") 104 | metric_name_map = { 105 | "used" if ("used" in split[0]) else "max": split[0], 106 | "used" if ("used" in split[1]) else "max": split[1] 107 | } 108 | 109 | timestamp = -1 110 | metrics = {} 111 | 112 | for metrics_data in data_json["metrics"]: 113 | if metric_name_map["used"] in metrics_data["metricname"]: 114 | metrics["used"] = metrics_data["metrics"].values()[0] 115 | current_timestamp = metrics_data["timestamp"] 116 | if timestamp == -1: 117 | timestamp = current_timestamp 118 | elif timestamp > current_timestamp: 119 | timestamp = current_timestamp 120 | continue 121 | if metric_name_map["max"] in metrics_data["metricname"]: 122 | metrics["max"] = metrics_data["metrics"].values()[0] 123 | current_timestamp = metrics_data["timestamp"] 124 | if timestamp == -1: 125 | timestamp = current_timestamp 126 | elif timestamp > current_timestamp: 127 | timestamp = current_timestamp 128 | continue 129 | 130 | if int(timestamp) == -1: 131 | return RESULT_STATE_UNKNOWN, ["There is not enough data"] 132 | 133 | current_time = int(time.time()) * 1000 134 | metric_period_ms = 10 * 1000 * 60 135 | difference = current_time - timestamp 136 | 137 | if difference > metric_period_ms: 138 | message = "Data retrieved is older than 10 minutes, please check the Solr node, and confirm that the Solr " \ 139 | "ambari metrics is running" 140 | return RESULT_STATE_WARNING, [message] 141 | 142 | memory_used = metrics["used"] / (1024 * 1024) 143 | memory_max = metrics["max"] / (1024 * 1024) 144 | memory_value = memory_used / memory_max 145 | response = 'Memory usage is {0:.2f} %'.format(memory_value) 146 | 147 | if int(memory_value) >= warning_threshold and int(memory_value) < critical_threshold: 148 | return RESULT_STATE_WARNING, [response] 149 | 150 | if memory_value >= critical_threshold: 151 | return RESULT_STATE_CRITICAL, [response] 152 | 153 | return RESULT_STATE_OK, [response] 154 | 155 | 156 | def get_collector_config(configurations): 157 | solr_package_dir = '/opt/lucidworks-hdpsearch' 158 | solr_metrics_config_conf_dir = format(configurations[SOLR_METRICS_CONF_DIR]) 159 | solr_metrics_props = "{0}/{1}".format(solr_metrics_config_conf_dir, "solr.metrics.properties") 160 | props = load_properties_from_file(solr_metrics_props) 161 | collector_hosts = props.get("collector.hosts") 162 | 163 | if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: 164 | collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] 165 | collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) 166 | else: 167 | collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") 168 | if valid_collector_webapp_address(collector_webapp_address): 169 | collector_host = select_metric_collector_hosts_from_hostnames(collector_hosts) 170 | collector_port = int(collector_webapp_address[1]) 171 | else: 172 | return None, None 173 | return collector_host, collector_port 174 | 175 | 176 | def valid_collector_webapp_address(webapp_address): 177 | if len(webapp_address) == 2 and webapp_address[0] != '127.0.0.1' and webapp_address[1].isdigit(): 178 | return True 179 | return False 180 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/service_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from resource_management.core.logger import Logger 6 | from resource_management.core.resources.system import Execute 7 | from resource_management.libraries.functions.format import format 8 | from resource_management.libraries.script import Script 9 | 10 | from solr_utils import exists_collection 11 | 12 | 13 | class ServiceCheck(Script): 14 | def service_check(self, env): 15 | import params 16 | env.set_params(params) 17 | 18 | if not os.path.isfile(params.solr_config_pid_file): 19 | Logger.error(format("PID file {solr_config_pid_file} does not exist")) 20 | exit(1) 21 | 22 | if not params.solr_collection_sample_create: 23 | Logger.info("Create sample collection unchecked, skipping ...") 24 | return 25 | 26 | if exists_collection(params.solr_collection_name): 27 | Logger.warning(format("Collection {solr_collection_name} already exists, skipping ...")) 28 | return 29 | 30 | if not params.solr_cloud_mode: 31 | Execute( 32 | format( 33 | '{solr_config_bin_dir}/solr create_core -c {solr_collection_name}' + 34 | ' -d {solr_collection_config_dir} -p {solr_config_port} >> {solr_config_service_log_file} 2>&1' 35 | ), 36 | environment={'JAVA_HOME': params.java64_home}, 37 | user=params.solr_config_user 38 | ) 39 | else: 40 | Execute(format( 41 | '{solr_config_bin_dir}/solr create_collection -c {solr_collection_name}' + 42 | ' -d {solr_collection_config_dir} -p {solr_config_port}' + 43 | ' -s {solr_collection_shards} -rf {solr_collection_replicas}' + 44 | ' >> {solr_config_service_log_file} 2>&1'), 45 | environment={'JAVA_HOME': params.java64_home}, 46 | user=params.solr_config_user 47 | ) 48 | 49 | 50 | if __name__ == "__main__": 51 | ServiceCheck().execute() 52 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/setup_solr.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Directory, File, Execute 2 | from resource_management.core.source import Template, InlineTemplate 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr(): 7 | import params 8 | 9 | Directory( 10 | [ 11 | params.solr_config_dir, 12 | params.solr_config_log_dir, 13 | params.solr_config_service_log_dir, 14 | params.solr_config_pid_dir, 15 | params.solr_config_conf_dir, 16 | params.solr_config_home_dir 17 | ], 18 | mode=0755, 19 | cd_access='a', 20 | owner=params.solr_config_user, 21 | group=params.solr_config_group, 22 | create_parents=True 23 | ) 24 | 25 | Execute( 26 | ('chmod', '-R', '777', params.solr_webapp_dir), 27 | sudo=True 28 | ) 29 | 30 | File( 31 | format("{solr_config_bin_dir}/solr.in.sh"), 32 | content=InlineTemplate(params.solr_config_in_sh), 33 | owner=params.solr_config_user 34 | ) 35 | 36 | File( 37 | format("{solr_config_conf_dir}/log4j.properties"), 38 | content=InlineTemplate(params.log4j_properties), 39 | owner=params.solr_config_user 40 | ) 41 | 42 | File( 43 | format("{solr_config_home_dir}/solr.xml"), 44 | content=InlineTemplate(params.solr_xml_template), 45 | owner=params.solr_config_user 46 | ) 47 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/setup_solr_cloud.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute 3 | from resource_management.core.shell import call 4 | from resource_management.libraries.functions.format import format 5 | 6 | 7 | def setup_solr_cloud(): 8 | import params 9 | 10 | code, output = call( 11 | format( 12 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{clusterstate_json}' 13 | ), 14 | env={'JAVA_HOME': params.java64_home}, 15 | timeout=60 16 | ) 17 | 18 | if not ("NoNodeException" in output): 19 | Logger.info( 20 | format( 21 | "ZK node {solr_cloud_zk_directory}{clusterstate_json} already exists, skipping ..." 22 | ) 23 | ) 24 | return 25 | 26 | Execute( 27 | format( 28 | '{zk_client_prefix} -cmd makepath {solr_cloud_zk_directory}' 29 | ), 30 | environment={'JAVA_HOME': params.java64_home}, 31 | ignore_failures=True, 32 | user=params.solr_config_user 33 | ) 34 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/setup_solr_hdfs_support.py: -------------------------------------------------------------------------------- 1 | def setup_solr_hdfs_support(): 2 | import params 3 | 4 | # Create a home directory for solr user 5 | params.HdfsResource( 6 | params.solr_hdfs_home_directory, 7 | type="directory", 8 | action="create_on_execute", 9 | owner=params.solr_config_user 10 | ) 11 | 12 | params.HdfsResource(params.solr_hdfs_directory, 13 | type="directory", 14 | action="create_on_execute", 15 | owner=params.solr_config_user 16 | ) 17 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/setup_solr_kerberos_auth.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute, File 3 | from resource_management.core.shell import call 4 | from resource_management.core.source import Template 5 | from resource_management.libraries.functions.format import format 6 | 7 | 8 | def setup_solr_kerberos_auth(): 9 | import params 10 | 11 | File(format("{solr_kerberos_jaas_config}"), 12 | content=Template("solr_server_jaas.conf.j2"), 13 | owner=params.solr_config_user 14 | ) 15 | 16 | if _has_security_json(): 17 | Logger.info("Solr Security Json was found, it will not be overridden") 18 | return 19 | 20 | command = format('{zk_client_prefix} -cmd put {solr_cloud_zk_directory}{security_json} ') 21 | command += format('\'{solr_security_json}\'') 22 | Execute(command, 23 | environment={'JAVA_HOME': params.java64_home}, 24 | ignore_failures=True, 25 | user=params.solr_config_user 26 | ) 27 | 28 | 29 | def remove_solr_kerberos_auth(): 30 | import params 31 | 32 | if not _has_security_json(): 33 | Logger.debug( 34 | format("Solr Security Json not found {solr_cloud_zk_directory}{security_json}")) 35 | return 36 | 37 | Execute(format('{zk_client_prefix} -cmd clear {solr_cloud_zk_directory}{security_json}'), 38 | environment={'JAVA_HOME': params.java64_home}, 39 | timeout=60, 40 | ignore_failures=True, 41 | user=params.solr_config_user 42 | ) 43 | 44 | 45 | def _has_security_json(): 46 | import params 47 | 48 | if not params.solr_cloud_mode: 49 | return False 50 | 51 | code, output = call( 52 | format('{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{security_json}'), 53 | env={'JAVA_HOME': params.java64_home}, 54 | timeout=60 55 | ) 56 | 57 | if "{}" == output: 58 | return False 59 | return "NoNodeException" not in output 60 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/setup_solr_metrics.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Directory, File 2 | from resource_management.core.source import InlineTemplate, Template 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr_metrics_support(): 7 | import params 8 | 9 | Directory( 10 | [ 11 | params.solr_metrics_config_conf_dir, 12 | params.solr_metrics_config_pid_dir, 13 | params.solr_metrics_config_log_dir 14 | ], 15 | mode=0755, 16 | cd_access='a', 17 | owner=params.solr_config_user, 18 | group=params.solr_config_group, 19 | create_parents=True 20 | ) 21 | 22 | File( 23 | format("{solr_metrics_config_conf_dir}/solr.metrics.properties"), 24 | content=InlineTemplate(params.solr_metrics_properties), 25 | owner=params.solr_config_user 26 | ) 27 | 28 | File( 29 | format("{solr_metrics_config_conf_dir}/log4j2.json"), 30 | content=InlineTemplate(params.solr_metrics_log4j2), 31 | owner=params.solr_config_user 32 | ) 33 | 34 | if params.security_enabled: 35 | File( 36 | format("{solr_metrics_kerberos_jaas_config}"), 37 | content=Template("solr_metrics_jaas.conf.j2"), 38 | owner=params.solr_config_user 39 | ) 40 | 41 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/setup_solr_ssl_support.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Execute 2 | from resource_management.core.shell import call 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr_ssl_support(): 7 | import params 8 | 9 | if not params.solr_cloud_mode: 10 | return 11 | 12 | Execute( 13 | format( 14 | '{zk_client_prefix}{solr_cloud_zk_directory} -cmd clusterprop -name urlScheme -val https' 15 | ), 16 | environment={'JAVA_HOME': params.java64_home}, 17 | ignore_failures=True, 18 | user=params.solr_config_user 19 | ) 20 | 21 | 22 | def remove_solr_ssl_support(): 23 | import params 24 | 25 | if not params.solr_cloud_mode: 26 | return 27 | 28 | code, output = call( 29 | format( 30 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{clusterprops_json}' 31 | ), 32 | env={'JAVA_HOME': params.java64_home}, 33 | timeout=60 34 | ) 35 | 36 | if "NoNodeException" in output: 37 | return 38 | 39 | Execute( 40 | format( 41 | '{zk_client_prefix} -cmd clear {solr_cloud_zk_directory}{clusterprops_json}' 42 | ), 43 | environment={'JAVA_HOME': params.java64_home}, 44 | ignore_failures=True, 45 | user=params.solr_config_user 46 | ) 47 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/solr.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute, File 3 | from resource_management.libraries.functions.check_process_status import check_process_status 4 | from resource_management.libraries.functions.format import format 5 | from resource_management.libraries.script.script import Script 6 | 7 | from setup_solr import setup_solr 8 | from setup_solr_cloud import setup_solr_cloud 9 | from setup_solr_hdfs_support import setup_solr_hdfs_support 10 | from setup_solr_kerberos_auth import setup_solr_kerberos_auth, remove_solr_kerberos_auth 11 | from setup_solr_metrics import setup_solr_metrics_support 12 | from setup_solr_ssl_support import setup_solr_ssl_support, remove_solr_ssl_support 13 | from solr_utils import is_solr_running, solr_port_validation, delete_write_lock_files 14 | 15 | 16 | class Solr(Script): 17 | def install(self, env): 18 | import params 19 | env.set_params(params) 20 | self.install_packages(env) 21 | 22 | def configure(self, env): 23 | import params 24 | env.set_params(params) 25 | setup_solr() 26 | 27 | if params.solr_cloud_mode: 28 | setup_solr_cloud() 29 | 30 | if params.solr_hdfs_enable: 31 | setup_solr_hdfs_support() 32 | 33 | if params.solr_ssl_enable: 34 | setup_solr_ssl_support() 35 | else: 36 | remove_solr_ssl_support() 37 | 38 | if params.security_enabled: 39 | setup_solr_kerberos_auth() 40 | else: 41 | remove_solr_kerberos_auth() 42 | 43 | if params.solr_hdfs_enable and params.solr_hdfs_delete_write_lock_files: 44 | delete_write_lock_files() 45 | 46 | if params.has_metric_collector and params.solr_enable_metrics: 47 | setup_solr_metrics_support() 48 | 49 | def start(self, env): 50 | import params 51 | env.set_params(params) 52 | self.configure(env) 53 | 54 | if not solr_port_validation(): 55 | exit(1) 56 | 57 | if is_solr_running(): 58 | Logger.info("Solr is running, it can not be started it again") 59 | exit(1) 60 | 61 | Logger.info("Starting Solr ... ") 62 | start_command = format('{solr_config_bin_dir}/solr start >> {solr_config_service_log_file} 2>&1') 63 | 64 | Execute( 65 | start_command, 66 | environment={'JAVA_HOME': params.java64_home}, 67 | user=params.solr_config_user 68 | ) 69 | 70 | if params.has_metric_collector and params.solr_enable_metrics: 71 | Logger.info('Starting Solr Metrics Sink.') 72 | 73 | if params.security_enabled: 74 | Execute( 75 | format('{kinit_path_local} -kt {solr_metrics_kerberos_keytab} {solr_metrics_kerberos_principal}'), 76 | user=params.solr_config_user 77 | ) 78 | 79 | Execute( 80 | format('{solr_metrics_sink_bin}/solr.metrics.sh start'), 81 | environment={ 82 | 'JAVA_HOME': params.java64_home, 83 | 'SOLR_METRICS_PID_DIR': params.solr_metrics_config_pid_dir, 84 | 'SOLR_METRICS_LOGS': params.solr_metrics_config_log_dir, 85 | 'SOLR_METRIC_CONF': params.solr_metrics_config_conf_dir 86 | }, 87 | user=params.solr_config_user 88 | ) 89 | 90 | def stop(self, env): 91 | import params 92 | env.set_params(params) 93 | 94 | if params.has_metric_collector and params.solr_enable_metrics: 95 | Logger.info('Stopping Solr Metrics Sink.') 96 | 97 | if params.security_enabled: 98 | Execute( 99 | format('{kinit_path_local} -kt {solr_metrics_kerberos_keytab} {solr_metrics_kerberos_principal}'), 100 | user=params.solr_config_user 101 | ) 102 | 103 | Execute( 104 | format('{solr_metrics_sink_bin}/solr.metrics.sh stop'), 105 | environment={ 106 | 'JAVA_HOME': params.java64_home, 107 | 'SOLR_METRICS_PID_DIR': params.solr_metrics_config_pid_dir, 108 | 'SOLR_METRICS_LOGS': params.solr_metrics_config_log_dir, 109 | 'SOLR_METRIC_CONF': params.solr_metrics_config_conf_dir 110 | }, 111 | ignore_failures=True, 112 | user=params.solr_config_user 113 | ) 114 | 115 | if not is_solr_running(): 116 | Logger.info("Solr is not running, it can not be stopped it again") 117 | return 118 | 119 | Execute( 120 | format( 121 | '{solr_config_bin_dir}/solr stop -all >> {solr_config_service_log_file} 2>&1'), 122 | environment={'JAVA_HOME': params.java64_home}, 123 | user=params.solr_config_user 124 | ) 125 | 126 | File( 127 | params.solr_config_pid_file, 128 | action="delete" 129 | ) 130 | 131 | def status(self, env): 132 | import status_params 133 | env.set_params(status_params) 134 | 135 | check_process_status(status_params.solr_config_pid_file) 136 | 137 | 138 | if __name__ == "__main__": 139 | Solr().execute() 140 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/solr_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from resource_management.core.logger import Logger 5 | from resource_management.core.resources.system import Execute 6 | from resource_management.core.shell import call 7 | from resource_management.libraries.functions.format import format 8 | 9 | COLLECTION_PATTERN = "{solr_hdfs_directory}/[a-zA-Z0-9\._-]+" 10 | CORE_PATTERN = "{collection_path}\/core_node[0-9]+" 11 | WRITE_LOCK_PATTERN = "{0}/data/index/write.lock " 12 | HOSTNAME_VERIFIER_PATTERN = "{core_node_name}\":{{((?![^_]shard|core_node).)*\"node_name\":\"{solr_hostname}" 13 | 14 | 15 | def solr_port_validation(): 16 | code, output = call( 17 | format('netstat -lnt | awk -v v1={solr_config_port} \'$6 == "LISTEN" && $4 ~ ":"v1\''), 18 | timeout=60) 19 | Logger.info(format("Solr port validation output: {output}")) 20 | 21 | if "LISTEN" in output: 22 | Logger.error(format("The port {solr_config_port} is not available")) 23 | return False 24 | 25 | return True 26 | 27 | 28 | def is_solr_running(): 29 | import params 30 | 31 | code, output = call( 32 | format('{solr_config_bin_dir}/solr status'), 33 | env={'JAVA_HOME': params.java64_home}, 34 | timeout=60 35 | ) 36 | Logger.info(format("Solr status output: {output}")) 37 | pattern_port = re.compile(format('running on port {solr_config_port}')) 38 | 39 | if pattern_port.search(output): 40 | Logger.error(format("A Solr instance is running on port {solr_config_port}")) 41 | return True 42 | 43 | return False 44 | 45 | 46 | def exists_collection(collection_name): 47 | import params 48 | 49 | if not params.solr_cloud_mode: 50 | if os.path.isdir(format("{solr_config_home_dir}/{collection_name}")): 51 | return True 52 | return False 53 | 54 | code, output = call(format( 55 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}'), 56 | env={'JAVA_HOME': params.java64_home}, 57 | timeout=60 58 | ) 59 | 60 | if "NoNodeException" in output: 61 | return False 62 | return True 63 | 64 | 65 | def get_collection_paths(hadoop_output): 66 | import params 67 | pattern = re.compile(format(COLLECTION_PATTERN)) 68 | 69 | collection_paths = re.findall(pattern, hadoop_output) 70 | return collection_paths 71 | 72 | 73 | def get_core_paths(hadoop_output, collection_path): 74 | pattern = re.compile(format(CORE_PATTERN)) 75 | core_paths = re.findall(pattern, hadoop_output) 76 | return core_paths 77 | 78 | 79 | def get_write_lock_files_solr_cloud(hadoop_prefix, collections): 80 | import params 81 | 82 | write_locks_to_delete = '' 83 | 84 | for collection_path in collections: 85 | code, output = call(format('{hadoop_prefix} -ls {collection_path}')) 86 | core_paths = get_core_paths(output, collection_path) 87 | 88 | collection_name = collection_path.replace(format('{solr_hdfs_directory}/'), '') 89 | zk_code, zk_output = call(format( 90 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}/state.json'), 91 | env={'JAVA_HOME': params.java64_home}, 92 | timeout=60 93 | ) 94 | if zk_code != 0: 95 | Logger.error(format('Cannot determine cores owned by [{solr_hostname}] in collection [{collection_name}] due to ZK error.')) 96 | continue 97 | 98 | for core_path in core_paths: 99 | core_node_name = core_path.replace(format('{collection_path}/'), '') 100 | pattern = re.compile(format(HOSTNAME_VERIFIER_PATTERN), re.MULTILINE|re.DOTALL) 101 | core_on_hostname = re.search(pattern, zk_output) 102 | if core_on_hostname is not None: 103 | write_locks_to_delete += WRITE_LOCK_PATTERN.format(core_path) 104 | 105 | return write_locks_to_delete 106 | 107 | 108 | def get_write_lock_files_solr_standalone(collections): 109 | write_locks_to_delete = '' 110 | 111 | for collection_path in collections: 112 | write_locks_to_delete += WRITE_LOCK_PATTERN.format(collection_path) 113 | 114 | return write_locks_to_delete 115 | 116 | 117 | def delete_write_lock_files(): 118 | import params 119 | 120 | if params.security_enabled: 121 | kinit_if_needed = format( 122 | '{kinit_path_local} {hdfs_principal_name} -kt {hdfs_user_keytab}; ') 123 | else: 124 | kinit_if_needed = '' 125 | 126 | hadoop_prefix = format('{kinit_if_needed}hadoop --config {hadoop_conf_dir} dfs') 127 | code, output = call(format('{hadoop_prefix} -ls {solr_hdfs_directory}')) 128 | collections = get_collection_paths(output) 129 | 130 | if params.solr_cloud_mode: 131 | write_locks_to_delete = get_write_lock_files_solr_cloud(hadoop_prefix, collections) 132 | else: 133 | write_locks_to_delete = get_write_lock_files_solr_standalone(collections) 134 | 135 | if len(write_locks_to_delete) > 1: 136 | Logger.info(format('For hostname: \'{solr_hostname}\' lock files \'{write_locks_to_delete}\' will be deleted.')) 137 | Execute( 138 | format('{hadoop_prefix} -rm -f {write_locks_to_delete}'), 139 | user=params.hdfs_user 140 | ) 141 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/scripts/status_params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from resource_management.libraries.functions.format import format 4 | from resource_management.libraries.script.script import Script 5 | 6 | config = Script.get_config() 7 | 8 | map_solr_config = config['configurations']['solr-config-env'] 9 | 10 | solr_config_port = map_solr_config['solr_config_port'] 11 | solr_config_pid_dir = map_solr_config['solr_config_pid_dir'] 12 | solr_config_pid_file = format('{solr_config_pid_dir}/solr-{solr_config_port}.pid') 13 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/templates/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | 9 | 10 | 12 | 13 | %d{ISO8601} - %-5p [%t:%C{1}@%L] - %X - %m%n 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/templates/solr_metrics_jaas.conf.j2: -------------------------------------------------------------------------------- 1 | SolrJClient { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{solr_metrics_kerberos_keytab}}" 5 | storeKey=true 6 | useTicketCache=false 7 | debug=false 8 | principal="{{solr_metrics_kerberos_principal}}"; 9 | }; -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/package/templates/solr_server_jaas.conf.j2: -------------------------------------------------------------------------------- 1 | Client { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{solr_kerberos_keytab}}" 5 | storeKey=true 6 | useTicketCache=true 7 | debug=false 8 | principal="{{solr_kerberos_principal}}"; 9 | }; 10 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/6.6.2/properties/solr.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 28 | 29 | 30 | 31 | 32 | 33 | ${host:} 34 | ${jetty.port:8983} 35 | ${hostContext:solr} 36 | 37 | ${genericCoreNodeNames:true} 38 | 39 | ${zkClientTimeout:30000} 40 | ${distribUpdateSoTimeout:600000} 41 | ${distribUpdateConnTimeout:60000} 42 | ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} 43 | ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} 44 | 45 | 46 | 47 | 49 | ${socketTimeout:600000} 50 | ${connTimeout:60000} 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/alerts.json: -------------------------------------------------------------------------------- 1 | { 2 | "SOLR": { 3 | "SOLR_SERVER": [ 4 | { 5 | "name": "solr", 6 | "label": "Solr Web UI", 7 | "description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.", 8 | "interval": 1, 9 | "scope": "ANY", 10 | "source": { 11 | "type": "WEB", 12 | "uri": { 13 | "http": "{{solr-config-env/solr_config_port}}", 14 | "https": "{{solr-config-env/solr_config_port}}", 15 | "https_property": "{{solr-ssl/solr_ssl_enable}}", 16 | "https_property_value": "true", 17 | "kerberos_keytab": "{{solr-config-env/solr_spnego_keytab_path}}", 18 | "kerberos_principal": "{{solr-config-env/solr_spnego_principal_name}}", 19 | "connection_timeout": 5.0, 20 | "default_port": 8983 21 | }, 22 | "reporting": { 23 | "ok": { 24 | "text": "HTTP {0} response in {2:.3f}s" 25 | }, 26 | "warning": { 27 | "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" 28 | }, 29 | "critical": { 30 | "text": "Connection failed to {1} ({3})" 31 | } 32 | } 33 | } 34 | }, 35 | { 36 | "name": "solr_cpu_usage", 37 | "label": "Solr CPU Utilization", 38 | "description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", 39 | "interval": 5, 40 | "scope": "ANY", 41 | "source": { 42 | "type": "SCRIPT", 43 | "path": "SOLR/7.4.0/package/alerts/alert_solr_cpu_metrics.py", 44 | "parameters": [ 45 | { 46 | "name": "connection.timeout", 47 | "value": 5.0, 48 | "type": "NUMERIC", 49 | "units": "seconds", 50 | "visibility": "HIDDEN" 51 | }, 52 | { 53 | "name": "app.id", 54 | "value": "solr-host-app", 55 | "type": "STRING", 56 | "visibility": "HIDDEN" 57 | }, 58 | { 59 | "name": "metric.names", 60 | "value": "solr.jvm.gauge.os.processCpuLoad", 61 | "type": "STRING", 62 | "visibility": "HIDDEN" 63 | }, 64 | { 65 | "name": "metric.solr.cpu.warning.threshold", 66 | "display_name": "Warning threshold", 67 | "type": "PERCENT", 68 | "units": "%", 69 | "value": 75, 70 | "description": "CPU usage is greater than 50%", 71 | "threshold": "WARNING" 72 | }, 73 | { 74 | "name": "metric.solr.cpu.critical.threshold", 75 | "display_name": "Critical threshold", 76 | "type": "PERCENT", 77 | "units": "%", 78 | "value": 90, 79 | "description": "CPU usage is greater than 75%", 80 | "threshold": "CRITICAL" 81 | } 82 | ] 83 | } 84 | }, 85 | { 86 | "name": "solr_memory", 87 | "label": "Solr Memory Utilization", 88 | "description": "This host-level alert is triggered if CPU utilization of the Solr exceeds certain warning and critical thresholds. It checks the Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", 89 | "interval": 5, 90 | "scope": "ANY", 91 | "source": { 92 | "type": "SCRIPT", 93 | "path": "SOLR/7.4.0/package/alerts/alert_solr_memory_metrics.py", 94 | "parameters": [ 95 | { 96 | "name": "connection.timeout", 97 | "value": 5.0, 98 | "type": "NUMERIC", 99 | "units": "seconds", 100 | "visibility": "HIDDEN" 101 | }, 102 | { 103 | "name": "app.id", 104 | "value": "solr-host-app", 105 | "type": "STRING", 106 | "visibility": "HIDDEN" 107 | }, 108 | { 109 | "name": "metric.names", 110 | "value": "solr.jvm.gauge.memory.total.used,solr.jvm.gauge.memory.total.max", 111 | "type": "STRING", 112 | "visibility": "HIDDEN" 113 | }, 114 | { 115 | "name": "metric.solr.memory.warning.threshold", 116 | "display_name": "Warning threshold", 117 | "type": "PERCENT", 118 | "units": "%", 119 | "value": 75, 120 | "description": "Memory usage is greater than 50%", 121 | "threshold": "WARNING" 122 | }, 123 | { 124 | "name": "metric.solr.memory.critical.threshold", 125 | "display_name": "Critical threshold", 126 | "type": "PERCENT", 127 | "units": "%", 128 | "value": 90, 129 | "description": "Memory usage is greater than 75%", 130 | "threshold": "CRITICAL" 131 | } 132 | ] 133 | } 134 | } 135 | ] 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/configuration/example-collection.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_collection_sample_create 7 | true 8 | True to create a sample collection when Solr is deployed. 9 | Create sample collection 10 | 11 | boolean 12 | false 13 | 14 | 15 | 16 | 17 | solr_collection_sample_name 18 | collection1 19 | Solr sample collection name. Mandatory. 20 | Sample collection name 21 | 22 | 23 | 24 | solr_collection_sample_config_directory 25 | _default 26 | Solr configuration set for the sample collection, which must be a directory 27 | that contains at least solrconfig.xml and a schema. Mandatory. 28 | This directory path is relative to /opt/lucidworks-hadoop/solr/server/solr/configsets. 29 | 30 | Solr configset 31 | 32 | 33 | 34 | solr_collection_sample_shards 35 | 2 36 | Number of shards (logical partitions of the collection) to create. For details refer to 37 | (https://lucene.apache.org/solr/guide/7_4/shards-and-indexing-data-in-solrcloud.html) 38 | 39 | 40 | int 41 | 42 | Number of shards 43 | 44 | 45 | 46 | solr_collection_sample_replicas 47 | 1 48 | Number of Solr replicas (physical copies of shards) to create. For details refer to 49 | (https://lucene.apache.org/solr/guide/7_4/shards-and-indexing-data-in-solrcloud.html) 50 | 51 | 52 | int 53 | 54 | Number of replicas 55 | 56 | 57 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/configuration/solr-log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | content 8 | Custom log4j2.xml 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.} %m%n 17 | 18 | 19 | 20 | 21 | 25 | 26 | 27 | %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.} %m%n 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 41 | 42 | 43 | %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.} %m%n 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | ]]> 68 | 69 | 70 | content 71 | false 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/configuration/solr-ssl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | solr_ssl_key_store_type 7 | JKS 8 | The keystore type. 9 | SSL keystore type 10 | 11 | 12 | 13 | solr_ssl_trust_store_type 14 | JKS 15 | The truststore type. 16 | SSL truststore type 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/dashboards/grafana-dashboards/grafana-solr-740-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": null, 3 | "title": "Solr 7.4.0 - Hosts", 4 | "originalTitle": "Solr 7.4.0 - Hosts", 5 | "tags": [ 6 | "solr", 7 | "solr-7.4.0" 8 | ], 9 | "style": "dark", 10 | "timezone": "browser", 11 | "editable": true, 12 | "hideControls": false, 13 | "sharedCrosshair": false, 14 | "rows": [ 15 | { 16 | "collapse": false, 17 | "editable": true, 18 | "height": "25px", 19 | "panels": [ 20 | { 21 | "content": "

Solr Hosts Dashboard

", 22 | "editable": true, 23 | "error": false, 24 | "id": 3, 25 | "isNew": true, 26 | "links": [], 27 | "mode": "html", 28 | "span": 12, 29 | "style": {}, 30 | "title": "", 31 | "type": "text" 32 | } 33 | ], 34 | "title": "New row" 35 | }, 36 | { 37 | "collapse": false, 38 | "editable": true, 39 | "height": "250px", 40 | "panels": [ 41 | { 42 | "aliasColors": {}, 43 | "bars": false, 44 | "datasource": null, 45 | "editable": true, 46 | "error": false, 47 | "fill": 1, 48 | "grid": { 49 | "leftLogBase": 1, 50 | "leftMax": null, 51 | "leftMin": null, 52 | "rightLogBase": 1, 53 | "rightMax": null, 54 | "rightMin": null, 55 | "threshold1": null, 56 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 57 | "threshold2": null, 58 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 59 | }, 60 | "id": 1, 61 | "isNew": true, 62 | "legend": { 63 | "avg": false, 64 | "current": false, 65 | "max": false, 66 | "min": false, 67 | "show": true, 68 | "total": false, 69 | "values": false 70 | }, 71 | "lines": true, 72 | "linewidth": 2, 73 | "links": [], 74 | "nullPointMode": "connected", 75 | "percentage": false, 76 | "pointradius": 5, 77 | "points": false, 78 | "renderer": "flot", 79 | "seriesOverrides": [], 80 | "span": 6, 81 | "stack": false, 82 | "steppedLine": false, 83 | "targets": [ 84 | { 85 | "aggregator": "none", 86 | "app": "solr-host-app", 87 | "downsampleAggregator": "avg", 88 | "errors": {}, 89 | "metric": "solr.jvm.gauge.os.processCpuLoad", 90 | "precision": "default", 91 | "refId": "A", 92 | "seriesAggregator": "none", 93 | "templatedHost": "", 94 | "transform": "none" 95 | } 96 | ], 97 | "timeFrom": null, 98 | "timeShift": null, 99 | "title": "Panel Title", 100 | "tooltip": { 101 | "shared": true, 102 | "value_type": "cumulative" 103 | }, 104 | "type": "graph", 105 | "x-axis": true, 106 | "y-axis": true, 107 | "y_formats": [ 108 | "percent", 109 | "none" 110 | ] 111 | }, 112 | { 113 | "aliasColors": {}, 114 | "bars": false, 115 | "datasource": null, 116 | "editable": true, 117 | "error": false, 118 | "fill": 1, 119 | "grid": { 120 | "leftLogBase": 1, 121 | "leftMax": null, 122 | "leftMin": null, 123 | "rightLogBase": 1, 124 | "rightMax": null, 125 | "rightMin": null, 126 | "threshold1": null, 127 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 128 | "threshold2": null, 129 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 130 | }, 131 | "id": 2, 132 | "isNew": true, 133 | "legend": { 134 | "avg": false, 135 | "current": false, 136 | "max": false, 137 | "min": false, 138 | "show": true, 139 | "total": false, 140 | "values": false 141 | }, 142 | "lines": true, 143 | "linewidth": 2, 144 | "links": [], 145 | "nullPointMode": "connected", 146 | "percentage": false, 147 | "pointradius": 5, 148 | "points": false, 149 | "renderer": "flot", 150 | "seriesOverrides": [], 151 | "span": 6, 152 | "stack": false, 153 | "steppedLine": false, 154 | "targets": [ 155 | { 156 | "aggregator": "none", 157 | "app": "solr-host-app", 158 | "downsampleAggregator": "avg", 159 | "errors": {}, 160 | "metric": "solr.jvm.gauge.memory.total.used", 161 | "precision": "default", 162 | "refId": "A", 163 | "seriesAggregator": "none", 164 | "templatedHost": "", 165 | "transform": "none" 166 | } 167 | ], 168 | "timeFrom": null, 169 | "timeShift": null, 170 | "title": "Panel Title", 171 | "tooltip": { 172 | "shared": true, 173 | "value_type": "cumulative" 174 | }, 175 | "type": "graph", 176 | "x-axis": true, 177 | "y-axis": true, 178 | "y_formats": [ 179 | "bytes", 180 | "bytes" 181 | ] 182 | } 183 | ], 184 | "title": "Row" 185 | } 186 | ], 187 | "time": { 188 | "from": "now-1h", 189 | "to": "now" 190 | }, 191 | "timepicker": { 192 | "now": true, 193 | "refresh_intervals": [ 194 | "5s", 195 | "10s", 196 | "30s", 197 | "1m", 198 | "5m", 199 | "15m", 200 | "30m", 201 | "1h", 202 | "2h", 203 | "1d" 204 | ], 205 | "time_options": [ 206 | "5m", 207 | "15m", 208 | "1h", 209 | "6h", 210 | "12h", 211 | "24h", 212 | "2d", 213 | "7d", 214 | "30d" 215 | ] 216 | }, 217 | "templating": { 218 | "list": [ 219 | { 220 | "allFormat": "glob", 221 | "current": { 222 | "text": "solr-host-app", 223 | "value": "solr-host-app" 224 | }, 225 | "datasource": null, 226 | "includeAll": false, 227 | "multi": false, 228 | "multiFormat": "glob", 229 | "name": "components", 230 | "options": [ 231 | { 232 | "selected": true, 233 | "text": "solr-host-app", 234 | "value": "solr-host-app" 235 | } 236 | ], 237 | "query": "solr-host-app", 238 | "refresh": false, 239 | "regex": "", 240 | "type": "custom" 241 | }, 242 | { 243 | "allFormat": "glob", 244 | "current": { 245 | }, 246 | "datasource": null, 247 | "includeAll": false, 248 | "multi": false, 249 | "multiFormat": "glob", 250 | "name": "hosts", 251 | "options": [ 252 | 253 | ], 254 | "query": "hosts", 255 | "refresh": true, 256 | "regex": "", 257 | "type": "query" 258 | } 259 | ] 260 | }, 261 | "annotations": { 262 | "list": [] 263 | }, 264 | "schemaVersion": 8, 265 | "version": 2, 266 | "links": [] 267 | } 268 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 2.0 18 | 19 | 20 | SOLR 21 | 7.4.0 22 | common-services/SOLR/6.6.2 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/alerts/alert_solr_cpu_metrics.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import httplib 3 | import json 4 | import time 5 | 6 | from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames 7 | from ambari_commons.ambari_metrics_helper import load_properties_from_file 8 | from resource_management.libraries.functions.format import format 9 | 10 | RESULT_STATE_OK = 'OK' 11 | RESULT_STATE_CRITICAL = 'CRITICAL' 12 | RESULT_STATE_WARNING = 'WARNING' 13 | RESULT_STATE_UNKNOWN = 'UNKNOWN' 14 | RESULT_STATE_SKIPPED = 'SKIPPED' 15 | 16 | KERBEROS_KEYTAB = '{{hdfs-site/dfs.web.authentication.kerberos.keytab}}' 17 | KERBEROS_PRINCIPAL = '{{hdfs-site/dfs.web.authentication.kerberos.principal}}' 18 | SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' 19 | SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' 20 | 21 | METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' 22 | METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' 23 | METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' 24 | 25 | SOLR_METRICS_CONF_DIR = '{{solr-metrics/solr_metrics_config_conf_dir}}' 26 | SOLR_METRICS_PERIOD = '{{solr-metrics/solr_metrics_period}}' 27 | 28 | AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s" 29 | 30 | CONNECTION_TIMEOUT_DEFAULT = 5.0 31 | CONNECTION_TIMEOUT_KEY = "connection.timeout" 32 | 33 | METRIC_NAMES_DEFAULT = "solr.jvm.gauge.os.processCpuLoad" 34 | METRIC_NAMES_KEY = "metric.names" 35 | 36 | APP_ID_DEFAULT = "solr-host-app" 37 | APP_ID_KEY = "app.id" 38 | 39 | WARNING_THRESHOLD_DEFAULT = 50 40 | WARNING_THRESHOLD_KEY = "metric.solr.cpu.warning.threshold" 41 | 42 | CRITICAL_THRESHOLD_DEFAULT = 75 43 | CRITICAL_THRESHOLD_KEY = "metric.solr.cpu.critical.threshold" 44 | 45 | 46 | def get_tokens(): 47 | """ 48 | Returns a tuple of tokens in the format {{site/property}} that will be used 49 | to build the dictionary passed into execute 50 | """ 51 | return (SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, METRICS_COLLECTOR_VIP_HOST_KEY, 52 | METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, SOLR_METRICS_CONF_DIR, 53 | SOLR_METRICS_PERIOD) 54 | 55 | 56 | def execute(configurations={}, parameters={}, host_name=None): 57 | collector_host, collector_port = get_collector_config(configurations) 58 | 59 | if collector_host is None or collector_port is None: 60 | return RESULT_STATE_UNKNOWN, ['Undefined collector host: {0} or collector port {1}'.format(collector_host, 61 | collector_port)] 62 | 63 | metric_name = METRIC_NAMES_DEFAULT 64 | if METRIC_NAMES_KEY in parameters: 65 | metric_name = parameters[METRIC_NAMES_KEY] 66 | 67 | app_id = APP_ID_DEFAULT 68 | if APP_ID_KEY in parameters: 69 | app_id = parameters[APP_ID_KEY] 70 | 71 | connection_timeout = CONNECTION_TIMEOUT_DEFAULT 72 | if CONNECTION_TIMEOUT_KEY in parameters: 73 | connection_timeout = parameters[CONNECTION_TIMEOUT_KEY] 74 | 75 | warning_threshold = WARNING_THRESHOLD_DEFAULT 76 | if WARNING_THRESHOLD_KEY in parameters: 77 | warning_threshold = parameters[WARNING_THRESHOLD_KEY] 78 | 79 | critical_threshold = CRITICAL_THRESHOLD_DEFAULT 80 | if CRITICAL_THRESHOLD_KEY in parameters: 81 | critical_threshold = parameters[CRITICAL_THRESHOLD_KEY] 82 | 83 | get_metrics_parameters = { 84 | "metricNames": metric_name, 85 | "appId": app_id, 86 | "hostname": host_name, 87 | "grouped": "true", 88 | } 89 | 90 | encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters) 91 | 92 | try: 93 | conn = httplib.HTTPConnection(collector_host, int(collector_port), timeout=float(connection_timeout)) 94 | conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters) 95 | response = conn.getresponse() 96 | data = response.read() 97 | conn.close() 98 | except Exception as e: 99 | message = "Unable to retrieve metrics from the Ambari Metrics service. Error: {}".format(str(e)) 100 | return RESULT_STATE_UNKNOWN, [message] 101 | 102 | data_json = json.loads(data) 103 | timestamp = -1 104 | 105 | for metrics_data in data_json["metrics"]: 106 | if metric_name in metrics_data["metricname"]: 107 | metrics = metrics_data["metrics"].values() 108 | timestamp = metrics_data["timestamp"] 109 | 110 | if int(timestamp) == -1: 111 | return RESULT_STATE_UNKNOWN, ["There is not enough data"] 112 | 113 | current_time = int(time.time()) * 1000 114 | metric_period_ms = 10 * 1000 * 60 115 | difference = current_time - timestamp 116 | 117 | if difference > metric_period_ms: 118 | message = "Data retrieved is older than 10 minutes, please check the Solr node, and confirm that the Solr " \ 119 | "ambari metrics is running" 120 | return RESULT_STATE_WARNING, [message] 121 | 122 | cpu_load_value = metrics[0] * 100 123 | response = 'CPU load {0:.2f} %'.format(cpu_load_value) 124 | 125 | if int(cpu_load_value) >= warning_threshold and int(cpu_load_value) < critical_threshold: 126 | return RESULT_STATE_WARNING, [response] 127 | 128 | if cpu_load_value >= critical_threshold: 129 | return RESULT_STATE_CRITICAL, [response] 130 | 131 | return RESULT_STATE_OK, [response] 132 | 133 | 134 | def get_collector_config(configurations): 135 | solr_package_dir = '/opt/lucidworks-hdpsearch' 136 | solr_metrics_config_conf_dir = format(configurations[SOLR_METRICS_CONF_DIR]) 137 | solr_metrics_props = "{0}/{1}".format(solr_metrics_config_conf_dir, "solr.metrics.properties") 138 | props = load_properties_from_file(solr_metrics_props) 139 | collector_hosts = props.get("collector.hosts") 140 | 141 | if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: 142 | collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] 143 | collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) 144 | else: 145 | collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") 146 | if valid_collector_webapp_address(collector_webapp_address): 147 | collector_host = select_metric_collector_hosts_from_hostnames(collector_hosts) 148 | collector_port = int(collector_webapp_address[1]) 149 | else: 150 | return None, None 151 | return collector_host, collector_port 152 | 153 | 154 | def valid_collector_webapp_address(webapp_address): 155 | if len(webapp_address) == 2 and webapp_address[0] != '127.0.0.1' and webapp_address[1].isdigit(): 156 | return True 157 | return False 158 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/alerts/alert_solr_memory_metrics.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import httplib 3 | import json 4 | import time 5 | 6 | from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames 7 | from ambari_commons.ambari_metrics_helper import load_properties_from_file 8 | from resource_management.libraries.functions.format import format 9 | 10 | RESULT_STATE_OK = 'OK' 11 | RESULT_STATE_CRITICAL = 'CRITICAL' 12 | RESULT_STATE_WARNING = 'WARNING' 13 | RESULT_STATE_UNKNOWN = 'UNKNOWN' 14 | RESULT_STATE_SKIPPED = 'SKIPPED' 15 | 16 | KERBEROS_KEYTAB = '{{hdfs-site/dfs.web.authentication.kerberos.keytab}}' 17 | KERBEROS_PRINCIPAL = '{{hdfs-site/dfs.web.authentication.kerberos.principal}}' 18 | SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' 19 | SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' 20 | 21 | METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' 22 | METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' 23 | METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' 24 | 25 | SOLR_METRICS_CONF_DIR = '{{solr-metrics/solr_metrics_config_conf_dir}}' 26 | SOLR_METRICS_PERIOD = "{{solr-metrics/solr_metrics_period}}" 27 | 28 | AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s" 29 | 30 | CONNECTION_TIMEOUT_DEFAULT = 5.0 31 | CONNECTION_TIMEOUT_KEY = "connection.timeout" 32 | 33 | METRIC_NAMES_DEFAULT = "solr.jvm.gauge.memory.total.used,solr.jvm.gauge.memory.total.max" 34 | METRIC_NAMES_KEY = "metric.names" 35 | 36 | APP_ID_DEFAULT = "solr-host-app" 37 | APP_ID_KEY = "app.id" 38 | 39 | WARNING_THRESHOLD_DEFAULT = 50 40 | WARNING_THRESHOLD_KEY = "metric.solr.memory.warning.threshold" 41 | 42 | CRITICAL_THRESHOLD_DEFAULT = 75 43 | CRITICAL_THRESHOLD_KEY = "metric.solr.memory.critical.threshold" 44 | 45 | def get_tokens(): 46 | """ 47 | Returns a tuple of tokens in the format {{site/property}} that will be used 48 | to build the dictionary passed into execute 49 | """ 50 | return (SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, METRICS_COLLECTOR_VIP_HOST_KEY, 51 | METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, SOLR_METRICS_CONF_DIR, 52 | SOLR_METRICS_PERIOD) 53 | 54 | 55 | def execute(configurations={}, parameters={}, host_name=None): 56 | collector_host, collector_port = get_collector_config(configurations) 57 | 58 | if collector_host is None or collector_port is None: 59 | return RESULT_STATE_UNKNOWN, ['Undefined collector host: {0} or collector port {1}'.format(collector_host, 60 | collector_port)] 61 | 62 | metric_name = METRIC_NAMES_DEFAULT 63 | if METRIC_NAMES_KEY in parameters: 64 | metric_name = parameters[METRIC_NAMES_KEY] 65 | 66 | app_id = APP_ID_DEFAULT 67 | if APP_ID_KEY in parameters: 68 | app_id = parameters[APP_ID_KEY] 69 | 70 | connection_timeout = CONNECTION_TIMEOUT_DEFAULT 71 | if CONNECTION_TIMEOUT_KEY in parameters: 72 | connection_timeout = parameters[CONNECTION_TIMEOUT_KEY] 73 | 74 | warning_threshold = WARNING_THRESHOLD_DEFAULT 75 | if WARNING_THRESHOLD_KEY in parameters: 76 | warning_threshold = parameters[WARNING_THRESHOLD_KEY] 77 | 78 | critical_threshold = CRITICAL_THRESHOLD_DEFAULT 79 | if CRITICAL_THRESHOLD_KEY in parameters: 80 | critical_threshold = parameters[CRITICAL_THRESHOLD_KEY] 81 | 82 | get_metrics_parameters = { 83 | "metricNames": metric_name, 84 | "appId": app_id, 85 | "hostname": host_name, 86 | "grouped": "true", 87 | } 88 | 89 | encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters) 90 | 91 | try: 92 | conn = httplib.HTTPConnection(collector_host, int(collector_port), timeout=float(connection_timeout)) 93 | conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters) 94 | response = conn.getresponse() 95 | data = response.read() 96 | conn.close() 97 | except Exception as e: 98 | message = "Unable to retrieve metrics from the Ambari Metrics service. Error: {}".format(str(e)) 99 | return RESULT_STATE_UNKNOWN, [message] 100 | 101 | data_json = json.loads(data) 102 | 103 | split = metric_name.split(",") 104 | metric_name_map = { 105 | "used" if ("used" in split[0]) else "max": split[0], 106 | "used" if ("used" in split[1]) else "max": split[1] 107 | } 108 | 109 | timestamp = -1 110 | metrics = {} 111 | 112 | for metrics_data in data_json["metrics"]: 113 | if metric_name_map["used"] in metrics_data["metricname"]: 114 | metrics["used"] = metrics_data["metrics"].values()[0] 115 | current_timestamp = metrics_data["timestamp"] 116 | if timestamp == -1: 117 | timestamp = current_timestamp 118 | elif timestamp > current_timestamp: 119 | timestamp = current_timestamp 120 | continue 121 | if metric_name_map["max"] in metrics_data["metricname"]: 122 | metrics["max"] = metrics_data["metrics"].values()[0] 123 | current_timestamp = metrics_data["timestamp"] 124 | if timestamp == -1: 125 | timestamp = current_timestamp 126 | elif timestamp > current_timestamp: 127 | timestamp = current_timestamp 128 | continue 129 | 130 | if int(timestamp) == -1: 131 | return RESULT_STATE_UNKNOWN, ["There is not enough data"] 132 | 133 | current_time = int(time.time()) * 1000 134 | metric_period_ms = 10 * 1000 * 60 135 | difference = current_time - timestamp 136 | 137 | if difference > metric_period_ms: 138 | message = "Data retrieved is older than 10 minutes, please check the Solr node, and confirm that the Solr " \ 139 | "ambari metrics is running" 140 | return RESULT_STATE_WARNING, [message] 141 | 142 | memory_used = metrics["used"] / (1024 * 1024) 143 | memory_max = metrics["max"] / (1024 * 1024) 144 | memory_value = memory_used / memory_max 145 | response = 'Memory usage is {0:.2f} %'.format(memory_value) 146 | 147 | if int(memory_value) >= warning_threshold and int(memory_value) < critical_threshold: 148 | return RESULT_STATE_WARNING, [response] 149 | 150 | if memory_value >= critical_threshold: 151 | return RESULT_STATE_CRITICAL, [response] 152 | 153 | return RESULT_STATE_OK, [response] 154 | 155 | 156 | def get_collector_config(configurations): 157 | solr_package_dir = '/opt/lucidworks-hdpsearch' 158 | solr_metrics_config_conf_dir = format(configurations[SOLR_METRICS_CONF_DIR]) 159 | solr_metrics_props = "{0}/{1}".format(solr_metrics_config_conf_dir, "solr.metrics.properties") 160 | props = load_properties_from_file(solr_metrics_props) 161 | collector_hosts = props.get("collector.hosts") 162 | 163 | if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: 164 | collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] 165 | collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) 166 | else: 167 | collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") 168 | if valid_collector_webapp_address(collector_webapp_address): 169 | collector_host = select_metric_collector_hosts_from_hostnames(collector_hosts) 170 | collector_port = int(collector_webapp_address[1]) 171 | else: 172 | return None, None 173 | return collector_host, collector_port 174 | 175 | 176 | def valid_collector_webapp_address(webapp_address): 177 | if len(webapp_address) == 2 and webapp_address[0] != '127.0.0.1' and webapp_address[1].isdigit(): 178 | return True 179 | return False 180 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/service_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from resource_management.core.logger import Logger 6 | from resource_management.core.resources.system import Execute 7 | from resource_management.libraries.functions.format import format 8 | from resource_management.libraries.script import Script 9 | 10 | from solr_utils import exists_collection 11 | 12 | 13 | class ServiceCheck(Script): 14 | def service_check(self, env): 15 | import params 16 | env.set_params(params) 17 | 18 | if not os.path.isfile(params.solr_config_pid_file): 19 | Logger.error(format("PID file {solr_config_pid_file} does not exist")) 20 | exit(1) 21 | 22 | if not params.solr_collection_sample_create: 23 | Logger.info("Create sample collection unchecked, skipping ...") 24 | return 25 | 26 | if exists_collection(params.solr_collection_name): 27 | Logger.warning(format("Collection {solr_collection_name} already exists, skipping ...")) 28 | return 29 | 30 | if not params.solr_cloud_mode: 31 | Execute( 32 | format( 33 | '{solr_config_bin_dir}/solr create_core -c {solr_collection_name}' + 34 | ' -d {solr_collection_config_dir} -p {solr_config_port} >> {solr_config_service_log_file} 2>&1' 35 | ), 36 | environment={'JAVA_HOME': params.java64_home}, 37 | user=params.solr_config_user 38 | ) 39 | else: 40 | Execute(format( 41 | '{solr_config_bin_dir}/solr create_collection -c {solr_collection_name}' + 42 | ' -d {solr_collection_config_dir} -p {solr_config_port}' + 43 | ' -s {solr_collection_shards} -rf {solr_collection_replicas}' + 44 | ' >> {solr_config_service_log_file} 2>&1'), 45 | environment={'JAVA_HOME': params.java64_home}, 46 | user=params.solr_config_user 47 | ) 48 | 49 | 50 | if __name__ == "__main__": 51 | ServiceCheck().execute() 52 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/setup_solr.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Directory, File, Execute 2 | from resource_management.core.source import Template, InlineTemplate 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr(): 7 | import params 8 | 9 | Directory( 10 | [ 11 | params.solr_config_dir, 12 | params.solr_config_log_dir, 13 | params.solr_config_service_log_dir, 14 | params.solr_config_pid_dir, 15 | params.solr_config_conf_dir, 16 | params.solr_config_home_dir 17 | ], 18 | mode=0755, 19 | cd_access='a', 20 | owner=params.solr_config_user, 21 | group=params.solr_config_group, 22 | create_parents=True 23 | ) 24 | 25 | Execute( 26 | ('chmod', '-R', '777', params.solr_webapp_dir), 27 | sudo=True 28 | ) 29 | 30 | File( 31 | format("{solr_config_bin_dir}/solr.in.sh"), 32 | content=InlineTemplate(params.solr_config_in_sh), 33 | owner=params.solr_config_user 34 | ) 35 | 36 | File( 37 | format("{solr_config_conf_dir}/log4j2.xml"), 38 | content=InlineTemplate(params.log4j_properties), 39 | owner=params.solr_config_user 40 | ) 41 | 42 | File( 43 | format("{solr_config_home_dir}/solr.xml"), 44 | content=InlineTemplate(params.solr_xml_template), 45 | owner=params.solr_config_user 46 | ) 47 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/setup_solr_cloud.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute 3 | from resource_management.core.shell import call 4 | from resource_management.libraries.functions.format import format 5 | 6 | 7 | def setup_solr_cloud(): 8 | import params 9 | 10 | code, output = call( 11 | format( 12 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{clusterstate_json}' 13 | ), 14 | env={'JAVA_HOME': params.java64_home}, 15 | timeout=60 16 | ) 17 | 18 | if not ("NoNodeException" in output): 19 | Logger.info( 20 | format( 21 | "ZK node {solr_cloud_zk_directory}{clusterstate_json} already exists, skipping ..." 22 | ) 23 | ) 24 | return 25 | 26 | Execute( 27 | format( 28 | '{zk_client_prefix} -cmd makepath {solr_cloud_zk_directory}' 29 | ), 30 | environment={'JAVA_HOME': params.java64_home}, 31 | ignore_failures=True, 32 | user=params.solr_config_user 33 | ) 34 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/setup_solr_hdfs_support.py: -------------------------------------------------------------------------------- 1 | def setup_solr_hdfs_support(): 2 | import params 3 | 4 | # Create a home directory for solr user 5 | params.HdfsResource( 6 | params.solr_hdfs_home_directory, 7 | type="directory", 8 | action="create_on_execute", 9 | owner=params.solr_config_user 10 | ) 11 | 12 | params.HdfsResource(params.solr_hdfs_directory, 13 | type="directory", 14 | action="create_on_execute", 15 | owner=params.solr_config_user 16 | ) 17 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/setup_solr_kerberos_auth.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute, File 3 | from resource_management.core.shell import call 4 | from resource_management.core.source import Template 5 | from resource_management.libraries.functions.format import format 6 | 7 | 8 | def setup_solr_kerberos_auth(): 9 | import params 10 | 11 | File(format("{solr_kerberos_jaas_config}"), 12 | content=Template("solr_server_jaas.conf.j2"), 13 | owner=params.solr_config_user 14 | ) 15 | 16 | if _has_security_json(): 17 | Logger.info("Solr Security Json was found, it will not be overridden") 18 | return 19 | 20 | command = format('{zk_client_prefix} -cmd put {solr_cloud_zk_directory}{security_json} ') 21 | command += format('\'{solr_security_json}\'') 22 | Execute(command, 23 | environment={'JAVA_HOME': params.java64_home}, 24 | ignore_failures=True, 25 | user=params.solr_config_user 26 | ) 27 | 28 | 29 | def remove_solr_kerberos_auth(): 30 | import params 31 | 32 | if not _has_security_json(): 33 | Logger.debug( 34 | format("Solr Security Json not found {solr_cloud_zk_directory}{security_json}")) 35 | return 36 | 37 | Execute(format('{zk_client_prefix} -cmd clear {solr_cloud_zk_directory}{security_json}'), 38 | environment={'JAVA_HOME': params.java64_home}, 39 | timeout=60, 40 | ignore_failures=True, 41 | user=params.solr_config_user 42 | ) 43 | 44 | 45 | def _has_security_json(): 46 | import params 47 | 48 | if not params.solr_cloud_mode: 49 | return False 50 | 51 | code, output = call( 52 | format('{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{security_json}'), 53 | env={'JAVA_HOME': params.java64_home}, 54 | timeout=60 55 | ) 56 | 57 | if "{}" == output: 58 | return False 59 | return "NoNodeException" not in output 60 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/setup_solr_metrics.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Directory, File 2 | from resource_management.core.source import InlineTemplate, Template 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr_metrics_support(): 7 | import params 8 | 9 | Directory( 10 | [ 11 | params.solr_metrics_config_conf_dir, 12 | params.solr_metrics_config_pid_dir, 13 | params.solr_metrics_config_log_dir 14 | ], 15 | mode=0755, 16 | cd_access='a', 17 | owner=params.solr_config_user, 18 | group=params.solr_config_group, 19 | create_parents=True 20 | ) 21 | 22 | File( 23 | format("{solr_metrics_config_conf_dir}/solr.metrics.properties"), 24 | content=InlineTemplate(params.solr_metrics_properties), 25 | owner=params.solr_config_user 26 | ) 27 | 28 | File( 29 | format("{solr_metrics_config_conf_dir}/log4j2.json"), 30 | content=InlineTemplate(params.solr_metrics_log4j2), 31 | owner=params.solr_config_user 32 | ) 33 | 34 | if params.security_enabled: 35 | File( 36 | format("{solr_metrics_kerberos_jaas_config}"), 37 | content=Template("solr_metrics_jaas.conf.j2"), 38 | owner=params.solr_config_user 39 | ) 40 | 41 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/setup_solr_ssl_support.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.resources.system import Execute 2 | from resource_management.core.shell import call 3 | from resource_management.libraries.functions.format import format 4 | 5 | 6 | def setup_solr_ssl_support(): 7 | import params 8 | 9 | if not params.solr_cloud_mode: 10 | return 11 | 12 | Execute( 13 | format( 14 | '{zk_client_prefix}{solr_cloud_zk_directory} -cmd clusterprop -name urlScheme -val https' 15 | ), 16 | environment={'JAVA_HOME': params.java64_home}, 17 | ignore_failures=True, 18 | user=params.solr_config_user 19 | ) 20 | 21 | 22 | def remove_solr_ssl_support(): 23 | import params 24 | 25 | if not params.solr_cloud_mode: 26 | return 27 | 28 | code, output = call( 29 | format( 30 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}{clusterprops_json}' 31 | ), 32 | env={'JAVA_HOME': params.java64_home}, 33 | timeout=60 34 | ) 35 | 36 | if "NoNodeException" in output: 37 | return 38 | 39 | Execute( 40 | format( 41 | '{zk_client_prefix} -cmd clear {solr_cloud_zk_directory}{clusterprops_json}' 42 | ), 43 | environment={'JAVA_HOME': params.java64_home}, 44 | ignore_failures=True, 45 | user=params.solr_config_user 46 | ) 47 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/solr.py: -------------------------------------------------------------------------------- 1 | from resource_management.core.logger import Logger 2 | from resource_management.core.resources.system import Execute, File 3 | from resource_management.libraries.functions.check_process_status import check_process_status 4 | from resource_management.libraries.functions.format import format 5 | from resource_management.libraries.script.script import Script 6 | 7 | from setup_solr import setup_solr 8 | from setup_solr_cloud import setup_solr_cloud 9 | from setup_solr_hdfs_support import setup_solr_hdfs_support 10 | from setup_solr_kerberos_auth import setup_solr_kerberos_auth, remove_solr_kerberos_auth 11 | from setup_solr_metrics import setup_solr_metrics_support 12 | from setup_solr_ssl_support import setup_solr_ssl_support, remove_solr_ssl_support 13 | from solr_utils import is_solr_running, solr_port_validation, delete_write_lock_files 14 | 15 | 16 | class Solr(Script): 17 | def install(self, env): 18 | import params 19 | env.set_params(params) 20 | self.install_packages(env) 21 | 22 | def configure(self, env): 23 | import params 24 | env.set_params(params) 25 | setup_solr() 26 | 27 | if params.solr_cloud_mode: 28 | setup_solr_cloud() 29 | 30 | if params.solr_hdfs_enable: 31 | setup_solr_hdfs_support() 32 | 33 | if params.solr_ssl_enable: 34 | setup_solr_ssl_support() 35 | else: 36 | remove_solr_ssl_support() 37 | 38 | if params.security_enabled: 39 | setup_solr_kerberos_auth() 40 | else: 41 | remove_solr_kerberos_auth() 42 | 43 | if params.solr_hdfs_enable and params.solr_hdfs_delete_write_lock_files: 44 | delete_write_lock_files() 45 | 46 | if params.has_metric_collector and params.solr_enable_metrics: 47 | setup_solr_metrics_support() 48 | 49 | def start(self, env): 50 | import params 51 | env.set_params(params) 52 | self.configure(env) 53 | 54 | if not solr_port_validation(): 55 | exit(1) 56 | 57 | if is_solr_running(): 58 | Logger.info("Solr is running, it can not be started it again") 59 | exit(1) 60 | 61 | Logger.info("Starting Solr ... ") 62 | start_command = format('{solr_config_bin_dir}/solr start >> {solr_config_service_log_file} 2>&1') 63 | 64 | Execute( 65 | start_command, 66 | environment={'JAVA_HOME': params.java64_home}, 67 | user=params.solr_config_user 68 | ) 69 | 70 | if params.has_metric_collector and params.solr_enable_metrics: 71 | Logger.info('Starting Solr Metrics Sink.') 72 | 73 | if params.security_enabled: 74 | Execute( 75 | format('{kinit_path_local} -kt {solr_metrics_kerberos_keytab} {solr_metrics_kerberos_principal}'), 76 | user=params.solr_config_user 77 | ) 78 | 79 | Execute( 80 | format('{solr_metrics_sink_bin}/solr.metrics.sh start'), 81 | environment={ 82 | 'JAVA_HOME': params.java64_home, 83 | 'SOLR_METRICS_PID_DIR': params.solr_metrics_config_pid_dir, 84 | 'SOLR_METRICS_LOGS': params.solr_metrics_config_log_dir, 85 | 'SOLR_METRIC_CONF': params.solr_metrics_config_conf_dir 86 | }, 87 | user=params.solr_config_user 88 | ) 89 | 90 | def stop(self, env): 91 | import params 92 | env.set_params(params) 93 | 94 | if params.has_metric_collector and params.solr_enable_metrics: 95 | Logger.info('Stopping Solr Metrics Sink.') 96 | 97 | if params.security_enabled: 98 | Execute( 99 | format('{kinit_path_local} -kt {solr_metrics_kerberos_keytab} {solr_metrics_kerberos_principal}'), 100 | user=params.solr_config_user 101 | ) 102 | 103 | Execute( 104 | format('{solr_metrics_sink_bin}/solr.metrics.sh stop'), 105 | environment={ 106 | 'JAVA_HOME': params.java64_home, 107 | 'SOLR_METRICS_PID_DIR': params.solr_metrics_config_pid_dir, 108 | 'SOLR_METRICS_LOGS': params.solr_metrics_config_log_dir, 109 | 'SOLR_METRIC_CONF': params.solr_metrics_config_conf_dir 110 | }, 111 | ignore_failures=True, 112 | user=params.solr_config_user 113 | ) 114 | 115 | if not is_solr_running(): 116 | Logger.info("Solr is not running, it can not be stopped it again") 117 | return 118 | 119 | Execute( 120 | format( 121 | '{solr_config_bin_dir}/solr stop -all >> {solr_config_service_log_file} 2>&1'), 122 | environment={'JAVA_HOME': params.java64_home}, 123 | user=params.solr_config_user 124 | ) 125 | 126 | File( 127 | params.solr_config_pid_file, 128 | action="delete" 129 | ) 130 | 131 | def status(self, env): 132 | import status_params 133 | env.set_params(status_params) 134 | 135 | check_process_status(status_params.solr_config_pid_file) 136 | 137 | 138 | if __name__ == "__main__": 139 | Solr().execute() 140 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/solr_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from resource_management.core.logger import Logger 5 | from resource_management.core.resources.system import Execute 6 | from resource_management.core.shell import call 7 | from resource_management.libraries.functions.format import format 8 | 9 | COLLECTION_PATTERN = "{solr_hdfs_directory}/[a-zA-Z0-9\._-]+" 10 | CORE_PATTERN = "{collection_path}\/core_node[0-9]+" 11 | WRITE_LOCK_PATTERN = "{0}/data/index/write.lock " 12 | HOSTNAME_VERIFIER_PATTERN = "{core_node_name}\":{{((?![^_]shard|core_node).)*\"node_name\":\"{solr_hostname}" 13 | 14 | 15 | def solr_port_validation(): 16 | code, output = call( 17 | format('netstat -lnt | awk -v v1={solr_config_port} \'$6 == "LISTEN" && $4 ~ ":"v1\''), 18 | timeout=60) 19 | Logger.info(format("Solr port validation output: {output}")) 20 | 21 | if "LISTEN" in output: 22 | Logger.error(format("The port {solr_config_port} is not available")) 23 | return False 24 | 25 | return True 26 | 27 | 28 | def is_solr_running(): 29 | import params 30 | 31 | code, output = call( 32 | format('{solr_config_bin_dir}/solr status'), 33 | env={'JAVA_HOME': params.java64_home}, 34 | timeout=60 35 | ) 36 | Logger.info(format("Solr status output: {output}")) 37 | pattern_port = re.compile(format('running on port {solr_config_port}')) 38 | 39 | if pattern_port.search(output): 40 | Logger.error(format("A Solr instance is running on port {solr_config_port}")) 41 | return True 42 | 43 | return False 44 | 45 | 46 | def exists_collection(collection_name): 47 | import params 48 | 49 | if not params.solr_cloud_mode: 50 | if os.path.isdir(format("{solr_config_home_dir}/{collection_name}")): 51 | return True 52 | return False 53 | 54 | code, output = call(format( 55 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}'), 56 | env={'JAVA_HOME': params.java64_home}, 57 | timeout=60 58 | ) 59 | 60 | if "NoNodeException" in output: 61 | return False 62 | return True 63 | 64 | 65 | def get_collection_paths(hadoop_output): 66 | import params 67 | pattern = re.compile(format(COLLECTION_PATTERN)) 68 | 69 | collection_paths = re.findall(pattern, hadoop_output) 70 | return collection_paths 71 | 72 | 73 | def get_core_paths(hadoop_output, collection_path): 74 | pattern = re.compile(format(CORE_PATTERN)) 75 | core_paths = re.findall(pattern, hadoop_output) 76 | return core_paths 77 | 78 | 79 | def get_write_lock_files_solr_cloud(hadoop_prefix, collections): 80 | import params 81 | 82 | write_locks_to_delete = '' 83 | 84 | for collection_path in collections: 85 | code, output = call(format('{hadoop_prefix} -ls {collection_path}')) 86 | core_paths = get_core_paths(output, collection_path) 87 | 88 | collection_name = collection_path.replace(format('{solr_hdfs_directory}/'), '') 89 | zk_code, zk_output = call(format( 90 | '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}/state.json'), 91 | env={'JAVA_HOME': params.java64_home}, 92 | timeout=60 93 | ) 94 | if zk_code != 0: 95 | Logger.error(format('Cannot determine cores owned by [{solr_hostname}] in collection [{collection_name}] due to ZK error.')) 96 | continue 97 | 98 | for core_path in core_paths: 99 | core_node_name = core_path.replace(format('{collection_path}/'), '') 100 | pattern = re.compile(format(HOSTNAME_VERIFIER_PATTERN), re.MULTILINE|re.DOTALL) 101 | core_on_hostname = re.search(pattern, zk_output) 102 | if core_on_hostname is not None: 103 | write_locks_to_delete += WRITE_LOCK_PATTERN.format(core_path) 104 | 105 | return write_locks_to_delete 106 | 107 | 108 | def get_write_lock_files_solr_standalone(collections): 109 | write_locks_to_delete = '' 110 | 111 | for collection_path in collections: 112 | write_locks_to_delete += WRITE_LOCK_PATTERN.format(collection_path) 113 | 114 | return write_locks_to_delete 115 | 116 | 117 | def delete_write_lock_files(): 118 | import params 119 | 120 | if params.security_enabled: 121 | kinit_if_needed = format( 122 | '{kinit_path_local} {hdfs_principal_name} -kt {hdfs_user_keytab}; ') 123 | else: 124 | kinit_if_needed = '' 125 | 126 | hadoop_prefix = format('{kinit_if_needed}hadoop --config {hadoop_conf_dir} dfs') 127 | code, output = call(format('{hadoop_prefix} -ls {solr_hdfs_directory}')) 128 | collections = get_collection_paths(output) 129 | 130 | if params.solr_cloud_mode: 131 | write_locks_to_delete = get_write_lock_files_solr_cloud(hadoop_prefix, collections) 132 | else: 133 | write_locks_to_delete = get_write_lock_files_solr_standalone(collections) 134 | 135 | if len(write_locks_to_delete) > 1: 136 | Logger.info(format('For hostname: \'{solr_hostname}\' lock files \'{write_locks_to_delete}\' will be deleted.')) 137 | Execute( 138 | format('{hadoop_prefix} -rm -f {write_locks_to_delete}'), 139 | user=params.hdfs_user 140 | ) 141 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/scripts/status_params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from resource_management.libraries.functions.format import format 4 | from resource_management.libraries.script.script import Script 5 | 6 | config = Script.get_config() 7 | 8 | map_solr_config = config['configurations']['solr-config-env'] 9 | 10 | solr_config_port = map_solr_config['solr_config_port'] 11 | solr_config_pid_dir = map_solr_config['solr_config_pid_dir'] 12 | solr_config_pid_file = format('{solr_config_pid_dir}/solr-{solr_config_port}.pid') 13 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/templates/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | 9 | 10 | 12 | 13 | %d{ISO8601} - %-5p [%t:%C{1}@%L] - %X - %m%n 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/templates/solr_metrics_jaas.conf.j2: -------------------------------------------------------------------------------- 1 | Client { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{solr_metrics_kerberos_keytab}}" 5 | storeKey=true 6 | useTicketCache=false 7 | debug=false 8 | principal="{{solr_metrics_kerberos_principal}}"; 9 | }; 10 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/package/templates/solr_server_jaas.conf.j2: -------------------------------------------------------------------------------- 1 | Client { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="{{solr_kerberos_keytab}}" 5 | storeKey=true 6 | useTicketCache=true 7 | debug=false 8 | principal="{{solr_kerberos_principal}}"; 9 | }; 10 | -------------------------------------------------------------------------------- /src/main/mpack/common-services/SOLR/7.4.0/properties/solr.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 28 | 29 | 30 | 31 | 32 | 33 | ${host:} 34 | ${jetty.port:8983} 35 | ${hostContext:solr} 36 | 37 | ${genericCoreNodeNames:true} 38 | 39 | ${zkClientTimeout:30000} 40 | ${distribUpdateSoTimeout:600000} 41 | ${distribUpdateConnTimeout:60000} 42 | ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} 43 | ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} 44 | 45 | 46 | 47 | 49 | ${socketTimeout:600000} 50 | ${connTimeout:60000} 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/main/mpack/custom-services/SOLR/5.5.5/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 2.0 18 | 19 | 20 | SOLR 21 | 5.5.5 22 | common-services/SOLR/5.5.5 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/main/mpack/custom-services/SOLR/6.6.2/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 2.0 18 | 19 | 20 | SOLR 21 | 6.6.2 22 | common-services/SOLR/6.6.2 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/main/mpack/custom-services/SOLR/7.4.0/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 2.0 18 | 19 | 20 | SOLR 21 | 7.4.0 22 | common-services/SOLR/7.4.0 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/main/mpack/extensions/SOLR/5.5.5/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | HDP 20 | 2.4 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/main/mpack/extensions/SOLR/6.6.2/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | HDP 20 | 2.5 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/main/mpack/extensions/SOLR/7.3.1/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | HDP 20 | 2.6 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/main/mpack/mpack.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "full-release", 3 | "name": "solr-ambari-mpack", 4 | "version": "{VERSION}", 5 | "description": "SOLR Management Pack", 6 | "prerequisites": { 7 | "min_ambari_version": "2.5.0.0", 8 | "max-ambari-version": "", 9 | "min_stack_versions": [ 10 | { 11 | "stack_name": "HDP", 12 | "stack_version": "2.4" 13 | } 14 | ] 15 | }, 16 | "artifacts": [ 17 | { 18 | "name": "solr-service-definitions ", 19 | "type": "service-definitions", 20 | "source_dir": "common-services" 21 | }, 22 | { 23 | "name": "solr-extension-definitions", 24 | "type": "extension-definitions", 25 | "source_dir": "extensions" 26 | }, 27 | { 28 | "name": "solr-addon-service-definitions", 29 | "type": "stack-addon-service-definitions", 30 | "source_dir": "custom-services", 31 | "service_versions_map": [ 32 | { 33 | "service_name" : "SOLR", 34 | "service_version" : "5.5.5", 35 | "applicable_stacks" : [ 36 | { 37 | "stack_name" : "HDP", 38 | "stack_version" : "2.4" 39 | } 40 | ] 41 | }, 42 | { 43 | "service_name": "SOLR", 44 | "service_version": "6.6.2", 45 | "applicable_stacks": [ 46 | { 47 | "stack_name": "HDP", 48 | "stack_version": "2.5" 49 | } 50 | ] 51 | }, 52 | { 53 | "service_name": "SOLR", 54 | "service_version": "7.4.0", 55 | "applicable_stacks": [ 56 | { 57 | "stack_name": "HDP", 58 | "stack_version": "3.0" 59 | } 60 | ] 61 | } 62 | ] 63 | } 64 | ] 65 | } 66 | -------------------------------------------------------------------------------- /src/main/template/repos/repoinfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | http://public-repo-1.hortonworks.com/{REPOID}/repos/centos6 22 | {REPOID} 23 | HDP-SOLR 24 | 25 | 26 | 27 | 28 | http://public-repo-1.hortonworks.com/{REPOID}/repos/centos7 29 | {REPOID} 30 | HDP-SOLR 31 | 32 | 33 | 34 | 35 | 36 | http://public-repo-1.hortonworks.com/{REPOID}/repos/suse11sp3 37 | {REPOID} 38 | HDP-SOLR 39 | 40 | 41 | 42 | 43 | http://public-repo-1.hortonworks.com/{REPOID}/repos/sles12 44 | {REPOID} 45 | HDP-SOLR 46 | 47 | 48 | 49 | 50 | http://public-repo-1.hortonworks.com/{REPOID}/repos/debian6 51 | {REPOID} 52 | HDP-SOLR 53 | 54 | 55 | 56 | 57 | http://public-repo-1.hortonworks.com/{REPOID}/repos/debian7 58 | {REPOID} 59 | HDP-SOLR 60 | 61 | 62 | 63 | 64 | http://public-repo-1.hortonworks.com/{REPOID}/repos/ubuntu12 65 | {REPOID} 66 | HDP-SOLR 67 | 68 | 69 | 70 | 71 | http://public-repo-1.hortonworks.com/{REPOID}/repos/ubuntu14 72 | {REPOID} 73 | HDP-SOLR 74 | 75 | 76 | 77 | --------------------------------------------------------------------------------