├── .gitignore ├── package ├── __init__.py └── scripts │ ├── __init__.py │ ├── download.ini │ ├── common.py │ ├── params.py │ ├── azkaban_executor.py │ └── azkaban_web.py ├── bin ├── start-web.sh ├── start-exec.sh ├── shutdown-web.sh ├── shutdown-exec.sh └── internal │ ├── internal-start-web.sh │ ├── internal-start-executor.sh │ ├── util.sh │ ├── stop-executor.sh │ └── stop-web.sh ├── README.md ├── quicklinks └── quicklinks.json ├── configuration ├── global.properties.xml ├── log4j.properties.xml ├── azkaban-db.xml ├── azkaban-users.xml ├── azkaban-web.properties.xml └── azkaban-executor.properties.xml └── metainfo.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /package/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /package/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/start-web.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | script_dir=$(dirname $0) 4 | 5 | ${script_dir}/internal/internal-start-web.sh >webServerLog_`date +%F+%T`.out 2>&1 & 6 | -------------------------------------------------------------------------------- /bin/start-exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | script_dir=$(dirname $0) 4 | 5 | # pass along command line arguments to the internal launch script. 6 | ${script_dir}/internal/internal-start-executor.sh "$@" >executorServerLog__`date +%F+%T`.out 2>&1 & 7 | 8 | -------------------------------------------------------------------------------- /bin/shutdown-web.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Shutdown script for azkaban web server 3 | set -o nounset 4 | 5 | script_dir=$(dirname $0) 6 | base_dir="${script_dir}/.." 7 | source "${script_dir}/internal/stop-web.sh" 8 | common_shutdown "web-server" ${base_dir} 9 | -------------------------------------------------------------------------------- /package/scripts/download.ini: -------------------------------------------------------------------------------- 1 | [download] 2 | azkaban_web_url = http://fp-bd6/azkaban/azkaban-web-server-3.38.0.tar.gz 3 | azkaban_executor_url = http://fp-bd6/azkaban/azkaban-exec-server-3.38.0.tar.gz 4 | azkaban_db_url = http://fp-bd6/azkaban/create-all-sql-3.38.0.sql 5 | -------------------------------------------------------------------------------- /bin/shutdown-exec.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Shutdown script for azkaban executor server 3 | set -o nounset 4 | 5 | script_dir=$(dirname $0) 6 | base_dir="${script_dir}/.." 7 | source "${script_dir}/internal/stop-executor.sh" 8 | common_shutdown "executor" ${base_dir} 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Intro 2 | Ambari 集成 Azkaban 3 | 4 | 使用前，先clone.代码到本地，选择合适你的分支版本。 5 | 6 | # Major Project Structure 7 | - configuration : azkaban 配置文件 8 | - bin : Azkaban脚本修改（单机部署web、exec需要更换） 9 | - package : 10 | - scripts : ambari 管理逻辑脚本 11 | - azkaban_executor.py 12 | - azkaban_web.py 13 | - common.py 14 | - download.ini 15 | - params.py 16 | 17 | # Deploy 18 | 19 | - 1台服务器同时安装web、executor（需要修改azkaban的脚本，同时启动有冲突） 20 | - web、executor分别部署在多台服务器（无需关心脚本） 21 | 22 | # Usage 23 | 24 | https://cwiki.apache.org/confluence/display/AMBARI/Overview 25 | -------------------------------------------------------------------------------- /quicklinks/quicklinks.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default", 3 | "description": "default quick links configuration", 4 | "configuration": { 5 | "protocol": { 6 | "type": "HTTP_ONLY" 7 | }, 8 | "links": [ 9 | { 10 | "name": "azkaban_web", 11 | "label": "Azkaban Web", 12 | "component_name": "AZKABAN_WEB", 13 | "requires_user_name": "false", 14 | "url": "%@://%@:%@", 15 | "port": { 16 | "http_property": "jetty.port", 17 | "regex": "^(\\d+)$", 18 | "site": "azkaban-web.properties" 19 | } 20 | } 21 | ] 22 | } 23 | } 24 | 25 | -------------------------------------------------------------------------------- /package/scripts/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | import ConfigParser 18 | 19 | script_dir = os.path.dirname(os.path.realpath(__file__)) 20 | config = ConfigParser.ConfigParser() 21 | config.readfp(open(os.path.join(script_dir, 'download.ini'))) 22 | 23 | AZKABAN_HOME = '/usr/hdp/current/azkaban' 24 | AZKABAN_NAME = 'azkaban' 25 | AZKABAN_SQL = 'azkaban.sql' 26 | AZKABAN_WEB_URL = config.get('download', 'azkaban_web_url') 27 | AZKABAN_EXECUTOR_URL = config.get('download', 'azkaban_executor_url') 28 | AZKABAN_DB_URL = config.get('download', 'azkaban_db_url') 29 | AZKABAN_CONF = AZKABAN_HOME + '/conf' 30 | -------------------------------------------------------------------------------- /configuration/global.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | content 21 | global.properties template 22 | Custom global.properties 23 | # 24 | 25 | content 26 | false 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /package/scripts/params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from resource_management.libraries.script.script import Script 17 | 18 | # config object that holds the configurations declared in the config xml file 19 | config = Script.get_config() 20 | 21 | azkaban_web_properties = config['configurations']['azkaban-web.properties'] 22 | azkaban_executor_properties = config['configurations']['azkaban-executor.properties'] 23 | azkaban_users = config['configurations']['azkaban-users'] 24 | azkaban_db = config['configurations']['azkaban-db'] 25 | global_properties = config['configurations']['global.properties'] 26 | log4j_properties = config['configurations']['log4j.properties'] 27 | 28 | host_info = config['clusterHostInfo'] 29 | host_level_params = config['hostLevelParams'] 30 | java_home = host_level_params['java_home'] 31 | -------------------------------------------------------------------------------- /configuration/log4j.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | content 21 | log4j.properties template 22 | Custom log4j.properties 23 | 24 | log4j.rootLogger=INFO, Console 25 | log4j.appender.Console=org.apache.log4j.ConsoleAppender 26 | log4j.appender.Console.layout=org.apache.log4j.PatternLayout 27 | log4j.appender.Console.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS Z} %p [%c{1}] %m%n 28 | log4j.category.velocity=INFO 29 | 30 | 31 | content 32 | false 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /configuration/azkaban-db.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | database.type 22 | mysql 23 | 24 | 25 | mysql.host 26 | fp-bd5 27 | 28 | 29 | mysql.port 30 | 3306 31 | 32 | 33 | mysql.database 34 | azkaban 35 | 36 | 37 | mysql.user 38 | azkaban 39 | 40 | 41 | mysql.password 42 | azkaban 43 | 44 | 45 | mysql.numconnections 46 | 100 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /configuration/azkaban-users.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | content 21 | azkaban-users.xml template 22 | Custom azkaban-users.xml 23 | 24 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | ]]> 43 | 44 | 45 | content 46 | false 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /bin/internal/internal-start-web.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | azkaban_dir=$(dirname $0)/../.. 4 | 5 | # Specifies location of azkaban.properties, log4j.properties files 6 | # Change if necessary 7 | conf=$azkaban_dir/conf 8 | filtr="exec" 9 | if [[ -z "$tmpdir" ]]; then 10 | tmpdir=/tmp 11 | fi 12 | 13 | for file in $azkaban_dir/lib/*.jar; 14 | do 15 | if [[ $file =~ $filtr ]]; 16 | then 17 | echo "" 18 | else 19 | CLASSPATH=$CLASSPATH:$file 20 | fi 21 | done 22 | 23 | for file in $azkaban_dir/extlib/*.jar; 24 | do 25 | CLASSPATH=$CLASSPATH:$file 26 | done 27 | 28 | for file in $azkaban_dir/plugins/*/*.jar; 29 | do 30 | CLASSPATH=$CLASSPATH:$file 31 | done 32 | 33 | if [ "$HADOOP_HOME" != "" ]; then 34 | echo "Using Hadoop from $HADOOP_HOME" 35 | CLASSPATH=$CLASSPATH:$HADOOP_HOME/conf:$HADOOP_HOME/* 36 | JAVA_LIB_PATH="-Djava.library.path=$HADOOP_HOME/lib/native/Linux-amd64-64" 37 | else 38 | echo "Error: HADOOP_HOME is not set. Hadoop job types will not run properly." 39 | fi 40 | 41 | if [ "$HIVE_HOME" != "" ]; then 42 | echo "Using Hive from $HIVE_HOME" 43 | CLASSPATH=$CLASSPATH:$HIVE_HOME/conf:$HIVE_HOME/lib/* 44 | fi 45 | 46 | echo $azkaban_dir; 47 | echo $CLASSPATH; 48 | 49 | executorport=`cat $conf/azkaban.properties | grep executor.port | cut -d = -f 2` 50 | serverpath=`pwd` 51 | 52 | if [[ -z "$AZKABAN_OPTS" ]]; then 53 | AZKABAN_OPTS="-Xmx4G" 54 | fi 55 | # Set the log4j configuration file 56 | if [ -f $conf/log4j.properties ]; then 57 | AZKABAN_OPTS="$AZKABAN_OPTS -Dlog4j.configuration=file:$conf/log4j.properties -Dlog4j.log.dir=$azkaban_dir/logs" 58 | else 59 | echo "Exit with error: $conf/log4j.properties file doesn't exist." 60 | exit 1; 61 | fi 62 | AZKABAN_OPTS="$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath" 63 | 64 | java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $conf $@ & 65 | 66 | echo $! > $azkaban_dir/web_currentpid 67 | 68 | -------------------------------------------------------------------------------- /bin/internal/internal-start-executor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | azkaban_dir=$(dirname $0)/../.. 4 | 5 | # Specifies location of azkaban.properties, log4j.properties files 6 | # Change if necessary 7 | conf=$azkaban_dir/conf 8 | filtr="web" 9 | if [[ -z "$tmpdir" ]]; then 10 | tmpdir=/tmp 11 | fi 12 | 13 | for file in $azkaban_dir/lib/*.jar; 14 | do 15 | if [[ $file =~ $filtr ]]; 16 | then 17 | echo "" 18 | else 19 | CLASSPATH=$CLASSPATH:$file 20 | fi 21 | done 22 | 23 | for file in $azkaban_dir/extlib/*.jar; 24 | do 25 | CLASSPATH=$CLASSPATH:$file 26 | done 27 | 28 | for file in $azkaban_dir/plugins/*/*.jar; 29 | do 30 | CLASSPATH=$CLASSPATH:$file 31 | done 32 | 33 | if [ "$HADOOP_HOME" != "" ]; then 34 | echo "Using Hadoop from $HADOOP_HOME" 35 | CLASSPATH=$CLASSPATH:$HADOOP_HOME/conf:$HADOOP_HOME/* 36 | JAVA_LIB_PATH="-Djava.library.path=$HADOOP_HOME/lib/native/Linux-amd64-64" 37 | else 38 | echo "Error: HADOOP_HOME is not set. Hadoop job types will not run properly." 39 | fi 40 | 41 | if [ "$HIVE_HOME" != "" ]; then 42 | echo "Using Hive from $HIVE_HOME" 43 | CLASSPATH=$CLASSPATH:$HIVE_HOME/conf:$HIVE_HOME/lib/* 44 | fi 45 | 46 | echo $azkaban_dir; 47 | echo $CLASSPATH; 48 | 49 | executorport=`cat $conf/azkaban.properties | grep executor.port | cut -d = -f 2` 50 | echo "Starting AzkabanExecutorServer on port $executorport ..." 51 | serverpath=`pwd` 52 | 53 | if [[ -z "$AZKABAN_OPTS" ]]; then 54 | AZKABAN_OPTS="-Xmx3G" 55 | fi 56 | # Set the log4j configuration file 57 | if [ -f $conf/log4j.properties ]; then 58 | AZKABAN_OPTS="$AZKABAN_OPTS -Dlog4j.configuration=file:$conf/log4j.properties -Dlog4j.log.dir=$azkaban_dir/logs" 59 | else 60 | echo "Exit with error: $conf/log4j.properties file doesn't exist." 61 | exit 1; 62 | fi 63 | AZKABAN_OPTS="$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath" 64 | 65 | java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.execapp.AzkabanExecutorServer -conf $conf $@ & 66 | 67 | echo $! > $azkaban_dir/currentpid 68 | 69 | -------------------------------------------------------------------------------- /bin/internal/util.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Common utils 3 | set -o nounset # exit the script if you try to use an uninitialised variable 4 | set -o errexit # exit the script if any statement returns a non-true return value 5 | 6 | #--- 7 | # is_process_running: Checks if a process is running 8 | # args: Process ID of running proccess 9 | # returns: returns 0 if process is running, 1 if not found 10 | #--- 11 | function is_process_running { 12 | local pid=$1 13 | kill -0 $pid > /dev/null 2>&1 #exit code ($?) is 0 if pid is running, 1 if not running 14 | local status=$? #because we are returning exit code, can use with if & no [ bracket 15 | return $status 16 | } 17 | 18 | #--- 19 | # args: Process name of a running process to shutdown, install directory 20 | # returns: returns 0 if success, 1 otherwise 21 | #--- 22 | function common_shutdown { 23 | process_name="$1" 24 | install_dir="$2" 25 | max_attempt=3 26 | pid=`cat ${install_dir}/currentpid` 27 | 28 | kill_process_with_retry "${pid}" "${process_name}" "${max_attempt}" 29 | 30 | if [[ $? == 0 ]]; then 31 | rm -f ${install_dir}/currentpid 32 | return 0 33 | else 34 | return 1 35 | fi 36 | } 37 | 38 | #--- 39 | # kill_process_with_retry: Checks and attempts to kill the running process 40 | # args: PID, process name, number of kill attempts 41 | # returns: returns 0 if kill succeds or nothing to kill, 1 if kill fails 42 | # exception: If passed a non-existant pid, function will forcefully exit 43 | #--- 44 | function kill_process_with_retry { 45 | local pid="$1" 46 | local pname="$2" 47 | local maxattempt="$3" 48 | local sleeptime=5 49 | 50 | if ! is_process_running $pid ; then 51 | echo "ERROR: process name ${pname} with pid: ${pid} not found" 52 | exit 1 53 | fi 54 | 55 | for try in $(seq 1 $maxattempt); do 56 | echo "Killing $pname. [pid: $pid], attempt: $try" 57 | kill ${pid} 58 | sleep 5 59 | if is_process_running $pid; then 60 | echo "$pname is not dead [pid: $pid]" 61 | echo "sleeping for $sleeptime seconds before retry" 62 | sleep $sleeptime 63 | else 64 | echo "shutdown succeeded" 65 | return 0 66 | fi 67 | done 68 | 69 | echo "Error: unable to kill process for $maxattempt attempt(s), killing the process with -9" 70 | kill -9 $pid 71 | sleep $sleeptime 72 | 73 | if is_process_running $pid; then 74 | echo "$pname is not dead even after kill -9 [pid: $pid]" 75 | return 1 76 | else 77 | echo "shutdown succeeded" 78 | return 0 79 | fi 80 | } 81 | 82 | -------------------------------------------------------------------------------- /bin/internal/stop-executor.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Common utils 3 | set -o nounset # exit the script if you try to use an uninitialised variable 4 | set -o errexit # exit the script if any statement returns a non-true return value 5 | 6 | #--- 7 | # is_process_running: Checks if a process is running 8 | # args: Process ID of running proccess 9 | # returns: returns 0 if process is running, 1 if not found 10 | #--- 11 | function is_process_running { 12 | local pid=$1 13 | kill -0 $pid > /dev/null 2>&1 #exit code ($?) is 0 if pid is running, 1 if not running 14 | local status=$? #because we are returning exit code, can use with if & no [ bracket 15 | return $status 16 | } 17 | 18 | #--- 19 | # args: Process name of a running process to shutdown, install directory 20 | # returns: returns 0 if success, 1 otherwise 21 | #--- 22 | function common_shutdown { 23 | process_name="$1" 24 | install_dir="$2" 25 | max_attempt=3 26 | if [ ! -f "${install_dir}/currentpid" ]; then 27 | return 0 28 | fi 29 | pid=`cat ${install_dir}/currentpid` 30 | kill_process_with_retry "${pid}" "${process_name}" "${max_attempt}" 31 | 32 | if [[ $? == 0 ]]; then 33 | rm -f ${install_dir}/currentpid 34 | return 0 35 | else 36 | return 1 37 | fi 38 | } 39 | 40 | #--- 41 | # kill_process_with_retry: Checks and attempts to kill the running process 42 | # args: PID, process name, number of kill attempts 43 | # returns: returns 0 if kill succeds or nothing to kill, 1 if kill fails 44 | # exception: If passed a non-existant pid, function will forcefully exit 45 | #--- 46 | function kill_process_with_retry { 47 | local pid="$1" 48 | local pname="$2" 49 | local maxattempt="$3" 50 | local sleeptime=5 51 | 52 | if ! is_process_running $pid ; then 53 | echo "ERROR: process name ${pname} with pid: ${pid} not found" 54 | exit 1 55 | fi 56 | 57 | for try in $(seq 1 $maxattempt); do 58 | echo "Killing $pname. [pid: $pid], attempt: $try" 59 | kill ${pid} 60 | sleep 5 61 | if is_process_running $pid; then 62 | echo "$pname is not dead [pid: $pid]" 63 | echo "sleeping for $sleeptime seconds before retry" 64 | sleep $sleeptime 65 | else 66 | echo "shutdown succeeded" 67 | return 0 68 | fi 69 | done 70 | 71 | echo "Error: unable to kill process for $maxattempt attempt(s), killing the process with -9" 72 | kill -9 $pid 73 | sleep $sleeptime 74 | 75 | if is_process_running $pid; then 76 | echo "$pname is not dead even after kill -9 [pid: $pid]" 77 | return 1 78 | else 79 | echo "shutdown succeeded" 80 | return 0 81 | fi 82 | } 83 | 84 | -------------------------------------------------------------------------------- /bin/internal/stop-web.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Common utils 3 | set -o nounset # exit the script if you try to use an uninitialised variable 4 | set -o errexit # exit the script if any statement returns a non-true return value 5 | 6 | #--- 7 | # is_process_running: Checks if a process is running 8 | # args: Process ID of running proccess 9 | # returns: returns 0 if process is running, 1 if not found 10 | #--- 11 | function is_process_running { 12 | local pid=$1 13 | kill -0 $pid > /dev/null 2>&1 #exit code ($?) is 0 if pid is running, 1 if not running 14 | local status=$? #because we are returning exit code, can use with if & no [ bracket 15 | return $status 16 | } 17 | 18 | #--- 19 | # args: Process name of a running process to shutdown, install directory 20 | # returns: returns 0 if success, 1 otherwise 21 | #--- 22 | function common_shutdown { 23 | process_name="$1" 24 | install_dir="$2" 25 | max_attempt=3 26 | if [ ! -f "${install_dir}/web_currentpid" ]; then 27 | return 0 28 | fi 29 | pid=`cat ${install_dir}/web_currentpid` 30 | kill_process_with_retry "${pid}" "${process_name}" "${max_attempt}" 31 | 32 | if [[ $? == 0 ]]; then 33 | rm -f ${install_dir}/web_currentpid 34 | return 0 35 | else 36 | return 1 37 | fi 38 | } 39 | 40 | #--- 41 | # kill_process_with_retry: Checks and attempts to kill the running process 42 | # args: PID, process name, number of kill attempts 43 | # returns: returns 0 if kill succeds or nothing to kill, 1 if kill fails 44 | # exception: If passed a non-existant pid, function will forcefully exit 45 | #--- 46 | function kill_process_with_retry { 47 | local pid="$1" 48 | local pname="$2" 49 | local maxattempt="$3" 50 | local sleeptime=5 51 | 52 | if ! is_process_running $pid ; then 53 | echo "ERROR: process name ${pname} with pid: ${pid} not found" 54 | exit 1 55 | fi 56 | 57 | for try in $(seq 1 $maxattempt); do 58 | echo "Killing $pname. [pid: $pid], attempt: $try" 59 | kill ${pid} 60 | sleep 5 61 | if is_process_running $pid; then 62 | echo "$pname is not dead [pid: $pid]" 63 | echo "sleeping for $sleeptime seconds before retry" 64 | sleep $sleeptime 65 | else 66 | echo "shutdown succeeded" 67 | return 0 68 | fi 69 | done 70 | 71 | echo "Error: unable to kill process for $maxattempt attempt(s), killing the process with -9" 72 | kill -9 $pid 73 | sleep $sleeptime 74 | 75 | if is_process_running $pid; then 76 | echo "$pname is not dead even after kill -9 [pid: $pid]" 77 | return 1 78 | else 79 | echo "shutdown succeeded" 80 | return 0 81 | fi 82 | } 83 | 84 | -------------------------------------------------------------------------------- /metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 2.0 19 | 20 | 21 | AZKABAN 22 | Azkaban 23 | Azkaban is a batch workflow job scheduler created at LinkedIn to run Hadoop jobs. Azkaban resolves 24 | the ordering through job dependencies and provides an easy to use web user interface to maintain and 25 | track your workflows. 26 | 27 | 3.81.4 28 | 29 | 30 | quicklinks.json 31 | true 32 | 33 | 34 | 35 | 36 | AZKABAN_WEB 37 | Azkaban Web Server 38 | MASTER 39 | 1 40 | true 41 | 42 | 43 | PYTHON 44 | 1200 45 | 46 | 47 | 48 | 49 | AZKABAN_EXECUTOR 50 | Azkaban Executor Server 51 | SLAVE 52 | 0+ 53 | true 54 | 55 | 56 | PYTHON 57 | 58 | 59 | 60 | 61 | 62 | 63 | azkaban-executor.properties.xml 64 | azkaban-users.xml 65 | azkaban-web.properties.xml 66 | log4j.properties.xml 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /package/scripts/azkaban_executor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os.path as path 16 | import time 17 | import socket 18 | from common import AZKABAN_EXECUTOR_URL, AZKABAN_NAME, AZKABAN_HOME, AZKABAN_CONF 19 | from resource_management.core.exceptions import ExecutionFailed, ComponentIsNotRunning 20 | from resource_management.core.resources.system import Execute 21 | from resource_management.libraries.script.script import Script 22 | 23 | 24 | class ExecutorServer(Script): 25 | def install(self, env): 26 | #from params import java_home 27 | java_home='/usr/local/jdk1.8.0_171' 28 | Execute('wget --no-check-certificate {0} -O /tmp/{1}'.format(AZKABAN_EXECUTOR_URL, AZKABAN_NAME)) 29 | Execute( 30 | 'mkdir -p {0} {1} {2} || echo "whatever"'.format( 31 | AZKABAN_HOME + '/conf', 32 | AZKABAN_HOME + '/extlib', 33 | AZKABAN_HOME + '/plugins/jobtypes', 34 | ) 35 | ) 36 | Execute('echo execute.as.user=false > {0} '.format(AZKABAN_HOME + '/plugins/jobtypes/commonprivate.properties')) 37 | Execute( 38 | 'export JAVA_HOME={0} && tar -xf /tmp/{1} -C {2} --strip-components 1'.format( 39 | java_home, 40 | AZKABAN_NAME, 41 | AZKABAN_HOME 42 | ) 43 | ) 44 | self.configure(env) 45 | 46 | def stop(self, env): 47 | self.configure(env) 48 | Execute('cd {0} && bin/shutdown-exec.sh'.format(AZKABAN_HOME)) 49 | 50 | def start(self, env): 51 | from params import azkaban_executor_properties 52 | self.configure(env) 53 | executor_port = int(azkaban_executor_properties['executor.port']) 54 | Execute('cd {0} && bin/start-exec.sh'.format(AZKABAN_HOME)) 55 | while 1: 56 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 57 | result = sock.connect_ex(('127.0.0.1',executor_port)) 58 | sock.close() 59 | if result == 0: 60 | Execute('curl http://localhost:{0}/executor?action=activate'.format(executor_port)) 61 | break 62 | else: 63 | time.sleep(5) 64 | 65 | def status(self, env): 66 | try: 67 | self.configure(env) 68 | Execute( 69 | 'export AZ_CNT=`ps -ef |grep -v grep |grep azkaban-exec-server | wc -l` && `if [ $AZ_CNT -ne 0 ];then exit 0;else exit 3;fi `' 70 | ) 71 | except ExecutionFailed as ef: 72 | if ef.code == 3: 73 | raise ComponentIsNotRunning("ComponentIsNotRunning") 74 | else: 75 | raise ef 76 | 77 | def configure(self, env): 78 | from params import azkaban_executor_properties, log4j_properties, azkaban_db 79 | key_val_template = '{0}={1}\n' 80 | 81 | with open(path.join(AZKABAN_CONF, 'azkaban.properties'), 'w') as f: 82 | for key, value in azkaban_db.iteritems(): 83 | f.write(key_val_template.format(key, value)) 84 | for key, value in azkaban_executor_properties.iteritems(): 85 | if key != 'content': 86 | f.write(key_val_template.format(key, value)) 87 | #f.write(azkaban_executor_properties['content']) 88 | if azkaban_executor_properties.has_key('content'): 89 | f.write(str(azkaban_executor_properties['content'])) 90 | 91 | with open(path.join(AZKABAN_CONF, 'log4j.properties'), 'w') as f: 92 | #f.write(log4j_properties['content']) 93 | if log4j_properties.has_key('content'): 94 | f.write(str(log4j_properties['content'])) 95 | 96 | 97 | if __name__ == '__main__': 98 | ExecutorServer().execute() 99 | -------------------------------------------------------------------------------- /package/scripts/azkaban_web.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os.path as path 16 | 17 | from common import AZKABAN_DB_URL, AZKABAN_WEB_URL, AZKABAN_NAME, AZKABAN_HOME, AZKABAN_CONF, AZKABAN_SQL 18 | from resource_management.core.exceptions import ExecutionFailed, ComponentIsNotRunning 19 | from resource_management.core.resources.system import Execute 20 | from resource_management.libraries.script.script import Script 21 | 22 | 23 | class WebServer(Script): 24 | def install(self, env): 25 | #from params import java_home, azkaban_db 26 | from params import azkaban_db 27 | java_home='/usr/local/jdk1.8.0_171' 28 | Execute('wget --no-check-certificate {0} -O /tmp/{1}'.format(AZKABAN_WEB_URL, AZKABAN_NAME)) 29 | Execute('wget --no-check-certificate {0} -O /tmp/{1}'.format(AZKABAN_DB_URL, AZKABAN_SQL)) 30 | Execute( 31 | 'mysql -h{0} -P{1} -D{2} -u{3} -p{4} < {5}'.format( 32 | azkaban_db['mysql.host'], 33 | azkaban_db['mysql.port'], 34 | azkaban_db['mysql.database'], 35 | azkaban_db['mysql.user'], 36 | azkaban_db['mysql.password'], 37 | '/tmp/{0}'.format(AZKABAN_SQL), 38 | ) 39 | ) 40 | Execute( 41 | 'mkdir -p {0} {1} {2} || echo "whatever"'.format( 42 | AZKABAN_HOME + '/conf', 43 | AZKABAN_HOME + '/extlib', 44 | AZKABAN_HOME + '/plugins/jobtypes', 45 | ) 46 | ) 47 | Execute('echo execute.as.user=false > {0} '.format(AZKABAN_HOME + '/plugins/jobtypes/commonprivate.properties')) 48 | Execute( 49 | 'export JAVA_HOME={0} && tar -xf /tmp/{1} -C {2} --strip-components 1'.format( 50 | java_home, 51 | AZKABAN_NAME, 52 | AZKABAN_HOME 53 | ) 54 | ) 55 | self.configure(env) 56 | 57 | def stop(self, env): 58 | self.configure(env) 59 | Execute('cd {0} && bin/shutdown-web.sh'.format(AZKABAN_HOME)) 60 | 61 | def start(self, env): 62 | self.configure(env) 63 | Execute('cd {0} && bin/start-web.sh'.format(AZKABAN_HOME)) 64 | 65 | def status(self, env): 66 | try: 67 | self.configure(env) 68 | Execute( 69 | 'export AZ_CNT=`ps -ef |grep -v grep |grep azkaban-web-server | wc -l` && `if [ $AZ_CNT -ne 0 ];then exit 0;else exit 3;fi `' 70 | ) 71 | except ExecutionFailed as ef: 72 | if ef.code == 3: 73 | raise ComponentIsNotRunning("ComponentIsNotRunning") 74 | else: 75 | raise ef 76 | 77 | def configure(self, env): 78 | from params import azkaban_db, azkaban_web_properties, azkaban_users, global_properties, log4j_properties 79 | key_val_template = '{0}={1}\n' 80 | 81 | with open(path.join(AZKABAN_CONF, 'azkaban.properties'), 'w') as f: 82 | for key, value in azkaban_db.iteritems(): 83 | f.write(key_val_template.format(key, value)) 84 | for key, value in azkaban_web_properties.iteritems(): 85 | if key != 'content': 86 | f.write(key_val_template.format(key, value)) 87 | #f.write(azkaban_web_properties['content']) 88 | if azkaban_web_properties.has_key('content'): 89 | f.write(str(azkaban_web_properties['content'])) 90 | 91 | with open(path.join(AZKABAN_CONF, 'azkaban-users.xml'), 'w') as f: 92 | if azkaban_users.has_key('content'): 93 | f.write(str(azkaban_users['content'])) 94 | 95 | with open(path.join(AZKABAN_CONF, 'global.properties'), 'w') as f: 96 | #f.write(global_properties['content']) 97 | if global_properties.has_key('content'): 98 | f.write(str(global_properties['content'])) 99 | 100 | with open(path.join(AZKABAN_CONF, 'log4j.properties'), 'w') as f: 101 | #f.write(log4j_properties['content']) 102 | if log4j_properties.has_key('content'): 103 | f.write(str(log4j_properties['content'])) 104 | 105 | 106 | if __name__ == '__main__': 107 | WebServer().execute() 108 | -------------------------------------------------------------------------------- /configuration/azkaban-web.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | java_home 23 | /usr/local/jdk1.8.0_171 24 | 25 | 26 | azkaban.name 27 | azkaban 28 | 29 | 30 | azkaban.label 31 | test 32 | 33 | 34 | azkaban.color 35 | #FF3601 36 | 37 | 38 | azkaban.default.servlet.path 39 | /index 40 | 41 | 42 | web.resource.dir 43 | web/ 44 | 45 | 46 | default.timezone.id 47 | Asia/Shanghai 48 | 49 | 50 | 51 | user.manager.class 52 | azkaban.user.XmlUserManager 53 | 54 | 55 | user.manager.xml.file 56 | conf/azkaban-users.xml 57 | 58 | 59 | 60 | executor.global.properties 61 | conf/global.properties 62 | 63 | 64 | azkaban.project.dir 65 | projects 66 | 67 | 68 | 69 | velocity.dev.mode 70 | false 71 | 72 | 73 | 74 | jetty.use.ssl 75 | false 76 | 77 | 78 | jetty.maxThreads 79 | 25 80 | 81 | 82 | jetty.port 83 | 10200 84 | 85 | 86 | 87 | 88 | mail.sender 89 | azkaban@email.com 90 | 91 | 92 | mail.host 93 | azkaban@email.com 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | job.failure.email 104 | azkaban@email.com 105 | 106 | 107 | job.success.email 108 | azkaban@email.com 109 | 110 | 111 | lockdown.create.projects 112 | false 113 | 114 | 115 | cache.directory 116 | cache 117 | 118 | 119 | 120 | jetty.connector.stats 121 | true 122 | 123 | 124 | executor.connector.stats 125 | true 126 | 127 | 128 | 129 | database.type 130 | mysql 131 | 132 | 133 | mysql.port 134 | 3306 135 | 136 | 137 | mysql.host 138 | localhost 139 | 140 | 141 | mysql.database 142 | azkaban 143 | 144 | 145 | mysql.user 146 | azkaban 147 | 148 | 149 | mysql.password 150 | azkaban 151 | 152 | 153 | mysql.numconnections 154 | 100 155 | 156 | 157 | executor.port 158 | 12321 159 | 160 | 161 | 162 | azkaban.use.multiple.executors 163 | true 164 | 165 | 166 | azkaban.executorselector.filters 167 | StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus 168 | 169 | 170 | azkaban.executorselector.comparator.NumberOfAssignedFlowComparator 171 | 1 172 | 173 | 174 | azkaban.executorselector.comparator.Memory 175 | 1 176 | 177 | 178 | azkaban.executorselector.comparator.LastDispatched 179 | 1 180 | 181 | 182 | azkaban.executorselector.comparator.CpuUsage 183 | 1 184 | 185 | 186 | 187 | -------------------------------------------------------------------------------- /configuration/azkaban-executor.properties.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | java_home 23 | /usr/local/jdk1.8.0_171 24 | 25 | 26 | azkaban.name 27 | azkaban 28 | 29 | 30 | azkaban.label 31 | test 32 | 33 | 34 | azkaban.color 35 | #FF3601 36 | 37 | 38 | azkaban.default.servlet.path 39 | /index 40 | 41 | 42 | web.resource.dir 43 | web/ 44 | 45 | 46 | default.timezone.id 47 | Asia/Shanghai 48 | 49 | 50 | 51 | user.manager.class 52 | azkaban.user.XmlUserManager 53 | 54 | 55 | user.manager.xml.file 56 | conf/azkaban-users.xml 57 | 58 | 59 | 60 | executor.global.properties 61 | conf/global.properties 62 | 63 | 64 | azkaban.project.dir 65 | projects 66 | 67 | 68 | 69 | velocity.dev.mode 70 | false 71 | 72 | 73 | 74 | jetty.use.ssl 75 | false 76 | 77 | 78 | jetty.maxThreads 79 | 25 80 | 81 | 82 | jetty.port 83 | 10200 84 | 85 | 86 | 87 | 88 | azkaban.webserver.url 89 | http://localhost:10200 90 | 91 | 92 | 93 | mail.sender 94 | azkaban@email.com 95 | 96 | 97 | mail.host 98 | azkaban@email.com 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | job.failure.email 109 | azkaban@email.com 110 | 111 | 112 | job.success.email 113 | azkaban@email.com 114 | 115 | 116 | lockdown.create.projects 117 | false 118 | 119 | 120 | cache.directory 121 | cache 122 | 123 | 124 | 125 | jetty.connector.stats 126 | true 127 | 128 | 129 | executor.connector.stats 130 | true 131 | 132 | 133 | 134 | azkaban.jobtype.plugin.dir 135 | plugins/jobtypes 136 | 137 | 138 | 139 | database.type 140 | mysql 141 | 142 | 143 | mysql.port 144 | 3306 145 | 146 | 147 | mysql.host 148 | localhost 149 | 150 | 151 | mysql.database 152 | azkaban 153 | 154 | 155 | mysql.user 156 | azkaban 157 | 158 | 159 | mysql.password 160 | azkaban 161 | 162 | 163 | mysql.numconnections 164 | 100 165 | 166 | 167 | 168 | executor.maxThreads 169 | 50 170 | 171 | 172 | executor.flow.threads 173 | 30 174 | 175 | 176 | executor.port 177 | 12321 178 | 179 | 180 | 181 | azkaban.use.multiple.executors 182 | true 183 | 184 | 185 | azkaban.executorselector.filters 186 | StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus 187 | 188 | 189 | azkaban.executorselector.comparator.NumberOfAssignedFlowComparator 190 | 1 191 | 192 | 193 | azkaban.executorselector.comparator.Memory 194 | 1 195 | 196 | 197 | azkaban.executorselector.comparator.LastDispatched 198 | 1 199 | 200 | 201 | azkaban.executorselector.comparator.CpuUsage 202 | 1 203 | 204 | 205 | --------------------------------------------------------------------------------