├── README.md ├── autoscaling-lifecycle-hook ├── config.json ├── drain-document.json ├── kube-lambda.zip ├── lambda │ └── index.js └── put-lifecyclehook.js └── memory-based-autoscaling.sh /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Scripts 2 | -------------------------------------------------------------------------------- /autoscaling-lifecycle-hook/config.json: -------------------------------------------------------------------------------- 1 | { "accessKeyId": "xxxxx", "secretAccessKey": "xxxxxxxxxxx", "region": "us-east-1" } 2 | -------------------------------------------------------------------------------- /autoscaling-lifecycle-hook/drain-document.json: -------------------------------------------------------------------------------- 1 | { 2 | "schemaVersion": "1.2", 3 | "description": "Draining Node", 4 | "parameters": { 5 | "nodename":{ 6 | "type":"String", 7 | "description":"Specify the Node name to drain" 8 | } 9 | }, 10 | "runtimeConfig": { 11 | "aws:runShellScript": { 12 | "properties": [ 13 | { 14 | "id": "0.aws:runShellScript", 15 | "runCommand": [ 16 | "#!/bin/bash", 17 | "kubectl drain --force {{ nodename }}" 18 | ] 19 | } 20 | ] 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /autoscaling-lifecycle-hook/kube-lambda.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerupcloud/kubernetes-1/da0239887a390ad2957302abc32f089e6143bc39/autoscaling-lifecycle-hook/kube-lambda.zip -------------------------------------------------------------------------------- /autoscaling-lifecycle-hook/lambda/index.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var ec2 = new AWS.EC2(); 3 | var as = new AWS.AutoScaling(); 4 | var ssm = new AWS.SSM(); 5 | 6 | var async = require('async'); 7 | var sleep = require('sleep'); 8 | 9 | var documentName = 'kubernetesDrainNode' ; //name of the document to be executed on nodes 10 | 11 | exports.handler = function(notification, context) { 12 | 13 | console.log("INFO: request Recieved.\nDetails:\n", JSON.stringify(notification)); 14 | var message = JSON.parse(notification.Records[0].Sns.Message); 15 | console.log("DEBUG: SNS message contents. \nMessage:\n", message); 16 | 17 | var instanceId = message.EC2InstanceId; 18 | 19 | console.log(instanceId); 20 | var lifecycleParams = { 21 | "AutoScalingGroupName": message.AutoScalingGroupName, 22 | "LifecycleHookName": message.LifecycleHookName, 23 | "LifecycleActionToken": message.LifecycleActionToken, 24 | "LifecycleActionResult": "CONTINUE" 25 | }; 26 | 27 | ec2.describeInstances({ 28 | InstanceIds: [instanceId] 29 | }, function(err, data) { 30 | if (err) console.log(err, err.stack); // an error occurred 31 | else { 32 | console.log(data); 33 | nodename = data.Reservations[0].Instances[0].PrivateDnsName; 34 | if(nodename == '') console.log("No Nodes to be drained"); 35 | else{ 36 | console.log("Node that is going to be drained: ", nodename); 37 | executeCommand(nodename, lifecycleParams, context); 38 | } 39 | } 40 | }); 41 | }; 42 | 43 | function executeCommand(nodename, lifecycleParams, context) { 44 | var ssmparams = { 45 | DocumentName: documentName, 46 | Comment: 'Draining Node', //any comment 47 | OutputS3BucketName: 'xxxxxxxx', //save the logs in this bucket 48 | OutputS3KeyPrefix: 'ssm-logs', //bucket prefix 49 | OutputS3Region: 'us-east-1', //region of bucket 50 | Targets: [{ 51 | Key: 'tag:master', 52 | Values: [ 53 | 'yes' 54 | ] 55 | }], // execute the command on the server with this tag 56 | Parameters: { 57 | 'nodename': [ 58 | nodename 59 | ] 60 | } 61 | }; 62 | ssm.sendCommand(ssmparams, function(err, data) { 63 | if (err) console.log(err, err.stack); 64 | else { 65 | console.log(data); 66 | commandid = data.Command.CommandId; 67 | waitCommandSuccess(commandid, function waitCommandReadyCallback(err) { 68 | if (err) { 69 | console.log("ERROR: Failure waiting for Command to be Success"); 70 | } else 71 | console.log("Command Status is Success"); 72 | completeAsLifecycleAction(lifecycleParams, function lifecycleActionResponseHandler(err) { 73 | if (err) { 74 | context.fail(); 75 | } else { 76 | //if we successfully notified AutoScaling of the instance status, tell lambda we succeeded 77 | //even if the operation on the instance failed 78 | context.succeed(); 79 | } 80 | }); 81 | }); 82 | } 83 | }); 84 | } 85 | 86 | function waitCommandSuccess(commandid, waitCommandReadyCallback) { 87 | var commandStatus = undefined; 88 | async.until( 89 | function isSuccess(err) { 90 | return commandStatus === "Success"; 91 | }, 92 | function getCommandStatus(getCommandStatusCallback) { 93 | ssm.listCommands({ 94 | CommandId: commandid 95 | }, function(err, data) { 96 | if (err) console.log(err, err.stack); 97 | else { 98 | console.log(data.Commands[0].Status); 99 | commandStatus = data.Commands[0].Status; 100 | sleep.sleep(2); 101 | getCommandStatusCallback(err) 102 | } 103 | }); 104 | }, 105 | function waitCommandReadyCallbackClosure(err) { 106 | if (err) { 107 | console.log("ERROR: error waiting for Command to be success:\n", err); 108 | } 109 | waitCommandReadyCallback(err); 110 | } 111 | ); 112 | } 113 | 114 | function completeAsLifecycleAction(lifecycleParams, callback) { 115 | //returns true on success or false on failure 116 | //notifies AutoScaling that it should either continue or abandon the instance 117 | as.completeLifecycleAction(lifecycleParams, function(err, data) { 118 | if (err) { 119 | console.log("ERROR: AS lifecycle completion failed.\nDetails:\n", err); 120 | console.log("DEBUG: CompleteLifecycleAction\nParams:\n", lifecycleParams); 121 | callback(err); 122 | } else { 123 | console.log("INFO: CompleteLifecycleAction Successful.\nReported:\n", data); 124 | callback(null); 125 | } 126 | }); 127 | } 128 | -------------------------------------------------------------------------------- /autoscaling-lifecycle-hook/put-lifecyclehook.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | AWS.config.loadFromPath('./config.json'); 3 | var as = new AWS.AutoScaling(); 4 | 5 | asg_name = "kubernetes-minion-group-us-east-1b" ; 6 | hook_name = "kube-lifecycle-hook" 7 | 8 | var params = { 9 | AutoScalingGroupName: asg_name, 10 | LifecycleHookName: hook_name, 11 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 12 | NotificationTargetARN: "arn:aws:sns:us-east-1:xxxxxxx:xxxx-kube-lifecyclehook01", 13 | RoleARN: "arn:aws:iam::xxxxxxxx:role/kubernetes-lifecycle-hook-role" 14 | }; 15 | as.putLifecycleHook(params, function(err, data) { 16 | if (err) console.log(err, err.stack); // an error occurred 17 | else console.log(data); // successful response 18 | /* 19 | data = { 20 | } 21 | */ 22 | }); 23 | -------------------------------------------------------------------------------- /memory-based-autoscaling.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | heapmem=0 4 | podmem=0 5 | TODAY=`date +%F` 6 | KUBECTL=/usr/local/bin/kubectl 7 | SCRIPT_HOME=/var/log/kube-deploy 8 | if [ ! -d $SCRIPT_HOME ]; then 9 | mkdir -p $SCRIPT_HOME 10 | fi 11 | #LOG_FILE=$SCRIPT_HOME/kube-$TODAY.log 12 | #touch $LOG_FILE 13 | RED='\033[01;31m' 14 | YELLOW='\033[0;33m' 15 | NONE='\033[00m' 16 | 17 | print_help(){ 18 | echo -e "${YELLOW}Use the following Command:" 19 | echo -e "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" 20 | echo -e "${RED}./ --action --deployment --scaleup --scaledown " 21 | echo -e "${YELLOW}+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" 22 | printf "Choose one of the available actions below:\n" 23 | printf " get-heapmemory\n get-podmemory\n deploy-heap-autoscaling\n deploy-pod-autoscaling\n" 24 | echo -e "You can get the list of existing deployments using command: kubectl get deployments${NONE}" 25 | } 26 | ARG="$#" 27 | if [[ $ARG -eq 0 ]]; then 28 | print_help 29 | exit 30 | fi 31 | 32 | while test -n "$1"; do 33 | case "$1" in 34 | --action) 35 | ACTION=$2 36 | shift 37 | ;; 38 | --deployment) 39 | DEPLOYMENT=$2 40 | shift 41 | ;; 42 | --scaleup) 43 | SCALEUPTHRESHOLD=$2 44 | shift 45 | ;; 46 | --scaledown) 47 | SCALEDOWNTHRESHOLD=$2 48 | shift 49 | ;; 50 | *) 51 | print_help 52 | exit 53 | ;; 54 | esac 55 | shift 56 | done 57 | 58 | LOG_FILE=$SCRIPT_HOME/kube-$DEPLOYMENT-$TODAY.log 59 | touch $LOG_FILE 60 | 61 | REPLICAS=`$KUBECTL get deployment -l name=$DEPLOYMENT | awk '{print $3}' | grep -v "CURRENT"` 62 | ######################################### 63 | #defining function to calculate heap memory 64 | 65 | calculate_heap(){ 66 | echo "===========================" >> $LOG_FILE 67 | pods=`$KUBECTL get pod -l name=$DEPLOYMENT | awk '{print $1}' | grep -v NAME` 68 | for i in $pods 69 | do 70 | echo "Pod: "$i >> $LOG_FILE 71 | 72 | PID=`$KUBECTL exec -it $i -- ps -ef | grep -v grep | grep java | awk '{print $2}'` >> $LOG_FILE 73 | 74 | TOTALHEAP=`$KUBECTL exec -t $i -- ps -ef | grep java | grep -v grep | awk -F'Xmx' '{print $2}' | awk '{print $1}' | grep -o '[0-9]\+[a-z]'` 75 | 76 | if [[ $TOTALHEAP =~ .*g.* ]]; then 77 | TOTALHEAPINGB=${TOTALHEAP//[!0-9]/} 78 | TOTALHEAPINMB=$((TOTALHEAPINGB * 1024)) 79 | echo "Total Heap Capacity Allocated: "$TOTALHEAPINMB"MB" >> $LOG_FILE 80 | elif [[ $TOTALHEAP =~ .*m.* ]]; then 81 | TOTALHEAPINMB=${TOTALHEAP//[!0-9]/} 82 | echo "Total Heap Capacity Allocated: "$TOTALHEAPINMB"MB" >> $LOG_FILE 83 | fi 84 | 85 | USEDHEAP=`$KUBECTL exec -it $i -- jstat -gc $PID | tail -n 1 | awk '{ print ($3 + $4 + $6 + $8 + $10) / 1024 }'` 86 | echo "Used Heap Memory: "$USEDHEAP"MB" >> $LOG_FILE 87 | 88 | UTILIZEDHEAP=$(awk "BEGIN { pc=100*${USEDHEAP}/${TOTALHEAPINMB}; i=int(pc); print (pc-i<0.5)?i:i+1 }") 89 | echo "Heap memory Percent: "$UTILIZEDHEAP"%" >> $LOG_FILE 90 | 91 | heapmem=$((heapmem+UTILIZEDHEAP)) 92 | echo "===========================" >> $LOG_FILE 93 | done 94 | AVGHEAPMEM=$(( $heapmem/$REPLICAS )) 95 | echo "Average Heap Memory: "$AVGHEAPMEM >> $LOG_FILE 96 | } 97 | 98 | ######################################### 99 | #defining function to autoscale based on heap memory 100 | 101 | heapmemory_autoscale(){ 102 | if [ $AVGHEAPMEM -gt $SCALEUPTHRESHOLD ] 103 | then 104 | echo "Memory is greater than the threshold" >> $LOG_FILE 105 | count=$((REPLICAS+1)) 106 | echo "Updated No. of Replicas will be: "$count >> $LOG_FILE 107 | scale=`$KUBECTL scale --replicas=$count deployment/$DEPLOYMENT` 108 | echo "Deployment Scaled Up" >> $LOG_FILE 109 | 110 | elif [ $AVGHEAPMEM -lt $SCALEDOWNTHRESHOLD ] && [ $REPLICAS -gt 2 ] 111 | then 112 | echo "Memory is less than threshold" >> $LOG_FILE 113 | count=$((REPLICAS-1)) 114 | echo "Updated No. of Replicas will be: "$count >> $LOG_FILE 115 | scale=`$KUBECTL scale --replicas=$count deployment/$DEPLOYMENT` 116 | echo "Deployment Scaled Down" >> $LOG_FILE 117 | else 118 | echo "Heap Memory is not crossing the threshold. No Scaling Done." >> $LOG_FILE 119 | fi 120 | } 121 | 122 | ########################################## 123 | #defining function to calculate pod memory 124 | 125 | calculate_podmemory(){ 126 | pods=`$KUBECTL top pod -l name=$DEPLOYMENT | awk '{print $3}' | grep -o '[0-9]\+'` 127 | 128 | TOTALMEM=`$KUBECTL describe pod -l name=$DEPLOYMENT | grep -A 2 "Limits:" | grep memory | grep -o '[0-9]\+[A-Z]' | head -1` 129 | if [[ $TOTALMEM =~ .*G.* ]]; then 130 | TOTALMEMINGB=${TOTALMEM//[!0-9]/} 131 | TOTALMEMINMB=$((TOTALMEMINGB * 1024)) 132 | echo "Total Pod Memory Allocated: "$TOTALMEMINMB"MB" >> $LOG_FILE 133 | echo "===========================" >> $LOG_FILE 134 | elif [[ $TOTALMEM =~ .*M.* ]]; then 135 | TOTALMEMINMB=${TOTALMEM//[!0-9]/} 136 | echo "Total Pod Memory Allocated: "$TOTALMEMINMB"MB" >> $LOG_FILE 137 | echo "===========================" >> $LOG_FILE 138 | fi 139 | 140 | for i in $pods 141 | do 142 | podmem=$((podmem+i)) 143 | echo "Used Pod Memory: "$podmem >> $LOG_FILE 144 | UTILIZEDPODMEM=$(awk "BEGIN { pc=100*${podmem}/${TOTALMEMINMB}; i=int(pc); print (pc-i<0.5)?i:i+1 }") 145 | echo "Pod memory Percent: "$UTILIZEDPODMEM"%" >> $LOG_FILE 146 | echo "===========================" >> $LOG_FILE 147 | done 148 | AVGPODMEM=$(( $UTILIZEDPODMEM/$REPLICAS )) 149 | echo "Average Pod Memory: "$AVGPODMEM >> $LOG_FILE 150 | } 151 | 152 | ########################################## 153 | #defining function to autoscale based on pod memory 154 | 155 | podmemory_autoscale(){ 156 | if [ $AVGPODMEM -gt $SCALEUPTHRESHOLD ] 157 | then 158 | echo "Memory is greater than threshold" >> $LOG_FILE 159 | count=$((REPLICAS+1)) 160 | echo "Updated No. of Replicas will be: "$count >> $LOG_FILE 161 | scale=`$KUBECTL scale --replicas=$count deployment/$DEPLOYMENT` 162 | echo "Deployment Scaled Up" >> $LOG_FILE 163 | 164 | elif [ $AVGPODMEM -lt $SCALEDOWNTHRESHOLD ] && [ $REPLICAS -gt 2 ] 165 | then 166 | echo "Memory is less than threshold" >> $LOG_FILE 167 | count=$((REPLICAS-1)) 168 | echo "Updated No. of Replicas will be: "$count >> $LOG_FILE 169 | scale=`$KUBECTL scale --replicas=$count deployment/$DEPLOYMENT` 170 | echo "Deployment Scaled Down" >> $LOG_FILE 171 | else 172 | echo "Memory is not crossing the threshold. No Scaling done." >> $LOG_FILE 173 | fi 174 | } 175 | 176 | ########################################## 177 | #Calling Functions 178 | 179 | 180 | if [[ $REPLICAS ]]; then 181 | if [ "$ACTION" = "deploy-heap-autoscaling" ];then 182 | if [ $ARG -ne 8 ] 183 | then 184 | echo "Incorrect No. of Arguments Provided" 185 | print_help 186 | exit 1 187 | fi 188 | calculate_heap 189 | heapmemory_autoscale 190 | elif [ "$ACTION" = "get-heapmemory" ];then 191 | if [ $ARG -ne 4 ] 192 | then 193 | echo "Incorrect No. of Arguments Provided" 194 | print_help 195 | exit 1 196 | fi 197 | calculate_heap 198 | elif [ "$ACTION" = "get-podmemory" ];then 199 | if [ $ARG -ne 4 ] 200 | then 201 | echo "Incorrect No. of Arguments Provided" 202 | print_help 203 | exit 1 204 | fi 205 | calculate_podmemory 206 | elif [ "$ACTION" = "deploy-pod-autoscaling" ];then 207 | if [ $ARG -ne 8 ] 208 | then 209 | echo "Incorrect No. of Arguments Provided" 210 | print_help 211 | exit 1 212 | fi 213 | calculate_podmemory 214 | podmemory_autoscale 215 | else 216 | echo "Unknown Action" 217 | print_help 218 | fi 219 | else 220 | echo "No Deployment exists with name: "$DEPLOYMENT 221 | print_help 222 | fi 223 | --------------------------------------------------------------------------------