├── README.md
├── prime.sh
├── ssec2
├── ssec2B
├── ssec2C
├── ssec2L
└── wrap_cmd.sh


/README.md:
--------------------------------------------------------------------------------
  1 | 
  2 | Amazon's elastic cloud compute(EC2) provides on-demand and reasonably priced
  3 | general purpose computing resources.  However, it can be a pain to use.  Stupid
  4 | Simple Elastic Cloud Compute was written as a minimal wrapper around EC2 so
  5 | that it mimics a typical compute cluster where users can either log into nodes
  6 | directly, or simply submit work (in the form of a bash script) to be completed.
  7 | 
  8 | # Usage
  9 | 
 10 | There are two cattegories of commands, launch and connect.
 11 | 
 12 | ```
 13 | $ ssec2 -h
 14 | 
 15 | usage: ssec2 SUBCOMMAND OPTIONS
 16 | 
 17 |     launch      Launch a new instance
 18 |     connect     Conntect to instances
 19 | ```
 20 | 
 21 | `launch` spins up instances that you can send work to (bash script) or log into
 22 | 
 23 | ```
 24 | $ ssec2 launch -h
 25 | 
 26 | usage: ssec2L OPTIONS <script> <arg 1> <arg 2> <...>
 27 | 
 28 | OPTIONS can be:
 29 |     -h      Show this message
 30 |     -i      Image id (default Ubuntu Server:ami-83687be9)
 31 |     -t      Type (default t2.micro)
 32 |     -k      SSH key name (default lumpy_conf_test)
 33 |     -r      AMI region (default us-east-1)
 34 |     -s      Security group (default ssh_in_all_out)
 35 |     -n      Run name (required when submitting a script)
 36 |     -o      Output JSON file
 37 |     -d      Dry runkk
 38 | ```
 39 | 
 40 | `connect` gives basic monitoring and control capabilties
 41 | 
 42 | ```
 43 | $ ssec2 connect -h
 44 | 
 45 | usage: /Users/rl6sf/bin/ssec2C OPTIONS
 46 | 
 47 | OPTIONS can be:
 48 |     -h      Show this message
 49 |     -l      Get full listing
 50 |     -i      Instance ID
 51 |     -k      SSH key path (default /Users/rl6sf/.aws)
 52 |     -r      Region (default us-east-1)
 53 |     -u      User name (default ubuntu)
 54 |     -t      Terminate instance
 55 |     -s      Stop instance
 56 |     -T      Time frame for instance metrics (default 10min)
 57 | ```
 58 | 
 59 | # Dependencies
 60 | 
 61 | ## Software
 62 | https://aws.amazon.com/cli/
 63 | 
 64 | https://stedolan.github.io/jq/
 65 | 
 66 | ## AWS
 67 | You will need:
 68 | * an [s3 bucket](http://docs.aws.amazon.com/AmazonS3/latest/gsg/CreatingABucket.html)
 69 | * an [ec2 key pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html)
 70 | * a [security group](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html) with a rule that AT LEAST allows in ssh (I allow only ssh in and all out)
 71 | 
 72 | # Setup
 73 | 
 74 | Set the following values in `$HOME/.ssec2`
 75 | 
 76 | ```
 77 | DATA_STORE_KEY_ID=
 78 | DATA_STORE_ACCESS_KEY=
 79 | DATA_REGION=
 80 | DATA_BUCKET=
 81 | KEY_NAME=
 82 | KEY_PATH=
 83 | KEY_LOCATION=$KEY_PATH/$KEY_NAME.pem
 84 | AMI=
 85 | AMI_REGION=
 86 | AMI_USER_NAME=
 87 | AMI_WORKING_DIR=
 88 | SECURITY_GROUP=
 89 | LOG_FILE=$HOME/.ssec2_history
 90 | BATCH_LOG_GROUP_NAME=
 91 | ```
 92 | 
 93 | For example:
 94 | 
 95 | ```
 96 | DATA_STORE_KEY_ID=YOURKEYIDHEREISISLONGANDALLCAPS
 97 | DATA_STORE_ACCESS_KEY=YourKeyHereItIsLongerAndAMixOfLettersAndNumbers
 98 | DATA_REGION=us-east-1
 99 | DATA_BUCKET=ssec2
100 | KEY_NAME=ssec2_key
101 | KEY_PATH=~/.ssh
102 | KEY_LOCATION=$KEY_PATH/$KEY_NAME.pem
103 | AMI_REGION=us-east-1
104 | AMI=ami-fce3c696
105 | AMI_REGION=us-east-1
106 | AMI_USER_NAME=ubuntu
107 | AMI_WORKING_DIR=/home/ubuntu
108 | SECURITY_GROUP=ssh_in_all_out
109 | LOG_FILE=$HOME/.ssec2_history
110 | BATCH_LOG_GROUP_NAME=/aws/batch/job
111 | ```
112 | # Examples
113 | 
114 | ## Launch an instance, get the info, connect, then terminate
115 | 
116 | Spin up a new instance 
117 | ```
118 | $ ssec2 launch
119 | ```
120 | 
121 | Get the instance id
122 | ```
123 | $ ssec2 connect -l
124 | 
125 | ID          Launch Time               State           Type            CPU
126 | i-a55be038  2016-04-22T21:26:38.000Z  running         t2.micro        3.915
127 | ```
128 | 
129 | Connect to the instance
130 | ```
131 | $ ssec2 connect -i i-a55be038
132 | ```
133 | 
134 | Terminate it
135 | ```
136 | $ ssec2 connect -i i-a55be038 -t
137 | 
138 | $ ssec2 connect -l
139 | ID          Launch Time               State           Type            CPU
140 | i-a55be038  2016-04-22T21:26:38.000Z  shutting-down   t2.micro        .
141 | ```
142 | 
143 | ## Get the current status
144 | 
145 | When nothing is given to `ssec2` a summary of the current status is given
146 | ```
147 | $ ssec2
148 | 
149 | us-east-1   Fri Apr 22 15:27:05 MDT 2016
150 | Running        0
151 | Pending        1
152 | ```
153 | 
154 | or
155 | 
156 | ```
157 | $ ssec2 connect
158 | 
159 | us-east-1   Fri Apr 22 15:27:30 MDT 2016
160 | Running        1
161 | Pending        0
162 | ```
163 | 
164 | The `-l` option gives per instance information, but is slow
165 | 
166 | ```
167 | $ ssec2 connect -l
168 | 
169 | ID          Launch Time               State           Type            CPU
170 | i-a55be038  2016-04-22T21:26:38.000Z  running         t2.micro        3.915
171 | ```
172 | 
173 | ### Have an instance run a script then terminate
174 | 
175 | Launch an instance that runs a sctip (in this case `prime.sh`)  on a `m3.medium` instance script:
176 | 
177 | ```
178 | source $HOME/.ssec2
179 | 
180 | $ ssec2 launch \
181 |     -t m3.medium \
182 |     -n find_primes_to_1000 \
183 |     prime.sh \
184 |     1000
185 | ```
186 | 
187 | `-n ` is the name given to this run, which is required
188 | 
189 | `prime.sh` is the script that will run (as root in `$AMI_WORKING_DIR`)
190 | 
191 | The rest of the options are passed directly to `prime.sh` as command line
192 | arguments which `prime.sh` is ready to accept.
193 | 
194 | Every instance launched with a script will also retain the output from STDERR
195 | and STDOUT and store them to
196 | `s3://$DATA_BUCKET/find_primes_to_1000.instance_id.err` and
197 | `s3://$DATA_BUCKET/find_primes_to_1000.instance_id.out`.
198 | 
199 | If the instance was named `i-2394b13`, then you should be able to get the
200 | results by
201 | 
202 | ```
203 | $ aws s3 cp s3://find_primes_to_1000.i-2394b13.out .
204 | $ cat find_primes_to_1000.i-2394b13.out 2 3 5 7 11 13 17 19 23 29 31 37 41 43
205 | 47 53 59 61 67 71 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151
206 | 157 163 167 173 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263
207 | 269 271 277 281 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383
208 | 389 397 401 409 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503
209 | 509 521 523 541 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641
210 | 643 647 653 659 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769
211 | 773 787 797 809 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911
212 | 919 929 937 941 947 953 967 971 977 983 991 997
213 | ```
214 | 
215 | ## Use spot instances
216 | First get the historical pricing for the targer machine type in your region by passing `?` as the bid price.
217 | 
218 | ```
219 | $ ssec2 launch -t m3.2xlarge -b ?
220 | Region      Seconds Ago  Cost($/hr)
221 | us-east-1b  0            0.011600
222 | us-east-1b  25           0.011500
223 | us-east-1b  416          0.108200
224 | us-east-1b  492          0.059200
225 | 
226 | us-east-1c  3            0.011600
227 | us-east-1c  8            0.060200
228 | us-east-1c  489          0.108200
229 | 
230 | us-east-1d  0            0.108300
231 | us-east-1d  11           0.108200
232 | us-east-1d  40           0.059200
233 | us-east-1d  228          0.013000
234 | 
235 | us-east-1e  0            0.108600
236 | us-east-1e  32           0.013000
237 | us-east-1e  180          0.108500
238 | us-east-1e  245          0.059200
239 | ```
240 | 
241 | Choose your price, and submit the same job again with the bid price.
242 | 
243 | ```
244 | $ ssec2 launch \
245 |     -t m3.medium \
246 |     -n find_primes_to_1000 \
247 |     -b 0.05 \
248 |     prime.sh \
249 |     1000
250 | ```
251 | 


--------------------------------------------------------------------------------
/prime.sh:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | #http://rosettacode.org/wiki/Sieve_of_Eratosthenes#UNIX_Shell
 4 | 
 5 | function primes {
 6 |         typeset a i=2 j m=$1
 7 |         # No for (( ... )) loop in pdksh. Use while loop.
 8 |         while (( i <= m )); do
 9 |                 a[$i]=$i
10 |                 (( i++ ))
11 |         done
12 |  
13 |         i=2
14 |         while (( j = i * i, j <= m )); do
15 |                 if [[ -n ${a[$i]} ]]; then
16 |                         while (( j <= m )); do
17 |                                 unset a[$j]
18 |                                 (( j += i ))
19 |                         done
20 |                 fi
21 |                 (( i++ ))
22 |         done
23 |         # No print command in bash. Use echo command.
24 |         echo ${a[*]}
25 | }
26 |  
27 | primes $1
28 | 


--------------------------------------------------------------------------------
/ssec2:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | THIS_PATH="`dirname \"$0\"`"
 4 | THIS_PATH="`( cd \"$THIS_PATH\" && pwd )`" 
 5 | 
 6 | usage()
 7 | {
 8 |     cat << EOF
 9 | 
10 | usage: $0 SUBCOMMAND OPTIONS
11 | 
12 |     launch      Launch a new instance
13 |     connect     Conntect to instances
14 | EOF
15 | }
16 | 
17 | # Check options passed in.
18 | while getopts "h" OPTION
19 | do
20 |     case $OPTION in
21 |         h)
22 |             usage
23 |             exit 1
24 |             ;;
25 |         ?)
26 |             usage
27 |             exit
28 |             ;;
29 |     esac
30 | done
31 | 
32 | if [ "$#" -eq "0" ]
33 | then
34 |     $THIS_PATH/ssec2C
35 | elif [ $1 == "connect" ]
36 | then
37 |     $THIS_PATH/ssec2C ${@:2}
38 | elif [ $1 == "launch" ]
39 | then
40 |     $THIS_PATH/ssec2L ${@:2}
41 | else
42 |     echo "Unknown subcommand"
43 |     usage
44 |     exit
45 | fi
46 | 


--------------------------------------------------------------------------------
/ssec2B:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | set -e
  3 | 
  4 | ERR()
  5 | {
  6 |     rm -f $O
  7 | }
  8 | 
  9 | trap ERR EXIT
 10 | 
 11 | TO_SOURCE_FILE=$HOME/.ssec2
 12 | 
 13 | if [ ! -z "$SSEC2_PROFILE" ]; then
 14 |     echo $SSEC2_PROFILE
 15 |     TO_SOURCE_FILE=$HOME/.ssec2.$SSEC2_PROFILE
 16 | fi
 17 | 
 18 | if [ -f $TO_SOURCE_FILE ]; then
 19 |     source $TO_SOURCE_FILE
 20 | else
 21 |     echo "$TO_SOURCE_FILE not found"
 22 |     exit 1
 23 | fi
 24 | 
 25 | THIS_PATH="`dirname \"$0\"`"
 26 | THIS_PATH="`( cd \"$THIS_PATH\" && pwd )`" 
 27 | 
 28 | usage()
 29 | {
 30 |     cat << EOF
 31 | 
 32 | usage: $0 OPTIONS
 33 | 
 34 | OPTIONS can be:
 35 |     -h      Show this message
 36 |     -l      Get listing
 37 |     -s      Status
 38 |     -L      Get log
 39 |     -t      Terminate job
 40 |     -r      Terminate reason (default: Terminating job.)
 41 |     -j      Job id
 42 |     -q      Queue name
 43 |     -n      Number
 44 | EOF
 45 | }
 46 | 
 47 | 
 48 | LIST=
 49 | QUEUE=
 50 | STATUS=
 51 | VERBOSE=
 52 | NUMBER=
 53 | GET_LOG=
 54 | TERM_JOB=
 55 | TERM_JOB_REASON="Terminating job."
 56 | JOB_ID=
 57 | 
 58 | while getopts "hlq:s:vn:Ltr:j:" OPTION
 59 | do
 60 |     case $OPTION in
 61 |         h)
 62 |             usage
 63 |             exit 1
 64 |             ;;
 65 |         l)
 66 |             LIST=1
 67 |             ;;
 68 |         q)
 69 |             QUEUE=$OPTARG
 70 |             ;;
 71 |         s)
 72 |             STATUS=$OPTARG
 73 |             ;;
 74 |         v)
 75 |             VERBOSE=1
 76 |             ;;
 77 |         n)
 78 |             NUMBER=$OPTARG
 79 |             ;;
 80 |         L)
 81 |             GET_LOG=1
 82 |             ;;
 83 |         t)
 84 |             TERM_JOB=1
 85 |             ;;
 86 |         r)
 87 |             TERM_JOB_REASON=$OPTARG
 88 |             ;;
 89 |         j)
 90 |             JOB_ID=$OPTARG
 91 |             ;;
 92 |         ?)
 93 |             usage
 94 |             exit
 95 |             ;;
 96 |     esac
 97 | done
 98 | 
 99 | 
100 | O=ssec2C.$RANDOM.out
101 | 
102 | if [ ! -z "$GET_LOG" ]; then
103 |     if [ -s "$JOB_ID" ]; then
104 |         >&2 echo "ERROR: Cannot get log. No job id given" 
105 |         exit 1
106 |     fi
107 | 
108 |     JOB_DEF=`AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
109 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
110 |     aws batch describe-jobs --jobs $JOB_ID`
111 | 
112 | 
113 |     TASK_ARN=`echo $JOB_DEF \
114 |     | jq --raw-output ".jobs[] | select(.jobId == \""$JOB_ID"\") | .container.taskArn | split(\"/\") | .[1]"`
115 | 
116 |     JOB_NAME=`echo $JOB_DEF \
117 |     | jq --raw-output ".jobs[] | select(.jobId == \""$JOB_ID"\") | .jobName"`
118 | 
119 |     AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
120 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
121 |     aws logs get-log-events --log-group-name "$BATCH_LOG_GROUP_NAME" --log-stream-name $JOB_NAME/$JOB_ID/$TASK_ARN \
122 |     > $O
123 | 
124 |     if [ -z "$NUMBER" ];then 
125 |         paste <( cat $O | jq --raw-output '.events | sort_by(.timestamp) | .[].timestamp') \
126 |               <( cat $O | jq --raw-output '.events | sort_by(.timestamp) | .[].message') \
127 |         | awk -F"\t" '{print strftime("%c", ( $1 + 500 ) / 1000 ), $2;}' 
128 |     else
129 |         paste <( cat $O | jq --raw-output '.events | sort_by(.timestamp) | .[].timestamp') \
130 |               <( cat $O | jq --raw-output '.events | sort_by(.timestamp) | .[].message') \
131 |         | tail -n $NUMBER \
132 |         | awk -F"\t" '{print strftime("%c", ( $1 + 500 ) / 1000 ), $2;}' 
133 |     fi
134 |  
135 | elif [ ! -z "$TERM_JOB" ]; then
136 |     if [ -s "$JOB_ID" ]; then
137 |         >&2 echo "ERROR: Cannot terminate job. No job id given" 
138 |         exit 1
139 |     fi
140 | 
141 |     AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
142 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
143 |     aws batch terminate-job --job-id $JOB_ID --reason "$TERM_JOB_REASON"
144 | 
145 | elif [ ! -z "$LIST" ]; then
146 |     if [ -z "$STATUS" ]; then 
147 |         STATUSS="SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED"
148 |     else
149 |         STATUSS=$STATUS
150 |     fi
151 | 
152 |     JOB_QUEUES=""
153 | 
154 |     if [ ! -z "$QUEUE" ]; then
155 |         JOB_QUEUES=$QUEUE
156 |     else
157 |         JOB_QUEUES=`AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
158 |         AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
159 |         aws batch describe-job-queues  \
160 |         | jq --raw-output '.jobQueues[].jobQueueName'`
161 |     fi
162 | 
163 |     for Q in $JOB_QUEUES; do
164 |         if [ -z "$VERBOSE" ]; then
165 |             echo -en "Queue\t"
166 | 
167 |             if [ -z "$VERBOSE" ]; then
168 |                 echo $STATUSS | tr ' ' '\t'
169 |             fi
170 | 
171 |             echo -n "$Q"
172 | 
173 |             for S in $STATUSS; do
174 |                 NUM=`AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
175 |                 AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
176 |                 aws batch list-jobs --job-queue $Q --job-status $S \
177 |                 | jq --raw-output '.jobSummaryList | length'`
178 |                 echo -en "\t$NUM"
179 |             done
180 | 
181 |             echo
182 | 
183 |         else
184 |             echo "$Q:$STATUSS"
185 |             JOB_IDS=""
186 |             for S in $STATUSS; do
187 |                 JOB_IDS=`AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
188 |                 AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
189 |                 aws batch list-jobs --job-queue $Q --job-status $S \
190 |                 | jq --raw-output '.jobSummaryList[].jobId'`
191 |             done
192 | 
193 |             LEN=`echo -e "$JOB_IDS" | wc -l`
194 | 
195 |             LOOPS=1
196 |             STEP=$LEN
197 |             if [ "$LEN" -gt 100 ]; then
198 |                 STEP=100
199 |                 LOOPS=`expr $LEN / $STEP`
200 |             fi
201 |             FIRST=1
202 |             echo -e "Name\tID\tStarted At"
203 | 
204 |             (for i in `seq 1 $LOOPS`; do
205 |                 JOB_IDS_PAG=`echo -e "$JOB_IDS" | tail -n+$FIRST | head -n $STEP | paste -sd " " -`
206 |                 AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
207 |                 AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
208 |                 aws batch describe-jobs --jobs `echo $JOB_IDS_PAG` \
209 |                 | jq --raw-output '.jobs[] | [.jobName,.jobId,.startedAt|tostring] | join("\t")' 
210 |                 FIRST=$(( FIRST + $STEP ))
211 |             done) \
212 |             | sort -k 3,3n \
213 |             > $O 
214 | 
215 |             if [ -z "$NUMBER" ];then
216 |                 cat $O \
217 |                 | awk '{
218 |                     OFS="\t";
219 |                     print $1,$2,strftime("%c", ( $3 + 500 ) / 1000 );}' 
220 |             else
221 |                 tail -n $NUMBER $O \
222 |                 | awk '{
223 |                     OFS="\t";
224 |                     print $1,$2,strftime("%c", ( $3 + 500 ) / 1000 );}' 
225 |             fi
226 |         fi
227 |     done
228 | fi
229 | 


--------------------------------------------------------------------------------
/ssec2C:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | set -e
  3 | 
  4 | ERR()
  5 | {
  6 |     rm -f $O
  7 | }
  8 | 
  9 | TO_SOURCE_FILE=$HOME/.ssec2
 10 | 
 11 | if [ ! -z "$SSEC2_PROFILE" ]; then
 12 |     echo $SSEC2_PROFILE
 13 |     TO_SOURCE_FILE=$HOME/.ssec2.$SSEC2_PROFILE
 14 | fi
 15 | 
 16 | if [ -f $TO_SOURCE_FILE ]; then
 17 |     source $TO_SOURCE_FILE
 18 | else
 19 |     echo "$TO_SOURCE_FILE not found"
 20 |     exit 1
 21 | fi
 22 | 
 23 | 
 24 | trap ERR EXIT
 25 | 
 26 | THIS_PATH="`dirname \"$0\"`"
 27 | THIS_PATH="`( cd \"$THIS_PATH\" && pwd )`" 
 28 | 
 29 | usage()
 30 | {
 31 |     cat << EOF
 32 | 
 33 | usage: $0 OPTIONS
 34 | 
 35 | OPTIONS can be:
 36 |     -h      Show this message
 37 |     -l      Get full listing
 38 |     -i      Instance ID
 39 |     -k      SSH key path (default $KEY_PATH)
 40 |     -r      Region (default $AMI_REGION)
 41 |     -u      User name (default $AMI_USER_NAME)
 42 |     -f      secure copy local file to instance
 43 |     -F      secure copy remote file freom instance
 44 |     -c      run a remote command
 45 |     -t      Terminate instance
 46 |     -T      Terminate all instances
 47 |     -s      Stop instance
 48 | EOF
 49 | }
 50 | 
 51 | TIME=10
 52 | TERM_I=0
 53 | TERM_ALL=0
 54 | STOP_I=0
 55 | INSTANCE=
 56 | LIST=0
 57 | LOCAL_FILE=
 58 | REMOTE_FILE=
 59 | REMOTE_CMD=
 60 | 
 61 | # Check options passed in.
 62 | while getopts "h i:k:r:u:tsTlf:F:c:" OPTION
 63 | do
 64 |     case $OPTION in
 65 |         h)
 66 |             usage
 67 |             exit 1
 68 |             ;;
 69 |         c)
 70 |             REMOTE_CMD=$OPTARG
 71 |             ;;
 72 |         i)
 73 |             INSTANCE=$OPTARG
 74 |             ;;
 75 |         k)
 76 |             KEY_LOCATION=$OPTARG
 77 |             ;;
 78 |         r)
 79 |             AMI_REGION=$OPTARG
 80 |             ;;
 81 |         u)
 82 |             AMI_USER_NAME=$OPTARG
 83 |             ;;
 84 |         f)
 85 |             LOCAL_FILE=$OPTARG
 86 |             ;;
 87 |         F)
 88 |             REMOTE_FILE=$OPTARG
 89 |             ;;
 90 |         t)
 91 |             TERM_I=1
 92 |             ;;
 93 |         s)
 94 |             STOP_I=1
 95 |             ;;
 96 |         l)
 97 |             LIST=1
 98 |             ;;
 99 |         T)
100 |             TERM_ALL=1
101 |             ;;
102 |         ?)
103 |             usage
104 |             exit
105 |             ;;
106 |     esac
107 | done
108 | 
109 | O=ssec2C.$RANDOM.out
110 | AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
111 | AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
112 | aws ec2 describe-instances --region $AMI_REGION > $O
113 | 
114 | if [ "$TERM_ALL" -eq "1" ]
115 | then
116 |     INSTANCES=`cat $O | jq -r ".Reservations[].Instances[] | select(.State.Name == \"running\") | .InstanceId" | tr '\n' ' '`
117 |     CMD="AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
118 |         AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
119 |         aws ec2 terminate-instances --instance-ids $INSTANCES --region $AMI_REGION"
120 |     eval $CMD 
121 | elif [ "$LIST" -eq "1" ]
122 | then
123 |     THEN=
124 |     NOW=
125 |     
126 |     if [ "`uname`" == "Linux" ]
127 |     then
128 |         THEN=$(date -u --date "-$TIME minutes" "+%Y-%m-%dT%H:%M:%SUTC")
129 |         NOW=$(date -u "+%Y-%m-%dT%H:%M:%SUTC")
130 |     else
131 |         THEN=$(date -u -j -r $((`date -u +%s`-$TIME*60)) "+%Y-%m-%dT%H:%M:%SUTC")
132 |         NOW=$(date -u -j -r $(date -u +%s) "+%Y-%m-%dT%H:%M:%SUTC")
133 |     fi
134 | 
135 |     if [ ! -z "$INSTANCE" ]
136 |     then
137 |         cat $O | jq ".Reservations[].Instances[] |  select(.InstanceId == \"$INSTANCE\")"
138 |     else
139 |         JR_Q=".Reservations[].Instances[] | select(.State.Name != \"terminated\") | .InstanceId + \",\" + .LaunchTime + \",\" + .State.Name + \",\" + .PublicDnsName + \",\" + .InstanceType"
140 |         printf "%-15s %-15s %-15s %-15s %-10s\n" "ID" "Launch Time" "State" "Type" "CPU"
141 |         for INSTANCE_STATE_STR in `cat $O | jq -r "$JR_Q"`
142 |         do
143 |             IFS=',' read -r -a INSTANCE_STATE <<< "$INSTANCE_STATE_STR"
144 |             printf "%-15s %-15s %-15s %-10s " "${INSTANCE_STATE[0]}" "${INSTANCE_STATE[1]}" "${INSTANCE_STATE[2]}" "${INSTANCE_STATE[4]}"
145 | 
146 |             CPU="."
147 |             if [ "${INSTANCE_STATE[2]}" == "running" ]
148 |             then
149 |                 INSTANCE="${INSTANCE_STATE[0]}"
150 |                 CPU=`AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
151 |                     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
152 |                     aws cloudwatch get-metric-statistics \
153 |                     --namespace "AWS/EC2" \
154 |                     --metric-name CPUUtilization \
155 |                     --region $AMI_REGION \
156 |                     --dimensions Name=InstanceId,Value=$INSTANCE \
157 |                     --start-time $THEN \
158 |                     --end-time $NOW \
159 |                     --statistics Average \
160 |                     --period 60  | jq '.Datapoints[].Average' 2>/dev/null \
161 |                     | awk '{x+=$0;y+=$0^2}END{if (NR>0) print x/NR}'`
162 |             fi
163 | 
164 |             printf "%-10s\n" "$CPU"
165 |         done
166 |     fi
167 | elif [ ! -z "$INSTANCE" ]
168 | then
169 |     CMD="AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
170 | AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
171 | "
172 |     JR_Q=
173 |     
174 |     if [ "$TERM_I" -eq "1" ]
175 |     then
176 |         CMD+="aws ec2 terminate-instances --instance-ids $INSTANCE --region $AMI_REGION"
177 |         JRQ=".TerminatingInstances[] | .InstanceId + \" \" + .PreviousState + \"->\" + .CurrentState"
178 |     elif [ "$STOP_I" -eq "1" ]
179 |     then
180 |         CMD+="aws ec2 stop-instances --instance-ids $INSTANCE --region $AMI_REGION"
181 |     elif [ ! -z "$LOCAL_FILE" ]
182 |     then
183 |         CMD=`cat $O \
184 |             | jq ".Reservations[].Instances[] | select(.InstanceId == \"$INSTANCE\")" \
185 |             | jq -r "\"scp -o StrictHostKeyChecking=no -i $KEY_PATH/\" + .KeyName + \".pem $LOCAL_FILE $AMI_USER_NAME@\" + .PublicDnsName + \":.\""`
186 |     elif [ ! -z "$REMOTE_FILE" ]
187 |     then
188 |         CMD=`cat $O \
189 |             | jq ".Reservations[].Instances[] | select(.InstanceId == \"$INSTANCE\")" \
190 |             | jq -r "\"scp -o StrictHostKeyChecking=no -i $KEY_PATH/\" + .KeyName + \".pem $AMI_USER_NAME@\" + .PublicDnsName + \":$REMOTE_FILE .\""`
191 |     else
192 |         CMD=`cat $O \
193 |             | jq ".Reservations[].Instances[] | select(.InstanceId == \"$INSTANCE\")" \
194 |             | jq -r "\"ssh -o StrictHostKeyChecking=no -i $KEY_PATH/\" + .KeyName + \".pem $AMI_USER_NAME@\" + .PublicDnsName"`
195 | 
196 |         echo $CMD
197 | 
198 |         if [ ! -z "$REMOTE_CMD" ]
199 |         then
200 |             CMD+=" \"$REMOTE_CMD\""
201 |         fi
202 |     fi
203 |     eval $CMD
204 |     rm $O
205 | else 
206 |     printf "%s\t%s\n" "$AMI_REGION" "$(date)"
207 |     printf "\nInstances\n"
208 | 
209 |     #printf "Running "
210 |     #JR_Q=".Reservations[].Instances[] | select(.State.Name == \"running\") | .InstanceId "
211 |     #printf "%s\n" "$(cat $O | jq -r "$JR_Q" | wc -l)"
212 | 
213 |     #printf "Pending "
214 |     #JR_Q=".Reservations[].Instances[] | select(.State.Name == \"pending\") | .InstanceId "
215 |     #printf "%s\n" "$(cat $O | jq -r "$JR_Q" | wc -l)"
216 | 
217 |     JR_Q=".Reservations[].Instances[].State.Name"
218 |     cat $O | jq -r "$JR_Q" | sort | uniq -c 
219 | 
220 |     printf "\nSpot Requests\n"
221 |     AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
222 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
223 |     aws ec2 describe-spot-instance-requests --region $AMI_REGION > $O
224 |     JR_Q=".SpotInstanceRequests[].Status.Code"
225 |     cat $O | jq -r "$JR_Q" | sort | uniq -c 
226 | fi
227 | 


--------------------------------------------------------------------------------
/ssec2L:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | TO_SOURCE_FILE=$HOME/.ssec2
  4 | 
  5 | if [ ! -z "$SSEC2_PROFILE" ]; then
  6 |     echo $SSEC2_PROFILE
  7 |     TO_SOURCE_FILE=$HOME/.ssec2.$SSEC2_PROFILE
  8 | fi
  9 | 
 10 | if [ -f $TO_SOURCE_FILE ]; then
 11 |     source $TO_SOURCE_FILE
 12 | else
 13 |     echo "$TO_SOURCE_FILE not found"
 14 |     exit 1
 15 | fi
 16 | 
 17 | 
 18 | THIS_PATH="`dirname \"$0\"`"
 19 | THIS_PATH="`( cd \"$THIS_PATH\" && pwd )`" 
 20 | 
 21 | usage()
 22 | {
 23 |     cat << EOF
 24 | 
 25 | usage: $0 OPTIONS <script> <arg 1> <arg 2> <...>
 26 | 
 27 | OPTIONS can be:
 28 |     -h      Show this message
 29 |     -i      Image id (default Ubuntu Server:$AMI)
 30 |     -t      Type (default t2.micro)
 31 |     -k      SSH key name (default $KEY_NAME)
 32 |     -r      AMI region (default $AMI_REGION)
 33 |     -s      Security group (default $SECURITY_GROUP)
 34 |     -n      Run name (required when submitting a script)
 35 |     -o      Output JSON file
 36 |     -b      Bid price for a spot instance (e.g., 0.01)
 37 |     -x      Extra options to pass
 38 |     -d      Dry run
 39 | EOF
 40 | }
 41 | 
 42 | #if test -z "$1"
 43 | #then
 44 | #    usage
 45 | #    exit
 46 | #fi
 47 | 
 48 | TYPE="t2.micro"
 49 | JSON_OUT=
 50 | DRYRUN=0
 51 | NAME=
 52 | FILE_NAME=
 53 | BID=
 54 | EXTRA=
 55 | 
 56 | # Check options passed in.
 57 | while getopts "h i:t:k:r:s:u:o:n:db:x:" OPTION
 58 | do
 59 |     case $OPTION in
 60 |         h)
 61 |             usage
 62 |             exit 1
 63 |             ;;
 64 |         i)
 65 |             AMI=$OPTARG
 66 |             ;;
 67 |         t)
 68 |             TYPE=$OPTARG
 69 |             ;;
 70 |         k)
 71 |             KEY_NAME=`echo $OPTARG | xargs -I{} basename {} .pem`
 72 |             ;;
 73 |         r)
 74 |             AMI_REGION=$OPTARG
 75 |             ;;
 76 |         s)
 77 |             SECURITY_GROUP=$OPTARG
 78 |             ;;
 79 |         o)
 80 |             JSON_OUT=$OPTARG
 81 |             ;;
 82 |         n)
 83 |             NAME=$OPTARG
 84 |             ;;
 85 |         d)
 86 |             DRYRUN=1
 87 |             ;;
 88 |         b)
 89 |             BID=$OPTARG
 90 |             ;;
 91 |         x)
 92 |             EXTRA=$OPTARG
 93 |             ;;
 94 |         ?)
 95 |             usage
 96 |             exit
 97 |             ;;
 98 |     esac
 99 | done
100 | 
101 | shift $(($OPTIND - 1))
102 | 
103 | if [ -z "$KEY_NAME" ]
104 | then
105 |     echo "No key name"
106 |     usage
107 |     exit
108 | fi
109 | 
110 | if [ -z "$SECURITY_GROUP" ]
111 | then
112 |     echo "No security group"
113 |     usage
114 |     exit
115 | fi
116 | 
117 | CMD="AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
118 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
119 | "
120 | 
121 | if [ -z "$BID" ]
122 | then
123 |     CMD+="aws ec2 run-instances \
124 |         $EXTRA \
125 |         --image-id $AMI \
126 |         --key-name $KEY_NAME  \
127 |         --instance-type $TYPE \
128 |         --region $AMI_REGION \
129 |         --security-groups $SECURITY_GROUP \
130 |         --instance-initiated-shutdown-behavior terminate \
131 |     "
132 |     if [ "$#" -gt "0" ]
133 |     then
134 |         if [ -z "$NAME" ]
135 |         then
136 |             echo "No name given."
137 |             usage
138 |             exit
139 |         fi
140 | 
141 |         SCRIPT=$1
142 |         ARGS=${@:2}
143 | 
144 |         if [ ! -e $SCRIPT ]
145 |         then
146 |             echo "File $SCRIPT does not exist"
147 |             exit
148 |         fi
149 | 
150 |         FILE_NAME=launch.$RANDOM.sh
151 |         $THIS_PATH/wrap_cmd.sh $NAME $SCRIPT $ARGS > $FILE_NAME
152 |         CMD+="\
153 |         --user-data file://$FILE_NAME"
154 |     fi
155 | else
156 | 
157 |     if [ "$BID" = "?" ]
158 |     then
159 |         THEN=
160 |         NOW=
161 | 
162 |         if [ "`uname`" == "Linux" ]
163 |         then
164 |             THEN=$(date -u --date "-60 minutes" "+%Y-%m-%dT%H:%M:%S")
165 |             NOW=$(date -u "+%Y-%m-%dT%H:%M:%S")
166 |         else
167 |             THEN=$(date -u -jf "%s" $((`date -u +%s`-60))  "+%Y-%m-%dT%H:%M:%SZ")
168 |             NOW=$(date -u -jf "%s" $(date -u +%s) "+%Y-%m-%dT%H:%M:%SZ")
169 |         fi
170 | 
171 |         CMD="aws ec2 \
172 |         describe-spot-price-history 
173 |         --region $AMI_REGION \
174 |         --instance-types $TYPE \
175 |         --start-time $THEN
176 |         --end-time $NOW  \
177 |         --no-paginate"
178 |         O=`eval $CMD`
179 | 
180 |         R=`echo $O | jq -r '.SpotPriceHistory[] | .AvailabilityZone + "\t" + .Timestamp + "\t" + .SpotPrice'`
181 | 
182 |         ZONE=`echo -e "$R" | cut -f1`
183 |         TIMES=`echo -e "$R" \
184 |             | cut -f2 \
185 |             | sed -e "s/\.[0-9]*Z$//" \
186 |             | xargs -I{} bash -c 'echo $(( ($(date -u "+%s") - $(date -u -jf "%Y-%m-%dT%H:%M:%S" "{}" "+%s"))/60 ))'`
187 |             
188 |         PRICE=`echo -e "$R" | cut -f3`
189 | 
190 |         (echo -e "Region\tSeconds Ago\tCost($/hr)";
191 |         paste <(echo -e "$ZONE") <(echo -e "$TIMES") <(echo -e "$PRICE") \
192 |             | sort -k 1,1 -k 2,2n )\
193 |         | column -t -s $'\t' \
194 |         | awk 'BEGIN {region="";}
195 |                 {
196 |                     if (($1 != region) && 
197 |                         (region != "") && 
198 |                         (region != "Region")) {
199 |                         print "";
200 |                     }
201 |                     region=$1;
202 |                     print $0;
203 |                 }'
204 |         exit
205 |     fi
206 | 
207 |     CMD+="aws ec2 request-spot-instances \
208 |         --spot-price $BID \
209 |         --instance-count 1 \
210 |         --region $AMI_REGION \
211 |         --launch-specification \
212 |             \"{ \\\"ImageId\\\":\\\"$AMI\\\", \
213 |                 \\\"InstanceType\\\":\\\"$TYPE\\\", \
214 |                 \\\"KeyName\\\":\\\"$KEY_NAME\\\", \
215 |                 \\\"SecurityGroups\\\": [\\\"$SECURITY_GROUP\\\"], \
216 |                 \\\"Monitoring\\\": { \\\"Enabled\\\": true }"
217 |     if [ "$#" -gt "0" ]
218 |     then
219 | 
220 |         SCRIPT=$1
221 |         ARGS=${@:2}
222 | 
223 |         if [ ! -e $SCRIPT ]
224 |         then
225 |             echo "File $SCRIPT does not exist"
226 |             exit
227 |         fi
228 | 
229 |         FILE_NAME=launch.$RANDOM.sh
230 |         $THIS_PATH/wrap_cmd.sh $NAME $SCRIPT $ARGS > $FILE_NAME
231 | 
232 |         CMD+=",\
233 |                 \\\"UserData\\\":\\\"`base64 $FILE_NAME`\\\""
234 |     fi
235 |     CMD+="}\""
236 |     echo $CMD
237 | fi
238 | 
239 | if [ "$DRYRUN" -eq "1" ]
240 | then
241 |     CMD+="\
242 |     --dry-run"
243 | fi
244 | 
245 | O=ec2_launch.$RANDOM.json
246 | eval $CMD > $O
247 | 
248 | date >> $LOG_FILE
249 | echo $CMD >> $LOG_FILE
250 | 
251 | if [ -z "$BID" ]
252 | then
253 |     cat $O | jq -r .Instances[0].InstanceId >> $LOG_FILE
254 |     cat $O | jq -r .Instances[0].InstanceId
255 | 
256 |     aws ec2 monitor-instances --region $AMI_REGION \
257 |         --instance-ids `cat $O | jq -r .Instances[0].InstanceId` \
258 |     | jq -r "\"Monitoring state: \" + .InstanceMonitorings[].Monitoring.State"
259 | 
260 |     if [ ! -z "$NAME" ]
261 |     then
262 |     aws ec2 create-tags \
263 |         --resources `cat $O  | jq -r .Instances[0].InstanceId` \
264 |         --tags Key=Name,Value=$NAME --region $AMI_REGION 
265 |     fi
266 | fi
267 | 
268 | rm $O
269 | 
270 | if [ ! -z $FILE_NAME ]
271 | then
272 |     rm -f $FILE_NAME
273 | fi
274 | 


--------------------------------------------------------------------------------
/wrap_cmd.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -o nounset
 3 | 
 4 | NAME=$1
 5 | SCRIPT=$2
 6 | 
 7 | source $HOME/.ssec2
 8 | 
 9 | echo "#!/bin/bash
10 | 
11 | if [ -z \$(which aws) ]
12 | then
13 |     wget https://bootstrap.pypa.io/get-pip.py
14 |     python get-pip.py
15 |     pip install awscli
16 | fi
17 | 
18 | 
19 | ERR()
20 | {
21 |     set +e
22 | 
23 |     AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
24 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
25 |     aws s3 cp --region us-west-1 \
26 |     $NAME.\$EC2_INSTANCE_ID.out s3://$DATA_BUCKET/logs/
27 | 
28 |     AWS_ACCESS_KEY_ID=$DATA_STORE_KEY_ID \
29 |     AWS_SECRET_ACCESS_KEY=$DATA_STORE_ACCESS_KEY \
30 |     aws s3 cp --region us-west-1 \
31 |     $NAME.\$EC2_INSTANCE_ID.err s3://$DATA_BUCKET/logs/
32 | 
33 | 
34 |     shutdown -h now
35 | }
36 | 
37 | trap ERR EXIT
38 | "
39 | 
40 | echo '
41 | EC2_INSTANCE_ID="`wget -q -O - http://instance-data/latest/meta-data/instance-id || die \"wget instance-id has failed: $?\"`"
42 | '
43 | 
44 | echo "
45 | cd $AMI_WORKING_DIR
46 | 
47 | touch .running
48 | 
49 | SSEC2_F()
50 | {
51 | "
52 | 
53 | cat $SCRIPT
54 | 
55 | echo "}
56 | 
57 | SSEC2_F ${@:3} > $NAME.\$EC2_INSTANCE_ID.out 2> $NAME.\$EC2_INSTANCE_ID.err"
58 | 
59 | 


--------------------------------------------------------------------------------