├── 2015 ├── README.md ├── dev-ec2-instance ├── latest-amis ├── remove-sgs └── setup-dev-ec2-instance ├── 2016 ├── README.md └── auto-configure ├── 2017 ├── README.md ├── alias └── clone ├── .github └── PULL_REQUEST_TEMPLATE.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt └── README.md /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /2015/README.md: -------------------------------------------------------------------------------- 1 | # AWS CLI code samples from re:Invent 2015 2 | 3 | This directory contains the AWS CLI code samples I 4 | presented from the "Automating AWS with the AWS CLI" 5 | session at re:Invent 2015. 6 | 7 | The slides for this talk can be found 8 | on slideshare: http://www.slideshare.net/AmazonWebServices/dev301-automating-aws-with-the-aws-cli/ 9 | 10 | The video for this talk is available 11 | on youtube: https://www.youtube.com/watch?v=TnfqJYPjD9I 12 | 13 | Some of these samples require ``jp``, which can 14 | be installed here: https://github.com/jmespath/jp/releases 15 | Mac users can also just run: 16 | 17 | ``` 18 | brew tap jmespath/jmespath 19 | brew install jmespath 20 | ``` 21 | -------------------------------------------------------------------------------- /2015/dev-ec2-instance: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docs="usage: dev-ec2-instance [setup] 4 | 5 | This command makes it easy to launch dev EC2 instances 6 | that can be used for testing purposes, debugging, etc. 7 | 8 | You can run this command with no arguments to launch a new 9 | instance. If you do this, then: 10 | 11 | * The latest Amazon Linux AMI will be used. 12 | * The id_rsa keypair will be used. 13 | * The first security group tagged with 14 | dev-ec2-instance:non-windows will be used. 15 | * The IAM role named \"dev-ec2-instance\" will be used. 16 | 17 | " 18 | set -e 19 | 20 | readonly KEY_NAME="id_rsa" 21 | readonly INSTANCE_TYPE="m3.medium" 22 | readonly ROLE_NAME="dev-ec2-instance" 23 | 24 | 25 | usage() { 26 | echo "$docs" 27 | } 28 | 29 | 30 | errexit() { 31 | echo "error: $(basename "$0") (line ${LINENO}): ${1:-"Unknown Error"}" 1>&2 32 | exit 1 33 | } 34 | 35 | 36 | wait_for_port() { 37 | local port="$2" 38 | local host="$1" 39 | while ! nc -z "$host" "$port" >/dev/null; do 40 | sleep 5 41 | done 42 | } 43 | 44 | 45 | find_tagged_ec2_group() { 46 | aws ec2 describe-security-groups \ 47 | --filter Name=tag:dev-ec2-instance,Values=linux \ 48 | --query "SecurityGroups[].GroupId | [0]" \ 49 | --output text 50 | 51 | } 52 | 53 | 54 | ssh_to_instance() { 55 | # First we'll look for the SSH key fingerprint in the 56 | # console output before SSHing to the instance. 57 | local instance_id="$1" 58 | local hostname="$2" 59 | echo -n "Waiting for console output to be available..." 60 | aws ec2 wait console-output-available --instance-id "$instance_id" 61 | echo "DONE" 62 | expected_fingerprint=$(aws ec2 get-console-output \ 63 | --instance-id "$instance_id" --query Output --output text \ 64 | | grep '^ec2.*RSA' | cut -d ' ' -f 3) 65 | ssh-keyscan -t rsa "$hostname" > "/tmp/${instance_id}.pub" 66 | actual_fingerprint_line=$(ssh-keygen -lf "/tmp/${instance_id}.pub") 67 | actual_fingerprint=$(cut -d' ' -f 2 <<< "$actual_fingerprint_line") 68 | if [ "$expected_fingerprint" != "$actual_fingerprint" ]; then 69 | errexit "SSH fingerprint does not match for $hostname" 70 | fi 71 | cat < "/tmp/${instance_id}.pub" >> ~/.ssh/known_hosts 72 | # Only Mac OS X supported for desktop notifications for now. 73 | # You can "brew install terminal-notifier" if you want 74 | # desktop notifications on a Mac. 75 | command -v terminal-notifier >/dev/null 2>&1 && \ 76 | terminal-notifier -title dev-ec2-instance -message "EC2 instance is now available" 77 | ssh "ec2-user@${hostname}" 78 | } 79 | 80 | 81 | emit_desktop_notification() { 82 | # Only supported on Mac OS X for now. 83 | if is_mac_os_x; then 84 | echo "you are on a mac" 85 | else 86 | echo "not on mac" 87 | fi 88 | } 89 | 90 | 91 | 92 | launch_instance() { 93 | local image_id="$1" 94 | local security_groups=$(find_tagged_ec2_group) || \ 95 | errexit "Can't find tagged security group" 96 | 97 | local instance_id=$(aws ec2 run-instances \ 98 | --image-id "$image_id" \ 99 | --key-name "$KEY_NAME" \ 100 | --security-group-ids "$security_groups" \ 101 | --instance-type "$INSTANCE_TYPE" \ 102 | --iam-instance-profile "Name=$ROLE_NAME" \ 103 | --output text \ 104 | --query "Instances[0].InstanceId") 105 | 106 | # After that we tag the instance with purpose=dev-ec2-instance 107 | # so we can clean it up later. 108 | aws ec2 create-tags \ 109 | --resources "$instance_id" \ 110 | --tags Key=purpose,Value=dev-ec2-instance 111 | 112 | echo -n "Waiting for instance to be in the running state..." 113 | aws ec2 wait instance-running \ 114 | --instance-ids "$instance_id" 115 | echo "Done" 116 | # Once it's done we need to do another describe call 117 | # to get the actual public dns name. 118 | local instance_hostname=$(aws ec2 describe-instances \ 119 | --instance-ids "$instance_id" \ 120 | --output text \ 121 | --query Reservations[0].Instances[0].PublicDnsName) 122 | echo -n "Waiting for SSH port to be available at $instance_hostname..." 123 | wait_for_port "$instance_hostname" 22 124 | echo "Done" 125 | echo "" 126 | # Now actually SSH to instance 127 | ssh_to_instance "$instance_id" "$instance_hostname" 128 | } 129 | 130 | 131 | # Assumes that latest-amis is in the same directory as 132 | # the dev-ec2-instance script, so that by adding the cwd 133 | # to the PATH, we can invoke "latest-amis". 134 | export PATH="$PATH:$(pwd)" 135 | region=${AWS_DEFAULT_REGION:-$(aws configure get region)} 136 | latest_amis=$(latest-amis -r "$region") 137 | latest_ami=$(grep -v rc <<< "$latest_amis" | head -n 1) 138 | # Format is: \t\t\t 139 | echo "Launching: $(cut -f 4 <<< "$latest_ami" )" 140 | launch_instance "$(cut -f 2 <<< "$latest_ami")" 141 | -------------------------------------------------------------------------------- /2015/latest-amis: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will retrieve the latest Amazon Linux AMIs. 4 | # Each AMI will be displayed one per line, and for each line, 5 | # you'll get tab separated output: 6 | # 7 | # region image_id name description 8 | # 9 | # 10 | # If you run this script with no arguments you'll get the 11 | # AMIs in every region. Otherwise you can provide 12 | # a "-r | --region" option that will only display 13 | # images for the specified region. 14 | 15 | list_amis() { 16 | # Tab separated output 17 | # region image_id name description 18 | local region_name="$1" 19 | aws ec2 describe-images \ 20 | --filters \ 21 | Name=owner-alias,Values=amazon \ 22 | Name=name,Values="amzn-ami-hvm-*" \ 23 | Name=architecture,Values=x86_64 \ 24 | Name=virtualization-type,Values=hvm \ 25 | Name=root-device-type,Values=ebs \ 26 | Name=block-device-mapping.volume-type,Values=gp2 \ 27 | --region "$region_name" \ 28 | --query "reverse(sort_by(Images[? !contains(Name, 'rc')], &CreationDate))[*].['$region_name',ImageId,Name,Description]" \ 29 | --output text 30 | } 31 | 32 | 33 | if [ -z "$1" ]; then 34 | for region_name in $(aws ec2 describe-regions --query "sort(Regions[].RegionName)" --output text); do 35 | list_amis "$region_name" 36 | done 37 | else 38 | case "$1" in 39 | -r|--region) 40 | shift 41 | region_name="$1" 42 | ;; 43 | *) 44 | echo "usage: latest-amis [-r | --region]" 1>&2 45 | exit 1 46 | esac 47 | list_amis "$region_name" 48 | fi 49 | -------------------------------------------------------------------------------- /2015/remove-sgs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is the script that was used 4 | # for this part of the "Automating AWS with the AWS CLI" 5 | # talk: https://youtu.be/TnfqJYPjD9I?t=1962 6 | 7 | # We require the --region argument because this is such 8 | # a destructive thing you have to be absolutely sure you 9 | # want to do this. 10 | docs="usage: remove-sgs appname 11 | 12 | This command will remove every single security group in a given 13 | region that's tagged with appname=. 14 | Be VERY careful using this command. 15 | " 16 | if [[ -z "$1" ]]; then 17 | echo "$docs" 18 | exit 1 19 | fi 20 | 21 | tag_value_name="$1" 22 | 23 | errexit() { 24 | echo "ERROR: $(basename "$0") (line ${LINENO}): ${1:-"Unknown Error"}" 1>&2 25 | exit 1 26 | } 27 | 28 | query() { 29 | jp -u "$2" <<<"$1" 30 | } 31 | 32 | security_groups=$(aws ec2 describe-security-groups \ 33 | --query SecurityGroups \ 34 | --filters "Name=tag:appname,Values=$tag_value_name") \ 35 | || errexit "Could not find security groups" 36 | 37 | echo "$security_groups" 38 | 39 | echo "" 40 | echo "You are about to remove ALL these security groups in the region $region_name" 41 | echo -n "To confirm, please type 'yes' and press enter: " 42 | read confirmation 43 | if [[ "$confirmation" != "yes" ]] 44 | then 45 | echo "Didn't receive a confirmation, exiting." 46 | exit 2 47 | fi 48 | 49 | 50 | num_groups=$(query "$security_groups" "length(@)") 51 | 52 | for ((i = 0 ; i < "$num_groups" ; i++)); do 53 | group_id=$(query "$security_groups" "[$i].GroupId") 54 | ip_permissions=$(query "$security_groups" "[$i].IpPermissions") 55 | echo "Revoking ingress rules for security group: $group_id" 56 | aws ec2 revoke-security-group-ingress --group-id "$group_id" --ip-permissions "$ip_permissions" 57 | done 58 | 59 | for ((i = 0 ; i < "$num_groups" ; i++)); do 60 | group_id=$(query "$security_groups" "[$i].GroupId") 61 | echo "Removing security group: $group_id" 62 | aws ec2 delete-security-group --group-id "$group_id" 63 | done 64 | -------------------------------------------------------------------------------- /2015/setup-dev-ec2-instance: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: you need "jp" installed 4 | # to run this command. You can 5 | # brew tap jmespath/jmespath && brew install jp 6 | # to install on a Mac. 7 | # Or you can download prebuilt binaries 8 | # from: https://github.com/jmespath/jp/releases 9 | 10 | docs="usage: setup-dev-ec2-instance 11 | 12 | This command will set up the necessary resources needed to run 13 | dev-ec2-instance. You can also use this script to check that 14 | all the necessary resources are setup. If any resources 15 | are missing, it will prompt you to set up the necessary parts. 16 | " 17 | 18 | TRUST_POLICY='{ 19 | "Version": "2008-10-17", 20 | "Statement": [ 21 | { 22 | "Action": "sts:AssumeRole", 23 | "Principal": { 24 | "Service": "ec2.amazonaws.com" 25 | }, 26 | "Effect": "Allow", 27 | "Sid": "" 28 | } 29 | ] 30 | }' 31 | 32 | errexit() { 33 | echo "ERROR: $(basename "$0") (line ${LINENO}): ${1:-"Unknown Error"}" 1>&2 34 | exit 1 35 | } 36 | 37 | usage() { 38 | echo "$docs" 39 | } 40 | 41 | resource_exists() { 42 | # Run a command with an added --query length(...) 43 | # and check if it results in at least a 1 element list. 44 | local command 45 | local num_matches 46 | set -e 47 | if [[ -z "$2" ]] 48 | then 49 | command="$1 --query length(*[0])" 50 | else 51 | command="$1 --query length($2)" 52 | fi 53 | num_matches=$($command) 54 | if [[ "$?" -ne 0 ]] 55 | then 56 | echo "Could not check if resource exists, exiting." 57 | exit 2 58 | fi 59 | set +e 60 | if [[ "$num_matches" -gt 0 ]] 61 | then 62 | # RC of 0 mean the resource exists, "success" 63 | return 0 64 | else 65 | return 1 66 | fi 67 | } 68 | 69 | has_new_enough_openssl() { 70 | # This is a quirk of openssl. 71 | # RC of 1 (an error) means the command 72 | # exists. Otherwise an RC of 0 means 73 | # the command does *not* exist. So 74 | # we have to reverse this because we want 75 | # RC 0 -> success/true 76 | # RC 1 -> fail/false 77 | openssl pkey -foobar 2>/dev/null 78 | if [[ "$?" -eq 0 ]]; then 79 | return 1 80 | fi 81 | return 0 82 | } 83 | 84 | import_key_pair() { 85 | echo -n "Would you like to import ~/.ssh/id_rsa.pub? [y/N]: " 86 | read confirmation 87 | if [[ "$confirmation" != "y" ]] 88 | then 89 | return 90 | fi 91 | aws ec2 import-key-pair \ 92 | --key-name id_rsa \ 93 | --public-key-material file://~/.ssh/id_rsa.pub 94 | } 95 | 96 | create_instance_profile() { 97 | echo -n "Would you like to create an IAM instance profile? [y/N]: " 98 | read confirmation 99 | if [[ "$confirmation" != "y" ]]; then 100 | return 101 | fi 102 | aws iam create-role --role-name dev-ec2-instance \ 103 | --assume-role-policy-document "$TRUST_POLICY" || errexit "Could not create Role" 104 | 105 | # Use a managed policy 106 | policies=$(aws iam list-policies --scope AWS) 107 | admin_policy_arn=$(jp -u \ 108 | "Policies[?PolicyName=='AdministratorAccess'].Arn | [0]" <<< "$policies") 109 | aws iam attach-role-policy \ 110 | --role-name dev-ec2-instance \ 111 | --policy-arn "$admin_policy_arn" || errexit "Could not attach role policy" 112 | 113 | # Then we need to create an instance profile from the role. 114 | aws iam create-instance-profile \ 115 | --instance-profile-name dev-ec2-instance || \ 116 | errexit "Could not create instance profile." 117 | # And add it to the role 118 | aws iam add-role-to-instance-profile \ 119 | --role-name dev-ec2-instance \ 120 | --instance-profile-name dev-ec2-instance || \ 121 | errexit "Could not add role to instance profile." 122 | } 123 | 124 | compute_key_fingerprint() { 125 | # Computes the fingerprint of a public SSH key given a private 126 | # RSA key. This can be used to compare against the output given 127 | # from aws ec2 describe-key-pair. 128 | openssl pkey -in ~/.ssh/id_rsa -pubout -outform DER | \ 129 | openssl md5 -c | \ 130 | cut -d = -f 2 | \ 131 | tr -d '[:space:]' 132 | } 133 | 134 | tag_security_group() { 135 | aws ec2 describe-security-groups --query "SecurityGroups[].[GroupName,GroupId,VpcId]" --output text | column -t 136 | echo -n "Enter the security group ID to tag: " 137 | read response 138 | if [ -z "$response" ]; then 139 | return 140 | fi 141 | echo "Tagging security group" 142 | aws ec2 create-tags --resources "$response" --tags Key=dev-ec2-instance,Value=linux 143 | } 144 | 145 | do_setup() { 146 | echo "Checking for required resources..." 147 | echo "" 148 | # 1. Check if a security group is found for 149 | # both windows and non-windows tags. 150 | # If not, we'll eventually give the option to 151 | # configure this. 152 | if resource_exists "aws ec2 describe-security-groups \ 153 | --filter Name=tag:dev-ec2-instance,Values=linux"; then 154 | echo "Security groups exists." 155 | else 156 | echo "Security group not found." 157 | tag_security_group 158 | fi 159 | 160 | # 2. Make sure the keypair is imported. 161 | if [[ ! -f ~/.ssh/id_rsa ]]; then 162 | echo "Missing ~/.ssh/id_rsa key pair." 163 | elif has_new_enough_openssl; then 164 | fingerprint=$(compute_key_fingerprint ~/.ssh/id_rsa) 165 | if resource_exists "aws ec2 describe-key-pairs \ 166 | --filter Name=fingerprint,Values=$fingerprint"; then 167 | echo "Key pair exists." 168 | else 169 | echo "~/.ssh/id_rsa key pair does not appear to be imported." 170 | import_key_pair 171 | fi 172 | else 173 | echo "Can't check if SSH key has been imported." 174 | echo "You need at least openssl 1.0.0 that has a \"pkey\" command." 175 | echo "Please upgrade your version of openssl." 176 | fi 177 | 178 | # 3. Check that they have an IAM role called dev-ec2-instance. 179 | # There is no server side filter for this like we have with the EC2 180 | # APIs, so we have to manually use a --query option for us. 181 | if resource_exists "aws iam list-instance-profiles" \ 182 | "InstanceProfiles[?InstanceProfileName=='dev-ec2-instance']"; then 183 | echo "Instance profile exists." 184 | else 185 | echo "Missing IAM instance profile 'dev-ec2-instance'" 186 | create_instance_profile 187 | fi 188 | echo "Setup complete, you can now run: ./dev-ec2-instance" 189 | } 190 | 191 | do_setup 192 | -------------------------------------------------------------------------------- /2016/README.md: -------------------------------------------------------------------------------- 1 | # AWS CLI code samples from re:Invent 2015 2 | 3 | This directory contains the AWS CLI code samples presented from the 4 | "The Effective AWS CLI User" session at re:Invent 2016. 5 | 6 | The video for this talk is available on youtube: 7 | https://www.youtube.com/watch?v=Xc1dHtWa9-Q 8 | -------------------------------------------------------------------------------- /2016/auto-configure: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Script for auto-configuring ``s3.max_concurrent_requests`` variable based 4 | on transfer speed. Note that this script only accounts for upload 5 | speed. To run this script, the installed AWS CLI version must be 6 | 1.11.24 or higher. 7 | """ 8 | import os 9 | import re 10 | import shutil 11 | import subprocess 12 | import sys 13 | import tempfile 14 | import time 15 | 16 | 17 | BUCKET = None 18 | KEY = None 19 | TRANSFER_SPEED_REGEX = re.compile('\((.*)/s\)') 20 | MAX_POLLING_INTERVAL = 10 21 | TEMPFILE_SIZE = 5 * (1024 ** 3) 22 | TRANSFER_SPEED_DISPLAY = 'Transfer speed: %s/s' 23 | MAX_THREAD_INCREASE = 3 24 | 25 | SIZE_SUFFIX = { 26 | 'kb': 1024, 27 | 'mb': 1024 ** 2, 28 | 'gb': 1024 ** 3, 29 | 'tb': 1024 ** 4, 30 | 'kib': 1024, 31 | 'mib': 1024 ** 2, 32 | 'gib': 1024 ** 3, 33 | 'tib': 1024 ** 4, 34 | } 35 | 36 | 37 | assert BUCKET, ( 38 | 'Please set the BUCKET variable in the auto-configure script to a ' 39 | 'bucket that you can upload to') 40 | assert KEY, ( 41 | 'Please set the KEY variable in the auto-configure script to ' 42 | 'a key that you want to upload to') 43 | 44 | 45 | def human_readable_to_bytes(value): 46 | value = value.lower() 47 | if value[-2:] == 'ib': 48 | # Assume IEC suffix. 49 | suffix = value[-3:].lower() 50 | else: 51 | suffix = value[-2:].lower() 52 | has_size_identifier = ( 53 | len(value) >= 2 and suffix in SIZE_SUFFIX) 54 | if not has_size_identifier: 55 | try: 56 | return int(value) 57 | except ValueError: 58 | raise ValueError("Invalid size value: %s" % value) 59 | else: 60 | multiplier = SIZE_SUFFIX[suffix] 61 | return float(value[:-len(suffix)]) * multiplier 62 | 63 | 64 | def create_transfer_file(directory): 65 | filename = os.path.join(directory, 'file') 66 | with open(filename, 'a+') as f: 67 | f.truncate(TEMPFILE_SIZE) 68 | return filename 69 | 70 | 71 | def determine_transfer_speed(filename): 72 | p = upload(filename) 73 | try: 74 | return _get_transfer_speed_from_process(p) 75 | except BaseException: 76 | p.terminate() 77 | raise 78 | 79 | 80 | def _get_transfer_speed_from_process(p): 81 | start_time = time.time() 82 | total_time = 0 83 | current_transfer_speed = '' 84 | len_previous_print_statement = 0 85 | while total_time < MAX_POLLING_INTERVAL: 86 | line = p.stdout.read(40).decode('utf-8') 87 | transfer_speed_match = TRANSFER_SPEED_REGEX.search(line) 88 | if transfer_speed_match: 89 | transfer_speed = transfer_speed_match.group(1) 90 | print_statement = TRANSFER_SPEED_DISPLAY % transfer_speed 91 | print_statement = print_statement.ljust( 92 | len_previous_print_statement, ' ') 93 | sys.stdout.write(print_statement + '\r') 94 | sys.stdout.flush() 95 | len_previous_print_statement = len(print_statement) 96 | current_transfer_speed = transfer_speed 97 | total_time = time.time() - start_time 98 | p.terminate() 99 | sys.stdout.write('\n') 100 | return current_transfer_speed 101 | 102 | 103 | def upload(filename): 104 | return subprocess.Popen( 105 | 'aws s3 cp %s s3://%s/%s' % (filename, BUCKET, KEY), 106 | stdout=subprocess.PIPE, shell=True) 107 | 108 | 109 | def set_max_concurrent_requests(num): 110 | print('Setting max_concurrent_requests to: %s' % num) 111 | subprocess.check_call( 112 | 'aws configure set s3.max_concurrent_requests %s' % num, shell=True) 113 | 114 | 115 | def get_next_max_concurrent_requests(current, stats, history): 116 | stats_in_bytes = _get_stats_to_bytes(stats) 117 | return _calculate_next_max_concurrent_requests( 118 | current, stats_in_bytes, history) 119 | 120 | 121 | def _calculate_next_max_concurrent_requests(current, stats, history): 122 | if not history: 123 | return current + MAX_THREAD_INCREASE 124 | previous = history[-1] 125 | ratio_difference = stats[current] / stats[previous] 126 | multiplier = 0 127 | if ratio_difference > 1.75: 128 | multiplier = 1 129 | elif ratio_difference > 1.50: 130 | multiplier = 0.67 131 | elif ratio_difference > 1.25: 132 | multiplier = 0.33 133 | elif ratio_difference <= 1: 134 | return history[-1] 135 | return current + int(round(MAX_THREAD_INCREASE * multiplier)) 136 | 137 | 138 | def _get_stats_to_bytes(stats): 139 | stats_in_bytes = {} 140 | for num_threads, speed in stats.items(): 141 | stats_in_bytes[num_threads] = human_readable_to_bytes( 142 | speed.replace(' ', '')) 143 | return stats_in_bytes 144 | 145 | 146 | def _cleanup_orphan_multipart_uploads(): 147 | subprocess.check_call( 148 | 'aws s3api list-multipart-uploads --bucket %s ' 149 | '--prefix %s --query Uploads[].[UploadId] --output text | ' 150 | 'xargs -I {} aws s3api abort-multipart-upload --bucket ' 151 | '%s --key %s --upload-id {}' % ( 152 | BUCKET, KEY, BUCKET, KEY), shell=True 153 | ) 154 | 155 | 156 | def autoconfigure(): 157 | temp_directory = tempfile.mkdtemp() 158 | stats = {} 159 | history = [] 160 | max_concurrent_requests = 1 161 | try: 162 | print('Creating temporary file for uploads...') 163 | filename = create_transfer_file(temp_directory) 164 | print('Created temporary file: %s' % filename) 165 | while True: 166 | set_max_concurrent_requests(max_concurrent_requests) 167 | print('Starting upload...') 168 | stats[max_concurrent_requests] = determine_transfer_speed(filename) 169 | next_max_concurrent_requests = get_next_max_concurrent_requests( 170 | max_concurrent_requests, stats, history) 171 | if next_max_concurrent_requests <= max_concurrent_requests: 172 | break 173 | history.append(max_concurrent_requests) 174 | max_concurrent_requests = next_max_concurrent_requests 175 | print('Done tuning max_concurrent_requests...') 176 | set_max_concurrent_requests(next_max_concurrent_requests) 177 | except BaseException: 178 | raise 179 | finally: 180 | print('Cleaning up temporary file and any remaining multipart ' 181 | 'uploads...') 182 | shutil.rmtree(temp_directory) 183 | _cleanup_orphan_multipart_uploads() 184 | 185 | 186 | if __name__ == '__main__': 187 | autoconfigure() 188 | -------------------------------------------------------------------------------- /2017/README.md: -------------------------------------------------------------------------------- 1 | # AWS CLI code samples from re:Invent 2017 2 | 3 | This directory contains the AWS CLI code samples presented from the 4 | "AWS CLI: 2017 and Beyond" session at re:Invent 2017. 5 | 6 | The video for this talk is available on youtube: 7 | https://www.youtube.com/watch?v=W8IyScUGuGI 8 | -------------------------------------------------------------------------------- /2017/alias: -------------------------------------------------------------------------------- 1 | [toplevel] 2 | 3 | keychain = 4 | !f() { 5 | ACCESS_KEY=$(security find-generic-password -s "aws demo access" -w reinvent) 6 | SECRET_KEY=$(security find-generic-password -s "aws demo secret" -w reinvent) 7 | echo \{\"AccessKeyId\":\""$ACCESS_KEY"\",\"SecretAccessKey\":\""$SECRET_KEY"\",\"Version\":1\} 8 | }; f 9 | -------------------------------------------------------------------------------- /2017/clone: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will list all of your AWS CodeCommit repositories and clone 4 | # each one into your current working directory. 5 | # 6 | # If you provide a "-r | --repository-name" option, it will only clone the 7 | # repository matching the provided name in your current working directory. 8 | 9 | clone() { 10 | local repo_name="$1" 11 | ssh_url=$(aws codecommit get-repository --repository-name "$repo_name" \ 12 | --query repositoryMetadata.cloneUrlSsh --output text) 13 | git clone "$ssh_url" 14 | } 15 | 16 | 17 | if [ -z "$1" ]; then 18 | for repo_name in $(aws codecommit list-repositories --query "repositories[].repositoryName" --output text); do 19 | clone "$repo_name" 20 | done 21 | else 22 | case "$1" in 23 | -r|--repository-name) 24 | shift 25 | repo_name="$1" 26 | ;; 27 | *) 28 | echo "usage: clone [-r | --repository-name]" 1>&2 29 | exit 1 30 | esac 31 | clone "$repo_name" 32 | fi 33 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/issues), or [recently closed](https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"). You 4 | may not use this file except in compliance with the License. A copy of 5 | the License is located at 6 | 7 | http://aws.amazon.com/apache2.0/ 8 | 9 | or in the "license" file accompanying this file. This file is 10 | distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | ANY KIND, either express or implied. See the License for the specific 12 | language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS CLI code samples from past re:Invent CLI talks 2 | 3 | This repo contains code samples from past re:Invent talks. Navigate to 4 | the directory of the specific re:Invent year for sample code and information 5 | for that year's re:Invent talk. 6 | --------------------------------------------------------------------------------