├── .gitignore ├── assets ├── openspot.gif ├── openspot-logo.png └── openspot-tech-arch.png ├── aws ├── assets │ ├── EC2-Trust.json │ ├── spot-instance-specification-template.json │ ├── iam-instance-role-ec2-volume-policy.json │ ├── pull-secret.txt │ ├── post_install.sh │ └── user-data-template.sh ├── mac_client.sh ├── destroy.sh └── launch.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | aws/assets/spot-instance-specification.json 2 | aws/assets/user-data.sh 3 | -------------------------------------------------------------------------------- /assets/openspot.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksingh7/openspot/HEAD/assets/openspot.gif -------------------------------------------------------------------------------- /assets/openspot-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksingh7/openspot/HEAD/assets/openspot-logo.png -------------------------------------------------------------------------------- /assets/openspot-tech-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksingh7/openspot/HEAD/assets/openspot-tech-arch.png -------------------------------------------------------------------------------- /aws/assets/EC2-Trust.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "", 6 | "Effect": "Allow", 7 | "Principal": { 8 | "Service": "ec2.amazonaws.com" 9 | }, 10 | "Action": "sts:AssumeRole" 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /aws/assets/spot-instance-specification-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImageId": "AMI_ID", 3 | "KeyName": "KEY_PAIR_NAME", 4 | "SecurityGroupIds": [ "SG_ID" ], 5 | "UserData": "USER_DATA_BASE_64", 6 | "InstanceType": "INSTANCE_TYPE", 7 | "Placement": { 8 | "AvailabilityZone": "AZ_NAME" 9 | }, 10 | "IamInstanceProfile": { 11 | "Arn": "arn:aws:iam::AWS_ACCOUNT_NUMBER:instance-profile/crc-Instance-Profile" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /aws/assets/iam-instance-role-ec2-volume-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "StsDecodeEC2ListInstances", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "sts:DecodeAuthorizationMessage", 9 | "ec2:DescribeInstances", 10 | "ec2:DescribeVolumes", 11 | "ec2:AttachVolume", 12 | "ec2:CreateVolume", 13 | "ec2:CreateTags" 14 | ], 15 | "Resource": "*" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /aws/mac_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | brew install dnsmasq ; 3 | mkdir -p /usr/local/etc/dnsmasq.d ; 4 | rm /usr/local/etc/dnsmasq.d/crc.conf ; 5 | EIP=$(aws ec2 describe-instances --filters "Name=instance-type,Values=c5n.metal" --query "Reservations[*].Instances[*].PublicIpAddress" --output=text) ; 6 | echo "address=/apps-crc.testing/$EIP" > /usr/local/etc/dnsmasq.d/crc.conf ; 7 | echo "address=/api.crc.testing/$EIP" >> /usr/local/etc/dnsmasq.d/crc.conf ; 8 | sudo brew services restart dnsmasq ; 9 | dig apps-crc.testing @127.0.0.1 ; 10 | dig console-openshift-console.apps-crc.testing @127.0.0.1 ; 11 | -------------------------------------------------------------------------------- /aws/assets/pull-secret.txt: -------------------------------------------------------------------------------- 1 | {"auths":{"cloud.openshift.com":{"auth":"t007b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K2thcmFzaW5ncmVkaGF0Y29tMWx4ZmlqcmE4eGpmbGNnOGxxeGJ3eG9kbnJyOkVHWlJKUkNFSzFMTjVPQUc4RThDQjlWVk9IVkhBRzBSQTFWRVNJTzAwVzFLMFcwSlpRNlE3S0NYVTU1RVRZQzU=","email":"karasing@redhat.com"},"quay.io":{"auth":"b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K2thcmFzaW5ncmVkaGF0Y29tMWx4ZmlqcmE4eGpmbGNnOGxxeGJ3eG9kbnJyOkVHWlJKUkNFSzFMTjVPQUc4RThDQjlWVk9IVkhBRzBSQTFWRVNJTzAwVzFLMFcwSlpRNlE3S0NYVTU1RVRZQzU=","email":"karasing@redhat.com"},"registry.connect.redhat.com":{"auth":"Nzk2MzU2OHx1aGMtMUxYRklqcmE4eGpGTGNHOGxReEJ3eE9kTnJSOmV5SmhiR2NpT2lKU1V6VXhNaUo5LmV5SnpkV0lpT2lJNE5XVmtOakkyWmpSbFlXTTBNR1V6WVRKbVlURmhOekU0WkRrNVlUUTNOU0o5LklHdTgzTUZsaWNzam9oM0hIOUE3QTVhNmpwZTJlX0dvQldjRXRVaU5GM0ExUndCUDZEMEFTZDdEMVRIc2NLcWhfeE5HZUZKazVYSU5fa1pEZm1LYjdyY2wtUXhnVktBLU1LbFFrcUtQMzNjVE1laWd6LUFkVzNLV1g1U2NJTm4tX0lJaS1Pd0RGWEpMYXoyeC1DMm05Q3A4blZrZkhhSjZIYzhxOEFqbS1nZUczM283ekdWTEJrNmthOWh0ekYwOHNCeVRDa0xkeWJjc0Y2aVdDbnAzNS1PdE9YMWstSHR2TVpLNVRSTTI5UVJtaEZ0QUpJTEZWX3NqenBacmZlN05OLTBjOTdZay11NkFMV005OHpCZENrT3lQRTVMRWpUWm04TmxVZlpXNDlUUTdNY1BKN1h2Z1FuRlpHSDQ4bjlfUkNwbDluSlFmUm5yNFFvd2VyMXEzYWhEV1RGaGVyU05pV3NzVWJfVGJDQ1ZDdFJVMnJSODZYdHdyVEZiSS1mcXlSal9XclE3YUVHRm5TSlBnRHAxZV95SEo3QzJHZllXelBvRmNkU1MzejdfbHNfU2MwbnNpUHYyTVE1TGFmdUNsVEFVcWRlUUMzYkJtRnduOTQwdVVrbHdoZGYtcXBuT1lnRXNsX2VDVmxZOFJ3bkx4MjZuQjY1WWtPdnRRT2lfOWlXTG9oWURibTJJMkhWU3dXS3AzalVqaFFvS1FoWFJHVmIzUmRkMkcyQ0s5WWJtNlhTdTNrdHB0Rk4wSTJ0eXQwOG5MUDItcEpnOHQtdlJsV2lwSGEtZkt1eWQzQkFZNmszQXpCanA4RzdaZllvQXBTZF8xWG1YckNteER5U0pnMmpHU2FwMjBNbGJtam9rRHdHdzdIaTE4RWx0eWxEdXdKWEhSaF9WSkk4","email":"karasing@redhat.com"},"registry.redhat.io":{"auth":"Nzk2MzU2OHx1aGMtMUxYRklqcmE4eGpGTGNHOGxReEJ3eE9kTnJSOmV5SmhiR2NpT2lKU1V6VXhNaUo5LmV5SnpkV0lpT2lJNE5XVmtOakkyWmpSbFlXTTBNR1V6WVRKbVlURmhOekU0WkRrNVlUUTNOU0o5LklHdTgzTUZsaWNzam9oM0hIOUE3QTVhNmpwZTJlX0dvQldjRXRVaU5GM0ExUndCUDZEMEFTZDdEMVRIc2NLcWhfeE5HZUZKazVYSU5fa1pEZm1LYjdyY2wtUXhnVktBLU1LbFFrcUtQMzNjVE1laWd6LUFkVzNLV1g1U2NJTm4tX0lJaS1Pd0RGWEpMYXoyeC1DMm05Q3A4blZrZkhhSjZIYzhxOEFqbS1nZUczM283ekdWTEJrNmthOWh0ekYwOHNCeVRDa0xkeWJjc0Y2aVdDbnAzNS1PdE9YMWstSHR2TVpLNVRSTTI5UVJtaEZ0QUpJTEZWX3NqenBacmZlN05OLTBjOTdZay11NkFMV005OHpCZENrT3lQRTVMRWpUWm04TmxVZlpXNDlUUTdNY1BKN1h2Z1FuRlpHSDQ4bjlfUkNwbDluSlFmUm5yNFFvd2VyMXEzYWhEV1RGaGVyU05pV3NzVWJfVGJDQ1ZDdFJVMnJSODZYdHdyVEZiSS1mcXlSal9XclE3YUVHRm5TSlBnRHAxZV95SEo3QzJHZllXelBvRmNkU1MzejdfbHNfU2MwbnNpUHYyTVE1TGFmdUNsVEFVcWRlUUMzYkJtRnduOTQwdVVrbHdoZGYtcXBuT1lnRXNsX2VDVmxZOFJ3bkx4MjZuQjY1WWtPdnRRT2lfOWlXTG9oWURibTJJMkhWU3dXS3AzalVqaFFvS1FoWFJHVmIzUmRkMkcyQ0s5WWJtNlhTdTNrdHB0Rk4wSTJ0eXQwOG5MUDItcEpnOHQtdlJsV2lwSGEtZkt1eWQzQkFZNmszQXpCanA4RzdaZllvQXBTZF8xWG1YckNteER5U0pnMmpHU2FwMjBNbGJtam9rRHdHdzdIaTE4RWx0eWxEdXdKWEhSaF9WSkk4","email":"noname@noname.com"}}} -------------------------------------------------------------------------------- /aws/assets/post_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # bash post_install.sh ~/pull-secret.txt true 3 | PULL_SECRET_PATH=$1 4 | EXPAND_CRC_DISK_SIZE=$2 5 | DEFAULT_PULL_SECRET_FILE=~/pull-secret.txt 6 | 7 | CRC_XML_FILE=~/crc.xml 8 | if [ -f "$CRC_XML_FILE" ]; then 9 | echo "Previously configured CRC environment detected ... " 10 | echo "Reconfiguring CRC ..." 11 | for service in libvirtd.service libvirtd.socket ; do sudo systemctl start $service ; done 12 | for service in libvirtd.service libvirtd.socket ; do sudo systemctl enable $service ; done 13 | sudo virsh define ~/crc.xml 14 | fi 15 | 16 | echo "Setting up CRC ... " 17 | crc setup 18 | 19 | if [ -n "$PULL_SECRET_PATH" ]; then 20 | echo "Starting CRC ... " 21 | crc config set pull-secret-file $PULL_SECRET_PATH 22 | crc start 23 | CRC_IP=$(crc ip) 24 | elif [ -f "$DEFAULT_PULL_SECRET_FILE" ]; then 25 | echo "Starting CRC ... " 26 | crc config set pull-secret-file $DEFAULT_PULL_SECRET_FILE 27 | crc start 28 | CRC_IP=$(crc ip) 29 | else 30 | echo "Error: CRC Pull Secret not provided, Exiting..." 31 | echo "Get your pull secret from https://cloud.redhat.com/openshift/create/local and save it as ~/pull-secret.txt" 32 | exit 1 33 | fi 34 | 35 | if [ ! -f "$CRC_XML_FILE" ]; then 36 | echo "Stopping CRC temporarily ... " 37 | crc stop 38 | 39 | if [[ "$EXPAND_CRC_DISK_SIZE" == "true" ]]; then 40 | echo "Expanding CRC ROOT Disk Size by +60G ..." 41 | # Increase DISK size 42 | CRC_MACHINE_IMAGE=${HOME}/.crc/machines/crc/crc.qcow2 43 | # This resize is thin-provisioned 44 | sudo qemu-img resize ${CRC_MACHINE_IMAGE} +60G 45 | sudo cp ${CRC_MACHINE_IMAGE} ${CRC_MACHINE_IMAGE}.ORIGINAL 46 | #increase the /dev/sda4 (known as vda4 in the VM) disk partition size by an additional 20GB 47 | sudo virt-resize --expand /dev/sda4 ${CRC_MACHINE_IMAGE}.ORIGINAL ${CRC_MACHINE_IMAGE} 48 | sudo rm ${CRC_MACHINE_IMAGE}.ORIGINAL 49 | fi 50 | 51 | echo "Listing Libvirt VMs ... " 52 | sudo virsh list --all 53 | echo "Dumping VM config in XML ... " 54 | sudo virsh dumpxml crc > ~/crc.xml 55 | echo "Starting CRC ... " 56 | crc config set pull-secret-file $DEFAULT_PULL_SECRET_FILE 57 | crc start 58 | fi 59 | 60 | sleep 10 61 | 62 | echo "Setting up HAPROXY on host machine ..." 63 | SERVER_IP=0.0.0.0 64 | sudo cp /etc/haproxy/haproxy.cfg{,.bak} 65 | sudo semanage port -a -t http_port_t -p tcp 6443 66 | sudo tee /etc/haproxy/haproxy.cfg &>/dev/null < /dev/null 33 | aws ec2 terminate-instances --instance-ids $EC2_INSTANCE_ID > /dev/null 34 | 35 | echo "Deleting Spot Request ..." 36 | SPOT_REQUEST_ID=$(aws ec2 describe-spot-instance-requests --filters "Name=state,Values=open,active" "Name=tag:environment,Values=crc" "Name=availability-zone-group,Values=$REGION" --query "SpotInstanceRequests[*].[SpotInstanceRequestId]" --output text) 37 | aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SPOT_REQUEST_ID > /dev/null 38 | 39 | EBS_VOLUME_ID=$(aws ec2 describe-volumes --filters "Name=tag:environment,Values=crc" "Name=availability-zone,Values=$AZ_NAME" --query "Volumes[*].{ID:VolumeId}" --output text) 40 | 41 | echo "Detaching volume" 42 | aws ec2 detach-volume --volume-id $EBS_VOLUME_ID > /dev/null 43 | 44 | if [[ "$DELETE_EBS" == "true" ]]; then 45 | echo "Destroying EBS Volume ..." 46 | aws ec2 delete-volume --volume-id $EBS_VOLUME_ID > /dev/null 47 | fi 48 | 49 | echo "Removing Role from Instance Profile ... [Done]" 50 | aws iam remove-role-from-instance-profile --instance-profile-name crc-Instance-Profile --role-name crc-ec2-volume-role > /dev/null 51 | 52 | echo "Deleting Role Policy ... [Done]" 53 | aws iam delete-role-policy --role-name crc-ec2-volume-role --policy-name crc-ec20-volume-policy > /dev/null 54 | 55 | echo "Deleting Role ... [Done]" 56 | aws iam delete-role --role-name crc-ec2-volume-role > /dev/null 57 | 58 | echo "Deleting Instance Profile ... [Done]" 59 | aws iam delete-instance-profile --instance-profile-name crc-Instance-Profile > /dev/null 60 | 61 | echo "Deleting Key Pair ... [Done]" 62 | aws --region $REGION ec2 delete-key-pair --key-name crc-key-pair 63 | 64 | echo "Deleting Security Group... [Done]" 65 | aws ec2 delete-security-group --group-name crc-sg > /dev/null 2>&1 66 | 67 | } 68 | 69 | usage() { 70 | cat << EOT 71 | usage $0 -r "AWS_Region_Name" -a "AWS_AZ_NAME" -d "true or false" -v "true or false" 72 | OPTIONS 73 | -r "AWS Region Name : Optional, if not provided, will use AWS CLI default value" 74 | -a "AWS Availablity Zone Name : Optional, if not provided, will use AWS CLI default value" 75 | -d "Delete EBS Volume (true or false), default = false" 76 | -v "Optional, used for verbose output (true or false), default = false" 77 | -h "Show help menu" 78 | EOT 79 | } 80 | 81 | while getopts r:a:d:v:h option; do 82 | case $option in 83 | r) 84 | REGION="$OPTARG" 85 | ;; 86 | a) 87 | AZ_NAME="$OPTARG" 88 | ;; 89 | d) 90 | DELETE_EBS="$OPTARG" 91 | ;; 92 | v) 93 | DEBUG="$OPTARG" 94 | ;; 95 | \?) 96 | echo "wrong option." 97 | usage 98 | exit 1 99 | ;; 100 | h) 101 | usage 102 | exit 0 103 | ;; 104 | esac 105 | done 106 | shift $(($OPTIND - 1)) 107 | 108 | main -------------------------------------------------------------------------------- /aws/assets/user-data-template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | EBS_SIZE=100 3 | set -x 4 | rm /var/log/crc* > /dev/null 2>&1 5 | rm -rf aws* > /dev/null 2>&1 6 | touch /var/log/crc_setup.log 7 | chown fedora:fedora /var/log/crc_setup.log 8 | 9 | touch /var/log/crc_status 10 | chown fedora:fedora /var/log/crc_status 11 | echo "progressing" >> /var/log/crc_status 2>&1 12 | 13 | echo "Installing required packages ..." >> /var/log/crc_setup.log 2>&1 14 | dnf update -y > /dev/null 2>&1 15 | dnf install -y NetworkManager wget git haproxy vim unzip bc libvirt libvirt-daemon-kvm qemu-kvm libguestfs-tools guestfs-tools /usr/sbin/semanage > /dev/null 2>&1 16 | echo "Installing required packages ... [Done]" >> /var/log/crc_setup.log 2>&1 17 | 18 | echo "Setting up AWS Cli..." >> /var/log/crc_setup.log 2>&1 19 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/awscliv2.zip" > /dev/null 2>&1 20 | unzip /awscliv2.zip > /dev/null 2>&1 21 | ./aws/install > /dev/null 2>&1 22 | echo "Setting up AWS Cli... [Done]" >> /var/log/crc_setup.log 2>&1 23 | #EBS_VOLUME_ID=$(aws --region=REGION ec2 describe-volumes --filters "Name=tag:environment,Values=crc" --query "Volumes[*].{ID:VolumeId}" --output text) 24 | 25 | EBS_VOLUME_ID=$(aws --region=REGION ec2 describe-volumes --filters "Name=tag:environment,Values=crc" "Name=availability-zone,Values=AZ_NAME" --query "Volumes[*].{ID:VolumeId}" --output text) 26 | 27 | #EBS_VOLUME_AZ=$(aws --region=REGION ec2 describe-volumes --filters "Name=tag:environment,Values=crc" --query "Volumes[*].{ID:AvailabilityZone}" --output text) 28 | 29 | if [ -n "$EBS_VOLUME_ID" ]; then 30 | echo "Using existing EBS Volume ..." >> /var/log/crc_setup.log 2>&1 31 | else 32 | echo "Creating EBS Volume for CRC ..." >> /var/log/crc_setup.log 2>&1 33 | aws --region=REGION ec2 create-volume --volume-type gp2 --size $EBS_SIZE --availability-zone AZ_NAME --tag-specifications 'ResourceType=volume,Tags=[{Key="environment",Value="crc"}]' >> /var/log/crc_setup.log 2>&1 34 | fi 35 | sleep 10 36 | 37 | EC2_INSTANCE_ID=$(aws --region=REGION ec2 describe-instances --filters "Name=instance-type,Values=INSTANCE_TYPE" "Name=instance-state-code,Values=16" --query 'Reservations[*].Instances[*].{Instance:InstanceId}' --output text) 38 | echo "Instance ID :" $EC2_INSTANCE_ID >> /var/log/crc_setup.log 2>&1 39 | 40 | EBS_VOLUME_ID=$(aws --region=REGION ec2 describe-volumes --filters "Name=tag:environment,Values=crc" "Name=availability-zone,Values=AZ_NAME" --query "Volumes[*].{ID:VolumeId}" --output text) 41 | echo "EBS Volume ID :"$EBS_VOLUME_ID >> /var/log/crc_setup.log 2>&1 42 | 43 | echo "Attaching EBS Volume to CRC Spot Instance ..." >> /var/log/crc_setup.log 2>&1 44 | aws --region=REGION ec2 attach-volume --volume-id $EBS_VOLUME_ID --instance-id $EC2_INSTANCE_ID --device /dev/xvdb >> /var/log/crc_setup.log 2>&1 45 | 46 | sleep 10 47 | 48 | if [ $(blkid /dev/nvme1n1 | awk '{print $4}' | cut -d '"' -f 2) = "xfs" ]; then 49 | echo "XFS Filesystem found, just mounting volume ..." >> /var/log/crc_setup.log 2>&1 50 | echo "/dev/nvme1n1 /home xfs defaults 0 0" >> /etc/fstab 51 | mount -a 52 | sudo -u fedora sudo cp /home/fedora/crc /usr/bin/crc 53 | sudo -u fedora sudo cp /home/fedora/oc /usr/bin/oc 54 | POST_INSTALL="true" 55 | else 56 | echo "No Filesystem found, creating XFS filesystem ..." >> /var/log/crc_setup.log 2>&1 57 | mkfs.xfs /dev/nvme1n1 >> /var/log/crc_setup.log 2>&1 58 | mount /dev/nvme1n1 /mnt 59 | cp -rp /home/fedora /mnt/ 60 | echo "/dev/nvme1n1 /home xfs defaults 0 0" >> /etc/fstab 61 | umount /mnt 62 | mount -a >> /var/log/crc_setup.log 2>&1 63 | sudo -u fedora echo "Downloading latest version of CRC ..." >> /var/log/crc_setup.log 2>&1 64 | sudo -u fedora wget https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-linux-amd64.tar.xz -q --show-progres -O /home/fedora/crc-linux-amd64.tar.xz 65 | 66 | sudo -u fedora echo "Downloading latest version of OC client ..." >> /var/log/crc_setup.log 2>&1 67 | sudo -u fedora wget https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz -q --show-progress -O /home/fedora/openshift-client-linux.tar.gz 68 | 69 | sudo -u fedora echo "Extracting CRC binary ..." >> /var/log/crc_setup.log 2>&1 70 | sudo -u fedora tar xvf /home/fedora/crc-linux-amd64.tar.xz -C /home/fedora >> /var/log/crc_setup.log 2>&1 71 | sudo -u fedora sudo cp /home/fedora/crc-linux-*-amd64/crc /usr/bin/crc 72 | sudo -u fedora sudo cp /home/fedora/crc-linux-*-amd64/crc /home/fedora/crc 73 | 74 | sudo -u fedora echo "Extracting OC binary ..." >> /var/log/crc_setup.log 2>&1 75 | sudo -u fedora tar xvf /home/fedora/openshift-client-linux.tar.gz -C /home/fedora >> /var/log/crc_setup.log 2>&1 76 | sudo -u fedora sudo cp /home/fedora/oc /usr/bin/oc 77 | 78 | sudo -u fedora rm -rf /home/fedora/crc-linux* > /dev/null 2>&1 79 | sudo -u fedora rm /home/fedora/openshift-client* > /dev/null 2>&1 80 | 81 | sudo -u fedora echo "Cleaning up leftovers ... [Done]" >> /var/log/crc_setup.log 2>&1 82 | fi 83 | 84 | echo "Calculating CPU cores for CRC usage ..." >> /var/log/crc_setup.log 2>&1 85 | CPU_TEMP=$(echo "$(lscpu | grep -v "NUMA" | grep -i "CPU(s):" | awk '{print $2}')*0.90" | bc) 86 | CPU=$(printf '%.0f\n' $CPU_TEMP) 87 | 88 | echo "Calculating Memory for CRC usage ..." >> /var/log/crc_setup.log 2>&1 89 | MEMORY_TEMP=$(echo "$(free -m | grep -i mem | awk '{print $2}') * 0.90" | bc) 90 | MEMORY=$(printf '%.0f\n' $MEMORY_TEMP) 91 | 92 | sudo -u fedora echo "Setting up CRC ..." >> /var/log/crc_setup.log 2>&1 93 | sudo -u fedora crc config set cpus $CPU >> /var/log/crc_setup.log 2>&1 94 | sudo -u fedora crc config set memory $MEMORY >> /var/log/crc_setup.log 2>&1 95 | sudo -u fedora crc config set enable-cluster-monitoring true >> /var/log/crc_setup.log 2>&1 96 | sudo -u fedora crc config set consent-telemetry yes >> /var/log/crc_setup.log 2>&1 97 | sudo -u fedora crc config set kubeadmin-password kubeadmin >> /var/log/crc_setup.log 2>&1 98 | sudo -u fedora crc config view >> /var/log/crc_setup.log 2>&1 99 | 100 | sudo -u fedora echo "===== Starting CRC ... Please wait ====" >> /var/log/crc_setup.log 2>&1 101 | sudo -u fedora wget https://raw.githubusercontent.com/ksingh7/openspot/main/aws/assets/post_install.sh -O /home/fedora/post_install.sh 102 | sudo -u fedora chmod +x /home/fedora/post_install.sh 103 | sudo -u fedora wget https://raw.githubusercontent.com/ksingh7/openspot/main/aws/assets/pull-secret.txt -O /home/fedora/pull-secret.txt 104 | sudo -u fedora bash /home/fedora/post_install.sh /home/fedora/pull-secret.txt true >> /var/log/crc_setup.log 2>&1 105 | 106 | sudo -u fedora echo "===== CRC Setup Completed ====" >> /var/log/crc_setup.log 2>&1 107 | sudo -u fedora echo "#############################################################################################" >> /var/log/crc_setup.log 2>&1 108 | sudo -u fedora echo "### You should now configure your local machine DNS records inorder to use this CRC instance ###" >> /var/log/crc_setup.log 2>&1 109 | sudo -u fedora echo "### If you are using MacOS, please execute mac_client.sh script to configure your local DNS ###" >> /var/log/crc_setup.log 2>&1 110 | sudo -u fedora echo "#############################################################################################" >> /var/log/crc_setup.log 2>&1 111 | 112 | ################################################################################# 113 | ### You should now configure your local machine DNS records inorder to use this CRC instance ### 114 | ### If you are using MacOS, please execute mac_client.sh script to configure your local DNS ### 115 | ################################################################################# 116 | 117 | 118 | # if [[ "$POST_INSTALL" == "true" ]]; then 119 | # #sudo -u fedora echo "===== Running Post Install ... Please wait ====" >> /var/log/crc_setup.log 2>&1 120 | # #sudo -u fedora bash /home/fedora/post_install.sh >> /var/log/crc_setup.log 2>&1 121 | # sudo -u fedora echo "===== You can now Exit from logs tail command by presing Ctrl+C ====" >> /var/log/crc_setup.log 2>&1 122 | # else 123 | # sudo -u fedora wget https://raw.githubusercontent.com/ksingh7/openspot/main/aws/assets/post_install.sh -O /home/fedora/post_install.sh 124 | # sudo -u fedora chmod +x /home/fedora/post_install.sh 125 | # sudo -u fedora echo "===== CRC Setup Completed ====" >> /var/log/crc_setup.log 2>&1 126 | # sudo -u fedora echo "===== You can now SSH into the instance for post installation setup ====" >> /var/log/crc_setup.log 2>&1 127 | # sudo -u fedora echo "===== Post Installation scrip file location : /home/fedora/post_install.sh ====" >> /var/log/crc_setup.log 2>&1 128 | # sudo -u fedora echo "===== You can now Exit from logs tail command by presing Ctrl+C ====" >> /var/log/crc_setup.log 2>&1 129 | # fi 130 | 131 | # echo "completed" > /var/log/crc_status 2>&1 -------------------------------------------------------------------------------- /aws/launch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #bash launch.sh -r ap-south-1 -a ap-south-1a -v false 3 | # Author: karan.singh731987@gmail.com , karan@redhat.com (Karan Singh) 4 | 5 | main() { 6 | OS=$(uname) 7 | #REGION=$1 8 | #AZ_NAME=$2 9 | #DEBUG=$3 10 | #AMI_ID=ami-01d3bd808e1fd393c # Default Fedora-Cloud-Base-34-1.2.x86_64-hvm-ap-south-1-gp2-0 11 | #AMI_ID=ami-0de1a1ee38e9d0267 # ap-southeast-1 Fedora-Cloud-Base-34-1.2.x86_64-hvm-ap-southeast-1-gp2-0 12 | 13 | KEY_PAIR_NAME="crc-key-pair" 14 | AWS_ACCOUNT_NUMBER=$(aws sts get-caller-identity --query "Account" --output text) 15 | INSTANCE_TYPE=c5n.metal #cheapest x86 baremetal instance from AWS 16 | 17 | if [ "$DEBUG" == "true" ]; then 18 | echo "Enabling Verbose output ..." 19 | set -x 20 | fi 21 | 22 | # if [ -n "$SPOTPRICE" ]; then 23 | # echo "Launching instance with SPOT PRICE of : "$"$REGION ..." 24 | # else 25 | # echo "Error : SPOT Price missing , please provide SPOT price" 26 | # echo "------- You can run the below command to get spot price history ------- " 27 | # echo 'aws ec2 describe-spot-price-history --start-time=$(date +%s) --instance-types $INSTANCE_TYPE --product-descriptions="Linux/UNIX"' 28 | # exit 1 29 | # fi 30 | 31 | if [ -n "$REGION" ]; then 32 | : 33 | else 34 | REGION=$(aws configure get region) 35 | echo "No Region provided, launching instance in Region : $REGION ..." 36 | fi 37 | 38 | if [ -n "$AZ_NAME" ]; then 39 | : 40 | else 41 | AZ_NAME="$REGION"a 42 | echo "No AZ provided, launching instance in AZ : $AZ_NAME ..." 43 | fi 44 | 45 | if [ -n "$AMI_ID" ]; then 46 | : 47 | else 48 | AMI_ID=ami-01d3bd808e1fd393c 49 | echo "Using default AMI ID : $AMI_ID -- Fedora-Cloud-Base-34-1.2.x86_64-hvm-ap-south-1-gp2-0" 50 | fi 51 | 52 | if [ -z $PUB_KEY_PATH ]; then 53 | echo "Need your SSH Public Key absolute path to create AWS Key Pair in the selected Region (ex: $HOME/.ssh/id_rsa.pub) : " 54 | read -p "Enter SSH Public Key Path [$HOME/.ssh/id_rsa.pub]: " PUB_KEY_PATH 55 | PUB_KEY_PATH=${PUB_KEY_PATH:-$HOME/.ssh/id_rsa.pub} 56 | fi 57 | 58 | #in this way if $PUB_KEY_PATH is set from outside the script the path won't be asked (useful for CI/CD pipelines and unattended installations) 59 | if [ -a $PUB_KEY_PATH ]; then 60 | aws --region $REGION ec2 import-key-pair --key-name $KEY_PAIR_NAME --public-key-material fileb://$PUB_KEY_PATH --tag-specifications 'ResourceType=key-pair,Tags=[{Key="environment",Value="crc"}]' > /dev/null 61 | echo "New key-pair named "$KEY_PAIR_NAME" created in region "$REGION"..." 62 | else 63 | echo "Invalid SSH Public Key path or Key file does not exists ... exiting" 64 | exit 1 65 | fi 66 | 67 | if [[ "$IS_IAM_ROLE_EXISTS" == "crc-ec2-volume-role" ]]; then 68 | echo "IAM Role, Policy, Instance Profile, Already Exists, Skipping ..." 69 | else 70 | echo "Creating IAM Role ..." 71 | aws iam create-role --role-name crc-ec2-volume-role --assume-role-policy-document file://assets/EC2-Trust.json > /dev/null 72 | echo "Adding policy to IAM Role ..." 73 | aws iam put-role-policy --role-name crc-ec2-volume-role --policy-name crc-ec20-volume-policy --policy-document file://assets/iam-instance-role-ec2-volume-policy.json > /dev/null 74 | echo "Creating Instance Profile ..." 75 | aws iam create-instance-profile --instance-profile-name crc-Instance-Profile > /dev/null 76 | echo "Adding Role to Instance Profile ..." 77 | aws iam add-role-to-instance-profile --instance-profile-name crc-Instance-Profile --role-name crc-ec2-volume-role > /dev/null 78 | fi 79 | 80 | aws ec2 delete-security-group --group-name crc-sg > /dev/null 2>&1 81 | 82 | IS_SG_EXISTS=$(aws ec2 describe-security-groups --filters "Name=tag:environment,Values=crc" --query "SecurityGroups[*].{Name:GroupName}" --output text) 83 | 84 | if [[ "$IS_SG_EXISTS" == "crc-sg" ]]; then 85 | echo "Security Group Already Exists, Skipping ..." 86 | SG_ID=$(aws ec2 describe-security-groups --filters "Name=tag:environment,Values=crc" --query "SecurityGroups[*].{Name:GroupId}" --output text) 87 | else 88 | echo "Creating Security Group ..." 89 | SG_ID=$(aws ec2 create-security-group --group-name crc-sg --description "CRC Security Group" --tag-specifications 'ResourceType=security-group,Tags=[{Key="environment",Value="crc"}]' | jq -r .GroupId) 90 | for PORT in 22 80 443 6443 ; do aws ec2 authorize-security-group-ingress --group-id $SG_ID --protocol tcp --port $PORT --cidr '0.0.0.0/0' --output text > /dev/null 2>&1 ; done 91 | fi 92 | 93 | echo "Generating User-Data script file ..." 94 | if [[ $OS == "Linux" ]] 95 | then 96 | sed 's/REGION/'$REGION'/g' assets/user-data-template.sh > assets/user-data.sh 97 | sed -i 's/AZ_NAME/'$AZ_NAME'/g' assets/user-data.sh 98 | sed -i 's/INSTANCE_TYPE/'$INSTANCE_TYPE'/g' assets/user-data.sh 99 | USER_DATA_BASE_64=$(base64 assets/user-data.sh) 100 | ## Todo - Improve this (replacing should be made with jq) 101 | echo "Generating Launch Specification file ..." 102 | sed 's/SG_ID/'$SG_ID'/' assets/spot-instance-specification-template.json > assets/spot-instance-specification.json 103 | sed -i 's/KEY_PAIR_NAME/'$KEY_PAIR_NAME'/' assets/spot-instance-specification.json 104 | sed -i 's/INSTANCE_TYPE/'$INSTANCE_TYPE'/' assets/spot-instance-specification.json 105 | sed -i 's/AWS_ACCOUNT_NUMBER/'$AWS_ACCOUNT_NUMBER'/' assets/spot-instance-specification.json 106 | #used awk to avoid strange behaviour on escape characters 107 | #new lines replaced in the variable expansion with tr (escaped) 108 | awk -i inplace -v DATA_BASE_64="${USER_DATA_BASE_64//[$'\t\r\n ']}" '{ sub(/USER_DATA_BASE_64/, DATA_BASE_64); print; }' assets/spot-instance-specification.json 109 | sed -i 's/AZ_NAME/'$AZ_NAME'/' assets/spot-instance-specification.json 110 | sed -i 's/AMI_ID/'$AMI_ID'/' assets/spot-instance-specification.json 111 | else 112 | #Not sure if these sed calls are meant to work with BSD sed but on linux they seem to not work 113 | sed 's/REGION/'$REGION'/g' assets/user-data-template.sh > assets/user-data.sh 114 | sed -i ' ' 's/AZ_NAME/'$AZ_NAME'/g' assets/user-data.sh 115 | sed -i ' ' 's/INSTANCE_TYPE/'$INSTANCE_TYPE'/g' assets/user-data.sh 116 | USER_DATA_BASE_64=$(base64 assets/user-data.sh) 117 | ## Todo - Improve this 118 | echo "Generating Launch Specification file ..." 119 | sed 's/SG_ID/'$SG_ID'/' assets/spot-instance-specification-template.json > assets/spot-instance-specification.json 120 | sed -i ' ' 's/KEY_PAIR_NAME/'$KEY_PAIR_NAME'/' assets/spot-instance-specification.json 121 | sed -i ' ' 's/INSTANCE_TYPE/'$INSTANCE_TYPE'/' assets/spot-instance-specification.json 122 | sed -i ' ' 's/AWS_ACCOUNT_NUMBER/'$AWS_ACCOUNT_NUMBER'/' assets/spot-instance-specification.json 123 | sed -i ' ' 's/USER_DATA_BASE_64/'$USER_DATA_BASE_64'/' assets/spot-instance-specification.json 124 | sed -i ' ' 's/AZ_NAME/'$AZ_NAME'/' assets/spot-instance-specification.json 125 | sed -i ' ' 's/AMI_ID/'$AMI_ID'/' assets/spot-instance-specification.json 126 | fi 127 | echo "Launching SPOT Instance, Please Wait ..." 128 | sleep 10 129 | aws ec2 request-spot-instances --availability-zone-group $REGION --instance-count 1 --type "one-time" --launch-specification file://assets/spot-instance-specification.json --tag-specifications 'ResourceType=spot-instances-request,Tags=[{Key="environment",Value="crc"}]' > /dev/null 130 | rm -f assets/spot-instance-specification.json 131 | rm -f assets/user-data.sh 132 | 133 | # Todo : If instance is not provisioned due to capacity or other issues, 134 | # Add logic to delete last spot request and submit a new one 135 | 136 | #SPOT_REQUEST_OUTPUT=$(aws ec2 describe-spot-instance-requests --filters "Name=state,Values=open,active" "Name=tag:environment,Values=crc" "Name=availability-zone-group,Values=$REGION") 137 | #echo $SPOT_REQUEST_ID 138 | echo "Please allow 5 minutes for instance configuration" 139 | sleep 180 140 | echo "Trying to tail instance setup logs ... " 141 | sleep 10 142 | 143 | echo "Applying TAG to Instance" 144 | EC2_INSTANCE_ID=$(aws --region=$REGION ec2 describe-instances --filters "Name=instance-type,Values=$INSTANCE_TYPE" "Name=instance-state-code,Values=16" --query 'Reservations[*].Instances[*].{Instance:InstanceId}' --output text) 145 | aws ec2 create-tags --resources $EC2_INSTANCE_ID --tags 'Key=environment,Value=crc' 'Key=availability-zone,Value=$AZ_NAME' > /dev/null 146 | 147 | EIP=$(aws ec2 describe-instances --filters "Name=instance-type,Values=$INSTANCE_TYPE" "Name=availability-zone,Values=$AZ_NAME" --query "Reservations[*].Instances[*].PublicIpAddress" --output=text) 148 | ssh -o "StrictHostKeyChecking no" fedora@$EIP tail -50f /var/log/crc_setup.log 149 | 150 | } 151 | 152 | usage() { 153 | cat << EOT 154 | usage $0 -r "AWS_Region_Name" -a "AWS_AZ_NAME" -v "true or false" 155 | OPTIONS 156 | -r "AWS Region Name : Optional, if not provided, will use AWS CLI default value" 157 | -a "AWS Availablity Zone Name : Optional, if not provided, will use AWS CLI default value" 158 | -v "Optional : Verbose Output, set either true or false, default value is false" 159 | -i "Optional : AMI ID to use, default: ami-01d3bd808e1fd393c Fedora-Cloud-Base-34-1.2.x86_64-hvm-ap-south-1-gp2-0" 160 | -h "Show help menu" 161 | EOT 162 | } 163 | 164 | while getopts r:a:v:h:i: option; do 165 | case $option in 166 | r) 167 | REGION="$OPTARG" 168 | ;; 169 | a) 170 | AZ_NAME="$OPTARG" 171 | ;; 172 | v) 173 | DEBUG="$OPTARG" 174 | ;; 175 | i) 176 | AMI_ID="$OPTARG" 177 | ;; 178 | \?) 179 | echo "wrong option." 180 | usage 181 | exit 1 182 | ;; 183 | h) 184 | usage 185 | exit 0 186 | ;; 187 | esac 188 | done 189 | shift $(($OPTIND - 1)) 190 | 191 | main 192 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![OpenSpot](assets/openspot.gif) 2 | ## tldr; 3 | 1. Make sure AWS CLI is configured properly with right set of credentials and access to launch resources 4 | 2. Execute the following to launch CRC on AWS Spot Instance 5 | ``` 6 | git clone https://github.com/ksingh7/openspot.git 7 | cd openspot/aws 8 | time bash launch.sh -r ap-south-1 -a ap-south-1a -v false 9 | ``` 10 | ## Preamble 11 | ### What is crc 12 | Red Hat CodeReady Containers (CRC) brings a minimal single node OpenShift 4 cluster to your local computer. This cluster provides a minimal OpenShift environment for development and testing purposes. [Read more](https://developers.redhat.com/products/codeready-containers/overview) 13 | 14 | ### What are Spot Instances 15 | Amazon EC2 Spot Instances let you take advantage of unused EC2 capacity in the AWS datacenter. Spot Instances are available at up to a 90% discount compared to On-Demand prices. [Read more](https://aws.amazon.com/ec2/spot/) 16 | ### What is OpenSpot 17 | OpenSpot [Open~~Shift on~~ Spot ~~Instance~~] is a tool that helps you deploy CRC on AWS Spot Instances in a fully automated & resilient manner. 18 | The core idea of openspot is automate all the steps required to launch AWS Spot Instance and automatically configure CRC. 19 | OpenSpot allows you to tear down CRC spot instance and their associated resources, when you do not need it, that translates to further cost saving. 20 | 21 | For example : You are a developer/architect, need a CRC environment daily for 4 hours. You can use OpenSpot, launch your CRC environment and terminate after 4 hours, and just repeat this step daily. OpenSpot uses EBS volume to persist your CRC instance, destroying and re-creating OpenSpot instance, do not destroy the CRC VM that you are working on daily (unless you are deleting EBS volume) 22 | 23 | ##### Features of OpenSpot 24 | - Select Region & AZ of your choice 25 | - Sequentially provision all AWS resources required (as pre-requisite) 26 | - Key-pair 27 | - IAM role, policy, instance-policy 28 | - Security Group 29 | - user-data as Template 30 | - Launch Spot Instnace 31 | - Configure OS 32 | - Install all required packages 33 | - Get CRC/OC binaries 34 | - EBS volume 35 | - Provision, Attach 36 | - Detect/Create/Mount filesystem 37 | - CRC Setup 38 | - Dynamically set CPU/Memory 39 | - Expand CRC root disk 40 | - Configure Haproxy & make CRC instance available remotely 41 | - Handle Spot Instance Termination 42 | - Provision new Spot Instnace 43 | - Detect previous instance of CRC and resume that 44 | - Cleanup 45 | - Destroy all AWS resources requested by openspot 46 | 47 | ![OpenSpot Technical Architecture](assets/openspot-tech-arch.png) 48 | ## Setup 49 | ### Prerequisite 50 | - AWS CLI must be configured on local machines 51 | - AWS Admin Access & Secret Key 52 | - If you do not have Admin access, make sure your AWS ID has right capabilites to provision resources like `IAM roles, policies,instance-policy,key-pair,security group,spot-instance,EBS` 53 | - Test configuration of AWS CLI 54 | ``` 55 | aws ec2 describe-instances 56 | ``` 57 | - Get OpenSpot 58 | ``` 59 | git clone https://github.com/ksingh7/openspot.git 60 | cd openspot/aws 61 | ``` 62 | ## Deploy CRC on Spot Instance 63 | 64 | ``` 65 | usage launch.sh -r "AWS_Region_Name" -a "AWS_AZ_NAME" -v "true or false" 66 | OPTIONS 67 | -r "AWS Region Name : Optional, if not provided, will use AWS CLI default value" 68 | -a "AWS Availablity Zone Name : Optional, if not provided, will use AWS CLI default value" 69 | -v "Optional : Verbose Output, set either true or false, default value is false" 70 | -h "Show help menu" 71 | -i "AMI ID" 72 | ``` 73 | - To launch AWS Spot Instance in `ap-south-1` region and `ap-south-1a` availablity zone, execute 74 | ``` 75 | time bash launch.sh -r ap-south-1 -a ap-south-1a -v false 76 | ``` 77 | Sample output 78 | ``` 79 | $ time bash launch.sh -r ap-south-1 -a ap-south-1a -v false 80 | Need your SSH Public Key absolute path to create AWS Key Pair in the selected Region (ex: /Users/karasing/.ssh/id_rsa.pub) : 81 | Enter SSH Public Key Path [/Users/karasing/.ssh/id_rsa.pub]: 82 | New key-pair named crc-key-pair created in region ap-south-1... 83 | Creating IAM Role ... 84 | Adding policy to IAM Role ... 85 | Creating Instance Profile ... 86 | Adding Role to Instance Profile ... 87 | Creating Security Group ... 88 | Generating User-Data script file ... 89 | Generating Launch Specification file ... 90 | Launching SPOT Instance, Please Wait ... 91 | Please allow 5 minutes for instance configuration 92 | Trying to tail instance setup logs ... 93 | Applying TAG to Instance 94 | Warning: Permanently added '13.232.50.164' (ECDSA) to the list of known hosts. 95 | Installing required packages ... [Done] 96 | Setting up AWS Cli... [Done] 97 | Using existing EBS Volume ... 98 | ... 99 | ... 100 | Started the OpenShift cluster. 101 | 102 | The server is accessible via web console at: 103 | https://console-openshift-console.apps-crc.testing 104 | 105 | Log in as administrator: 106 | Username: kubeadmin 107 | Password: kubeadmin 108 | 109 | Log in as user: 110 | Username: developer 111 | Password: developer 112 | 113 | Use the 'oc' command line interface: 114 | $ eval $(crc oc-env) 115 | $ oc login -u developer https://api.crc.testing:6443 116 | Setting up HAPROXY on host machine ... 117 | Starting HAPROXY Service ... 118 | ========= Post Launch Configuration Completed Successfully ============== 119 | ===== You can now Exit from logs tail command by presing Ctrl+C ==== 120 | bash launch.sh -r ap-south-1 -a ap-south-1a -v false 121 | 9.87s user 12.99s system 3% cpu 9:55.69 total 122 | ``` 123 | ## Configure Local Machine to remotely connect to CRC Instance 124 | - Automated Instructions for Macos Client to connect to CRC Instance 125 | ``` 126 | bash aws/mac_client.sh 127 | ``` 128 | - Manual Instructions for configuring `MacOS` client to connet to remote CRC 129 | ``` 130 | brew install dnsmasq 131 | mkdir -p /usr/local/etc/dnsmasq.d 132 | touch /usr/local/etc/dnsmasq.d/crc.conf 133 | EIP=$(aws ec2 describe-instances --filters "Name=instance-type,Values=c5n.metal" --query "Reservations[*].Instances[*].PublicIpAddress" --output=text) ; 134 | echo "address=/apps-crc.testing/$EIP" > /usr/local/etc/dnsmasq.d/crc.conf ; 135 | echo "address=/api.crc.testing/$EIP" >> /usr/local/etc/dnsmasq.d/crc.conf ; 136 | sudo brew services restart dnsmasq ; 137 | dig apps-crc.testing @127.0.0.1 ; 138 | dig console-openshift-console.apps-crc.testing @127.0.0.1 ; 139 | ``` 140 | - Login to OC 141 | ``` 142 | oc login -u kubeadmin -p kubeadmin https://api.crc.testing:6443 143 | oc login -u developer -p developer https://api.crc.testing:6443 144 | ``` 145 | - Manual Instructions for Linux `Fedora` 146 | ``` 147 | sudo dnf install dnsmasq 148 | 149 | sudo tee /etc/NetworkManager/conf.d/use-dnsmasq.conf &>/dev/null </dev/null <