├── cloudfuzzer.conf ├── .gitignore ├── tests ├── volume_container_rsync │ ├── Dockerfile │ ├── test-script.sh │ └── run-test-local.sh └── gce-tests │ ├── scripts │ └── get-gce-ip-address.sh │ └── gce-setup.sh ├── bastion ├── run-containers.sh ├── get-stats.sh ├── get-results.sh ├── autocomplete.bash.inc ├── distribute-docker-image.sh ├── setup-swarm.sh └── helpers.sh ├── fuzzvm ├── sync-results.sh ├── run-containers.sh ├── get-stats.sh ├── swarm-create.sh └── functions.bash.inc ├── scripts ├── get-directory.sh ├── autocomplete.bash.inc ├── create-keys.sh ├── send-docker-data.sh └── functions.bash.inc ├── packer ├── provisioner-scripts │ ├── bastion-setup.sh │ └── fuzzvm-setup.sh ├── ubuntu-16.04.preseed.cfg ├── packer-fuzzvm.json └── packer-bastion.json ├── doc ├── work-flow.md └── draft1.txt ├── README.md └── LICENSE /cloudfuzzer.conf: -------------------------------------------------------------------------------- 1 | SSH_OPTS="-o StrictHostKeyChecking=no -i $CLOUDFUZZER_DIR/vm-keys/bastion-key" 2 | BASTION_USER="ubuntu" 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vm-keys/ 2 | packer/packer_cache 3 | packer/bastion-img 4 | packer/fuzzvm-img 5 | context/ 6 | test-results/ 7 | user.conf 8 | -------------------------------------------------------------------------------- /tests/volume_container_rsync/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | ADD ./test-script.sh /usr/local/bin/run 4 | 5 | ENTRYPOINT ["/usr/local/bin/run"] -------------------------------------------------------------------------------- /bastion/run-containers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | COUNT=$1 7 | 8 | MASTER_ADDRESS=$(cat $HOME/address_master) 9 | 10 | scp $HOME/docker-arguments $MASTER_ADDRESS: 11 | ssh $MASTER_ADDRESS "./scripts/run-containers.sh $COUNT" -------------------------------------------------------------------------------- /fuzzvm/sync-results.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BASTION_ADDRESS=$(cat $HOME/address_bastion) 4 | OWN_ADDRESS=$(hostname -I | awk '{print $1}') 5 | 6 | ssh -o StrictHostKeyChecking=no ubuntu@$BASTION_ADDRESS "scripts/helpers.sh get-results $OWN_ADDRESS" 7 | 8 | echo "Done syncing, shutting down." 9 | -------------------------------------------------------------------------------- /scripts/get-directory.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | REMOTE_DIRECTORY=$1 7 | LOCAL_DIRECTORY=$2 8 | 9 | echo "Fetching $REMOTE_DIRECTORY to $LOCAL_DIRECTORY" 10 | 11 | rsync -auP -z --no-links --min-size=1 -r $BASTION_USER@$BASTION_ADDRESS:$REMOTE_DIRECTORY/* $LOCAL_DIRECTORY; -------------------------------------------------------------------------------- /bastion/get-stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | MASTER_ADDRESS=$(cat $HOME/address_master) 7 | 8 | ssh $MASTER_ADDRESS "./scripts/get-stats.sh" 9 | 10 | if [ -d $HOME/stats ]; then 11 | rm -rf $HOME/stats 12 | fi 13 | 14 | mkdir $HOME/stats 15 | 16 | scp $MASTER_ADDRESS:stats/* $HOME/stats -------------------------------------------------------------------------------- /bastion/get-results.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | if [ $# -eq 0 ] 7 | then 8 | NODES=$(cat $HOME/address_nodes) 9 | else 10 | NODES=$@ 11 | fi 12 | 13 | # Rsync nodes 14 | for node in $NODES; do 15 | rsync -auP --no-links --min-size=1 rsync://$node:10873/volume/ results/ 16 | done 17 | -------------------------------------------------------------------------------- /tests/gce-tests/scripts/get-gce-ip-address.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | #Helper for fetching of gce instance IP addresses. 7 | #Note: This script is for usage with gce-setup.sh, different instance configuration, 8 | #might result return values other than IP addresses. 9 | 10 | INSTANCE_NAME_PREFIX=$1; 11 | 12 | gcloud compute instances list | awk '$1 ~ /'$INSTANCE_NAME_PREFIX'/ {print $(--NF)}' | xargs -------------------------------------------------------------------------------- /scripts/autocomplete.bash.inc: -------------------------------------------------------------------------------- 1 | _cloudfuzzer() 2 | { 3 | local cur prev opts 4 | COMPREPLY=() 5 | cur="${COMP_WORDS[COMP_CWORD]}" 6 | prev="${COMP_WORDS[COMP_CWORD-1]}" 7 | opts="create-keys send-docker-data bastion ssh get-results get-stats help" 8 | 9 | 10 | if [[ ${cur} == * ]] ; then 11 | COMPREPLY=( $(compgen -W "${opts}" ${cur}) ) 12 | return 0 13 | fi 14 | } 15 | complete -F _cloudfuzzer cloudfuzzer 16 | -------------------------------------------------------------------------------- /bastion/autocomplete.bash.inc: -------------------------------------------------------------------------------- 1 | _cloudfuzzer() 2 | { 3 | local cur prev opts 4 | COMPREPLY=() 5 | cur="${COMP_WORDS[COMP_CWORD]}" 6 | prev="${COMP_WORDS[COMP_CWORD-1]}" 7 | opts="setup-swarm run-containers distribute-docker-image get-results get-stats ssh-to-master help" 8 | 9 | 10 | if [[ ${cur} == * ]] ; then 11 | COMPREPLY=( $(compgen -W "${opts}" ${cur}) ) 12 | return 0 13 | fi 14 | } 15 | complete -F _cloudfuzzer cloudfuzzer 16 | -------------------------------------------------------------------------------- /bastion/distribute-docker-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | # 7 | #Sends docker image from bastion to swarm nodes. Removes image after sending. 8 | # 9 | # 10 | 11 | #Usage: ./distribute-docker-image.sh 12 | 13 | 14 | IMAGE="$1" 15 | 16 | 17 | for node in $(cat $HOME/address_nodes) 18 | do 19 | echo "FuzzVM: $node" 20 | cat $IMAGE | ssh $node "gunzip | docker load;"; 21 | done 22 | 23 | echo "Removing image from bastion." 24 | rm $IMAGE; -------------------------------------------------------------------------------- /fuzzvm/run-containers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | COUNT=$1 4 | 5 | if [ -z $COUNT ]; then 6 | COUNT=1; 7 | fi 8 | 9 | DOCKER_COMMAND="docker service create --name fuzz-service --replicas $COUNT \ 10 | --mount type=volume,source=rsync-volume-container,destination=/output \ 11 | --restart-condition none $(cat $HOME/docker-arguments);" 12 | 13 | echo "Starting fuzz-service:" 14 | echo "Instances: $COUNT" 15 | echo "Full command:" 16 | echo "$DOCKER_COMMAND" 17 | 18 | bash -c "$DOCKER_COMMAND"; 19 | 20 | 21 | -------------------------------------------------------------------------------- /packer/provisioner-scripts/bastion-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | ## 7 | ##Bastion install script. 8 | ## 9 | sudo apt-get update; 10 | sudo apt-get upgrade -y; 11 | 12 | mkdir -p $HOME/.ssh; 13 | mv /tmp/bastion-key $HOME/.ssh/id_rsa 14 | chmod 600 $HOME/.ssh/id_rsa 15 | cat /tmp/bastion-key.pub >> $HOME/.ssh/authorized_keys 16 | cat /tmp/fuzzvm-key.pub >> $HOME/.ssh/authorized_keys 17 | 18 | mv /tmp/scripts $HOME/ 19 | 20 | sudo passwd ubuntu -l 21 | sudo sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config 22 | 23 | echo 'source $HOME/scripts/helpers.sh' >> $HOME/.bashrc 24 | -------------------------------------------------------------------------------- /fuzzvm/get-stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | # 7 | #Collect various logs from our docker swarm 8 | # 9 | 10 | if [ -d $HOME/stats ]; then 11 | rm -rf $HOME/stats 12 | fi 13 | 14 | mkdir $HOME/stats 15 | 16 | COMMANDS=( 17 | "docker service list" 18 | "docker service inspect fuzz-service --pretty" 19 | "docker service ps fuzz-service" 20 | ) 21 | 22 | for COMMAND in "${COMMANDS[@]}"; do 23 | echo $COMMAND | tee -a $HOME/stats/cloudfuzzer-stats.log 24 | timeout 10s $COMMAND | tee -a $HOME/stats/cloudfuzzer-stats.log 25 | echo "" | tee -a $HOME/stats/cloudfuzzer-stats.log 26 | done 27 | 28 | timeout 10s docker service logs fuzz-service | gzip > $HOME/stats/fuzz-service-log.gz -------------------------------------------------------------------------------- /doc/work-flow.md: -------------------------------------------------------------------------------- 1 | # Cloudfuzzer work flow 2 | 3 | ## Pre work 4 | * packer build images on the cloud 5 | 6 | ## Work flow 7 | * docker build <image> 8 | * create-N-fuzzvm <user-defined> 9 | * create-1-bastion <user-defined> 10 | * ssh bastion "scripts/setup-swarm <nodes>" 11 | * docker save <img> | ssh bastion "scripts/distribute-docker-image.sh" 12 | * ssh bastion "scripts/health.sh" 13 | * ssh bastion "scripts/fuzz.sh" 14 | * sleep 10000 15 | * get-results.sh <user-defined> 16 | * ssh bastion "scripts/stop.sh" 17 | * exit 18 | 19 | ## Result sync 20 | * rsync 21 | * no-overwrite 22 | * no delete sync 23 | ## fuzzvm 24 | * sync on terminate 25 | * sync time interval 26 | * clean local 27 | * sync on bastion command 28 | -------------------------------------------------------------------------------- /scripts/create-keys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname "$(readlink -f "$0")"); 4 | 5 | 6 | #Create vm-keys folder, if it doesn't exist. 7 | if [ ! -d ../vm-keys ]; then 8 | mkdir -p ../vm-keys; 9 | fi 10 | cd ../vm-keys; 11 | 12 | #Create keys for bastion 13 | if [ ! -f ./bastion-key ]; then 14 | echo "No keys for bastion..." 15 | echo "Generating: $(pwd)/bastion-key and $(pwd)/bastion-key.pub" 16 | ssh-keygen -N '' -t rsa -b 4096 -C "bastion" -f bastion-key 17 | else 18 | echo "Keys for bastion already exist..." 19 | fi 20 | 21 | #Create keys for fuzzvm 22 | if [ ! -f ./fuzzvm-key ]; then 23 | echo "No keys for fuzzvm..." 24 | echo "Generating: $(pwd)/fuzzvm-key and $(pwd)/fuzzvm-key.pub" 25 | ssh-keygen -N '' -t rsa -b 4096 -C "fuzzvm" -f fuzzvm-key 26 | else 27 | echo "Keys for fuzzvm already exist..." 28 | fi 29 | -------------------------------------------------------------------------------- /tests/volume_container_rsync/test-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | echo "Starting test..."; 5 | 6 | #We just write dummy-files to the samples and results folders. 7 | mkdir -p /output/samples; 8 | mkdir -p /output/results; 9 | 10 | count=0; 11 | SCID=$(cat /etc/hostname); 12 | while true; do 13 | let count++; 14 | echo "Adding sample: sample-$SCID-$count" 15 | echo "$count" > /output/samples/sample-$SCID-$count; 16 | 17 | if ! ((count % 2)); then 18 | echo "Adding result: crash-$SCID-$count" 19 | echo "$count" > /output/results/crash-$SCID-$count; 20 | fi 21 | 22 | sleep 0.2; 23 | if [ $count -gt 10 ]; then 24 | break; 25 | fi 26 | done 27 | 28 | #Store file list as seen inside the container 29 | touch /output/file-list-test-container; 30 | cd /; 31 | ls output/* | sort > /output/file-list-test-container; -------------------------------------------------------------------------------- /scripts/send-docker-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | source $CLOUDFUZZER_DIR/scripts/functions.bash.inc 7 | 8 | #Usage: bash send-docker-data.sh 9 | #Example: bash send-docker-data.sh ./fuzz-data 10 | 11 | #Requirements: 12 | # docker-data-directory must have the following structure: 13 | # docker-data-directory: 14 | # docker-image 15 | # docker-arguments 16 | 17 | #docker-image: 18 | # docker image saved with: docker save | gzip > docker-image 19 | 20 | #docker-arguments: 21 | # docker run options: docker run 22 | # 23 | 24 | if [ -z "$1" ]; then 25 | echo "Missing required argument." 26 | echo "You must specify the docker-data-directory." 27 | return 1 28 | fi 29 | 30 | DIRECTORY="$1"; 31 | 32 | if [ ! -f "$DIRECTORY/docker-arguments" ]; then 33 | echo "docker-arguments: No such file." 34 | echo "docker-arguments file should contain arguments that are given to docker run" 35 | echo "docker run " 36 | return 1 37 | elif [ ! -f "$DIRECTORY/docker-image" ]; then 38 | echo "docker-image: No such file." 39 | echo "To save the image, you can use: docker save | gzip > docker-image" 40 | return 1 41 | fi 42 | 43 | scp -o StrictHostKeyChecking=no -o User=$BASTION_USER -Cr $DIRECTORY/* $BASTION_ADDRESS: 44 | ssh -o StrictHostKeyChecking=no -o User=$BASTION_USER $BASTION_ADDRESS 'scripts/helpers.sh distribute-docker-image ./docker-image' 45 | -------------------------------------------------------------------------------- /bastion/setup-swarm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | ## 6 | ##Swarm setup script 7 | ## 8 | 9 | #usage: setup-swarm.sh ... 10 | 11 | #example: ssh bastion "./setup-swarm.sh $(./get-gcloud-fuzzvm-ips.sh)" 12 | 13 | #First checks connection to each FuzzVM and that ssh works without passwd. 14 | #Selects first FuzzVM as swarm manager and triggers swarm-create on manager. 15 | 16 | ADDRESSES="$@" 17 | 18 | OWN_ADDRESS=$(hostname -I | awk '{print $1}') 19 | 20 | echo "Bastion: $OWN_ADDRESS" 21 | echo "FuzzVMs: $ADDRESSES" 22 | 23 | echo "Checking connection to all FuzzVMs." 24 | 25 | for fuzzvm in $ADDRESSES; do 26 | echo "Checking $fuzzvm" 27 | ssh $fuzzvm -o StrictHostKeyChecking=no -o ConnectTimeout=10 'echo -n $HOSTNAME\($(hostname -I | awk '\''{print $1}'\'')\);' && echo " - Online" 28 | if [ $? -ne 0 ]; then 29 | echo "Couldn't reach host: $fuzzvm" 30 | echo "Before rerunning this script:" 31 | echo "Check that you provided valid address for FuzzVM instance, in a form that can be used in ssh
." 32 | echo "Also check that you are using correct username, that has the ssh-keys set." 33 | return 1; 34 | fi 35 | done 36 | 37 | SWARM_MASTER=$1; 38 | shift; 39 | SWARM_NODES=$@; 40 | 41 | echo $SWARM_MASTER > $HOME/address_master; 42 | echo $SWARM_MASTER $SWARM_NODES > $HOME/address_nodes; 43 | echo $OWN_ADDRESS > $HOME/address_bastion 44 | 45 | scp $HOME/address_* $SWARM_MASTER: 46 | 47 | echo "Selected $SWARM_MASTER as swarm master" 48 | echo "Starting swarm-create.sh on swarm master" 49 | ssh $SWARM_MASTER "scripts/swarm-create.sh"; 50 | -------------------------------------------------------------------------------- /tests/gce-tests/gce-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if (( $# == 2 )); then 4 | source ../../scripts/functions.bash.inc 5 | 6 | set -o errexit 7 | set -o nounset 8 | 9 | PACKERENV=$1 10 | 11 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 12 | 13 | cd $CLOUDFUZZER_DIR/packer/ 14 | 15 | packer build -only=gcloud -force -var-file=$PACKERENV $CLOUDFUZZER_DIR/packer/packer-bastion.json 16 | packer build -only=gcloud -force -var-file=$PACKERENV $CLOUDFUZZER_DIR/packer/packer-fuzzvm.json 17 | 18 | cd $DIR 19 | 20 | gcloud compute instances create fuzzvm-1 fuzzvm-2 fuzzvm-3 fuzzvm-4 fuzzvm-5 \ 21 | --zone europe-west1-d --image=cloudfuzzer-fuzzvm --no-address \ 22 | --metadata-from-file shutdown-script=$CLOUDFUZZER_DIR/fuzzvm/sync-results.sh 23 | 24 | gcloud compute instances create bastion \ 25 | --zone europe-west1-d --image=cloudfuzzer-bastion 26 | 27 | 28 | echo "Waiting for the machines to spin up..." 29 | sleep 30; 30 | 31 | FUZZVM_ADDRESSES=`scripts/get-gce-ip-address.sh fuzzvm` 32 | export BASTION_ADDRESS=`scripts/get-gce-ip-address.sh bastion` 33 | 34 | cloudfuzzer bastion setup-swarm $FUZZVM_ADDRESSES 35 | 36 | #Test context in ./context contains docker-image and docker-options files. 37 | cloudfuzzer send-docker-data $2; 38 | 39 | cloudfuzzer bastion run-containers 7; 40 | 41 | rm -rf ./test-results/; 42 | mkdir -p ./test-results/; 43 | 44 | sleep 10; 45 | 46 | cloudfuzzer get-stats ./test-results; 47 | 48 | cloudfuzzer get-results ./test-results; 49 | else 50 | echo "Illegal number of arguments given." 51 | echo "Usage: ./gce-setup " 52 | fi 53 | -------------------------------------------------------------------------------- /fuzzvm/swarm-create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Script for swarm master setup 4 | 5 | #TODO: Investigate if there can be timing issues. 6 | 7 | BASTION_ADDRESS=$(cat $HOME/address_bastion); 8 | MASTER_ADDRESS=$(cat $HOME/address_master); 9 | NODE_ADDRESSES=$(cat $HOME/address_nodes); 10 | 11 | 12 | docker swarm init --advertise-addr $MASTER_ADDRESS 1> /dev/null; 13 | 14 | WORKER_TOKEN=$(docker swarm join-token worker -q) 15 | 16 | echo "Setting up nodes." 17 | for node in $NODE_ADDRESSES; do 18 | if [ $node == $MASTER_ADDRESS ]; then 19 | continue 20 | fi 21 | #Connect each swarm node and run swarm join script. 22 | scp -o StrictHostKeyChecking=no $HOME/address_* $node:; 23 | ssh -o StrictHostKeyChecking=no $node "docker swarm join --token $WORKER_TOKEN $MASTER_ADDRESS:2377"; 24 | done 25 | 26 | echo "Starting rsync-volume-containers." 27 | docker service create --mode global --publish mode=host,target=873,published=10873 --name rsync-volume-container \ 28 | --mount type=volume,source=rsync-volume-container,destination=/output \ 29 | -e VOLUME=/output -e ALLOW="$BASTION_ADDRESS" nabeken/docker-volume-container-rsync; 30 | 31 | echo "Swarm setup completed. Waiting for rsync-volume-container to be started on all nodes." 32 | sleep 10; 33 | 34 | NODE_COUNT=$(echo $NODE_ADDRESSES | wc -w) 35 | 36 | WAITS=0; 37 | while (( $NODE_COUNT != $(docker service ps rsync-volume-container | grep 'Running\s\+Running' | wc -l) )); do 38 | echo "Nodes are not ready yet..." 39 | let WAITS++; 40 | if (( $WAITS == 12 )); then 41 | echo "For some reason rsync-volume-containers haven't come up yet. Aborting." 42 | exit; 43 | fi 44 | echo "Waiting another 10s." 45 | sleep 10; 46 | done 47 | 48 | echo "All nodes are ready." 49 | -------------------------------------------------------------------------------- /doc/draft1.txt: -------------------------------------------------------------------------------- 1 | 2 | (Some notes...) 3 | 4 | Images: 5 | FuzzImage: 6 | Works as a golden image for docker swarm machines 7 | Contains all required components to run as a docker swarm-master, or as a swarm-node 8 | In initialization N swarm-machine instances are created from FuzzImage, one of them is selected as a swarm-master, by Bastion 9 | 10 | Can be used in Google Compute Preemptible instances and Amazon Spot Instances. 11 | 12 | swarm-master: 13 | Uses docker-machine to set up docker-swarm, including all swarm-machine instances 14 | Runs docker swarm discovery service 15 | Distributes fuzzing jobs, once recieved from Bastion 16 | 17 | swarm-node: 18 | Runs fuzzing docker container(s) 19 | Syncs results with Bastion 20 | *Time interval 21 | *Preemptible and SpotInstance shutdown detection 22 | 23 | 24 | Bastion: 25 | Works as a SSH gateway between outside world and fuzzing cluster 26 | Delivers docker image from user to swarm-machines 27 | Stores results 28 | 29 | 30 | FuzzImage: 31 | 32 | OS: Ubuntu 16.04 33 | 34 | Contains: 35 | docker.io 36 | docker-machine 37 | swarm 38 | swarm discovery service (runs only on swarm-master) 39 | scripts for results syncing 40 | ssh 41 | rsync 42 | 43 | Bastion: 44 | OS: Ubuntu 16.04 45 | 46 | Contains: 47 | ssh 48 | scripts for swarm-master control 49 | scripts for docker images distribution 50 | 51 | Note: User should be able to run whole, or at least partial, implementation on local machine, without too large overhead. 52 | -------------------------------------------------------------------------------- /packer/provisioner-scripts/fuzzvm-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | ## 7 | ##FuzzVM install script. 8 | ## 9 | ##Note: This script is for Ubuntu 16.04 10 | 11 | sudo apt-get update; 12 | sudo apt-get upgrade -y; 13 | 14 | #Disable apport 15 | sudo systemctl disable apport.service 16 | 17 | #For afl support change fuzzvm core_pattern, 18 | #the default pattern triggers apport, which we disable by default. 19 | echo "core" | sudo tee /proc/sys/kernel/core_pattern 20 | 21 | #Add bastion and fuzzvm keys to authorized keys 22 | mkdir -p $HOME/.ssh; 23 | mv /tmp/fuzzvm-key $HOME/.ssh/id_rsa 24 | chmod 600 $HOME/.ssh/id_rsa 25 | cat /tmp/bastion-key.pub >> $HOME/.ssh/authorized_keys 26 | cat /tmp/fuzzvm-key.pub >> $HOME/.ssh/authorized_keys 27 | 28 | #CloudFuzzer uses docker functionality that requires newer docker than what is available at Ubuntu apt repos. 29 | wget https://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_17.03.0~ce-0~ubuntu-xenial_amd64.deb && \ 30 | sudo dpkg --install docker-engine_17.03.0~ce-0~ubuntu-xenial_amd64.deb && rm docker-engine_17.03.0~ce-0~ubuntu-xenial_amd64.deb 31 | sudo apt-get -f install -y 32 | 33 | #Add user to docker group 34 | sudo groupadd -f docker 35 | sudo gpasswd -a ${USER} docker 36 | 37 | #In docker-engine 17.03, some of the docker service functionality is only available 38 | #in experimental daemon mode, so we need to enable it until those features 39 | #land stable. 40 | sudo sh -c 'echo "{\"experimental\":true}" > /etc/docker/daemon.json' 41 | 42 | sudo systemctl restart docker; 43 | 44 | sudo docker pull nabeken/docker-volume-container-rsync; 45 | 46 | sudo systemctl stop docker; 47 | 48 | #Remove docker key.json. This forces docker to regenerate 49 | #ID when the fuzzvm is started, otherwise multiple daemons 50 | #with same ID will cause conflict in docker swarm. 51 | sudo rm /etc/docker/key.json 52 | 53 | #Add fuzzvm scripts 54 | mv /tmp/scripts $HOME/ 55 | chmod +x $HOME/scripts/* 56 | 57 | #Remove passwd and disable ssh passwd login. 58 | sudo passwd ubuntu -l 59 | sudo sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config 60 | -------------------------------------------------------------------------------- /tests/volume_container_rsync/run-test-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | #This script is for local test of result sync with docker volume container rsync 5 | # 6 | 7 | set -o errexit 8 | set -o nounset 9 | 10 | CLEANUP=${CLEANUP:=true}; 11 | DOCKER0_IP=$(ip addr list docker0 | grep "inet" | cut -d" " -f6 | head -1) 12 | 13 | #Start volume-container 14 | VOLUME_CID=$(docker run -d -p 10873:873 -e ALLOW="$DOCKER0_IP" --volume /output -e VOLUME=/output nabeken/docker-volume-container-rsync); 15 | #Create results and samples directories 16 | docker exec $VOLUME_CID bash -c "mkdir /output/results; mkdir /output/samples;"; 17 | 18 | #Build sample_sync_test docker container 19 | docker build -t sample_sync_test ./; 20 | 21 | #Create output folder for results and tmp 22 | [ ! -d ./output/ ] && mkdir ./output/; 23 | 24 | #Start sample_sync_test 25 | docker run --cidfile="./output/TEST_CID" -it --volumes-from $VOLUME_CID sample_sync_test; 26 | docker run --cidfile="./output/TEST_CID2" -it --volumes-from $VOLUME_CID sample_sync_test; 27 | 28 | #Store test container CIDs 29 | TEST_CID=$(cat ./output/TEST_CID); 30 | TEST_CID2=$(cat ./output/TEST_CID2); 31 | 32 | #Remove temp files 33 | rm ./output/TEST_CID; 34 | rm ./output/TEST_CID2; 35 | 36 | 37 | #rsync results and samples 38 | rsync -r rsync://127.0.0.1:10873/volume ./output; 39 | 40 | #Get file lists from local and volume-container 41 | ls output/* | sort > ./output/file-list-local; 42 | docker exec $VOLUME_CID bash -c "cd /; ls output/* | sort" > ./output/file-list-volume-container; 43 | 44 | #diff rsync'd files to file list from volume-container, and then against file list that test containers wrote 45 | diff ./output/file-list-local ./output/file-list-volume-container && diff ./output/file-list-local ./output/file-list-test-container; 46 | 47 | if [ $? -eq 0 ]; then 48 | echo "Got:" 49 | ls output/* 50 | echo "File lists match." 51 | else 52 | echo "Something went wrong. See the diff above." 53 | fi 54 | 55 | 56 | if [ $CLEANUP = true ]; then 57 | docker rm -f $TEST_CID; 58 | docker rm -f $TEST_CID2; 59 | docker rm -f $VOLUME_CID; 60 | docker rmi sample_sync_test; 61 | docker rmi nabeken/docker-volume-container-rsync; 62 | docker volume prune -f; 63 | rm -rf ./output; 64 | fi -------------------------------------------------------------------------------- /packer/ubuntu-16.04.preseed.cfg: -------------------------------------------------------------------------------- 1 | # A close copy of: https://github.com/geerlingguy/packer-ubuntu-1604/blob/0bcad82234c38b6de792b76660e8cfd14826973b/http/preseed.cfg 2 | 3 | choose-mirror-bin mirror/http/proxy string 4 | d-i base-installer/kernel/override-image string linux-server 5 | d-i clock-setup/utc boolean true 6 | d-i clock-setup/utc-auto boolean true 7 | d-i finish-install/reboot_in_progress note 8 | d-i grub-installer/only_debian boolean true 9 | d-i grub-installer/with_other_os boolean true 10 | d-i partman-auto-lvm/guided_size string max 11 | d-i partman-auto/choose_recipe select atomic 12 | d-i partman-auto/method string lvm 13 | d-i partman-lvm/confirm boolean true 14 | d-i partman-lvm/confirm boolean true 15 | d-i partman-lvm/confirm_nooverwrite boolean true 16 | d-i partman-lvm/device_remove_lvm boolean true 17 | d-i partman/choose_partition select finish 18 | d-i partman/confirm boolean true 19 | d-i partman/confirm_nooverwrite boolean true 20 | d-i partman/confirm_write_new_label boolean true 21 | d-i pkgsel/include string openssh-server cryptsetup build-essential libssl-dev libreadline-dev zlib1g-dev linux-source dkms nfs-common 22 | d-i pkgsel/install-language-support boolean false 23 | d-i pkgsel/update-policy select none 24 | d-i pkgsel/upgrade select full-upgrade 25 | d-i time/zone string UTC 26 | tasksel tasksel/first multiselect standard, ubuntu-server 27 | 28 | d-i console-setup/ask_detect boolean false 29 | d-i keyboard-configuration/layoutcode string us 30 | d-i keyboard-configuration/modelcode string pc105 31 | d-i debian-installer/locale string en_US 32 | 33 | # Create ubunt user 34 | d-i passwd/user-fullname string ubuntu 35 | d-i passwd/username string ubuntu 36 | d-i passwd/user-password password ubuntu 37 | d-i passwd/user-password-again password ubuntu 38 | d-i user-setup/allow-password-weak boolean true 39 | d-i user-setup/encrypt-home boolean false 40 | d-i passwd/user-default-groups ubuntu sudo 41 | d-i passwd/user-uid string 900 42 | 43 | # The systemd networking.service expects old-style link naming (eth0, eth1, ...). 44 | # Also passwordless sudo is expected by some install scripts. 45 | d-i preseed/late_command string \ 46 | ln -sf /dev/null /target/etc/systemd/network/99-default.link; \ 47 | echo 'Defaults:ubuntu !requiretty' > /target/etc/sudoers.d/ubuntu; \ 48 | echo 'ubuntu ALL=(ALL) NOPASSWD: ALL' >> /target/etc/sudoers.d/ubuntu; \ 49 | chmod 440 /target/etc/sudoers.d/ubuntu 50 | 51 | -------------------------------------------------------------------------------- /bastion/helpers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="${BASH_SOURCE%/*}" 4 | if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi 5 | 6 | cloudfuzzer () { 7 | case $1 in 8 | "setup-swarm") 9 | "$DIR/setup-swarm.sh" ${@:2} 10 | ;; 11 | "run-containers") 12 | "$DIR/run-containers.sh" $2 13 | ;; 14 | "distribute-docker-image") 15 | "$DIR/distribute-docker-image.sh" $2 16 | ;; 17 | "get-results") 18 | "$DIR/get-results.sh" ${@:2} 19 | ;; 20 | "get-stats") 21 | "$DIR/get-stats.sh" 22 | ;; 23 | "ssh-to-master") 24 | if [ -z "$PS1" ]; then 25 | echo "This command can only be used with interative shell" 26 | else 27 | ssh $(cat $HOME/address_master) 28 | fi 29 | ;; 30 | "help" | "") 31 | print_help $2 32 | ;; 33 | *) 34 | echo "Unknown argument." 35 | ;; 36 | esac 37 | } 38 | 39 | function print_help () { 40 | case $1 in 41 | "setup-swarm") 42 | echo "First checks connection to each FuzzVM and that ssh works without passwd." 43 | echo "Selects first FuzzVM as swarm manager and triggers swarm-create on manager." 44 | echo "Usage cloudfuzzer setup-swarm ..." 45 | ;; 46 | "run-containers") 47 | echo "Run containers. Takes number of containers to be run as an argument." 48 | ;; 49 | "distribute-docker-image") 50 | echo "Sends docker image from bastion to swarm nodes. Removes image after sending." 51 | echo "Usage: cloudfuzzer distribute-local-docker-image.sh " 52 | ;; 53 | "get-results") 54 | echo "Get results from fuzzvm's. When no arguments are given results from all fuzzvm's are fetched." 55 | echo "Otherwise it gets results from ip addresses of fuzzvm's given as arguments" 56 | ;; 57 | "get-stats") 58 | echo "Get stats from master fuzzvm" 59 | ;; 60 | "ssh-to-master") 61 | echo "ssh to fuzzvm swarm master" 62 | ;; 63 | *) 64 | echo "Available commands:" 65 | echo " get-results (fuzzvm1) (fuzzvm2) ..." 66 | echo " get-stats" 67 | echo " distribute-docker-image" 68 | echo " run-containers " 69 | echo " setup-swarm ..." 70 | if [ ! -z "$PS1" ]; then 71 | echo " ssh-to-master" 72 | fi 73 | ;; 74 | esac 75 | } 76 | 77 | case $- in 78 | *i*) ;; 79 | *) 80 | cloudfuzzer $@ 81 | 82 | ;; 83 | esac 84 | -------------------------------------------------------------------------------- /fuzzvm/functions.bash.inc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | NODE_ADDRESSES=$(cat $HOME/address_nodes); 4 | BASTION_ADDRESS=$(cat $HOME/address_bastion); 5 | MASTER_ADDRESS=$(cat $HOME/address_master); 6 | 7 | DIR="${BASH_SOURCE%/*}" 8 | if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi 9 | 10 | cloudfuzzer () { 11 | case $1 in 12 | "check-internet") 13 | "$DIR/check-internet.sh" 14 | ;; 15 | "collect-results-from-remote") 16 | "$DIR/collect-results-from-remote.sh" 17 | ;; 18 | "collect-results") 19 | "$DIR/collect-results.sh" 20 | ;; 21 | "collect-samples") 22 | "$DIR/collect-samples.sh" 23 | ;; 24 | "get-samples-from-container") 25 | "$DIR/get-samples-from-container.sh" $2 26 | ;; 27 | "remove-containers") 28 | "$DIR/remove-containers.sh" 29 | ;; 30 | "run-containers") 31 | "$DIR/run-containers.sh" $2 32 | ;; 33 | "setup-node") 34 | "$DIR/setup-node.sh" $2 $3 35 | ;; 36 | "stop-containers") 37 | "$DIR/stop-containers.sh" 38 | ;; 39 | "swarm-create") 40 | "$DIR/swarm-create.sh" ${@:2} 41 | ;; 42 | "help") 43 | print_help $2 44 | ;; 45 | *) 46 | echo "Unknown argument." 47 | ;; 48 | esac 49 | } 50 | 51 | function print_help () { 52 | case $1 in 53 | "check-internet") 54 | echo "Run to check if the node is connected to Internet. There should be no Internet connection in fuzzing nodes (fuzzvm)." 55 | ;; 56 | "collect-results-from-remote") 57 | echo "Collects results to master. Used for debugging." 58 | ;; 59 | "collect-results") 60 | echo "Collects results to bastion." 61 | ;; 62 | "collect-samples") 63 | echo "Collects results to bastion." 64 | ;; 65 | "get-samples-from-container") 66 | echo "Collect diff of samples from container." 67 | ;; 68 | "remove-containers") 69 | echo "Remove the containers." 70 | ;; 71 | "run-containers") 72 | echo "Run the containers." 73 | ;; 74 | "setup-node") 75 | echo "help setup-node" 76 | ;; 77 | "stop-containers") 78 | echo "Stop running containers." 79 | ;; 80 | "swarm-create") 81 | echo "Script for swarm master setup" 82 | ;; 83 | *) 84 | echo "Available commands:" 85 | echo "check-internet" 86 | echo "collect-results-from-remote" 87 | echo "collect-results" 88 | echo "collect-samples" 89 | echo "get-samples-from-container" 90 | echo "remove-containers" 91 | echo "run-containers" 92 | echo "setup-node" 93 | echo "stop-containers" 94 | echo "swarm-create" 95 | echo "help " 96 | ;; 97 | esac 98 | } 99 | -------------------------------------------------------------------------------- /packer/packer-fuzzvm.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "account_file": "", 4 | "project_id": "", 5 | "aws_access_key": "", 6 | "aws_secret_key": "" 7 | }, 8 | "builders": [{ 9 | "name": "gcloud", 10 | "type": "googlecompute", 11 | "account_file": "{{user `account_file`}}", 12 | "project_id": "{{user `project_id`}}", 13 | "source_image": "ubuntu-1604-xenial-v20161020", 14 | "zone": "europe-west1-d", 15 | "disk_size": "10", 16 | "disk_type": "pd-standard", 17 | "image_description": "cloudfuzzer fuzzing image", 18 | "image_family": "cloudfuzzer-fuzzvm", 19 | "image_name": "cloudfuzzer-fuzzvm", 20 | "machine_type": "n1-standard-1", 21 | "ssh_username": "ubuntu" 22 | }, 23 | { 24 | "name": "aws", 25 | "type": "amazon-ebs", 26 | "access_key": "{{user `aws_access_key`}}", 27 | "secret_key": "{{user `aws_secret_key`}}", 28 | "region": "eu-central-1", 29 | "instance_type": "t2.micro", 30 | "source_ami": "ami-b87881d7", 31 | "ami_name": "cloudfuzzer-fuzzvm", 32 | "ssh_username": "ubuntu" 33 | }, 34 | { 35 | "name": "virtualbox", 36 | "type": "virtualbox-iso", 37 | "guest_os_type": "Ubuntu_64", 38 | "iso_url": "http://releases.ubuntu.com/16.04/ubuntu-16.04.2-server-amd64.iso", 39 | "iso_checksum": "737ae7041212c628de5751d15c3016058b0e833fdc32e7420209b76ca3d0a535", 40 | "iso_checksum_type": "sha256", 41 | "headless": "false", 42 | "output_directory": "fuzzvm-img", 43 | "vm_name": "fuzzvm", 44 | "disk_size": "4000", 45 | "ssh_timeout": "30m", 46 | "ssh_username": "ubuntu", 47 | "ssh_password": "ubuntu", 48 | "shutdown_command": "sudo shutdown -P now", 49 | "http_directory": ".", 50 | "boot_command": [ 51 | "", 52 | "", 53 | "", 54 | "", 55 | "", 56 | "", 57 | "", 58 | "", 59 | "", 60 | "", 61 | "", 62 | "/install/vmlinuz ", 63 | "initrd=/install/initrd.gz ", 64 | "net.ifnames=0 ", 65 | "auto-install/enable=true ", 66 | "debconf/priority=critical ", 67 | "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu-16.04.preseed.cfg ", 68 | ""] 69 | }], 70 | "provisioners": [ 71 | { 72 | "type": "file", 73 | "source": "../vm-keys/fuzzvm-key", 74 | "destination": "/tmp/fuzzvm-key" 75 | }, 76 | { 77 | "type": "file", 78 | "source": "../fuzzvm", 79 | "destination": "/tmp/scripts" 80 | }, 81 | { 82 | "type": "file", 83 | "source": "../vm-keys/bastion-key.pub", 84 | "destination": "/tmp/bastion-key.pub" 85 | }, 86 | { 87 | "type": "file", 88 | "source": "../vm-keys/fuzzvm-key.pub", 89 | "destination": "/tmp/fuzzvm-key.pub" 90 | }, 91 | { 92 | "type": "shell", 93 | "script": "provisioner-scripts/fuzzvm-setup.sh" 94 | }] 95 | } 96 | -------------------------------------------------------------------------------- /packer/packer-bastion.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "account_file": "", 4 | "project_id": "", 5 | "aws_access_key": "", 6 | "aws_secret_key": "" 7 | }, 8 | "builders": [{ 9 | "name": "gcloud", 10 | "type": "googlecompute", 11 | "account_file": "{{user `account_file`}}", 12 | "project_id": "{{user `project_id`}}", 13 | "source_image": "ubuntu-1604-xenial-v20161020", 14 | "zone": "europe-west1-d", 15 | "disk_size": "10", 16 | "disk_type": "pd-standard", 17 | "image_description": "cloudfuzzer bastion image", 18 | "image_family": "cloudfuzzer-bastion", 19 | "image_name": "cloudfuzzer-bastion", 20 | "machine_type": "n1-standard-1", 21 | "ssh_username": "ubuntu" 22 | }, 23 | { 24 | "name": "aws", 25 | "type": "amazon-ebs", 26 | "access_key": "{{user `aws_access_key`}}", 27 | "secret_key": "{{user `aws_secret_key`}}", 28 | "region": "eu-central-1", 29 | "instance_type": "t2.micro", 30 | "source_ami": "ami-b87881d7", 31 | "ami_name": "cloudfuzzer-bastion", 32 | "ssh_username": "ubuntu" 33 | }, 34 | { 35 | "name": "virtualbox", 36 | "type": "virtualbox-iso", 37 | "guest_os_type": "Ubuntu_64", 38 | "iso_url": "http://releases.ubuntu.com/16.04/ubuntu-16.04.2-server-amd64.iso", 39 | "iso_checksum": "737ae7041212c628de5751d15c3016058b0e833fdc32e7420209b76ca3d0a535", 40 | "iso_checksum_type": "sha256", 41 | "headless": "false", 42 | "output_directory": "bastion-img", 43 | "vm_name": "bastion", 44 | "disk_size": "4000", 45 | "ssh_timeout": "30m", 46 | "ssh_username": "ubuntu", 47 | "ssh_password": "ubuntu", 48 | "shutdown_command": "sudo shutdown -P now", 49 | "http_directory": ".", 50 | "boot_command": [ 51 | "", 52 | "", 53 | "", 54 | "", 55 | "", 56 | "", 57 | "", 58 | "", 59 | "", 60 | "", 61 | "", 62 | "/install/vmlinuz ", 63 | "initrd=/install/initrd.gz ", 64 | "net.ifnames=0 ", 65 | "auto-install/enable=true ", 66 | "debconf/priority=critical ", 67 | "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu-16.04.preseed.cfg ", 68 | ""] 69 | }], 70 | "provisioners": [ 71 | { 72 | "type": "file", 73 | "source": "../vm-keys/bastion-key", 74 | "destination": "/tmp/bastion-key" 75 | }, 76 | { 77 | "type": "file", 78 | "source": "../bastion", 79 | "destination": "/tmp/scripts" 80 | }, 81 | { 82 | "type": "file", 83 | "source": "../vm-keys/bastion-key.pub", 84 | "destination": "/tmp/bastion-key.pub" 85 | }, 86 | { 87 | "type": "file", 88 | "source": "../vm-keys/fuzzvm-key.pub", 89 | "destination": "/tmp/fuzzvm-key.pub" 90 | }, 91 | { 92 | "type": "shell", 93 | "script": "provisioner-scripts/bastion-setup.sh" 94 | }] 95 | 96 | 97 | } 98 | -------------------------------------------------------------------------------- /scripts/functions.bash.inc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CLOUDFUZZER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" 4 | 5 | CLOUDFUZZER_CONF="$CLOUDFUZZER_DIR/cloudfuzzer.conf" 6 | USER_CONF="$CLOUDFUZZER_DIR/user.conf" 7 | 8 | if [ -f $CLOUDFUZZER_CONF ] 9 | then 10 | source $CLOUDFUZZER_CONF 11 | else 12 | echo "ERROR: $CLOUDFUZZER_CONF not found. Aborted." 13 | return 1 14 | fi 15 | 16 | if [ -f $USER_CONF ] 17 | then 18 | source $USER_CONF 19 | fi 20 | 21 | if [ -z $BASTION_ADDRESS ]; then 22 | echo "BASTION_ADDRESS not set." 23 | echo "You have to set it, fefore running using send-docker-data, or any cloudfuzzer bastion " 24 | echo "You can set it to $CLOUDFUZZER_CONF or $USER_CONF, and rerun source command." 25 | echo "Or later use: export BASTION_ADDRESS=" 26 | else 27 | export BASTION_ADDRESS=$BASTION_ADDRESS 28 | fi 29 | 30 | if [ -z $BASTION_USER ]; then 31 | echo "BASTION_USER not set." 32 | echo "You have to set it, fefore running using send-docker-data, or any cloudfuzzer bastion " 33 | echo "You can set it to $CLOUDFUZZER_CONF or $USER_CONF, and rerun source command." 34 | echo "Or later use: export BASTION_USER=, to set it" 35 | else 36 | export BASTION_USER=$BASTION_USER 37 | fi 38 | 39 | is_bastion_env_set (){ 40 | if [ -z $BASTION_ADDRESS ]; then 41 | echo "BASTION_ADDRESS not set." 42 | return 1 43 | elif [ -z $BASTION_USER ]; then 44 | echo "BASTION_USER not set." 45 | return 1 46 | fi 47 | return 0 48 | } 49 | 50 | cloudfuzzer () { 51 | case $1 in 52 | "create-keys") 53 | if [ "$2" == "help" ]; then 54 | echo "Create keys in vm-keys folder to be used in bastion and fuzzvm. If keys already exist no kew keys are created." 55 | else 56 | "$CLOUDFUZZER_DIR/scripts/create-keys.sh" 57 | fi 58 | ;; 59 | "send-docker-data") 60 | if [ "$2" == "help" ]; then 61 | echo "Send docker data directory to bastion." 62 | echo "Usage: cloudfuzzer send-docker-data " 63 | echo "Directory must contain two files: docker-image and docker-arguments." 64 | echo "docker-image: docker save | gzip > docker-image" 65 | echo "docker-arguments: file must contain docker run arguments: docker run " 66 | else 67 | $CLOUDFUZZER_DIR/scripts/send-docker-data.sh ${@:2} 68 | fi 69 | ;; 70 | "bastion") 71 | is_bastion_env_set || return 1; 72 | ssh $SSH_OPTS $BASTION_USER@$BASTION_ADDRESS "scripts/helpers.sh ${@:2}" 73 | ;; 74 | "ssh") 75 | if [ "$2" == "help" ]; then 76 | echo "ssh to bastion" 77 | else 78 | is_bastion_env_set || return 1; 79 | ssh $SSH_OPTS $BASTION_USER@$BASTION_ADDRESS 80 | fi 81 | ;; 82 | "get-results") 83 | if [ "$2" == "help" ]; then 84 | echo "Get results from cloudfuzzer" 85 | echo "Usage: cloudfuzzer get-results " 86 | else 87 | is_bastion_env_set || return 1; 88 | cloudfuzzer bastion get-results 89 | $CLOUDFUZZER_DIR/scripts/get-directory.sh results $2 90 | fi 91 | ;; 92 | "get-stats") 93 | if [ "$2" == "help" ]; then 94 | echo "Get status and logs from cloudfuzzer" 95 | echo "Usage: cloudfuzzer get-stats " 96 | else 97 | is_bastion_env_set || return 1; 98 | cloudfuzzer bastion get-stats 99 | $CLOUDFUZZER_DIR/scripts/get-directory.sh stats $2 100 | fi 101 | ;; 102 | "help") 103 | echo "Available commands:" 104 | echo " bastion - run on bastion." 105 | echo " create-keys - create new vm-keys" 106 | echo " get-results - get results from cloudfuzzer" 107 | echo " get-stats - get various stats and logs from cloudfuzzer fuzzvm docker" 108 | echo " help - print this" 109 | echo " send-docker-data - send docker data directory to bastion" 110 | echo " ssh - ssh to bastion" 111 | ;; 112 | *) 113 | echo " Unknown argument." 114 | ;; 115 | esac 116 | } 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloudfuzzer 2 | 3 | Cloudfuzzer is a cloud fuzzing framework. Purpose of cloudfuzzer is to make it possible to easily run automated fuzz-testing in cloud environment. 4 | 5 | In cloud environment __bastion__ instance works as a SSH gateway between outside world and fuzzing cluster. It is used to deliver docker image from user to swarm machines. Bastion is also used for storing fuzzing results. 6 | 7 | __Fuzzvm__ instances consist of __one__ __swarm-master__ and __N__ __swarm-nodes__. Swarm-master is used to set up docker-swarm, including all swarm-machine instances. It distributes fuzzing jobs, once received from Bastion. Swarm-nodes run fuzzing docker containers and sync results with bastion. Swarm-nodes can be run as Preemptible/SpotInstance instances because they have shutdown detection and they sync results before shutdown. 8 | 9 | # Getting started 10 | 11 | ## Example script for GCE 12 | 13 | Example script for setup in GCE is found in [tests/gce-tests/gce-setup.sh](tests/gce-tests/gce-setup.sh) 14 | 15 | ## config files 16 | 17 | Config file of cloudfuzzer is named cloudfuzzer.conf 18 | user.conf overrides cloudfuzzer.conf. 19 | 20 | Following variables are used: 21 | ``` 22 | BASTION_ADDRESS="xxx.xxx.xxx.xxx" # IP address of bastion 23 | SSH_OPTS="-o StrictHostKeyChecking=no -i $CLOUDFUZZER_DIR/vm-keys/bastion-key" # SSH options 24 | BASTION_USER="ubuntu" # Bastion username 25 | ``` 26 | ## cloudfuzzer 27 | 28 | Use following command to source cloudfuzzer functions 29 | ``` 30 | $ source scripts/functions.bash.inc 31 | ``` 32 | 33 | Available commands (usage: cloudfuzzer <command>): 34 | ``` 35 | bastion - run on bastion. 36 | create-keys - create new vm-keys 37 | get-results - get results from cloudfuzzer 38 | get-stats - get various stats and logs from cloudfuzzer fuzzvm docker 39 | help - print this 40 | send-docker-data - send docker data directory to bastion 41 | ssh - ssh to bastion 42 | ``` 43 | 44 | To get help for specific command type 45 | ``` 46 | $ cloudfuzzer help 47 | ``` 48 | 49 | ## ssh-keys 50 | 51 | Packer is used to provision ssh keys to the bastion and fuzzvm images. 52 | 53 | By default keys should be named bastion-key, bastion-key.pub and fuzzvm-key, fuzzvm-key.pub and should locate in folder ./vm-keys. 54 | 55 | You can use following command to create rsa 4096 keys for you. 56 | 57 | ``` 58 | $ cloudfuzzer create-keys 59 | ``` 60 | 61 | Keys are provisioned so that bastion can access all machines created from fuzzvm-image, and fuzzvm can access all other fuzzvms and bastion. 62 | (Currently there are no separate users for different operations, so you get full root access with these keys.) 63 | 64 | ## Images 65 | 66 | Packer is used for creating images to cloud environment. You must build images for bastion and fuzzvm. You find the packer files from [packer/](packer/) directory. 67 | 68 | By default, packer files for bastion and fuzzvm use use_variables for account_file and project_id. One way to use them is to make a separate json-file: 69 | ``` 70 | { 71 | "account_file": "/path/to/your/account_file.json", 72 | "project_id": "your_cloudfuzzer_project_id" 73 | } 74 | ``` 75 | 76 | After that you can build images with following commands (Google Cloud) 77 | 78 | __Bastion__ 79 | ``` 80 | $ packer build -only=gcloud -var-file=/path/to/your/variables.json packer-bastion.json 81 | ``` 82 | 83 | __Fuzzvm__ 84 | ``` 85 | $ packer build -only=gcloud -var-file=/path/to/your/variables.json packer-fuzzvm.json 86 | ``` 87 | 88 | * If you want to use aws use -only=aws 89 | * You can use -force if you want Packer to rewrite existing images in cloud platform. 90 | * Using Google Compute Engine with Packer: https://www.packer.io/docs/builders/googlecompute.html 91 | 92 | ## Instances 93 | 94 | After creating images with packer you should setup running instances in cloud environment. 95 | * 1x bastion 96 | * Nx fuzzvm 97 | 98 | Bastion should have access public ip so it can be accessed from outside network while fuzzvm should only have internal network ip. 99 | 100 | ## Setting it up 101 | 102 | ``` 103 | $ cloudfuzzer bastion setup-swarm 104 | ``` 105 | 106 | List of ip addresses of nodes should be given as argument for setup-swarm.sh 107 | 108 | ## Distributing docker image 109 | 110 | Save docker image to cloudfuzzer/context : 111 | ``` 112 | $ docker save $image | gzip > cloudfuzzer/context/docker-image 113 | ``` 114 | 115 | Docker arguments should be defined in context/docker-options 116 | Example: 117 | ``` 118 | -d -m 3g volume_container_rsync 119 | ``` 120 | 121 | Upload context: 122 | ``` 123 | $ send-docker-data cloudfuzzer/context 124 | ``` 125 | 126 | ## Run containers 127 | 128 | Run number of containers 129 | 130 | ``` 131 | $ cloudfuzzer bastion run-containers 132 | ``` 133 | 134 | ## Get results 135 | 136 | Get results from fuzzvm's. If no ip-adresses are given as argument results from all fuzzvm's are fetched. 137 | 138 | ``` 139 | $ cloudfuzzer bastion get-results (fuzzvm1) (fuzzvm2) ... 140 | ``` 141 | 142 | ## Get stats 143 | 144 | Get stats 145 | 146 | ``` 147 | $ cloudfuzzer get-stats 148 | docker service list 149 | ID NAME MODE REPLICAS IMAGE 150 | nsbg1dh6t2k1 fuzz-service replicated 70/70 p7zip-fuzz 151 | xl3l65kufl66 rsync-volume-container global 10/10 nabeken/docker-volume-container-rsync 152 | 153 | docker service inspect fuzz-service --pretty 154 | 155 | ID: nsbg1dh6t2k1biiroyf92oukf 156 | Name: fuzz-service 157 | Service Mode: Replicated 158 | Replicas: 70 159 | Placement: 160 | UpdateConfig: 161 | Parallelism: 1 162 | On failure: pause 163 | Max failure ratio: 0 164 | ContainerSpec: 165 | Image: p7zip-fuzz 166 | Mounts: 167 | Target = /output 168 | Source = rsync-volume-container 169 | ReadOnly = false 170 | Type = volume 171 | Resources: 172 | Reservations: 173 | Memory: 500 MiB 174 | Endpoint Mode: vip 175 | 176 | docker service ps fuzz-service 177 | ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS 178 | 7iadtdvc3dsc fuzz-service.1 p7zip-fuzz fuzzvm-8 Running Running 2 hours ago 179 | 81qsodiixxee fuzz-service.2 p7zip-fuzz fuzzvm-6 Running Running 2 hours ago 180 | ``` 181 | 182 | 183 | # Requirements 184 | 185 | * [Packer](https://www.packer.io/) 0.11.0 186 | * Cloud service 187 | 188 | License 189 | ---- 190 | Apache 2.0 191 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------