├── .gitignore ├── files ├── jenkins_jobs.ini ├── add_slave.yaml ├── build_jenkins_slave.yaml ├── build_jenkins_master.yaml └── jenkins-slave.conf ├── scripts ├── install-packer.sh ├── slave-install.sh ├── setup-jobs.sh ├── create-ssh-key.sh ├── jenkins-install.sh ├── swarm-service.sh └── plugins-install.sh ├── ubuntu-jenkins-ami.json ├── ubuntu-slave-ami.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.iso 2 | *.box 3 | *.pem 4 | -------------------------------------------------------------------------------- /files/jenkins_jobs.ini: -------------------------------------------------------------------------------- 1 | [jenkins] 2 | user=job-builder 3 | password=yourpasswordhere 4 | url=http://localhost:8080/ 5 | -------------------------------------------------------------------------------- /scripts/install-packer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Installing Packer 0.5.2 - unfortunately version is hardcoded right now" 4 | wget https://dl.bintray.com/mitchellh/packer/0.5.2_linux_amd64.zip 5 | sudo unzip 0.5.2_linux_amd64.zip -d /usr/local/bin/ 6 | 7 | 8 | -------------------------------------------------------------------------------- /files/add_slave.yaml: -------------------------------------------------------------------------------- 1 | - job: 2 | name: add_slave 3 | description: 'Do not edit this job through the web!' 4 | project-type: freestyle 5 | block-downstream: false 6 | builders: 7 | - shell: | 8 | ec2-run-instances 9 | echo "still need to figure how do get the AMI name back to this job" 10 | -------------------------------------------------------------------------------- /scripts/slave-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Sometimes ssh starts a bit too fast, so lets sleep for 10 seconds before starting" 4 | sleep 10 5 | 6 | 7 | sudo apt-get -y update 8 | sudo apt-get -y upgrade 9 | sudo apt-get install -y openjdk-7-jre 10 | sudo useradd -m jenkins 11 | sudo -u jenkins mkdir /home/jenkins/.ssh 12 | 13 | -------------------------------------------------------------------------------- /scripts/setup-jobs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "We're going to use Openstack's jenkins-job builder to setup our initial jobs." 4 | sudo apt-get install -y jenkins-job-builder 5 | 6 | echo "Setup the preconfigured jobs" 7 | sudo mkdir /etc/jenkins_jobs 8 | sudo mv /tmp/jenkins_jobs.ini /etc/jenkins_jobs 9 | jenkins-jobs update /tmp/build_jenkins_master.yaml 10 | jenkins-jobs update /tmp/build_jenkins_slave.yaml 11 | jenkins-jobs update /tmp/add_slave.yaml 12 | -------------------------------------------------------------------------------- /scripts/create-ssh-key.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Setting up the jenkins user .ssh directory" 4 | sudo -u jenkins mkdir /var/lib/jenkins/.ssh 5 | 6 | echo "Generating the ssh keypair and add it to agent" 7 | ssh-keygen -f id_rsa -t rsa -N '' 8 | eval `ssh-agent -s` 9 | ssh-add id_rsa 10 | 11 | echo "move the keys to the correct place in the jenkins root" 12 | sudo mv id_rsa* /var/lib/jenkins/.ssh 13 | sudo chown jenkins:nogroup -R /var/lib/jenkins/.ssh 14 | 15 | -------------------------------------------------------------------------------- /scripts/jenkins-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Sometimes ssh comes up a bit too quick so we sleep for 10 seconds before we begin..." 4 | sleep 10 5 | 6 | echo "Installing Jenkins" 7 | wget -q -O - http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add - 8 | sudo sh -c 'echo deb http://pkg.jenkins-ci.org/debian binary/ > /etc/apt/sources.list.d/jenkins.list' 9 | sudo apt-get update -y 10 | sudo apt-get install -y jenkins git 11 | 12 | sudo service jenkins start 13 | -------------------------------------------------------------------------------- /files/build_jenkins_slave.yaml: -------------------------------------------------------------------------------- 1 | - job: 2 | name: build_slave 3 | description: 'Do not edit this job through the web!' 4 | project-type: freestyle 5 | block-downstream: false 6 | scm: 7 | - git: 8 | skip-tag: false 9 | url: https://github.com/phiche/jenkins-autobuilder.git 10 | branches: 11 | - master 12 | triggers: 13 | - pollscm: '@hourly' 14 | builders: 15 | - shell: | 16 | packer build ubuntu-slave-ami.json 17 | -------------------------------------------------------------------------------- /files/build_jenkins_master.yaml: -------------------------------------------------------------------------------- 1 | - job: 2 | name: build-jenkins-master 3 | description: 'Do not edit this job through the web!' 4 | project-type: freestyle 5 | block-downstream: false 6 | scm: 7 | - git: 8 | skip-tag: false 9 | url: https://github.com/phiche/jenkins-autobuilder.git 10 | branches: 11 | - master 12 | triggers: 13 | - pollscm: '@hourly' 14 | builders: 15 | - shell: | 16 | packer build ubuntu-jenkins-ami.json 17 | -------------------------------------------------------------------------------- /scripts/swarm-service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Download swarm client, move it to the right place and set correct permissions" 4 | wget http://maven.jenkins-ci.org/content/repositories/releases/org/jenkins-ci/plugins/swarm-client/1.20/swarm-client-1.20-jar-with-dependencies.jar 5 | sudo mv swarm-client-1.20-jar-with-dependencies.jar /usr/local/bin 6 | sudo chown root:root /usr/local/bin/swarm-client-1.20-jar-with-dependencies.jar 7 | sudo chmod +x /usr/local/bin/swarm-client-1.20-jar-with-dependencies.jar 8 | 9 | echo "Move jenkins-slave.conf service to correct location and set permissions" 10 | sudo mv /tmp/jenkins-slave.conf /etc/init/ 11 | sudo chown root:root /etc/init/jenkins-slave.conf 12 | 13 | -------------------------------------------------------------------------------- /files/jenkins-slave.conf: -------------------------------------------------------------------------------- 1 | # jenkins-slave starts the swarm client as a service on the slave 2 | 3 | description "jenkins slave swarm client" 4 | 5 | #Fedora/Centos/RHEL 6 | #start on started network 7 | #Debian/Ubuntu 8 | start on started networking 9 | 10 | stop on runlevel [016] 11 | 12 | respawn 13 | respawn limit 5 10 14 | 15 | pre-start script 16 | logger -is -t "$UPSTART_JOB" "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Starting" 17 | end script 18 | 19 | exec java -jar /usr/local/bin/swarm-client-1.20-jar-with-dependencies.jar -master http://name.of.server.here:8080/ -fsroot /home/jenkins -name `hostname` -description boom 20 | 21 | pre-stop script 22 | logger -is -t "$UPSTART_JOB" "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Stopping" 23 | end script 24 | -------------------------------------------------------------------------------- /ubuntu-jenkins-ami.json: -------------------------------------------------------------------------------- 1 | { 2 | "builders": [ 3 | { 4 | "type": "amazon-ebs", 5 | "region": "eu-west-1", 6 | "source_ami": "ami-f6b11181", 7 | "instance_type": "m1.medium", 8 | "ssh_username": "ubuntu", 9 | "ssh_timeout": "5m", 10 | "ami_name": "ubuntu-jenkins-master {{timestamp}}" 11 | } 12 | ], 13 | "provisioners": [ 14 | { 15 | "type": "file", 16 | "source": "files/", 17 | "destination": "/tmp" 18 | }, 19 | { 20 | "type": "shell", 21 | "scripts": [ 22 | "scripts/jenkins-install.sh", 23 | "scripts/create-ssh-key.sh", 24 | "scripts/plugins-install.sh", 25 | "scripts/install-packer.sh", 26 | "scripts/setup-jobs.sh" 27 | ] 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /ubuntu-slave-ami.json: -------------------------------------------------------------------------------- 1 | { 2 | "builders": [ 3 | { 4 | "type": "amazon-ebs", 5 | "region": "eu-west-1", 6 | "source_ami": "ami-f6b11181", 7 | "instance_type": "m1.medium", 8 | "ssh_username": "ubuntu", 9 | "ssh_timeout": "5m", 10 | "ami_name": "ubuntu-jenkins-slave {{timestamp}}" 11 | } 12 | ], 13 | "provisioners": [ 14 | { 15 | "type": "shell", 16 | "scripts": [ 17 | "scripts/slave-install.sh" 18 | ] 19 | }, 20 | { 21 | "type": "file", 22 | "source": "files/jenkins-slave.conf", 23 | "destination": "/tmp/jenkins-slave.conf" 24 | }, 25 | { 26 | "type": "shell", 27 | "scripts": [ 28 | "scripts/swarm-service.sh" 29 | ] 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /scripts/plugins-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo apt-get install -y unzip 4 | 5 | wget http://s3.amazonaws.com/ec2-downloads/ec2-api-tools.zip 6 | sudo mkdir /usr/local/ec2 7 | sudo unzip ec2-api-tools.zip -d /usr/local/ec2 8 | 9 | echo "It takes some time for jenkins to come up so we need to look through this a few times" 10 | wget "http://`hostname -f`:8080/jnlpJars/jenkins-cli.jar" 11 | while [ $? -ne 0 ] 12 | do 13 | echo "Jenkins not up yet. Sleeping for 5 sec" 14 | sleep 5 15 | wget "http://`hostname -f`:8080/jnlpJars/jenkins-cli.jar" 16 | done 17 | echo "Jenkins is up! Downloading plugins" 18 | 19 | echo "There is some bug with the plugin installer so we have to follow procedure here https://gist.github.com/rowan-m/1026918" 20 | curl -L http://updates.jenkins-ci.org/update-center.json | sed '1d;$d' | curl -X POST -H 'Accept: application/json' -d @- http://localhost:8080/updateCenter/byId/default/postBack 21 | java -jar jenkins-cli.jar -s http://localhost:8080/ install-plugin Git Swarm Bitbucket 22 | #java -jar jenkins-cli.jar -s http://localhost:8080/ install-plugin http://updates.jenkins-ci.org/download/plugins/git/2.0.4/git.hpi http://updates.jenkins-ci.org/download/plugins/swarm/1.15/swarm.hpi http://updates.jenkins-ci.org/download/plugins/bitbucket/1.0/bitbucket.hpi 23 | #cd /var/lib/jenkins/plugins 24 | #wget http://updates.jenkins-ci.org/download/plugins/git/2.0.4/git.hpi 25 | #wget http://updates.jenkins-ci.org/download/plugins/swarm/1.15/swarm.hpi 26 | #wget http://updates.jenkins-ci.org/download/plugins/bitbucket/1.0/bitbucket.hpi 27 | #curl http://`hostname -f`:8080/reload 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | jenkins-autobuilder 2 | =================== 3 | 4 | This project will build and configure a new jenkins master and it's slaves on AWS using packer. Please read the documentation at http://www.packer.io/docs to learn how packer works. 5 | 6 | This project is still a work in progress. 7 | 8 | Requirements 9 | ------------ 10 | * Packer 11 | * EC2 command line tools 12 | * EC2 account 13 | 14 | Tested on: 15 | * OSX Mavericks 16 | 17 | Install Dependencies 18 | ------------------- 19 | #### Create AWS account 20 | If you don't already have an account on AWS go sign up first [here](http://aws.amazon.com) 21 | #### Install EC2 command line tools 22 | Follow [these directions](http://docs.aws.amazon.com/AWSEC2/latest/CommandLineReference/set-up-ec2-cli-linux.html) to set up the EC2 command line tools. Alternatively, if you are on OSX and have homebrew installed, you can just run "brew install ec2-api-tools". 23 | #### Install Packer 24 | Install Packer from [http://www.packer.io](http://www.packer.io). Again, if you are on OSX with homebrew, just perform a `brew install packer` and you'll get the latest version. 25 | 26 | Instructions 27 | ------------------- 28 | Clone this repo and run `packer build ubuntu-jenkins-ami.json`. Packer will then build you a new ami with a jenkins master bootstrapped on the base-ami specified in the json file. 29 | 30 | At the end of the packer build, the console output will tell you the name of the ami it has create. Go and start this instance on EC2 and you will discover you have some preconfigured jobs there already there. 31 | * to build a new master (using this exact same method described here) 32 | * build a new jenkins slave which connects to the master using the swarm plugin (still a manual step to add the necessary environment variables and fix ports so that this works) 33 | * add a new jenkins slave which will spin up the instance create and auto connect to the jenkins master 34 | * you need to allow traffic into the master from the slaves on ports 8080 for discovery and 42529 to complete the connection. 35 | 36 | TODOs 37 | ------------------- 38 | There is still some manual work involved to get the jobs building on the master on AWS. If you are familiar with jenkins then this should be pretty straight forward for you, however the intention is to Automate All The Things. 39 | Porting the project to other providers such as openstack and docker are also on the agenda. 40 | 41 | Licence 42 | ------------------- 43 | The MIT License (MIT) 44 | 45 | Copyright (c) 2015 Philip Cheong 46 | 47 | Permission is hereby granted, free of charge, to any person obtaining a copy 48 | of this software and associated documentation files (the "Software"), to deal 49 | in the Software without restriction, including without limitation the rights 50 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 51 | copies of the Software, and to permit persons to whom the Software is 52 | furnished to do so, subject to the following conditions: 53 | 54 | The above copyright notice and this permission notice shall be included in all 55 | copies or substantial portions of the Software. 56 | 57 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 58 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 59 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 60 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 61 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 62 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 63 | SOFTWARE. 64 | --------------------------------------------------------------------------------