├── deploy-make
├── README.md
└── deploy-helpers


/deploy-make:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Use this like:
 4 | #
 5 | #  bash deploy-make
 6 | #
 7 | # If you want to download and use then:
 8 | #
 9 | # bash <(curl https://raw.github.com/nicferrier/docker-shell-deploy/master/deploy-make)
10 | 
11 | [ -f Dockerfile ] || { echo "No Dockerfile" ; exit 1; }
12 | dockerExPort=$(sed -rne 's/EXPOSE ([0-9]+)/\1/p' Dockerfile)
13 | [ $dockerExPort -gt 1000 ] || { echo "Bad exported port in Dockerfile" ; exit 1; }
14 | 
15 | # Now read some stuff
16 | read -p "docker image: " dockerImage
17 | read -p "nginx config: " nginxConfig
18 | read -p "remote host name: " hostName
19 | 
20 | # Flatten the Dockerfile volumes in to mappings in a single string
21 | dockerVolumes=""
22 | exec 4<&0-
23 | while read mapping
24 | do
25 |     [ -n $dockerVolumes ] && dockerVolumes="${dockerVolumes}+$mapping"
26 |     [ -z $dockerVolumes ] && dockerVolumes="$mapping"
27 | done < <(sed -rne 's/VOLUME (.*)/\1/p' Dockerfile | while read volume
28 | do
29 |   read -u 4 -p "host mapping for ${volume}: " mapping
30 |   [ -z $mapping ] || echo "$mapping:$volume"
31 | done)
32 | 
33 | 
34 | # Should we test anything about the hostName here?
35 | # possibly:
36 | #
37 | #   ssh hostName sudo echo done
38 | # or:
39 | #   ssh hostName test -f $nginxConfig
40 | 
41 | url="https://raw.githubusercontent.com/nicferrier/docker-shell-deploy/master/deploy-helpers"
42 | cat <<EOF > deploy
43 | #!/bin/bash
44 | # Docker deploy script generated by deploy-make
45 | 
46 | [ -f ./.deploy-test ] && source ./.deploy-test
47 | [ -f ./.deploy ] \
48 |     || curl $url -o ./.deploy \
49 |     || { echo "can't http the deployscript" ; exit 1; }
50 | . ./.deploy
51 | dockerImage=$dockerImage
52 | dockerExPort=$dockerExPort
53 | nginxConfig=$nginxConfig
54 | hostName=$hostName
55 | dockerVolumes=$dockerVolumes
56 | deploy \${1:-"deploy"} $dockerImage $dockerExPort $nginxConfig $hostName $dockerVolumes
57 | EOF
58 | 
59 | chmod u+x deploy
60 | 
61 | echo deploy script has been written, now just:  bash deploy
62 | 
63 | # End
64 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # docker deployer
  2 | 
  3 | This is some tools for deploying docker containers to a host which is
  4 | providing access to the docker with nginx.
  5 | 
  6 | I wanted something similar to heroku, rapid deployment of new things,
  7 | but without being bound to heroku and having to pay for an account to
  8 | scale and all those things.
  9 | 
 10 | 
 11 | ## changelog
 12 | 
 13 | *2014 September* - make the deployment start a daemon on the remote
 14 | host to continually ping the docker's HTTP and restart it. The daemon
 15 | has a fifo that it reads for commands, you can get a list of commands
 16 | by sending it "help". Everything is stored in `/tmp/ddctrl/{name-of-image}`
 17 | 
 18 | *2014 August* - handle nginx configs with upstreams instead of direct
 19 | HTTP addresses in backend statements. The support isn't great but
 20 | without parsing the nginx config totally (and the best placed thing to
 21 | do that would be nginx) there's not a lot we can do better.
 22 | 
 23 | 
 24 | ### requirements
 25 | 
 26 | You need:
 27 | 
 28 | * `bash`
 29 | * `curl`
 30 | * `jq`
 31 | * `docker` - duh
 32 | * an app that has a `Dockerfile`
 33 | * something that exports a port with Docker, probably a webapp because this uses nginx
 34 | * a live host to host your container
 35 | * an ssh key relationship with your remote host such that you can `ssh remotehost`
 36 | * an account on the docker registry
 37 | 
 38 | ### how to
 39 | 
 40 | ```shell-session
 41 | $ bash <(curl -L https://raw.github.com/nicferrier/docker-shell-deploy/master/deploy-make)
 42 | ```
 43 | 
 44 | will download the script and ask a few questions:
 45 | 
 46 | ```
 47 | docker image: nicferrier/elnode-gnudoc
 48 | nginx config: /etc/nginx/sites-enabled/gnudoc.conf
 49 | remote host name: po5.ferrier.me.uk
 50 | ```
 51 | 
 52 | * `docker image` - the name of the docker image you want to build, this will be pushed to the docker registry
 53 | * `nginx config` - the nginx config on the live host which proxies the docker
 54 | * `remote host name` - the host name of the live host for the docker, the docker will be pulled here from the docker registry
 55 | 
 56 | If you have VOLUME export statements in your `Dockerfile` it will also
 57 | ask you where thet are mapped.
 58 | 
 59 | The script then creates a minimal `deploy` script which you can run to
 60 | deploy your docker app to the host you specified:
 61 | 
 62 | ```ShellSession
 63 | $ bash deploy
 64 | ```
 65 | 
 66 | You can safely check that into version control.
 67 | 
 68 | ### what does the deploy script do?
 69 | 
 70 | Here's an example:
 71 | 
 72 | ```bash
 73 | #!/bin/bash
 74 | # Docker deploy script generated by deploy-make
 75 | 
 76 | [ -f ./.deploy-test ] && source ./.deploy-test
 77 | [ -f ./.deploy ]     || curl https://raw.githubusercontent.com/nicferrier/docker-shell-deploy/master/deploy-helpers -o ./.deploy     || { echo "can't http the deployscript" ; exit 1; }
 78 | . ./.deploy
 79 | dockerImage=nicferrier/elnode-linky
 80 | dockerExPort=8005
 81 | nginxConfig=/etc/nginx/sites-enabled/linky
 82 | hostName=po5.ferrier.me.uk
 83 | dockerVolumes=;/home/nferrier/linky/db:/home/emacs/elnode-linky/db
 84 | deploy ${1:-"deploy"} nicferrier/elnode-linky 8005 /etc/nginx/sites-enabled/linky.conf po5.ferrier.me.uk /home/nferrier/linky/db:/home/emacs/elnode-linky/db
 85 | ```
 86 | 
 87 | So what does it do?
 88 | 
 89 | * builds the current Dockerfile to `docker image`
 90 | * pushes the `docker image` to the [docker registry](https://registry.hub.docker.com/)
 91 | * pushes a function to the `remote host name` with ssh and executes it to start a daemon to:
 92 |  * pull the new `docker image` from the docker registry
 93 |  * start the pulled `docker image` as a container
 94 |  * alter the `nginx config` to proxy the newly exported port
 95 |  * restart nginx
 96 |  * monitor the started docker for http
 97 |  * restart the docker when it fails (and update nginx)
 98 | 
 99 | ### other stuff the deploy script supports
100 | 
101 | The `deploy` script that gets created actually supports a few more
102 | tricks than just the deployment.
103 | 
104 | You can run just the build step of the deployment:
105 | 
106 | ```ShellSession
107 | $ bash deploy build
108 | ```
109 | 
110 | This will not push to the docker registry.
111 | 
112 | You can also run the build and push to docker registry, without doing
113 | the deploy:
114 | 
115 | ```ShellSession
116 | $ bash deploy push
117 | ```
118 | 
119 | ### stuff the deploy doesn't take care of
120 | 
121 | * creating your live environment outside the docker
122 |  * no nginx setup
123 |  * no creation of volume exported directories
124 | 
125 | ## thoughts
126 | 
127 | This is very imperfect. It has a lot of assumptions:
128 | 
129 | * you're using a webapp
130 | * you're exporting one port
131 | * you're proxying with nginx
132 | * you're using the docker registry
133 | * ... and that's just for starters
134 | 
135 | However, it's a start and it is repeatable.
136 | 
137 | ### is the download safe?
138 | 
139 | Perhaps you've heard that doing:
140 | 
141 | ```ShellSession
142 | $ curl http://something | bash -
143 | ```
144 | 
145 | is unsafe. Is:
146 | 
147 | ```ShellSession
148 | bash <(curl http://something)
149 | ```
150 | 
151 | any safer?
152 | 
153 | NO! Never use a curl and a bash together if you don't know what
154 | they're doing. It's completely mad.
155 | 
156 | However, once you are confident of the script then you can do it.
157 | 
158 | If you're not confident of what the script does
159 | then
160 | [go look at it](https://github.com/nicferrier/docker-shell-deploy/blob/master/deploy-make).
161 | 
162 | ### justification
163 | 
164 | Why not just use Heroku? or some other sexy PAAS?
165 | 
166 | Because this is just as usable and probably more scalable. By that I
167 | mean that I can persuade it do new things easier than I can get Heroku
168 | to do new things.
169 | 
170 | With Heroku I'm working with someone else's constraints. With this I'm
171 | working with mine.
172 | 


--------------------------------------------------------------------------------
/deploy-helpers:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # Functions to help with docker deployment.
  4 | #
  5 | # function deploy rebuilds a dockerfile, pushes the image to docker
  6 | # and then pulls the same docker image on a remote host, rebinds the
  7 | # nginx config there and restarts nginx.
  8 | #
  9 | # function remoteDeploy is a helper function for operating on the
 10 | # remote host.
 11 | 
 12 | 
 13 | # This is the script run on the remote machine
 14 | # 
 15 | # Call with:
 16 | #
 17 | # $1  the docker image name, eg: nicferrier/elnode-gnudoc
 18 | # $2  the docker exported port which you will connect to nginx
 19 | # $3  the nginx config name that you'll connect to
 20 | # $4  docker volume mounts, like: "/host:/container+/host:/container"
 21 | function remoteDeploy {
 22 |     local dockerImage=$1
 23 |     local dockerExPort=$2
 24 |     local nginxConfig=$3
 25 |     local dockerVolumes=$4
 26 | 
 27 |     [ -x /etc/init.d/nginx ] || { echo "no nginx?" >&2 ; return 1; }
 28 |     [ -x $(which jq) ] || { echo "no jq?" >&2 ; return 1; }
 29 |     [ -x $(which docker || which docker.io) ] || { echo "no docker?" >&2 ; return 1; }
 30 | 
 31 |     local docker="sudo $(which docker || which docker.io)"
 32 |     local jq="$(which jq)"
 33 |     local stateDir=/tmp/ddctrl/$(echo $dockerImage | sed -rne "s|/|__|pg")
 34 | 
 35 |     # FIXME: this is not a great way to identify the current
 36 |     # container... we could ask nginx?
 37 |     local containerId=$($docker ps | grep $dockerImage | cut -d ' ' -f 1 | head -n1)
 38 |     
 39 |     function daemon {
 40 |         echo "inside daemon"
 41 |         local ETAG=$(date '+%Y%m%d%H%M%S%N')
 42 | 
 43 |         # Daemon stuff
 44 |         mkdir -p $stateDir
 45 |         [ -f $stateDir/pid ] && mv $stateDir/pid $stateDir/old.pid.$(date '+%Y%m%d-%H%M-%S')
 46 |         echo $$ > $stateDir/pid
 47 |         exec 3>&-   # close stdin
 48 |         exec 2>>$stateDir/stderr
 49 |         exec 1>>$stateDir/stdout
 50 |         echo $(date) daemonized
 51 | 
 52 |         # Setup the fifo if it doesn't exist
 53 |         local fifo=$stateDir/fifo
 54 |         [ -p $fifo ] || mkfifo $fifo
 55 | 
 56 |         # Check docker volume mounts and turn them into docker opts
 57 |         [ -z $dockerVolumes ] || {
 58 |             echo $dockerVolumes | tr '+' '\n' | tr ':' ' ' | while read host container
 59 |             do
 60 |                 [ -d $host ] || { echo "$host should be a directory and isn't" >&2 ; return 1; }
 61 |             done
 62 | 
 63 |             # Everything ok? make a valid docker options string
 64 |             dockerVolumes=$(echo $dockerVolumes | sed -rne 's/^|\+/ -v /pg')
 65 |         }
 66 | 
 67 |         # Now the library of functions we need
 68 |         function containerToHost {
 69 |             local containerIdent=$1
 70 |             $docker inspect $containerIdent | jq -r '.[0] | .NetworkSettings.IPAddress'
 71 |         }
 72 | 
 73 |         function curlCheck {
 74 |             local containerIp=$1
 75 |             local port=$2
 76 |             local connectTimeout=${3:-20}
 77 |             local timeout=$(expr $connectTimeout + 10)
 78 |             local output=""
 79 |             local curlResult=0
 80 |             local awkResult=0
 81 |             local http=$stateDir/httpout
 82 |             curl -i -m $timeout \
 83 |                 --connect-timeout $connectTimeout \
 84 |                 http://$containerIp:$port 2> /dev/null > $http
 85 |             curlResult=$?
 86 |             local tries=0
 87 |             while [[ $curlResult -eq 7 && $tries < 3 ]] # if it was connect fail try again
 88 |             do
 89 |                 sleep 1
 90 |                 curl -i -m $timeout \
 91 |                     --connect-timeout $connectTimeout \
 92 |                     http://$containerIp:$port 2> /dev/null > $http
 93 |                 curlResult=$?
 94 |                 tries=$(expr $tries + 1)
 95 |             done
 96 |             output=$(awk 'BEGIN { lastline="start"; }
 97 | /^HTTP[/]1.1 [45]0[0-9]/ { print $0; exit(9); }
 98 | /<[Hh]1>Server error<[/][Hh]1>/ { print $0; exit(9); }
 99 | { gsub("[\r\n]$", ""); }
100 | lastline=="" { print $0; exit(0); }
101 | { lastline=$0; }' $http)
102 |             awkResult=$?
103 |             if [ $curlResult -ne 0 ]
104 |             then
105 |                 echo ">>> [$curlResult]"
106 |                 cat $http
107 |                 echo "<<<"
108 |             fi
109 |             local returnResult=$(($curlResult||$awkResult))
110 |             echo "$(date) curlCheck $containerIp $port $connectTimeout $timeout => [$output] [$returnResult]"
111 |             return $returnResult
112 |         }
113 | 
114 |         function nginx {
115 |             local newIp=$1
116 |             local upstreams=0
117 | 
118 |             grep "upstream" $nginxConfig > /dev/null
119 |             upstreams=$?
120 |             if [ $upstreams -eq 0 ]
121 |             then
122 |                 sudo sed -ibk \
123 |                     -re "s|server [^:;]*:[0-9]+|server ${newIp}:${dockerExPort}|" $nginxConfig
124 |             else
125 |                 sudo sed -ibk \
126 |                     -re "s|proxy_pass http://[^:;]*:[0-9]+|proxy_pass http://$newIp:${dockerExPort}|" $nginxConfig
127 |             fi        
128 |             echo "$(date) nginx $nginxConfig upstreams=$upstreams"
129 |             sudo /etc/init.d/nginx reload
130 |         }
131 | 
132 |         function dockerRun {
133 |             $docker run -d --env ETAG=${ETAG} ${dockerVolumes} -t $dockerImage
134 |         }
135 | 
136 |         function dockerStart {
137 |             local newContainerId=$(dockerRun)
138 |             local newIp=$(containerToHost $newContainerId)
139 |             echo $(date) dockerStart newIp: $newIp newContainerId: $newContainerId
140 |             sleep 3
141 |             curlCheck $newIp $dockerExPort 450 && nginx $newIp
142 |             [ $? -eq 0 ] && containerId=$newContainerId
143 |         }
144 | 
145 |         function dockerClean {
146 |             local newIp=$(containerToHost $containerId)
147 |             $docker ps -q |  while read dockId
148 |             do 
149 |                 ( $docker inspect -f '{{ .Config.Image }}' $dockId \
150 |                     | grep $dockerImage > /dev/null ) && echo $dockId
151 |             done | while read dockId 
152 |             do
153 |                 # $dockId is a $dockerImage container
154 |                 printf "$dockId " 
155 |                 $docker inspect $dockId | $jq -r '.[0] | .NetworkSettings.IPAddress'
156 |             done | while read dockId ipAddress
157 |             do
158 |                 echo "$(date) dockerClean $newIp $dockId $ipAddress"
159 |                 # Now we have the dockId and the ipAddress
160 |                 [ "$newIp" != "$ipAddress" ] && {
161 |                     # killing $dockId because it's not on $newIp
162 |                     $docker kill $dockId;
163 |                 }
164 |             done
165 |         }
166 | 
167 |         function dockerRestart {
168 |             echo $(date) dockerRestart
169 |             dockerStart
170 |             dockerClean
171 |         }
172 | 
173 |         function dockerCheck {
174 |             echo "$(date) dockerCheck [$containerId]"
175 |             [ "$containerId" == "" ] && dockerRestart
176 |             local containerIp=$(containerToHost $containerId)
177 |             curlCheck $containerIp $dockerExPort 10
178 |             if [ $? -ne 0 ]
179 |             then
180 |                  dockerRestart
181 |             else
182 |                 echo "$(date) dockerCheck done - all ok"
183 |             fi
184 |         }
185 | 
186 |         # Daemon loop
187 |         while [ 1 ]
188 |         do
189 |             read -t 30 command args <>$fifo
190 |             case $command in
191 |                 run)
192 |                     dockerRun
193 |                     ;;
194 | 
195 |                 start)
196 |                     dockerStart
197 |                     ;;
198 | 
199 |                 check)
200 |                     dockerCheck
201 |                     ;;
202 | 
203 |                 stop)
204 |                     break
205 |                     ;;
206 | 
207 |                 pull)
208 |                     sudo docker pull $dockerImage
209 |                     ;;
210 | 
211 |                 restart)
212 |                     dockerRestart
213 |                     ;;
214 |                 
215 |                 help)
216 |                     echo $dockerImage control daemon
217 |                     echo "command: start|stop|help|pull|run|check to $fifo"
218 |                     ;;
219 |             esac
220 |             
221 |             # Check every now and then... maybe this needs to be a separate daemon?
222 |             dockerCheck
223 |         done
224 | 
225 |         # Must be exiting?
226 |         mv $stateDir/pid $stateDir/old.pid.$(date '+%Y%m%d-%H%M-%S')
227 |     }
228 | 
229 |     function daemonCheck {
230 |         [ -f $stateDir/pid ] && ps -$(cat $stateDir/pid) > /dev/null
231 |     }
232 | 
233 |     daemonCheck
234 |     [ $? -ne 0 ] || { 
235 |         echo "daemon already running";
236 |         echo stop > $stateDir/fifo; 
237 |     }
238 | 
239 |     daemon &
240 |     sudo docker pull $dockerImage # by the time this has finished we've got the daemon up
241 |     echo restart > $stateDir/fifo
242 | }
243 | 
244 | 
245 | # Function to the initial deploy
246 | # 
247 | # Call with:
248 | #
249 | # $1  the command, either "build" (just builds the docker)
250 | #       or "push"  (builds and pushes to remote but doesn't restart)
251 | #       or "deploy", the default, which builds, pushes and restarts
252 | # $2  the docker image name to build
253 | # $3  the docker exported port to connect to
254 | # $4  the nginx config to hack on the remote
255 | # $5  the host name of the remote
256 | # $6  the volumes to be mounted separated by +
257 | function deploy {
258 |     [ -x $(which docker || which docker.io) ] || { echo "no docker?" >&2 ; return 1; }
259 |     local docker="sudo $(which docker || which docker.io)"
260 |     
261 |     local command=$1
262 |     local dockerImage=$2
263 |     local dockerExPort=$3
264 |     local nginxConfig=$4
265 |     local hostName=$5
266 |     local dockerVolumes=$6
267 | 
268 |     if [ "$command" != "remote" ]
269 |     then
270 |         cd $(dirname $0)
271 |         [ -f Dockerfile ] || { echo "no Dockerfile?" >&2  ; return 1; }
272 |         
273 |         $docker build --no-cache -t $dockerImage .
274 |         
275 |         [ "$command" == "build" ] && echo "docker built" >&2 && exit 1
276 |         
277 |         $docker push $dockerImage
278 |         
279 |         [ "$command" == "push" ] && echo "docker pushed to $hostName" >&2 && exit 2
280 |     fi
281 | 
282 |     # Now the remote parts
283 |     ( typeset -fp remoteDeploy ; \
284 |         echo remoteDeploy $dockerImage $dockerExPort $nginxConfig $dockerVolumes \
285 |         ) | ssh $hostName bash -s
286 | }
287 | 
288 | # deploy ends here
289 | 


--------------------------------------------------------------------------------