├── cli.js ├── package.json ├── readme.md └── index.js /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict' 4 | 5 | var through = require('through2') 6 | 7 | var deployStream = require('./')() 8 | 9 | deployStream 10 | .on('error', function (err) { 11 | if (err.code === 403) { 12 | console.error(new Error('Authentication failed. Did you provide a `keyFile` or `credentials` object?')) 13 | } else { 14 | console.error(err) 15 | } 16 | }) 17 | .on('file', function (file) { 18 | console.log('Backup created:', file.bucket.name + '/' + file.name) 19 | }) 20 | .on('vm', function (vm) { 21 | console.log('VM ready:', vm.name) 22 | }) 23 | .on('start', function (url) { 24 | console.log('Deployed successfully!', url) 25 | }) 26 | .pipe(through(function (logLine, enc, next) { 27 | var vm = deployStream.vm 28 | var url = deployStream.url 29 | 30 | // replace some verbosity with what's probably more helpful-- the IP 31 | logLine = String(logLine).replace(new RegExp(vm.name + '[^:]*', 'g'), '(' + url + ')').trim() 32 | next(null, '\n' + logLine) 33 | })) 34 | .pipe(process.stdout) 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gcloud-deploy", 3 | "version": "0.7.0", 4 | "description": "Quickly deploy a Node.js project on Google Compute Engine", 5 | "main": "index.js", 6 | "bin": { 7 | "gcloud-deploy": "./cli.js" 8 | }, 9 | "keywords": [ 10 | "google", 11 | "google cloud", 12 | "google cloud platform", 13 | "compute engine", 14 | "google compute engine", 15 | "container", 16 | "deploy", 17 | "vm", 18 | "virtual machine", 19 | "instance" 20 | ], 21 | "author": "Stephen Sawchuk ", 22 | "repository": "stephenplusplus/gcloud-deploy", 23 | "license": "MIT", 24 | "dependencies": { 25 | "@google-cloud/compute": "^0.9.0", 26 | "@google-cloud/storage": "^1.5.2", 27 | "archiver": "^0.15.1", 28 | "async": "^1.4.2", 29 | "deep-assign": "^1.0.0", 30 | "gce-output-stream": "^0.1.0", 31 | "multiline": "^1.0.2", 32 | "pumpify": "^1.3.3", 33 | "slug": "^0.9.1", 34 | "string-format-obj": "^1.0.1", 35 | "through2": "^2.0.0" 36 | }, 37 | "devDependencies": { 38 | "standard": "^5.3.1" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # gcloud-deploy 2 | > Quickly deploy a Node.js project on Google Compute Engine 3 | 4 | ## Getting Started 5 | 6 | - [How it works](#how-it-works) 7 | - [Prerequisites](#prerequisites) 8 | - [Configuration](#configuration) 9 | - [API](#api) 10 | - [Contributions](#contributions) 11 | 12 | 13 | ### Quick Start 14 | 15 | See [gcloud-deploy Boilerplate](https://github.com/stephenplusplus/gcloud-deploy-boilerplate). 16 | 17 | ### CLI 18 | ```sh 19 | $ npm install -g gcloud-deploy 20 | $ gcloud-deploy 21 | Backup created: grape-spaceship-123-gcloud-deploy-tars/1444765984324.tar 22 | VM created: my-app-1444765984324 23 | Deployed successfully! http://146.148.48.123 24 | ``` 25 | 26 | ### Programmatically 27 | ```sh 28 | $ npm install --save gcloud-deploy 29 | ``` 30 | ```js 31 | var gcloudDeploy = require('gcloud-deploy') 32 | 33 | // Give it a Node.js project 34 | gcloudDeploy('./') 35 | // A VM was created (`vm` is a gcloud-node VM object) 36 | .on('vm', function (vm) {}) 37 | 38 | // App is being served at `url` 39 | .on('start', function (url) {}) 40 | 41 | // raw output from the server while it initializes & starts your app 42 | .pipe(process.stdout) 43 | ``` 44 | 45 | ### npm script 46 | ```sh 47 | $ npm install --save-dev gcloud-deploy 48 | ``` 49 | ```json 50 | { 51 | "name": "my-app", 52 | "devDependencies": { 53 | "gcloud-deploy": "*" 54 | }, 55 | "scripts": { 56 | "deploy": "gcloud-deploy" 57 | } 58 | } 59 | ``` 60 | ```sh 61 | $ npm run deploy 62 | ``` 63 | 64 | 65 | ## How it works 66 | 67 | This module... 68 | 69 | 1. makes a tarball of your project 70 | 1. uploads it to a bucket 71 | 1. creates a Compute Engine instance with a startup script to: 72 | 1. install the latest stable Node.js (customizable) 73 | 1. unpack the tarball 74 | 1. run `npm start` 75 | 76 | 77 | ## Prerequisites 78 | 79 | There are only two things required to use `gcloud-deploy`: 80 | 81 | - A Google Developers Console project ID to deploy your project to 82 | - A key file that contains credentials to authenticate API requests 83 | 84 | If you haven't already, you will need to create a project in the [Google Developers Console](https://console.developers.google.com/project). 85 | 86 | For a more detailed guide, see the *"On Your Own Server"* section of [gcloud-node's Authentication document](https://googlecloudplatform.github.io/gcloud-node/#/authentication). 87 | 88 | The APIs that **must be enabled** are: 89 | 90 | - **Google Compute Engine** 91 | - **Google Cloud Storage** 92 | 93 | The guide linked above will also guide you through creating a JSON keyfile. 94 | 95 | 96 | ## Configuration 97 | 98 | This library tries to provide sane defaults for your VM. As explained in the `Prerequisites` section, all that is required are two properties: 99 | 100 | - `projectId` - The project to deploy the VM to. 101 | - `keyFile` - A path to a JSON, PEM, or P12 key file. 102 | 103 | If you need further customization beyond the defaults, we accept configuration in a few different ways, which are listed below with examples. 104 | 105 | These two links will be important: 106 | 107 | - [Connection configuration](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.24.0?method=gcloud) 108 | - [VM configuration](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.24.0/compute/zone?method=createVM) 109 | 110 | 111 | ### Configuration Object 112 | 113 | When running programmatically, this may be the simplest, most consistent option. You can provide explicit configuration with a `config` object. 114 | 115 | ```js 116 | var config = { 117 | gcloud: { 118 | // Same as the `config` object documented here: 119 | // https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.24.0?method=gcloud 120 | }, 121 | 122 | vm: { 123 | // Same as the `config` object documented here: 124 | // https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.24.0/compute/zone?method=createVM 125 | } 126 | } 127 | 128 | gcloudDeploy(config) 129 | ``` 130 | 131 | Additionally, you can provide a `config.vm.zone` string to specify the zone to create your VM in. 132 | 133 | #### Defaults 134 | 135 | See how the default configuration is trumped by the [package.json's `gcloudDeploy`](#packageJson) object, then finally the [`config` object](#config). 136 | 137 | ```js 138 | var defaults = { 139 | root: process.cwd(), 140 | nodeVersion: 'stable', // the latest Node.js stable release 141 | 142 | gcloud: { 143 | projectId: process.env.GCLOUD_PROJECT_ID, 144 | keyFile: process.env.GCLOUD_KEY_FILE 145 | }, 146 | 147 | vm: { 148 | zone: process.env.GCLOUD_ZONE || 'us-central1-a', 149 | name: slugify(packageJson.name) + '-' + Date.now(), 150 | os: 'centos', 151 | http: true, 152 | https: true 153 | } 154 | } 155 | 156 | deepExtend(defaults, packageJson.gcloudDeploy, config) 157 | ``` 158 | 159 | 160 | ### package.json 161 | 162 | You may also create `gcloud` and `vm` properties inside of the deployed project's `package.json` in the same format as described above in [Configuration Object](#config). 163 | 164 | An example `package.json`: 165 | 166 | ```json 167 | { 168 | "name": "my-app", 169 | "version": "0.0.0", 170 | "dependencies": { 171 | "express": "^4.13.3" 172 | }, 173 | "gcloudDeploy": { 174 | "nodeVersion": 4, 175 | "gcloud": { 176 | "projectId": "grape-spaceship-123", 177 | "keyFile": "~/key.json" 178 | }, 179 | "vm": { 180 | "os": "ubuntu", 181 | "zone": "us-central1-b" 182 | } 183 | } 184 | } 185 | ``` 186 | 187 | ### Environment variables 188 | 189 | - **GCLOUD_PROJECT_ID** (required) - maps to `config.projectId` 190 | - **GCLOUD_KEY_FILE** - maps to `config.keyFile` 191 | - **GCLOUD_ZONE** - maps to `config.vm.zone` 192 | 193 | With just `GCLOUD_PROJECT_ID` and `GCLOUD_KEYFILE`, you can ignore all of the other configuration options described above. 194 | 195 | However, you are still free to provide further customization. Any values specified with the other techniques will take precedence over the environment variables. 196 | 197 | 198 | ## API 199 | 200 | #### `gcloudDeploy = require('gcloud-deploy')([config])` 201 | 202 | #### config 203 | - Type: `String|Object` 204 | - *Optional* 205 | 206 | If a string, it is treated as the package root (`config.root`); the directory to where the project's `package.json` can be found. 207 | 208 | If an object, See [**Configuration Object**](#config). 209 | 210 | ##### config.nodeVersion 211 | - Type: `String` 212 | - *Optional* 213 | - Default: `stable` 214 | 215 | The version of Node.js to run on the deployed VM via [nvm](https://github.com/creationix/nvm). If you need a specific version, provide that here, e.g. `0.12.7`. 216 | 217 | ##### config.root 218 | - Type: `String` 219 | - Default: `process.cwd()` 220 | 221 | The directory to where the project's `package.json` can be found. 222 | 223 | ##### config.startupScript 224 | - Type: `String` 225 | 226 | Define a custom start up script that will execute when your VM is launched and restarted. 227 | 228 | #### gcloudDeploy 229 | - Type: `Stream` 230 | 231 | A stream is returned that will **not end unless you end it**. It is a constant pouring of output from the created VM using [gce-output-stream](http://gitnpm.com/gce-output-stream). To end it, just abort the process (easy for the CLI), or programmatically: 232 | 233 | ```js 234 | gcloudDeploy() 235 | .on('data', function (outputLine) { 236 | if (outputLine.indexOf('node server.js') > -1) { 237 | // Looks like the server started 238 | // No need to poll for more output 239 | this.end() 240 | } 241 | }) 242 | ``` 243 | 244 | ##### .on('error', function (err) {}) 245 | - Type: `Error` 246 | 247 | An error occurred during the deploy process. 248 | 249 | ##### .on('bucket', function (bucket) {}) 250 | - Type: [`Bucket`](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.23.0/storage/bucket) 251 | 252 | A bucket was successfully created (or re-used) to hold the tarball snapshots we take of your project. 253 | 254 | *See the [gcloud-node Bucket docs](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.23.0/storage/bucket).* 255 | 256 | ##### .on('file', function (file) {}) 257 | - Type: [`File`](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.23.0/storage/file) 258 | 259 | The tarball snapshot of your project was uploaded successfully. After being used by the VM's startup script, it is deleted. 260 | 261 | *See the [gcloud-node File docs](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.23.0/storage/file).* 262 | 263 | ##### .on('vm', function (vm) {}) 264 | - Type: [`VM`](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.23.0/compute/vm) 265 | 266 | The VM that was created to host your project. Get the name of the VM from the `name` property (`vm.name`). 267 | 268 | *See the [gcloud-node VM docs](https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.23.0/compute/vm).* 269 | 270 | ##### .on('start', function (url) {}) 271 | - Type: `String` 272 | 273 | The URL to your project. If your app listens on port 80, you can get right to it from this URL. 274 | 275 | 276 | ## Contributions 277 | 278 | Desperately seeking help with the following tasks: 279 | 280 | - Modularize the startup script (maybe use [this one?](https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/gce/startup-script.sh)) 281 | - Don't make the tarball public 282 | - Expand CLI to: 283 | - Show running VMs 284 | - Stop/start VMs 285 | - Delete VMs 286 | 287 | If you're interested in helping out, please open an issue so our efforts don't collide. Plus, it'd be nice to meet you! 288 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var archiver = require('archiver') 4 | var assign = require('deep-assign') 5 | var async = require('async') 6 | var format = require('string-format-obj') 7 | var multiline = require('multiline') 8 | var outputStream = require('gce-output-stream') 9 | var path = require('path') 10 | var pumpify = require('pumpify') 11 | var slug = require('slug') 12 | var through = require('through2') 13 | 14 | var Compute = require('@google-cloud/compute') 15 | var Storage = require('@google-cloud/storage') 16 | 17 | var resolveConfig = function (pkg, explicitConfig) { 18 | var config = { 19 | root: process.cwd(), 20 | nodeVersion: 'stable', 21 | 22 | gcloud: { 23 | projectId: process.env.GCLOUD_PROJECT_ID, 24 | keyFile: process.env.GCLOUD_KEY_FILE 25 | }, 26 | 27 | vm: { 28 | zone: process.env.GCLOUD_ZONE || 'us-central1-a', 29 | os: 'centos', 30 | http: true, 31 | https: true 32 | } 33 | } 34 | 35 | assign(config, pkg.gcloudDeploy, explicitConfig) 36 | 37 | // gcloud wants `keyFilename` 38 | config.gcloud.keyFilename = config.gcloud.keyFile 39 | delete config.gcloud.keyFile 40 | 41 | if (!config.gcloud.projectId) { 42 | throw new Error('A projectId is required') 43 | } 44 | 45 | if (!config.gcloud.credentials && !config.gcloud.keyFilename) { 46 | throw new Error('Authentication with a credentials object or keyFile path is required') 47 | } 48 | 49 | return config 50 | } 51 | 52 | module.exports = function (config) { 53 | if (typeof config !== 'object') config = { root: config || process.cwd() } 54 | 55 | var pkg = require(path.join(config.root, 'package.json')) 56 | config = resolveConfig(pkg, config) 57 | 58 | var gcloudConfig = config.gcloud 59 | var pkgRoot = config.root 60 | var uniqueId = slug(pkg.name, { lower: true }) + '-' + Date.now() 61 | 62 | var gce = new Compute(gcloudConfig) 63 | var gcs = new Storage(gcloudConfig) 64 | 65 | var deployStream = pumpify() 66 | 67 | async.waterfall([ 68 | createTarStream, 69 | uploadTar, 70 | createVM, 71 | startVM 72 | ], function (err, vm) { 73 | if (err) return deployStream.destroy(err) 74 | 75 | var outputCfg = assign({}, gcloudConfig, { name: vm.name, zone: vm.zone.name }) 76 | 77 | outputCfg.authConfig = {} 78 | if (gcloudConfig.credentials) outputCfg.authConfig.credentials = gcloudConfig.credentials 79 | if (gcloudConfig.keyFilename) outputCfg.authConfig.keyFile = gcloudConfig.keyFilename 80 | 81 | deployStream.setPipeline(outputStream(outputCfg), through()) 82 | 83 | // sniff the output stream for when it's safe to delete the tar file 84 | deleteTarFile(outputStream(outputCfg)) 85 | }) 86 | 87 | function createTarStream (callback) { 88 | var tarStream = archiver.create('tar', { gzip: true }) 89 | tarStream.bulk([{ expand: true, cwd: pkgRoot, src: ['**', '!node_modules/**'] }]) 90 | tarStream.finalize() 91 | callback(null, tarStream) 92 | } 93 | 94 | function uploadTar (tarStream, callback) { 95 | var bucketName = gcloudConfig.projectId + '-gcloud-deploy-tars' 96 | var bucket = gcs.bucket(bucketName) 97 | 98 | bucket.get({ autoCreate: true }, function (err) { 99 | if (err) return callback(err) 100 | 101 | deployStream.emit('bucket', bucket) 102 | deployStream.bucket = bucket 103 | 104 | var tarFile = bucket.file(uniqueId + '.tar') 105 | var writeStream = tarFile.createWriteStream({ 106 | gzip: true, 107 | public: true 108 | }) 109 | 110 | tarStream.pipe(writeStream) 111 | .on('error', callback) 112 | .on('finish', function () { 113 | deployStream.emit('file', tarFile) 114 | deployStream.file = tarFile 115 | 116 | callback(null, tarFile) 117 | }) 118 | }) 119 | } 120 | 121 | function createVM (file, callback) { 122 | var vmCfg = config.vm 123 | 124 | // most node apps will have dependencies that requires compiling. without 125 | // these build tools, the libraries might not install 126 | var installBuildEssentialsCommands = { 127 | debian: multiline.stripIndent(function () {/* 128 | apt-get update 129 | apt-get install -yq build-essential git-core 130 | */}), 131 | 132 | fedora: multiline.stripIndent(function () {/* 133 | yum -y groupinstall "Development Tools" "Development Libraries" 134 | */}), 135 | 136 | suse: multiline.stripIndent(function () {/* 137 | sudo zypper --non-interactive addrepo http://download.opensuse.org/distribution/13.2/repo/oss/ repo 138 | sudo zypper --non-interactive --no-gpg-checks rm product:SLES-12-0.x86_64 cpp48-4.8.3+r212056-11.2.x86_64 suse-build-key-12.0-4.1.noarch 139 | sudo zypper --non-interactive --no-gpg-checks install --auto-agree-with-licenses --type pattern devel_basis 140 | */}) 141 | } 142 | 143 | var installBuildEssentialsCommand 144 | 145 | switch (vmCfg.os) { 146 | case 'centos': 147 | case 'centos-cloud': 148 | case 'redhat': 149 | case 'rhel': 150 | case 'rhel-cloud': 151 | installBuildEssentialsCommand = installBuildEssentialsCommands.fedora 152 | break 153 | 154 | case 'suse': 155 | case 'suse-cloud': 156 | case 'opensuse': 157 | case 'opensuse-cloud': 158 | installBuildEssentialsCommand = installBuildEssentialsCommands.suse 159 | break 160 | 161 | case 'debian': 162 | case 'debian-cloud': 163 | case 'ubuntu': 164 | case 'ubuntu-cloud': 165 | case 'ubuntu-os-cloud': 166 | default: 167 | installBuildEssentialsCommand = installBuildEssentialsCommands.debian 168 | break 169 | } 170 | 171 | var startupScript = format(multiline.stripIndent(function () {/* 172 | #! /bin/bash 173 | set -v 174 | {installBuildEssentialsCommand} 175 | {customStartupScript} 176 | export NVM_DIR=/usr/local/nvm 177 | export HOME=/root 178 | export GCLOUD_VM=true 179 | curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.31.0/install.sh | bash 180 | source /usr/local/nvm/nvm.sh 181 | nvm install {version} 182 | if [ ! -d /opt/app ]; then 183 | mkdir /opt/app 184 | fi 185 | cd /opt/app 186 | curl https://storage.googleapis.com/{bucketName}/{fileName} | tar -xz 187 | npm install --only-production 188 | npm start & 189 | */}), { 190 | installBuildEssentialsCommand: installBuildEssentialsCommand, 191 | customStartupScript: config.startupScript || '', 192 | bucketName: file.bucket.name, 193 | fileName: file.name, 194 | version: config.nodeVersion 195 | }) 196 | 197 | vmCfg.metadata = vmCfg.metadata || {} 198 | vmCfg.metadata.items = vmCfg.metadata.items || [] 199 | vmCfg.metadata.items.push({ key: 'startup-script', value: startupScript }) 200 | 201 | var zone = gce.zone(vmCfg.zone) 202 | 203 | var onVMReady = function (vm) { 204 | deployStream.emit('vm', vm) 205 | deployStream.vm = vm 206 | 207 | callback(null, vm) 208 | } 209 | 210 | var vm = zone.vm(vmCfg.name || uniqueId) 211 | 212 | if (vmCfg.name) { 213 | // re-use an existing VM 214 | // @tood implement `setMetadata` in gcloud-node#vm 215 | vm.setMetadata({ 216 | 'startup-script': startupScript 217 | }, _onOperationComplete(function (err) { 218 | if (err) return callback(err) 219 | onVMReady(vm) 220 | })) 221 | } else { 222 | // create a VM 223 | vm.create(vmCfg, _onOperationComplete(function (err) { 224 | if (err) return callback(err) 225 | onVMReady(vm) 226 | })) 227 | } 228 | } 229 | 230 | function startVM (vm, callback) { 231 | // if re-using a VM, we have to stop & start to apply the new startup script 232 | vm.stop(_onOperationComplete(function (err) { 233 | if (err) return callback(err) 234 | 235 | vm.start(_onOperationComplete(function (err) { 236 | if (err) return callback(err) 237 | 238 | vm.getMetadata(function (err, metadata) { 239 | if (err) return callback(err) 240 | 241 | var url = 'http://' + metadata.networkInterfaces[0].accessConfigs[0].natIP 242 | deployStream.emit('start', url) 243 | deployStream.url = url 244 | 245 | callback(null, vm) 246 | }) 247 | })) 248 | })) 249 | } 250 | 251 | function deleteTarFile (outputStream) { 252 | var tarFile = deployStream.file 253 | var startupScriptStarted = false 254 | 255 | outputStream.pipe(through(function (outputLine, enc, next) { 256 | outputLine = outputLine.toString('utf8') 257 | 258 | startupScriptStarted = startupScriptStarted || outputLine.indexOf('Starting Google Compute Engine user scripts') > -1 259 | 260 | // if npm install is running, the file has already been downloaded 261 | if (startupScriptStarted && outputLine.indexOf('npm install') > -1) { 262 | outputStream.end() 263 | 264 | tarFile.delete(function (err, apiResponse) { 265 | if (err) { 266 | var error = new Error('The tar file (' + tarFile.name + ') could not be deleted') 267 | error.response = apiResponse 268 | deployStream.destroy(error) 269 | } 270 | }) 271 | } else { 272 | next() 273 | } 274 | })) 275 | } 276 | 277 | return deployStream 278 | } 279 | 280 | // helper to wait for an operation to complete before executing the callback 281 | // this also supports creation callbacks, specifically `createVM`, which has an 282 | // extra arg with the instance object of the created VM 283 | function _onOperationComplete (callback) { 284 | return function (err, operation, apiResponse) { 285 | if (err) return callback(err) 286 | 287 | if (arguments.length === 4) { 288 | operation = apiResponse 289 | } 290 | 291 | operation 292 | .on('error', callback) 293 | .on('complete', callback.bind(null, null)) 294 | } 295 | } 296 | --------------------------------------------------------------------------------