├── .gitignore ├── Dockerfile ├── LICENSE ├── index.js ├── package.json ├── readme.md └── test └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | docker-compose.yml 3 | job-definition.json 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:6.11.0-alpine 2 | 3 | RUN mkdir -p /usr/src/app 4 | WORKDIR /usr/src/app 5 | 6 | ONBUILD ARG NODE_ENV 7 | ONBUILD ENV NODE_ENV $NODE_ENV 8 | ONBUILD COPY package.json /usr/src/app/ 9 | ONBUILD RUN npm install && npm cache clean 10 | ONBUILD COPY . /usr/src/app 11 | 12 | CMD [ "npm", "start" ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Turner 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const fs = require('fs') 4 | const yaml = require('js-yaml') 5 | 6 | let dockerComposeYaml = '' 7 | 8 | //read bits from stdin 9 | process.stdin.setEncoding('utf8') 10 | process.stdin.on('readable', () => { 11 | let chunk = process.stdin.read() 12 | if (chunk !== null) dockerComposeYaml += chunk 13 | }) 14 | 15 | process.stdin.on('end', () => { 16 | //parse docker-compose.yml 17 | let compose 18 | try { 19 | compose = yaml.safeLoad(dockerComposeYaml, 'utf8') 20 | } catch (e) { 21 | console.error(e) 22 | } 23 | 24 | //transform docker-compose format to job definition format 25 | jobDefinition = transform(compose) 26 | 27 | //write job definition to stdout 28 | console.log(JSON.stringify(jobDefinition, undefined, 2)) 29 | }) 30 | 31 | function transform(compose) { 32 | let result = {} 33 | 34 | const serviceName = Object.keys(compose.services)[0] 35 | const service = compose.services[serviceName] 36 | 37 | result.jobDefinitionName = serviceName 38 | result.type = 'container' 39 | result.containerProperties = { 40 | image: service.image, 41 | environment: [], 42 | vcpus: 2, 43 | memory: 2000 44 | } 45 | 46 | //environment variables 47 | if (service.environment) { 48 | Object.keys(service.environment).forEach(env => { 49 | //ignore dynamic environment variables (for now) 50 | if (!service.environment[env].startsWith('${')) { 51 | result.containerProperties.environment.push({ 52 | name: env, 53 | value: service.environment[env] 54 | }) 55 | } 56 | }) 57 | } 58 | 59 | //optional 60 | if (service.command) result.containerProperties.command = service.command 61 | 62 | if (service.labels) { 63 | Object.keys(service.labels).forEach(label => { 64 | if (label === 'composeToBatch.vcpus') 65 | result.containerProperties.vcpus = parseInt(service.labels[label]) 66 | 67 | if (label === 'composeToBatch.memory') 68 | result.containerProperties.memory = parseInt(service.labels[label]) 69 | 70 | if (label === 'composeToBatch.jobRoleArn') 71 | result.containerProperties.jobRoleArn = service.labels[label] 72 | }) 73 | } 74 | 75 | return result 76 | } 77 | 78 | module.exports = transform 79 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "compose-to-batch", 3 | "version": "0.1.2", 4 | "description": "A CLI tool to transform a docker-compose.yml into an AWS Batch job definition", 5 | "main": "index.js", 6 | "bin": { 7 | "compose-to-batch": "index.js" 8 | }, 9 | "scripts": { 10 | "start": "node .", 11 | "fmt": "prettier --single-quote --no-semi --write index.js && prettier --single-quote --no-semi --write test/index.js", 12 | "test": "mocha", 13 | "watch": "watch 'npm run fmt && npm test' ." 14 | }, 15 | "engines": { 16 | "node": ">=6.11.0" 17 | }, 18 | "author": "John Ritsema ", 19 | "license": "Apache-2", 20 | "keywords": [ 21 | "docker-compose", 22 | "docker-compose.yml", 23 | "aws", 24 | "batch", 25 | "job", 26 | "job-definition.json" 27 | ], 28 | "devDependencies": { 29 | "mocha": "^3.4.2", 30 | "watch": "^1.0.2" 31 | }, 32 | "dependencies": { 33 | "js-yaml": "^3.8.4" 34 | }, 35 | "repository": { 36 | "type": "git", 37 | "url": "git+https://github.com/turnerlabs/compose-to-batch.git" 38 | }, 39 | "homepage": "https://github.com/turnerlabs/compose-to-batch#readme", 40 | "bugs": { 41 | "url": "https://github.com/turnerlabs/compose-to-batch/issues" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | compose-to-batch 2 | ============== 3 | 4 | A CLI tool to transform a [`docker-compose.yml`](https://docs.docker.com/compose/compose-file/compose-file-v2/) into an [`AWS Batch job definition`](http://docs.aws.amazon.com/batch/latest/userguide/job_definitions.html). 5 | 6 | This tool allows you to develop your job code on your `laptop` using docker: 7 | ``` 8 | $ docker-compose up 9 | ``` 10 | 11 | And then when you're ready to deploy to `AWS Batch`: 12 | ``` 13 | $ docker-compose config | compose-to-batch > job-definition.json 14 | $ aws batch register-job-definition --cli-input-json file://job-definition.json 15 | ``` 16 | 17 | ### usage 18 | 19 | install 20 | ``` 21 | npm install -g compose-to-batch 22 | ``` 23 | 24 | given the following `docker-compose.yml` 25 | 26 | ```yaml 27 | version: "2" 28 | services: 29 | my-job: 30 | build: . 31 | image: 12345678910.dkr.ecr.us-east-1.amazonaws.com/my-job:0.1.0 32 | environment: 33 | REGION: us-east-1 34 | ``` 35 | 36 | running the following command 37 | 38 | ```bash 39 | docker-compose config | compose-to-batch 40 | ``` 41 | 42 | will output the following 43 | 44 | ```json 45 | { 46 | "jobDefinitionName": "my-job", 47 | "type": "container", 48 | "containerProperties": { 49 | "image": "12345678910.dkr.ecr.us-east-1.amazonaws.com/my-job:0.1.0", 50 | "environment": [ 51 | { 52 | "name": "REGION", 53 | "value": "us-east-1" 54 | } 55 | ], 56 | "vcpus": "2", 57 | "memory": "2000" 58 | } 59 | } 60 | ``` 61 | 62 | ### mapping 63 | 64 | - service -> jobDefinitionName 65 | - service.image -> containerProperties.image 66 | - service.command -> containerProperties.command 67 | - service.environment -> containerProperties.environment 68 | - service.labels.composeToBatch.vcpus -> containerProperties.vcpus 69 | - service.labels.composeToBatch.memory -> containerProperties.memory 70 | - service.labels.composeToBatch.jobRoleArn -> containerProperties.jobRoleArn 71 | 72 | 73 | ### local development using IAM keys and Batch with jobRoleArn 74 | 75 | A common scenario is to do local development using IAM keys specified as environment variables in your container. You typically don't want to check in your secrets to source control so a `.env` file is a good option here. When you're ready to deploy to Batch, you also typically don't want to expose your secret IAM keys. With this in mind, `compose-to-batch` will exclude any environment variables that reference dynamic variables. For example: 76 | 77 | ``` 78 | AWS_ACCESS_KEY_ID=xyz 79 | AWS_SECRET_ACCESS_KEY=xyz 80 | ``` 81 | 82 | ```yaml 83 | version: "2" 84 | services: 85 | my-job: 86 | build: . 87 | image: 12345678910.dkr.ecr.us-east-1.amazonaws.com/my-job:0.1.0 88 | environment: 89 | REGION: us-east-1 90 | S3_BUCKET: some-bucket 91 | AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} 92 | AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} 93 | labels: 94 | composeToBatch.jobRoleArn: arn:aws:iam::12345678910:role/my-role 95 | ``` 96 | 97 | When you're ready to deploy to Batch, the following command will safely exclude your secret keys from the generated batch job definition and use the job role instead. 98 | 99 | ``` 100 | $ cat docker-compose.yml | compose-to-batch 101 | ``` 102 | 103 | ```json 104 | { 105 | "jobDefinitionName": "my-job", 106 | "type": "container", 107 | "containerProperties": { 108 | "image": "12345678910.dkr.ecr.us-east-1.amazonaws.com/my-job:0.1.0", 109 | "environment": [ 110 | { 111 | "name": "REGION", 112 | "value": "us-east-1" 113 | }, 114 | { 115 | "name": "S3_BUCKET", 116 | "value": "some-bucket" 117 | } 118 | ], 119 | "vcpus": "2", 120 | "memory": "2000", 121 | "jobRoleArn": "arn:aws:iam::12345678910:role/my-role" 122 | } 123 | } 124 | ``` 125 | 126 | However, if you DO want to include substituted environment variables, you can use `docker-compose config` instead since it returns the result of the variable substitution. 127 | 128 | ``` 129 | $ docker-compose.config | compose-to-batch 130 | ``` -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert') 2 | const yaml = require('js-yaml') 3 | const transform = require('../.') 4 | 5 | const service = 'my-job' 6 | const vcpus = 4 7 | const memory = 8000 8 | const jobRoleArn = 'foo' 9 | const envVarName = 'NAME1' 10 | const envVarValue = 'VALUE1' 11 | const envVarName2 = 'NAME2' 12 | const envVarValue2 = 'VALUE2' 13 | 14 | const dockerComposeYaml = ` 15 | version: "2" 16 | services: 17 | ${service}: 18 | build: . 19 | image: 12345678910.dkr.ecr.us-east-1.amazonaws.com/my-job:0.1.0 20 | environment: 21 | ${envVarName}: ${envVarValue} 22 | ${envVarName2}: ${envVarValue2} 23 | labels: 24 | composeToBatch.vcpus: ${vcpus} 25 | composeToBatch.memory: ${memory} 26 | composeToBatch.jobRoleArn: ${jobRoleArn} 27 | ` 28 | const compose = yaml.safeLoad(dockerComposeYaml, 'utf8') 29 | 30 | describe('composeToBatch', () => { 31 | describe('transform()', () => { 32 | it('should map to the first service', () => { 33 | jobDefinition = transform(compose) 34 | assert.equal(jobDefinition.jobDefinitionName, service) 35 | }) 36 | 37 | it('should map the image', () => { 38 | jobDefinition = transform(compose) 39 | assert.equal(jobDefinition.image, compose.image) 40 | }) 41 | 42 | it('should map the env vars', () => { 43 | jobDefinition = transform(compose) 44 | 45 | assert.equal( 46 | jobDefinition.containerProperties.environment[0].name, 47 | envVarName 48 | ) 49 | assert.equal( 50 | jobDefinition.containerProperties.environment[0].value, 51 | envVarValue 52 | ) 53 | 54 | assert.equal( 55 | jobDefinition.containerProperties.environment[1].name, 56 | envVarName2 57 | ) 58 | assert.equal( 59 | jobDefinition.containerProperties.environment[1].value, 60 | envVarValue2 61 | ) 62 | }) 63 | 64 | it('should map the vcpus as a number', () => { 65 | jobDefinition = transform(compose) 66 | assert.equal(jobDefinition.containerProperties.vcpus, vcpus) 67 | assert.equal(typeof jobDefinition.containerProperties.vcpus, typeof vcpus) 68 | }) 69 | 70 | it('should map the memory as a number', () => { 71 | jobDefinition = transform(compose) 72 | assert.equal(jobDefinition.containerProperties.memory, memory) 73 | assert.equal(typeof jobDefinition.containerProperties.vcpus, typeof vcpus) 74 | }) 75 | 76 | it('should map the jobRoleArn', () => { 77 | jobDefinition = transform(compose) 78 | assert.equal(jobDefinition.containerProperties.jobRoleArn, jobRoleArn) 79 | }) 80 | 81 | it('should map the command', () => { 82 | const arg1 = 'foo' 83 | const arg2 = 'bar' 84 | const dockerComposeYaml = ` 85 | version: "2" 86 | services: 87 | service: 88 | command: [ "${arg1}", "${arg2}" ] 89 | ` 90 | const compose = yaml.safeLoad(dockerComposeYaml, 'utf8') 91 | jobDefinition = transform(compose) 92 | assert.equal(jobDefinition.containerProperties.command[0], arg1) 93 | assert.equal(jobDefinition.containerProperties.command[1], arg2) 94 | }) 95 | 96 | it('should ignore dynamic environment variables', () => { 97 | const dockerComposeYaml = ` 98 | version: "2" 99 | services: 100 | service: 101 | environment: 102 | FOO: \${foo} 103 | BAR: bar 104 | ` 105 | const compose = yaml.safeLoad(dockerComposeYaml, 'utf8') 106 | jobDefinition = transform(compose) 107 | assert.equal(jobDefinition.containerProperties.environment[0].name, 'BAR') 108 | }) 109 | }) 110 | }) 111 | --------------------------------------------------------------------------------