├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | test.sh 3 | node_modules 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # docker-cloudwatchlogs 2 | # 3 | # VERSION 0.1.0 4 | 5 | FROM node:0.10-onbuild 6 | MAINTAINER Damian Beresford <@dberesford> 7 | 8 | ENTRYPOINT ["/usr/src/app/index.js"] 9 | CMD [] 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) nearForm Ltd. and Contributors 4 | All rights reserved. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-cloudwatchlogs 2 | 3 | Forward all your logs to [CloudWatch Logs](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/WhatIsCloudWatchLogs.html), like a breeze. 4 | 5 | ## Usage as a Container 6 | 7 | The simplest way to forward all your container's log to Cloudwatch is to 8 | run this repository as a container, with: 9 | 10 | ```sh 11 | docker run -v /var/run/docker.sock:/var/run/docker.sock dberesford/docker-cloudwatchlogs -a ACCESSKEY -s SECRET_KEY -r REGION -g GROUP_NAME -t STREAM_NAME 12 | ``` 13 | 14 | ### Running container in a restricted environment. 15 | Some environments(such as Google Compute Engine) does not allow to access the docker socket without special privileges. You will get EACCES(`Error: read EACCES`) error if you try to run the container. 16 | To run the container in such environments add --privileged to the `docker run` command. 17 | 18 | Example: 19 | ```sh 20 | docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock dberesford/docker-cloudwatchlogs -a ACCESSKEY -s SECRET_KEY -r REGION -g GROUP_NAME -t STREAM_NAME 21 | ``` 22 | 23 | ## Usage as a CLI 24 | 25 | 1. `npm install docker-cloudwatchlogs -g` 26 | 2. `docker-cloudwatchlogs -a ACCESSKEY -s SECRET_KEY -r REGION -g GROUP_NAME -t STREAM_NAME` 27 | 3. ..there is no step 3 28 | 29 | 30 | ## Embedded usage 31 | 32 | Install it with: `npm install docker-cloudwatchlogs --save` 33 | 34 | Then, in your JS file: 35 | 36 | ``` 37 | var cloudwatchlogs = require('docker-cloudwatchlogs')({ 38 | 'accessKeyId': 'ACCESS_KEY', 39 | 'secretAccessKey': 'SECRET_KEY', 40 | 'region': 'REGION', 41 | 'logGroupName': 'GROUP_NAME', 42 | 'logStreamName': 'STREAM_NAME', 43 | 'bulkIndex': BULK_INDEX, 44 | 'timeout': TIMEOUT 45 | }) 46 | 47 | // cloudwatch is the source stream with all the 48 | // log lines 49 | 50 | setTimeout(function() { 51 | cloudwatchlogs.destroy() 52 | }, 5000) 53 | ``` 54 | 55 | ## Building a docker repo from this repository 56 | 57 | First clone this repository, then: 58 | 59 | ```bash 60 | docker build -t cloudwatchlogs . 61 | docker run -v /var/run/docker.sock:/var/run/docker.sock cloudwatchlogs -a ACCESSKEY -s SECRET_KEY -r REGION -g GROUP_NAME -t STREAM_NAME 62 | ``` 63 | 64 | ## How it works 65 | 66 | This module wraps four [Docker APIs](https://docs.docker.com/reference/api/docker_remote_api_v1.17/): 67 | 68 | * `POST /containers/{id}/attach`, to fetch the logs 69 | * `GET /containers/{id}/stats`, to fetch the stats of the container 70 | * `GET /containers/json`, to detect the containers that are running when 71 | this module starts 72 | * `GET /events`, to detect new containers that will start after the 73 | module has started 74 | 75 | This module wraps: 76 | 77 | * [docker-loghose](https://github.com/mcollina/docker-loghose) 78 | * [docker-stats](https://github.com/pelger/docker-stats) to fetch the logs and the stats as a never ending stream of data 79 | * [cloudwatchlogs-stream](https://github.com/nearform/cloudwatchlogs-stream) to stream the logs to CloudWatch 80 | 81 | All the originating requests are wrapped in [never-ending-stream](https://github.com/mcollina/never-ending-stream). 82 | 83 | ## License 84 | 85 | MIT 86 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | var util = require('util'); 4 | var tls = require('tls'); 5 | var net = require('net'); 6 | var eos = require('end-of-stream'); 7 | var through = require('through2'); 8 | var minimist = require('minimist'); 9 | var allContainers = require('docker-allcontainers'); 10 | var statsFactory = require('docker-stats'); 11 | var logFactory = require('docker-loghose'); 12 | var CloudWatchLogs = require('cloudwatchlogs-stream'); 13 | 14 | function start(opts){ 15 | var logsToken = opts.logstoken || opts.token; 16 | var statsToken = opts.statstoken || opts.token; 17 | var out; 18 | var noRestart = function() {}; 19 | var filter = through.obj(function(obj, enc, cb) { 20 | addAll(opts.add, obj); 21 | if (obj.line) { 22 | this.push(obj.line + '\n'); 23 | } 24 | cb() 25 | }); 26 | 27 | var events = allContainers(opts); 28 | opts.events = events; 29 | 30 | var loghose = logFactory(opts); 31 | loghose.pipe(filter); 32 | 33 | if (opts.stats !== false) { 34 | var stats = statsFactory(opts); 35 | stats.pipe(filter); 36 | } 37 | 38 | pipe(); 39 | 40 | // destroy out if loghose is destroyed 41 | eos(loghose, function() { 42 | noRestart() 43 | out.destroy(); 44 | }); 45 | 46 | return loghose; 47 | 48 | function addAll(proto, obj) { 49 | if (!proto) { return; } 50 | 51 | var key; 52 | for (key in proto) { 53 | if (proto.hasOwnProperty(key)) { 54 | obj[key] = proto[key]; 55 | } 56 | } 57 | } 58 | 59 | function pipe() { 60 | if (out) { 61 | filter.unpipe(out); 62 | } 63 | 64 | out = new CloudWatchLogs(opts); 65 | out.on('error', function(err) { 66 | throw new Error(err); 67 | }); 68 | 69 | filter.pipe(out, { end: false }); 70 | 71 | // automatically reconnect on socket failure 72 | noRestart = eos(out, pipe); 73 | } 74 | } 75 | 76 | function cli() { 77 | var argv = minimist(process.argv.slice(2), { 78 | boolean: ['json', 'stats'], 79 | alias: { 80 | 'accessKeyId': 'a', 81 | 'secretAccessKey': 's', 82 | 'region': 'r', 83 | 'logGroupName': 'g', 84 | 'logStreamName': 't', 85 | 'bulkIndex': 'b', 86 | 'timeout': 'o' 87 | //'json': 'j', 88 | //'stats': 't' 89 | //'add': 'a' 90 | }, 91 | default: { 92 | json: false, 93 | stats: false, 94 | add: [] 95 | } 96 | }); 97 | 98 | if (!(argv.accesskey || argv.secretkey || argv.groupname || argv.streamname || argv.region)) { 99 | console.log('Usage: docker-cloudwatch [-a ACCESS_KEY] [-k SECRET_KEY]\n' + 100 | ' [-r REGION] [-g GROUP_NAME] [-s STREAM_NAME] [--json]\n' + 101 | ' [-b BULK_INDEX] [-o TIMEOUT]'); 102 | // ' [--no-stats] [-a KEY=VALUE]'); 103 | process.exit(1); 104 | } 105 | 106 | if (argv.add && !Array.isArray(argv.add)) { 107 | argv.add = [argv.add]; 108 | } 109 | 110 | argv.add = argv.add.reduce(function(acc, arg) { 111 | arg = arg.split('='); 112 | acc[arg[0]] = arg[1]; 113 | return acc 114 | }, {}); 115 | 116 | start(argv); 117 | } 118 | 119 | module.exports = start; 120 | 121 | if (require.main === module) { 122 | cli(); 123 | } 124 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "docker-cloudwatchlogs", 3 | "version": "1.2.0", 4 | "description": "Forward all logs from all running docker containers to Cloudwatch", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/nearform/docker-cloudwatchlogs.git" 8 | }, 9 | "main": "index.js", 10 | "scripts": { 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "author": [ 14 | "Matteo Collina", 15 | "Damian Beresford" 16 | ], 17 | "license": "MIT", 18 | "dependencies": { 19 | "docker-allcontainers": "^0.2.0", 20 | "docker-loghose": "^0.4.0", 21 | "docker-stats": "^0.2.0", 22 | "end-of-stream": "^1.1.0", 23 | "minimist": "^1.1.0", 24 | "through2": "^0.6.3", 25 | "cloudwatchlogs-stream": "^1.2.0" 26 | } 27 | } 28 | --------------------------------------------------------------------------------