├── .gitignore
├── README.md
├── base
    ├── lambda-task-runner.zip
    ├── lambda-task-runner
    │   └── index.js
    └── main.tf
└── scheduled-task
    ├── main.tf
    └── task.tpl


/.gitignore:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jbrook/ecs-task-scheduler-tf/4356a402894a5dad58c09a2dfe1aa5686b2994e4/.gitignore


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # AWS ECS Task Runner Terraform Module
 2 | 
 3 | Allows scheduling standalone AWS EC2 Container Service (ECS) tasks with
 4 | Terraform. There are two modules:
 5 | 
 6 | 1. A [base](./base/main.tf) module to provision shared resources - a generic
 7 | lambda function to run AWS ECS tasks and it's associated IAM role and policies.
 8 | 2. A [scheduled-task](./scheduled-task/main.tf) module to provision indivdual
 9 | scheduled tasks.
10 |     * Creates a Cloudwatch scheduled event with a cron-style or rate expression.
11 |     * Targets the scheduled event at the Lambda function with the arguments necessary to run the specified ECS task.
12 | 
13 | ## Usage Example
14 | 
15 | ```
16 | /**
17 |  * Shared resources to be used by ECS scheduled jobs.
18 |  * A lambda function and it's IAM role and policies.
19 |  */
20 | module "ecs_task_scheduler_base_resources" {
21 |   source = "git::ssh://git@github.com/jbrook/ecs-task-scheduler-tf.git//base"
22 |   stack_name = "${var.stack_name}"
23 |   ecs_cluster_id = "${var.ecs_cluster_id}"
24 | }
25 | 
26 | /**
27 |  * Periodically runs a job
28 |  */
29 | module "schedule_dw_access_token_refresh" {
30 |   source = "git::ssh://git@github.com/jbrook/ecs-task-scheduler-tf.git//scheduled-task"
31 |   job_identifier = "my-job"
32 |   ecs_task_def = "<task def ARN or name:revision"
33 |   stack_name = "${var.stack_name}"
34 |   region = "${var.region}"
35 |   ecs_cluster_id = "${module.core-vpc.ecs_cluster_id}"
36 |   schedule_expression = "rate(10 minutes)"
37 |   container_name = "my-container" # needs to match the container you want to run in the task-def
38 |   container_cmd = ["echo","'Hello, World!'"]
39 |   lambda_function_name = "${module.ecs_task_scheduler_base_resources.lambda_function_name}"
40 |   lambda_function_arn = "${module.ecs_task_scheduler_base_resources.lambda_function_arn}"
41 | }
42 | ```
43 | 
44 | ## Preparing and updating the Lambda function 
45 | 
46 | If you update the nodejs Lambda function code in the "base/lambda-task-runner" folder after you have already applied a module with Terraform, then you need to take the following steps to "re-provision" the Lambda function.
47 | 
48 | In this module folder:
49 | ```
50 | cd lambda-task-runner
51 | zip -9 lambda-task-runner.zip index.js
52 | mv lambda-task-runner.zip ../
53 | ```
54 | 
55 | In the directory for the terraform environment you are provisioning (the resource names will actually be different (have a look at 'terraform state list'):
56 | ```
57 | terraform taint aws_cloudwatch_event_target.call_task_runner_scheduler
58 | terraform taint aws_lambda_function.task_runner 
59 | ```
60 | 
61 | Then you can make a terraform plan and apply as you ususually would.
62 | 
63 | ## TODO
64 | 
65 | 1. Allow overriding task environment variables too.
66 | 2. Lambda function could be extended to poll 'describe-tasks' and look for an exit code. The function could then report a success/failure metric to the Cloudwatch logs.
67 | 


--------------------------------------------------------------------------------
/base/lambda-task-runner.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jbrook/ecs-task-scheduler-tf/4356a402894a5dad58c09a2dfe1aa5686b2994e4/base/lambda-task-runner.zip


--------------------------------------------------------------------------------
/base/lambda-task-runner/index.js:
--------------------------------------------------------------------------------
 1 | var AWS = require('aws-sdk')
 2 | var ecs = new AWS.ECS();
 3 | 
 4 | exports.handler = (events, context) => {
 5 |     /*
 6 |     {
 7 |       "job_identifier": "${job_identifier}",
 8 |       "region": "${region}",
 9 |       "cluster": "${cluster}",
10 |       "ecs_task_def": "${ecs_task_def}",
11 |       "overrides": {
12 |         "containerOverrides": [
13 |           {
14 |             "name": "${container_name}",
15 |             "command": "${container_cmd}"
16 |           }
17 |         ]
18 |       }
19 |     }
20 |     */
21 |     
22 |   console.log(events)
23 |   var ecs_task_def = events.ecs_task_def
24 |   var exec_region  = events.region || 'undefined'
25 |   var cluster      = events.cluster || 'undefined'
26 |   var overrides    = events.overrides || 'undefined'
27 |   console.log(ecs_task_def, exec_region)
28 |   var params = {
29 |       taskDefinition: ecs_task_def,
30 |       cluster: cluster,
31 |       overrides: overrides
32 |   }
33 |   ecs.runTask(params, function(err, data) {
34 |       if (err) console.log(err, err.stack); // an error occurred
35 |       else     console.log(data);           // successful response 
36 |       context.done(err, data)
37 |   })
38 |     
39 | }
40 | 


--------------------------------------------------------------------------------
/base/main.tf:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Module: ecs-task-scheduler-tf/base
  3 |  *
  4 |  * This module is a base module to provision shared resources that will be used
  5 |  * when scheduling ECS tasks. This base module is intended to be used with
  6 |  * ecs-task-scheduler/scheduled-task. The scheduled-task companion module could
  7 |  * be used multiple times in a single terraform environment but it does not make
  8 |  * provision generic resources like the lambda function and it's IAM role and
  9 |  * policies multiple times. Instead they are created by this module and can then
 10 |  * be passed on to scheduled-task modules.
 11 |  */
 12 | 
 13 | 
 14 | variable "stack_name" {}
 15 | variable "ecs_cluster_id" {}
 16 | 
 17 | /**
 18 |  * Lambda function to run an ECS task with the AWS SDK.
 19 |  */
 20 | resource "aws_lambda_function" "task_runner" {
 21 |   function_name = "${var.stack_name}-ECSTaskRunner"
 22 |   filename = "${path.module}/lambda-task-runner.zip"
 23 |   runtime = "nodejs4.3"
 24 |   timeout = 30
 25 |   description = "Runs an ECS task with specified overrides"
 26 |   role = "${aws_iam_role.task_runner_execution.arn}"
 27 |   handler = "index.handler"
 28 |   environment {
 29 |     variables = {
 30 |       SOME_VAR = "SOME_VALUE"
 31 |     }
 32 |   }
 33 |   lifecycle {
 34 |     # Attempt to workaround - https://github.com/hashicorp/terraform/issues/7613
 35 |     ignore_changes = ["filename"]
 36 |   }
 37 | }
 38 | 
 39 | /**
 40 |  * Execution role - a role that will allow the Lambda function
 41 |  * to be executed and to run ECS tasks.
 42 |  */
 43 | resource "aws_iam_role" "task_runner_execution" {
 44 |   name = "${var.stack_name}-task-runner-execution"
 45 |   assume_role_policy = <<EOF
 46 | {
 47 |   "Version": "2012-10-17",
 48 |   "Statement": [
 49 |     {
 50 |       "Action": "sts:AssumeRole",
 51 |       "Principal": {
 52 |         "Service": "lambda.amazonaws.com"
 53 |       },
 54 |       "Effect": "Allow",
 55 |       "Sid": ""
 56 |     }
 57 |   ]
 58 | }
 59 | EOF
 60 | }
 61 | 
 62 | /**
 63 |  * Policy document for running ECS tasks on this specific cluster.
 64 |  */
 65 | data "aws_iam_policy_document" "run_ecs_task_policy_document" {
 66 | 
 67 |   statement {
 68 |     sid = "RunTasksOnECSCluster"
 69 |     actions = [
 70 |       "ecs:RunTask",
 71 |     ]
 72 |     resources = [ "*" ] // We could restrict to a specific task family if we wanted.
 73 |     condition {
 74 |       test = "ArnEquals"
 75 |       variable = "ecs:cluster"
 76 |       values = ["${var.ecs_cluster_id}"]
 77 |     }
 78 |   }
 79 | 
 80 |   statement = {
 81 |     effect = "Allow"
 82 |     actions = [
 83 |       "logs:CreateLogGroup",
 84 |       "logs:CreateLogStream",
 85 |       "logs:PutLogEvents",
 86 |       "logs:DescribeLogStreams"
 87 |     ]
 88 |     resources = [
 89 |         "arn:aws:logs:*:*:*"
 90 |     ]
 91 |   }
 92 | 
 93 | }
 94 | 
 95 | /**
 96 |  * Policy allows our task runner-Lambda function to run
 97 |  * ECS tasks on this cluster and log to Cloudwatch.
 98 |  */
 99 | resource "aws_iam_policy" "run_ecs_task_policy" {
100 |   name = "${var.stack_name}-task-runner-execution-ecs-execution-policy"
101 |   description = "Policy to allow task-runner to run ECS tasks and log to Cloudwatch"
102 |   path = "/"
103 |   policy = "${data.aws_iam_policy_document.run_ecs_task_policy_document.json}"
104 | }
105 | 
106 | /**
107 |  * Attach the policy to the IAM role.
108 |  */
109 | resource "aws_iam_role_policy_attachment" "run_ecs_task_policy" {
110 |   role = "${aws_iam_role.task_runner_execution.name}"
111 |   policy_arn = "${aws_iam_policy.run_ecs_task_policy.arn}"
112 | }
113 | 
114 | output "lambda_function_name" {
115 |   value = "${aws_lambda_function.task_runner.function_name}"
116 | }
117 | 
118 | output "lambda_function_arn" {
119 |   value = "${aws_lambda_function.task_runner.arn}"
120 | }
121 | 
122 | 


--------------------------------------------------------------------------------
/scheduled-task/main.tf:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Module: ecs-task-scheduler-tf/scheduled-task
  3 |  *
  4 |  * This module allows EC2 Container Service tasks to be run on a schedule.
  5 |  * The schedule is specified using scheduled Cloudwatch events. This allows
  6 |  * events to be scheduled (using cron or rate expressions) and associated with
  7 |  * a target. In this case the target is a Lambda function which can execute an
  8 |  * ECS task on a cluster.
  9 |  * 
 10 |  * The lambda function and its IAM role and policies should be created separately
 11 |  * using the 'ecs-task-scheduler-tf/base' module and then provided to this module
 12 |  * through input variables.
 13 |  */
 14 | 
 15 | variable "stack_name" {}
 16 | variable "ecs_cluster_id" {}
 17 | variable "region" {}
 18 | 
 19 | /**
 20 |  * The name and ARN of the lambda function to target.
 21 |  */
 22 | variable "lambda_function_arn" {}
 23 | variable "lambda_function_name" {}
 24 | 
 25 | /**
 26 |  * A short name to identify the job. E.g. "reindex-es-users".
 27 |  */
 28 |  variable "job_identifier" {}
 29 | 
 30 | /**
 31 |  * A cron or rate expression.
 32 |  * See: http://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html
 33 |  */
 34 | variable "schedule_expression" {}
 35 | 
 36 | /**
 37 |  * The family and revision (family:revision) or full Amazon Resource Name (ARN) of the
 38 |  * task definition to run. If a revision is not specified, the latest ACTIVE
 39 |  * revision is used.
 40 |  */
 41 | variable "ecs_task_def" {}
 42 | 
 43 | /**
 44 |  * The container name. Used when overriding the command. 
 45 |  */
 46 | variable "container_name" {}
 47 | 
 48 | /**
 49 |  * The container command to run. It should be a terraform array of strings, e.g.
 50 |  *  container_cmd = ["/app/ops/spaaza","ops:refreshDemandwareAccessTokens"]
 51 |  */
 52 | variable "container_cmd" {
 53 |   type = "list"
 54 | }
 55 | 
 56 | 
 57 | /**
 58 |  * A rule for a cloudwatch schedule event.
 59 |  */
 60 | resource "aws_cloudwatch_event_rule" "task_runner_scheduler" {
 61 |   name = "${var.stack_name}-${var.job_identifier}"
 62 |   description = "${var.stack_name}-${var.job_identifier}-schedule"
 63 |   schedule_expression = "${var.schedule_expression}"
 64 | }
 65 | 
 66 | /**
 67 |  * Target the lambda function with the schedule.
 68 |  */
 69 | resource "aws_cloudwatch_event_target" "call_task_runner_scheduler" {
 70 |   rule = "${aws_cloudwatch_event_rule.task_runner_scheduler.name}"
 71 |   target_id = "${var.lambda_function_name}"
 72 |   arn = "${var.lambda_function_arn}"
 73 |   input = "${data.template_file.task_json.rendered}"
 74 | }
 75 | 
 76 | data "template_file" "task_json" {
 77 |     template = "${file("${path.module}/task.tpl")}"
 78 | 
 79 |     vars {
 80 |         job_identifier = "${var.job_identifier}"
 81 |         region         = "${var.region}"
 82 |         cluster        = "${var.ecs_cluster_id}"
 83 |         ecs_task_def   = "${var.ecs_task_def}"
 84 |         container_name = "${var.container_name}"
 85 |         container_cmd  = "${jsonencode(var.container_cmd)}"
 86 |     }
 87 | }
 88 | 
 89 | /**
 90 |  * Permission to allow Cloudwatch events to trigger the task runner
 91 |  * Lambda function.
 92 |  */
 93 | resource "aws_lambda_permission" "allow_cloudwatch_to_call_task_runner" {
 94 |   statement_id = "${var.job_identifier}-AllowExecutionFromCloudWatch"
 95 |   action = "lambda:InvokeFunction"
 96 |   function_name = "${var.lambda_function_name}"
 97 |   principal = "events.amazonaws.com"
 98 |   source_arn = "${aws_cloudwatch_event_rule.task_runner_scheduler.arn}"
 99 | }
100 | 


--------------------------------------------------------------------------------
/scheduled-task/task.tpl:
--------------------------------------------------------------------------------
 1 | {
 2 |   "job_identifier": "${job_identifier}",
 3 |   "region": "${region}",
 4 |   "cluster": "${cluster}",
 5 |   "ecs_task_def": "${ecs_task_def}",
 6 |   "overrides": {
 7 |     "containerOverrides": [
 8 |       {
 9 |         "name": "${container_name}",
10 |         "command": ${container_cmd}
11 |       }
12 |     ]
13 |   }
14 | }
15 | 


--------------------------------------------------------------------------------